diff --git a/CMakeLists.txt b/CMakeLists.txt index fb35b6c..092151f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,4 +126,26 @@ else() message(STATUS "NRD Denoiser Disabled") endif() +# OpenXR SDK +option(MCVR_ENABLE_OPENXR "Enable OpenXR VR support" ON) + +if (MCVR_ENABLE_OPENXR) + message(STATUS "OpenXR Support Enabled") + include(FetchContent) + FetchContent_Declare( + openxr_sdk + GIT_REPOSITORY https://github.com/KhronosGroup/OpenXR-SDK.git + GIT_TAG release-1.1.43 + GIT_SHALLOW TRUE + EXCLUDE_FROM_ALL + ) + set(BUILD_TESTS OFF CACHE BOOL "" FORCE) + set(BUILD_API_LAYERS OFF CACHE BOOL "" FORCE) + set(BUILD_CONFORMANCE_TESTS OFF CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(openxr_sdk) + set(OPENXR_INCLUDE_DIR ${openxr_sdk_SOURCE_DIR}/include) +else() + message(STATUS "OpenXR Support Disabled") +endif() + add_subdirectory(src) diff --git a/src/common/shared.hpp b/src/common/shared.hpp index 87fc418..6b9f716 100644 --- a/src/common/shared.hpp +++ b/src/common/shared.hpp @@ -247,6 +247,15 @@ namespace Data { T_UINT endPortalTextureID; T_UINT pad4; T_UINT pad5; + + // VR stereo fields + T_MAT4 eyeViewOffsets[2]; // per-eye view offset (translate ±ipd/2) + T_MAT4 eyeProjOffsets[2]; // per-eye projection offset (identity for symmetric) + T_FLOAT ipd; + T_UINT stereoEnabled; // 0=mono, 1=stereo + T_FLOAT foveatedInnerRadius; // normalized radius of full-res inner circle (1.0 = full screen) + T_UINT foveatedOuterBlockSize; // block size for outer region (1=off, 2=2x2, 4=4x4) + T_VEC2 foveatedCenter; // normalised gaze centre (0.5,0.5 = screen centre) }; struct SkyUBO { diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index a14bb2e..7c9c03c 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -18,6 +18,11 @@ if (MCVR_ENABLE_NRD) target_compile_definitions(core PUBLIC NRD_STATIC_LIBRARY=1) target_compile_definitions(core PUBLIC MCVR_ENABLE_NRD) endif () +if (MCVR_ENABLE_OPENXR) + target_link_libraries(core PUBLIC openxr_loader) + target_include_directories(core PUBLIC ${OPENXR_INCLUDE_DIR}) + target_compile_definitions(core PUBLIC MCVR_ENABLE_OPENXR XR_USE_GRAPHICS_API_VULKAN) +endif () if (MSVC) target_compile_definitions(core PRIVATE WIN32_LEAN_AND_MEAN NOMINMAX) target_compile_options(core PRIVATE /utf-8) diff --git a/src/core/middleware/com_radiance_client_proxy_vulkan_VRProxy.cpp b/src/core/middleware/com_radiance_client_proxy_vulkan_VRProxy.cpp new file mode 100644 index 0000000..6bff785 --- /dev/null +++ b/src/core/middleware/com_radiance_client_proxy_vulkan_VRProxy.cpp @@ -0,0 +1,292 @@ +#include "com_radiance_client_proxy_vulkan_VRProxy.h" + +#include "core/render/renderer.hpp" +#include "core/render/render_framework.hpp" + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetEnabled(JNIEnv *, + jclass, + jboolean enabled) { + Renderer::options.vrEnabled = enabled; + +#ifdef MCVR_ENABLE_OPENXR + if (Renderer::is_initialized()) { + auto framework = Renderer::instance().framework(); + if (framework != nullptr && !enabled) { + framework->stopXRSession(); + } + } +#endif +} + +JNIEXPORT jboolean JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeStartXRSession(JNIEnv *, + jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return JNI_FALSE; + auto framework = Renderer::instance().framework(); + if (framework == nullptr) return JNI_FALSE; + return framework->startXRSession() ? JNI_TRUE : JNI_FALSE; +#else + return JNI_FALSE; +#endif +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeStopXRSession(JNIEnv *, + jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return; + auto framework = Renderer::instance().framework(); + if (framework == nullptr) return; + framework->stopXRSession(); +#endif +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetRenderScale(JNIEnv *, + jclass, + jfloat renderScale) { + Renderer::options.vrRenderScale = renderScale; +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetIPD(JNIEnv *, + jclass, + jfloat ipd) { + Renderer::options.vrIPD = ipd; +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetWorldScale(JNIEnv *, + jclass, + jfloat worldScale) { + Renderer::options.vrWorldScale = worldScale; +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetWorldOrientation( + JNIEnv *, jclass, jfloat qx, jfloat qy, jfloat qz, jfloat qw) { + if (!Renderer::is_initialized()) return; + Renderer::instance().vrSystem().worldOrientation = glm::normalize(glm::quat(qw, qx, qy, qz)); +} + +JNIEXPORT jboolean JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeIsEnabled(JNIEnv *, + jclass) { + return Renderer::options.vrEnabled; +} + +JNIEXPORT jint JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetEyeCount(JNIEnv *, + jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return Renderer::options.vrEnabled ? 2 : 1; + auto framework = Renderer::instance().framework(); + if (framework != nullptr && framework->isXRSessionRunning()) return 2; + return 1; +#else + return Renderer::options.vrEnabled ? 2 : 1; +#endif +} + +JNIEXPORT jint JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetEyeRenderWidth(JNIEnv *, + jclass) { + if (!Renderer::is_initialized()) return 0; + return static_cast(Renderer::instance().vrSystem().eyeRenderWidth()); +} + +JNIEXPORT jint JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetEyeRenderHeight(JNIEnv *, + jclass) { + if (!Renderer::is_initialized()) return 0; + return static_cast(Renderer::instance().vrSystem().eyeRenderHeight()); +} + +JNIEXPORT jfloat JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetRefreshRate(JNIEnv *, + jclass) { + if (!Renderer::is_initialized()) return 0.0f; + return Renderer::instance().vrSystem().config.refreshRate; +} + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetHeadPose(JNIEnv *env, + jclass) { + jfloatArray result = env->NewFloatArray(7); + if (!Renderer::is_initialized()) return result; + const auto &vr = Renderer::instance().vrSystem(); + const auto &pose = vr.headPose; + if (!pose.valid) return result; + + // Return raw tracking-space pose — worldOrientation is applied only + // in the C++ rendering path (buffers.cpp). Java-side VRData handles + // the worldRotation transform independently. + const glm::vec3 &pos = pose.position; + const glm::quat &ori = pose.orientation; + + float data[7] = {pos.x, pos.y, pos.z, ori.x, ori.y, ori.z, ori.w}; + env->SetFloatArrayRegion(result, 0, 7, data); + return result; +} + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetEyeFov(JNIEnv *env, + jclass, + jint eye) { + jfloatArray result = env->NewFloatArray(4); + if (!Renderer::is_initialized() || eye < 0 || eye > 1) return result; + const auto &ep = Renderer::instance().vrSystem().eyes[eye]; + float data[4] = {ep.tanLeft, ep.tanRight, ep.tanUp, ep.tanDown}; + env->SetFloatArrayRegion(result, 0, 4, data); + return result; +} + +JNIEXPORT jintArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetRecommendedResolution(JNIEnv *env, + jclass) { + jintArray result = env->NewIntArray(2); + if (!Renderer::is_initialized()) return result; + const auto &eye0 = Renderer::instance().vrSystem().eyes[0]; + jint data[2] = {static_cast(eye0.recommendedWidth), static_cast(eye0.recommendedHeight)}; + env->SetIntArrayRegion(result, 0, 2, data); + return result; +} + +// ---- Controller input ---- + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetControllerPose( + JNIEnv *env, jclass, jint hand) { + // Returns [px, py, pz, qx, qy, qz, qw, vx, vy, vz, valid] = 11 floats + jfloatArray result = env->NewFloatArray(11); + if (!Renderer::is_initialized() || hand < 0 || hand > 1) return result; + const auto &vr = Renderer::instance().vrSystem(); + const auto &ctrl = vr.controllers[hand]; + + // Return raw tracking-space pose — worldOrientation is applied only + // in the C++ rendering path (buffers.cpp). Java-side VRData handles + // the worldRotation transform independently. + const glm::vec3 &pos = ctrl.position; + const glm::quat &ori = ctrl.orientation; + const glm::vec3 &vel = ctrl.linearVelocity; + + float data[11] = { + pos.x, pos.y, pos.z, + ori.x, ori.y, ori.z, ori.w, + vel.x, vel.y, vel.z, + ctrl.valid ? 1.0f : 0.0f + }; + env->SetFloatArrayRegion(result, 0, 11, data); + return result; +} + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetControllerButtons( + JNIEnv *env, jclass, jint hand) { + // Returns [triggerValue, gripValue, thumbstickX, thumbstickY, + // triggerPressed, gripPressed, primaryButton, secondaryButton, + // thumbstickClick, menuButton] = 10 floats + jfloatArray result = env->NewFloatArray(10); + if (!Renderer::is_initialized() || hand < 0 || hand > 1) return result; + const auto &ctrl = Renderer::instance().vrSystem().controllers[hand]; + float data[10] = { + ctrl.triggerValue, ctrl.gripValue, + ctrl.thumbstick.x, ctrl.thumbstick.y, + ctrl.triggerPressed ? 1.0f : 0.0f, + ctrl.gripPressed ? 1.0f : 0.0f, + ctrl.primaryButton ? 1.0f : 0.0f, + ctrl.secondaryButton ? 1.0f : 0.0f, + ctrl.thumbstickClick ? 1.0f : 0.0f, + ctrl.menuButton ? 1.0f : 0.0f + }; + env->SetFloatArrayRegion(result, 0, 10, data); + return result; +} + +// ---- Haptics ---- + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeVibrate( + JNIEnv *, jclass, jint hand, jfloat amplitude, jlong durationNs, jfloat frequency) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized() || hand < 0 || hand > 1) return; + auto *xrCtx = Renderer::instance().framework()->xrContext(); + if (!xrCtx || !xrCtx->isSessionRunning()) return; + xrCtx->input().vibrate( + xrCtx->session(), static_cast(hand), amplitude, durationNs, frequency); +#endif +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeStopVibration( + JNIEnv *, jclass, jint hand) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized() || hand < 0 || hand > 1) return; + auto *xrCtx = Renderer::instance().framework()->xrContext(); + if (!xrCtx || !xrCtx->isSessionRunning()) return; + xrCtx->input().stopVibration(xrCtx->session(), static_cast(hand)); +#endif +} + +// ---- Performance stats ---- + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetPerformanceStats( + JNIEnv *env, jclass) { + // Returns [gpuFrameTimeMs, cpuFrameTimeMs, cpuWorkMs, cpuWaitMs, compositorTargetMs, fps, droppedFrames, headroom] = 8 floats + jfloatArray result = env->NewFloatArray(8); + if (!Renderer::is_initialized()) return result; + const auto &stats = Renderer::instance().vrSystem().perfStats; + float data[8] = { + stats.gpuFrameTimeMs, stats.cpuFrameTimeMs, + stats.cpuWorkMs, stats.cpuWaitMs, + stats.compositorTargetMs, stats.fps, + static_cast(stats.droppedFrames), stats.headroom + }; + env->SetFloatArrayRegion(result, 0, 8, data); + return result; +} + +// ---- World position offset ---- + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeSetWorldPosition( + JNIEnv *, jclass, jfloat x, jfloat y, jfloat z) { + if (!Renderer::is_initialized()) return; + Renderer::instance().vrSystem().worldPosition = glm::vec3(x, y, z); +} + +JNIEXPORT jfloatArray JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetWorldPosition( + JNIEnv *env, jclass) { + jfloatArray result = env->NewFloatArray(3); + if (!Renderer::is_initialized()) return result; + const auto &pos = Renderer::instance().vrSystem().worldPosition; + float data[3] = {pos.x, pos.y, pos.z}; + env->SetFloatArrayRegion(result, 0, 3, data); + return result; +} + +JNIEXPORT void JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeRecenter( + JNIEnv *, jclass) { + if (!Renderer::is_initialized()) return; + Renderer::instance().vrSystem().recenter(); +} + +// ---- Session state / device info ---- + +JNIEXPORT jfloat JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetFloorHeight( + JNIEnv *, jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return 0.0f; + auto *xrCtx = Renderer::instance().framework()->xrContext(); + if (!xrCtx) return 0.0f; + return xrCtx->floorHeight(); +#else + return 0.0f; +#endif +} + +JNIEXPORT jint JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetSessionState( + JNIEnv *, jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return 0; + auto *xrCtx = Renderer::instance().framework()->xrContext(); + if (!xrCtx) return 0; + return static_cast(xrCtx->sessionState()); +#else + return 0; +#endif +} + +JNIEXPORT jstring JNICALL Java_com_radiance_client_proxy_vulkan_VRProxy_nativeGetSystemName( + JNIEnv *env, jclass) { +#ifdef MCVR_ENABLE_OPENXR + if (!Renderer::is_initialized()) return env->NewStringUTF("unknown"); + auto *xrCtx = Renderer::instance().framework()->xrContext(); + if (!xrCtx) return env->NewStringUTF("unknown"); + return env->NewStringUTF(xrCtx->systemName().c_str()); +#else + return env->NewStringUTF("simulation"); +#endif +} diff --git a/src/core/render/buffers.cpp b/src/core/render/buffers.cpp index 0a5f9cd..199d02e 100644 --- a/src/core/render/buffers.cpp +++ b/src/core/render/buffers.cpp @@ -346,6 +346,61 @@ void Buffers::setAndUploadWorldUniformBuffer(vk::Data::WorldUBO &ubo) { ubo.cameraPos.z = world->getCameraPos().z; ubo.cameraPos.w = 0; + // VR stereo fields + auto &vr = Renderer::instance().vrSystem(); + if (vr.enabled && vr.eyeCount > 1) { + // VR path: headView (physical HMD tracking) * worldOrientation (game world rotation). + // worldOrientation is set from Java each frame and encodes all game-world + // rotation (player body yaw, stick/mouse turning, elytra roll, etc.). + // The MC cameraViewMat is NOT used here — VR has its own rotation source. + if (vr.headPose.valid) { + glm::mat4 headView = vr.headPose.viewMatrix(); + glm::mat4 worldRot = glm::mat4_cast(glm::inverse(vr.worldOrientation)); + ubo.cameraViewMat = headView * worldRot; + ubo.cameraEffectedViewMat = headView * worldRot; + ubo.cameraViewMatInv = glm::inverse(ubo.cameraViewMat); + ubo.cameraEffectedViewMatInv = glm::inverse(ubo.cameraEffectedViewMat); + + // Apply physical HMD offset (relative to worldPosition) to cameraPos. + // This lets small head movements (lean, duck) shift the camera naturally. + glm::vec3 trackingOffset = vr.headPose.position - vr.worldPosition; + // Rotate tracking-space offset into game world space via worldOrientation + glm::vec3 worldOffset = vr.worldOrientation * trackingOffset; + worldOffset *= vr.config.worldScale; + ubo.cameraPos.x += worldOffset.x; + ubo.cameraPos.y += worldOffset.y; + ubo.cameraPos.z += worldOffset.z; + } + + ubo.stereoEnabled = 1; + ubo.ipd = vr.config.ipd; + for (uint32_t eye = 0; eye < 2; eye++) { + ubo.eyeViewOffsets[eye] = vr.eyes[eye].viewOffset(); + // Compute per-eye projection correction relative to the base camera projection. + // eyeProjOffsets[eye] = eyeProjection * inverse(cameraProjMat) + // This lets the shader transform from the base projection to the per-eye asymmetric one. + glm::mat4 eyeProj = vr.eyes[eye].projectionMatrix(0.05f, 1000.0f); + ubo.eyeProjOffsets[eye] = eyeProj * glm::inverse(ubo.cameraProjMat); + } + } else { + ubo.stereoEnabled = 0; + ubo.ipd = 0.0f; + for (uint32_t eye = 0; eye < 2; eye++) { + ubo.eyeViewOffsets[eye] = glm::mat4(1.0f); + ubo.eyeProjOffsets[eye] = glm::mat4(1.0f); + } + } + + ubo.foveatedInnerRadius = foveatedInnerRadius_; + ubo.foveatedOuterBlockSize = foveatedOuterBlockSize_; + + // Foveated centre: use eye gaze if valid, otherwise screen centre + if (vr.enabled && vr.gazeValid) { + ubo.foveatedCenter = {vr.gazePoint.x, vr.gazePoint.y}; + } else { + ubo.foveatedCenter = {0.5f, 0.5f}; + } + if (worldUniformBuffer_[context->frameIndex] == nullptr) { worldUniformBuffer_[context->frameIndex] = vk::HostVisibleBuffer::create(vma, device, sizeof(vk::Data::WorldUBO), @@ -531,4 +586,12 @@ std::shared_ptr Buffers::lightMapUniformBuffer() { void Buffers::setUseJitter(bool useJitter) { useJitter_ = useJitter; +} + +void Buffers::setFoveatedInnerRadius(float radius) { + foveatedInnerRadius_ = radius; +} + +void Buffers::setFoveatedOuterBlockSize(uint32_t blockSize) { + foveatedOuterBlockSize_ = blockSize; } \ No newline at end of file diff --git a/src/core/render/buffers.hpp b/src/core/render/buffers.hpp index 52c0190..b11cc3c 100644 --- a/src/core/render/buffers.hpp +++ b/src/core/render/buffers.hpp @@ -51,6 +51,8 @@ class Buffers : public SharedObject { std::shared_ptr lightMapUniformBuffer(); void setUseJitter(bool useJitter); + void setFoveatedInnerRadius(float radius); + void setFoveatedOuterBlockSize(uint32_t blockSize); private: static constexpr uint32_t baseBlockSize = 16 * 1024; @@ -74,4 +76,6 @@ class Buffers : public SharedObject { std::shared_ptr>> importantIndexVertexBuffer_; bool useJitter_ = true; + float foveatedInnerRadius_ = 1.0f; + uint32_t foveatedOuterBlockSize_ = 2; }; diff --git a/src/core/render/modules/world/dlss/dlss_module.cpp b/src/core/render/modules/world/dlss/dlss_module.cpp index 526ce22..7b5c7bc 100644 --- a/src/core/render/modules/world/dlss/dlss_module.cpp +++ b/src/core/render/modules/world/dlss/dlss_module.cpp @@ -62,8 +62,6 @@ void DLSSModule::init(std::shared_ptr framework, std::shared_ptr> &images, @@ -211,7 +209,12 @@ void DLSSModule::build() { dlssRRInitInfo.inputSize = {inputWidth_, inputHeight_}; dlssRRInitInfo.outputSize = {outputWidth_, outputHeight_}; dlssRRInitInfo.quality = mode_; - ngxContext_->initDlssRR(dlssRRInitInfo, framework->mainCommandPool(), dlss_); + + dlssInstances_.resize(eyeCount_); + for (uint32_t eye = 0; eye < eyeCount_; eye++) { + dlssInstances_[eye] = DlssRR::create(); + ngxContext_->initDlssRR(dlssRRInitInfo, framework->mainCommandPool(), dlssInstances_[eye]); + } contexts_.resize(size); @@ -230,7 +233,10 @@ void DLSSModule::bindTexture(std::shared_ptr sampler, int index) {} void DLSSModule::preClose() { - dlss_->deinit(); + for (auto &dlss : dlssInstances_) { + dlss->deinit(); + } + dlssInstances_.clear(); } DLSSModuleContext::DLSSModuleContext(std::shared_ptr frameworkContext, @@ -238,6 +244,7 @@ DLSSModuleContext::DLSSModuleContext(std::shared_ptr framework std::shared_ptr dlssModule) : WorldModuleContext(frameworkContext, worldPipelineContext), dLSSModule(dlssModule), + eyeCount_(dlssModule->eyeCount()), hdrImage(dlssModule->hdrImages_[frameworkContext->frameIndex]), diffuseAlbedoImage(dlssModule->diffuseAlbedoImages_[frameworkContext->frameIndex]), specularAlbedoImage(dlssModule->specularAlbedoImages_[frameworkContext->frameIndex]), @@ -250,12 +257,17 @@ DLSSModuleContext::DLSSModuleContext(std::shared_ptr framework upscaledFirstHitDepthImage(dlssModule->upscaledFirstHitDepthImages_[frameworkContext->frameIndex]) {} void DLSSModuleContext::render() { + renderEye(0); +} + +void DLSSModuleContext::renderEye(uint32_t eyeIndex) { auto context = frameworkContext.lock(); auto framework = context->framework.lock(); auto worldCommandBuffer = context->worldCommandBuffer; auto mainQueueIndex = framework->physicalDevice()->mainQueueIndex(); auto module = dLSSModule.lock(); + uint32_t viewIdx = eyeCount_ > 1 ? 1 + eyeIndex : 0; { worldCommandBuffer->barriersBufferImage( @@ -355,21 +367,24 @@ void DLSSModuleContext::render() { linearDepthImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; processedImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; - module->dlss_->setResource(DlssRR::RESOURCE_COLOR_IN, hdrImage); - module->dlss_->setResource(DlssRR::RESOURCE_COLOR_OUT, processedImage); - module->dlss_->setResource(DlssRR::RESOURCE_DIFFUSE_ALBEDO, diffuseAlbedoImage); - module->dlss_->setResource(DlssRR::RESOURCE_SPECULAR_ALBEDO, specularAlbedoImage); - module->dlss_->setResource(DlssRR::RESOURCE_NORMALROUGHNESS, normalRoughnessImage); - module->dlss_->setResource(DlssRR::RESOURCE_MOTIONVECTOR, motionVectorImage); - module->dlss_->setResource(DlssRR::RESOURCE_LINEARDEPTH, linearDepthImage); - module->dlss_->setResource(DlssRR::RESOURCE_SPECULAR_HITDISTANCE, specularHitDepthImage); + auto &dlss = module->dlssInstances_[eyeIndex]; + dlss->setResource(DlssRR::RESOURCE_COLOR_IN, hdrImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_COLOR_OUT, processedImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_DIFFUSE_ALBEDO, diffuseAlbedoImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_SPECULAR_ALBEDO, specularAlbedoImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_NORMALROUGHNESS, normalRoughnessImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_MOTIONVECTOR, motionVectorImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_LINEARDEPTH, linearDepthImage, viewIdx); + dlss->setResource(DlssRR::RESOURCE_SPECULAR_HITDISTANCE, specularHitDepthImage, viewIdx); auto worldUBOBuffer = Renderer::instance().buffers()->worldUniformBuffer(); auto worldUBO = static_cast(worldUBOBuffer->mappedPtr()); if (worldUBO != nullptr) { glm::vec2 jitter = worldUBO->cameraJitter; - module->dlss_->denoise(worldCommandBuffer, glm::uvec2{module->inputWidth_, module->inputHeight_}, jitter, - worldUBO->cameraViewMat, worldUBO->cameraProjMat); + glm::mat4 eyeView = worldUBO->eyeViewOffsets[eyeIndex] * worldUBO->cameraViewMat; + glm::mat4 eyeProj = worldUBO->eyeProjOffsets[eyeIndex] * worldUBO->cameraProjMat; + dlss->denoise(worldCommandBuffer, glm::uvec2{module->inputWidth_, module->inputHeight_}, jitter, + eyeView, eyeProj); } } @@ -407,14 +422,14 @@ void DLSSModuleContext::render() { VkImageBlit imageBlit{}; imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.srcSubresource.mipLevel = 0; - imageBlit.srcSubresource.baseArrayLayer = 0; + imageBlit.srcSubresource.baseArrayLayer = eyeIndex; imageBlit.srcSubresource.layerCount = 1; imageBlit.srcOffsets[0] = {0, 0, 0}; imageBlit.srcOffsets[1] = {static_cast(firstHitDepthImage->width()), static_cast(firstHitDepthImage->height()), 1}; imageBlit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.dstSubresource.mipLevel = 0; - imageBlit.dstSubresource.baseArrayLayer = 0; + imageBlit.dstSubresource.baseArrayLayer = eyeIndex; imageBlit.dstSubresource.layerCount = 1; imageBlit.dstOffsets[0] = {0, 0, 0}; imageBlit.dstOffsets[1] = {static_cast(upscaledFirstHitDepthImage->width()), diff --git a/src/core/render/modules/world/dlss/dlss_module.hpp b/src/core/render/modules/world/dlss/dlss_module.hpp index 2eede54..6a8bc85 100644 --- a/src/core/render/modules/world/dlss/dlss_module.hpp +++ b/src/core/render/modules/world/dlss/dlss_module.hpp @@ -48,6 +48,8 @@ class DLSSModule : public WorldModule, public SharedObject { void preClose() override; + StereoMode stereoMode() const override { return StereoMode::DualInstance; } + private: static std::shared_ptr ngxContext_; @@ -62,7 +64,7 @@ class DLSSModule : public WorldModule, public SharedObject { std::vector> firstHitDepthImages_; // dlss - std::shared_ptr dlss_; + std::vector> dlssInstances_; NgxContext::SupportedSizes supportedSizes_{}; NVSDK_NGX_PerfQuality_Value mode_ = NVSDK_NGX_PerfQuality_Value_Balanced; @@ -79,6 +81,9 @@ class DLSSModule : public WorldModule, public SharedObject { struct DLSSModuleContext : public WorldModuleContext, SharedObject { std::weak_ptr dLSSModule; + StereoMode stereoMode() const override { return StereoMode::DualInstance; } + void renderEye(uint32_t eyeIndex) override; + // input std::shared_ptr hdrImage; std::shared_ptr diffuseAlbedoImage; @@ -98,4 +103,7 @@ struct DLSSModuleContext : public WorldModuleContext, SharedObject dLSSModule); void render() override; + + private: + uint32_t eyeCount_; }; \ No newline at end of file diff --git a/src/core/render/modules/world/dlss/dlss_wrapper.cpp b/src/core/render/modules/world/dlss/dlss_wrapper.cpp index 03ac256..53e4112 100644 --- a/src/core/render/modules/world/dlss/dlss_wrapper.cpp +++ b/src/core/render/modules/world/dlss/dlss_wrapper.cpp @@ -353,13 +353,18 @@ DlssRR::~DlssRR() { assert(!m_dlssdHandle && "Must call deinit"); } -void DlssRR::setResource(DlssResource resourceId, std::shared_ptr image) { +void DlssRR::setResource(DlssResource resourceId, std::shared_ptr image, uint32_t viewIndex) { assert(m_dlssdHandle); VkExtent2D size = resourceId == RESOURCE_COLOR_OUT ? m_outputSize : m_inputSize; + VkImageSubresourceRange range = vk::wholeColorSubresourceRange; + if (viewIndex > 0) { + range = vk::colorSubresourceRangeForLayer(viewIndex - 1); + } + NVSDK_NGX_Resource_VK resource = NVSDK_NGX_Create_ImageView_Resource_VK( - image->vkImageView(), image->vkImage(), vk::wholeColorSubresourceRange, image->vkFormat(), size.width, + image->vkImageView(viewIndex), image->vkImage(), range, image->vkFormat(), size.width, size.height, resourceId == RESOURCE_COLOR_OUT /*readWrite*/); m_resources[resourceId] = resource; diff --git a/src/core/render/modules/world/dlss/dlss_wrapper.hpp b/src/core/render/modules/world/dlss/dlss_wrapper.hpp index b00891e..833a092 100644 --- a/src/core/render/modules/world/dlss/dlss_wrapper.hpp +++ b/src/core/render/modules/world/dlss/dlss_wrapper.hpp @@ -168,8 +168,9 @@ class DlssRR : public SharedObject { RESOURCE_NUM }; - // Associate a DlssRR resource with a Vulkan texture - void setResource(DlssResource resourceId, std::shared_ptr image); + // Associate a DlssRR resource with a Vulkan texture. + // viewIndex: 0 = whole image (default), 1+ = per-layer view for stereo array images + void setResource(DlssResource resourceId, std::shared_ptr image, uint32_t viewIndex = 0); void resetResource(DlssResource resourceId); // Perform the actual denoising. diff --git a/src/core/render/modules/world/fsr_upscaler/upscaler_module.cpp b/src/core/render/modules/world/fsr_upscaler/upscaler_module.cpp index e19be42..5695e74 100644 --- a/src/core/render/modules/world/fsr_upscaler/upscaler_module.cpp +++ b/src/core/render/modules/world/fsr_upscaler/upscaler_module.cpp @@ -56,9 +56,16 @@ bool UpscalerModule::setOrCreateInputImages(std::vectorswapchain()->vkExtent(); - displayWidth_ = extent.width; - displayHeight_ = extent.height; + const auto &vr = Renderer::instance().vrSystem(); + if (vr.enabled && eyeCount_ > 1) { + displayWidth_ = vr.eyeRenderWidth(); + displayHeight_ = vr.eyeRenderHeight(); + } + if (displayWidth_ == 0 || displayHeight_ == 0) { + VkExtent2D extent = fw->swapchain()->vkExtent(); + displayWidth_ = extent.width; + displayHeight_ = extent.height; + } } if (renderWidth_ == 0 || renderHeight_ == 0) { @@ -77,8 +84,9 @@ bool UpscalerModule::setOrCreateInputImages(std::vectordevice(), fw->vma(), false, renderWidth_, renderHeight_, 1, formats[i], + fw->device(), fw->vma(), false, renderWidth_, renderHeight_, eyeCount_, formats[i], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[i]->createPerLayerViews(); } else if (images[i]->width() != renderWidth_ || images[i]->height() != renderHeight_) { return false; } @@ -104,18 +112,26 @@ bool UpscalerModule::setOrCreateOutputImages(std::vectorswapchain()->vkExtent(); - displayWidth_ = extent.width; - displayHeight_ = extent.height; + const auto &vr = Renderer::instance().vrSystem(); + if (vr.enabled && eyeCount_ > 1) { + displayWidth_ = vr.eyeRenderWidth(); + displayHeight_ = vr.eyeRenderHeight(); + } + if (displayWidth_ == 0 || displayHeight_ == 0) { + VkExtent2D extent = fw->swapchain()->vkExtent(); + displayWidth_ = extent.width; + displayHeight_ = extent.height; + } } } for (uint32_t i = 0; i < images.size(); i++) { if (images[i] == nullptr) { images[i] = vk::DeviceLocalImage::create( - fw->device(), fw->vma(), false, displayWidth_, displayHeight_, 1, formats[i], + fw->device(), fw->vma(), false, displayWidth_, displayHeight_, eyeCount_, formats[i], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + if (eyeCount_ > 1) images[i]->createPerLayerViews(); } else if (images[i]->width() != displayWidth_ || images[i]->height() != displayHeight_) { return false; } @@ -130,7 +146,8 @@ void UpscalerModule::build() { auto wp = worldPipeline_.lock(); uint32_t size = fw->swapchain()->imageCount(); - fsr3_ = std::make_shared(); + // Create per-eye FSR3 instances (DualInstance) + fsr3Upscalers_.resize(eyeCount_); mcvr::UpscalerConfig config{}; config.device = fw->device()->vkDevice(); @@ -152,13 +169,18 @@ void UpscalerModule::build() { if (!fsr3Enabled_) { initialized_ = false; - } else if (!fsr3_->initialize(config)) { - std::cerr << "UpscalerModule: Failed to initialize FSR3" << std::endl; - initialized_ = false; } else { initialized_ = true; + for (uint32_t eye = 0; eye < eyeCount_; eye++) { + fsr3Upscalers_[eye] = std::make_shared(); + if (!fsr3Upscalers_[eye]->initialize(config)) { + std::cerr << "UpscalerModule: Failed to initialize FSR3 for eye " << eye << std::endl; + initialized_ = false; + } + } } + uint32_t totalCtx = size * eyeCount_; initDescriptorTables(); initImages(); initPipeline(); @@ -176,18 +198,20 @@ void UpscalerModule::build() { contexts_[i]->outputImage = outputImages_[i][0]; contexts_[i]->upscaledFirstHitDepthImage = outputImages_[i][1]; - contexts_[i]->deviceDepthImage = deviceDepthImages_[i]; - contexts_[i]->fsrMotionVectorImage = fsrMotionVectorImages_[i]; - contexts_[i]->depthDescriptorTable = depthDescriptorTables_[i]; + // Store eye-0 references for mono compatibility + contexts_[i]->deviceDepthImage = deviceDepthImages_[i * eyeCount_]; + contexts_[i]->fsrMotionVectorImage = fsrMotionVectorImages_[i * eyeCount_]; + contexts_[i]->depthDescriptorTable = depthDescriptorTables_[i * eyeCount_]; } } void UpscalerModule::initDescriptorTables() { auto fw = framework_.lock(); uint32_t size = fw->swapchain()->imageCount(); - depthDescriptorTables_.resize(size); + uint32_t totalCtx = size * eyeCount_; + depthDescriptorTables_.resize(totalCtx); - for (uint32_t i = 0; i < size; i++) { + for (uint32_t i = 0; i < totalCtx; i++) { depthDescriptorTables_[i] = vk::DescriptorTableBuilder{} .beginDescriptorLayoutSet() .beginDescriptorLayoutSetBinding() @@ -229,8 +253,14 @@ void UpscalerModule::initDescriptorTables() { void UpscalerModule::initImages() { auto fw = framework_.lock(); uint32_t size = fw->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; - for (uint32_t i = 0; i < size; i++) { + deviceDepthImages_.resize(totalCtx); + fsrMotionVectorImages_.resize(totalCtx); + fsr3ColorImages_.resize(totalCtx); + fsr3OutputImages_.resize(totalCtx); + + for (uint32_t i = 0; i < totalCtx; i++) { deviceDepthImages_[i] = vk::DeviceLocalImage::create( fw->device(), fw->vma(), false, renderWidth_, renderHeight_, 1, VK_FORMAT_R32_SFLOAT, VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); @@ -240,6 +270,15 @@ void UpscalerModule::initImages() { fsrMotionVectorImages_[i] = vk::DeviceLocalImage::create( fw->device(), fw->vma(), false, renderWidth_, renderHeight_, 1, VK_FORMAT_R16G16_SFLOAT, VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + + // Per-eye single-layer intermediates for FSR3 SDK (can't handle array images) + fsr3ColorImages_[i] = vk::DeviceLocalImage::create( + fw->device(), fw->vma(), false, renderWidth_, renderHeight_, 1, VK_FORMAT_R16G16B16A16_SFLOAT, + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + + fsr3OutputImages_[i] = vk::DeviceLocalImage::create( + fw->device(), fw->vma(), false, displayWidth_, displayHeight_, 1, VK_FORMAT_R16G16B16A16_SFLOAT, + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT); } } @@ -293,10 +332,15 @@ void UpscalerModule::bindTexture(std::shared_ptr sampler, std::shar int index) {} void UpscalerModule::preClose() { - if (fsr3_) { - fsr3_->destroy(); - fsr3_.reset(); + for (auto &fsr : fsr3Upscalers_) { + if (fsr) { + fsr->destroy(); + fsr.reset(); + } } + fsr3Upscalers_.clear(); + fsr3ColorImages_.clear(); + fsr3OutputImages_.clear(); initialized_ = false; } @@ -328,6 +372,7 @@ bool UpscalerModuleContext::checkCameraReset(const glm::vec3 &cameraPos, const g module->firstFrame_ = false; module->lastCameraPos_ = cameraPos; module->lastCameraDir_ = cameraDir; + lastResetResult_ = true; return true; } @@ -337,6 +382,7 @@ bool UpscalerModuleContext::checkCameraReset(const glm::vec3 &cameraPos, const g module->lastCameraPos_ = cameraPos; module->lastCameraDir_ = cameraDir; + lastResetResult_ = shouldReset; return shouldReset; } @@ -361,6 +407,10 @@ float UpscalerModuleContext::getSmoothDeltaTime() { } void UpscalerModuleContext::render() { + renderEye(0); +} + +void UpscalerModuleContext::renderEye(uint32_t eyeIndex) { auto module = upscalerModule_.lock(); if (!module) return; @@ -368,6 +418,13 @@ void UpscalerModuleContext::render() { auto worldCommandBuffer = fwContext->worldCommandBuffer; auto mainQueueIndex = fwContext->framework.lock()->physicalDevice()->mainQueueIndex(); + uint32_t dtIdx = fwContext->frameIndex * module->eyeCount() + eyeIndex; + auto eyeDeviceDepth = module->deviceDepthImages_[dtIdx]; + auto eyeFsrMotionVector = module->fsrMotionVectorImages_[dtIdx]; + auto eyeDepthDT = module->depthDescriptorTables_[dtIdx]; + auto eyeColorIntermediate = module->fsr3ColorImages_[dtIdx]; + auto eyeOutputIntermediate = module->fsr3OutputImages_[dtIdx]; + if (!module->fsr3Enabled_) { worldCommandBuffer->barriersBufferImage( {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR, @@ -417,10 +474,10 @@ void UpscalerModuleContext::render() { upscaledFirstHitDepthImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; VkImageBlit colorBlit{}; - colorBlit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + colorBlit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; colorBlit.srcOffsets[1] = {static_cast(inputColorImage->width()), static_cast(inputColorImage->height()), 1}; - colorBlit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + colorBlit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; colorBlit.dstOffsets[1] = {static_cast(outputImage->width()), static_cast(outputImage->height()), 1}; @@ -429,10 +486,10 @@ void UpscalerModuleContext::render() { VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &colorBlit, VK_FILTER_LINEAR); VkImageBlit depthBlit{}; - depthBlit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + depthBlit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; depthBlit.srcOffsets[1] = {static_cast(inputFirstHitDepthImage->width()), static_cast(inputFirstHitDepthImage->height()), 1}; - depthBlit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + depthBlit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; depthBlit.dstOffsets[1] = {static_cast(upscaledFirstHitDepthImage->width()), static_cast(upscaledFirstHitDepthImage->height()), 1}; @@ -456,19 +513,19 @@ void UpscalerModuleContext::render() { return; } - if (!module->initialized_ || !module->fsr3_) return; + if (!module->initialized_ || module->fsr3Upscalers_.size() <= eyeIndex || !module->fsr3Upscalers_[eyeIndex]) return; auto buffers = Renderer::instance().buffers(); auto worldUBO = static_cast(buffers->worldUniformBuffer()->mappedPtr()); - // Barrier for inputs and outputs (prepare for depth conversion + FSR3) + // Barrier shared pipeline inputs + per-eye intermediates to GENERAL worldCommandBuffer->barriersBufferImage( {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = inputColorImage->imageLayout(), - .newLayout = VK_IMAGE_LAYOUT_GENERAL, // keep GENERAL until FSR3 barrier + .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, .image = inputColorImage, @@ -488,7 +545,7 @@ void UpscalerModuleContext::render() { .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, .oldLayout = inputMotionVectorImage->imageLayout(), - .newLayout = VK_IMAGE_LAYOUT_GENERAL, // keep GENERAL until FSR3 barrier + .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, .image = inputMotionVectorImage, @@ -497,45 +554,73 @@ void UpscalerModuleContext::render() { .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, - .oldLayout = deviceDepthImage->imageLayout(), + .oldLayout = eyeDeviceDepth->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = deviceDepthImage, + .image = eyeDeviceDepth, .subresourceRange = vk::wholeColorSubresourceRange}, {.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, - .oldLayout = fsrMotionVectorImage->imageLayout(), + .oldLayout = eyeFsrMotionVector->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = fsrMotionVectorImage, + .image = eyeFsrMotionVector, .subresourceRange = vk::wholeColorSubresourceRange}, {.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .oldLayout = outputImage->imageLayout(), + .oldLayout = eyeOutputIntermediate->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = outputImage, + .image = eyeOutputIntermediate, .subresourceRange = vk::wholeColorSubresourceRange}}); inputColorImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; inputDepthImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; inputMotionVectorImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; - deviceDepthImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; - fsrMotionVectorImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; - outputImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; + eyeDeviceDepth->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; + eyeFsrMotionVector->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; + eyeOutputIntermediate->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; + + // Copy inputColorImage layer[eyeIndex] → per-eye single-layer intermediate for FSR3 + { + VkImageCopy copyRegion{}; + copyRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; + copyRegion.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copyRegion.extent = {module->renderWidth_, module->renderHeight_, 1}; + + worldCommandBuffer->barriersBufferImage( + {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .oldLayout = eyeColorIntermediate->imageLayout(), + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .image = eyeColorIntermediate, + .subresourceRange = vk::wholeColorSubresourceRange}}); - // Linear to Device Depth + FSR motion vector prepare - depthDescriptorTable->bindImage(inputDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0); - depthDescriptorTable->bindImage(deviceDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 1); - depthDescriptorTable->bindImage(inputMotionVectorImage, VK_IMAGE_LAYOUT_GENERAL, 0, 2); - depthDescriptorTable->bindImage(fsrMotionVectorImage, VK_IMAGE_LAYOUT_GENERAL, 0, 3); + vkCmdCopyImage(worldCommandBuffer->vkCommandBuffer(), + inputColorImage->vkImage(), VK_IMAGE_LAYOUT_GENERAL, + eyeColorIntermediate->vkImage(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, ©Region); + + eyeColorIntermediate->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + } + + // Depth conversion: bind per-layer views for array inputs + uint32_t viewIdx = (module->eyeCount() > 1) ? (1 + eyeIndex) : 0; + eyeDepthDT->bindImage(inputDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0, viewIdx); + eyeDepthDT->bindImage(eyeDeviceDepth, VK_IMAGE_LAYOUT_GENERAL, 0, 1); + eyeDepthDT->bindImage(inputMotionVectorImage, VK_IMAGE_LAYOUT_GENERAL, 0, 2, viewIdx); + eyeDepthDT->bindImage(eyeFsrMotionVector, VK_IMAGE_LAYOUT_GENERAL, 0, 3); struct PushConstants { float cameraNear; float cameraFar; @@ -546,10 +631,10 @@ void UpscalerModuleContext::render() { } pushConstants{0.1f, 10000.0f, module->renderWidth_, module->renderHeight_, worldUBO->cameraJitter.x, worldUBO->cameraJitter.y}; - worldCommandBuffer->bindDescriptorTable(depthDescriptorTable, VK_PIPELINE_BIND_POINT_COMPUTE) + worldCommandBuffer->bindDescriptorTable(eyeDepthDT, VK_PIPELINE_BIND_POINT_COMPUTE) ->bindComputePipeline(module->depthConversionPipeline_); - vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), depthDescriptorTable->vkPipelineLayout(), + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), eyeDepthDT->vkPipelineLayout(), VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pushConstants), &pushConstants); vkCmdDispatch(worldCommandBuffer->vkCommandBuffer(), (module->renderWidth_ + 15) / 16, @@ -560,81 +645,137 @@ void UpscalerModuleContext::render() { .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT}}); - // Transition FSR3 inputs to shader read-only + // Transition per-eye intermediates to shader read-only for FSR3 worldCommandBuffer->barriersBufferImage( - {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, - .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_SHADER_READ_BIT, + {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, - .oldLayout = inputColorImage->imageLayout(), + .oldLayout = eyeColorIntermediate->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = inputColorImage, + .image = eyeColorIntermediate, .subresourceRange = vk::wholeColorSubresourceRange}, {.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_SHADER_READ_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, - .oldLayout = deviceDepthImage->imageLayout(), + .oldLayout = eyeDeviceDepth->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = deviceDepthImage, + .image = eyeDeviceDepth, .subresourceRange = vk::wholeColorSubresourceRange}, {.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_SHADER_READ_BIT, .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, - .oldLayout = fsrMotionVectorImage->imageLayout(), + .oldLayout = eyeFsrMotionVector->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = fsrMotionVectorImage, + .image = eyeFsrMotionVector, .subresourceRange = vk::wholeColorSubresourceRange}}); - inputColorImage->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - deviceDepthImage->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - fsrMotionVectorImage->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + eyeColorIntermediate->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + eyeDeviceDepth->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + eyeFsrMotionVector->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; - // FSR3 Dispatch + // FSR3 Dispatch with per-eye single-layer intermediates mcvr::UpscalerInput input{}; input.commandBuffer = worldCommandBuffer->vkCommandBuffer(); - input.colorImage = inputColorImage->vkImage(); - input.colorImageView = inputColorImage->vkImageView(); - input.colorLayout = inputColorImage->imageLayout(); - input.depthImage = deviceDepthImage->vkImage(); - input.depthImageView = deviceDepthImage->vkImageView(); - input.depthFormat = deviceDepthImage->vkFormat(); - input.motionVectorImage = fsrMotionVectorImage->vkImage(); - input.motionVectorImageView = fsrMotionVectorImage->vkImageView(); - input.outputImage = outputImage->vkImage(); - input.outputImageView = outputImage->vkImageView(); + input.colorImage = eyeColorIntermediate->vkImage(); + input.colorImageView = eyeColorIntermediate->vkImageView(); + input.colorLayout = eyeColorIntermediate->imageLayout(); + input.depthImage = eyeDeviceDepth->vkImage(); + input.depthImageView = eyeDeviceDepth->vkImageView(); + input.depthFormat = eyeDeviceDepth->vkFormat(); + input.motionVectorImage = eyeFsrMotionVector->vkImage(); + input.motionVectorImageView = eyeFsrMotionVector->vkImageView(); + input.outputImage = eyeOutputIntermediate->vkImage(); + input.outputImageView = eyeOutputIntermediate->vkImageView(); input.outputLayout = VK_IMAGE_LAYOUT_GENERAL; input.renderWidth = module->renderWidth_; input.renderHeight = module->renderHeight_; input.displayWidth = module->displayWidth_; input.displayHeight = module->displayHeight_; - // FSR expects jitter in the camera offset convention (sign may differ from our ray jitter) input.jitterOffsetX = -worldUBO->cameraJitter.x; input.jitterOffsetY = -worldUBO->cameraJitter.y; - input.motionVectorScaleX = 1.0f; // Shader outputs pixel-space MVs + input.motionVectorScaleX = 1.0f; input.motionVectorScaleY = 1.0f; input.cameraNear = 0.1f; input.cameraFar = 10000.0f; - input.cameraFovVertical = 2.0f * atan(1.0f / worldUBO->cameraProjMat[1][1]); + { + glm::mat4 eyeProj = worldUBO->eyeProjOffsets[eyeIndex] * worldUBO->cameraProjMat; + input.cameraFovVertical = 2.0f * atan(1.0f / eyeProj[1][1]); + } input.frameTimeDelta = getSmoothDeltaTime(); input.preExposure = module->preExposure_; - input.reset = checkCameraReset(glm::vec3(worldUBO->cameraPos), - glm::vec3(worldUBO->cameraViewMat[0][2], worldUBO->cameraViewMat[1][2], - worldUBO->cameraViewMat[2][2])); + input.reset = (eyeIndex == 0) + ? checkCameraReset(glm::vec3(worldUBO->cameraPos), + glm::vec3(worldUBO->cameraViewMat[0][2], worldUBO->cameraViewMat[1][2], + worldUBO->cameraViewMat[2][2])) + : lastResetResult_; input.enableSharpening = true; input.sharpness = module->sharpness_; - module->fsr3_->dispatch(input); + module->fsr3Upscalers_[eyeIndex]->dispatch(input); + eyeOutputIntermediate->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; + + // Copy per-eye FSR3 output → outputImage layer[eyeIndex] + { + worldCommandBuffer->barriersBufferImage( + {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .oldLayout = eyeOutputIntermediate->imageLayout(), + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .image = eyeOutputIntermediate, + .subresourceRange = vk::wholeColorSubresourceRange}, + {.srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .oldLayout = outputImage->imageLayout(), + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .image = outputImage, + .subresourceRange = vk::wholeColorSubresourceRange}}); + + eyeOutputIntermediate->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + outputImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + + VkImageCopy copyRegion{}; + copyRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copyRegion.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; + copyRegion.extent = {module->displayWidth_, module->displayHeight_, 1}; + + vkCmdCopyImage(worldCommandBuffer->vkCommandBuffer(), + eyeOutputIntermediate->vkImage(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + outputImage->vkImage(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, ©Region); + } + + // Transition output to GENERAL for downstream + worldCommandBuffer->barriersBufferImage( + {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .oldLayout = outputImage->imageLayout(), + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .image = outputImage, + .subresourceRange = vk::wholeColorSubresourceRange}}); outputImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; - // Blit first hit depth + // Blit first hit depth per-layer worldCommandBuffer->barriersBufferImage( {}, {{.srcStageMask = VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR, .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, @@ -661,9 +802,9 @@ void UpscalerModuleContext::render() { upscaledFirstHitDepthImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; VkImageBlit blit{}; - blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; blit.srcOffsets[1] = {static_cast(module->renderWidth_), static_cast(module->renderHeight_), 1}; - blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; blit.dstOffsets[1] = {static_cast(module->displayWidth_), static_cast(module->displayHeight_), 1}; vkCmdBlitImage(worldCommandBuffer->vkCommandBuffer(), inputFirstHitDepthImage->vkImage(), diff --git a/src/core/render/modules/world/fsr_upscaler/upscaler_module.hpp b/src/core/render/modules/world/fsr_upscaler/upscaler_module.hpp index d696836..77cf1ba 100644 --- a/src/core/render/modules/world/fsr_upscaler/upscaler_module.hpp +++ b/src/core/render/modules/world/fsr_upscaler/upscaler_module.hpp @@ -54,6 +54,8 @@ class UpscalerModule : public WorldModule, public SharedObject { void preClose() override; + StereoMode stereoMode() const override { return StereoMode::DualInstance; } + static void getRenderResolution(uint32_t displayWidth, uint32_t displayHeight, QualityMode mode, uint32_t* outRenderWidth, uint32_t* outRenderHeight); @@ -75,16 +77,20 @@ class UpscalerModule : public WorldModule, public SharedObject { float preExposure_ = 1.0f; bool fsr3Enabled_ = true; - // FSR3 implementation - std::shared_ptr fsr3_; + // FSR3 implementation (per-eye for stereo DualInstance) + std::vector> fsr3Upscalers_; bool initialized_ = false; - // Depth conversion resources + // Depth conversion resources (indexed by frameIndex * eyeCount + eyeIndex) std::vector> deviceDepthImages_; std::vector> fsrMotionVectorImages_; std::vector> depthDescriptorTables_; std::shared_ptr depthConversionPipeline_; + // Per-eye single-layer intermediates for FSR3 (can't handle array images) + std::vector> fsr3ColorImages_; + std::vector> fsr3OutputImages_; + // Camera state for reset detection glm::vec3 lastCameraPos_ = glm::vec3(0.0f); glm::vec3 lastCameraDir_ = glm::vec3(0.0f, 0.0f, -1.0f); @@ -102,6 +108,8 @@ class UpscalerModuleContext : public WorldModuleContext { std::shared_ptr upscalerModule); void render() override; + StereoMode stereoMode() const override { return StereoMode::DualInstance; } + void renderEye(uint32_t eyeIndex) override; // Inputs (render resolution) std::shared_ptr inputColorImage; @@ -127,4 +135,5 @@ class UpscalerModuleContext : public WorldModuleContext { std::deque frameTimes_; std::chrono::high_resolution_clock::time_point lastFrameTime_; bool timingFirstFrame_ = true; + bool lastResetResult_ = false; }; diff --git a/src/core/render/modules/world/nrd/nrd_module.cpp b/src/core/render/modules/world/nrd/nrd_module.cpp index 9b2a797..02c53a5 100644 --- a/src/core/render/modules/world/nrd/nrd_module.cpp +++ b/src/core/render/modules/world/nrd/nrd_module.cpp @@ -37,9 +37,10 @@ bool NrdModule::setOrCreateInputImages(std::vector 1) images[index]->createPerLayerViews(); }; for (uint32_t i = 0; i < images.size(); i++) { @@ -101,10 +102,25 @@ void NrdModule::build() { auto worldPipeline = worldPipeline_.lock(); uint32_t size = framework->swapchain()->imageCount(); - m_wrapper = std::make_shared(); + // Create one NrdWrapper per eye + m_wrappers.resize(eyeCount_); + for (uint32_t eye = 0; eye < eyeCount_; eye++) { + m_wrappers[eye] = std::make_shared(); + bool ok = m_wrappers[eye]->init(m_device, m_vma, framework->physicalDevice(), width_, height_, size); + if (!ok) { + std::cerr << "[NrdModule] init failed for eye " << eye << std::endl; + m_wrappers[eye].reset(); + } + } + + uint32_t totalCtx = size * eyeCount_; + + // Per-eye denoised images: indexed by [frameIndex * eyeCount_ + eyeIndex] + denoisedDiffuseRadianceImages_.resize(totalCtx); + denoisedSpecularRadianceImages_.resize(totalCtx); auto createInternal = [&](std::vector> &images) { - for (uint32_t i = 0; i < size; i++) { + for (uint32_t i = 0; i < totalCtx; i++) { if (images[i] != nullptr) continue; images[i] = vk::DeviceLocalImage::create(m_device, m_vma, false, width_, height_, 1, VK_FORMAT_R16G16B16A16_SFLOAT, @@ -114,14 +130,19 @@ void NrdModule::build() { createInternal(denoisedDiffuseRadianceImages_); createInternal(denoisedSpecularRadianceImages_); - bool ok = m_wrapper->init(m_device, m_vma, framework->physicalDevice(), width_, height_, size); - if (!ok) { - std::cerr << "[NrdModule] init failed." << std::endl; - m_wrapper.reset(); + // Per-eye single-layer copy of linearDepth for NRD IN_VIEWZ (because NRD can't use array image views) + if (eyeCount_ > 1 && linearDepthImages_[0]) { + VkFormat depthFmt = linearDepthImages_[0]->vkFormat(); + m_nrdLinearDepthImages.resize(totalCtx); + for (uint32_t i = 0; i < totalCtx; i++) { + m_nrdLinearDepthImages[i] = vk::DeviceLocalImage::create( + m_device, m_vma, false, width_, height_, 1, depthFmt, + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + } } - createCompositionPipeline(m_device, size); - createPreparePipeline(m_device, size); + createCompositionPipeline(m_device, totalCtx); + createPreparePipeline(m_device, totalCtx); contexts_.resize(size); for (int i = 0; i < size; i++) { @@ -166,7 +187,8 @@ void NrdModule::bindTexture(std::shared_ptr sampler, int index) {} void NrdModule::preClose() { - m_wrapper.reset(); + m_wrappers.clear(); + m_nrdLinearDepthImages.clear(); composeDescriptorTables_.clear(); prepareDescriptorTables_.clear(); composeSamplers_ = {}; @@ -192,12 +214,16 @@ NrdModuleContext::NrdModuleContext(std::shared_ptr frameworkCo diffuseHitDepthImage(nrdModule->diffuseHitDepthImages_[frameworkContext->frameIndex]), specularHitDepthImage(nrdModule->specularHitDepthImages_[frameworkContext->frameIndex]), denoisedRadianceImage(nrdModule->denoisedRadianceImages_[frameworkContext->frameIndex]), - denoisedDiffuseRadianceImage(nrdModule->denoisedDiffuseRadianceImages_[frameworkContext->frameIndex]), - denoisedSpecularRadianceImage(nrdModule->denoisedSpecularRadianceImages_[frameworkContext->frameIndex]) {} + denoisedDiffuseRadianceImage(nrdModule->denoisedDiffuseRadianceImages_[frameworkContext->frameIndex * nrdModule->eyeCount()]), + denoisedSpecularRadianceImage(nrdModule->denoisedSpecularRadianceImages_[frameworkContext->frameIndex * nrdModule->eyeCount()]) {} void NrdModuleContext::render() { + renderEye(0); +} + +void NrdModuleContext::renderEye(uint32_t eyeIndex) { auto module = nrdModule.lock(); - if (!module || !module->wrapper()) return; + if (!module || !module->wrapper(eyeIndex)) return; auto context = frameworkContext.lock(); auto worldCommandBuffer = context->worldCommandBuffer; @@ -209,13 +235,21 @@ void NrdModuleContext::render() { if (module->width_ == 0 || module->height_ == 0) return; + uint32_t dtIdx = context->frameIndex * module->eyeCount_ + eyeIndex; + + // Compute per-eye view and projection matrices + glm::mat4 eyeView = worldUBO->eyeViewOffsets[eyeIndex] * worldUBO->cameraViewMat; + glm::mat4 eyeProj = worldUBO->eyeProjOffsets[eyeIndex] * worldUBO->cameraProjMat; + glm::mat4 lastEyeView = lastWorldUBO->eyeViewOffsets[eyeIndex] * lastWorldUBO->cameraViewMat; + glm::mat4 lastEyeProj = lastWorldUBO->eyeProjOffsets[eyeIndex] * lastWorldUBO->cameraProjMat; + nrd::CommonSettings commonSettings = {}; for (int i = 0; i < 4; ++i) { for (int j = 0; j < 4; ++j) { - commonSettings.viewToClipMatrix[i * 4 + j] = worldUBO->cameraProjMat[i][j]; - commonSettings.viewToClipMatrixPrev[i * 4 + j] = lastWorldUBO->cameraProjMat[i][j]; - commonSettings.worldToViewMatrix[i * 4 + j] = worldUBO->cameraViewMat[i][j]; - commonSettings.worldToViewMatrixPrev[i * 4 + j] = lastWorldUBO->cameraViewMat[i][j]; + commonSettings.viewToClipMatrix[i * 4 + j] = eyeProj[i][j]; + commonSettings.viewToClipMatrixPrev[i * 4 + j] = lastEyeProj[i][j]; + commonSettings.worldToViewMatrix[i * 4 + j] = eyeView[i][j]; + commonSettings.worldToViewMatrixPrev[i * 4 + j] = lastEyeView[i][j]; } } @@ -265,25 +299,27 @@ void NrdModuleContext::render() { reblurSettings.historyFixFrameNum = 3; reblurSettings.hitDistanceReconstructionMode = nrd::HitDistanceReconstructionMode::AREA_5X5; + uint32_t viewIdx = (module->eyeCount_ > 1) ? (1 + eyeIndex) : 0; + { - auto prepareTable = module->prepareDescriptorTables_[context->frameIndex]; - prepareTable->bindImage(motionVectorImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0); - prepareTable->bindImage(normalRoughnessImage, VK_IMAGE_LAYOUT_GENERAL, 0, 1); - prepareTable->bindImage(linearDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 2); - prepareTable->bindImage(diffuseRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 3); - prepareTable->bindImage(specularRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 4); - prepareTable->bindImage(module->m_nrdMotionVectorImages[context->frameIndex], VK_IMAGE_LAYOUT_GENERAL, 0, 5); - prepareTable->bindImage(module->m_nrdNormalRoughnessImages[context->frameIndex], VK_IMAGE_LAYOUT_GENERAL, 0, 6); - prepareTable->bindImage(module->m_nrdDiffuseRadianceImages[context->frameIndex], VK_IMAGE_LAYOUT_GENERAL, 0, 7); - prepareTable->bindImage(module->m_nrdSpecularRadianceImages[context->frameIndex], VK_IMAGE_LAYOUT_GENERAL, 0, + auto prepareTable = module->prepareDescriptorTables_[dtIdx]; + prepareTable->bindImage(motionVectorImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0, viewIdx); + prepareTable->bindImage(normalRoughnessImage, VK_IMAGE_LAYOUT_GENERAL, 0, 1, viewIdx); + prepareTable->bindImage(linearDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 2, viewIdx); + prepareTable->bindImage(diffuseRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 3, viewIdx); + prepareTable->bindImage(specularRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 4, viewIdx); + prepareTable->bindImage(module->m_nrdMotionVectorImages[dtIdx], VK_IMAGE_LAYOUT_GENERAL, 0, 5); + prepareTable->bindImage(module->m_nrdNormalRoughnessImages[dtIdx], VK_IMAGE_LAYOUT_GENERAL, 0, 6); + prepareTable->bindImage(module->m_nrdDiffuseRadianceImages[dtIdx], VK_IMAGE_LAYOUT_GENERAL, 0, 7); + prepareTable->bindImage(module->m_nrdSpecularRadianceImages[dtIdx], VK_IMAGE_LAYOUT_GENERAL, 0, 8); - prepareTable->bindImage(diffuseAlbedoImage, VK_IMAGE_LAYOUT_GENERAL, 0, 9); - prepareTable->bindImage(specularAlbedoImage, VK_IMAGE_LAYOUT_GENERAL, 0, 10); - prepareTable->bindImage(directRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 11); - prepareTable->bindImage(clearRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 12); + prepareTable->bindImage(diffuseAlbedoImage, VK_IMAGE_LAYOUT_GENERAL, 0, 9, viewIdx); + prepareTable->bindImage(specularAlbedoImage, VK_IMAGE_LAYOUT_GENERAL, 0, 10, viewIdx); + prepareTable->bindImage(directRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 11, viewIdx); + prepareTable->bindImage(clearRadianceImage, VK_IMAGE_LAYOUT_GENERAL, 0, 12, viewIdx); prepareTable->bindBuffer(Renderer::instance().buffers()->worldUniformBuffer(), 0, 13); - prepareTable->bindImage(diffuseHitDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 14); - prepareTable->bindImage(specularHitDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 15); + prepareTable->bindImage(diffuseHitDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 14, viewIdx); + prepareTable->bindImage(specularHitDepthImage, VK_IMAGE_LAYOUT_GENERAL, 0, 15, viewIdx); worldCommandBuffer->bindDescriptorTable(prepareTable, VK_PIPELINE_BIND_POINT_COMPUTE) ->bindComputePipeline(module->preparePipeline_); @@ -300,7 +336,7 @@ void NrdModuleContext::render() { .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = module->m_nrdMotionVectorImages[context->frameIndex], + .image = module->m_nrdMotionVectorImages[dtIdx], .subresourceRange = vk::wholeColorSubresourceRange, }, { @@ -312,7 +348,7 @@ void NrdModuleContext::render() { .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = module->m_nrdNormalRoughnessImages[context->frameIndex], + .image = module->m_nrdNormalRoughnessImages[dtIdx], .subresourceRange = vk::wholeColorSubresourceRange, }, { @@ -324,7 +360,7 @@ void NrdModuleContext::render() { .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = module->m_nrdDiffuseRadianceImages[context->frameIndex], + .image = module->m_nrdDiffuseRadianceImages[dtIdx], .subresourceRange = vk::wholeColorSubresourceRange, }, { @@ -336,26 +372,60 @@ void NrdModuleContext::render() { .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = module->m_nrdSpecularRadianceImages[context->frameIndex], + .image = module->m_nrdSpecularRadianceImages[dtIdx], + .subresourceRange = vk::wholeColorSubresourceRange, + }}); + } + + module->wrapper(eyeIndex)->updateSettings(commonSettings, reblurSettings); + + // For stereo, copy the relevant layer of linearDepthImage to a single-layer image + // because NRD wrapper uses the default image view (which would be 2D_ARRAY for array images) + auto viewZImage = linearDepthImage; + if (module->eyeCount_ > 1 && !module->m_nrdLinearDepthImages.empty()) { + auto dst = module->m_nrdLinearDepthImages[dtIdx]; + VkImageCopy region{}; + region.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eyeIndex, 1}; + region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.extent = {module->width_, module->height_, 1}; + vkCmdCopyImage(worldCommandBuffer->vkCommandBuffer(), + linearDepthImage->vkImage(), VK_IMAGE_LAYOUT_GENERAL, + dst->vkImage(), VK_IMAGE_LAYOUT_GENERAL, + 1, ®ion); + worldCommandBuffer->barriersBufferImage( + {}, {{ + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst, .subresourceRange = vk::wholeColorSubresourceRange, }}); + viewZImage = dst; } - module->wrapper()->updateSettings(commonSettings, reblurSettings); + auto denoisedDiffuse = module->denoisedDiffuseRadianceImages_[dtIdx]; + auto denoisedSpecular = module->denoisedSpecularRadianceImages_[dtIdx]; std::map> userTextures; - userTextures[nrd::ResourceType::IN_MV] = module->m_nrdMotionVectorImages[context->frameIndex]; - userTextures[nrd::ResourceType::IN_NORMAL_ROUGHNESS] = module->m_nrdNormalRoughnessImages[context->frameIndex]; - userTextures[nrd::ResourceType::IN_VIEWZ] = linearDepthImage; - userTextures[nrd::ResourceType::IN_DIFF_RADIANCE_HITDIST] = module->m_nrdDiffuseRadianceImages[context->frameIndex]; + userTextures[nrd::ResourceType::IN_MV] = module->m_nrdMotionVectorImages[dtIdx]; + userTextures[nrd::ResourceType::IN_NORMAL_ROUGHNESS] = module->m_nrdNormalRoughnessImages[dtIdx]; + userTextures[nrd::ResourceType::IN_VIEWZ] = viewZImage; + userTextures[nrd::ResourceType::IN_DIFF_RADIANCE_HITDIST] = module->m_nrdDiffuseRadianceImages[dtIdx]; userTextures[nrd::ResourceType::IN_SPEC_RADIANCE_HITDIST] = - module->m_nrdSpecularRadianceImages[context->frameIndex]; - userTextures[nrd::ResourceType::OUT_DIFF_RADIANCE_HITDIST] = denoisedDiffuseRadianceImage; - userTextures[nrd::ResourceType::OUT_SPEC_RADIANCE_HITDIST] = denoisedSpecularRadianceImage; + module->m_nrdSpecularRadianceImages[dtIdx]; + userTextures[nrd::ResourceType::OUT_DIFF_RADIANCE_HITDIST] = denoisedDiffuse; + userTextures[nrd::ResourceType::OUT_SPEC_RADIANCE_HITDIST] = denoisedSpecular; - module->wrapper()->denoise(worldCommandBuffer->vkCommandBuffer(), context->frameIndex, userTextures); + module->wrapper(eyeIndex)->denoise(worldCommandBuffer->vkCommandBuffer(), context->frameIndex, userTextures); std::vector barriers; + bool stereo = module->eyeCount_ > 1; + auto ensureGeneral = [&](std::shared_ptr img) { if (!img) return; if (img->imageLayout() == VK_IMAGE_LAYOUT_GENERAL) return; @@ -399,23 +469,30 @@ void NrdModuleContext::render() { }); img->imageLayout() = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; }; - addBarrier(diffuseAlbedoImage); - addBarrier(specularAlbedoImage); - addBarrier(normalRoughnessImage); - addBarrier(linearDepthImage); - addBarrier(denoisedDiffuseRadianceImage); - addBarrier(denoisedSpecularRadianceImage); - addBarrier(clearRadianceImage); - addBarrier(baseEmissionImage); - if (directRadianceImage->imageLayout() == VK_IMAGE_LAYOUT_GENERAL) { addBarrier(directRadianceImage); } + // In stereo, shared pipeline input images stay in GENERAL (both eyes need them). + // Compose will bind them with GENERAL layout. + if (!stereo) { + addBarrier(diffuseAlbedoImage); + addBarrier(specularAlbedoImage); + addBarrier(normalRoughnessImage); + addBarrier(linearDepthImage); + addBarrier(clearRadianceImage); + addBarrier(baseEmissionImage); + if (directRadianceImage->imageLayout() == VK_IMAGE_LAYOUT_GENERAL) { addBarrier(directRadianceImage); } + } + // Per-eye denoised images always need barrier + addBarrier(denoisedDiffuse); + addBarrier(denoisedSpecular); worldCommandBuffer->barriersBufferImage({}, barriers); + VkImageLayout samplerLayout = stereo ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + std::map> composeInputs; composeInputs["direct"] = directRadianceImage; - composeInputs["diffuse"] = denoisedDiffuseRadianceImage; - composeInputs["specular"] = denoisedSpecularRadianceImage; + composeInputs["diffuse"] = denoisedDiffuse; + composeInputs["specular"] = denoisedSpecular; composeInputs["albedo"] = diffuseAlbedoImage; composeInputs["specularAlbedo"] = specularAlbedoImage; composeInputs["normal"] = normalRoughnessImage; @@ -423,8 +500,8 @@ void NrdModuleContext::render() { composeInputs["clearRadiance"] = clearRadianceImage; composeInputs["baseEmission"] = baseEmissionImage; - module->dispatchComposition(worldCommandBuffer, context->frameIndex, composeInputs, buffers->worldUniformBuffer(), - denoisedRadianceImage); + module->dispatchComposition(worldCommandBuffer, context->frameIndex, eyeIndex, composeInputs, + buffers->worldUniformBuffer(), denoisedRadianceImage); denoisedRadianceImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; } @@ -502,6 +579,7 @@ void NrdModule::createCompositionPipeline(std::shared_ptr device, ui void NrdModule::dispatchComposition(std::shared_ptr cmd, uint32_t frameIndex, + uint32_t eyeIndex, const std::map> &images, std::shared_ptr worldUBO, std::shared_ptr outputImage) { @@ -511,28 +589,35 @@ void NrdModule::dispatchComposition(std::shared_ptr cmd, return; } - auto descriptorTable = composeDescriptorTables_[frameIndex]; + uint32_t dtIdx = frameIndex * eyeCount_ + eyeIndex; + uint32_t viewIdx = (eyeCount_ > 1) ? (1 + eyeIndex) : 0; + bool stereo = eyeCount_ > 1; + // In stereo: shared pipeline inputs stay GENERAL, per-eye denoised images are SHADER_READ_ONLY + VkImageLayout sharedLayout = stereo ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkImageLayout denoisedLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + + auto descriptorTable = composeDescriptorTables_[dtIdx]; - descriptorTable->bindImage(outputImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0); + descriptorTable->bindImage(outputImage, VK_IMAGE_LAYOUT_GENERAL, 0, 0, viewIdx); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("direct"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 1, 0); + sharedLayout, 0, 1, 0, viewIdx); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("diffuse"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 2, 0); + denoisedLayout, 0, 2, 0); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("specular"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 3, 0); + denoisedLayout, 0, 3, 0); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("albedo"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 4, 0); + sharedLayout, 0, 4, 0, viewIdx); descriptorTable->bindSamplerImage(composeSamplers_[1], images.at("normal"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 5, 0); - descriptorTable->bindSamplerImage(composeSamplers_[1], images.at("depth"), VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - 0, 6, 0); + sharedLayout, 0, 5, 0, viewIdx); + descriptorTable->bindSamplerImage(composeSamplers_[1], images.at("depth"), + sharedLayout, 0, 6, 0, viewIdx); descriptorTable->bindBuffer(worldUBO, 0, 7); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("specularAlbedo"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 8, 0); + sharedLayout, 0, 8, 0, viewIdx); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("clearRadiance"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 9, 0); + sharedLayout, 0, 9, 0, viewIdx); descriptorTable->bindSamplerImage(composeSamplers_[0], images.at("baseEmission"), - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 10, 0); + sharedLayout, 0, 10, 0, viewIdx); cmd->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_COMPUTE)->bindComputePipeline(composePipeline_); diff --git a/src/core/render/modules/world/nrd/nrd_module.hpp b/src/core/render/modules/world/nrd/nrd_module.hpp index 0cfd27d..438d7e0 100644 --- a/src/core/render/modules/world/nrd/nrd_module.hpp +++ b/src/core/render/modules/world/nrd/nrd_module.hpp @@ -33,12 +33,15 @@ class NrdModule : public WorldModule, public SharedObject { bindTexture(std::shared_ptr sampler, std::shared_ptr image, int index) override; void preClose() override; - std::shared_ptr wrapper() { - return m_wrapper; + std::shared_ptr wrapper(uint32_t eyeIndex = 0) { + return m_wrappers.empty() ? nullptr : m_wrappers[eyeIndex]; } + StereoMode stereoMode() const override { return StereoMode::DualInstance; } + void dispatchComposition(std::shared_ptr cmd, uint32_t frameIndex, + uint32_t eyeIndex, const std::map> &images, std::shared_ptr worldUBO, std::shared_ptr outputImage); @@ -52,7 +55,8 @@ class NrdModule : public WorldModule, public SharedObject { std::shared_ptr m_vma; std::vector> contexts_; - std::shared_ptr m_wrapper; + // m_wrappers[eyeIndex] — one NRD instance per eye for stereo + std::vector> m_wrappers; uint32_t maxAccumulatedFrameNum_ = 31; uint32_t maxFastAccumulatedFrameNum_ = 4; @@ -78,15 +82,19 @@ class NrdModule : public WorldModule, public SharedObject { std::vector> denoisedSpecularRadianceImages_; std::shared_ptr composePipeline_; + // composeDescriptorTables_[frameIndex * eyeCount_ + eyeIndex] std::vector> composeDescriptorTables_; std::array, 2> composeSamplers_; std::shared_ptr preparePipeline_; + // prepareDescriptorTables_[frameIndex * eyeCount_ + eyeIndex] std::vector> prepareDescriptorTables_; + // NRD intermediate images: [frameIndex * eyeCount_ + eyeIndex] std::vector> m_nrdMotionVectorImages; std::vector> m_nrdNormalRoughnessImages; std::vector> m_nrdDiffuseRadianceImages; std::vector> m_nrdSpecularRadianceImages; + std::vector> m_nrdLinearDepthImages; void createCompositionPipeline(std::shared_ptr device, uint32_t contextCount); void createPreparePipeline(std::shared_ptr device, uint32_t contextCount); @@ -98,7 +106,9 @@ class NrdModuleContext : public WorldModuleContext, public SharedObject worldPipelineContext, std::shared_ptr nrdModule); + StereoMode stereoMode() const override { return StereoMode::DualInstance; } void render() override; + void renderEye(uint32_t eyeIndex) override; private: std::weak_ptr nrdModule; diff --git a/src/core/render/modules/world/post_render/post_render_module.cpp b/src/core/render/modules/world/post_render/post_render_module.cpp index e41d2e6..e430131 100644 --- a/src/core/render/modules/world/post_render/post_render_module.cpp +++ b/src/core/render/modules/world/post_render/post_render_module.cpp @@ -29,8 +29,9 @@ bool PostRenderModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[0], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[0], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[0]->createPerLayerViews(); } else { if (images[0]->width() != width_ || images[0]->height() != height_) return false; ldrImages_[frameIndex] = images[0]; @@ -38,8 +39,9 @@ bool PostRenderModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[1], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[1], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[1]->createPerLayerViews(); } else { if (images[1]->width() != width_ || images[1]->height() != height_) return false; firstHitDepthImages_[frameIndex] = images[1]; @@ -93,7 +95,8 @@ void PostRenderModule::bindTexture(std::shared_ptr sampler, auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); - for (int i = 0; i < size; i++) { + uint32_t totalCtx = size * eyeCount_; + for (int i = 0; i < totalCtx; i++) { if (descriptorTables_[i] != nullptr) descriptorTables_[i]->bindSamplerImage(sampler, image, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 0, 0, index); @@ -105,11 +108,12 @@ void PostRenderModule::preClose() {} void PostRenderModule::initDescriptorTables() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; - descriptorTables_.resize(size); - samplers_.resize(size); + descriptorTables_.resize(totalCtx); + samplers_.resize(totalCtx); - for (int i = 0; i < size; i++) { + for (int i = 0; i < totalCtx; i++) { descriptorTables_[i] = vk::DescriptorTableBuilder{} .beginDescriptorLayoutSet() // set 0 .beginDescriptorLayoutSetBinding() @@ -165,6 +169,11 @@ void PostRenderModule::initDescriptorTables() { }) .endDescriptorLayoutSetBinding() .endDescriptorLayoutSet() + .definePushConstant({ + .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + .offset = 0, + .size = sizeof(PostRenderPushConstant), + }) .build(framework->device()); samplers_[i] = vk::Sampler::create(framework->device(), VK_FILTER_LINEAR, VK_SAMPLER_MIPMAP_MODE_LINEAR, @@ -179,18 +188,24 @@ void PostRenderModule::initImages() { uint32_t size = framework->swapchain()->imageCount(); worldLightMapImages_.resize(size); - worldPostDepthImages_.resize(size); + uint32_t totalCtx = size * eyeCount_; + worldPostDepthImages_.resize(totalCtx); for (int i = 0; i < size; i++) { worldLightMapImages_[i] = vk::DeviceLocalImage::create(device, vma, false, 16, 16, 1, VK_FORMAT_R8G8B8A8_UNORM, VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); - descriptorTables_[i]->bindSamplerImageForShader(samplers_[i], worldLightMapImages_[i], 0, 1); - descriptorTables_[i]->bindImage(firstHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 0, 2); + for (uint32_t e = 0; e < eyeCount_; e++) { + uint32_t dtIdx = i * eyeCount_ + e; + uint32_t viewIdx = eyeCount_ > 1 ? 1 + e : 0; - worldPostDepthImages_[i] = vk::DeviceLocalImage::create( - device, vma, false, width_, height_, 1, VK_FORMAT_D32_SFLOAT, VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT); + descriptorTables_[dtIdx]->bindSamplerImageForShader(samplers_[dtIdx], worldLightMapImages_[i], 0, 1); + descriptorTables_[dtIdx]->bindImage(firstHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 0, 2, viewIdx); + + worldPostDepthImages_[dtIdx] = vk::DeviceLocalImage::create( + device, vma, false, width_, height_, 1, VK_FORMAT_D32_SFLOAT, VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT); + } } } @@ -431,9 +446,11 @@ void PostRenderModule::initFrameBuffers() { auto device = framework->device(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; + worldLightMapFramebuffers_.resize(size); - worldPostColorToDepthFramebuffers_.resize(size); - worldPostFramebuffers_.resize(size); + worldPostColorToDepthFramebuffers_.resize(totalCtx); + worldPostFramebuffers_.resize(totalCtx); for (int i = 0; i < size; i++) { worldLightMapFramebuffers_[i] = vk::FramebufferBuilder{} @@ -442,18 +459,23 @@ void PostRenderModule::initFrameBuffers() { .endAttachment() .build(device, worldLightMapRenderPass_); - worldPostColorToDepthFramebuffers_[i] = vk::FramebufferBuilder{} - .beginAttachment() - .defineAttachment(worldPostDepthImages_[i]) - .endAttachment() - .build(device, worldPostColorToDepthRenderPass_); - - worldPostFramebuffers_[i] = vk::FramebufferBuilder{} - .beginAttachment() - .defineAttachment(postRenderedImages_[i]) - .defineAttachment(worldPostDepthImages_[i]) - .endAttachment() - .build(device, worldPostRenderPass_); + for (uint32_t e = 0; e < eyeCount_; e++) { + uint32_t idx = i * eyeCount_ + e; + uint32_t viewIdx = eyeCount_ > 1 ? 1 + e : 0; + + worldPostColorToDepthFramebuffers_[idx] = vk::FramebufferBuilder{} + .beginAttachment() + .defineAttachment(worldPostDepthImages_[idx]) + .endAttachment() + .build(device, worldPostColorToDepthRenderPass_); + + worldPostFramebuffers_[idx] = vk::FramebufferBuilder{} + .beginAttachment() + .defineAttachment(postRenderedImages_[i], viewIdx) + .defineAttachment(worldPostDepthImages_[idx]) + .endAttachment() + .build(device, worldPostRenderPass_); + } } } @@ -686,15 +708,24 @@ PostRenderModuleContext::PostRenderModuleContext(std::shared_ptrldrImages_[frameworkContext->frameIndex]), firstHitDepthImage(postRenderModule->firstHitDepthImages_[frameworkContext->frameIndex]), worldLightMapImage(postRenderModule->worldLightMapImages_[frameworkContext->frameIndex]), - worldPostDepthImage(postRenderModule->worldPostDepthImages_[frameworkContext->frameIndex]), - descriptorTable(postRenderModule->descriptorTables_[frameworkContext->frameIndex]), worldLightMapFramebuffer(postRenderModule->worldLightMapFramebuffers_[frameworkContext->frameIndex]), - worldPostColorToDepthFramebuffer( - postRenderModule->worldPostColorToDepthFramebuffers_[frameworkContext->frameIndex]), - worldPostFramebuffer(postRenderModule->worldPostFramebuffers_[frameworkContext->frameIndex]), - postRenderedImage(postRenderModule->postRenderedImages_[frameworkContext->frameIndex]) {} + postRenderedImage(postRenderModule->postRenderedImages_[frameworkContext->frameIndex]), + eyeCount_(postRenderModule->eyeCount_) { + uint32_t fi = frameworkContext->frameIndex; + uint32_t base = fi * eyeCount_; + for (uint32_t e = 0; e < eyeCount_; e++) { + descriptorTables.push_back(postRenderModule->descriptorTables_[base + e]); + worldPostDepthImages.push_back(postRenderModule->worldPostDepthImages_[base + e]); + worldPostColorToDepthFramebuffers.push_back(postRenderModule->worldPostColorToDepthFramebuffers_[base + e]); + worldPostFramebuffers.push_back(postRenderModule->worldPostFramebuffers_[base + e]); + } +} void PostRenderModuleContext::render() { + render3D(1); +} + +void PostRenderModuleContext::render3D(uint32_t eyeCount) { auto context = frameworkContext.lock(); auto framework = context->framework.lock(); auto worldCommandBuffer = context->worldCommandBuffer; @@ -792,29 +823,34 @@ void PostRenderModuleContext::render() { ensureLayout(firstHitDepthImage, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT); - if (worldPostDepthImage && worldPostDepthImage->imageLayout() != VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL) { - VkPipelineStageFlags2 srcStage = 0; - VkAccessFlags2 srcAccess = 0; - chooseSrc(worldPostDepthImage->imageLayout(), VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, srcStage, srcAccess); - worldCommandBuffer->barriersBufferImage( - {}, {{.srcStageMask = srcStage, - .srcAccessMask = srcAccess, - .dstStageMask = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT, - .dstAccessMask = VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, - .oldLayout = worldPostDepthImage->imageLayout(), - .newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, - .srcQueueFamilyIndex = mainQueueIndex, - .dstQueueFamilyIndex = mainQueueIndex, - .image = worldPostDepthImage, - .subresourceRange = vk::wholeDepthSubresourceRange}}); - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + for (uint32_t e = 0; e < eyeCount; e++) { + auto &eyeDepth = worldPostDepthImages[e]; + if (eyeDepth && eyeDepth->imageLayout() != VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL) { + VkPipelineStageFlags2 srcStage = 0; + VkAccessFlags2 srcAccess = 0; + chooseSrc(eyeDepth->imageLayout(), VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, srcStage, srcAccess); + worldCommandBuffer->barriersBufferImage( + {}, {{.srcStageMask = srcStage, + .srcAccessMask = srcAccess, + .dstStageMask = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT, + .dstAccessMask = VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .oldLayout = eyeDepth->imageLayout(), + .newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .image = eyeDepth, + .subresourceRange = vk::wholeDepthSubresourceRange}}); + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + } } - descriptorTable->bindBuffer(buffers->worldUniformBuffer(), 1, 0); - descriptorTable->bindBuffer(buffers->skyUniformBuffer(), 1, 1); - descriptorTable->bindBuffer(buffers->lightMapUniformBuffer(), 1, 2); - descriptorTable->bindBuffer(Renderer::instance().buffers()->textureMappingBuffer(), 2, 0); + for (uint32_t e = 0; e < eyeCount; e++) { + descriptorTables[e]->bindBuffer(buffers->worldUniformBuffer(), 1, 0); + descriptorTables[e]->bindBuffer(buffers->skyUniformBuffer(), 1, 1); + descriptorTables[e]->bindBuffer(buffers->lightMapUniformBuffer(), 1, 2); + descriptorTables[e]->bindBuffer(Renderer::instance().buffers()->textureMappingBuffer(), 2, 0); + } // render light map { @@ -849,16 +885,24 @@ void PostRenderModuleContext::render() { worldLightMapImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; worldCommandBuffer->bindGraphicsPipeline(module->worldLightMapPipeline_) - ->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) + ->bindDescriptorTable(descriptorTables[0], VK_PIPELINE_BIND_POINT_GRAPHICS) ->draw(3, 1) ->endRenderPass(); worldLightMapImage->imageLayout() = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + // ---- PER-EYE LOOP ---- + for (uint32_t eye = 0; eye < eyeCount; eye++) { + auto &eyeDT = descriptorTables[eye]; + auto &eyeDepth = worldPostDepthImages[eye]; + auto &eyeC2DFB = worldPostColorToDepthFramebuffers[eye]; + auto &eyePostFB = worldPostFramebuffers[eye]; + uint32_t eyeLayer = eyeCount > 1 ? eye : 0; + // cast linear depth from color to depth format { VkPipelineStageFlags2 srcStageDepth = 0; VkAccessFlags2 srcAccessDepth = 0; - chooseSrc(worldPostDepthImage->imageLayout(), + chooseSrc(eyeDepth->imageLayout(), VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR | VK_PIPELINE_STAGE_2_TRANSFER_BIT, VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, srcStageDepth, srcAccessDepth); @@ -876,11 +920,11 @@ void PostRenderModuleContext::render() { .srcAccessMask = srcAccessDepth, .dstStageMask = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .oldLayout = worldPostDepthImage->imageLayout(), + .oldLayout = eyeDepth->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = worldPostDepthImage, + .image = eyeDepth, .subresourceRange = vk::wholeDepthSubresourceRange, }, { @@ -896,22 +940,22 @@ void PostRenderModuleContext::render() { .subresourceRange = vk::wholeColorSubresourceRange, }}); } - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; firstHitDepthImage->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; worldCommandBuffer->beginRenderPass({ .renderPass = module->worldPostColorToDepthRenderPass_, - .framebuffer = worldPostColorToDepthFramebuffer, - .renderAreaExtent = {worldPostDepthImage->width(), worldPostDepthImage->height()}, + .framebuffer = eyeC2DFB, + .renderAreaExtent = {eyeDepth->width(), eyeDepth->height()}, .clearValues = {{.depthStencil = {.depth = 1.0f}}}, }); - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - worldCommandBuffer->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) + worldCommandBuffer->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_GRAPHICS) ->bindGraphicsPipeline(module->worldPostColorToDepthPipeline_) ->draw(3, 1) ->endRenderPass(); - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; // copy input to output { @@ -964,13 +1008,13 @@ void PostRenderModuleContext::render() { VkImageBlit imageBlit{}; imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.srcSubresource.mipLevel = 0; - imageBlit.srcSubresource.baseArrayLayer = 0; + imageBlit.srcSubresource.baseArrayLayer = eyeLayer; imageBlit.srcSubresource.layerCount = 1; imageBlit.srcOffsets[0] = {0, 0, 0}; imageBlit.srcOffsets[1] = {static_cast(ldrImage->width()), static_cast(ldrImage->height()), 1}; imageBlit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.dstSubresource.mipLevel = 0; - imageBlit.dstSubresource.baseArrayLayer = 0; + imageBlit.dstSubresource.baseArrayLayer = eyeLayer; imageBlit.dstSubresource.layerCount = 1; imageBlit.dstOffsets[0] = {0, 0, 0}; imageBlit.dstOffsets[1] = {static_cast(postRenderedImage->width()), @@ -1034,17 +1078,23 @@ void PostRenderModuleContext::render() { worldCommandBuffer->beginRenderPass({ .renderPass = module->worldPostRenderPass_, - .framebuffer = worldPostFramebuffer, - .renderAreaExtent = {worldPostDepthImage->width(), worldPostDepthImage->height()}, + .framebuffer = eyePostFB, + .renderAreaExtent = {eyeDepth->width(), eyeDepth->height()}, .clearValues = {}, }); postRenderedImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; std::shared_ptr entityPostRenderDataBatch = Renderer::instance().world()->entities()->entityPostBatch(); if (entityPostRenderDataBatch != nullptr) { - worldCommandBuffer->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) + PostRenderPushConstant pc{}; + pc.eyeIndex = eye; + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), eyeDT->vkPipelineLayout(), + VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 0, sizeof(PostRenderPushConstant), &pc); + + worldCommandBuffer->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_GRAPHICS) ->bindGraphicsPipeline(module->worldPostPipeline_); for (int i = 0; i < entityPostRenderDataBatch->entities.size(); i++) { @@ -1067,7 +1117,7 @@ void PostRenderModuleContext::render() { #else postRenderedImage->imageLayout() = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; #endif - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; // render post star field { @@ -1124,14 +1174,21 @@ void PostRenderModuleContext::render() { worldCommandBuffer->beginRenderPass({ .renderPass = module->worldPostRenderPass_, - .framebuffer = worldPostFramebuffer, - .renderAreaExtent = {worldPostDepthImage->width(), worldPostDepthImage->height()}, + .framebuffer = eyePostFB, + .renderAreaExtent = {eyeDepth->width(), eyeDepth->height()}, .clearValues = {}, }); postRenderedImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - worldCommandBuffer->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) + { + PostRenderPushConstant pc{}; + pc.eyeIndex = eye; + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), eyeDT->vkPipelineLayout(), + VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, + 0, sizeof(PostRenderPushConstant), &pc); + } + worldCommandBuffer->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_GRAPHICS) ->bindGraphicsPipeline(module->worldPostStarFieldPipeline_) ->bindVertexBuffers(module->starFieldVertexBuffer) @@ -1143,5 +1200,7 @@ void PostRenderModuleContext::render() { #else postRenderedImage->imageLayout() = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; #endif - worldPostDepthImage->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + eyeDepth->imageLayout() = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; + + } // end per-eye loop } diff --git a/src/core/render/modules/world/post_render/post_render_module.hpp b/src/core/render/modules/world/post_render/post_render_module.hpp index 29b4f93..bc765ff 100644 --- a/src/core/render/modules/world/post_render/post_render_module.hpp +++ b/src/core/render/modules/world/post_render/post_render_module.hpp @@ -13,6 +13,10 @@ class FrameworkContext; class WorldPipeline; struct WorldModuleContext; +struct PostRenderPushConstant { + uint32_t eyeIndex; +}; + struct PostRenderModuleContext; class PostRenderModule : public WorldModule, public SharedObject { @@ -45,6 +49,8 @@ class PostRenderModule : public WorldModule, public SharedObject { std::weak_ptr postRenderModule; + StereoMode stereoMode() const override { return StereoMode::SingleInstance3DDispatch; } + void render3D(uint32_t eyeCount) override; + // input std::shared_ptr ldrImage; std::shared_ptr firstHitDepthImage; - // post render + // post render (per-eye indexed) std::shared_ptr worldLightMapImage; - std::shared_ptr worldPostDepthImage; - std::shared_ptr descriptorTable; + std::vector> worldPostDepthImages; + std::vector> descriptorTables; std::shared_ptr worldLightMapFramebuffer; - std::shared_ptr worldPostColorToDepthFramebuffer; - std::shared_ptr worldPostFramebuffer; + std::vector> worldPostColorToDepthFramebuffers; + std::vector> worldPostFramebuffers; // output std::shared_ptr postRenderedImage; @@ -125,4 +134,7 @@ struct PostRenderModuleContext : public WorldModuleContext, SharedObject postRenderModule); void render() override; + + private: + uint32_t eyeCount_; }; diff --git a/src/core/render/modules/world/ray_tracing/ray_tracing_module.cpp b/src/core/render/modules/world/ray_tracing/ray_tracing_module.cpp index 8813a8f..7ab4b6e 100644 --- a/src/core/render/modules/world/ray_tracing/ray_tracing_module.cpp +++ b/src/core/render/modules/world/ray_tracing/ray_tracing_module.cpp @@ -62,8 +62,9 @@ bool RayTracingModule::setOrCreateOutputImages(std::vectordevice(), framework->vma(), false, width, height, 1, formats[i], + framework->device(), framework->vma(), false, width, height, eyeCount_, formats[i], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[i]->createPerLayerViews(); } } @@ -99,6 +100,11 @@ void RayTracingModule::setAttributes(int attributeCount, std::vectorsetUseJitter(useJitter_); + } else if (key == "render_pipeline.module.ray_tracing.attribute.foveated_inner_radius") { + Renderer::instance().buffers()->setFoveatedInnerRadius(std::stof(value)); + } else if (key == "render_pipeline.module.ray_tracing.attribute.foveated_outer_block_size") { + Renderer::instance().buffers()->setFoveatedOuterBlockSize( + static_cast(std::stoi(value))); } } } @@ -114,6 +120,7 @@ void RayTracingModule::build() { contexts_.resize(size); initDescriptorTables(); + createVisibilityMaskImage(); initImages(); initPipeline(); initSBT(); @@ -141,14 +148,145 @@ void RayTracingModule::bindTexture(std::shared_ptr sampler, uint32_t size = framework->swapchain()->imageCount(); for (int i = 0; i < size; i++) { + // Bind to unified descriptor table (no per-eye iteration) if (rayTracingDescriptorTables_[i] != nullptr) rayTracingDescriptorTables_[i]->bindSamplerImage(sampler, image, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, - 0, 0, index); + TEXTURES_SET, 0, index); } } void RayTracingModule::preClose() {} +void RayTracingModule::createVisibilityMaskImage() { + auto framework = framework_.lock(); + auto device = framework->device(); + auto vma = framework->vma(); + + uint32_t w = hdrNoisyOutputImages_[0]->width(); + uint32_t h = hdrNoisyOutputImages_[0]->height(); + // Each row is packed into ceil(w/32) uint32 words (1 bit per pixel) + uint32_t wPacked = (w + 31) / 32; + + // Allocate bitmask buffer: all visible (all bits set) initially + std::vector masks(static_cast(wPacked) * h * eyeCount_, 0xFFFFFFFFu); + + // Fill hidden triangles only when running the stereo XR path. + bool useVisibilityMask = false; +#ifdef MCVR_ENABLE_OPENXR + auto *xr = Renderer::instance().framework()->xrContext(); + useVisibilityMask = + (eyeCount_ > 1) && Renderer::options.vrEnabled && xr && xr->isSessionRunning() && xr->hasVisibilityMask(); +#endif + + if (useVisibilityMask) { + for (uint32_t eye = 0; eye < eyeCount_; eye++) { + auto &verts = xr->visibilityMaskVertices(eye); + auto &indices = xr->visibilityMaskIndices(eye); + if (verts.empty() || indices.empty()) continue; + + // Derive coordinate range from vertex bounding box + // The mesh includes a bounding rectangle covering the full FOV + float minVx = verts[0].x, maxVx = verts[0].x; + float minVy = verts[0].y, maxVy = verts[0].y; + for (auto &v : verts) { + minVx = std::min(minVx, v.x); maxVx = std::max(maxVx, v.x); + minVy = std::min(minVy, v.y); maxVy = std::max(maxVy, v.y); + } + float rangeX = maxVx - minVx; + float rangeY = maxVy - minVy; + if (rangeX < 1e-6f || rangeY < 1e-6f) continue; + + uint32_t *layer = masks.data() + static_cast(wPacked) * h * eye; + + for (size_t t = 0; t + 2 < indices.size(); t += 3) { + // Map from tangent-angle space to pixel coords using vertex bounding box + auto toPixel = [&](const glm::vec2 &v) -> glm::vec2 { + float px = (v.x - minVx) / rangeX * static_cast(w); + float py = (maxVy - v.y) / rangeY * static_cast(h); + return {px, py}; + }; + glm::vec2 p0 = toPixel(verts[indices[t + 0]]); + glm::vec2 p1 = toPixel(verts[indices[t + 1]]); + glm::vec2 p2 = toPixel(verts[indices[t + 2]]); + + // Bounding box + int minX = std::max(0, static_cast(std::floor(std::min({p0.x, p1.x, p2.x})))); + int maxX = std::min(static_cast(w) - 1, + static_cast(std::ceil(std::max({p0.x, p1.x, p2.x})))); + int minY = std::max(0, static_cast(std::floor(std::min({p0.y, p1.y, p2.y})))); + int maxY = std::min(static_cast(h) - 1, + static_cast(std::ceil(std::max({p0.y, p1.y, p2.y})))); + + float denom = (p1.y - p2.y) * (p0.x - p2.x) + (p2.x - p1.x) * (p0.y - p2.y); + if (std::abs(denom) < 1e-6f) continue; + float invDenom = 1.0f / denom; + + for (int y = minY; y <= maxY; y++) { + for (int x = minX; x <= maxX; x++) { + float px = x + 0.5f, py = y + 0.5f; + float a = ((p1.y - p2.y) * (px - p2.x) + (p2.x - p1.x) * (py - p2.y)) * invDenom; + float b = ((p2.y - p0.y) * (px - p2.x) + (p0.x - p2.x) * (py - p2.y)) * invDenom; + float c = 1.0f - a - b; + if (a >= 0.0f && b >= 0.0f && c >= 0.0f) { + layer[y * wPacked + x / 32] &= ~(1u << (x % 32u)); // clear bit = hidden + } + } + } + } + } + } + + // Create device image: width=wPacked (one uint32 per 32 pixels), R32_UINT + visMaskImage_ = vk::DeviceLocalImage::create( + device, vma, true, wPacked, h, eyeCount_, VK_FORMAT_R32_UINT, + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT); + if (eyeCount_ > 1) visMaskImage_->createPerLayerViews(); + + // Upload bitmask data to staging buffer + visMaskImage_->uploadToStagingBuffer(masks.data()); + + // One-shot command buffer: transition → copy → transition + auto physDev = framework->physicalDevice(); + auto cmdPool = vk::CommandPool::create(physDev, device); + auto cmd = vk::CommandBuffer::create(device, cmdPool); + + uint32_t qIdx = physDev->mainQueueIndex(); + + cmd->begin(VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT); + cmd->barriersBufferImage({}, {{ + .srcStageMask = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, + .srcAccessMask = 0, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = qIdx, + .dstQueueFamilyIndex = qIdx, + .image = visMaskImage_, + .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, eyeCount_}, + }}); + + visMaskImage_->uploadToImage(cmd); + + cmd->barriersBufferImage({}, {{ + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_RAY_TRACING_SHADER_BIT_KHR, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = qIdx, + .dstQueueFamilyIndex = qIdx, + .image = visMaskImage_, + .subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, eyeCount_}, + }}); + cmd->end(); + + cmd->submitMainQueueIndividual(device); + vkQueueWaitIdle(device->mainVkQueue()); + visMaskImage_->imageLayout() = VK_IMAGE_LAYOUT_GENERAL; +} + void RayTracingModule::initDescriptorTables() { auto framework = framework_.lock(); @@ -156,6 +294,7 @@ void RayTracingModule::initDescriptorTables() { rayTracingDescriptorTables_.resize(size); for (int i = 0; i < size; i++) { + // Create unified descriptor table for 3D dispatch (no per-eye tables) rayTracingDescriptorTables_[i] = vk::DescriptorTableBuilder{} .beginDescriptorLayoutSet() // set 0 @@ -163,7 +302,7 @@ void RayTracingModule::initDescriptorTables() { .defineDescriptorLayoutSetBinding({ .binding = 0, .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, - .descriptorCount = 4096, // a very big number + .descriptorCount = 512, // Reasonable texture limit (was 4096) .stageFlags = VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR | VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, @@ -381,6 +520,12 @@ void RayTracingModule::initDescriptorTables() { VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | VK_SHADER_STAGE_FRAGMENT_BIT, }) + .defineDescriptorLayoutSetBinding({ + .binding = 14, // binding 14: visibilityMaskImage + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_RAYGEN_BIT_KHR, + }) .endDescriptorLayoutSetBinding() .endDescriptorLayoutSet() .definePushConstant({ @@ -388,7 +533,7 @@ void RayTracingModule::initDescriptorTables() { VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | VK_SHADER_STAGE_INTERSECTION_BIT_KHR, .offset = 0, - .size = sizeof(uint32_t), + .size = sizeof(RayTracingPushConstant), }) .build(framework->device()); } @@ -400,26 +545,31 @@ void RayTracingModule::initImages() { uint32_t size = framework->swapchain()->imageCount(); for (int i = 0; i < size; i++) { - rayTracingDescriptorTables_[i]->bindSamplerImageForShader(atmosphere_->atmLUTImageSampler_, - atmosphere_->atmLUTImage_, 0, 1); - rayTracingDescriptorTables_[i]->bindSamplerImageForShader(atmosphere_->atmCubeMapImageSamplers_[i], - atmosphere_->atmCubeMapImages_[i], 0, 2, 7); - - rayTracingDescriptorTables_[i]->bindImage(hdrNoisyOutputImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 0); - rayTracingDescriptorTables_[i]->bindImage(diffuseAlbedoImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 1); - rayTracingDescriptorTables_[i]->bindImage(specularAlbedoImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 2); - rayTracingDescriptorTables_[i]->bindImage(normalRoughnessImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 3); - rayTracingDescriptorTables_[i]->bindImage(motionVectorImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 4); - rayTracingDescriptorTables_[i]->bindImage(linearDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 5); - rayTracingDescriptorTables_[i]->bindImage(specularHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 6); - rayTracingDescriptorTables_[i]->bindImage(firstHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 7); - rayTracingDescriptorTables_[i]->bindImage(firstHitDiffuseDirectLightImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 8); - rayTracingDescriptorTables_[i]->bindImage(firstHitDiffuseIndirectLightImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, - 9); - rayTracingDescriptorTables_[i]->bindImage(firstHitSpecularImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 10); - rayTracingDescriptorTables_[i]->bindImage(firstHitClearImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 11); - rayTracingDescriptorTables_[i]->bindImage(firstHitBaseEmissionImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 12); - rayTracingDescriptorTables_[i]->bindImage(directLightDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, 3, 13); + // Unified descriptor table for 3D dispatch + auto& dt = rayTracingDescriptorTables_[i]; + + dt->bindSamplerImageForShader(atmosphere_->atmLUTImageSampler_, + atmosphere_->atmLUTImage_, TEXTURES_SET, 1); + dt->bindSamplerImageForShader(atmosphere_->atmCubeMapImageSamplers_[i], + atmosphere_->atmCubeMapImages_[i], TEXTURES_SET, 2, 7); + + // set 3: bind full image array views for 3D dispatch + // In 3D dispatch, shader will use gl_LaunchIDEXT.z to select the layer + dt->bindImage(hdrNoisyOutputImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 0, 0); + dt->bindImage(diffuseAlbedoImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 1, 0); + dt->bindImage(specularAlbedoImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 2, 0); + dt->bindImage(normalRoughnessImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 3, 0); + dt->bindImage(motionVectorImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 4, 0); + dt->bindImage(linearDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 5, 0); + dt->bindImage(specularHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 6, 0); + dt->bindImage(firstHitDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 7, 0); + dt->bindImage(firstHitDiffuseDirectLightImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 8, 0); + dt->bindImage(firstHitDiffuseIndirectLightImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 9, 0); + dt->bindImage(firstHitSpecularImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 10, 0); + dt->bindImage(firstHitClearImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 11, 0); + dt->bindImage(firstHitBaseEmissionImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 12, 0); + dt->bindImage(directLightDepthImages_[i], VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 13, 0); + dt->bindImage(visMaskImage_, VK_IMAGE_LAYOUT_GENERAL, OUTPUT_IMAGES_SET, 14, 0); } } @@ -553,6 +703,10 @@ RayTracingModuleContext::RayTracingModuleContext(std::shared_ptrworldPrepare_->contexts_[frameworkContext->frameIndex]) {} void RayTracingModuleContext::render() { + render3D(1); +} + +void RayTracingModuleContext::render3D(uint32_t eyeCount) { atmosphereContext->render(); worldPrepareContext->render(); @@ -568,27 +722,23 @@ void RayTracingModuleContext::render() { auto module = rayTracingModule.lock(); - rayTracingDescriptorTable->bindAS(worldPrepareContext->tlas, 1, 0); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->blasOffsetsBuffer, 1, 1); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->vertexBufferAddr, 1, 2); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->indexBufferAddr, 1, 3); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->lastVertexBufferAddr, 1, 4); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->lastIndexBufferAddr, 1, 5); - rayTracingDescriptorTable->bindBuffer(worldPrepareContext->lastObjToWorldMat, 1, 6); - + // Bind per-frame data to unified descriptor table auto buffers = Renderer::instance().buffers(); auto worldBuffer = buffers->worldUniformBuffer(); - rayTracingDescriptorTable->bindBuffer(buffers->textureMappingBuffer(), 1, 7); - rayTracingDescriptorTable->bindBuffer(worldBuffer, 2, 0); - rayTracingDescriptorTable->bindBuffer(buffers->lastWorldUniformBuffer(), 2, 1); - rayTracingDescriptorTable->bindBuffer(buffers->skyUniformBuffer(), 2, 2); + auto& dt = rayTracingDescriptorTable; + dt->bindAS(worldPrepareContext->tlas, ACCELERATION_SET, 0); + dt->bindBuffer(worldPrepareContext->blasOffsetsBuffer, ACCELERATION_SET, 1); + dt->bindBuffer(worldPrepareContext->vertexBufferAddr, ACCELERATION_SET, 2); + dt->bindBuffer(worldPrepareContext->indexBufferAddr, ACCELERATION_SET, 3); + dt->bindBuffer(worldPrepareContext->lastVertexBufferAddr, ACCELERATION_SET, 4); + dt->bindBuffer(worldPrepareContext->lastIndexBufferAddr, ACCELERATION_SET, 5); + dt->bindBuffer(worldPrepareContext->lastObjToWorldMat, ACCELERATION_SET, 6); - vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), rayTracingDescriptorTable->vkPipelineLayout(), - VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR | - VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | - VK_SHADER_STAGE_INTERSECTION_BIT_KHR, - 0, sizeof(uint32_t), &module->numRayBounces_); + dt->bindBuffer(buffers->textureMappingBuffer(), ACCELERATION_SET, 7); + dt->bindBuffer(worldBuffer, UNIFORMS_SET, 0); + dt->bindBuffer(buffers->lastWorldUniformBuffer(), UNIFORMS_SET, 1); + dt->bindBuffer(buffers->skyUniformBuffer(), UNIFORMS_SET, 2); auto chooseSrc = [](VkImageLayout oldLayout, VkPipelineStageFlags2 fallbackStage, VkAccessFlags2 fallbackAccess, VkPipelineStageFlags2 &outStage, VkAccessFlags2 &outAccess) { @@ -641,7 +791,23 @@ void RayTracingModuleContext::render() { if (!barriers.empty()) { worldCommandBuffer->barriersBufferImage({}, barriers); } + // 3D DISPATCH + // Single 3D dispatch instead of per-eye loop to eliminate: + + RayTracingPushConstant pc{}; + pc.numRayBounces = static_cast(module->numRayBounces_); + pc.useJitter = module->useJitter_ ? 1 : 0; + // eyeIndex is determined by gl_LaunchIDEXT.z in 3D dispatch mode + + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), + rayTracingDescriptorTable->vkPipelineLayout(), + VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR | + VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | + VK_SHADER_STAGE_INTERSECTION_BIT_KHR, + 0, sizeof(RayTracingPushConstant), &pc); + + // Single descriptor table bind and single 3D dispatch worldCommandBuffer->bindDescriptorTable(rayTracingDescriptorTable, VK_PIPELINE_BIND_POINT_RAY_TRACING_KHR) ->bindRTPipeline(module->rayTracingPipeline_) - ->raytracing(sbt, hdrNoisyOutputImage->width(), hdrNoisyOutputImage->height(), 1); + ->raytracing(sbt, hdrNoisyOutputImage->width(), hdrNoisyOutputImage->height(), eyeCount); } diff --git a/src/core/render/modules/world/ray_tracing/ray_tracing_module.hpp b/src/core/render/modules/world/ray_tracing/ray_tracing_module.hpp index e04bdf3..43bfc65 100644 --- a/src/core/render/modules/world/ray_tracing/ray_tracing_module.hpp +++ b/src/core/render/modules/world/ray_tracing/ray_tracing_module.hpp @@ -24,6 +24,12 @@ struct RayTracingPushConstant { int useJitter; }; +// Descriptor set and binding constants +constexpr uint32_t TEXTURES_SET = 0; +constexpr uint32_t ACCELERATION_SET = 1; +constexpr uint32_t UNIFORMS_SET = 2; +constexpr uint32_t OUTPUT_IMAGES_SET = 3; + class RayTracingModule : public WorldModule, public SharedObject { friend RayTracingModuleContext; friend Atmosphere; @@ -56,11 +62,14 @@ class RayTracingModule : public WorldModule, public SharedObject worldLightMapVertShader_; std::shared_ptr worldLightMapFragShader_; + // rayTracingDescriptorTables_[frameIndex] - unified descriptor table for 3D dispatch std::vector> rayTracingDescriptorTables_; std::shared_ptr rayTracingPipeline_; std::vector> sbts_; @@ -127,6 +137,9 @@ class RayTracingModule : public WorldModule, public SharedObject> firstHitBaseEmissionImages_; std::vector> directLightDepthImages_; + // visibility mask (static, shared across frames) + std::shared_ptr visMaskImage_; + // submodules std::shared_ptr atmosphere_; std::shared_ptr worldPrepare_; @@ -141,6 +154,7 @@ struct RayTracingModuleContext : public WorldModuleContext, SharedObject rayTracingDescriptorTable; std::shared_ptr sbt; @@ -168,5 +182,7 @@ struct RayTracingModuleContext : public WorldModuleContext, SharedObject worldPipelineContext, std::shared_ptr rayTracingModule); + StereoMode stereoMode() const override { return StereoMode::SingleInstance3DDispatch; } void render() override; + void render3D(uint32_t eyeCount) override; }; \ No newline at end of file diff --git a/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.cpp b/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.cpp index 599e30b..41a9f47 100644 --- a/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.cpp +++ b/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.cpp @@ -27,8 +27,9 @@ bool TemporalAccumulationModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[0], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[0], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[0]->createPerLayerViews(); } else { if (images[0]->width() != width_ || images[0]->height() != height_) return false; hdrNoisyImages_[frameIndex] = images[0]; @@ -36,8 +37,9 @@ bool TemporalAccumulationModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[1], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[1], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[1]->createPerLayerViews(); } else { if (images[1]->width() != width_ || images[1]->height() != height_) return false; motionVectorImages_[frameIndex] = images[1]; @@ -45,8 +47,9 @@ bool TemporalAccumulationModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[2], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[2], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[2]->createPerLayerViews(); } else { if (images[2]->width() != width_ || images[2]->height() != height_) return false; normalRoughnessImages_[frameIndex] = images[2]; @@ -102,10 +105,11 @@ void TemporalAccumulationModule::preClose() {} void TemporalAccumulationModule::initDescriptorTables() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; - descriptorTables_.resize(size); + descriptorTables_.resize(totalCtx); - for (int i = 0; i < size; i++) { + for (uint32_t i = 0; i < totalCtx; i++) { descriptorTables_[i] = vk::DescriptorTableBuilder{} .beginDescriptorLayoutSet() // set 0 .beginDescriptorLayoutSetBinding() @@ -156,28 +160,40 @@ void TemporalAccumulationModule::initDescriptorTables() { void TemporalAccumulationModule::initImages() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; accumulatedRadianceImage_ = vk::DeviceLocalImage::create( - framework->device(), framework->vma(), false, hdrNoisyImages_[0]->width(), hdrNoisyImages_[0]->height(), 1, - hdrNoisyImages_[0]->vkFormat(), - VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + framework->device(), framework->vma(), false, hdrNoisyImages_[0]->width(), hdrNoisyImages_[0]->height(), + eyeCount_, hdrNoisyImages_[0]->vkFormat(), + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT); + if (eyeCount_ > 1) accumulatedRadianceImage_->createPerLayerViews(); accumulatedNormalImage_ = vk::DeviceLocalImage::create( - framework->device(), framework->vma(), false, hdrNoisyImages_[0]->width(), hdrNoisyImages_[0]->height(), 1, - normalRoughnessImages_[0]->vkFormat(), - VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + framework->device(), framework->vma(), false, hdrNoisyImages_[0]->width(), hdrNoisyImages_[0]->height(), + eyeCount_, normalRoughnessImages_[0]->vkFormat(), + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT); + if (eyeCount_ > 1) accumulatedNormalImage_->createPerLayerViews(); - for (int i = 0; i < size; i++) { - descriptorTables_[i]->bindSamplerImageForShader(sampler_, hdrNoisyImages_[i], 0, 0); - descriptorTables_[i]->bindSamplerImageForShader(sampler_, accumulatedRadianceImage_, 0, 1); - descriptorTables_[i]->bindSamplerImageForShader(sampler_, motionVectorImages_[i], 0, 2); - descriptorTables_[i]->bindSamplerImageForShader(sampler_, normalRoughnessImages_[i], 0, 3); - descriptorTables_[i]->bindSamplerImageForShader(sampler_, accumulatedNormalImage_, 0, 4); + accumulatedNormalOutImages_.resize(totalCtx); + + for (uint32_t i = 0; i < totalCtx; i++) { + uint32_t frameIdx = i / eyeCount_; + uint32_t eyeIdx = i % eyeCount_; + uint32_t viewIdx = (eyeCount_ > 1) ? (1 + eyeIdx) : 0; + + descriptorTables_[i]->bindSamplerImageForShader(sampler_, hdrNoisyImages_[frameIdx], 0, 0, viewIdx); + descriptorTables_[i]->bindSamplerImageForShader(sampler_, accumulatedRadianceImage_, 0, 1, viewIdx); + descriptorTables_[i]->bindSamplerImageForShader(sampler_, motionVectorImages_[frameIdx], 0, 2, viewIdx); + descriptorTables_[i]->bindSamplerImageForShader(sampler_, normalRoughnessImages_[frameIdx], 0, 3, viewIdx); + descriptorTables_[i]->bindSamplerImageForShader(sampler_, accumulatedNormalImage_, 0, 4, viewIdx); accumulatedNormalOutImages_[i] = vk::DeviceLocalImage::create( framework->device(), framework->vma(), false, hdrNoisyImages_[0]->width(), hdrNoisyImages_[0]->height(), 1, normalRoughnessImages_[0]->vkFormat(), - VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT); } } @@ -225,13 +241,18 @@ void TemporalAccumulationModule::initRenderPass() { void TemporalAccumulationModule::initFrameBuffers() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; - framebuffers_.resize(size); + framebuffers_.resize(totalCtx); + + for (uint32_t i = 0; i < totalCtx; i++) { + uint32_t frameIdx = i / eyeCount_; + uint32_t eyeIdx = i % eyeCount_; + int viewIdx = (eyeCount_ > 1) ? static_cast(1 + eyeIdx) : 0; - for (int i = 0; i < size; i++) { framebuffers_[i] = vk::FramebufferBuilder{} .beginAttachment() - .defineAttachment(accumulatedRadianceOutImages_[i]) + .defineAttachment(accumulatedRadianceOutImages_[frameIdx], viewIdx) .defineAttachment(accumulatedNormalOutImages_[i]) .endAttachment() .build(framework->device(), renderPass_); @@ -240,6 +261,12 @@ void TemporalAccumulationModule::initFrameBuffers() { void TemporalAccumulationModule::initPipeline() { auto framework = framework_.lock(); + uint32_t targetWidth = framework->swapchain()->vkExtent().width; + uint32_t targetHeight = framework->swapchain()->vkExtent().height; + if (!accumulatedRadianceOutImages_.empty() && accumulatedRadianceOutImages_[0] != nullptr) { + targetWidth = accumulatedRadianceOutImages_[0]->width(); + targetHeight = accumulatedRadianceOutImages_[0]->height(); + } std::filesystem::path shaderPath = Renderer::folderPath / "shaders"; vertShader_ = vk::Shader::create(framework->device(), (shaderPath / "world/temporal_accumulation/tmp_acc_vert.spv").string()); @@ -258,15 +285,15 @@ void TemporalAccumulationModule::initPipeline() { { .x = 0, .y = 0, - .width = static_cast(framework->swapchain()->vkExtent().width), - .height = static_cast(framework->swapchain()->vkExtent().height), + .width = static_cast(targetWidth), + .height = static_cast(targetHeight), .minDepth = 0.0, .maxDepth = 1.0, }, .scissor = { .offset = {.x = 0, .y = 0}, - .extent = framework->swapchain()->vkExtent(), + .extent = {targetWidth, targetHeight}, }, }) .defineDepthStencilState({ @@ -291,31 +318,38 @@ TemporalAccumulationModuleContext::TemporalAccumulationModuleContext( temporalAccumulationModule(temporalAccumulationModule), hdrNoisyImage(temporalAccumulationModule->hdrNoisyImages_[frameworkContext->frameIndex]), motionVectorImage(temporalAccumulationModule->motionVectorImages_[frameworkContext->frameIndex]), - descriptorTable(temporalAccumulationModule->descriptorTables_[frameworkContext->frameIndex]), - framebuffer(temporalAccumulationModule->framebuffers_[frameworkContext->frameIndex]), + descriptorTable(temporalAccumulationModule->descriptorTables_[frameworkContext->frameIndex * temporalAccumulationModule->eyeCount()]), + framebuffer(temporalAccumulationModule->framebuffers_[frameworkContext->frameIndex * temporalAccumulationModule->eyeCount()]), accumulatedRadianceImage(temporalAccumulationModule->accumulatedRadianceImage_), accumulatedNormalImage(temporalAccumulationModule->accumulatedNormalImage_), - accumulatedNormalOutImage(temporalAccumulationModule->accumulatedNormalOutImages_[frameworkContext->frameIndex]), + accumulatedNormalOutImage(temporalAccumulationModule->accumulatedNormalOutImages_[frameworkContext->frameIndex * temporalAccumulationModule->eyeCount()]), accumulatedRadianceOutImage( temporalAccumulationModule->accumulatedRadianceOutImages_[frameworkContext->frameIndex]) {} void TemporalAccumulationModuleContext::render() { + renderEye(0); +} + +void TemporalAccumulationModuleContext::renderEye(uint32_t eyeIndex) { auto context = frameworkContext.lock(); auto framework = context->framework.lock(); auto worldCommandBuffer = context->worldCommandBuffer; auto mainQueueIndex = framework->physicalDevice()->mainQueueIndex(); auto module = temporalAccumulationModule.lock(); + uint32_t dtIdx = context->frameIndex * module->eyeCount() + eyeIndex; + + auto eyeDT = module->descriptorTables_[dtIdx]; + auto eyeFB = module->framebuffers_[dtIdx]; + auto eyeNormalOut = module->accumulatedNormalOutImages_[dtIdx]; TemporalAccumulationPushConstant pc{}; pc.alpha = module->alpha_; pc.threshold = module->threshould_; - vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), descriptorTable->vkPipelineLayout(), - VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR | - VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | - VK_SHADER_STAGE_INTERSECTION_BIT_KHR, - 0, sizeof(uint32_t), &pc); + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), eyeDT->vkPipelineLayout(), + VK_SHADER_STAGE_FRAGMENT_BIT, + 0, sizeof(TemporalAccumulationPushConstant), &pc); worldCommandBuffer->barriersBufferImage( {}, {{ @@ -379,14 +413,14 @@ void TemporalAccumulationModuleContext::render() { worldCommandBuffer->beginRenderPass({ .renderPass = module->renderPass_, - .framebuffer = framebuffer, + .framebuffer = eyeFB, .renderAreaExtent = {accumulatedRadianceOutImage->width(), accumulatedRadianceOutImage->height()}, .clearValues = {{.color = {0.1f, 0.1f, 0.1f, 1.0f}}, {.depthStencil = {.depth = 1.0f}}}, }); accumulatedRadianceOutImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; worldCommandBuffer->bindGraphicsPipeline(module->pipeline_) - ->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) + ->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_GRAPHICS) ->draw(3, 1) ->endRenderPass(); accumulatedRadianceOutImage->imageLayout() = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; @@ -413,11 +447,11 @@ void TemporalAccumulationModuleContext::render() { .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .oldLayout = accumulatedNormalOutImage->imageLayout(), + .oldLayout = eyeNormalOut->imageLayout(), .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = mainQueueIndex, .dstQueueFamilyIndex = mainQueueIndex, - .image = accumulatedNormalOutImage, + .image = eyeNormalOut, .subresourceRange = vk::wholeColorSubresourceRange, }, { @@ -445,23 +479,23 @@ void TemporalAccumulationModuleContext::render() { .subresourceRange = vk::wholeColorSubresourceRange, }}); accumulatedRadianceOutImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; - accumulatedNormalOutImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + eyeNormalOut->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; accumulatedRadianceImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; accumulatedNormalImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - // TODO: add to command buffer + // blit accumulatedRadianceOutImage layer[eyeIndex] → accumulatedRadianceImage layer[eyeIndex] { VkImageBlit imageBlit{}; imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.srcSubresource.mipLevel = 0; - imageBlit.srcSubresource.baseArrayLayer = 0; + imageBlit.srcSubresource.baseArrayLayer = eyeIndex; imageBlit.srcSubresource.layerCount = 1; imageBlit.srcOffsets[0] = {0, 0, 0}; imageBlit.srcOffsets[1] = {static_cast(accumulatedRadianceOutImage->width()), static_cast(accumulatedRadianceOutImage->height()), 1}; imageBlit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.dstSubresource.mipLevel = 0; - imageBlit.dstSubresource.baseArrayLayer = 0; + imageBlit.dstSubresource.baseArrayLayer = eyeIndex; imageBlit.dstSubresource.layerCount = 1; imageBlit.dstOffsets[0] = {0, 0, 0}; imageBlit.dstOffsets[1] = {static_cast(accumulatedRadianceImage->width()), @@ -472,6 +506,7 @@ void TemporalAccumulationModuleContext::render() { accumulatedRadianceImage->imageLayout(), 1, &imageBlit, VK_FILTER_LINEAR); } + // blit eyeNormalOut (single-layer) → accumulatedNormalImage layer[eyeIndex] { VkImageBlit imageBlit{}; imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; @@ -479,18 +514,18 @@ void TemporalAccumulationModuleContext::render() { imageBlit.srcSubresource.baseArrayLayer = 0; imageBlit.srcSubresource.layerCount = 1; imageBlit.srcOffsets[0] = {0, 0, 0}; - imageBlit.srcOffsets[1] = {static_cast(accumulatedNormalOutImage->width()), - static_cast(accumulatedNormalOutImage->height()), 1}; + imageBlit.srcOffsets[1] = {static_cast(eyeNormalOut->width()), + static_cast(eyeNormalOut->height()), 1}; imageBlit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; imageBlit.dstSubresource.mipLevel = 0; - imageBlit.dstSubresource.baseArrayLayer = 0; + imageBlit.dstSubresource.baseArrayLayer = eyeIndex; imageBlit.dstSubresource.layerCount = 1; imageBlit.dstOffsets[0] = {0, 0, 0}; imageBlit.dstOffsets[1] = {static_cast(accumulatedNormalImage->width()), static_cast(accumulatedNormalImage->height()), 1}; - vkCmdBlitImage(worldCommandBuffer->vkCommandBuffer(), accumulatedNormalOutImage->vkImage(), - accumulatedNormalOutImage->imageLayout(), accumulatedNormalImage->vkImage(), + vkCmdBlitImage(worldCommandBuffer->vkCommandBuffer(), eyeNormalOut->vkImage(), + eyeNormalOut->imageLayout(), accumulatedNormalImage->vkImage(), accumulatedNormalImage->imageLayout(), 1, &imageBlit, VK_FILTER_LINEAR); } } \ No newline at end of file diff --git a/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.hpp b/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.hpp index 6921127..f33c3f5 100644 --- a/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.hpp +++ b/src/core/render/modules/world/temporal_accumulation/temporal_accumulation_module.hpp @@ -48,6 +48,8 @@ class TemporalAccumulationModule : public WorldModule, public SharedObject worldPipelineContext, std::shared_ptr temporalAccumulationModule); + StereoMode stereoMode() const override { return StereoMode::SingleInstanceMultiDispatch; } void render() override; + void renderEye(uint32_t eyeIndex) override; }; \ No newline at end of file diff --git a/src/core/render/modules/world/tone_mapping/tone_mapping_module.cpp b/src/core/render/modules/world/tone_mapping/tone_mapping_module.cpp index c996cda..7e0eac0 100644 --- a/src/core/render/modules/world/tone_mapping/tone_mapping_module.cpp +++ b/src/core/render/modules/world/tone_mapping/tone_mapping_module.cpp @@ -23,8 +23,9 @@ bool ToneMappingModule::setOrCreateInputImages(std::vectordevice(), framework->vma(), false, width_, height_, 1, formats[0], + framework->device(), framework->vma(), false, width_, height_, eyeCount_, formats[0], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT); + if (eyeCount_ > 1) images[0]->createPerLayerViews(); } else { if (images[0]->width() != width_ || images[0]->height() != height_) return false; hdrImages_[frameIndex] = images[0]; @@ -91,11 +92,12 @@ void ToneMappingModule::preClose() {} void ToneMappingModule::initDescriptorTables() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); + uint32_t totalCtx = size * eyeCount_; - descriptorTables_.resize(size); - samplers_.resize(size); + descriptorTables_.resize(totalCtx); + samplers_.resize(totalCtx); - for (int i = 0; i < size; i++) { + for (int i = 0; i < totalCtx; i++) { descriptorTables_[i] = vk::DescriptorTableBuilder{} .beginDescriptorLayoutSet() // set 0 .beginDescriptorLayoutSetBinding() @@ -135,8 +137,12 @@ void ToneMappingModule::initImages() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); - for (int i = 0; i < size; i++) { - descriptorTables_[i]->bindSamplerImageForShader(samplers_[i], hdrImages_[i], 0, 0); + for (uint32_t i = 0; i < size; i++) { + for (uint32_t e = 0; e < eyeCount_; e++) { + uint32_t dtIdx = i * eyeCount_ + e; + uint32_t viewIdx = eyeCount_ > 1 ? 1 + e : 0; + descriptorTables_[dtIdx]->bindSamplerImageForShader(samplers_[dtIdx], hdrImages_[i], 0, 0, viewIdx); + } } } @@ -152,13 +158,15 @@ void ToneMappingModule::initBuffers() { vk::DeviceLocalBuffer::create(vma, device, sizeof(ToneMappingModuleExposureData), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - for (int i = 0; i < size; i++) { + for (uint32_t i = 0; i < size; i++) { histBuffers_[i] = vk::DeviceLocalBuffer::create(vma, device, histSize * sizeof(uint32_t), VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - descriptorTables_[i]->bindBuffer(histBuffers_[i], 0, 1); - - descriptorTables_[i]->bindBuffer(exposureData_, 0, 2); + for (uint32_t e = 0; e < eyeCount_; e++) { + uint32_t dtIdx = i * eyeCount_ + e; + descriptorTables_[dtIdx]->bindBuffer(histBuffers_[i], 0, 1); + descriptorTables_[dtIdx]->bindBuffer(exposureData_, 0, 2); + } } } @@ -200,21 +208,32 @@ void ToneMappingModule::initRenderPass() { void ToneMappingModule::initFrameBuffers() { auto framework = framework_.lock(); uint32_t size = framework->swapchain()->imageCount(); - - framebuffers_.resize(size); - - for (int i = 0; i < size; i++) { - framebuffers_[i] = vk::FramebufferBuilder{} - .beginAttachment() - .defineAttachment(ldrImages_[i]) - .endAttachment() - .build(framework->device(), renderPass_); + uint32_t totalCtx = size * eyeCount_; + + framebuffers_.resize(totalCtx); + + for (uint32_t i = 0; i < size; i++) { + for (uint32_t e = 0; e < eyeCount_; e++) { + uint32_t idx = i * eyeCount_ + e; + uint32_t viewIdx = eyeCount_ > 1 ? 1 + e : 0; + framebuffers_[idx] = vk::FramebufferBuilder{} + .beginAttachment() + .defineAttachment(ldrImages_[i], viewIdx) + .endAttachment() + .build(framework->device(), renderPass_); + } } } void ToneMappingModule::initPipeline() { auto framework = framework_.lock(); auto device = framework->device(); + uint32_t targetWidth = framework->swapchain()->vkExtent().width; + uint32_t targetHeight = framework->swapchain()->vkExtent().height; + if (!hdrImages_.empty() && hdrImages_[0] != nullptr) { + targetWidth = hdrImages_[0]->width(); + targetHeight = hdrImages_[0]->height(); + } std::filesystem::path shaderPath = Renderer::folderPath / "shaders"; histShader_ = vk::Shader::create(framework->device(), (shaderPath / "world/tone_mapping/hist_comp.spv").string()); @@ -245,15 +264,15 @@ void ToneMappingModule::initPipeline() { { .x = 0, .y = 0, - .width = static_cast(framework->swapchain()->vkExtent().width), - .height = static_cast(framework->swapchain()->vkExtent().height), + .width = static_cast(targetWidth), + .height = static_cast(targetHeight), .minDepth = 0.0, .maxDepth = 1.0, }, .scissor = { .offset = {.x = 0, .y = 0}, - .extent = framework->swapchain()->vkExtent(), + .extent = {targetWidth, targetHeight}, }, }) .defineDepthStencilState({ @@ -275,13 +294,22 @@ ToneMappingModuleContext::ToneMappingModuleContext(std::shared_ptr toneMappingModule) : WorldModuleContext(frameworkContext, worldPipelineContext), toneMappingModule(toneMappingModule), + eyeCount_(toneMappingModule->eyeCount()), hdrImage(toneMappingModule->hdrImages_[frameworkContext->frameIndex]), - descriptorTable(toneMappingModule->descriptorTables_[frameworkContext->frameIndex]), - framebuffer(toneMappingModule->framebuffers_[frameworkContext->frameIndex]), histBuffer(toneMappingModule->histBuffers_[frameworkContext->frameIndex]), - ldrImage(toneMappingModule->ldrImages_[frameworkContext->frameIndex]) {} + ldrImage(toneMappingModule->ldrImages_[frameworkContext->frameIndex]) { + uint32_t base = frameworkContext->frameIndex * eyeCount_; + for (uint32_t e = 0; e < eyeCount_; e++) { + descriptorTables.push_back(toneMappingModule->descriptorTables_[base + e]); + framebuffers.push_back(toneMappingModule->framebuffers_[base + e]); + } +} void ToneMappingModuleContext::render() { + render3D(1); +} + +void ToneMappingModuleContext::render3D(uint32_t eyeCount) { auto context = frameworkContext.lock(); auto framework = context->framework.lock(); auto worldCommandBuffer = context->worldCommandBuffer; @@ -397,57 +425,66 @@ void ToneMappingModuleContext::render() { pc.minExposure = 1e-4f; pc.maxExposure = 2.0f; - vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), descriptorTable->vkPipelineLayout(), - VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(ToneMappingModulePushConstant), &pc); - - worldCommandBuffer->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_COMPUTE) - ->bindComputePipeline(module->histPipeline_); - - uint32_t groupX = (module->width_ + 16 - 1) / 16; - uint32_t groupY = (module->height_ + 16 - 1) / 16; - vkCmdDispatch(worldCommandBuffer->vkCommandBuffer(), groupX, groupY, 1); - - worldCommandBuffer->barriersBufferImage( - {{ - .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, - .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_2_TRANSFER_BIT, - .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .srcQueueFamilyIndex = mainQueueIndex, - .dstQueueFamilyIndex = mainQueueIndex, - .buffer = histBuffer, - }}, - {}); - - worldCommandBuffer->bindComputePipeline(module->exposurePipeline_); - vkCmdDispatch(worldCommandBuffer->vkCommandBuffer(), 1, 1, 1); - - worldCommandBuffer->barriersBufferImage( - {{ - .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, - .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_2_TRANSFER_BIT, - .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, - .srcQueueFamilyIndex = mainQueueIndex, - .dstQueueFamilyIndex = mainQueueIndex, - .buffer = module->exposureData_, - }}, - {}); + // Eye 0: histogram → exposure → tone mapping + // Eye 1+: tone mapping only (reuse eye 0's exposure for consistent brightness) + for (uint32_t eye = 0; eye < eyeCount; eye++) { + auto &eyeDT = descriptorTables[eye]; + auto &eyeFB = framebuffers[eye]; + + if (eye == 0) { + vkCmdPushConstants(worldCommandBuffer->vkCommandBuffer(), eyeDT->vkPipelineLayout(), + VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(ToneMappingModulePushConstant), &pc); + + worldCommandBuffer->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_COMPUTE) + ->bindComputePipeline(module->histPipeline_); + + uint32_t groupX = (module->width_ + 16 - 1) / 16; + uint32_t groupY = (module->height_ + 16 - 1) / 16; + vkCmdDispatch(worldCommandBuffer->vkCommandBuffer(), groupX, groupY, 1); + + worldCommandBuffer->barriersBufferImage( + {{ + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .buffer = histBuffer, + }}, + {}); + + worldCommandBuffer->bindComputePipeline(module->exposurePipeline_); + vkCmdDispatch(worldCommandBuffer->vkCommandBuffer(), 1, 1, 1); + + worldCommandBuffer->barriersBufferImage( + {{ + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | + VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT, + .srcQueueFamilyIndex = mainQueueIndex, + .dstQueueFamilyIndex = mainQueueIndex, + .buffer = module->exposureData_, + }}, + {}); + } - worldCommandBuffer->beginRenderPass({ - .renderPass = module->renderPass_, - .framebuffer = framebuffer, - .renderAreaExtent = {ldrImage->width(), ldrImage->height()}, - .clearValues = {{.color = {0.1f, 0.1f, 0.1f, 1.0f}}, {.depthStencil = {.depth = 1.0f}}}, - }); - ldrImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + worldCommandBuffer->beginRenderPass({ + .renderPass = module->renderPass_, + .framebuffer = eyeFB, + .renderAreaExtent = {ldrImage->width(), ldrImage->height()}, + .clearValues = {{.color = {0.1f, 0.1f, 0.1f, 1.0f}}, {.depthStencil = {.depth = 1.0f}}}, + }); + + worldCommandBuffer->bindGraphicsPipeline(module->pipeline_) + ->bindDescriptorTable(eyeDT, VK_PIPELINE_BIND_POINT_GRAPHICS) + ->draw(3, 1) + ->endRenderPass(); + } - worldCommandBuffer->bindGraphicsPipeline(module->pipeline_) - ->bindDescriptorTable(descriptorTable, VK_PIPELINE_BIND_POINT_GRAPHICS) - ->draw(3, 1) - ->endRenderPass(); #ifdef USE_AMD ldrImage->imageLayout() = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; #else diff --git a/src/core/render/modules/world/tone_mapping/tone_mapping_module.hpp b/src/core/render/modules/world/tone_mapping/tone_mapping_module.hpp index 7128b32..4f04901 100644 --- a/src/core/render/modules/world/tone_mapping/tone_mapping_module.hpp +++ b/src/core/render/modules/world/tone_mapping/tone_mapping_module.hpp @@ -66,6 +66,8 @@ class ToneMappingModule : public WorldModule, public SharedObject { std::weak_ptr toneMappingModule; + StereoMode stereoMode() const override { return StereoMode::SingleInstance3DDispatch; } + void render3D(uint32_t eyeCount) override; + // input std::shared_ptr hdrImage; - // tone mapping - std::shared_ptr descriptorTable; - std::shared_ptr framebuffer; + // tone mapping (per-eye indexed) + std::vector> descriptorTables; + std::vector> framebuffers; std::shared_ptr histBuffer; // output @@ -132,4 +137,7 @@ struct ToneMappingModuleContext : public WorldModuleContext, SharedObject toneMappingModule); void render() override; + + private: + uint32_t eyeCount_; }; \ No newline at end of file diff --git a/src/core/render/modules/world/world_module.hpp b/src/core/render/modules/world/world_module.hpp index 27f18b4..9331414 100644 --- a/src/core/render/modules/world/world_module.hpp +++ b/src/core/render/modules/world/world_module.hpp @@ -14,6 +14,12 @@ struct WorldPipelineContext; struct WorldModuleContext; +enum class StereoMode { + SingleInstance3DDispatch, + SingleInstanceMultiDispatch, + DualInstance +}; + class WorldModule { public: WorldModule(); @@ -35,10 +41,14 @@ class WorldModule { virtual void bindTexture(std::shared_ptr sampler, std::shared_ptr image, int index) = 0; - // release resources that must be released before deconstruction virtual void preClose() = 0; + virtual StereoMode stereoMode() const { return StereoMode::SingleInstance3DDispatch; } + virtual uint32_t eyeCount() const { return eyeCount_; } + virtual void setEyeCount(uint32_t count) { eyeCount_ = count; } + protected: + uint32_t eyeCount_ = 1; std::weak_ptr framework_; std::weak_ptr worldPipeline_; }; @@ -50,5 +60,11 @@ struct WorldModuleContext { WorldModuleContext(std::shared_ptr frameworkContext, std::shared_ptr worldPipelineContext); + virtual StereoMode stereoMode() const { return StereoMode::SingleInstance3DDispatch; } + virtual void render() = 0; + virtual void render3D(uint32_t eyeCount) { render(); } + virtual void renderEye(uint32_t eyeIndex) { currentEyeIndex = eyeIndex; render(); } + + uint32_t currentEyeIndex = 0; }; \ No newline at end of file diff --git a/src/core/render/openxr_context.cpp b/src/core/render/openxr_context.cpp new file mode 100644 index 0000000..e032589 --- /dev/null +++ b/src/core/render/openxr_context.cpp @@ -0,0 +1,807 @@ +#ifdef MCVR_ENABLE_OPENXR + +#include "core/render/openxr_context.hpp" +#include "core/render/renderer.hpp" + +#include +#include +#include +#include +#include + +// Tee streambuf: writes to both a file and another streambuf +class TeeStreambuf : public std::streambuf { +public: + TeeStreambuf(std::streambuf *a, std::streambuf *b) : a_(a), b_(b) {} +protected: + int overflow(int c) override { + if (c == EOF) return !EOF; + int r1 = a_->sputc(c); + int r2 = b_->sputc(c); + return (r1 == EOF || r2 == EOF) ? EOF : c; + } + int sync() override { a_->pubsync(); b_->pubsync(); return 0; } +private: + std::streambuf *a_; + std::streambuf *b_; +}; + +static std::ofstream &xrLogFile() { + static std::ofstream file("openxr_debug.log", std::ios::out | std::ios::trunc); + return file; +} + +static std::ostream &xrCout() { + static TeeStreambuf tee(std::cout.rdbuf(), xrLogFile().rdbuf()); + static std::ostream teeStream(&tee); + return teeStream << "[OpenXR] "; +} + +static std::ostream &xrCerr() { + static TeeStreambuf tee(std::cerr.rdbuf(), xrLogFile().rdbuf()); + static std::ostream teeStream(&tee); + return teeStream << "[OpenXR][ERROR] "; +} + +static const char *xrResultStr(XrResult result) { + switch (result) { + case XR_SUCCESS: return "XR_SUCCESS"; + case XR_ERROR_FORM_FACTOR_UNAVAILABLE: return "XR_ERROR_FORM_FACTOR_UNAVAILABLE"; + case XR_ERROR_RUNTIME_UNAVAILABLE: return "XR_ERROR_RUNTIME_UNAVAILABLE"; + case XR_ERROR_INSTANCE_LOST: return "XR_ERROR_INSTANCE_LOST"; + case XR_ERROR_SESSION_LOST: return "XR_ERROR_SESSION_LOST"; + default: return "XR_UNKNOWN"; + } +} + +#define XR_CHECK(call, msg) \ + do { \ + XrResult _r = (call); \ + if (XR_FAILED(_r)) { \ + xrCerr() << (msg) << " failed: " << xrResultStr(_r) << std::endl; \ + return false; \ + } \ + } while (0) + +#define XR_CHECK_VOID(call, msg) \ + do { \ + XrResult _r = (call); \ + if (XR_FAILED(_r)) { \ + xrCerr() << (msg) << " failed: " << xrResultStr(_r) << std::endl; \ + return; \ + } \ + } while (0) + +// Split a space-delimited string of extensions into a vector of individual names. +static std::vector splitExtensions(const char *str) { + std::vector result; + std::istringstream ss(str); + std::string token; + while (ss >> token) { result.push_back(token); } + return result; +} + +// ---- Stage A: preVulkanInit ---- + +bool OpenXRContext::preVulkanInit() { + // 1. Create XrInstance + XrInstanceCreateInfo instanceCI{XR_TYPE_INSTANCE_CREATE_INFO}; + std::strncpy(instanceCI.applicationInfo.applicationName, "MCVR", XR_MAX_APPLICATION_NAME_SIZE); + instanceCI.applicationInfo.applicationVersion = 1; + std::strncpy(instanceCI.applicationInfo.engineName, "Radiance", XR_MAX_ENGINE_NAME_SIZE); + instanceCI.applicationInfo.engineVersion = 1; + instanceCI.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 1, 0); + + // Request Vulkan graphics binding extension (v1 — matches our use of v1 query functions) + // Also request optional extensions: visibility mask, eye gaze + std::vector requestedExts = {XR_KHR_VULKAN_ENABLE_EXTENSION_NAME}; + + // Check available extensions and add optional ones + uint32_t availExtCount = 0; + xrEnumerateInstanceExtensionProperties(nullptr, 0, &availExtCount, nullptr); + std::vector availExts(availExtCount, {XR_TYPE_EXTENSION_PROPERTIES}); + xrEnumerateInstanceExtensionProperties(nullptr, availExtCount, &availExtCount, availExts.data()); + for (auto &ext : availExts) { + if (std::strcmp(ext.extensionName, "XR_KHR_visibility_mask") == 0) { + requestedExts.push_back("XR_KHR_visibility_mask"); + visMaskAvailable_ = true; + xrCout() << "Visibility mask extension available" << std::endl; + } + if (std::strcmp(ext.extensionName, "XR_EXT_eye_gaze_interaction") == 0) { + requestedExts.push_back("XR_EXT_eye_gaze_interaction"); + xrCout() << "Eye gaze interaction extension available" << std::endl; + } + } + + instanceCI.enabledExtensionCount = static_cast(requestedExts.size()); + instanceCI.enabledExtensionNames = requestedExts.data(); + + XR_CHECK(xrCreateInstance(&instanceCI, &xrInstance_), "xrCreateInstance"); + xrCout() << "XrInstance created" << std::endl; + + // 2. Get system (HMD) + XrSystemGetInfo systemGI{XR_TYPE_SYSTEM_GET_INFO}; + systemGI.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + XR_CHECK(xrGetSystem(xrInstance_, &systemGI, &systemId_), "xrGetSystem"); + xrCout() << "XrSystem acquired (id=" << systemId_ << ")" << std::endl; + + // Query system properties (device name) + XrSystemProperties sysProps{XR_TYPE_SYSTEM_PROPERTIES}; + if (xrGetSystemProperties(xrInstance_, systemId_, &sysProps) == XR_SUCCESS) { + systemName_ = sysProps.systemName; + xrCout() << "System name: " << systemName_ << std::endl; + } + + // 3. Query Vulkan instance extensions required by OpenXR + PFN_xrGetVulkanInstanceExtensionsKHR xrGetVulkanInstanceExtensionsKHR = nullptr; + XR_CHECK(xrGetInstanceProcAddr(xrInstance_, "xrGetVulkanInstanceExtensionsKHR", + reinterpret_cast(&xrGetVulkanInstanceExtensionsKHR)), + "get xrGetVulkanInstanceExtensionsKHR"); + + uint32_t bufSize = 0; + XR_CHECK(xrGetVulkanInstanceExtensionsKHR(xrInstance_, systemId_, 0, &bufSize, nullptr), + "xrGetVulkanInstanceExtensionsKHR(size)"); + std::vector instExtBuf(bufSize); + XR_CHECK(xrGetVulkanInstanceExtensionsKHR(xrInstance_, systemId_, bufSize, &bufSize, instExtBuf.data()), + "xrGetVulkanInstanceExtensionsKHR(data)"); + requiredInstanceExts_ = splitExtensions(instExtBuf.data()); + + xrCout() << "Required Vulkan instance extensions:" << std::endl; + for (auto &e : requiredInstanceExts_) { xrCout() << " " << e << std::endl; } + + // 4. Query Vulkan device extensions required by OpenXR + PFN_xrGetVulkanDeviceExtensionsKHR xrGetVulkanDeviceExtensionsKHR = nullptr; + XR_CHECK(xrGetInstanceProcAddr(xrInstance_, "xrGetVulkanDeviceExtensionsKHR", + reinterpret_cast(&xrGetVulkanDeviceExtensionsKHR)), + "get xrGetVulkanDeviceExtensionsKHR"); + + bufSize = 0; + XR_CHECK(xrGetVulkanDeviceExtensionsKHR(xrInstance_, systemId_, 0, &bufSize, nullptr), + "xrGetVulkanDeviceExtensionsKHR(size)"); + std::vector devExtBuf(bufSize); + XR_CHECK(xrGetVulkanDeviceExtensionsKHR(xrInstance_, systemId_, bufSize, &bufSize, devExtBuf.data()), + "xrGetVulkanDeviceExtensionsKHR(data)"); + requiredDeviceExts_ = splitExtensions(devExtBuf.data()); + + xrCout() << "Required Vulkan device extensions:" << std::endl; + for (auto &e : requiredDeviceExts_) { xrCout() << " " << e << std::endl; } + + // 5. Query view configuration (stereo) to get recommended resolution + uint32_t viewCount = 0; + XR_CHECK(xrEnumerateViewConfigurationViews(xrInstance_, systemId_, viewConfigType_, 0, &viewCount, nullptr), + "xrEnumerateViewConfigurationViews(count)"); + if (viewCount != 2) { + xrCerr() << "Expected 2 stereo views, got " << viewCount << std::endl; + return false; + } + configViews_[0] = {XR_TYPE_VIEW_CONFIGURATION_VIEW}; + configViews_[1] = {XR_TYPE_VIEW_CONFIGURATION_VIEW}; + XR_CHECK(xrEnumerateViewConfigurationViews(xrInstance_, systemId_, viewConfigType_, 2, &viewCount, + configViews_.data()), + "xrEnumerateViewConfigurationViews(data)"); + + xrCout() << "Recommended per-eye resolution: " + << configViews_[0].recommendedImageRectWidth << "x" + << configViews_[0].recommendedImageRectHeight << std::endl; + + return true; +} + +VkPhysicalDevice OpenXRContext::getXRPhysicalDevice(VkInstance vkInstance) const { + PFN_xrGetVulkanGraphicsDeviceKHR xrGetVulkanGraphicsDeviceKHR = nullptr; + xrGetInstanceProcAddr(xrInstance_, "xrGetVulkanGraphicsDeviceKHR", + reinterpret_cast(&xrGetVulkanGraphicsDeviceKHR)); + VkPhysicalDevice xrDevice = VK_NULL_HANDLE; + XrResult r = xrGetVulkanGraphicsDeviceKHR(xrInstance_, systemId_, vkInstance, &xrDevice); + if (XR_FAILED(r)) { + xrCerr() << "xrGetVulkanGraphicsDeviceKHR failed" << std::endl; + return VK_NULL_HANDLE; + } + return xrDevice; +} + +// ---- Stage B: postVulkanInit ---- + +bool OpenXRContext::postVulkanInit(VkInstance vkInstance, VkPhysicalDevice vkPhysicalDevice, + VkDevice vkDevice, uint32_t queueFamilyIndex, + uint32_t queueIndex) { + vkInstance_ = vkInstance; + vkPhysicalDevice_ = vkPhysicalDevice; + vkDevice_ = vkDevice; + queueFamilyIndex_ = queueFamilyIndex; + queueIndex_ = queueIndex; + vulkanReady_ = true; + xrCout() << "OpenXR Vulkan bridge ready (session deferred)" << std::endl; + return true; +} + +bool OpenXRContext::createSession(VkInstance vkInstance, VkPhysicalDevice vkPhysicalDevice, + VkDevice vkDevice, uint32_t queueFamilyIndex, + uint32_t queueIndex) { + // Vulkan graphics binding + XrGraphicsBindingVulkanKHR binding{XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR}; + binding.instance = vkInstance; + binding.physicalDevice = vkPhysicalDevice; + binding.device = vkDevice; + binding.queueFamilyIndex = queueFamilyIndex; + binding.queueIndex = queueIndex; + + // Spec requires calling xrGetVulkanGraphicsRequirementsKHR before xrCreateSession + PFN_xrGetVulkanGraphicsRequirementsKHR xrGetVulkanGraphicsRequirementsKHR = nullptr; + XR_CHECK(xrGetInstanceProcAddr(xrInstance_, "xrGetVulkanGraphicsRequirementsKHR", + reinterpret_cast(&xrGetVulkanGraphicsRequirementsKHR)), + "get xrGetVulkanGraphicsRequirementsKHR"); + XrGraphicsRequirementsVulkanKHR graphicsReqs{XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN_KHR}; + XR_CHECK(xrGetVulkanGraphicsRequirementsKHR(xrInstance_, systemId_, &graphicsReqs), + "xrGetVulkanGraphicsRequirementsKHR"); + xrCout() << "Vulkan requirements: min=" << XR_VERSION_MAJOR(graphicsReqs.minApiVersionSupported) + << "." << XR_VERSION_MINOR(graphicsReqs.minApiVersionSupported) + << " max=" << XR_VERSION_MAJOR(graphicsReqs.maxApiVersionSupported) + << "." << XR_VERSION_MINOR(graphicsReqs.maxApiVersionSupported) << std::endl; + + XrSessionCreateInfo sessionCI{XR_TYPE_SESSION_CREATE_INFO}; + sessionCI.next = &binding; + sessionCI.systemId = systemId_; + XR_CHECK(xrCreateSession(xrInstance_, &sessionCI, &session_), "xrCreateSession"); + xrCout() << "XrSession created" << std::endl; + + // Create reference space (LOCAL = seated, STAGE = standing/room-scale) + XrReferenceSpaceCreateInfo spaceCI{XR_TYPE_REFERENCE_SPACE_CREATE_INFO}; + spaceCI.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL; + spaceCI.poseInReferenceSpace = {{0, 0, 0, 1}, {0, 0, 0}}; + XR_CHECK(xrCreateReferenceSpace(session_, &spaceCI, &appSpace_), "xrCreateReferenceSpace"); + xrCout() << "Reference space created (LOCAL)" << std::endl; + + return true; +} + +bool OpenXRContext::createSwapchains() { + // Enumerate supported swapchain formats — prefer SRGB, fall back to UNORM + uint32_t formatCount = 0; + XR_CHECK(xrEnumerateSwapchainFormats(session_, 0, &formatCount, nullptr), "xrEnumerateSwapchainFormats(count)"); + std::vector formats(formatCount); + XR_CHECK(xrEnumerateSwapchainFormats(session_, formatCount, &formatCount, formats.data()), + "xrEnumerateSwapchainFormats(data)"); + + // Prefer R8G8B8A8_SRGB, then R8G8B8A8_UNORM, then B8G8R8A8_SRGB + int64_t selectedFormat = formats[0]; + const int64_t preferred[] = { + VK_FORMAT_R8G8B8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_B8G8R8A8_UNORM, + }; + for (int64_t pref : preferred) { + for (int64_t fmt : formats) { + if (fmt == pref) { selectedFormat = pref; goto found; } + } + } + found: + xrCout() << "Swapchain format: " << selectedFormat << std::endl; + + // Create per-eye swapchains + for (uint32_t eye = 0; eye < 2; eye++) { + auto &es = eyeSwapchains_[eye]; + es.width = configViews_[eye].recommendedImageRectWidth; + es.height = configViews_[eye].recommendedImageRectHeight; + + XrSwapchainCreateInfo swapCI{XR_TYPE_SWAPCHAIN_CREATE_INFO}; + swapCI.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + swapCI.format = selectedFormat; + swapCI.sampleCount = 1; + swapCI.width = es.width; + swapCI.height = es.height; + swapCI.faceCount = 1; + swapCI.arraySize = 1; + swapCI.mipCount = 1; + XR_CHECK(xrCreateSwapchain(session_, &swapCI, &es.handle), "xrCreateSwapchain"); + + // Enumerate VkImages owned by the swapchain + uint32_t imgCount = 0; + XR_CHECK(xrEnumerateSwapchainImages(es.handle, 0, &imgCount, nullptr), "xrEnumerateSwapchainImages(count)"); + std::vector xrImages(imgCount, {XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR}); + XR_CHECK(xrEnumerateSwapchainImages(es.handle, imgCount, &imgCount, + reinterpret_cast(xrImages.data())), + "xrEnumerateSwapchainImages(data)"); + es.images.resize(imgCount); + for (uint32_t i = 0; i < imgCount; i++) { es.images[i] = xrImages[i].image; } + + xrCout() << "Eye " << eye << " swapchain: " << es.width << "x" << es.height + << ", " << imgCount << " images" << std::endl; + } + return true; +} + +// ---- Per-frame operations ---- + +void OpenXRContext::pollEvents() { + XrEventDataBuffer event{XR_TYPE_EVENT_DATA_BUFFER}; + while (xrPollEvent(xrInstance_, &event) == XR_SUCCESS) { + switch (event.type) { + case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED: { + auto *stateEvent = reinterpret_cast(&event); + handleSessionStateChange(stateEvent->state); + break; + } + case XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING: + xrCerr() << "XR instance loss pending!" << std::endl; + break; + default: + break; + } + event = {XR_TYPE_EVENT_DATA_BUFFER}; + } +} + +void OpenXRContext::handleSessionStateChange(XrSessionState newState) { + sessionState_ = newState; + xrCout() << "Session state -> " << static_cast(newState) << std::endl; + + switch (newState) { + case XR_SESSION_STATE_READY: { + if (sessionRequested_) { + XrSessionBeginInfo beginInfo{XR_TYPE_SESSION_BEGIN_INFO}; + beginInfo.primaryViewConfigurationType = viewConfigType_; + XrResult r = xrBeginSession(session_, &beginInfo); + if (XR_SUCCEEDED(r)) { + sessionRunning_ = true; + xrCout() << "Session started" << std::endl; + } + } else { + xrCout() << "Session READY but not requested, waiting..." << std::endl; + } + break; + } + case XR_SESSION_STATE_STOPPING: + if (session_ != XR_NULL_HANDLE) { + xrEndSession(session_); + } + sessionRunning_ = false; + xrCout() << "Session stopped" << std::endl; + if (destroyPending_ && !sessionRequested_) { + destroySessionResources(); + } + break; + case XR_SESSION_STATE_EXITING: + case XR_SESSION_STATE_LOSS_PENDING: + sessionRunning_ = false; + if (destroyPending_) { + destroySessionResources(); + } + break; + default: + break; + } +} + +bool OpenXRContext::requestSessionStart() { + if (!vulkanReady_) { + xrCerr() << "Cannot start XR session: Vulkan bridge is not ready" << std::endl; + return false; + } + + sessionRequested_ = true; + destroyPending_ = false; + + if (session_ == XR_NULL_HANDLE) { + if (!createSession(vkInstance_, vkPhysicalDevice_, vkDevice_, queueFamilyIndex_, queueIndex_)) { + xrCerr() << "Failed to create XR session on demand" << std::endl; + sessionRequested_ = false; + return false; + } + if (!createSwapchains()) { + xrCerr() << "Failed to create XR swapchains on demand" << std::endl; + destroySessionResources(); + sessionRequested_ = false; + return false; + } + + if (input_.createActions(xrInstance_, session_) && input_.attachToSession(session_)) { + inputInitialized_ = true; + } + + if (visMaskAvailable_) { + queryVisibilityMask(); + } + } + + if (sessionRunning_) return true; + + // If runtime is already READY, begin immediately. Otherwise pollEvents() + // will begin when READY arrives. + if (sessionState_ == XR_SESSION_STATE_READY) { + XrSessionBeginInfo beginInfo{XR_TYPE_SESSION_BEGIN_INFO}; + beginInfo.primaryViewConfigurationType = viewConfigType_; + XrResult r = xrBeginSession(session_, &beginInfo); + if (XR_SUCCEEDED(r)) { + sessionRunning_ = true; + xrCout() << "Session started (requested immediately)" << std::endl; + return true; + } + xrCerr() << "xrBeginSession failed on request" << std::endl; + sessionRequested_ = false; + return false; + } + + return true; +} + +void OpenXRContext::requestSessionStop() { + sessionRequested_ = false; + destroyPending_ = true; + + if (session_ == XR_NULL_HANDLE) { + destroyPending_ = false; + return; + } + + if (frameState_ == FRAME_LATCHED) { + endFrame(); + } + + for (auto &es : eyeSwapchains_) { + if (es.handle != XR_NULL_HANDLE && es.imageAcquired) { + XrSwapchainImageReleaseInfo releaseInfo{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; + XrResult rr = xrReleaseSwapchainImage(es.handle, &releaseInfo); + if (XR_FAILED(rr)) { + xrCerr() << "xrReleaseSwapchainImage during stop failed" << std::endl; + } + es.imageAcquired = false; + } + } + + if (session_ != XR_NULL_HANDLE && sessionRunning_) { + // Ask runtime to transition to STOPPING first. + XrResult r = xrRequestExitSession(session_); + if (XR_FAILED(r)) { + xrCerr() << "xrRequestExitSession failed, forcing xrEndSession" << std::endl; + xrEndSession(session_); + sessionRunning_ = false; + } + } + + if (!sessionRunning_) { + destroySessionResources(); + destroyPending_ = false; + } +} + +void OpenXRContext::destroySessionResources() { + if (inputInitialized_) { + input_.shutdown(); + inputInitialized_ = false; + } + + for (auto &es : eyeSwapchains_) { + if (es.handle != XR_NULL_HANDLE && es.imageAcquired) { + XrSwapchainImageReleaseInfo releaseInfo{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; + xrReleaseSwapchainImage(es.handle, &releaseInfo); + es.imageAcquired = false; + } + if (es.handle != XR_NULL_HANDLE) { + xrDestroySwapchain(es.handle); + es.handle = XR_NULL_HANDLE; + } + es.images.clear(); + es.width = 0; + es.height = 0; + es.acquiredIndex = 0; + es.imageAcquired = false; + } + + if (appSpace_ != XR_NULL_HANDLE) { + xrDestroySpace(appSpace_); + appSpace_ = XR_NULL_HANDLE; + } + + if (session_ != XR_NULL_HANDLE) { + xrDestroySession(session_); + session_ = XR_NULL_HANDLE; + } + + sessionRunning_ = false; + sessionState_ = XR_SESSION_STATE_UNKNOWN; + frameState_ = FRAME_IDLE; + viewsValid_ = false; + destroyPending_ = false; + visMaskVertices_[0].clear(); + visMaskVertices_[1].clear(); + visMaskIndices_[0].clear(); + visMaskIndices_[1].clear(); +} + +void OpenXRContext::beginFrameRecording() { + frameState_ = FRAME_RECORDING; + viewsValid_ = false; // Will be updated in latchPose + lastWaitFrameMs_ = 0.0f; + lastSwapchainWaitMs_ = 0.0f; + + // Initialize XR frame state for upcoming latchPose() call + xrFrameState_ = {XR_TYPE_FRAME_STATE}; +} + +bool OpenXRContext::latchPose() { + // Consolidate session state validation + if (!sessionRunning_ || sessionState_ < XR_SESSION_STATE_READY || sessionState_ > XR_SESSION_STATE_FOCUSED) { + return false; + } + + if (frameState_ != FRAME_RECORDING) { + xrCerr() << "latchPose called without beginFrameRecording()" << std::endl; + return false; + } + + // CRITICAL: This is where we call the blocking xrWaitFrame + XrFrameWaitInfo waitInfo{XR_TYPE_FRAME_WAIT_INFO}; + auto waitFrameStart = std::chrono::high_resolution_clock::now(); + XrResult r = xrWaitFrame(session_, &waitInfo, &xrFrameState_); + lastWaitFrameMs_ = std::chrono::duration( + std::chrono::high_resolution_clock::now() - waitFrameStart).count(); + if (XR_FAILED(r)) { + xrCerr() << "xrWaitFrame failed" << std::endl; + return false; + } + + XrFrameBeginInfo beginInfo{XR_TYPE_FRAME_BEGIN_INFO}; + r = xrBeginFrame(session_, &beginInfo); + if (XR_FAILED(r)) { + xrCerr() << "xrBeginFrame failed" << std::endl; + return false; + } + + frameState_ = FRAME_LATCHED; + + // Locate views (head pose + per-eye FOV) with latest timing + viewsValid_ = false; + XrViewLocateInfo locateInfo{XR_TYPE_VIEW_LOCATE_INFO}; + locateInfo.viewConfigurationType = viewConfigType_; + locateInfo.displayTime = xrFrameState_.predictedDisplayTime; + locateInfo.space = appSpace_; + + XrViewState viewState{XR_TYPE_VIEW_STATE}; + uint32_t viewCount = 0; + views_[0] = {XR_TYPE_VIEW}; + views_[1] = {XR_TYPE_VIEW}; + r = xrLocateViews(session_, &locateInfo, &viewState, 2, &viewCount, views_.data()); + if (XR_SUCCEEDED(r) && (viewState.viewStateFlags & XR_VIEW_STATE_ORIENTATION_VALID_BIT)) { + viewsValid_ = true; + } + + // Sync controller input with latest timing + if (inputInitialized_) { + input_.syncAndUpdate(session_, appSpace_, xrFrameState_.predictedDisplayTime); + } + + return true; +} + +VkImage OpenXRContext::acquireSwapchainImage(uint32_t eye) { + auto &es = eyeSwapchains_[eye]; + if (es.handle == XR_NULL_HANDLE || es.images.empty()) { + xrCerr() << "acquireSwapchainImage called without a valid swapchain" << std::endl; + return VK_NULL_HANDLE; + } + + XrSwapchainImageAcquireInfo acquireInfo{XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO}; + XrResult r = xrAcquireSwapchainImage(es.handle, &acquireInfo, &es.acquiredIndex); + if (XR_FAILED(r)) { + xrCerr() << "xrAcquireSwapchainImage failed" << std::endl; + return VK_NULL_HANDLE; + } + + XrSwapchainImageWaitInfo waitInfo{XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO}; + waitInfo.timeout = XR_INFINITE_DURATION; + auto waitSwapchainStart = std::chrono::high_resolution_clock::now(); + r = xrWaitSwapchainImage(es.handle, &waitInfo); + lastSwapchainWaitMs_ += std::chrono::duration( + std::chrono::high_resolution_clock::now() - waitSwapchainStart).count(); + if (XR_FAILED(r)) { + xrCerr() << "xrWaitSwapchainImage failed" << std::endl; + return VK_NULL_HANDLE; + } + + if (es.acquiredIndex >= es.images.size()) { + xrCerr() << "acquired swapchain image index out of range" << std::endl; + return VK_NULL_HANDLE; + } + + es.imageAcquired = true; + + return es.images[es.acquiredIndex]; +} + +void OpenXRContext::releaseSwapchainImage(uint32_t eye) { + auto &es = eyeSwapchains_[eye]; + if (es.handle == XR_NULL_HANDLE || !es.imageAcquired) { + return; + } + + XrSwapchainImageReleaseInfo releaseInfo{XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO}; + XrResult r = xrReleaseSwapchainImage(es.handle, &releaseInfo); + if (XR_FAILED(r)) { + xrCerr() << "xrReleaseSwapchainImage failed" << std::endl; + return; + } + es.imageAcquired = false; +} + +void OpenXRContext::endFrame() { + if (frameState_ != FRAME_LATCHED) return; + frameState_ = FRAME_IDLE; + + // Build projection views referencing each eye's swapchain + for (uint32_t eye = 0; eye < 2; eye++) { + projViews_[eye] = {XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW}; + projViews_[eye].pose = views_[eye].pose; + projViews_[eye].fov = views_[eye].fov; + projViews_[eye].subImage.swapchain = eyeSwapchains_[eye].handle; + projViews_[eye].subImage.imageRect.offset = {0, 0}; + projViews_[eye].subImage.imageRect.extent = { + static_cast(eyeSwapchains_[eye].width), + static_cast(eyeSwapchains_[eye].height)}; + projViews_[eye].subImage.imageArrayIndex = 0; + } + + XrCompositionLayerProjection projLayer{XR_TYPE_COMPOSITION_LAYER_PROJECTION}; + projLayer.space = appSpace_; + projLayer.viewCount = 2; + projLayer.views = projViews_.data(); + + const XrCompositionLayerBaseHeader *layers[] = { + reinterpret_cast(&projLayer)}; + + XrFrameEndInfo endInfo{XR_TYPE_FRAME_END_INFO}; + endInfo.displayTime = xrFrameState_.predictedDisplayTime; + endInfo.environmentBlendMode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + if (xrFrameState_.shouldRender == XR_TRUE && viewsValid_) { + endInfo.layerCount = 1; + endInfo.layers = layers; + } else { + endInfo.layerCount = 0; + endInfo.layers = nullptr; + } + + XrResult r = xrEndFrame(session_, &endInfo); + if (XR_FAILED(r)) { + xrCerr() << "xrEndFrame failed: " << xrResultStr(r) << std::endl; + } +} + +// ---- Query helpers ---- + +void OpenXRContext::getEyeParams(VREyeParams eyes[2]) const { + // Compute head pose to derive per-eye RELATIVE offsets. + // headView is applied separately (in buffers.cpp), so eyeViewOffset must only + // contain the small IPD translation, not the full world-space eye pose. + glm::vec3 headPos{0.0f}; + glm::quat headOri{1.0f, 0.0f, 0.0f, 0.0f}; + if (viewsValid_) { + const XrPosef &left = views_[0].pose; + const XrPosef &right = views_[1].pose; + headPos = {(left.position.x + right.position.x) * 0.5f, + (left.position.y + right.position.y) * 0.5f, + (left.position.z + right.position.z) * 0.5f}; + headOri = {left.orientation.w, left.orientation.x, + left.orientation.y, left.orientation.z}; + } + glm::quat headOriInv = glm::inverse(headOri); + + for (uint32_t eye = 0; eye < 2; eye++) { + auto &cv = configViews_[eye]; + eyes[eye].recommendedWidth = cv.recommendedImageRectWidth; + eyes[eye].recommendedHeight = cv.recommendedImageRectHeight; + + if (viewsValid_) { + const XrFovf &fov = views_[eye].fov; + eyes[eye].tanLeft = std::tan(fov.angleLeft); + eyes[eye].tanRight = std::tan(fov.angleRight); + eyes[eye].tanUp = std::tan(fov.angleUp); + eyes[eye].tanDown = std::tan(fov.angleDown); + + const XrPosef &pose = views_[eye].pose; + glm::vec3 eyePos = {pose.position.x, pose.position.y, pose.position.z}; + glm::quat eyeOri = {pose.orientation.w, pose.orientation.x, + pose.orientation.y, pose.orientation.z}; + + // Store offsets RELATIVE to head pose (typically just ±IPD/2 along X) + eyes[eye].positionOffset = headOriInv * (eyePos - headPos); + eyes[eye].orientationOffset = headOriInv * eyeOri; + } + } +} + +void OpenXRContext::getHeadPose(VRHeadPose &pose) const { + if (!viewsValid_ || !sessionRunning_) { + pose = VRHeadPose{}; + return; + } + // Head pose is the average of left and right eye poses (approximation). + // For most runtimes, views[0].pose and views[1].pose share orientation + // and differ only in the IPD translation. We use the left eye's orientation + // and the midpoint position. + const XrPosef &left = views_[0].pose; + const XrPosef &right = views_[1].pose; + pose.position = {(left.position.x + right.position.x) * 0.5f, + (left.position.y + right.position.y) * 0.5f, + (left.position.z + right.position.z) * 0.5f}; + pose.orientation = {left.orientation.w, left.orientation.x, + left.orientation.y, left.orientation.z}; + pose.valid = true; +} + +// ---- Visibility Mask ---- + +void OpenXRContext::queryVisibilityMask() { + if (!visMaskAvailable_ || session_ == XR_NULL_HANDLE) return; + + PFN_xrGetVisibilityMaskKHR xrGetVisibilityMaskKHR = nullptr; + XrResult r = xrGetInstanceProcAddr(xrInstance_, "xrGetVisibilityMaskKHR", + reinterpret_cast(&xrGetVisibilityMaskKHR)); + if (XR_FAILED(r) || !xrGetVisibilityMaskKHR) { + xrCerr() << "Could not get xrGetVisibilityMaskKHR" << std::endl; + visMaskAvailable_ = false; + return; + } + + for (uint32_t eye = 0; eye < 2; eye++) { + XrVisibilityMaskKHR mask{XR_TYPE_VISIBILITY_MASK_KHR}; + // Query sizes first (HIDDEN_TRIANGLE_MESH gives the non-visible area) + r = xrGetVisibilityMaskKHR(session_, viewConfigType_, eye, + XR_VISIBILITY_MASK_TYPE_HIDDEN_TRIANGLE_MESH_KHR, &mask); + if (XR_FAILED(r) || mask.vertexCountOutput == 0) { + xrCout() << "No visibility mask for eye " << eye << std::endl; + continue; + } + + std::vector verts(mask.vertexCountOutput); + std::vector indices(mask.indexCountOutput); + mask.vertexCapacityInput = static_cast(verts.size()); + mask.vertices = verts.data(); + mask.indexCapacityInput = static_cast(indices.size()); + mask.indices = indices.data(); + + r = xrGetVisibilityMaskKHR(session_, viewConfigType_, eye, + XR_VISIBILITY_MASK_TYPE_HIDDEN_TRIANGLE_MESH_KHR, &mask); + if (XR_FAILED(r)) { + xrCerr() << "Failed to get visibility mask data for eye " << eye << std::endl; + continue; + } + + // Convert to glm::vec2 + visMaskVertices_[eye].resize(mask.vertexCountOutput); + for (uint32_t i = 0; i < mask.vertexCountOutput; i++) { + visMaskVertices_[eye][i] = {verts[i].x, verts[i].y}; + } + visMaskIndices_[eye] = std::move(indices); + + xrCout() << "Eye " << eye << " visibility mask: " + << mask.vertexCountOutput << " verts, " + << mask.indexCountOutput << " indices" << std::endl; + } +} + +// ---- Query ---- + +float OpenXRContext::floorHeight() const { + // In STAGE reference space, the Y coordinate of the head pose IS the height + // above the floor. Return the last known value from the renderer's VR system. + const auto &vr = Renderer::instance().vrSystem(); + return vr.headPose.valid ? vr.headPose.position.y : 0.0f; +} + +// ---- Lifecycle ---- + +void OpenXRContext::shutdown() { + destroySessionResources(); + if (xrInstance_ != XR_NULL_HANDLE) { + xrDestroyInstance(xrInstance_); + xrInstance_ = XR_NULL_HANDLE; + } + xrCout() << "OpenXR shutdown complete" << std::endl; +} + +OpenXRContext::~OpenXRContext() { + shutdown(); +} + +#endif // MCVR_ENABLE_OPENXR diff --git a/src/core/render/openxr_context.hpp b/src/core/render/openxr_context.hpp new file mode 100644 index 0000000..e36613a --- /dev/null +++ b/src/core/render/openxr_context.hpp @@ -0,0 +1,204 @@ +#pragma once + +#ifdef MCVR_ENABLE_OPENXR + +#include "core/render/vr_system.hpp" +#include "core/render/openxr_input.hpp" + +#include + +#include +#include + +#include +#include +#include + +// Manages the full OpenXR lifecycle: instance, session, swapchains, and per-frame operations. +// Initialization is split into two stages because OpenXR needs to provide required Vulkan +// extensions BEFORE VkInstance/VkDevice creation: +// Stage A preVulkanInit() — creates XrInstance, queries Vulkan extension requirements +// Stage B postVulkanInit() — creates XrSession, reference space, and per-eye swapchains +struct OpenXRContext { + // ---- Stage A: before Vulkan init ---- + + // Creates XrInstance and XrSystemId; queries required Vulkan instance/device extensions. + // Returns false if no HMD is connected or the runtime is unavailable. + bool preVulkanInit(); + + // Extension lists populated by preVulkanInit(), to be merged into Vulkan creation. + const std::vector &requiredInstanceExtensions() const { return requiredInstanceExts_; } + const std::vector &requiredDeviceExtensions() const { return requiredDeviceExts_; } + + // After VkInstance is created, let OpenXR verify/select the physical device. + // Returns the XR-preferred VkPhysicalDevice (may differ from app's choice). + VkPhysicalDevice getXRPhysicalDevice(VkInstance vkInstance) const; + + // ---- Stage B: after Vulkan init ---- + + // Creates XrSession, reference space, and per-eye swapchains. + bool postVulkanInit(VkInstance vkInstance, + VkPhysicalDevice vkPhysicalDevice, + VkDevice vkDevice, + uint32_t queueFamilyIndex, + uint32_t queueIndex); + + // ---- Per-frame operations ---- + + // Poll and process OpenXR events (session state transitions, etc.). + void pollEvents(); + + // --- PIPELINED FRAMING API --- + // Split frame operations into two phases for late-latch optimization: + + // Phase 1: Begin frame recording without waiting for compositor. + // Allows CPU to start recording commands while GPU executes previous frame. + // This is called early in acquireContext(). + void beginFrameRecording(); + + // Phase 2: Late-latch pose data just before submission. + // Calls xrWaitFrame + xrBeginFrame + xrLocateViews to get latest pose. + // Returns true if compositor wants us to render this frame. + // This is called in submitCommand() just before vkQueueSubmit. + bool latchPose(); + + // Acquire the current XR swapchain image for the given eye. + // Returns the VkImage to blit into. Must call releaseSwapchainImage() after blit. + VkImage acquireSwapchainImage(uint32_t eye); + + // Release the swapchain image after rendering/blitting. + void releaseSwapchainImage(uint32_t eye); + + // End the XR frame: constructs projection layers and calls xrEndFrame. + void endFrame(); + + // ---- Query ---- + + // Fill VREyeParams from the latest xrLocateViews result. + void getEyeParams(VREyeParams eyes[2]) const; + + // Fill VRHeadPose from the latest xrLocateViews result. + void getHeadPose(VRHeadPose &pose) const; + + // Per-eye swapchain dimensions (recommended by the runtime). + uint32_t swapchainWidth() const { return eyeSwapchains_[0].width; } + uint32_t swapchainHeight() const { return eyeSwapchains_[0].height; } + + bool isSessionRunning() const { return sessionRunning_; } + bool shouldRender() const { return xrFrameState_.shouldRender == XR_TRUE; } + XrSession session() const { return session_; } + + // Session control: keep runtime prepared but do not enter XR session + // until the game requests it (e.g. on world enter). + bool requestSessionStart(); + void requestSessionStop(); + bool sessionRequested() const { return sessionRequested_; } + + // Session state for Java-side pause/resume logic + XrSessionState sessionState() const { return sessionState_; } + + // HMD system name (e.g. "Oculus Quest 3", "Valve Index") + const std::string &systemName() const { return systemName_; } + + // Floor height: in STAGE reference space, headPose.position.y IS the height + // above the floor. This returns the last known head Y (or 0 if not valid). + float floorHeight() const; + + // Input subsystem (controllers, haptics, eye gaze) + OpenXRInput &input() { return input_; } + + // Predicted display period in nanoseconds (for performance stats) + int64_t predictedDisplayPeriodNs() const { return xrFrameState_.predictedDisplayPeriod; } + float lastWaitFrameMs() const { return lastWaitFrameMs_; } + float lastSwapchainWaitMs() const { return lastSwapchainWaitMs_; } + + // Visibility mask vertices for an eye (empty if extension not supported) + const std::vector &visibilityMaskVertices(uint32_t eye) const { return visMaskVertices_[eye]; } + const std::vector &visibilityMaskIndices(uint32_t eye) const { return visMaskIndices_[eye]; } + bool hasVisibilityMask() const { return visMaskAvailable_; } + + // ---- Lifecycle ---- + + void shutdown(); + ~OpenXRContext(); + +private: + // XR core handles + XrInstance xrInstance_ = XR_NULL_HANDLE; + XrSystemId systemId_ = XR_NULL_SYSTEM_ID; + XrSession session_ = XR_NULL_HANDLE; + XrSpace appSpace_ = XR_NULL_HANDLE; + + // Per-eye swapchain + struct EyeSwapchain { + XrSwapchain handle = XR_NULL_HANDLE; + std::vector images; + uint32_t width = 0; + uint32_t height = 0; + uint32_t acquiredIndex = 0; + bool imageAcquired = false; + }; + std::array eyeSwapchains_{}; + + // Per-frame state + XrFrameState xrFrameState_{XR_TYPE_FRAME_STATE}; + std::array views_{}; + std::array projViews_{}; + // Pipelined frame state + enum FrameState { + FRAME_IDLE, // No frame operations + FRAME_RECORDING, // Recording started, waiting for pose latch + FRAME_LATCHED, // Pose latched, ready for submit + FRAME_STARTED // XR frame started (legacy path) + }; + FrameState frameState_ = FRAME_IDLE; + bool viewsValid_ = false; + + // Session state + XrSessionState sessionState_ = XR_SESSION_STATE_UNKNOWN; + bool sessionRunning_ = false; + bool sessionRequested_ = false; + bool destroyPending_ = false; + + // System info (queried once in preVulkanInit) + std::string systemName_; + + // Extension requirements discovered in preVulkanInit + std::vector requiredInstanceExts_; + std::vector requiredDeviceExts_; + + // View configuration + XrViewConfigurationType viewConfigType_ = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + std::array configViews_{}; + + // Helpers + bool createSession(VkInstance vkInstance, VkPhysicalDevice vkPhysicalDevice, + VkDevice vkDevice, uint32_t queueFamilyIndex, uint32_t queueIndex); + bool createSwapchains(); + void destroySessionResources(); + void handleSessionStateChange(XrSessionState newState); + void queryVisibilityMask(); + + // Cached Vulkan handles for deferred session creation. + VkInstance vkInstance_ = VK_NULL_HANDLE; + VkPhysicalDevice vkPhysicalDevice_ = VK_NULL_HANDLE; + VkDevice vkDevice_ = VK_NULL_HANDLE; + uint32_t queueFamilyIndex_ = 0; + uint32_t queueIndex_ = 0; + bool vulkanReady_ = false; + + // Input subsystem + OpenXRInput input_; + bool inputInitialized_ = false; + + // Visibility mask (XR_KHR_visibility_mask) + bool visMaskAvailable_ = false; + std::array, 2> visMaskVertices_; + std::array, 2> visMaskIndices_; + + // Timing diagnostics for CPU wait analysis. + float lastWaitFrameMs_ = 0.0f; + float lastSwapchainWaitMs_ = 0.0f; +}; + +#endif // MCVR_ENABLE_OPENXR diff --git a/src/core/render/openxr_input.cpp b/src/core/render/openxr_input.cpp new file mode 100644 index 0000000..ed2528c --- /dev/null +++ b/src/core/render/openxr_input.cpp @@ -0,0 +1,410 @@ +#ifdef MCVR_ENABLE_OPENXR + +#include "core/render/openxr_input.hpp" + +#include +#include +#include +#include +#include + +static std::ostream &inputLog() { + static std::ofstream file("openxr_debug.log", std::ios::out | std::ios::app); + // Write to both cout and the log file + std::cout << "[OpenXR Input] "; + file << "[OpenXR Input] "; + // We return cout; the file gets a duplicate line via the caller flushing. + return std::cout; +} + +// ---- Helpers ---- + +XrPath OpenXRInput::toPath(XrInstance instance, const char *str) { + XrPath path = XR_NULL_PATH; + xrStringToPath(instance, str, &path); + return path; +} + +XrAction OpenXRInput::createAction(XrActionSet set, const char *name, const char *localizedName, + XrActionType type, uint32_t subactionPathCount, + const XrPath *subactionPaths) { + XrActionCreateInfo ci{XR_TYPE_ACTION_CREATE_INFO}; + std::strncpy(ci.actionName, name, XR_MAX_ACTION_NAME_SIZE); + std::strncpy(ci.localizedActionName, localizedName, XR_MAX_LOCALIZED_ACTION_NAME_SIZE); + ci.actionType = type; + ci.countSubactionPaths = subactionPathCount; + ci.subactionPaths = subactionPaths; + XrAction action = XR_NULL_HANDLE; + XrResult r = xrCreateAction(set, &ci, &action); + if (XR_FAILED(r)) { + inputLog() << "Failed to create action: " << name << std::endl; + } + return action; +} + +bool OpenXRInput::suggestBindings(XrInstance instance, const char *profilePath, + const std::vector &bindings) { + XrPath profile = toPath(instance, profilePath); + XrInteractionProfileSuggestedBinding suggestion{XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING}; + suggestion.interactionProfile = profile; + suggestion.countSuggestedBindings = static_cast(bindings.size()); + suggestion.suggestedBindings = bindings.data(); + XrResult r = xrSuggestInteractionProfileBindings(instance, &suggestion); + if (XR_FAILED(r)) { + inputLog() << "Failed to suggest bindings for " << profilePath << std::endl; + return false; + } + return true; +} + +// ---- createActions ---- + +bool OpenXRInput::createActions(XrInstance instance, XrSession session) { + // Check if eye gaze extension is available + uint32_t extCount = 0; + xrEnumerateInstanceExtensionProperties(nullptr, 0, &extCount, nullptr); + std::vector exts(extCount, {XR_TYPE_EXTENSION_PROPERTIES}); + xrEnumerateInstanceExtensionProperties(nullptr, extCount, &extCount, exts.data()); + for (auto &ext : exts) { + if (std::strcmp(ext.extensionName, "XR_EXT_eye_gaze_interaction") == 0) { + hasEyeGaze = true; + } + } + + // Hand sub-action paths + handPaths_[0] = toPath(instance, "/user/hand/left"); + handPaths_[1] = toPath(instance, "/user/hand/right"); + + // Create action set + XrActionSetCreateInfo asCI{XR_TYPE_ACTION_SET_CREATE_INFO}; + std::strncpy(asCI.actionSetName, "gameplay", XR_MAX_ACTION_SET_NAME_SIZE); + std::strncpy(asCI.localizedActionSetName, "Gameplay", XR_MAX_LOCALIZED_ACTION_SET_NAME_SIZE); + asCI.priority = 0; + XrResult r = xrCreateActionSet(instance, &asCI, &actionSet_); + if (XR_FAILED(r)) { inputLog() << "Failed to create action set" << std::endl; return false; } + + // Create actions + aimPoseAction_ = createAction(actionSet_, "aim_pose", "Aim Pose", XR_ACTION_TYPE_POSE_INPUT, 2, handPaths_); + gripPoseAction_ = createAction(actionSet_, "grip_pose", "Grip Pose", XR_ACTION_TYPE_POSE_INPUT, 2, handPaths_); + triggerAction_ = createAction(actionSet_, "trigger", "Trigger", XR_ACTION_TYPE_FLOAT_INPUT, 2, handPaths_); + gripAction_ = createAction(actionSet_, "grip", "Grip", XR_ACTION_TYPE_FLOAT_INPUT, 2, handPaths_); + thumbstickAction_ = createAction(actionSet_, "thumbstick", "Thumbstick", XR_ACTION_TYPE_VECTOR2F_INPUT,2, handPaths_); + primaryAction_ = createAction(actionSet_, "primary_button", "Primary Button", XR_ACTION_TYPE_BOOLEAN_INPUT, 2, handPaths_); + secondaryAction_ = createAction(actionSet_, "secondary_button", "Secondary Button", XR_ACTION_TYPE_BOOLEAN_INPUT, 2, handPaths_); + thumbstickClickAction_= createAction(actionSet_, "thumbstick_click", "Thumbstick Click", XR_ACTION_TYPE_BOOLEAN_INPUT, 2, handPaths_); + menuAction_ = createAction(actionSet_, "menu", "Menu", XR_ACTION_TYPE_BOOLEAN_INPUT, 2, handPaths_); + hapticAction_ = createAction(actionSet_, "haptic", "Haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT, 2, handPaths_); + + // Create action spaces for pose actions + for (uint32_t hand = 0; hand < 2; hand++) { + XrActionSpaceCreateInfo spaceCI{XR_TYPE_ACTION_SPACE_CREATE_INFO}; + spaceCI.poseInActionSpace = {{0, 0, 0, 1}, {0, 0, 0}}; + spaceCI.subactionPath = handPaths_[hand]; + + spaceCI.action = aimPoseAction_; + xrCreateActionSpace(session, &spaceCI, &aimSpaces_[hand]); + + spaceCI.action = gripPoseAction_; + xrCreateActionSpace(session, &spaceCI, &gripSpaces_[hand]); + } + + // ---- Suggest interaction profile bindings ---- + // PLACEHOLDER_BINDINGS_START + auto p = [&](const char *s) { return toPath(instance, s); }; + + // Oculus Touch (Quest 2/3/Pro) + { + std::vector bindings = { + {aimPoseAction_, p("/user/hand/left/input/aim/pose")}, + {aimPoseAction_, p("/user/hand/right/input/aim/pose")}, + {gripPoseAction_, p("/user/hand/left/input/grip/pose")}, + {gripPoseAction_, p("/user/hand/right/input/grip/pose")}, + {triggerAction_, p("/user/hand/left/input/trigger/value")}, + {triggerAction_, p("/user/hand/right/input/trigger/value")}, + {gripAction_, p("/user/hand/left/input/squeeze/value")}, + {gripAction_, p("/user/hand/right/input/squeeze/value")}, + {thumbstickAction_, p("/user/hand/left/input/thumbstick")}, + {thumbstickAction_, p("/user/hand/right/input/thumbstick")}, + {primaryAction_, p("/user/hand/left/input/x/click")}, + {primaryAction_, p("/user/hand/right/input/a/click")}, + {secondaryAction_, p("/user/hand/left/input/y/click")}, + {secondaryAction_, p("/user/hand/right/input/b/click")}, + {thumbstickClickAction_, p("/user/hand/left/input/thumbstick/click")}, + {thumbstickClickAction_, p("/user/hand/right/input/thumbstick/click")}, + {menuAction_, p("/user/hand/left/input/menu/click")}, + {hapticAction_, p("/user/hand/left/output/haptic")}, + {hapticAction_, p("/user/hand/right/output/haptic")}, + }; + suggestBindings(instance, "/interaction_profiles/oculus/touch_controller", bindings); + } + // PLACEHOLDER_BINDINGS_MID + + // Valve Index Controller + { + std::vector bindings = { + {aimPoseAction_, p("/user/hand/left/input/aim/pose")}, + {aimPoseAction_, p("/user/hand/right/input/aim/pose")}, + {gripPoseAction_, p("/user/hand/left/input/grip/pose")}, + {gripPoseAction_, p("/user/hand/right/input/grip/pose")}, + {triggerAction_, p("/user/hand/left/input/trigger/value")}, + {triggerAction_, p("/user/hand/right/input/trigger/value")}, + {gripAction_, p("/user/hand/left/input/squeeze/value")}, + {gripAction_, p("/user/hand/right/input/squeeze/value")}, + {thumbstickAction_, p("/user/hand/left/input/thumbstick")}, + {thumbstickAction_, p("/user/hand/right/input/thumbstick")}, + {primaryAction_, p("/user/hand/left/input/a/click")}, + {primaryAction_, p("/user/hand/right/input/a/click")}, + {secondaryAction_, p("/user/hand/left/input/b/click")}, + {secondaryAction_, p("/user/hand/right/input/b/click")}, + {thumbstickClickAction_, p("/user/hand/left/input/thumbstick/click")}, + {thumbstickClickAction_, p("/user/hand/right/input/thumbstick/click")}, + {hapticAction_, p("/user/hand/left/output/haptic")}, + {hapticAction_, p("/user/hand/right/output/haptic")}, + }; + suggestBindings(instance, "/interaction_profiles/valve/index_controller", bindings); + } + + // HTC Vive Controller + { + std::vector bindings = { + {aimPoseAction_, p("/user/hand/left/input/aim/pose")}, + {aimPoseAction_, p("/user/hand/right/input/aim/pose")}, + {gripPoseAction_, p("/user/hand/left/input/grip/pose")}, + {gripPoseAction_, p("/user/hand/right/input/grip/pose")}, + {triggerAction_, p("/user/hand/left/input/trigger/value")}, + {triggerAction_, p("/user/hand/right/input/trigger/value")}, + {gripAction_, p("/user/hand/left/input/squeeze/click")}, + {gripAction_, p("/user/hand/right/input/squeeze/click")}, + {thumbstickAction_, p("/user/hand/left/input/trackpad")}, + {thumbstickAction_, p("/user/hand/right/input/trackpad")}, + {thumbstickClickAction_, p("/user/hand/left/input/trackpad/click")}, + {thumbstickClickAction_, p("/user/hand/right/input/trackpad/click")}, + {menuAction_, p("/user/hand/left/input/menu/click")}, + {menuAction_, p("/user/hand/right/input/menu/click")}, + {hapticAction_, p("/user/hand/left/output/haptic")}, + {hapticAction_, p("/user/hand/right/output/haptic")}, + }; + suggestBindings(instance, "/interaction_profiles/htc/vive_controller", bindings); + } + + // Khronos Simple Controller (minimal fallback) + { + std::vector bindings = { + {aimPoseAction_, p("/user/hand/left/input/aim/pose")}, + {aimPoseAction_, p("/user/hand/right/input/aim/pose")}, + {gripPoseAction_, p("/user/hand/left/input/grip/pose")}, + {gripPoseAction_, p("/user/hand/right/input/grip/pose")}, + {triggerAction_, p("/user/hand/left/input/select/click")}, + {triggerAction_, p("/user/hand/right/input/select/click")}, + {menuAction_, p("/user/hand/left/input/menu/click")}, + {menuAction_, p("/user/hand/right/input/menu/click")}, + {hapticAction_, p("/user/hand/left/output/haptic")}, + {hapticAction_, p("/user/hand/right/output/haptic")}, + }; + suggestBindings(instance, "/interaction_profiles/khr/simple_controller", bindings); + } + + inputLog() << "Actions and bindings created" << std::endl; + return true; +} + +// ---- attachToSession ---- + +bool OpenXRInput::attachToSession(XrSession session) { + XrSessionActionSetsAttachInfo attachInfo{XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO}; + attachInfo.countActionSets = 1; + attachInfo.actionSets = &actionSet_; + XrResult r = xrAttachSessionActionSets(session, &attachInfo); + if (XR_FAILED(r)) { + inputLog() << "Failed to attach action sets to session" << std::endl; + return false; + } + inputLog() << "Action sets attached to session" << std::endl; + return true; +} + +// ---- syncAndUpdate ---- + +void OpenXRInput::syncAndUpdate(XrSession session, XrSpace appSpace, XrTime predictedTime) { + // Sync actions + XrActiveActionSet activeSet{}; + activeSet.actionSet = actionSet_; + activeSet.subactionPath = XR_NULL_PATH; + XrActionsSyncInfo syncInfo{XR_TYPE_ACTIONS_SYNC_INFO}; + syncInfo.countActiveActionSets = 1; + syncInfo.activeActionSets = &activeSet; + XrResult r = xrSyncActions(session, &syncInfo); + if (XR_FAILED(r)) return; + + // Update each hand + for (uint32_t hand = 0; hand < 2; hand++) { + auto &ctrl = controllers[hand]; + ctrl = VRControllerState{}; + + // Aim pose + XrSpaceLocation loc{XR_TYPE_SPACE_LOCATION}; + XrSpaceVelocity vel{XR_TYPE_SPACE_VELOCITY}; + loc.next = &vel; + r = xrLocateSpace(aimSpaces_[hand], appSpace, predictedTime, &loc); + if (XR_SUCCEEDED(r) && (loc.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) && + (loc.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT)) { + ctrl.valid = true; + ctrl.position = {loc.pose.position.x, loc.pose.position.y, loc.pose.position.z}; + ctrl.orientation = {loc.pose.orientation.w, loc.pose.orientation.x, + loc.pose.orientation.y, loc.pose.orientation.z}; + if (vel.velocityFlags & XR_SPACE_VELOCITY_LINEAR_VALID_BIT) + ctrl.linearVelocity = {vel.linearVelocity.x, vel.linearVelocity.y, vel.linearVelocity.z}; + if (vel.velocityFlags & XR_SPACE_VELOCITY_ANGULAR_VALID_BIT) + ctrl.angularVelocity = {vel.angularVelocity.x, vel.angularVelocity.y, vel.angularVelocity.z}; + } + // PLACEHOLDER_SYNC_CONTINUE + + // Trigger (float) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = triggerAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateFloat state{XR_TYPE_ACTION_STATE_FLOAT}; + if (XR_SUCCEEDED(xrGetActionStateFloat(session, &gi, &state)) && state.isActive) { + ctrl.triggerValue = state.currentState; + ctrl.triggerPressed = state.currentState > 0.8f; + } + } + + // Grip (float) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = gripAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateFloat state{XR_TYPE_ACTION_STATE_FLOAT}; + if (XR_SUCCEEDED(xrGetActionStateFloat(session, &gi, &state)) && state.isActive) { + ctrl.gripValue = state.currentState; + ctrl.gripPressed = state.currentState > 0.8f; + } + } + + // Thumbstick (vec2) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = thumbstickAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateVector2f state{XR_TYPE_ACTION_STATE_VECTOR2F}; + if (XR_SUCCEEDED(xrGetActionStateVector2f(session, &gi, &state)) && state.isActive) { + ctrl.thumbstick = {state.currentState.x, state.currentState.y}; + } + } + + // Primary button (boolean) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = primaryAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_SUCCEEDED(xrGetActionStateBoolean(session, &gi, &state)) && state.isActive) { + ctrl.primaryButton = state.currentState == XR_TRUE; + } + } + + // Secondary button (boolean) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = secondaryAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_SUCCEEDED(xrGetActionStateBoolean(session, &gi, &state)) && state.isActive) { + ctrl.secondaryButton = state.currentState == XR_TRUE; + } + } + + // Thumbstick click (boolean) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = thumbstickClickAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_SUCCEEDED(xrGetActionStateBoolean(session, &gi, &state)) && state.isActive) { + ctrl.thumbstickClick = state.currentState == XR_TRUE; + } + } + + // Menu button (boolean) + { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = menuAction_; + gi.subactionPath = handPaths_[hand]; + XrActionStateBoolean state{XR_TYPE_ACTION_STATE_BOOLEAN}; + if (XR_SUCCEEDED(xrGetActionStateBoolean(session, &gi, &state)) && state.isActive) { + ctrl.menuButton = state.currentState == XR_TRUE; + } + } + } + + // Eye gaze (if available — requires extension enabled at instance level) + gazeValid = false; + gazePoint = {0.5f, 0.5f}; + if (hasEyeGaze && gazeAction_ != XR_NULL_HANDLE && gazeSpace_ != XR_NULL_HANDLE) { + XrActionStateGetInfo gi{XR_TYPE_ACTION_STATE_GET_INFO}; + gi.action = gazeAction_; + XrActionStatePose state{XR_TYPE_ACTION_STATE_POSE}; + if (XR_SUCCEEDED(xrGetActionStatePose(session, &gi, &state)) && state.isActive) { + XrSpaceLocation loc{XR_TYPE_SPACE_LOCATION}; + if (XR_SUCCEEDED(xrLocateSpace(gazeSpace_, appSpace, predictedTime, &loc)) && + (loc.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT)) { + // Extract gaze direction from orientation, project onto screen plane. + // gazeSpace orientation's -Z is the gaze direction. + float qx = loc.pose.orientation.x, qy = loc.pose.orientation.y; + float qz = loc.pose.orientation.z, qw = loc.pose.orientation.w; + // Forward vector from quaternion + float fx = 2.0f * (qx * qz + qw * qy); + float fy = 2.0f * (qy * qz - qw * qx); + float fz = 1.0f - 2.0f * (qx * qx + qy * qy); + // Project: gaze point is where the gaze ray hits Z=-1 plane + if (std::abs(fz) > 1e-4f) { + gazePoint.x = 0.5f + 0.5f * (fx / (-fz)); + gazePoint.y = 0.5f - 0.5f * (fy / (-fz)); + gazePoint.x = std::clamp(gazePoint.x, 0.0f, 1.0f); + gazePoint.y = std::clamp(gazePoint.y, 0.0f, 1.0f); + gazeValid = true; + } + } + } + } +} + +// ---- Haptics ---- + +void OpenXRInput::vibrate(XrSession session, uint32_t hand, float amplitude, + int64_t durationNs, float frequency) { + if (hand > 1 || hapticAction_ == XR_NULL_HANDLE) return; + XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO}; + info.action = hapticAction_; + info.subactionPath = handPaths_[hand]; + XrHapticVibration vibration{XR_TYPE_HAPTIC_VIBRATION}; + vibration.amplitude = std::clamp(amplitude, 0.0f, 1.0f); + vibration.duration = durationNs; + vibration.frequency = frequency; + xrApplyHapticFeedback(session, &info, reinterpret_cast(&vibration)); +} + +void OpenXRInput::stopVibration(XrSession session, uint32_t hand) { + if (hand > 1 || hapticAction_ == XR_NULL_HANDLE) return; + XrHapticActionInfo info{XR_TYPE_HAPTIC_ACTION_INFO}; + info.action = hapticAction_; + info.subactionPath = handPaths_[hand]; + xrStopHapticFeedback(session, &info); +} + +// ---- Shutdown ---- + +void OpenXRInput::shutdown() { + for (uint32_t h = 0; h < 2; h++) { + if (aimSpaces_[h] != XR_NULL_HANDLE) { xrDestroySpace(aimSpaces_[h]); aimSpaces_[h] = XR_NULL_HANDLE; } + if (gripSpaces_[h] != XR_NULL_HANDLE) { xrDestroySpace(gripSpaces_[h]); gripSpaces_[h] = XR_NULL_HANDLE; } + } + if (gazeSpace_ != XR_NULL_HANDLE) { xrDestroySpace(gazeSpace_); gazeSpace_ = XR_NULL_HANDLE; } + // Actions and action sets are destroyed when the XrInstance is destroyed + actionSet_ = XR_NULL_HANDLE; +} + +#endif // MCVR_ENABLE_OPENXR + diff --git a/src/core/render/openxr_input.hpp b/src/core/render/openxr_input.hpp new file mode 100644 index 0000000..989163e --- /dev/null +++ b/src/core/render/openxr_input.hpp @@ -0,0 +1,84 @@ +#pragma once + +#ifdef MCVR_ENABLE_OPENXR + +#include + +#include +#include + +#include +#include +#include + +#include "core/render/vr_system.hpp" + +// Manages OpenXR action sets, action bindings, and per-frame input sync. +// Also handles haptic feedback and eye gaze tracking. +struct OpenXRInput { + // Initialise action sets, actions, and suggest interaction profile bindings. + // Must be called BEFORE xrAttachSessionActionSets. + bool createActions(XrInstance instance, XrSession session); + + // Attach action sets to the session. Must be called once after createActions. + bool attachToSession(XrSession session); + + // Sync actions and locate hand spaces. Call once per frame after xrSyncActions. + void syncAndUpdate(XrSession session, XrSpace appSpace, XrTime predictedTime); + + // Trigger haptic vibration on a hand (0=left, 1=right). + // duration in nanoseconds, amplitude in [0,1], frequency in Hz (0 = runtime default). + void vibrate(XrSession session, uint32_t hand, float amplitude, int64_t durationNs, float frequency); + void stopVibration(XrSession session, uint32_t hand); + + // Eye gaze point in normalised per-eye coordinates (0.5, 0.5 = centre). + // Falls back to (0.5, 0.5) when eye tracking is unavailable. + glm::vec2 gazePoint{0.5f, 0.5f}; + bool gazeValid = false; + + // Latest controller states + std::array controllers{}; + + void shutdown(); + + // Extension support flags (set during createActions based on instance extensions) + bool hasEyeGaze = false; + +private: + // Action set + XrActionSet actionSet_ = XR_NULL_HANDLE; + + // Hand paths (/user/hand/left, /user/hand/right) + XrPath handPaths_[2]{}; + + // Per-hand actions + XrAction aimPoseAction_ = XR_NULL_HANDLE; + XrAction gripPoseAction_ = XR_NULL_HANDLE; + XrAction triggerAction_ = XR_NULL_HANDLE; + XrAction gripAction_ = XR_NULL_HANDLE; + XrAction thumbstickAction_ = XR_NULL_HANDLE; + XrAction primaryAction_ = XR_NULL_HANDLE; + XrAction secondaryAction_ = XR_NULL_HANDLE; + XrAction thumbstickClickAction_ = XR_NULL_HANDLE; + XrAction menuAction_ = XR_NULL_HANDLE; + XrAction hapticAction_ = XR_NULL_HANDLE; + + // Eye gaze action + XrAction gazeAction_ = XR_NULL_HANDLE; + + // Per-hand action spaces (for locating poses) + XrSpace aimSpaces_[2]{}; + XrSpace gripSpaces_[2]{}; + + // Eye gaze space + XrSpace gazeSpace_ = XR_NULL_HANDLE; + + // Helpers + XrAction createAction(XrActionSet set, const char *name, const char *localizedName, + XrActionType type, uint32_t subactionPathCount, const XrPath *subactionPaths); + bool suggestBindings(XrInstance instance, const char *profilePath, + const std::vector &bindings); + XrPath toPath(XrInstance instance, const char *str); +}; + +#endif // MCVR_ENABLE_OPENXR diff --git a/src/core/render/pipeline.cpp b/src/core/render/pipeline.cpp index 652d57f..8a9211e 100644 --- a/src/core/render/pipeline.cpp +++ b/src/core/render/pipeline.cpp @@ -13,6 +13,7 @@ #include "core/render/modules/world/temporal_accumulation/temporal_accumulation_module.hpp" #include "core/render/modules/world/tone_mapping/tone_mapping_module.hpp" +#include #include #include #include @@ -22,6 +23,8 @@ WorldPipelineBlueprint::WorldPipelineBlueprint(WorldPipelineBuildParams *params) auto framework = Renderer::instance().framework(); auto pipeline = framework->pipeline(); + eyeCount_ = static_cast(params->eyeCount > 0 ? params->eyeCount : 1); + for (int i = 0; i < params->moduleCount; i++) { std::string moduleName = params->moduleNames[i]; moduleNames_.emplace_back(moduleName); @@ -73,13 +76,65 @@ void WorldPipeline::init(std::shared_ptr framework, std::shared_ptr

worldPipelineBlueprint(); uint32_t frameNum = framework->swapchain()->imageCount(); + bool useStereo = false; +#ifdef MCVR_ENABLE_OPENXR + if (auto *xr = framework->xrContext(); xr != nullptr) { + useStereo = Renderer::options.vrEnabled && xr->isSessionRunning(); + } +#endif + eyeCount_ = useStereo ? 2u : 1u; + + VkExtent2D mirrorExtent = framework->swapchain()->vkExtent(); + uint32_t targetWidth = mirrorExtent.width; + uint32_t targetHeight = mirrorExtent.height; + + // Sync VRSystem state from static options + { + auto &vr = Renderer::instance().vrSystem(); + auto &opts = Renderer::options; + vr.enabled = useStereo; + vr.config.renderScale = opts.vrRenderScale; + vr.config.ipd = opts.vrIPD; + vr.config.worldScale = opts.vrWorldScale; + vr.eyeCount = eyeCount_; + + if (vr.enabled && eyeCount_ > 1) { + bool xrResolutionReady = false; +#ifdef MCVR_ENABLE_OPENXR + if (auto *xr = framework->xrContext(); xr != nullptr) { + const uint32_t xrW = xr->swapchainWidth(); + const uint32_t xrH = xr->swapchainHeight(); + if (xrW > 0 && xrH > 0) { + for (uint32_t eye = 0; eye < 2; eye++) { + vr.eyes[eye].recommendedWidth = xrW; + vr.eyes[eye].recommendedHeight = xrH; + } + xrResolutionReady = true; + } + } +#endif + if (!xrResolutionReady) { + // No XR runtime/session yet: keep legacy simulation fallback for startup safety. + vr.updateSimulation(mirrorExtent.width, mirrorExtent.height, glm::radians(70.0f)); + } + + const uint32_t baseWidth = std::max(vr.eyes[0].recommendedWidth, vr.eyes[1].recommendedWidth); + const uint32_t baseHeight = std::max(vr.eyes[0].recommendedHeight, vr.eyes[1].recommendedHeight); + const float scale = std::max(0.1f, vr.config.renderScale); + targetWidth = std::max(1u, static_cast(static_cast(baseWidth) * scale)); + targetHeight = std::max(1u, static_cast(static_cast(baseHeight) * scale)); + } else if (vr.enabled) { + vr.updateSimulation(mirrorExtent.width, mirrorExtent.height, glm::radians(70.0f)); + } + } + worldModules_.resize(blueprint->moduleNames_.size()); sharedImages_.resize(frameNum, std::vector>(blueprint->imageFormats_.size(), nullptr)); contexts_.resize(frameNum); // Determine initial render resolution from upscaler quality (if present) - VkExtent2D extent = framework->swapchain()->vkExtent(); + VkExtent2D extent{targetWidth, targetHeight}; uint32_t renderWidth = extent.width; uint32_t renderHeight = extent.height; size_t upscalerIndex = std::numeric_limits::max(); @@ -104,12 +159,13 @@ void WorldPipeline::init(std::shared_ptr framework, std::shared_ptr

device(), framework->vma(), false, extent.width, extent.height, 1, blueprint->imageFormats_[0], + framework->device(), framework->vma(), false, extent.width, extent.height, eyeCount_, blueprint->imageFormats_[0], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT #ifdef USE_AMD | VK_IMAGE_USAGE_TRANSFER_SRC_BIT #endif ); + if (eyeCount_ > 1) sharedImages_[frameIndex][0]->createPerLayerViews(); } // Pre-create outputs for modules before upscaler at render resolution @@ -124,19 +180,21 @@ void WorldPipeline::init(std::shared_ptr framework, std::shared_ptr

device(), framework->vma(), false, renderWidth, renderHeight, 1, + framework->device(), framework->vma(), false, renderWidth, renderHeight, eyeCount_, blueprint->imageFormats_[idx], VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT #ifdef USE_AMD | VK_IMAGE_USAGE_TRANSFER_SRC_BIT #endif ); + if (eyeCount_ > 1) sharedImages_[frameIndex][idx]->createPerLayerViews(); } } } for (int i = blueprint->moduleNames_.size() - 1; i >= 0; i--) { worldModules_[i] = Pipeline::worldModuleConstructors[blueprint->moduleNames_[i]](framework, shared_from_this()); + worldModules_[i]->setEyeCount(eyeCount_); auto &moduleInputIndices = blueprint->modulesInputIndices_[i]; auto &moduleOutputIndices = blueprint->modulesOutputIndices_[i]; @@ -202,6 +260,7 @@ WorldPipelineContext::WorldPipelineContext(std::shared_ptr fra std::shared_ptr worldPipeline) : frameworkContext(frameworkContext), worldPipeline(worldPipeline), + eyeCount_(worldPipeline->eyeCount()), outputImage(worldPipeline->sharedImages_[frameworkContext->frameIndex][0]) { auto &worldModules = worldPipeline->worldModules(); for (int i = 0; i < worldModules.size(); i++) { @@ -250,7 +309,24 @@ void WorldPipelineContext::render() { outputImage->imageLayout() = targetLayout; } - for (int i = 0; i < worldModuleContexts.size(); i++) { worldModuleContexts[i]->render(); } + for (int i = 0; i < worldModuleContexts.size(); i++) { + if (eyeCount_ > 1) { + auto mode = worldModuleContexts[i]->stereoMode(); + switch (mode) { + case StereoMode::SingleInstance3DDispatch: + worldModuleContexts[i]->render3D(eyeCount_); + break; + case StereoMode::SingleInstanceMultiDispatch: + case StereoMode::DualInstance: + for (uint32_t eye = 0; eye < eyeCount_; eye++) { + worldModuleContexts[i]->renderEye(eye); + } + break; + } + } else { + worldModuleContexts[i]->render(); + } + } worldCommandBuffer->barriersBufferImage( {}, {{ @@ -497,26 +573,40 @@ void PipelineContext::fuseWorld() { uiModuleContext->overlayDrawColorImage->imageLayout() = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - // TODO: add to command buffer - VkImageBlit imageBlit{}; - imageBlit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageBlit.srcSubresource.mipLevel = 0; - imageBlit.srcSubresource.baseArrayLayer = 0; - imageBlit.srcSubresource.layerCount = 1; - imageBlit.srcOffsets[0] = {0, 0, 0}; - imageBlit.srcOffsets[1] = {static_cast(worldPipelineContext->outputImage->width()), - static_cast(worldPipelineContext->outputImage->height()), 1}; - imageBlit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - imageBlit.dstSubresource.mipLevel = 0; - imageBlit.dstSubresource.baseArrayLayer = 0; - imageBlit.dstSubresource.layerCount = 1; - imageBlit.dstOffsets[0] = {0, 0, 0}; - imageBlit.dstOffsets[1] = {static_cast(uiModuleContext->overlayDrawColorImage->width()), - static_cast(uiModuleContext->overlayDrawColorImage->height()), 1}; - - vkCmdBlitImage(overlayCommandBuffer->vkCommandBuffer(), worldPipelineContext->outputImage->vkImage(), - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, uiModuleContext->overlayDrawColorImage->vkImage(), - uiModuleContext->overlayDrawColorImage->imageLayout(), 1, &imageBlit, VK_FILTER_LINEAR); + auto srcImage = worldPipelineContext->outputImage; + auto dstImage = uiModuleContext->overlayDrawColorImage; + int srcW = static_cast(srcImage->width()); + int srcH = static_cast(srcImage->height()); + int dstW = static_cast(dstImage->width()); + int dstH = static_cast(dstImage->height()); + + if (worldPipelineContext->eyeCount_ > 1) { + // SBS: layer 0 → left half, layer 1 → right half + int halfW = dstW / 2; + VkImageBlit blits[2]{}; + for (uint32_t eye = 0; eye < 2; eye++) { + blits[eye].srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eye, 1}; + blits[eye].srcOffsets[0] = {0, 0, 0}; + blits[eye].srcOffsets[1] = {srcW, srcH, 1}; + blits[eye].dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blits[eye].dstOffsets[0] = {static_cast(eye * halfW), 0, 0}; + blits[eye].dstOffsets[1] = {static_cast((eye + 1) * halfW), dstH, 1}; + } + vkCmdBlitImage(overlayCommandBuffer->vkCommandBuffer(), srcImage->vkImage(), + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dstImage->vkImage(), + dstImage->imageLayout(), 2, blits, VK_FILTER_LINEAR); + } else { + VkImageBlit imageBlit{}; + imageBlit.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + imageBlit.srcOffsets[0] = {0, 0, 0}; + imageBlit.srcOffsets[1] = {srcW, srcH, 1}; + imageBlit.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + imageBlit.dstOffsets[0] = {0, 0, 0}; + imageBlit.dstOffsets[1] = {dstW, dstH, 1}; + vkCmdBlitImage(overlayCommandBuffer->vkCommandBuffer(), srcImage->vkImage(), + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dstImage->vkImage(), + dstImage->imageLayout(), 1, &imageBlit, VK_FILTER_LINEAR); + } overlayCommandBuffer->barriersBufferImage( {}, { diff --git a/src/core/render/pipeline.hpp b/src/core/render/pipeline.hpp index 1e8464c..bca9faa 100644 --- a/src/core/render/pipeline.hpp +++ b/src/core/render/pipeline.hpp @@ -10,7 +10,7 @@ struct WorldPipelineBuildParams { int moduleCount; - int padding; + int eyeCount; char **moduleNames; int *imageFormats; int **inputIndices; @@ -39,6 +39,7 @@ class WorldPipelineBlueprint : public SharedObject { WorldPipelineBlueprint(WorldPipelineBuildParams *params); private: + uint32_t eyeCount_ = 1; std::vector moduleNames_; std::vector> modulesInputIndices_; std::vector> modulesOutputIndices_; @@ -58,12 +59,14 @@ class WorldPipeline : public SharedObject { std::vector> &worldModules(); std::vector> &contexts(); + uint32_t eyeCount() const { return eyeCount_; } void bindTexture(std::shared_ptr sampler, std::shared_ptr image, int index); private: void dumpSharedImages(const char *label) const; + uint32_t eyeCount_ = 1; std::vector> worldModules_; std::vector>> sharedImages_; @@ -74,6 +77,7 @@ struct WorldPipelineContext : public SharedObject { std::weak_ptr frameworkContext; std::weak_ptr worldPipeline; + uint32_t eyeCount_ = 1; std::shared_ptr outputImage; std::vector> worldModuleContexts; diff --git a/src/core/render/render_framework.cpp b/src/core/render/render_framework.cpp index e01b6d4..b004fbe 100644 --- a/src/core/render/render_framework.cpp +++ b/src/core/render/render_framework.cpp @@ -11,7 +11,9 @@ #include "core/render/world.hpp" #include +#include #include +#include std::ostream &renderFrameworkCout() { return std::cout << "[Render Framework] "; @@ -21,6 +23,14 @@ std::ostream &renderFrameworkCerr() { return std::cerr << "[Render Framework] "; } +#ifdef MCVR_ENABLE_OPENXR +static void xrInitLog(const std::string &msg) { + std::cout << "[Render Framework] " << msg << std::endl; + std::ofstream file("openxr_debug.log", std::ios::out | std::ios::app); + file << "[Render Framework] " << msg << std::endl; +} +#endif + FrameworkContext::FrameworkContext(std::shared_ptr framework, uint32_t frameIndex) : framework(framework), frameIndex(frameIndex), @@ -35,9 +45,9 @@ FrameworkContext::FrameworkContext(std::shared_ptr framework, uint32_ commandProcessedSemaphore(framework->commandProcessedSemaphores_[frameIndex]), commandFinishedFence(framework->commandFinishedFences_[frameIndex]), uploadCommandBuffer(framework->uploadCommandBuffers_[frameIndex]), - overlayCommandBuffer(framework->overlayCommandBuffers_[frameIndex]), worldCommandBuffer(framework->worldCommandBuffers_[frameIndex]), - fuseCommandBuffer(framework->fuseCommandBuffers_[frameIndex]) {} + overlayCommandBuffer(framework->overlayCommandBuffers_[frameIndex]), + fuseCommandBuffer(framework->overlayCommandBuffers_[frameIndex]) {} FrameworkContext::~FrameworkContext() { #ifdef DEBUG @@ -147,10 +157,79 @@ void FrameworkContext::fuseFinal() { Framework::Framework() {} void Framework::init(GLFWwindow *window) { +#ifdef MCVR_ENABLE_OPENXR + // Stage A: pre-Vulkan OpenXR init — queries required Vulkan extensions + if (Renderer::options.vrEnabled) { + xrInitLog("VR enabled, attempting OpenXR init..."); + try { + xrContext_ = std::make_unique(); + if (xrContext_->preVulkanInit()) { + // Inject OpenXR-required extensions into Vulkan creation + vk::Instance::extraExtensions = xrContext_->requiredInstanceExtensions(); + vk::Device::extraExtensions = xrContext_->requiredDeviceExtensions(); + } else { + xrInitLog("ERROR: OpenXR pre-init failed, falling back to non-VR"); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } + } catch (const std::exception &e) { + xrInitLog(std::string("ERROR: OpenXR pre-init exception: ") + e.what()); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } catch (...) { + xrInitLog("ERROR: OpenXR pre-init unknown exception"); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } + } +#endif + instance_ = vk::Instance::create(); window_ = vk::Window::create(instance_, window); + +#ifdef MCVR_ENABLE_OPENXR + // Let OpenXR choose the physical device if available + if (xrContext_) { + try { + VkPhysicalDevice xrDev = xrContext_->getXRPhysicalDevice(instance_->vkInstance()); + if (xrDev != VK_NULL_HANDLE) { vk::PhysicalDevice::overrideDevice = xrDev; } + } catch (...) { + xrInitLog("ERROR: xrGetVulkanGraphicsDeviceKHR exception, ignoring"); + } + } +#endif + physicalDevice_ = vk::PhysicalDevice::create(instance_, window_); device_ = vk::Device::create(instance_, window_, physicalDevice_); + +#ifdef MCVR_ENABLE_OPENXR + // Stage B: post-Vulkan OpenXR init — creates session, swapchains + if (xrContext_) { + try { + if (!xrContext_->postVulkanInit(instance_->vkInstance(), physicalDevice_->vkPhysicalDevice(), + device_->vkDevice(), physicalDevice_->mainQueueIndex(), 0)) { + xrInitLog("ERROR: OpenXR post-init failed, falling back to non-VR"); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } else { + xrInitLog("OpenXR runtime prepared; session will start on explicit request"); + } + } catch (const std::exception &e) { + xrInitLog(std::string("ERROR: OpenXR post-init exception: ") + e.what()); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } catch (...) { + xrInitLog("ERROR: OpenXR post-init unknown exception"); + xrContext_.reset(); + Renderer::options.vrEnabled = false; + } + } + // Clear statics after use + vk::Instance::extraExtensions.clear(); + vk::Device::extraExtensions.clear(); + vk::PhysicalDevice::overrideDevice = VK_NULL_HANDLE; +#endif + vma_ = vk::VMA::create(instance_, physicalDevice_, device_); swapchain_ = vk::Swapchain::create(physicalDevice_, device_, window_); mainCommandPool_ = vk::CommandPool::create(physicalDevice_, device_); @@ -164,7 +243,6 @@ void Framework::init(GLFWwindow *window) { uploadCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); overlayCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); worldCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); - fuseCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); } worldAsyncCommandBuffer_ = vk::CommandBuffer::create(device_, asyncCommandPool_); @@ -175,6 +253,19 @@ void Framework::init(GLFWwindow *window) { for (int i = 0; i < imageCount; i++) { contexts_.push_back(FrameworkContext::create(shared_from_this(), i)); } pipeline_ = Pipeline::create(shared_from_this()); + +#ifdef MCVR_ENABLE_OPENXR + // Create GPU timestamp query pool for frame timing (2 queries per frame-in-flight) + if (xrContext_) { + timestampPeriodNs_ = physicalDevice_->properties().limits.timestampPeriod; + VkQueryPoolCreateInfo qpci{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + qpci.queryType = VK_QUERY_TYPE_TIMESTAMP; + qpci.queryCount = imageCount * 2; + vkCreateQueryPool(device_->vkDevice(), &qpci, nullptr, &gpuTimestampPool_); + // Reset all queries to "unavailable" + vkResetQueryPool(device_->vkDevice(), gpuTimestampPool_, 0, imageCount * 2); + } +#endif } Framework::~Framework() { @@ -186,15 +277,93 @@ Framework::~Framework() { void Framework::acquireContext() { if (!running_) return; + // Use acquireContext entry as the frame boundary so all waits in this function + // are included in the same CPU frame interval window. + static auto lastFrameBoundary = std::chrono::high_resolution_clock::now(); + auto frameBoundaryNow = std::chrono::high_resolution_clock::now(); + float cpuFrameIntervalMs = + std::chrono::duration(frameBoundaryNow - lastFrameBoundary).count(); + lastFrameBoundary = frameBoundaryNow; + +#ifdef MCVR_ENABLE_OPENXR + // Poll OpenXR events and begin XR frame (updates head pose + FOV) + if (xrContext_) { + // Keep VRSystem config synchronized with options regardless of session state. + auto &vr = Renderer::instance().vrSystem(); + vr.config.ipd = Renderer::options.vrIPD; + vr.config.renderScale = Renderer::options.vrRenderScale; + vr.config.worldScale = Renderer::options.vrWorldScale; + + xrContext_->pollEvents(); + bool sessionRunning = xrContext_->isSessionRunning(); + + // Runtime transition point: session on/off means eyeCount/resolution path changed. + if (sessionRunning != xrLastSessionRunning_) { + xrLastSessionRunning_ = sessionRunning; + Renderer::options.needRecreate = true; + if (pipeline_ != nullptr) pipeline_->needRecreate = true; + + vr.enabled = Renderer::options.vrEnabled && sessionRunning; + vr.eyeCount = vr.enabled ? 2u : 1u; + + if (sessionRunning) { + vr.worldOrientation = glm::quat(1.0f, 0.0f, 0.0f, 0.0f); + vr.worldPosition = glm::vec3(0.0f); + } else { + vr.headPose = VRHeadPose{}; + vr.worldOrientation = glm::quat(1.0f, 0.0f, 0.0f, 0.0f); + vr.worldPosition = glm::vec3(0.0f); + } + } + + if (sessionRunning) { + uint32_t xrW = xrContext_->swapchainWidth(); + uint32_t xrH = xrContext_->swapchainHeight(); + if (xrW > 0 && xrH > 0 && (xrW != xrLastSwapchainWidth_ || xrH != xrLastSwapchainHeight_)) { + xrLastSwapchainWidth_ = xrW; + xrLastSwapchainHeight_ = xrH; + Renderer::options.needRecreate = true; + if (pipeline_ != nullptr) pipeline_->needRecreate = true; + } + } + + if (xrContext_->isSessionRunning()) { + // ======== PIPELINED FRAMING OPTIMIZATION ======== + // Start frame recording early without waiting for compositor. + // This allows CPU to start recording commands while GPU executes previous frame. + // The critical xrWaitFrame() call will happen later in submitCommand(). + xrContext_->beginFrameRecording(); + + // Note: Pose data will be latched later in submitCommand() + // via latchPose() for minimal motion-to-photon latency + auto &vr = Renderer::instance().vrSystem(); + // Temporarily copy previous frame's data until latchPose() updates it + if (!vr.headPose.valid) { + vr.headPose = VRHeadPose{}; + vr.headPose.valid = true; // Will be properly updated in latchPose() + } + + // Note: Controller states and gaze data will be updated in submitCommand() + // after pose latch to ensure synchronization with latest compositor timing + + } else { + vr.enabled = false; + vr.eyeCount = 1; + vr.headPose = VRHeadPose{}; + } + } +#endif + std::shared_ptr lastContext; if (currentContext_) lastContext = currentContext_; VkResult result; std::shared_ptr imageAcquiredSemaphore = acquireSemaphore(); uint32_t imageIndex; - result = vkAcquireNextImageKHR(device_->vkDevice(), swapchain_->vkSwapchain(), UINT64_MAX, + result = vkAcquireNextImageKHR(device_->vkDevice(), swapchain_->vkSwapchain(), + 5'000'000'000ull, // 5 second timeout (avoid TDR on alt-tab/minimize) imageAcquiredSemaphore->vkSemaphore(), VK_NULL_HANDLE, &imageIndex); - if (result == VK_ERROR_OUT_OF_DATE_KHR) { + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_TIMEOUT || result == VK_NOT_READY) { recycleSemaphore(imageAcquiredSemaphore); recreate(); return; @@ -206,7 +375,12 @@ void Framework::acquireContext() { } std::shared_ptr fence = contexts_[imageIndex]->commandFinishedFence; - result = vkWaitForFences(device_->vkDevice(), 1, &fence->vkFence(), true, UINT64_MAX); + result = vkWaitForFences(device_->vkDevice(), 1, &fence->vkFence(), true, + 5'000'000'000ull); // 5 second timeout + if (result == VK_TIMEOUT) { + std::cerr << "vkWaitForFences timed out (possible TDR recovery), skipping frame" << std::endl; + return; + } if (result != VK_SUCCESS) { std::cout << "vkWaitForFences failed with error: " << std::dec << result << std::endl; waitDeviceIdle(); @@ -217,6 +391,23 @@ void Framework::acquireContext() { indexHistory_.push(imageIndex); if (indexHistory_.size() > swapchain_->imageCount()) indexHistory_.pop(); +#ifdef MCVR_ENABLE_OPENXR + // Read back GPU timestamps from the previous use of this frame slot + if (gpuTimestampPool_ != VK_NULL_HANDLE && timestampPeriodNs_ > 0.0f) { + uint64_t ts[2] = {0, 0}; + VkResult tsResult = vkGetQueryPoolResults( + device_->vkDevice(), gpuTimestampPool_, + imageIndex * 2, 2, + sizeof(ts), ts, sizeof(uint64_t), + VK_QUERY_RESULT_64_BIT); + if (tsResult == VK_SUCCESS && ts[1] > ts[0]) { + auto &vr = Renderer::instance().vrSystem(); + vr.perfStats.gpuFrameTimeMs = + static_cast(static_cast(ts[1] - ts[0]) * timestampPeriodNs_ / 1e6); + } + } +#endif + if (currentContext_->imageAcquiredSemaphore != VK_NULL_HANDLE) { recycleSemaphore(currentContext_->imageAcquiredSemaphore); currentContext_->imageAcquiredSemaphore = VK_NULL_HANDLE; @@ -226,7 +417,16 @@ void Framework::acquireContext() { currentContext_->uploadCommandBuffer->begin(); currentContext_->worldCommandBuffer->begin(); currentContext_->overlayCommandBuffer->begin(); - currentContext_->fuseCommandBuffer->begin(); + +#ifdef MCVR_ENABLE_OPENXR + // Reset + write GPU timestamp BEGIN in the first command buffer + if (gpuTimestampPool_ != VK_NULL_HANDLE) { + VkCommandBuffer cmd = currentContext_->uploadCommandBuffer->vkCommandBuffer(); + vkCmdResetQueryPool(cmd, gpuTimestampPool_, currentContextIndex_ * 2, 2); + vkCmdWriteTimestamp(cmd, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + gpuTimestampPool_, currentContextIndex_ * 2); + } +#endif auto pipelineContext = pipeline_->acquirePipelineContext(currentContext_); std::shared_ptr lastUIContext = @@ -244,6 +444,18 @@ void Framework::acquireContext() { static int frames = 0; static auto lastTime = std::chrono::high_resolution_clock::now(); +#ifdef MCVR_ENABLE_OPENXR + // Track CPU frame interval. + if (xrContext_ && Renderer::options.vrEnabled) { + auto &vr = Renderer::instance().vrSystem(); + float cpuMs = cpuFrameIntervalMs; + vr.perfStats.cpuFrameTimeMs = cpuMs; + if (cpuMs > 0.0f) { + vr.perfStats.fps = 1000.0f / cpuMs; + } + } +#endif + frames++; auto currentTime = std::chrono::high_resolution_clock::now(); std::chrono::duration elapsed = currentTime - lastTime; @@ -266,18 +478,215 @@ void Framework::submitCommand() { Renderer::instance().textures()->performQueuedUpload(); Renderer::instance().buffers()->performQueuedUpload(); + +#ifdef MCVR_ENABLE_OPENXR + // LATE-LATCH POSE + bool xrShouldRender = false; + if (xrContext_ && xrContext_->isSessionRunning()) { + xrShouldRender = xrContext_->latchPose(); + + // xrWaitFrame() populates predictedDisplayPeriod in latchPose(), so target + // frame time must be captured here instead of acquireContext(). + auto &vr = Renderer::instance().vrSystem(); + int64_t periodNs = xrContext_->predictedDisplayPeriodNs(); + if (periodNs > 0) { + vr.perfStats.compositorTargetMs = static_cast(periodNs) / 1e6f; + } + + // Split CPU frame interval into active work vs. compositor pacing wait. + // cpuWaitMs = time blocked in xrWaitFrame (compositor VSync alignment, not real work). + // cpuWorkMs = actual CPU computation time = interval - wait. + vr.perfStats.cpuWaitMs = xrContext_->lastWaitFrameMs(); + vr.perfStats.cpuWorkMs = std::max(0.0f, vr.perfStats.cpuFrameTimeMs - vr.perfStats.cpuWaitMs); + + // Headroom and dropped-frame detection use max(cpuWorkMs, gpuFrameTimeMs). + // cpuFrameTimeMs includes xrWaitFrame sleep and is ≈ targetMs by design, + // so using it here would always yield ~0 headroom — cpuWorkMs is correct. + if (vr.perfStats.compositorTargetMs > 0.0f) { + float workMs = std::max(vr.perfStats.cpuWorkMs, vr.perfStats.gpuFrameTimeMs); + vr.perfStats.headroom = + (vr.perfStats.compositorTargetMs - workMs) / vr.perfStats.compositorTargetMs; + if (workMs > vr.perfStats.compositorTargetMs) { + vr.perfStats.droppedFrames++; + } + } + + if (xrShouldRender) { + // Update VRSystem with latest pose data (fresh from compositor) + xrContext_->getHeadPose(vr.headPose); + xrContext_->getEyeParams(vr.eyes.data()); + vr.config.ipd = glm::distance(vr.eyes[0].positionOffset, vr.eyes[1].positionOffset); + + // Update controller states with latest input data + auto &input = xrContext_->input(); + for (uint32_t h = 0; h < 2; h++) { + auto &src = input.controllers[h]; + auto &dst = vr.controllers[h]; + dst.valid = src.valid; + dst.position = src.position; + dst.orientation = src.orientation; + dst.linearVelocity = src.linearVelocity; + dst.angularVelocity = src.angularVelocity; + dst.triggerValue = src.triggerValue; + dst.gripValue = src.gripValue; + dst.thumbstick = src.thumbstick; + dst.triggerPressed = src.triggerPressed; + dst.gripPressed = src.gripPressed; + dst.primaryButton = src.primaryButton; + dst.secondaryButton = src.secondaryButton; + dst.thumbstickClick = src.thumbstickClick; + dst.menuButton = src.menuButton; + } + + // Update eye gaze data + vr.gazeValid = input.gazeValid; + vr.gazePoint = input.gazePoint; + } + } +#endif + + // Build uniform buffers AFTER pose latch to capture latest VR data Renderer::instance().buffers()->buildAndUploadOverlayUniformBuffer(); +#ifdef MCVR_ENABLE_OPENXR + // Performance validation: log timing in debug builds + #ifdef DEBUG + if (xrContext_ && xrShouldRender) { + static auto lastLatchTime = std::chrono::high_resolution_clock::now(); + auto currentTime = std::chrono::high_resolution_clock::now(); + auto latchInterval = std::chrono::duration(currentTime - lastLatchTime).count(); + if (latchInterval > 0) { + // This should be close to the display refresh rate (e.g., ~11ms for 90Hz) + std::cout << "[XR Late-Latch] Pose latch interval: " << latchInterval << "ms" << std::endl; + } + lastLatchTime = currentTime; + } + #endif +#endif + auto pipelineContext = pipeline_->acquirePipelineContext(currentContext_); if (Renderer::instance().world()->shouldRender()) pipelineContext->worldPipelineContext->render(); pipelineContext->uiModuleContext->end(); +#ifdef MCVR_ENABLE_OPENXR + // Blit world render output (array image layers) to XR swapchain images. + if (xrContext_ && xrContext_->isSessionRunning() && xrContext_->shouldRender()) { + auto outputImage = pipelineContext->worldPipelineContext->outputImage; + if (outputImage) { + auto cmdBuf = currentContext_->worldCommandBuffer->vkCommandBuffer(); + auto mainQueueIndex = physicalDevice_->mainQueueIndex(); + + VkImage xrImages[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + xrImages[0] = xrContext_->acquireSwapchainImage(0); + if (xrImages[0] != VK_NULL_HANDLE) + xrImages[1] = xrContext_->acquireSwapchainImage(1); + + if (xrImages[0] != VK_NULL_HANDLE && xrImages[1] != VK_NULL_HANDLE) { + const uint32_t xrW = xrContext_->swapchainWidth(); + const uint32_t xrH = xrContext_->swapchainHeight(); + const bool sameSize = (outputImage->width() == xrW && outputImage->height() == xrH); + const VkImageSubresourceRange xrRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + VkImageMemoryBarrier preBlit[3] = {}; + preBlit[0].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + preBlit[0].srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + preBlit[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + preBlit[0].oldLayout = outputImage->imageLayout(); + preBlit[0].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + preBlit[0].srcQueueFamilyIndex = mainQueueIndex; + preBlit[0].dstQueueFamilyIndex = mainQueueIndex; + preBlit[0].image = outputImage->vkImage(); + preBlit[0].subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 2}; // both layers + preBlit[1].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + preBlit[1].srcAccessMask = 0; + preBlit[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + preBlit[1].oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + preBlit[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + preBlit[1].srcQueueFamilyIndex = mainQueueIndex; + preBlit[1].dstQueueFamilyIndex = mainQueueIndex; + preBlit[1].image = xrImages[0]; + preBlit[1].subresourceRange = xrRange; + preBlit[2] = preBlit[1]; + preBlit[2].image = xrImages[1]; + + vkCmdPipelineBarrier(cmdBuf, + VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 3, preBlit); + + // Both transfers back-to-back with no intervening barriers. + for (uint32_t eye = 0; eye < 2; eye++) { + if (sameSize) { + VkImageCopy copyRegion{}; + copyRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eye, 1}; + copyRegion.srcOffset = {0, 0, 0}; + copyRegion.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copyRegion.dstOffset = {0, 0, 0}; + copyRegion.extent = {xrW, xrH, 1}; + vkCmdCopyImage(cmdBuf, + outputImage->vkImage(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + xrImages[eye], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, ©Region); + } else { + VkImageBlit blitRegion{}; + blitRegion.srcSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, eye, 1}; + blitRegion.srcOffsets[0] = {0, 0, 0}; + blitRegion.srcOffsets[1] = {static_cast(outputImage->width()), + static_cast(outputImage->height()), 1}; + blitRegion.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + blitRegion.dstOffsets[0] = {0, 0, 0}; + blitRegion.dstOffsets[1] = {static_cast(xrW), static_cast(xrH), 1}; + vkCmdBlitImage(cmdBuf, + outputImage->vkImage(), VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + xrImages[eye], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + 1, &blitRegion, VK_FILTER_LINEAR); + } + } + + VkImageMemoryBarrier postBlit[3] = {}; + postBlit[0].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + postBlit[0].srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + postBlit[0].dstAccessMask = 0; + postBlit[0].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + postBlit[0].newLayout = outputImage->imageLayout(); + postBlit[0].srcQueueFamilyIndex = mainQueueIndex; + postBlit[0].dstQueueFamilyIndex = mainQueueIndex; + postBlit[0].image = outputImage->vkImage(); + postBlit[0].subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 2}; // both layers + postBlit[1].sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + postBlit[1].srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + postBlit[1].dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + postBlit[1].oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + postBlit[1].newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + postBlit[1].srcQueueFamilyIndex = mainQueueIndex; + postBlit[1].dstQueueFamilyIndex = mainQueueIndex; + postBlit[1].image = xrImages[0]; + postBlit[1].subresourceRange = xrRange; + postBlit[2] = postBlit[1]; + postBlit[2].image = xrImages[1]; + + vkCmdPipelineBarrier(cmdBuf, + VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, 0, nullptr, 0, nullptr, 3, postBlit); + } + } + } +#endif + currentContext_->fuseFinal(); +#ifdef MCVR_ENABLE_OPENXR + // Write GPU timestamp END in the last command buffer + if (gpuTimestampPool_ != VK_NULL_HANDLE) { + vkCmdWriteTimestamp(currentContext_->fuseCommandBuffer->vkCommandBuffer(), + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + gpuTimestampPool_, currentContextIndex_ * 2 + 1); + } +#endif + currentContext_->uploadCommandBuffer->end(); currentContext_->worldCommandBuffer->end(); currentContext_->overlayCommandBuffer->end(); - currentContext_->fuseCommandBuffer->end(); std::vector waitSemaphores = {currentContext_->imageAcquiredSemaphore->vkSemaphore()}; std::vector waitStageMasks = {VK_PIPELINE_STAGE_ALL_COMMANDS_BIT}; @@ -286,7 +695,6 @@ void Framework::submitCommand() { currentContext_->uploadCommandBuffer->vkCommandBuffer(), currentContext_->worldCommandBuffer->vkCommandBuffer(), currentContext_->overlayCommandBuffer->vkCommandBuffer(), - currentContext_->fuseCommandBuffer->vkCommandBuffer(), }; VkSubmitInfo vkSubmitInfo = {}; @@ -302,6 +710,14 @@ void Framework::submitCommand() { std::shared_ptr fence = currentContext_->commandFinishedFence; vkResetFences(device_->vkDevice(), 1, &fence->vkFence()); vkQueueSubmit(device_->mainVkQueue(), 1, &vkSubmitInfo, fence->vkFence()); + +#ifdef MCVR_ENABLE_OPENXR + if (xrContext_ && xrContext_->isSessionRunning()) { + for (uint32_t eye = 0; eye < 2; eye++) { + xrContext_->releaseSwapchainImage(eye); + } + } +#endif } void Framework::present() { @@ -318,6 +734,13 @@ void Framework::present() { VkResult result = vkQueuePresentKHR(device_->mainVkQueue(), &presentInfo); +#ifdef MCVR_ENABLE_OPENXR + // End the XR frame (submit layers to compositor) + if (xrContext_ && xrContext_->isSessionRunning()) { + xrContext_->endFrame(); + } +#endif + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR || vk::Window::framebufferResized || Renderer::options.needRecreate || pipeline_->needRecreate) { recreate(); @@ -342,7 +765,16 @@ void Framework::recreate() { int width = 0, height = 0; GLFW_GetFramebufferSize(window_->window(), &width, &height); + // In VR mode, don't block on minimized window — VR rendering doesn't need the desktop window. + // Without this, minimizing/alt-tabbing causes an infinite loop that triggers the watchdog. + int waitAttempts = 0; while (width == 0 || height == 0) { + if (waitAttempts++ > 50) { // ~5 seconds with typical event wait timing + std::cerr << "Window framebuffer still 0x0 after 50 attempts, using fallback size" << std::endl; + width = 1; + height = 1; + break; + } GLFW_GetFramebufferSize(window_->window(), &width, &height); GLFW_WaitEvents(); } @@ -354,7 +786,6 @@ void Framework::recreate() { uploadCommandBuffers_.clear(); overlayCommandBuffers_.clear(); worldCommandBuffers_.clear(); - fuseCommandBuffers_.clear(); commandFinishedFences_.clear(); commandProcessedSemaphores_.clear(); @@ -367,7 +798,6 @@ void Framework::recreate() { uploadCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); overlayCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); worldCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); - fuseCommandBuffers_.emplace_back(vk::CommandBuffer::create(device_, mainCommandPool_)); } // create fence for each context @@ -397,9 +827,51 @@ void Framework::waitBackendQueueIdle() { void Framework::close() { if (running_) { pipeline_->close(); } +#ifdef MCVR_ENABLE_OPENXR + if (gpuTimestampPool_ != VK_NULL_HANDLE) { + vkDestroyQueryPool(device_->vkDevice(), gpuTimestampPool_, nullptr); + gpuTimestampPool_ = VK_NULL_HANDLE; + } + if (xrContext_) { xrContext_->shutdown(); xrContext_.reset(); } +#endif running_ = false; } +#ifdef MCVR_ENABLE_OPENXR +bool Framework::startXRSession() { + if (!running_ || xrContext_ == nullptr) return false; + if (!Renderer::options.vrEnabled) return false; + + auto &vr = Renderer::instance().vrSystem(); + vr.worldOrientation = glm::quat(1.0f, 0.0f, 0.0f, 0.0f); + vr.worldPosition = glm::vec3(0.0f); + + bool ok = xrContext_->requestSessionStart(); + Renderer::options.needRecreate = true; + if (pipeline_ != nullptr) pipeline_->needRecreate = true; + return ok; +} + +void Framework::stopXRSession() { + if (!running_ || xrContext_ == nullptr) return; + + xrContext_->requestSessionStop(); + auto &vr = Renderer::instance().vrSystem(); + vr.enabled = false; + vr.eyeCount = 1; + vr.headPose = VRHeadPose{}; + vr.worldOrientation = glm::quat(1.0f, 0.0f, 0.0f, 0.0f); + vr.worldPosition = glm::vec3(0.0f); + + Renderer::options.needRecreate = true; + if (pipeline_ != nullptr) pipeline_->needRecreate = true; +} + +bool Framework::isXRSessionRunning() const { + return xrContext_ != nullptr && xrContext_->isSessionRunning(); +} +#endif + bool Framework::isRunning() { return running_; } diff --git a/src/core/render/render_framework.hpp b/src/core/render/render_framework.hpp index 48d3878..82ea268 100644 --- a/src/core/render/render_framework.hpp +++ b/src/core/render/render_framework.hpp @@ -7,6 +7,11 @@ #include "core/vulkan/all_core_vulkan.hpp" #include "core/render/modules/world/dlss/dlss_wrapper.hpp" +#ifdef MCVR_ENABLE_OPENXR +#include "core/render/openxr_context.hpp" +#include +#endif + #include #include @@ -78,6 +83,12 @@ class Framework : public SharedObject { void takeScreenshot(bool withUI, int width, int height, int channel, void *dstPointer); + #ifdef MCVR_ENABLE_OPENXR + bool startXRSession(); + void stopXRSession(); + bool isXRSessionRunning() const; + #endif + std::recursive_mutex &recreateMtx(); std::shared_ptr instance(); @@ -117,7 +128,6 @@ class Framework : public SharedObject { std::vector> uploadCommandBuffers_; std::vector> overlayCommandBuffers_; std::vector> worldCommandBuffers_; - std::vector> fuseCommandBuffers_; std::shared_ptr worldAsyncCommandBuffer_; std::shared_ptr pipeline_; @@ -137,6 +147,19 @@ class Framework : public SharedObject { bool running_ = true; std::shared_ptr gc_; + +#ifdef MCVR_ENABLE_OPENXR + std::unique_ptr xrContext_; + bool xrVRPopulated_ = false; + VkQueryPool gpuTimestampPool_ = VK_NULL_HANDLE; + float timestampPeriodNs_ = 0.0f; + bool xrLastSessionRunning_ = false; + uint32_t xrLastSwapchainWidth_ = 0; + uint32_t xrLastSwapchainHeight_ = 0; +public: + OpenXRContext *xrContext() { return xrContext_.get(); } +private: +#endif }; template diff --git a/src/core/render/renderer.hpp b/src/core/render/renderer.hpp index aa3628a..9eae688 100644 --- a/src/core/render/renderer.hpp +++ b/src/core/render/renderer.hpp @@ -11,6 +11,8 @@ class Framework; class Buffers; class World; +#include "core/render/vr_system.hpp" + struct Options { uint32_t maxFps = 1e6; uint32_t inactivityFpsLimit = 1e6; @@ -25,6 +27,11 @@ struct Options { uint32_t chunkBuildingBatchSize = 2; uint32_t chunkBuildingTotalBatches = 4; + + bool vrEnabled = false; + float vrRenderScale = 0.5f; + float vrIPD = 0.063f; + float vrWorldScale = 1.0f; }; class Renderer : public Singleton { @@ -40,6 +47,7 @@ class Renderer : public Singleton { std::shared_ptr textures(); std::shared_ptr buffers(); std::shared_ptr world(); + VRSystem &vrSystem() { return vrSystem_; } void close(); @@ -50,4 +58,5 @@ class Renderer : public Singleton { std::shared_ptr textures_; std::shared_ptr buffers_; std::shared_ptr world_; + VRSystem vrSystem_; }; diff --git a/src/core/render/vr_system.cpp b/src/core/render/vr_system.cpp new file mode 100644 index 0000000..43f2a93 --- /dev/null +++ b/src/core/render/vr_system.cpp @@ -0,0 +1,106 @@ +#include "core/render/vr_system.hpp" + +#include +#include + +// ---- VREyeParams ---- + +glm::mat4 VREyeParams::projectionMatrix(float nearZ, float farZ) const { + // Build asymmetric frustum from tangent values + float left = tanLeft * nearZ; + float right = tanRight * nearZ; + float top = tanUp * nearZ; + float bottom = tanDown * nearZ; + + // Manual frustum construction (Vulkan clip space: Y down, Z [0,1]) + float width = right - left; + float height = top - bottom; + float depth = farZ - nearZ; + + glm::mat4 result(0.0f); + result[0][0] = 2.0f * nearZ / width; + result[1][1] = -2.0f * nearZ / height; // Vulkan Y flip + result[2][0] = (right + left) / width; + result[2][1] = -(top + bottom) / height; // Negated for Vulkan Y-flip (entire row 1 must be negated) + result[2][2] = -farZ / depth; + result[2][3] = -1.0f; + result[3][2] = -(farZ * nearZ) / depth; + + return result; +} + +glm::mat4 VREyeParams::viewOffset() const { + glm::mat4 rotation = glm::mat4_cast(glm::inverse(orientationOffset)); + glm::mat4 translation = glm::translate(glm::mat4(1.0f), -positionOffset); + return rotation * translation; +} + +uint32_t VREyeParams::renderWidth(float renderScale) const { + return std::max(1u, static_cast(recommendedWidth * renderScale)); +} + +uint32_t VREyeParams::renderHeight(float renderScale) const { + return std::max(1u, static_cast(recommendedHeight * renderScale)); +} + +// ---- VRHeadPose ---- + +glm::mat4 VRHeadPose::viewMatrix() const { + if (!valid) return glm::mat4(1.0f); + glm::mat4 rotation = glm::mat4_cast(glm::inverse(orientation)); + glm::mat4 translation = glm::translate(glm::mat4(1.0f), -position); + return rotation * translation; +} + +// ---- VRSystem ---- + +void VRSystem::updateSimulation(uint32_t windowWidth, uint32_t windowHeight, float fovY) { + if (!enabled || eyeCount <= 1) return; + + float halfIPD = config.ipd * 0.5f; + float aspect = static_cast(windowWidth) / static_cast(windowHeight); + float tanHalfFovY = std::tan(fovY * 0.5f); + float tanHalfFovX = tanHalfFovY * aspect; + + for (uint32_t eye = 0; eye < 2; eye++) { + auto &e = eyes[eye]; + e.recommendedWidth = windowWidth; + e.recommendedHeight = windowHeight; + e.tanLeft = -tanHalfFovX; + e.tanRight = tanHalfFovX; + e.tanUp = tanHalfFovY; + e.tanDown = -tanHalfFovY; + + float offsetX = (eye == 0) ? -halfIPD : halfIPD; + e.positionOffset = glm::vec3(offsetX, 0.0f, 0.0f); + e.orientationOffset = glm::quat(1.0f, 0.0f, 0.0f, 0.0f); + } + + headPose = VRHeadPose{}; +} + +uint32_t VRSystem::eyeRenderWidth() const { + return eyes[0].renderWidth(config.renderScale); +} + +uint32_t VRSystem::eyeRenderHeight() const { + return eyes[0].renderHeight(config.renderScale); +} + +void VRSystem::updateFromOpenXR(const VRHeadPose &head, const VREyeParams inEyes[2]) { + headPose = head; + for (uint32_t eye = 0; eye < 2; eye++) { + eyes[eye] = inEyes[eye]; + } + // Derive IPD from the distance between the two eye positions + float ipdMeasured = glm::distance(eyes[0].positionOffset, eyes[1].positionOffset); + if (ipdMeasured > 0.0f) { + config.ipd = ipdMeasured; + } +} + +void VRSystem::recenter() { + if (headPose.valid) { + worldPosition = headPose.position; + } +} diff --git a/src/core/render/vr_system.hpp b/src/core/render/vr_system.hpp new file mode 100644 index 0000000..c28385c --- /dev/null +++ b/src/core/render/vr_system.hpp @@ -0,0 +1,129 @@ +#pragma once + +#include +#include + +#include +#include +#include + +// Per-hand controller state (populated from OpenXR input each frame) +struct VRControllerState { + bool valid = false; + glm::vec3 position{0.0f}; + glm::quat orientation{1.0f, 0.0f, 0.0f, 0.0f}; + glm::vec3 linearVelocity{0.0f}; + glm::vec3 angularVelocity{0.0f}; + + float triggerValue = 0.0f; + float gripValue = 0.0f; + glm::vec2 thumbstick{0.0f}; + + bool triggerPressed = false; + bool gripPressed = false; + bool primaryButton = false; + bool secondaryButton = false; + bool thumbstickClick = false; + bool menuButton = false; +}; + +// Performance statistics for monitoring/auto quality +struct VRPerformanceStats { + float gpuFrameTimeMs = 0.0f; + float cpuFrameTimeMs = 0.0f; // full frame interval (cpuWorkMs + cpuWaitMs) + float cpuWorkMs = 0.0f; // CPU active work time (excludes xrWaitFrame compositor pacing) + float cpuWaitMs = 0.0f; // CPU time blocked in xrWaitFrame (compositor pacing / VSync wait) + float compositorTargetMs = 0.0f; // from XrFrameState.predictedDisplayPeriod + float fps = 0.0f; + uint32_t droppedFrames = 0; + float headroom = 0.0f; // (targetMs - max(cpuWorkMs, gpuFrameTimeMs)) / targetMs +}; + +struct VREyeParams { + // Asymmetric FOV tangent values (negative left/down, positive right/up) + float tanLeft = -1.0f; + float tanRight = 1.0f; + float tanUp = 1.0f; + float tanDown = -1.0f; + + // Recommended render resolution from OpenXR (per-eye) + uint32_t recommendedWidth = 1920; + uint32_t recommendedHeight = 1080; + + // Eye position/orientation offset relative to head (head space) + glm::vec3 positionOffset{0.0f}; + glm::quat orientationOffset{1.0f, 0.0f, 0.0f, 0.0f}; + + // Build asymmetric projection matrix from tangent values + glm::mat4 projectionMatrix(float nearZ, float farZ) const; + + // Build view offset matrix (head → eye transform) + glm::mat4 viewOffset() const; + + // Actual render resolution = recommended × renderScale + uint32_t renderWidth(float renderScale) const; + uint32_t renderHeight(float renderScale) const; +}; + +struct VRHeadPose { + glm::vec3 position{0.0f}; + glm::quat orientation{1.0f, 0.0f, 0.0f, 0.0f}; + bool valid = false; + + glm::mat4 viewMatrix() const; +}; + +enum class TrackingOrigin : uint32_t { + Seated = 0, + Standing = 1, +}; + +struct VRConfig { + float ipd = 0.063f; + float renderScale = 0.5f; + float worldScale = 1.0f; + float refreshRate = 90.0f; + TrackingOrigin trackingOrigin = TrackingOrigin::Standing; +}; + +struct VRSystem { + bool enabled = false; + uint32_t eyeCount = 1; + + VRConfig config; + VRHeadPose headPose; + std::array eyes; + + // World orientation offset (tracking space → game world). + // Set from Java each frame; encodes player body yaw, stick turn, + // mouse yaw, and character pose (elytra roll, swimming, etc.). + glm::quat worldOrientation{1.0f, 0.0f, 0.0f, 0.0f}; + + // World position offset (tracking-space origin in game world). + // When recenter is called, this is set to the current headPose.position + // so subsequent physical movement is relative to that point. + glm::vec3 worldPosition{0.0f}; + + // Recenter: snapshot current head position/yaw as the new origin. + void recenter(); + + // Simulation mode: populate eyes[] from window size, FOV, and config.ipd + void updateSimulation(uint32_t windowWidth, uint32_t windowHeight, float fovY); + + // Update from real OpenXR data (called each frame when XR is active) + void updateFromOpenXR(const VRHeadPose &head, const VREyeParams inEyes[2]); + + // Actual per-eye render resolution (after renderScale) + uint32_t eyeRenderWidth() const; + uint32_t eyeRenderHeight() const; + + // Controller input (updated from OpenXR each frame) + std::array controllers; // 0=left, 1=right + + // Performance stats (updated each frame) + VRPerformanceStats perfStats; + + // Eye gaze foveated centre in normalised coords (0.5, 0.5 = screen centre) + glm::vec2 gazePoint{0.5f, 0.5f}; + bool gazeValid = false; +}; diff --git a/src/core/vulkan/device.cpp b/src/core/vulkan/device.cpp index 9bebaff..9f00251 100644 --- a/src/core/vulkan/device.cpp +++ b/src/core/vulkan/device.cpp @@ -9,6 +9,8 @@ #include #include +std::vector vk::Device::extraExtensions{}; + std::ostream &deviceCout() { return std::cout << "[Device] "; } @@ -54,6 +56,10 @@ vk::Device::Device(std::shared_ptr instance, } } + // Extra extensions (e.g. from OpenXR) + // Store as const char* pointers — the static vector owns the strings. + for (const auto &ext : extraExtensions) { enabledExtensions.push_back(ext.c_str()); } + uint32_t deviceExtensionCount = 0; vkEnumerateDeviceExtensionProperties(physicalDevice_->vkPhysicalDevice(), nullptr, &deviceExtensionCount, nullptr); std::vector deviceExtensions(deviceExtensionCount); diff --git a/src/core/vulkan/device.hpp b/src/core/vulkan/device.hpp index cfbcdca..f7ecc8e 100644 --- a/src/core/vulkan/device.hpp +++ b/src/core/vulkan/device.hpp @@ -2,6 +2,9 @@ #include "core/all_extern.hpp" +#include +#include + namespace vk { class Instance; class Window; @@ -9,6 +12,9 @@ class PhysicalDevice; class Device : public SharedObject { public: + // Extra device extensions to enable (set before create(), e.g. by OpenXR). + static std::vector extraExtensions; + Device(std::shared_ptr instance, std::shared_ptr window, std::shared_ptr physicalDevice); diff --git a/src/core/vulkan/image.cpp b/src/core/vulkan/image.cpp index e25c056..7cc6484 100644 --- a/src/core/vulkan/image.cpp +++ b/src/core/vulkan/image.cpp @@ -397,6 +397,31 @@ void vk::DeviceLocalImage::addImageView(VkImageViewCreateInfo info) { imageViews_.push_back(vkImageView); } +void vk::DeviceLocalImage::createPerLayerViews() { + if (layer_ <= 1) return; + for (uint32_t i = 0; i < layer_; i++) { + VkImageViewCreateInfo viewInfo{}; + viewInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + viewInfo.image = image_; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = format_; + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, + VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, i, 1}; + addImageView(viewInfo); + } +} + +VkImageView vk::DeviceLocalImage::perLayerView(uint32_t layerIndex) { + // Per-layer views start at index 1 (index 0 is the whole-image view) + uint32_t viewIndex = 1 + layerIndex; + if (viewIndex >= imageViews_.size()) { + imageCerr() << "perLayerView: layerIndex " << layerIndex << " out of range" << std::endl; + exit(EXIT_FAILURE); + } + return imageViews_[viewIndex]; +} + vk::Sampler::Sampler(std::shared_ptr device) : Sampler(device, VK_FILTER_LINEAR, VK_SAMPLER_MIPMAP_MODE_LINEAR, VK_SAMPLER_ADDRESS_MODE_REPEAT) {} diff --git a/src/core/vulkan/image.hpp b/src/core/vulkan/image.hpp index 2993f90..6026055 100644 --- a/src/core/vulkan/image.hpp +++ b/src/core/vulkan/image.hpp @@ -66,6 +66,10 @@ static VkImageSubresourceLayers wholeStencilSubresourceLayers = { .layerCount = VK_REMAINING_ARRAY_LAYERS, }; +static inline VkImageSubresourceRange colorSubresourceRangeForLayer(uint32_t layer) { + return {VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, layer, 1}; +} + class Image { public: virtual uint32_t width() = 0; @@ -167,6 +171,11 @@ class DeviceLocalImage : public Image, public SharedObject { void addImageView(VkImageViewCreateInfo info); + // Create per-layer views for stereo rendering (one 2D view per array layer) + void createPerLayerViews(); + // Get per-layer view index (1-based: viewIndex = 1 + layerIndex) + VkImageView perLayerView(uint32_t layerIndex); + private: std::shared_ptr device_; std::shared_ptr vma_; diff --git a/src/core/vulkan/instance.cpp b/src/core/vulkan/instance.cpp index 10642b7..47135f2 100644 --- a/src/core/vulkan/instance.cpp +++ b/src/core/vulkan/instance.cpp @@ -6,6 +6,8 @@ #include #include +std::vector vk::Instance::extraExtensions{}; + #ifndef NDEBUG const bool ENABLE_DEBUGGING = true; #else @@ -89,6 +91,9 @@ vk::Instance::Instance() { // repeated for dlss, but make sure extStorage.insert(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME); + // Extra extensions (e.g. from OpenXR) + for (const auto &ext : extraExtensions) { extStorage.insert(ext); } + // if (ENABLE_DEBUGGING) { push_ext(VK_EXT_DEBUG_REPORT_EXTENSION_NAME); } // Check for extensions diff --git a/src/core/vulkan/instance.hpp b/src/core/vulkan/instance.hpp index e689254..f578f38 100644 --- a/src/core/vulkan/instance.hpp +++ b/src/core/vulkan/instance.hpp @@ -2,9 +2,15 @@ #include "core/all_extern.hpp" +#include +#include + namespace vk { class Instance : public SharedObject { public: + // Extra instance extensions to enable (set before create(), e.g. by OpenXR). + static std::vector extraExtensions; + Instance(); ~Instance(); diff --git a/src/core/vulkan/physical_device.cpp b/src/core/vulkan/physical_device.cpp index ddc1073..af344ff 100644 --- a/src/core/vulkan/physical_device.cpp +++ b/src/core/vulkan/physical_device.cpp @@ -6,6 +6,8 @@ #include #include +VkPhysicalDevice vk::PhysicalDevice::overrideDevice = VK_NULL_HANDLE; + std::ostream &physicalDeviceCout() { return std::cout << "[PhysicalDevice] "; } @@ -61,6 +63,17 @@ bool isDeviceSuitable(VkPhysicalDevice device) { } void vk::PhysicalDevice::findPhysicalDevice() { + // If an override device was set (e.g. by OpenXR), use it directly. + if (overrideDevice != VK_NULL_HANDLE) { + physicalDevice_ = overrideDevice; + VkPhysicalDeviceProperties properties; + vkGetPhysicalDeviceProperties(physicalDevice_, &properties); +#ifdef DEBUG + physicalDeviceCout() << "Using override physical device: " << properties.deviceName << std::endl; +#endif + return; + } + uint32_t deviceCount = 0; if (vkEnumeratePhysicalDevices(instance_->vkInstance(), &deviceCount, nullptr) != VK_SUCCESS || deviceCount == 0) { physicalDeviceCerr() << "failed to get number of physical devices" << std::endl; diff --git a/src/core/vulkan/physical_device.hpp b/src/core/vulkan/physical_device.hpp index 67b5319..9ab43c9 100644 --- a/src/core/vulkan/physical_device.hpp +++ b/src/core/vulkan/physical_device.hpp @@ -15,6 +15,9 @@ class PhysicalDevice : public SharedObject { uint32_t mainQueueIndex(); uint32_t secondaryQueueIndex(); + // If set, use this physical device instead of auto-selecting (e.g. OpenXR override). + static VkPhysicalDevice overrideDevice; + void findPhysicalDevice(); void findQueueFamilies(); diff --git a/src/shader/world/post_render/world_post.vert b/src/shader/world/post_render/world_post.vert index 5b27a87..f85164f 100644 --- a/src/shader/world/post_render/world_post.vert +++ b/src/shader/world/post_render/world_post.vert @@ -12,6 +12,10 @@ layout(set = 1, binding = 0) uniform WorldUniform { WorldUBO worldUBO; }; +layout(push_constant) uniform PushConstant { + uint eyeIndex; +} pc; + layout(location = 0) in vec3 inPos; layout(location = 1) in uint inUseNorm; layout(location = 2) in vec3 inNorm; @@ -49,11 +53,18 @@ layout(location = 15) out vec4 lightMapColor; layout(location = 16) out vec4 overlayColor; void main() { + // Per-eye view offset + mat4 eyeViewOffset = worldUBO.eyeViewOffsets[pc.eyeIndex]; + mat4 eyeView = eyeViewOffset * worldUBO.cameraEffectedViewMat; + mat4 eyeViewOffsetInv = mat4(1.0); + eyeViewOffsetInv[3] = vec4(-eyeViewOffset[3].xyz, 1.0); + mat4 eyeViewInv = worldUBO.cameraViewMatInv * eyeViewOffsetInv; + vec3 pos = inPos + inPostBase; if (inCoordinate == 0) { - pos = pos - worldUBO.cameraViewMatInv[3].xyz; + pos = pos - eyeViewInv[3].xyz; } else if (inCoordinate == 1) { - pos = mat3(worldUBO.cameraViewMatInv) * pos; + pos = mat3(eyeViewInv) * pos; } else if (inCoordinate == 2) { pos = pos; } @@ -62,7 +73,7 @@ void main() { if (inCoordinate == 0 || inCoordinate == 2) { outNorm = inNorm; } else if (inCoordinate == 1) { - outNorm = normalize(mat3(worldUBO.cameraViewMatInv) * inNorm); + outNorm = normalize(mat3(eyeViewInv) * inNorm); } outUseColorLayer = inUseColorLayer; outColorLayer = inColorLayer; @@ -77,7 +88,7 @@ void main() { outUseLight = inUseLight; outLightUV = inLightUV; - gl_Position = worldUBO.cameraProjMat * worldUBO.cameraEffectedViewMat * vec4(pos, 1.0); + gl_Position = worldUBO.eyeProjOffsets[pc.eyeIndex] * worldUBO.cameraProjMat * eyeView * vec4(pos, 1.0); if (inUseLight > 0) { lightMapColor = texelFetch(lightMap, inLightUV / 16, 0); diff --git a/src/shader/world/post_render/world_post_star.vert b/src/shader/world/post_render/world_post_star.vert index af9826b..b2093ef 100644 --- a/src/shader/world/post_render/world_post_star.vert +++ b/src/shader/world/post_render/world_post_star.vert @@ -12,6 +12,10 @@ layout(set = 1, binding = 1) uniform SkyUniform { SkyUBO skyUBO; }; +layout(push_constant) uniform PushConstant { + uint eyeIndex; +} pc; + layout(location = 0) in vec3 inPos; layout(location = 4) in vec4 inColorLayer; @@ -45,7 +49,7 @@ void main() { mat3 R = rotationFromXToDir(skyUBO.sunDirection); vec3 pos = R * inPos; outPos = pos; - gl_Position = worldUBO.cameraProjMat * worldUBO.cameraEffectedViewMat * vec4(pos, 1.0); + gl_Position = worldUBO.eyeProjOffsets[pc.eyeIndex] * worldUBO.cameraProjMat * (worldUBO.eyeViewOffsets[pc.eyeIndex] * worldUBO.cameraEffectedViewMat) * vec4(pos, 1.0); float vis = starVisibilityFromSunY(skyUBO.sunDirection.y); outColorLayer = vec4(inColorLayer.rgb * vis, vis); diff --git a/src/shader/world/ray_tracing/end_gateway.rchit b/src/shader/world/ray_tracing/end_gateway.rchit index 1090a41..46b2c10 100644 --- a/src/shader/world/ray_tracing/end_gateway.rchit +++ b/src/shader/world/ray_tracing/end_gateway.rchit @@ -78,6 +78,12 @@ layout(std430, buffer_reference, buffer_reference_align = 8) readonly buffer Ind } indexBuffer; +layout(push_constant) uniform PushConstant { + int numRayBounces; + int useJitter; + uint eyeIndex; +} pc; + layout(location = 0) rayPayloadInEXT PrimaryRay mainRay; hitAttributeEXT vec2 attribs; @@ -159,8 +165,10 @@ void main() { vec3 localPos = baryCoords.x * v0.pos + baryCoords.y * v1.pos + baryCoords.z * v2.pos; vec3 worldPos = vec4(localPos, 1.0) * gl_ObjectToWorld3x4EXT; + mat4 eyeProj = worldUbo.eyeProjOffsets[pc.eyeIndex] * worldUbo.cameraProjMat; + mat4 eyeView = worldUbo.eyeViewOffsets[pc.eyeIndex] * worldUbo.cameraEffectedViewMat; vec4 texProj0 = - projection_from_position(worldUbo.cameraProjMat * worldUbo.cameraEffectedViewMat * vec4(worldPos, 1.0)); + projection_from_position(eyeProj * eyeView * vec4(worldPos, 1.0)); vec3 color = vec3(0.0); if (worldUbo.endSkyTextureID >= 0) color += textureProj(textures[nonuniformEXT(worldUbo.endSkyTextureID)], texProj0).rgb * COLORS[0]; diff --git a/src/shader/world/ray_tracing/end_portal.rchit b/src/shader/world/ray_tracing/end_portal.rchit index d1b57a2..bb8ac27 100644 --- a/src/shader/world/ray_tracing/end_portal.rchit +++ b/src/shader/world/ray_tracing/end_portal.rchit @@ -78,6 +78,12 @@ layout(std430, buffer_reference, buffer_reference_align = 8) readonly buffer Ind } indexBuffer; +layout(push_constant) uniform PushConstant { + int numRayBounces; + int useJitter; + uint eyeIndex; +} pc; + layout(location = 0) rayPayloadInEXT PrimaryRay mainRay; hitAttributeEXT vec2 attribs; @@ -159,8 +165,10 @@ void main() { vec3 localPos = baryCoords.x * v0.pos + baryCoords.y * v1.pos + baryCoords.z * v2.pos; vec3 worldPos = vec4(localPos, 1.0) * gl_ObjectToWorld3x4EXT; + mat4 eyeProj = worldUbo.eyeProjOffsets[pc.eyeIndex] * worldUbo.cameraProjMat; + mat4 eyeView = worldUbo.eyeViewOffsets[pc.eyeIndex] * worldUbo.cameraEffectedViewMat; vec4 texProj0 = - projection_from_position(worldUbo.cameraProjMat * worldUbo.cameraEffectedViewMat * vec4(worldPos, 1.0)); + projection_from_position(eyeProj * eyeView * vec4(worldPos, 1.0)); vec3 color = vec3(0.0); if (worldUbo.endSkyTextureID >= 0) color += textureProj(textures[nonuniformEXT(worldUbo.endSkyTextureID)], texProj0).rgb * COLORS[0]; diff --git a/src/shader/world/ray_tracing/world.rgen b/src/shader/world/ray_tracing/world.rgen index e1a26ef..047a93c 100644 --- a/src/shader/world/ray_tracing/world.rgen +++ b/src/shader/world/ray_tracing/world.rgen @@ -62,20 +62,21 @@ layout(set = 2, binding = 3) uniform LightMapUniform { LightMapUBO lightMapUBO; }; -layout(set = 3, binding = 0, rgba16f) uniform image2D outputImage; -layout(set = 3, binding = 1, rgba8) uniform image2D diffuseAlbedoImage; -layout(set = 3, binding = 2, rgba8) uniform image2D specularAlbedoImage; -layout(set = 3, binding = 3, rgba16f) uniform image2D normalRoughnessImage; -layout(set = 3, binding = 4, rg16f) uniform image2D motionVectorImage; -layout(set = 3, binding = 5, r16f) uniform image2D linearDepthImage; -layout(set = 3, binding = 6, r16f) uniform image2D specularHitDepthImage; -layout(set = 3, binding = 7, r16f) uniform image2D firstHitDepthImage; -layout(set = 3, binding = 8, rgba16f) uniform image2D firstHitDiffuseDirectLightImage; -layout(set = 3, binding = 9, rgba16f) uniform image2D firstHitDiffuseIndirectLightImage; -layout(set = 3, binding = 10, rgba16f) uniform image2D firstHitSpecularImage; -layout(set = 3, binding = 11, rgba16f) uniform image2D firstHitClearImage; -layout(set = 3, binding = 12, rgba16f) uniform image2D firstHitBaseEmissionImage; -layout(set = 3, binding = 13, r16f) uniform image2D directLightDepthImage; +layout(set = 3, binding = 0, rgba16f) uniform image2DArray outputImage; +layout(set = 3, binding = 1, rgba8) uniform image2DArray diffuseAlbedoImage; +layout(set = 3, binding = 2, rgba8) uniform image2DArray specularAlbedoImage; +layout(set = 3, binding = 3, rgba16f) uniform image2DArray normalRoughnessImage; +layout(set = 3, binding = 4, rg16f) uniform image2DArray motionVectorImage; +layout(set = 3, binding = 5, r16f) uniform image2DArray linearDepthImage; +layout(set = 3, binding = 6, r16f) uniform image2DArray specularHitDepthImage; +layout(set = 3, binding = 7, r16f) uniform image2DArray firstHitDepthImage; +layout(set = 3, binding = 8, rgba16f) uniform image2DArray firstHitDiffuseDirectLightImage; +layout(set = 3, binding = 9, rgba16f) uniform image2DArray firstHitDiffuseIndirectLightImage; +layout(set = 3, binding = 10, rgba16f) uniform image2DArray firstHitSpecularImage; +layout(set = 3, binding = 11, rgba16f) uniform image2DArray firstHitClearImage; +layout(set = 3, binding = 12, rgba16f) uniform image2DArray firstHitBaseEmissionImage; +layout(set = 3, binding = 13, r16f) uniform image2DArray directLightDepthImage; +layout(set = 3, binding = 14, r32ui) uniform readonly uimage2DArray visibilityMaskImage; layout(std430, buffer_reference, buffer_reference_align = 8) readonly buffer VertexBuffer { PBRTriangle vertices[]; @@ -89,6 +90,7 @@ indexBuffer; layout(push_constant) uniform PushConstant { int numRayBounces; + int useJitter; } pc; @@ -113,26 +115,80 @@ vec3 applyFog(vec3 radiance, float t, float fogStart, float dBase, float dFog, v } void main() { + // Per-eye view/projection with VR eye offset + // Use gl_LaunchIDEXT.z to get eye index in 3D dispatch mode + uint currentEyeIndex = gl_LaunchIDEXT.z; + ivec2 pixel = ivec2(gl_LaunchIDEXT.xy); vec2 pixelCenter = pixel + 0.5; vec2 unjitteredPixelCenter = pixelCenter; pixelCenter += worldUBO.cameraJitter; vec2 resolution = vec2(gl_LaunchSizeEXT.xy); - float fovY = fovYFromProj(worldUBO.cameraProjMat); - float fovX = fovXFromProj(worldUBO.cameraProjMat); + + // Visibility mask: skip hidden pixels outside the lens FOV. + // Bitmask texture has width=ceil(W/32); each uint32 covers 32 horizontal pixels. + uint visMask = imageLoad(visibilityMaskImage, ivec3(pixel.x >> 5, pixel.y, currentEyeIndex)).r; + if ((visMask & (1u << (uint(pixel.x) & 31u))) == 0u) { + imageStore(outputImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(diffuseAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(specularAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(normalRoughnessImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(motionVectorImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(linearDepthImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(specularHitDepthImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitDepthImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitDiffuseDirectLightImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitDiffuseIndirectLightImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitSpecularImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitClearImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(firstHitBaseEmissionImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + imageStore(directLightDepthImage, ivec3(pixel, currentEyeIndex), vec4(0.0)); + return; + } + + // Foveated rendering: determine block size for this pixel + uint blockSize = 1u; + if (worldUBO.foveatedOuterBlockSize > 1u) { + vec2 center = resolution * worldUBO.foveatedCenter; + float halfDiag = length(center); + uint bs = worldUBO.foveatedOuterBlockSize; + // Use block origin distance so all pixels in a block make the same decision + ivec2 blockOrigin = (pixel / ivec2(bs)) * ivec2(bs); + float originDist = length(vec2(blockOrigin) + 0.5 - center) / halfDiag; + if (originDist > worldUBO.foveatedInnerRadius) { + blockSize = bs; + if (pixel != blockOrigin) { + return; // non-origin pixel, origin will fill this + } + } + } + + // Per-eye view/projection with VR eye offset + mat4 eyeViewOffset = worldUBO.eyeViewOffsets[currentEyeIndex]; + mat4 eyeView = eyeViewOffset * worldUBO.cameraEffectedViewMat; + // Inverse of pure translation: negate the translation column + mat4 eyeViewOffsetInv = mat4(1.0); + eyeViewOffsetInv[3] = vec4(-eyeViewOffset[3].xyz, 1.0); + mat4 eyeViewInv = worldUBO.cameraEffectedViewMatInv * eyeViewOffsetInv; + + mat4 projMat = worldUBO.eyeProjOffsets[currentEyeIndex] * worldUBO.cameraProjMat; + mat4 projMatInv = inverse(projMat); + + float fovY = fovYFromProj(projMat); + float fovX = fovXFromProj(projMat); vec2 ndc = pixelCenter / resolution * 2.0 - 1.0; vec4 nearPoint = vec4(ndc, 0.0, 1.0); - vec4 viewNear = worldUBO.cameraProjMatInv * nearPoint; + vec4 viewNear = projMatInv * nearPoint; viewNear /= viewNear.w; - vec3 origin = vec3(worldUBO.cameraEffectedViewMatInv * - vec4(0, 0, 0, 1)); // even though camera-centered, view mat contains still small shifts + vec3 origin = vec3(eyeViewInv * + vec4(0, 0, 0, 1)); // per-eye camera origin with VR offset const vec3 eyePos = origin; - vec3 direction = normalize(vec3(worldUBO.cameraEffectedViewMatInv * vec4(viewNear.xyz, 0.0))); + vec3 direction = normalize(vec3(eyeViewInv * vec4(viewNear.xyz, 0.0))); const vec3 orgDirection = direction; const uint rayFlags = gl_RayFlagsCullBackFacingTrianglesEXT; @@ -153,6 +209,10 @@ void main() { vec3 firstHitIndirectRadiance = vec3(0.0); vec3 firstHitBaseEmission = vec3(0.0); + // Saved for foveated block-fill: per-pixel motion vector recomputation + vec4 savedMotionOrigin = vec4(0.0); + mat4 savedPrevRelativeMVP = mat4(1.0); + mainRay.seed = worldUBO.seed; for (int isHand = 0; isHand <= 1; isHand++) { // Result of trace @@ -247,16 +307,16 @@ void main() { float firstHitLinearDepth = INF_DISTANCE; if (psrDepth == 0 && mainRay.hitT != INF_DISTANCE) { vec3 hitPos = eyePos + orgDirection * T; - hitPos -= worldUBO.cameraViewMatInv[3].xyz; - firstHitLinearDepth = -(mat3(worldUBO.cameraEffectedViewMat) * hitPos).z; + hitPos -= eyeViewInv[3].xyz; + firstHitLinearDepth = -(mat3(eyeView) * hitPos).z; } if (isHand == 0) { if (psrDepth == 0) { - imageStore(firstHitDepthImage, pixel, vec4(firstHitLinearDepth)); + imageStore(firstHitDepthImage, ivec3(pixel, currentEyeIndex), vec4(firstHitLinearDepth)); } } else { if (psrDepth == 0 && mainRay.hitT != INF_DISTANCE) { - imageStore(firstHitDepthImage, pixel, vec4(firstHitLinearDepth)); + imageStore(firstHitDepthImage, ivec3(pixel, currentEyeIndex), vec4(firstHitLinearDepth)); } } @@ -281,17 +341,19 @@ void main() { psrMirror *= buildMirrorMatrix4(mainRay.normal, mainRay.worldPos); } while (psrDepth < NUM_PSR_PER_RAY); - imageStore(directLightDepthImage, pixel, vec4(mainRay.directLightHitT)); + imageStore(directLightDepthImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.directLightHitT)); vec3 virtualOrigin = eyePos + orgDirection * T; - // viewZ is the 'Z' of the world hit position in camera space - float viewDepth = -(worldUBO.cameraEffectedViewMat * vec4(virtualOrigin, 1.0)).z; + // viewZ is the 'Z' of the world hit position in camera space (per-eye) + float viewDepth = -(eyeView * vec4(virtualOrigin, 1.0)).z; // get camera pos and delta dvec3 currCameraPos = worldUBO.cameraPos.xyz; dvec3 lastCameraPos = lastWorldUBO.cameraPos.xyz; vec3 cameraDelta = vec3(currCameraPos - lastCameraPos); - mat4 prevRelativeMVP = lastWorldUBO.cameraProjMat * lastWorldUBO.cameraEffectedViewMat; + mat4 lastEyeView = lastWorldUBO.eyeViewOffsets[currentEyeIndex] * lastWorldUBO.cameraEffectedViewMat; + mat4 lastProjMat = lastWorldUBO.eyeProjOffsets[currentEyeIndex] * lastWorldUBO.cameraProjMat; + mat4 prevRelativeMVP = lastProjMat * lastEyeView; // Early out when hitting sky (even via mirrors) if (hitSky) { @@ -299,30 +361,32 @@ void main() { // 1. not for hand, can be directly or reflected towards sky // 2. for hand, must be reflected towards sky if (isHand == 0 || (isHand == 1 && psrDepth > 1)) { - imageStore(outputImage, pixel, vec4(mainRay.radiance, 1.0)); - imageStore(diffuseAlbedoImage, pixel, vec4(reinhardMax(mainRay.radiance), 0)); - imageStore(specularAlbedoImage, pixel, vec4(0)); - imageStore(normalRoughnessImage, pixel, vec4(0)); + imageStore(outputImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.radiance, 1.0)); + imageStore(diffuseAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(reinhardMax(mainRay.radiance), 0)); + imageStore(specularAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(0)); + imageStore(normalRoughnessImage, ivec3(pixel, currentEyeIndex), vec4(0)); // imageStore(dlssSpecHitDistance, pixel, vec4(0.0)); vec4 motionOrigin; if (!isPsr) { // This is the case when we hit the skybox directly. // Treat it like a point at infinity along the view vector - imageStore(linearDepthImage, pixel, vec4(INF_DISTANCE)); + imageStore(linearDepthImage, ivec3(pixel, currentEyeIndex), vec4(INF_DISTANCE)); // motion origin is point at infinity along view vector motionOrigin = vec4(orgDirection, 0.0); } else { // Here we hit the sky through a reflection on the primary surface // Pretend the sky is "texture mapped" onto the mirror and moves with it. - imageStore(linearDepthImage, pixel, vec4(viewDepth)); + imageStore(linearDepthImage, ivec3(pixel, currentEyeIndex), vec4(viewDepth)); vec3 virtualRelativeOrigin = origin + orgDirection * T; vec3 originRelativeToPrevCam = virtualRelativeOrigin + cameraDelta; motionOrigin = vec4(originRelativeToPrevCam, 1.0); } vec2 motionVec = computeCameraMotionVector(prevRelativeMVP, pixelCenter, motionOrigin); - imageStore(motionVectorImage, pixel, vec4(motionVec, vec2(0.0))); + imageStore(motionVectorImage, ivec3(pixel, currentEyeIndex), vec4(motionVec, vec2(0.0))); + savedMotionOrigin = motionOrigin; + savedPrevRelativeMVP = prevRelativeMVP; mainRay.intermediateRadiance = mainRay.radiance; psrHitSky = true; @@ -340,10 +404,10 @@ void main() { firstHitMat = mat; // BaseColor/Metalness Buffer. DLSS only needs the base color ("Diffuse Albedo") - imageStore(diffuseAlbedoImage, pixel, + imageStore(diffuseAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.albedoValue.rgb * mainRay.albedoValue.a, firstHitMat.metallic)); - imageStore(specularAlbedoImage, pixel, vec4(firstHitMat.f0, 0.0)); + imageStore(specularAlbedoImage, ivec3(pixel, currentEyeIndex), vec4(firstHitMat.f0, 0.0)); // Normal/Roughness buffer { @@ -352,11 +416,11 @@ void main() { vec3 worldNormal = (psrMirror * vec4(mainRay.normal, 0.0)).xyz; vec4 normalRoughness = vec4(worldNormal, firstHitMat.roughness); - imageStore(normalRoughnessImage, pixel, normalRoughness); + imageStore(normalRoughnessImage, ivec3(pixel, currentEyeIndex), normalRoughness); } // ViewZ buffer - imageStore(linearDepthImage, pixel, vec4(viewDepth)); + imageStore(linearDepthImage, ivec3(pixel, currentEyeIndex), vec4(viewDepth)); // Motion Vector Buffer { @@ -407,7 +471,9 @@ void main() { } vec2 motionVec = computeCameraMotionVector(prevRelativeMVP, pixelCenter, vec4(originRelativeToPrevCam, 1.0)); - imageStore(motionVectorImage, pixel, vec4(motionVec, 0.0, 0.0)); + imageStore(motionVectorImage, ivec3(pixel, currentEyeIndex), vec4(motionVec, 0.0, 0.0)); + savedMotionOrigin = vec4(originRelativeToPrevCam, 1.0); + savedPrevRelativeMVP = prevRelativeMVP; } // sightly make the first non-mirror hit brighter @@ -528,24 +594,69 @@ void main() { if (!psrHitSky && isFirstHitNoisy) { if (isFirstHitDiffuse) { - imageStore(firstHitDiffuseDirectLightImage, ivec2(pixel), vec4(firstHitDirectRadiance, 1.0)); - imageStore(firstHitDiffuseIndirectLightImage, ivec2(pixel), vec4(firstHitIndirectRadiance, 1.0)); - imageStore(firstHitSpecularImage, ivec2(pixel), vec4(vec3(0.0), 1.0)); - imageStore(firstHitClearImage, ivec2(pixel), vec4(vec3(0.0), 0.0)); + imageStore(firstHitDiffuseDirectLightImage, ivec3(pixel, currentEyeIndex), vec4(firstHitDirectRadiance, 1.0)); + imageStore(firstHitDiffuseIndirectLightImage, ivec3(pixel, currentEyeIndex), vec4(firstHitIndirectRadiance, 1.0)); + imageStore(firstHitSpecularImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 1.0)); + imageStore(firstHitClearImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 0.0)); } else { - imageStore(firstHitDiffuseDirectLightImage, ivec2(pixel), vec4(vec3(0.0), 1.0)); - imageStore(firstHitDiffuseIndirectLightImage, ivec2(pixel), vec4(vec3(0.0), 1.0)); - imageStore(firstHitSpecularImage, ivec2(pixel), vec4(mainRay.radiance, 1.0)); - imageStore(firstHitClearImage, ivec2(pixel), vec4(vec3(0.0), 0.0)); + imageStore(firstHitDiffuseDirectLightImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 1.0)); + imageStore(firstHitDiffuseIndirectLightImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 1.0)); + imageStore(firstHitSpecularImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.radiance, 1.0)); + imageStore(firstHitClearImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 0.0)); } } else { - imageStore(firstHitDiffuseDirectLightImage, ivec2(pixel), vec4(vec3(0.0), 0.0)); - imageStore(firstHitDiffuseIndirectLightImage, ivec2(pixel), vec4(vec3(0.0), 0.0)); - imageStore(firstHitSpecularImage, ivec2(pixel), vec4(vec3(0.0), 0.0)); - imageStore(firstHitClearImage, ivec2(pixel), vec4(mainRay.radiance, 1.0)); + imageStore(firstHitDiffuseDirectLightImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 0.0)); + imageStore(firstHitDiffuseIndirectLightImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 0.0)); + imageStore(firstHitSpecularImage, ivec3(pixel, currentEyeIndex), vec4(vec3(0.0), 0.0)); + imageStore(firstHitClearImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.radiance, 1.0)); } - imageStore(firstHitBaseEmissionImage, ivec2(pixel), vec4(firstHitEmission, 1.0)); + imageStore(firstHitBaseEmissionImage, ivec3(pixel, currentEyeIndex), vec4(firstHitEmission, 1.0)); float specHitDepth = hasSecondHit ? distFirstToSecond : (isFirstHitNoisy && !isFirstHitDiffuse ? 1000.0 : 0.0); - imageStore(specularHitDepthImage, ivec2(pixel), vec4(specHitDepth, vec3(1.0))); - imageStore(outputImage, ivec2(pixel), vec4(mainRay.radiance, 1.0)); + imageStore(specularHitDepthImage, ivec3(pixel, currentEyeIndex), vec4(specHitDepth, vec3(1.0))); + imageStore(outputImage, ivec3(pixel, currentEyeIndex), vec4(mainRay.radiance, 1.0)); + + // Foveated block-fill: copy origin pixel data to all other pixels in the block + if (blockSize > 1u) { + vec4 v_output = imageLoad(outputImage, ivec3(pixel, currentEyeIndex)); + vec4 v_diffuseAlbedo = imageLoad(diffuseAlbedoImage, ivec3(pixel, currentEyeIndex)); + vec4 v_specularAlbedo = imageLoad(specularAlbedoImage, ivec3(pixel, currentEyeIndex)); + vec4 v_normalRoughness = imageLoad(normalRoughnessImage, ivec3(pixel, currentEyeIndex)); + vec4 v_linearDepth = imageLoad(linearDepthImage, ivec3(pixel, currentEyeIndex)); + vec4 v_specHitDepth = imageLoad(specularHitDepthImage, ivec3(pixel, currentEyeIndex)); + vec4 v_firstHitDepth = imageLoad(firstHitDepthImage, ivec3(pixel, currentEyeIndex)); + vec4 v_diffDirect = imageLoad(firstHitDiffuseDirectLightImage, ivec3(pixel, currentEyeIndex)); + vec4 v_diffIndirect = imageLoad(firstHitDiffuseIndirectLightImage, ivec3(pixel, currentEyeIndex)); + vec4 v_specular = imageLoad(firstHitSpecularImage, ivec3(pixel, currentEyeIndex)); + vec4 v_clear = imageLoad(firstHitClearImage, ivec3(pixel, currentEyeIndex)); + vec4 v_baseEmission = imageLoad(firstHitBaseEmissionImage, ivec3(pixel, currentEyeIndex)); + vec4 v_directLightDepth = imageLoad(directLightDepthImage, ivec3(pixel, currentEyeIndex)); + + for (uint dy = 0u; dy < blockSize; dy++) { + for (uint dx = 0u; dx < blockSize; dx++) { + ivec2 fillPixel = pixel + ivec2(dx, dy); + if (fillPixel == pixel) continue; + if (fillPixel.x >= int(resolution.x) || fillPixel.y >= int(resolution.y)) continue; + + // Compute corrected motion vector for this fill pixel + vec2 fillPixelCenter = vec2(fillPixel) + 0.5 + worldUBO.cameraJitter; + vec2 fillMotionVec = computeCameraMotionVector( + savedPrevRelativeMVP, fillPixelCenter, savedMotionOrigin); + + imageStore(outputImage, ivec3(fillPixel, currentEyeIndex), v_output); + imageStore(diffuseAlbedoImage, ivec3(fillPixel, currentEyeIndex), v_diffuseAlbedo); + imageStore(specularAlbedoImage, ivec3(fillPixel, currentEyeIndex), v_specularAlbedo); + imageStore(normalRoughnessImage, ivec3(fillPixel, currentEyeIndex), v_normalRoughness); + imageStore(motionVectorImage, ivec3(fillPixel, currentEyeIndex), vec4(fillMotionVec, 0.0, 0.0)); + imageStore(linearDepthImage, ivec3(fillPixel, currentEyeIndex), v_linearDepth); + imageStore(specularHitDepthImage, ivec3(fillPixel, currentEyeIndex), v_specHitDepth); + imageStore(firstHitDepthImage, ivec3(fillPixel, currentEyeIndex), v_firstHitDepth); + imageStore(firstHitDiffuseDirectLightImage, ivec3(fillPixel, currentEyeIndex), v_diffDirect); + imageStore(firstHitDiffuseIndirectLightImage, ivec3(fillPixel, currentEyeIndex), v_diffIndirect); + imageStore(firstHitSpecularImage, ivec3(fillPixel, currentEyeIndex), v_specular); + imageStore(firstHitClearImage, ivec3(fillPixel, currentEyeIndex), v_clear); + imageStore(firstHitBaseEmissionImage, ivec3(fillPixel, currentEyeIndex), v_baseEmission); + imageStore(directLightDepthImage, ivec3(fillPixel, currentEyeIndex), v_directLightDepth); + } + } + } }