From e913b9e7c09a378e43c73ec5c4fc1dfbd61a79a8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 00:31:46 -0700 Subject: [PATCH 1/6] [WebRTC] Reduce remote rendering lag: input/draw coalescing, async encode, VP9 preference - Remove duplicate redraw on input (WebRTCWindowSystem.cpp) - Draw coalescing (BitmapWindowSystem.cpp) - Input (mouse event) coalescing (BitmapWindowSystem.cpp) - JS requestAnimationFrame coalescing / throttling (webrtcstreamer.js) - Async encoder thread (PeerConnectionManager.cpp/.h) - Data channel low-latency mode (webrtcstreamer.js) - Reduced startup delay (500ms -> 250ms) (WebRTCWindowSystem.cpp) - VP9 codec preference over VP8 (webrtcstreamer.js) --- .../visualization/gui/BitmapWindowSystem.cpp | 126 +++++++++++++++- .../webrtc_server/PeerConnectionManager.cpp | 59 ++++++-- .../webrtc_server/PeerConnectionManager.h | 14 ++ .../webrtc_server/WebRTCWindowSystem.cpp | 8 +- .../webrtc_server/html/webrtcstreamer.js | 136 +++++++++++++++--- 5 files changed, 302 insertions(+), 41 deletions(-) diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 3db9d0cefef..b73d680b381 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "open3d/geometry/Image.h" #include "open3d/utility/Logging.h" @@ -44,10 +45,21 @@ struct BitmapEvent { virtual void Execute() = 0; }; +// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue +// to clear the per-window pending-draw flag before calling OnDraw(). +struct BitmapEventQueue; + struct BitmapDrawEvent : public BitmapEvent { - BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {} + // queue_ is used to clear the pending-draw flag before OnDraw() so that + // a PostRedraw() called *inside* OnDraw() is not suppressed. + BitmapEventQueue *queue_; + + BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue) + : BitmapEvent(target), queue_(queue) {} - void Execute() override { event_target->o3d_window->OnDraw(); } + // Execute() is defined after BitmapEventQueue (below) because it calls + // clear_pending_draw(), which requires the full BitmapEventQueue type. + void Execute() override; }; struct BitmapResizeEvent : public BitmapEvent { @@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). pop_front() and -/// push() are protected by a mutex. push() may fail if the mutex cannot be -/// acquired immediately. empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). +/// pop_front() and push() are protected by a mutex. +/// push() may fail if the mutex cannot be acquired immediately. +/// empty() is not protected and is not reliable. +/// +/// Extended to support: +/// - Draw coalescing: at most one pending draw per window (push_draw). +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates +/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; +/// processing them forces stale frames to be encoded and sent. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; + // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -110,6 +130,7 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } + void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -117,10 +138,87 @@ struct BitmapEventQueue : public std::queue> { } } + // Push a draw event only if no draw is already pending for this window. + // Returns true if pushed. The caller must pass the shared_ptr to the draw + // event; this function inserts the window into pending_draw_windows_ so + // that BitmapDrawEvent::Execute() can clear it on completion. + bool push_draw(BitmapWindow *window, const value_t &event) { + if (evt_q_mutex_.try_lock()) { + bool pushed = false; + if (pending_draw_windows_.find(window) == + pending_draw_windows_.end()) { + pending_draw_windows_.insert(window); + super::push(event); + pushed = true; + } + evt_q_mutex_.unlock(); + return pushed; + } + return false; + } + + // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a + // redraw posted *during* drawing is not suppressed. + void clear_pending_draw(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // Remove all pending state for a window that is being destroyed. + void remove_window(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // For MOVE/DRAG: replace the last queued event of the same (target, type) + // with the new event (latest absolute position wins; camera controllers + // derive delta from absolute coords so intermediate positions are useless). + // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same + // (target, type) so that the total scroll amount is preserved even when + // multiple notches fire faster than the render loop. + // Falls back to a normal push when no matching event is at the back. + void replace_or_merge_mouse(const value_t &event) { + std::lock_guard lock(evt_q_mutex_); + auto *new_evt = static_cast(event.get()); + if (!super::c.empty()) { + auto *back_mouse = + dynamic_cast(super::c.back().get()); + if (back_mouse && + back_mouse->event_target == new_evt->event_target && + back_mouse->event.type == new_evt->event.type) { + if (new_evt->event.type == MouseEvent::WHEEL) { + // Accumulate scroll deltas; update cursor position and + // other fields to the latest event values. + back_mouse->event.wheel.dx += new_evt->event.wheel.dx; + back_mouse->event.wheel.dy += new_evt->event.wheel.dy; + back_mouse->event.x = new_evt->event.x; + back_mouse->event.y = new_evt->event.y; + back_mouse->event.modifiers = new_evt->event.modifiers; + back_mouse->event.wheel.isTrackpad = + new_evt->event.wheel.isTrackpad; + } else { + // Replace: only the latest absolute position matters. + back_mouse->event = new_evt->event; + } + return; + } + } + super::push(event); + } + private: std::mutex evt_q_mutex_; + // Windows with a draw event currently in the queue (not yet executed). + std::unordered_set pending_draw_windows_; }; +// Out-of-class definition: BitmapEventQueue is now fully defined so +// clear_pending_draw() can be called. +void BitmapDrawEvent::Execute() { + queue_->clear_pending_draw(event_target); + event_target->o3d_window->OnDraw(); +} + } // namespace struct BitmapWindowSystem::Impl { @@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) { while (!filtered_reversed.empty()) { impl_->event_queue_.push(filtered_reversed.pop_front()); } + // Clear any pending-draw entry for this window so the coalescing set + // does not hold a dangling pointer. + impl_->event_queue_.remove_window(the_deceased); // Requiem aeternam dona ei. Requiscat in pace. delete (BitmapWindow *)w; } void BitmapWindowSystem::PostRedrawEvent(OSWindow w) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw)); + // push_draw is a no-op when a draw event is already queued for this + // window, preventing redundant renders from piling up. + impl_->event_queue_.push_draw( + hw, std::make_shared(hw, &impl_->event_queue_)); } void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw, e)); + if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG || + e.type == MouseEvent::WHEEL) { + // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL + // accumulates dx/dy. Only the most recent state matters for rendering. + impl_->event_queue_.replace_or_merge_mouse( + std::make_shared(hw, e)); + } else { + impl_->event_queue_.push(std::make_shared(hw, e)); + } } void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) { diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index c29696c39e2..a3f3f5cbc32 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -195,9 +195,21 @@ PeerConnectionManager::PeerConnectionManager( } return this->HangUp(peerid); }; + + // Start async encoder thread. + encoder_running_ = true; + encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, + this); } -PeerConnectionManager::~PeerConnectionManager() {} +PeerConnectionManager::~PeerConnectionManager() { + // Stop async encoder thread before WebRTC resources are torn down. + encoder_running_ = false; + pending_frames_cv_.notify_all(); + if (encoder_thread_.joinable()) { + encoder_thread_.join(); + } +} // Return deviceList as JSON vector. const Json::Value PeerConnectionManager::GetMediaList() { @@ -729,19 +741,44 @@ void PeerConnectionManager::CloseWindowConnections( } } +// Encoder thread: wakes on each new frame, drains the per-window latest-frame +// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) +// off the render thread so frame delivery never blocks GUI redraws. +void PeerConnectionManager::EncoderThreadLoop() { + while (encoder_running_) { + std::unordered_map> snapshot; + { + std::unique_lock lock(pending_frames_mutex_); + pending_frames_cv_.wait(lock, [this] { + return !pending_frames_.empty() || !encoder_running_; + }); + if (!encoder_running_) break; + // Drain: take all pending frames in one batch; late-arriving frames + // for the same window have already overwritten earlier ones, so we + // encode only the latest per window (implicit frame coalescing). + snapshot = std::move(pending_frames_); + } + for (const auto &kv : snapshot) { + auto video_track_source = GetVideoTrackSource(kv.first); + if (video_track_source && kv.second) { + video_track_source->OnFrame(kv.second); + } + } + } +} + void PeerConnectionManager::OnFrame(const std::string &window_uid, const std::shared_ptr &im) { - // Get the WebRTC stream that corresponds to the window_uid. - // video_track_source is nullptr if the server is running but no client is - // connected. - rtc::scoped_refptr video_track_source = - GetVideoTrackSource(window_uid); - if (video_track_source) { - // TODO: this OnFrame(im); is a blocking call. Do we need to handle - // OnFrame in a separate thread? e.g. attach to a queue of frames, even - // if the queue size is just 1. - video_track_source->OnFrame(im); + // Skip if no peer is connected for this window. + if (!GetVideoTrackSource(window_uid)) return; + // Post the latest frame; overwrites any pending unencoded frame for this + // window (frame coalescing) so the encoder thread always sees the freshest + // content without blocking the render thread. + { + std::lock_guard lock(pending_frames_mutex_); + pending_frames_[window_uid] = im; } + pending_frames_cv_.notify_one(); } } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 0120e20b4fe..72a005e96c4 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -21,6 +21,8 @@ #include #include +#include +#include #include #include #include @@ -451,6 +453,18 @@ class PeerConnectionManager { const std::regex publish_filter_; std::map func_; std::string webrtc_port_range_; + + // Async encoder thread: decouples the render thread from the blocking + // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; + // the thread drains the map and calls video_track_source->OnFrame(). + std::unordered_map> + pending_frames_; + std::mutex pending_frames_mutex_; + std::condition_variable pending_frames_cv_; + std::atomic encoder_running_{false}; + std::thread encoder_thread_; + + void EncoderThreadLoop(); }; } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index 667a7c88be4..ab238201e7d 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,8 +364,10 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - const auto os_window = GetOSWindowByUID(window_uid); - if (os_window) PostRedrawEvent(os_window); + // Do not call PostRedrawEvent here. Mouse/keyboard callbacks + // already schedule a redraw via Window::OnMouseEvent → PostRedraw. + // An extra PostRedrawEvent here creates a duplicate draw event + // before the input event is even processed, causing backlog. return reply; } else { reply = fmt::format( @@ -405,7 +407,7 @@ void WebRTCWindowSystem::OnFrame(const std::string &window_uid, void WebRTCWindowSystem::SendInitFrames(const std::string &window_uid) { utility::LogInfo("Sending init frames to {}.", window_uid); static const int s_max_initial_frames = 5; - static const int s_sleep_between_frames_ms = 100; + static const int s_sleep_between_frames_ms = 50; const auto os_window = GetOSWindowByUID(window_uid); if (!os_window) return; for (int i = 0; os_window != nullptr && i < s_max_initial_frames; ++i) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d7bcd8f6c13..ec4e7e703b0 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -62,6 +62,13 @@ let WebRtcStreamer = (function() { // Open3D-specific functions. this.onClose = onClose; this.commsFetch = commsFetch; + + // Pending coalesced pointer/wheel events. A single requestAnimationFrame + // flushes both at most once per browser frame (~60 Hz), preventing a + // backlog of stale events from queuing up on the server. + this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) + this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) + this.rafPending = false; } const logAndReturn = function(value) { @@ -182,6 +189,26 @@ let WebRtcStreamer = (function() { } }; + // Schedule a requestAnimationFrame flush of pending coalesced events. + // Only one rAF is scheduled at a time; calling again while one is already + // pending is a no-op. When the frame fires, the latest pointer event and + // the accumulated wheel event (if any) are sent and cleared. + WebRtcStreamer.prototype._scheduleRafFlush = function() { + if (this.rafPending) return; + this.rafPending = true; + requestAnimationFrame(() => { + this.rafPending = false; + if (this.pendingPointerEvent !== null) { + this.sendJsonData(this.pendingPointerEvent); + this.pendingPointerEvent = null; + } + if (this.pendingWheelEvent !== null) { + this.sendJsonData(this.pendingWheelEvent); + this.pendingWheelEvent = null; + } + }); + }; + WebRtcStreamer.prototype.addEventListeners = function(windowUID) { if (this.videoElt) { var parentDivElt = this.videoElt.parentElement; @@ -333,7 +360,9 @@ let WebRtcStreamer = (function() { // - Open3D: L=1, M=2, R=4 // - JavaScript: L=1, R=2, M=4 event.preventDefault(); - var open3dMouseEvent = { + // Throttle to one event per animation frame. Only the latest + // absolute position matters; intermediate positions are stale. + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: event.buttons === 0 ? 'MOVE' : 'DRAG', @@ -344,7 +373,7 @@ let WebRtcStreamer = (function() { buttons: event.buttons, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('touchmove', (event) => { // TODO: Known differences. Currently only left-key drag works. @@ -352,7 +381,8 @@ let WebRtcStreamer = (function() { // - JavaScript: L=1, R=2, M=4 event.preventDefault(); var rect = event.target.getBoundingClientRect(); - var open3dMouseEvent = { + // Throttle to one event per animation frame (latest wins). + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: 'DRAG', @@ -363,7 +393,7 @@ let WebRtcStreamer = (function() { buttons: 1, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('mouseleave', (event) => { var open3dMouseEvent = { @@ -397,20 +427,36 @@ let WebRtcStreamer = (function() { dx = dx === 0 ? dx : (-dx / Math.abs(dx)) * 1; dy = dy === 0 ? dy : (-dy / Math.abs(dy)) * 1; - var open3dMouseEvent = { - window_uid: windowUID, - class_name: 'MouseEvent', - type: 'WHEEL', - x: event.offsetX, - y: event.offsetY, - modifiers: WebRtcStreamer._getModifiers(event), - wheel: { - dx: dx, - dy: dy, - isTrackpad: isTrackpad ? 1 : 0, - }, - }; - this.sendJsonData(open3dMouseEvent); + if (this.pendingWheelEvent === null) { + // First wheel event in this frame: create a new pending + // event and schedule a flush. + this.pendingWheelEvent = { + window_uid: windowUID, + class_name: 'MouseEvent', + type: 'WHEEL', + x: event.offsetX, + y: event.offsetY, + modifiers: WebRtcStreamer._getModifiers(event), + wheel: { + dx: dx, + dy: dy, + isTrackpad: isTrackpad ? 1 : 0, + }, + }; + this._scheduleRafFlush(); + } else { + // Subsequent wheel events in the same frame: accumulate + // dx/dy so the total scroll amount is preserved, and + // update cursor position to the latest coordinates. + this.pendingWheelEvent.wheel.dx += dx; + this.pendingWheelEvent.wheel.dy += dy; + this.pendingWheelEvent.x = event.offsetX; + this.pendingWheelEvent.y = event.offsetY; + this.pendingWheelEvent.modifiers = + WebRtcStreamer._getModifiers(event); + this.pendingWheelEvent.wheel.isTrackpad = + isTrackpad ? 1 : 0; + } }, {passive: false}); } }; @@ -464,6 +510,48 @@ let WebRtcStreamer = (function() { this.pc.addStream(stream); } + // Prefer VP9 via the standard setCodecPreferences() API. + // Must be called on the transceiver BEFORE createOffer(). + // For receive-only mode (no local stream), we create the video + // transceiver explicitly — offerToReceiveVideo creates it lazily + // inside createOffer(), which is too late to set preferences. + // Falls back silently to VP8 if the browser does not support the + // API or if VP9 is not in the browser's capabilities. + var pc = this.pc; + if (typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + var videoCaps = RTCRtpReceiver.getCapabilities('video'); + if (videoCaps) { + var vp9Codecs = videoCaps.codecs.filter(function(c) { + return c.mimeType === 'video/VP9'; + }); + if (vp9Codecs.length) { + var preferredCodecs = vp9Codecs.concat( + videoCaps.codecs.filter(function(c) { + return c.mimeType !== 'video/VP9'; + })); + if (stream) { + // Local video being sent: find the transceiver + // addStream() created and apply preferences. + pc.getTransceivers().forEach(function(t) { + if (t.sender && t.sender.track && + t.sender.track.kind === 'video' && + t.setCodecPreferences) { + t.setCodecPreferences(preferredCodecs); + } + }); + } else { + // Receive-only: create transceiver explicitly. + var vt = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (vt.setCodecPreferences) { + vt.setCodecPreferences(preferredCodecs); + } + } + } + } + } + // clear early candidates this.earlyCandidates.length = 0; @@ -599,9 +687,17 @@ let WebRtcStreamer = (function() { } }; - // Local datachannel sends data + // Local datachannel sends data. + // Use unordered, unreliable delivery for mouse/input events. Ordered + // reliable delivery causes head-of-line blocking: a lost packet stalls + // all subsequent events until it is retransmitted. For interactive + // mouse events the latest position is all that matters, so a dropped + // message is better than a delayed one. maxRetransmits: 0 means the + // browser sends once and moves on; ordered: false removes sequencing. try { - this.dataChannel = pc.createDataChannel('ClientDataChannel'); + this.dataChannel = pc.createDataChannel( + 'ClientDataChannel', + {ordered: false, maxRetransmits: 0}); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open'); From a14390e3f831448f9008d55c1ff415b3dc4436c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Wed, 20 May 2026 13:39:00 -0700 Subject: [PATCH 2/6] Explicitly warn and skip writing float textures (unsupported) in gltf. --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 75 ++++++++++++------- .../webrtc_server/PeerConnectionManager.cpp | 8 +- 2 files changed, 51 insertions(+), 32 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 1ce61332bd9..aff2fa3e6b6 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -16,6 +16,7 @@ #include #include +#include "open3d/core/Dtype.h" #include "open3d/core/ParallelFor.h" #include "open3d/core/TensorFunction.h" #include "open3d/t/io/ImageIO.h" @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP( return true; } -static void SetTextureMaterialProperty(aiMaterial* mat, - aiScene* scene, - int texture_idx, - aiTextureType tt, - t::geometry::Image& img) { +namespace { + +// Checks whether a texture map is present, not empty and uint8 or uint16 +// datatype. +bool HasValidTexture(const visualization::rendering::Material& material, + const std::string& key) { + if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) { + return false; + } + const auto& dt = material.GetTextureMap(key).GetDtype(); + if (dt != core::UInt8 && dt != core::UInt16) { + utility::LogWarning( + "Skipping texture map '{}' with unsupported data type '{}'. " + "Only uint8 and uint16 are supported.", + key, dt.ToString()); + return false; + } + return true; +} + +void SetTextureMaterialProperty(aiMaterial* mat, + aiScene* scene, + int texture_idx, + aiTextureType tt, + t::geometry::Image& img) { // Encode image as PNG std::vector img_buffer; WriteImageToPNGInMemory(img_buffer, img, 6); @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat, mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0)); } -namespace { // Add hash function for tuple key struct TupleHash { size_t operator()(const std::tuple& t) const { @@ -551,17 +571,18 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. + const auto& material = w_mesh.GetMaterial(); int n_textures = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures; - if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { + if (HasValidTexture(material, "albedo")) ++n_textures; + if (HasValidTexture(material, "normal")) ++n_textures; + if (HasValidTexture(material, "ao_rough_metal")) { ++n_textures; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { ++n_textures; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures; - if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures; + if (HasValidTexture(material, "roughness")) ++n_textures; + if (HasValidTexture(material, "metallic")) ++n_textures; } if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; @@ -573,23 +594,23 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, // Now embed the textures that are available... int current_idx = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) { - auto img = w_mesh.GetMaterial().GetAlbedoMap(); + if (HasValidTexture(material, "albedo")) { + auto img = material.GetAlbedoMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_DIFFUSE, img); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_BASE_COLOR, img); ++current_idx; } - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); + if (HasValidTexture(material, "ao_rough_metal")) { + auto img = material.GetAORoughnessMetalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap(); - auto metal = w_mesh.GetMaterial().GetMetallicMap(); + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { + auto rough = material.GetRoughnessMap(); + auto metal = material.GetMetallicMap(); auto rows = rough.GetRows(); auto cols = rough.GetCols(); auto rough_metal = @@ -620,21 +641,21 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, aiTextureType_UNKNOWN, rough_metal); ++current_idx; } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) { - auto img = w_mesh.GetMaterial().GetRoughnessMap(); + if (HasValidTexture(material, "roughness")) { + auto img = material.GetRoughnessMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } - if (w_mesh.GetMaterial().HasMetallicMap()) { - auto img = w_mesh.GetMaterial().GetMetallicMap(); + if (HasValidTexture(material, "metallic")) { + auto img = material.GetMetallicMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_UNKNOWN, img); ++current_idx; } } - if (w_mesh.GetMaterial().HasNormalMap()) { - auto img = w_mesh.GetMaterial().GetNormalMap(); + if (HasValidTexture(material, "normal")) { + auto img = material.GetNormalMap(); SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, aiTextureType_NORMALS, img); ++current_idx; diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index a3f3f5cbc32..9d5865d71ff 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -198,17 +198,15 @@ PeerConnectionManager::PeerConnectionManager( // Start async encoder thread. encoder_running_ = true; - encoder_thread_ = std::thread(&PeerConnectionManager::EncoderThreadLoop, - this); + encoder_thread_ = + std::thread(&PeerConnectionManager::EncoderThreadLoop, this); } PeerConnectionManager::~PeerConnectionManager() { // Stop async encoder thread before WebRTC resources are torn down. encoder_running_ = false; pending_frames_cv_.notify_all(); - if (encoder_thread_.joinable()) { - encoder_thread_.join(); - } + encoder_thread_.join(); } // Return deviceList as JSON vector. From 9469dce69204231b49377c5c314bf66dac4d7f66 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 13:49:11 -0700 Subject: [PATCH 3/6] Pin license-webpack-plugin - windows wheel build error. --- python/js/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/js/package.json b/python/js/package.json index d364052ec22..5556ea5df4a 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,7 +37,8 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6" + "abab": "^2.0.6", + "license-webpack-plugin": "^4.0.0" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", From 99b96d2bfaf0d4508f9efe4b9c2b507556f1df96 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 21 May 2026 16:11:14 -0700 Subject: [PATCH 4/6] [WebRTC] Optimize latency by adjusting pacing and jitter buffer settings sends zero playout delay in every RTP packet's header extension when adaptation is necessary, drops resolution rather than framerate prevents WebRTC's autonomous resolution scaling; Replaced playoutDelayHint with jitterBufferTarget = 0 --- .../webrtc_server/PeerConnectionManager.cpp | 23 +++++++++++++++++++ .../webrtc_server/html/webrtcstreamer.js | 21 +++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index 9d5865d71ff..68ff8256e29 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,18 @@ static IceServer GetIceServerFromUrl(const std::string &url) { static webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies() { + // Disable WebRTC's pacing delay and allow the pacer to drain its queue + // immediately. This reduces the added latency from packet smoothing, which + // is unnecessary for a local interactive streaming use case. + // Force playout delay to zero in the RTP header extension so the receiver + // renders frames immediately rather than buffering for jitter smoothing. + // Disable automatic resolution reduction so quality is only degraded if the + // encoder is genuinely overloaded (content hint kFluid still allows it). + webrtc::field_trial::InitFieldTrialsFromString( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/"); + webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = nullptr; dependencies.worker_thread = rtc::Thread::Current(); @@ -531,6 +544,10 @@ bool PeerConnectionManager::InitializePeerConnection() { PeerConnectionManager::PeerConnectionObserver * PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; + // Max bundle multiplexes all media and data channels on a single transport, + // eliminating separate ICE/DTLS handshakes per track and reducing latency. + config.bundle_policy = + webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; for (auto ice_server : ice_server_list_) { webrtc::PeerConnectionInterface::IceServer server; IceServer srv = GetIceServerFromUrl(ice_server); @@ -664,6 +681,12 @@ bool PeerConnectionManager::AddStreams( opts); video_track = peer_connection_factory_->CreateVideoTrack( window_uid + "_video", videoScaled); + // Prefer framerate over resolution when the encoder + // is under pressure (bandwidth or CPU constrained). + // For interactive 3D rendering, motion smoothness + // matters more than pixel-perfect resolution. + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); } if ((video_track) && (!stream->AddTrack(video_track))) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index ec4e7e703b0..45dea6bf245 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -677,11 +677,22 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { - if (recv.track && recv.track.kind === 'video' && - typeof recv.getParameters != 'undefined') { - console.log( - 'codecs:' + - JSON.stringify(recv.getParameters().codecs)); + if (recv.track && recv.track.kind === 'video') { + // Minimize browser jitter buffer to reduce playout + // latency. The server already sends RTP playout-delay + // header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial, but set + // jitterBufferTarget as well for browsers that honour + // the JS API over the in-band RTP extension. + if (typeof recv.jitterBufferTarget !== 'undefined') { + recv.jitterBufferTarget = 0; + } + if (typeof recv.getParameters != 'undefined') { + console.log( + 'codecs:' + + JSON.stringify( + recv.getParameters().codecs)); + } } }); } From 1a46e95ad7b273a8a6933230366bc01edebf99b8 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Thu, 21 May 2026 23:49:26 -0700 Subject: [PATCH 5/6] undo non-working windows jupyterlab fix. --- python/js/package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python/js/package.json b/python/js/package.json index 5556ea5df4a..0e2f7d78069 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -37,8 +37,7 @@ }, "resolutions": { "glob": "7.2.3", - "abab": "^2.0.6", - "license-webpack-plugin": "^4.0.0" + "abab": "^2.0.6" }, "dependencies": { "@jupyter-widgets/base": "^2 || ^3 || ^4 || ^5 || ^6", @@ -55,4 +54,4 @@ } } } -} \ No newline at end of file +} From 84f58be2feed823364588b2fd29472e2362f64c1 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sat, 23 May 2026 15:29:26 -0700 Subject: [PATCH 6/6] [WebRTC] Refactor texture handling in ASSIMP export and improve data channel reliability --- cpp/open3d/t/io/file_format/FileASSIMP.cpp | 148 +++++++++--------- .../webrtc_server/WebRTCWindowSystem.cpp | 6 +- .../webrtc_server/html/webrtcstreamer.js | 17 +- 3 files changed, 84 insertions(+), 87 deletions(-) diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 768ac51eca8..9cae51c804f 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -566,97 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE); } - // Count texture maps... + // Build a list of texture-embed actions in a single pass. Each action + // is a lambda that writes one texture slot into the assimp scene; the + // slot index is passed at call time. . + // // NOTE: GLTF2 expects a single combined roughness/metal map. If the // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. const auto& material = w_mesh.GetMaterial(); - int n_textures = 0; - if (HasValidTexture(material, "albedo")) ++n_textures; - if (HasValidTexture(material, "normal")) ++n_textures; - if (HasValidTexture(material, "ambient_occlusion")) ++n_textures; + using TextureAction = std::function; + std::vector texture_actions; + + if (HasValidTexture(material, "albedo")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAlbedoMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_DIFFUSE, img); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_BASE_COLOR, img); + }); + } + if (HasValidTexture(material, "ambient_occlusion")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAOMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_LIGHTMAP, img); + }); + } if (HasValidTexture(material, "ao_rough_metal")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto img = material.GetAORoughnessMetalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); } else if (HasValidTexture(material, "roughness") && HasValidTexture(material, "metallic")) { - ++n_textures; + texture_actions.push_back([&](int idx) { + auto rough = material.GetRoughnessMap().AsTensor(); + auto metal = material.GetMetallicMap().AsTensor(); + if (rough.GetShape() != metal.GetShape()) { + utility::LogError( + "RoughnessMap (shape={}) and MetallicMap " + "(shape={}) must have the same shape.", + rough.GetShape(), metal.GetShape()); + } + auto rows = rough.GetShape(0); + auto cols = rough.GetShape(1); + auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, + core::Dtype::UInt8); + rough_metal.Slice(2, 2, 3) = + metal.Slice(2, 0, 1); // blue channel is metal + rough_metal.Slice(2, 1, 2) = + rough.Slice(2, 0, 1); // green channel is roughness + geometry::Image rough_metal_img(rough_metal); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, + rough_metal_img); + }); } else { - if (HasValidTexture(material, "roughness")) ++n_textures; - if (HasValidTexture(material, "metallic")) ++n_textures; + if (HasValidTexture(material, "roughness")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetRoughnessMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + if (HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetMetallicMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } } + if (HasValidTexture(material, "normal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetNormalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_NORMALS, img); + }); + } + + int n_textures = static_cast(texture_actions.size()); if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; for (int i = 0; i < n_textures; ++i) { ai_scene->mTextures[i] = new aiTexture(); + texture_actions[i](i); } ai_scene->mNumTextures = n_textures; } - - // Now embed the textures that are available... - int current_idx = 0; - if (HasValidTexture(material, "albedo")) { - auto img = material.GetAlbedoMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_DIFFUSE, img); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_BASE_COLOR, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ambient_occlusion")) { - auto img = w_mesh.GetMaterial().GetAOMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_LIGHTMAP, img); - ++current_idx; - } - if (HasValidTexture(w_mesh.GetMaterial(), "ao_rough_metal")) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } else if (HasValidTexture(w_mesh.GetMaterial(), "roughness") && - HasValidTexture(w_mesh.GetMaterial(), "metallic")) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor(); - auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor(); - if (rough.GetShape() != metal.GetShape()) { - utility::LogError( - "RoughnessMap (shape={}) and MetallicMap (shape={}) " - "must have the same shape.", - rough.GetShape(), metal.GetShape()); - } - auto rows = rough.GetShape(0); - auto cols = rough.GetShape(1); - auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, - core::Dtype::UInt8); - rough_metal.Slice(2, 2, 3) = - metal.Slice(2, 0, 1); // blue channel is metal - rough_metal.Slice(2, 1, 2) = - rough.Slice(2, 0, 1); // green channel is roughness - - geometry::Image rough_metal_img(rough_metal); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, rough_metal_img); - ++current_idx; - } else { - if (HasValidTexture(material, "roughness")) { - auto img = material.GetRoughnessMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - if (HasValidTexture(material, "metallic")) { - auto img = material.GetMetallicMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - } - if (HasValidTexture(material, "normal")) { - auto img = material.GetNormalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_NORMALS, img); - ++current_idx; - } } ai_scene->mMaterials[0] = ai_mat; diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index ab238201e7d..c703ac5f66a 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,10 +364,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - // Do not call PostRedrawEvent here. Mouse/keyboard callbacks - // already schedule a redraw via Window::OnMouseEvent → PostRedraw. - // An extra PostRedrawEvent here creates a duplicate draw event - // before the input event is even processed, causing backlog. + // Custom callbacks that mutate GUI state (e.g. add/remove geometry) + // must call window->PostRedraw() or post_redraw() themselves. return reply; } else { reply = fmt::format( diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index 45dea6bf245..e3c385d6fb4 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -699,16 +699,15 @@ let WebRtcStreamer = (function() { }; // Local datachannel sends data. - // Use unordered, unreliable delivery for mouse/input events. Ordered - // reliable delivery causes head-of-line blocking: a lost packet stalls - // all subsequent events until it is retransmitted. For interactive - // mouse events the latest position is all that matters, so a dropped - // message is better than a delayed one. maxRetransmits: 0 means the - // browser sends once and moves on; ordered: false removes sequencing. + // Use reliable ordered delivery (the default). While unordered + + // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, + // the same channel carries discrete events (BUTTON_DOWN/UP, key events, + // resize) and application-level RPC (tensorboard/update_geometry etc.) + // that must not be lost or reordered. Mouse coalescing (rAF + server- + // side replace_or_merge_mouse) already prevents event backlog, so the + // extra reliability is free in practice on a local network. try { - this.dataChannel = pc.createDataChannel( - 'ClientDataChannel', - {ordered: false, maxRetransmits: 0}); + this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; dataChannel.onopen = function() { console.log('local datachannel open');