diff --git a/cpp/open3d/t/io/file_format/FileASSIMP.cpp b/cpp/open3d/t/io/file_format/FileASSIMP.cpp index 383483a1874..9cae51c804f 100644 --- a/cpp/open3d/t/io/file_format/FileASSIMP.cpp +++ b/cpp/open3d/t/io/file_format/FileASSIMP.cpp @@ -16,6 +16,7 @@ #include #include +#include "open3d/core/Dtype.h" #include "open3d/core/ParallelFor.h" #include "open3d/core/TensorFunction.h" #include "open3d/t/io/ImageIO.h" @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP( return true; } -static void SetTextureMaterialProperty(aiMaterial* mat, - aiScene* scene, - int texture_idx, - aiTextureType tt, - t::geometry::Image& img) { +namespace { + +// Checks whether a texture map is present, not empty and uint8 or uint16 +// datatype. +bool HasValidTexture(const visualization::rendering::Material& material, + const std::string& key) { + if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) { + return false; + } + const auto& dt = material.GetTextureMap(key).GetDtype(); + if (dt != core::UInt8 && dt != core::UInt16) { + utility::LogWarning( + "Skipping texture map '{}' with unsupported data type '{}'. " + "Only uint8 and uint16 are supported.", + key, dt.ToString()); + return false; + } + return true; +} + +void SetTextureMaterialProperty(aiMaterial* mat, + aiScene* scene, + int texture_idx, + aiTextureType tt, + t::geometry::Image& img) { // Encode image as PNG std::vector img_buffer; WriteImageToPNGInMemory(img_buffer, img, 6); @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat, mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0)); } -namespace { // Add hash function for tuple key struct TupleHash { size_t operator()(const std::tuple& t) const { @@ -546,96 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename, ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE); } - // Count texture maps... + // Build a list of texture-embed actions in a single pass. Each action + // is a lambda that writes one texture slot into the assimp scene; the + // slot index is passed at call time. . + // // NOTE: GLTF2 expects a single combined roughness/metal map. If the // model has one we just export it, otherwise if both roughness and // metal maps are available we combine them, otherwise if only one or // the other is available we just export the one map. - int n_textures = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures; - if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAOMap()) ++n_textures; - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - ++n_textures; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - ++n_textures; + const auto& material = w_mesh.GetMaterial(); + using TextureAction = std::function; + std::vector texture_actions; + + if (HasValidTexture(material, "albedo")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAlbedoMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_DIFFUSE, img); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_BASE_COLOR, img); + }); + } + if (HasValidTexture(material, "ambient_occlusion")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAOMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_LIGHTMAP, img); + }); + } + if (HasValidTexture(material, "ao_rough_metal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetAORoughnessMetalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } else if (HasValidTexture(material, "roughness") && + HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto rough = material.GetRoughnessMap().AsTensor(); + auto metal = material.GetMetallicMap().AsTensor(); + if (rough.GetShape() != metal.GetShape()) { + utility::LogError( + "RoughnessMap (shape={}) and MetallicMap " + "(shape={}) must have the same shape.", + rough.GetShape(), metal.GetShape()); + } + auto rows = rough.GetShape(0); + auto cols = rough.GetShape(1); + auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, + core::Dtype::UInt8); + rough_metal.Slice(2, 2, 3) = + metal.Slice(2, 0, 1); // blue channel is metal + rough_metal.Slice(2, 1, 2) = + rough.Slice(2, 0, 1); // green channel is roughness + geometry::Image rough_metal_img(rough_metal); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, + rough_metal_img); + }); } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures; - if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures; + if (HasValidTexture(material, "roughness")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetRoughnessMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + if (HasValidTexture(material, "metallic")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetMetallicMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_UNKNOWN, img); + }); + } + } + if (HasValidTexture(material, "normal")) { + texture_actions.push_back([&](int idx) { + auto img = material.GetNormalMap(); + SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx, + aiTextureType_NORMALS, img); + }); } + + int n_textures = static_cast(texture_actions.size()); if (n_textures > 0) { ai_scene->mTextures = new aiTexture*[n_textures]; for (int i = 0; i < n_textures; ++i) { ai_scene->mTextures[i] = new aiTexture(); + texture_actions[i](i); } ai_scene->mNumTextures = n_textures; } - - // Now embed the textures that are available... - int current_idx = 0; - if (w_mesh.GetMaterial().HasAlbedoMap()) { - auto img = w_mesh.GetMaterial().GetAlbedoMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_DIFFUSE, img); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_BASE_COLOR, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasAOMap()) { - auto img = w_mesh.GetMaterial().GetAOMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_LIGHTMAP, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) { - auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } else if (w_mesh.GetMaterial().HasRoughnessMap() && - w_mesh.GetMaterial().HasMetallicMap()) { - auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor(); - auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor(); - if (rough.GetShape() != metal.GetShape()) { - utility::LogError( - "RoughnessMap (shape={}) and MetallicMap (shape={}) " - "must have the same shape.", - rough.GetShape(), metal.GetShape()); - } - auto rows = rough.GetShape(0); - auto cols = rough.GetShape(1); - auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255, - core::Dtype::UInt8); - rough_metal.Slice(2, 2, 3) = - metal.Slice(2, 0, 1); // blue channel is metal - rough_metal.Slice(2, 1, 2) = - rough.Slice(2, 0, 1); // green channel is roughness - - geometry::Image rough_metal_img(rough_metal); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, rough_metal_img); - ++current_idx; - } else { - if (w_mesh.GetMaterial().HasRoughnessMap()) { - auto img = w_mesh.GetMaterial().GetRoughnessMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - if (w_mesh.GetMaterial().HasMetallicMap()) { - auto img = w_mesh.GetMaterial().GetMetallicMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_UNKNOWN, img); - ++current_idx; - } - } - if (w_mesh.GetMaterial().HasNormalMap()) { - auto img = w_mesh.GetMaterial().GetNormalMap(); - SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx, - aiTextureType_NORMALS, img); - ++current_idx; - } } ai_scene->mMaterials[0] = ai_mat; diff --git a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp index 3db9d0cefef..b73d680b381 100644 --- a/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp +++ b/cpp/open3d/visualization/gui/BitmapWindowSystem.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include "open3d/geometry/Image.h" #include "open3d/utility/Logging.h" @@ -44,10 +45,21 @@ struct BitmapEvent { virtual void Execute() = 0; }; +// Forward declaration: BitmapDrawEvent needs a pointer to BitmapEventQueue +// to clear the per-window pending-draw flag before calling OnDraw(). +struct BitmapEventQueue; + struct BitmapDrawEvent : public BitmapEvent { - BitmapDrawEvent(BitmapWindow *target) : BitmapEvent(target) {} + // queue_ is used to clear the pending-draw flag before OnDraw() so that + // a PostRedraw() called *inside* OnDraw() is not suppressed. + BitmapEventQueue *queue_; + + BitmapDrawEvent(BitmapWindow *target, BitmapEventQueue *queue) + : BitmapEvent(target), queue_(queue) {} - void Execute() override { event_target->o3d_window->OnDraw(); } + // Execute() is defined after BitmapEventQueue (below) because it calls + // clear_pending_draw(), which requires the full BitmapEventQueue type. + void Execute() override; }; struct BitmapResizeEvent : public BitmapEvent { @@ -93,15 +105,23 @@ struct BitmapTextInputEvent : public BitmapEvent { } }; -/// Thread safe event queue (multiple producers and consumers). pop_front() and -/// push() are protected by a mutex. push() may fail if the mutex cannot be -/// acquired immediately. empty() is not protected and is not reliable. +/// Thread safe event queue (multiple producers and consumers). +/// pop_front() and push() are protected by a mutex. +/// push() may fail if the mutex cannot be acquired immediately. +/// empty() is not protected and is not reliable. +/// +/// Extended to support: +/// - Draw coalescing: at most one pending draw per window (push_draw). +/// - Input coalescing: MOVE/DRAG replace latest, WHEEL accumulates +/// dx/dy (replace_or_merge_mouse). Old mouse positions are stale; +/// processing them forces stale frames to be encoded and sent. struct BitmapEventQueue : public std::queue> { using value_t = std::shared_ptr; using super = std::queue; using super::empty; // not reliable using super::super; + // pop + front needs to be atomic for thread safety. This is exception safe // since shared_ptr copy ctor is noexcept, when it is returned by value. value_t pop_front() { @@ -110,6 +130,7 @@ struct BitmapEventQueue : public std::queue> { super::pop(); return evt; } + void push(const value_t &event) { if (evt_q_mutex_.try_lock()) { super::push(event); @@ -117,10 +138,87 @@ struct BitmapEventQueue : public std::queue> { } } + // Push a draw event only if no draw is already pending for this window. + // Returns true if pushed. The caller must pass the shared_ptr to the draw + // event; this function inserts the window into pending_draw_windows_ so + // that BitmapDrawEvent::Execute() can clear it on completion. + bool push_draw(BitmapWindow *window, const value_t &event) { + if (evt_q_mutex_.try_lock()) { + bool pushed = false; + if (pending_draw_windows_.find(window) == + pending_draw_windows_.end()) { + pending_draw_windows_.insert(window); + super::push(event); + pushed = true; + } + evt_q_mutex_.unlock(); + return pushed; + } + return false; + } + + // Called by BitmapDrawEvent::Execute() just before OnDraw() so that a + // redraw posted *during* drawing is not suppressed. + void clear_pending_draw(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // Remove all pending state for a window that is being destroyed. + void remove_window(BitmapWindow *window) { + std::lock_guard lock(evt_q_mutex_); + pending_draw_windows_.erase(window); + } + + // For MOVE/DRAG: replace the last queued event of the same (target, type) + // with the new event (latest absolute position wins; camera controllers + // derive delta from absolute coords so intermediate positions are useless). + // For WHEEL: accumulate wheel.dx/dy into the last queued event of the same + // (target, type) so that the total scroll amount is preserved even when + // multiple notches fire faster than the render loop. + // Falls back to a normal push when no matching event is at the back. + void replace_or_merge_mouse(const value_t &event) { + std::lock_guard lock(evt_q_mutex_); + auto *new_evt = static_cast(event.get()); + if (!super::c.empty()) { + auto *back_mouse = + dynamic_cast(super::c.back().get()); + if (back_mouse && + back_mouse->event_target == new_evt->event_target && + back_mouse->event.type == new_evt->event.type) { + if (new_evt->event.type == MouseEvent::WHEEL) { + // Accumulate scroll deltas; update cursor position and + // other fields to the latest event values. + back_mouse->event.wheel.dx += new_evt->event.wheel.dx; + back_mouse->event.wheel.dy += new_evt->event.wheel.dy; + back_mouse->event.x = new_evt->event.x; + back_mouse->event.y = new_evt->event.y; + back_mouse->event.modifiers = new_evt->event.modifiers; + back_mouse->event.wheel.isTrackpad = + new_evt->event.wheel.isTrackpad; + } else { + // Replace: only the latest absolute position matters. + back_mouse->event = new_evt->event; + } + return; + } + } + super::push(event); + } + private: std::mutex evt_q_mutex_; + // Windows with a draw event currently in the queue (not yet executed). + std::unordered_set pending_draw_windows_; }; +// Out-of-class definition: BitmapEventQueue is now fully defined so +// clear_pending_draw() can be called. +void BitmapDrawEvent::Execute() { + queue_->clear_pending_draw(event_target); + event_target->o3d_window->OnDraw(); +} + } // namespace struct BitmapWindowSystem::Impl { @@ -187,18 +285,32 @@ void BitmapWindowSystem::DestroyWindow(OSWindow w) { while (!filtered_reversed.empty()) { impl_->event_queue_.push(filtered_reversed.pop_front()); } + // Clear any pending-draw entry for this window so the coalescing set + // does not hold a dangling pointer. + impl_->event_queue_.remove_window(the_deceased); // Requiem aeternam dona ei. Requiscat in pace. delete (BitmapWindow *)w; } void BitmapWindowSystem::PostRedrawEvent(OSWindow w) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw)); + // push_draw is a no-op when a draw event is already queued for this + // window, preventing redundant renders from piling up. + impl_->event_queue_.push_draw( + hw, std::make_shared(hw, &impl_->event_queue_)); } void BitmapWindowSystem::PostMouseEvent(OSWindow w, const MouseEvent &e) { auto hw = (BitmapWindow *)w; - impl_->event_queue_.push(std::make_shared(hw, e)); + if (e.type == MouseEvent::MOVE || e.type == MouseEvent::DRAG || + e.type == MouseEvent::WHEEL) { + // Coalesce: MOVE/DRAG replace latest (absolute position); WHEEL + // accumulates dx/dy. Only the most recent state matters for rendering. + impl_->event_queue_.replace_or_merge_mouse( + std::make_shared(hw, e)); + } else { + impl_->event_queue_.push(std::make_shared(hw, e)); + } } void BitmapWindowSystem::PostKeyEvent(OSWindow w, const KeyEvent &e) { diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp index c29696c39e2..68ff8256e29 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,18 @@ static IceServer GetIceServerFromUrl(const std::string &url) { static webrtc::PeerConnectionFactoryDependencies CreatePeerConnectionFactoryDependencies() { + // Disable WebRTC's pacing delay and allow the pacer to drain its queue + // immediately. This reduces the added latency from packet smoothing, which + // is unnecessary for a local interactive streaming use case. + // Force playout delay to zero in the RTP header extension so the receiver + // renders frames immediately rather than buffering for jitter smoothing. + // Disable automatic resolution reduction so quality is only degraded if the + // encoder is genuinely overloaded (content hint kFluid still allows it). + webrtc::field_trial::InitFieldTrialsFromString( + "WebRTC-Pacer-DrainQueue/Enabled/" + "WebRTC-ForceSendPlayoutDelay/min_ms:0,max_ms:0/" + "WebRTC-Video-DisableAutomaticResize/Enabled/"); + webrtc::PeerConnectionFactoryDependencies dependencies; dependencies.network_thread = nullptr; dependencies.worker_thread = rtc::Thread::Current(); @@ -195,9 +208,19 @@ PeerConnectionManager::PeerConnectionManager( } return this->HangUp(peerid); }; + + // Start async encoder thread. + encoder_running_ = true; + encoder_thread_ = + std::thread(&PeerConnectionManager::EncoderThreadLoop, this); } -PeerConnectionManager::~PeerConnectionManager() {} +PeerConnectionManager::~PeerConnectionManager() { + // Stop async encoder thread before WebRTC resources are torn down. + encoder_running_ = false; + pending_frames_cv_.notify_all(); + encoder_thread_.join(); +} // Return deviceList as JSON vector. const Json::Value PeerConnectionManager::GetMediaList() { @@ -521,6 +544,10 @@ bool PeerConnectionManager::InitializePeerConnection() { PeerConnectionManager::PeerConnectionObserver * PeerConnectionManager::CreatePeerConnection(const std::string &peerid) { webrtc::PeerConnectionInterface::RTCConfiguration config; + // Max bundle multiplexes all media and data channels on a single transport, + // eliminating separate ICE/DTLS handshakes per track and reducing latency. + config.bundle_policy = + webrtc::PeerConnectionInterface::kBundlePolicyMaxBundle; for (auto ice_server : ice_server_list_) { webrtc::PeerConnectionInterface::IceServer server; IceServer srv = GetIceServerFromUrl(ice_server); @@ -654,6 +681,12 @@ bool PeerConnectionManager::AddStreams( opts); video_track = peer_connection_factory_->CreateVideoTrack( window_uid + "_video", videoScaled); + // Prefer framerate over resolution when the encoder + // is under pressure (bandwidth or CPU constrained). + // For interactive 3D rendering, motion smoothness + // matters more than pixel-perfect resolution. + video_track->set_content_hint( + webrtc::VideoTrackInterface::ContentHint::kFluid); } if ((video_track) && (!stream->AddTrack(video_track))) { @@ -729,19 +762,44 @@ void PeerConnectionManager::CloseWindowConnections( } } +// Encoder thread: wakes on each new frame, drains the per-window latest-frame +// map, and calls video_track_source->OnFrame() (libyuv + WebRTC encode) +// off the render thread so frame delivery never blocks GUI redraws. +void PeerConnectionManager::EncoderThreadLoop() { + while (encoder_running_) { + std::unordered_map> snapshot; + { + std::unique_lock lock(pending_frames_mutex_); + pending_frames_cv_.wait(lock, [this] { + return !pending_frames_.empty() || !encoder_running_; + }); + if (!encoder_running_) break; + // Drain: take all pending frames in one batch; late-arriving frames + // for the same window have already overwritten earlier ones, so we + // encode only the latest per window (implicit frame coalescing). + snapshot = std::move(pending_frames_); + } + for (const auto &kv : snapshot) { + auto video_track_source = GetVideoTrackSource(kv.first); + if (video_track_source && kv.second) { + video_track_source->OnFrame(kv.second); + } + } + } +} + void PeerConnectionManager::OnFrame(const std::string &window_uid, const std::shared_ptr &im) { - // Get the WebRTC stream that corresponds to the window_uid. - // video_track_source is nullptr if the server is running but no client is - // connected. - rtc::scoped_refptr video_track_source = - GetVideoTrackSource(window_uid); - if (video_track_source) { - // TODO: this OnFrame(im); is a blocking call. Do we need to handle - // OnFrame in a separate thread? e.g. attach to a queue of frames, even - // if the queue size is just 1. - video_track_source->OnFrame(im); + // Skip if no peer is connected for this window. + if (!GetVideoTrackSource(window_uid)) return; + // Post the latest frame; overwrites any pending unencoded frame for this + // window (frame coalescing) so the encoder thread always sees the freshest + // content without blocking the render thread. + { + std::lock_guard lock(pending_frames_mutex_); + pending_frames_[window_uid] = im; } + pending_frames_cv_.notify_one(); } } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h index 0120e20b4fe..72a005e96c4 100644 --- a/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h +++ b/cpp/open3d/visualization/webrtc_server/PeerConnectionManager.h @@ -21,6 +21,8 @@ #include #include +#include +#include #include #include #include @@ -451,6 +453,18 @@ class PeerConnectionManager { const std::regex publish_filter_; std::map func_; std::string webrtc_port_range_; + + // Async encoder thread: decouples the render thread from the blocking + // libyuv + WebRTC encode path. OnFrame() posts the latest frame per window; + // the thread drains the map and calls video_track_source->OnFrame(). + std::unordered_map> + pending_frames_; + std::mutex pending_frames_mutex_; + std::condition_variable pending_frames_cv_; + std::atomic encoder_running_{false}; + std::thread encoder_thread_; + + void EncoderThreadLoop(); }; } // namespace webrtc_server diff --git a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp index 667a7c88be4..c703ac5f66a 100644 --- a/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp +++ b/cpp/open3d/visualization/webrtc_server/WebRTCWindowSystem.cpp @@ -364,8 +364,8 @@ std::string WebRTCWindowSystem::OnDataChannelMessage( if (impl_->data_channel_message_callbacks_.count(class_name) != 0) { reply = impl_->data_channel_message_callbacks_.at(class_name)( message); - const auto os_window = GetOSWindowByUID(window_uid); - if (os_window) PostRedrawEvent(os_window); + // Custom callbacks that mutate GUI state (e.g. add/remove geometry) + // must call window->PostRedraw() or post_redraw() themselves. return reply; } else { reply = fmt::format( @@ -405,7 +405,7 @@ void WebRTCWindowSystem::OnFrame(const std::string &window_uid, void WebRTCWindowSystem::SendInitFrames(const std::string &window_uid) { utility::LogInfo("Sending init frames to {}.", window_uid); static const int s_max_initial_frames = 5; - static const int s_sleep_between_frames_ms = 100; + static const int s_sleep_between_frames_ms = 50; const auto os_window = GetOSWindowByUID(window_uid); if (!os_window) return; for (int i = 0; os_window != nullptr && i < s_max_initial_frames; ++i) { diff --git a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js index d7bcd8f6c13..e3c385d6fb4 100755 --- a/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js +++ b/cpp/open3d/visualization/webrtc_server/html/webrtcstreamer.js @@ -62,6 +62,13 @@ let WebRtcStreamer = (function() { // Open3D-specific functions. this.onClose = onClose; this.commsFetch = commsFetch; + + // Pending coalesced pointer/wheel events. A single requestAnimationFrame + // flushes both at most once per browser frame (~60 Hz), preventing a + // backlog of stale events from queuing up on the server. + this.pendingPointerEvent = null; // MOVE or DRAG (latest wins) + this.pendingWheelEvent = null; // WHEEL (dx/dy accumulated) + this.rafPending = false; } const logAndReturn = function(value) { @@ -182,6 +189,26 @@ let WebRtcStreamer = (function() { } }; + // Schedule a requestAnimationFrame flush of pending coalesced events. + // Only one rAF is scheduled at a time; calling again while one is already + // pending is a no-op. When the frame fires, the latest pointer event and + // the accumulated wheel event (if any) are sent and cleared. + WebRtcStreamer.prototype._scheduleRafFlush = function() { + if (this.rafPending) return; + this.rafPending = true; + requestAnimationFrame(() => { + this.rafPending = false; + if (this.pendingPointerEvent !== null) { + this.sendJsonData(this.pendingPointerEvent); + this.pendingPointerEvent = null; + } + if (this.pendingWheelEvent !== null) { + this.sendJsonData(this.pendingWheelEvent); + this.pendingWheelEvent = null; + } + }); + }; + WebRtcStreamer.prototype.addEventListeners = function(windowUID) { if (this.videoElt) { var parentDivElt = this.videoElt.parentElement; @@ -333,7 +360,9 @@ let WebRtcStreamer = (function() { // - Open3D: L=1, M=2, R=4 // - JavaScript: L=1, R=2, M=4 event.preventDefault(); - var open3dMouseEvent = { + // Throttle to one event per animation frame. Only the latest + // absolute position matters; intermediate positions are stale. + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: event.buttons === 0 ? 'MOVE' : 'DRAG', @@ -344,7 +373,7 @@ let WebRtcStreamer = (function() { buttons: event.buttons, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('touchmove', (event) => { // TODO: Known differences. Currently only left-key drag works. @@ -352,7 +381,8 @@ let WebRtcStreamer = (function() { // - JavaScript: L=1, R=2, M=4 event.preventDefault(); var rect = event.target.getBoundingClientRect(); - var open3dMouseEvent = { + // Throttle to one event per animation frame (latest wins). + this.pendingPointerEvent = { window_uid: windowUID, class_name: 'MouseEvent', type: 'DRAG', @@ -363,7 +393,7 @@ let WebRtcStreamer = (function() { buttons: 1, // MouseButtons ORed together }, }; - this.sendJsonData(open3dMouseEvent); + this._scheduleRafFlush(); }, false); this.videoElt.addEventListener('mouseleave', (event) => { var open3dMouseEvent = { @@ -397,20 +427,36 @@ let WebRtcStreamer = (function() { dx = dx === 0 ? dx : (-dx / Math.abs(dx)) * 1; dy = dy === 0 ? dy : (-dy / Math.abs(dy)) * 1; - var open3dMouseEvent = { - window_uid: windowUID, - class_name: 'MouseEvent', - type: 'WHEEL', - x: event.offsetX, - y: event.offsetY, - modifiers: WebRtcStreamer._getModifiers(event), - wheel: { - dx: dx, - dy: dy, - isTrackpad: isTrackpad ? 1 : 0, - }, - }; - this.sendJsonData(open3dMouseEvent); + if (this.pendingWheelEvent === null) { + // First wheel event in this frame: create a new pending + // event and schedule a flush. + this.pendingWheelEvent = { + window_uid: windowUID, + class_name: 'MouseEvent', + type: 'WHEEL', + x: event.offsetX, + y: event.offsetY, + modifiers: WebRtcStreamer._getModifiers(event), + wheel: { + dx: dx, + dy: dy, + isTrackpad: isTrackpad ? 1 : 0, + }, + }; + this._scheduleRafFlush(); + } else { + // Subsequent wheel events in the same frame: accumulate + // dx/dy so the total scroll amount is preserved, and + // update cursor position to the latest coordinates. + this.pendingWheelEvent.wheel.dx += dx; + this.pendingWheelEvent.wheel.dy += dy; + this.pendingWheelEvent.x = event.offsetX; + this.pendingWheelEvent.y = event.offsetY; + this.pendingWheelEvent.modifiers = + WebRtcStreamer._getModifiers(event); + this.pendingWheelEvent.wheel.isTrackpad = + isTrackpad ? 1 : 0; + } }, {passive: false}); } }; @@ -464,6 +510,48 @@ let WebRtcStreamer = (function() { this.pc.addStream(stream); } + // Prefer VP9 via the standard setCodecPreferences() API. + // Must be called on the transceiver BEFORE createOffer(). + // For receive-only mode (no local stream), we create the video + // transceiver explicitly — offerToReceiveVideo creates it lazily + // inside createOffer(), which is too late to set preferences. + // Falls back silently to VP8 if the browser does not support the + // API or if VP9 is not in the browser's capabilities. + var pc = this.pc; + if (typeof RTCRtpReceiver !== 'undefined' && + RTCRtpReceiver.getCapabilities && pc.addTransceiver) { + var videoCaps = RTCRtpReceiver.getCapabilities('video'); + if (videoCaps) { + var vp9Codecs = videoCaps.codecs.filter(function(c) { + return c.mimeType === 'video/VP9'; + }); + if (vp9Codecs.length) { + var preferredCodecs = vp9Codecs.concat( + videoCaps.codecs.filter(function(c) { + return c.mimeType !== 'video/VP9'; + })); + if (stream) { + // Local video being sent: find the transceiver + // addStream() created and apply preferences. + pc.getTransceivers().forEach(function(t) { + if (t.sender && t.sender.track && + t.sender.track.kind === 'video' && + t.setCodecPreferences) { + t.setCodecPreferences(preferredCodecs); + } + }); + } else { + // Receive-only: create transceiver explicitly. + var vt = pc.addTransceiver( + 'video', {direction: 'recvonly'}); + if (vt.setCodecPreferences) { + vt.setCodecPreferences(preferredCodecs); + } + } + } + } + } + // clear early candidates this.earlyCandidates.length = 0; @@ -589,17 +677,35 @@ let WebRtcStreamer = (function() { const recvs = pc.getReceivers(); recvs.forEach((recv) => { - if (recv.track && recv.track.kind === 'video' && - typeof recv.getParameters != 'undefined') { - console.log( - 'codecs:' + - JSON.stringify(recv.getParameters().codecs)); + if (recv.track && recv.track.kind === 'video') { + // Minimize browser jitter buffer to reduce playout + // latency. The server already sends RTP playout-delay + // header extensions with min=max=0 via the + // WebRTC-ForceSendPlayoutDelay field trial, but set + // jitterBufferTarget as well for browsers that honour + // the JS API over the in-band RTP extension. + if (typeof recv.jitterBufferTarget !== 'undefined') { + recv.jitterBufferTarget = 0; + } + if (typeof recv.getParameters != 'undefined') { + console.log( + 'codecs:' + + JSON.stringify( + recv.getParameters().codecs)); + } } }); } }; - // Local datachannel sends data + // Local datachannel sends data. + // Use reliable ordered delivery (the default). While unordered + + // unreliable would reduce head-of-line blocking for mouse MOVE/DRAG, + // the same channel carries discrete events (BUTTON_DOWN/UP, key events, + // resize) and application-level RPC (tensorboard/update_geometry etc.) + // that must not be lost or reordered. Mouse coalescing (rAF + server- + // side replace_or_merge_mouse) already prevents event backlog, so the + // extra reliability is free in practice on a local network. try { this.dataChannel = pc.createDataChannel('ClientDataChannel'); var dataChannel = this.dataChannel; diff --git a/python/js/package.json b/python/js/package.json index d364052ec22..0e2f7d78069 100644 --- a/python/js/package.json +++ b/python/js/package.json @@ -54,4 +54,4 @@ } } } -} \ No newline at end of file +}