Skip to content

Reduce delays in webRTC remote display. #7496

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ssheorey wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Reduce delays in webRTC remote display. #7496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
e913b9e
[WebRTC] Reduce remote rendering lag: input/draw coalescing, async en…
ssheorey May 20, 2026
a14390e
Explicitly warn and skip writing float textures (unsupported) in gltf.
ssheorey May 20, 2026
7d564b5
Merge branch 'main' of https://github.com/isl-org/open3d into ss/fast…
ssheorey May 20, 2026
9469dce
Pin license-webpack-plugin - windows wheel build error.
ssheorey May 21, 2026
99b96d2
[WebRTC] Optimize latency by adjusting pacing and jitter buffer settings
ssheorey May 21, 2026
1a46e95
undo non-working windows jupyterlab fix.
ssheorey May 22, 2026
84f58be
[WebRTC] Refactor texture handling in ASSIMP export and improve data …
ssheorey May 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 104 additions & 83 deletions cpp/open3d/t/io/file_format/FileASSIMP.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <unordered_map>
#include <vector>

#include "open3d/core/Dtype.h"
#include "open3d/core/ParallelFor.h"
#include "open3d/core/TensorFunction.h"
#include "open3d/t/io/ImageIO.h"
Expand Down Expand Up @@ -177,11 +178,31 @@ bool ReadTriangleMeshUsingASSIMP(
return true;
}

static void SetTextureMaterialProperty(aiMaterial* mat,
aiScene* scene,
int texture_idx,
aiTextureType tt,
t::geometry::Image& img) {
namespace {

// Checks whether a texture map is present, not empty and uint8 or uint16
// datatype.
bool HasValidTexture(const visualization::rendering::Material& material,
const std::string& key) {
if (!material.HasTextureMap(key) || material.GetTextureMap(key).IsEmpty()) {
return false;
}
const auto& dt = material.GetTextureMap(key).GetDtype();
if (dt != core::UInt8 && dt != core::UInt16) {
utility::LogWarning(
"Skipping texture map '{}' with unsupported data type '{}'. "
"Only uint8 and uint16 are supported.",
key, dt.ToString());
return false;
}
return true;
}

void SetTextureMaterialProperty(aiMaterial* mat,
aiScene* scene,
int texture_idx,
aiTextureType tt,
t::geometry::Image& img) {
// Encode image as PNG
std::vector<uint8_t> img_buffer;
WriteImageToPNGInMemory(img_buffer, img, 6);
Expand All @@ -208,7 +229,6 @@ static void SetTextureMaterialProperty(aiMaterial* mat,
mat->AddProperty(&mode, 1, AI_MATKEY_MAPPINGMODE_V(tt, 0));
}

namespace {
// Add hash function for tuple key
struct TupleHash {
size_t operator()(const std::tuple<int64_t, float, float>& t) const {
Expand Down Expand Up @@ -546,96 +566,97 @@ bool WriteTriangleMeshUsingASSIMP(const std::string& filename,
ai_mat->AddProperty(&ac, 1, AI_MATKEY_COLOR_EMISSIVE);
}

// Count texture maps...
// Build a list of texture-embed actions in a single pass. Each action
// is a lambda that writes one texture slot into the assimp scene; the
// slot index is passed at call time. .
//
// NOTE: GLTF2 expects a single combined roughness/metal map. If the
// model has one we just export it, otherwise if both roughness and
// metal maps are available we combine them, otherwise if only one or
// the other is available we just export the one map.
int n_textures = 0;
if (w_mesh.GetMaterial().HasAlbedoMap()) ++n_textures;
if (w_mesh.GetMaterial().HasNormalMap()) ++n_textures;
if (w_mesh.GetMaterial().HasAOMap()) ++n_textures;
if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) {
++n_textures;
} else if (w_mesh.GetMaterial().HasRoughnessMap() &&
w_mesh.GetMaterial().HasMetallicMap()) {
++n_textures;
const auto& material = w_mesh.GetMaterial();
using TextureAction = std::function<void(int /*slot*/)>;
std::vector<TextureAction> texture_actions;

if (HasValidTexture(material, "albedo")) {
texture_actions.push_back([&](int idx) {
auto img = material.GetAlbedoMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_DIFFUSE, img);
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_BASE_COLOR, img);
});
}
if (HasValidTexture(material, "ambient_occlusion")) {
texture_actions.push_back([&](int idx) {
auto img = material.GetAOMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_LIGHTMAP, img);
});
}
if (HasValidTexture(material, "ao_rough_metal")) {
Comment on lines +577 to +597
texture_actions.push_back([&](int idx) {
auto img = material.GetAORoughnessMetalMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_UNKNOWN, img);
});
} else if (HasValidTexture(material, "roughness") &&
HasValidTexture(material, "metallic")) {
texture_actions.push_back([&](int idx) {
auto rough = material.GetRoughnessMap().AsTensor();
auto metal = material.GetMetallicMap().AsTensor();
if (rough.GetShape() != metal.GetShape()) {
utility::LogError(
"RoughnessMap (shape={}) and MetallicMap "
"(shape={}) must have the same shape.",
rough.GetShape(), metal.GetShape());
}
auto rows = rough.GetShape(0);
auto cols = rough.GetShape(1);
auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255,
core::Dtype::UInt8);
rough_metal.Slice(2, 2, 3) =
metal.Slice(2, 0, 1); // blue channel is metal
rough_metal.Slice(2, 1, 2) =
rough.Slice(2, 0, 1); // green channel is roughness
geometry::Image rough_metal_img(rough_metal);
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_UNKNOWN,
rough_metal_img);
});
} else {
if (w_mesh.GetMaterial().HasRoughnessMap()) ++n_textures;
if (w_mesh.GetMaterial().HasMetallicMap()) ++n_textures;
if (HasValidTexture(material, "roughness")) {
texture_actions.push_back([&](int idx) {
auto img = material.GetRoughnessMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_UNKNOWN, img);
});
}
if (HasValidTexture(material, "metallic")) {
texture_actions.push_back([&](int idx) {
auto img = material.GetMetallicMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_UNKNOWN, img);
});
}
}
if (HasValidTexture(material, "normal")) {
texture_actions.push_back([&](int idx) {
auto img = material.GetNormalMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), idx,
aiTextureType_NORMALS, img);
});
}

int n_textures = static_cast<int>(texture_actions.size());
if (n_textures > 0) {
ai_scene->mTextures = new aiTexture*[n_textures];
for (int i = 0; i < n_textures; ++i) {
ai_scene->mTextures[i] = new aiTexture();
texture_actions[i](i);
}
ai_scene->mNumTextures = n_textures;
}

// Now embed the textures that are available...
int current_idx = 0;
if (w_mesh.GetMaterial().HasAlbedoMap()) {
auto img = w_mesh.GetMaterial().GetAlbedoMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_DIFFUSE, img);
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_BASE_COLOR, img);
++current_idx;
}
if (w_mesh.GetMaterial().HasAOMap()) {
auto img = w_mesh.GetMaterial().GetAOMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_LIGHTMAP, img);
++current_idx;
}
if (w_mesh.GetMaterial().HasAORoughnessMetalMap()) {
auto img = w_mesh.GetMaterial().GetAORoughnessMetalMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_UNKNOWN, img);
++current_idx;
} else if (w_mesh.GetMaterial().HasRoughnessMap() &&
w_mesh.GetMaterial().HasMetallicMap()) {
auto rough = w_mesh.GetMaterial().GetRoughnessMap().AsTensor();
auto metal = w_mesh.GetMaterial().GetMetallicMap().AsTensor();
if (rough.GetShape() != metal.GetShape()) {
utility::LogError(
"RoughnessMap (shape={}) and MetallicMap (shape={}) "
"must have the same shape.",
rough.GetShape(), metal.GetShape());
}
auto rows = rough.GetShape(0);
auto cols = rough.GetShape(1);
auto rough_metal = core::Tensor::Full({rows, cols, 4}, 255,
core::Dtype::UInt8);
rough_metal.Slice(2, 2, 3) =
metal.Slice(2, 0, 1); // blue channel is metal
rough_metal.Slice(2, 1, 2) =
rough.Slice(2, 0, 1); // green channel is roughness

geometry::Image rough_metal_img(rough_metal);
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_UNKNOWN, rough_metal_img);
++current_idx;
} else {
if (w_mesh.GetMaterial().HasRoughnessMap()) {
auto img = w_mesh.GetMaterial().GetRoughnessMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_UNKNOWN, img);
++current_idx;
}
if (w_mesh.GetMaterial().HasMetallicMap()) {
auto img = w_mesh.GetMaterial().GetMetallicMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_UNKNOWN, img);
++current_idx;
}
}
if (w_mesh.GetMaterial().HasNormalMap()) {
auto img = w_mesh.GetMaterial().GetNormalMap();
SetTextureMaterialProperty(ai_mat, ai_scene.get(), current_idx,
aiTextureType_NORMALS, img);
++current_idx;
}
}
ai_scene->mMaterials[0] = ai_mat;

Expand Down
Loading
Loading