diff --git a/.superpowers/brainstorm/87111-1780403451/state/server.pid b/.superpowers/brainstorm/87111-1780403451/state/server.pid new file mode 100644 index 00000000..ae9583be --- /dev/null +++ b/.superpowers/brainstorm/87111-1780403451/state/server.pid @@ -0,0 +1 @@ +87111 diff --git a/.superpowers/brainstorm/87156-1780403468/content/layout-options.html b/.superpowers/brainstorm/87156-1780403468/content/layout-options.html new file mode 100644 index 00000000..018fbe35 --- /dev/null +++ b/.superpowers/brainstorm/87156-1780403468/content/layout-options.html @@ -0,0 +1,444 @@ + + + + +
+
+
+

Isaac Sim 初始实验室布局候选

+

+ 基于 4090 上已发现的 RoboArm Chem 04 URDF、Matterix 桌面/烧杯/热台资产,以及 LabUtopia 化学实验室场景。 + 这里先校准空间和展示路线,确认后再生成可执行的 Isaac 初始化布局脚本。 +

+
+
+ +
+
+ 机械臂 + /home/ubuntu/canonical/Uni-Lab-OS/robot_assets/roboarm_chem_04/urdf/roboarm_chem_04_query.urdf +
+
+ 工作台 + /home/ubuntu/Matterix/source/matterix_assets/data/infrastructure/tables/table-thorlabs-75x90/table.usda +
+
+ 仪器和容器 + hotplate_start_button.usda, beaker-500ml.usda, 后续可补泵、阀、天平、相机等 USD +
+
+ 背景实验室 + /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd +
+
+ +
+
+
+

A. 中央机械臂岛

+ 推荐录屏首版 +
+
+
后墙 / LabUtopia 背景
+
仪器背景墙
+
Thorlabs 75x90 主桌
+
+
RoboArm
Chem 04
+
热台
Hotplate
+
烧杯
500ml
+
试剂
托盘
+
仪器
占位
+
Transfer
Deck
+
Cam
+
前侧安全/观察通道
+
+
+
适合展示一镜到底展示真实 PNG、query 机械臂/仪器 prim pose、验证 Uni-Lab-OS 到 Isaac worker 的闭环。
+
Isaac 坐标建议机械臂 base 在世界原点附近;主桌中心对齐原点,仪器在 reach 半径内形成清晰语义区。
+
后续扩展最容易增加抓取点、放置区、相机视角和动作轨迹;不需要先解决复杂房间约束。
+
+
+ +
+
+

B. 线性实验台

+ 真实实验室感 +
+
+
后墙 / 长台背景
+
长工作台
+
+
RoboArm
Chem 04
+
热台
+
烧杯
+
泵/阀
占位
+
天平
占位
+
操作带
+
Cam
+
人员/录屏观察区
+
+
+
适合展示像真实化学台面,便于解释“机械臂面对一排仪器”的使用场景。
+
风险如果 RoboArm Chem 04 的有效工作半径偏小,后排仪器可能超出可达范围,需要实测 URDF 尺寸。
+
后续扩展适合做仪器 query 和视觉展示;复杂抓取可能需要重新调整仪器间距。
+
+
+ +
+
+

C. U 型自动化单元

+ 面向长期自动化 +
+
+
背景实验室
+
左侧台
+
后侧台
+
右侧台
+
+
RoboArm
Chem 04
+
热台
+
反应/容器区
+
检测
占位
+
耗材
+
废液/泵
+
中央转运区
+
Cam
+
安全门/观察口
+
+
+
适合展示最像完整自动化 workcell,后续可做安全区、任务队列、多仪器流程。
+
风险布局复杂,初版需要更多碰撞体、相机角度和仪器间距调参。
+
后续扩展适合 C6/C7 之后接入动作规划、仪器状态动画和多步骤实验流程。
+
+
+
+ +
+ RoboArm Chem 04 + 工作台 + 仪器 + 容器/耗材 + 机械臂 reach 估计 + 录屏相机 +
+ + +
diff --git a/.superpowers/brainstorm/87156-1780403468/state/server-stopped b/.superpowers/brainstorm/87156-1780403468/state/server-stopped new file mode 100644 index 00000000..b26a5d8d --- /dev/null +++ b/.superpowers/brainstorm/87156-1780403468/state/server-stopped @@ -0,0 +1 @@ +{"reason":"idle timeout","timestamp":1780405568919} diff --git a/.superpowers/brainstorm/87156-1780403468/state/server.pid b/.superpowers/brainstorm/87156-1780403468/state/server.pid new file mode 100644 index 00000000..2c027bed --- /dev/null +++ b/.superpowers/brainstorm/87156-1780403468/state/server.pid @@ -0,0 +1 @@ +87156 diff --git a/docs/demo/phase2_isaac_e2e_4090.md b/docs/demo/phase2_isaac_e2e_4090.md new file mode 100644 index 00000000..24a1fa5c --- /dev/null +++ b/docs/demo/phase2_isaac_e2e_4090.md @@ -0,0 +1,128 @@ +# Phase 2 Isaac 4090 端到端验收 Runbook + +## 前置条件 + +- Host: `ubuntu@172.20.0.39` +- Edge env: `/home/ubuntu/miniforge3/envs/unilab` +- Isaac env: `/home/ubuntu/miniforge3/envs/matterix` +- Test copy: `/tmp/Uni-Lab-OS-phase2-c3c5` +- Worker endpoint: `http://127.0.0.1:8092` +- Query gRPC: `127.0.0.1:50052` +- Scene: `/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd` + +不要覆盖 4090 上可能 dirty 的 `~/canonical/Uni-Lab-OS`。 +`8091` 可能已有其他 Isaac demo 占用,本 runbook 使用 `8092`。 + +## 1. 同步代码 + +```bash +rsync -az --delete --exclude .git --exclude .pytest_cache ./ \ + ubuntu@172.20.0.39:/tmp/Uni-Lab-OS-phase2-c3c5/ +``` + +## 2. 启动 Isaac Worker + +不要使用 `pkill -f "unilabos.sim.backends.isaac.worker"`,这个模式可能匹配并杀掉自己的 SSH shell。需要停旧 worker 时先用 `pgrep -af "[u]nilabos.sim.backends.isaac.worker.*8092"` 找到精确 PID。 + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + nohup /home/ubuntu/miniforge3/bin/conda run -n matterix env PYTHONPATH=. \ + python -m unilabos.sim.backends.isaac.worker \ + --host 127.0.0.1 \ + --port 8092 \ + --headless \ + --scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --camera /World/Camera \ + --rpc-timeout-s 600 \ + > /tmp/isaac_worker_phase2_c3c5_8092.log 2>&1 & + echo $! > /tmp/isaac_worker_phase2_c3c5_8092.pid +' +``` + +Isaac 启动较慢,等 60 秒后检查: + +```bash +ssh ubuntu@172.20.0.39 'sleep 70; ss -ltnp "( sport = :8092 )"; curl -sS http://127.0.0.1:8092/health' +``` + +## 3. 启动 Edge + +本地 graph + `--app_bridges fastapi` + 非 websocket 可以离线启动,不需要 `UNILAB_AK/UNILAB_SK`。 + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + nohup /home/ubuntu/miniforge3/bin/conda run --no-capture-output -n unilab env PYTHONPATH=. \ + unilab --graph unilabos/test/experiments/mock_devices/mock_all.json \ + --config unilabos/config/example_config.py \ + --backend ros \ + --mode sim \ + --sim_rate 10 \ + --physics isaac \ + --physics_endpoint http://127.0.0.1:8092 \ + --physics_scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --physics_timeout 300 \ + --query_labutopia_usd /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --query_grpc_port 50052 \ + --app_bridges fastapi \ + --visual disable \ + --skip_env_check \ + --disable_browser \ + --test_mode \ + --port 8002 \ + > /tmp/unilab_edge_c5_50052.log 2>&1 & + echo $! > /tmp/unilab_edge_c5_50052.pid +' +``` + +检查 gRPC: + +```bash +ssh ubuntu@172.20.0.39 'sleep 80; ss -ltnp "( sport = :50052 or sport = :8002 )"; tail -n 120 /tmp/unilab_edge_c5_50052.log' +``` + +## 4. 运行 Smoke + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + /home/ubuntu/miniforge3/bin/conda run -n unilab env PYTHONPATH=. \ + python scripts/smoke_sim_isaac_edge.py \ + --grpc 127.0.0.1:50052 \ + --physics-endpoint http://127.0.0.1:8092 \ + --state-target /World \ + --pose-target /World \ + --camera /World/Camera \ + --physics-timeout-s 300 \ + --out /tmp/labutopia-c5-e2e.png + file /tmp/labutopia-c5-e2e.png + ls -lh /tmp/labutopia-c5-e2e.png +' +``` + +## 5. 停止进程 + +```bash +ssh ubuntu@172.20.0.39 ' + kill $(cat /tmp/unilab_edge_c5_50052.pid) || true + pgrep -af "[u]nilabos.sim.backends.isaac.worker.*8092" +' +``` + +## 6. 最终回归 + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-phase2-c3c5 \ + python -m pytest tests/sim tests/queries tests/integration -q' +``` + +## 期望证据 + +- `ss -ltnp "( sport = :8092 )"` 能看到 Isaac worker。 +- `ss -ltnp "( sport = :50052 )"` 能看到 query gRPC。 +- smoke script 退出码为 0。 +- smoke 输出的 `state.source` 和 `pose.source` 是 `physics_live:isaac`。 +- `/tmp/labutopia-c5-e2e.png` 是 `PNG image data, 640 x 480, 8-bit/color RGBA`。 +- 最终回归通过。 diff --git a/docs/demo/phase2_isaac_project_summary.md b/docs/demo/phase2_isaac_project_summary.md new file mode 100644 index 00000000..3bf3b879 --- /dev/null +++ b/docs/demo/phase2_isaac_project_summary.md @@ -0,0 +1,296 @@ +# Phase 2 Isaac Bridge 项目改动与实现总览 + +更新时间:2026-06-02 + +本文档汇总本轮 Phase 2 Isaac Bridge 的整体目标、代码改动、已实现能力、4090 验收结果和演示命令。对应计划文档: + +- `docs/superpowers/plans/2026-06-02-phase2-isaac-c1-c2.md` +- `docs/superpowers/plans/2026-06-02-phase2-isaac-c3-worker.md` +- `docs/superpowers/plans/2026-06-02-phase2-isaac-c4-edge-integration.md` +- `docs/superpowers/plans/2026-06-02-phase2-isaac-c5-e2e-render.md` +- `docs/demo/phase2_isaac_e2e_4090.md` + +## 一句话结果 + +我们已经完成 Route A 链路:Uni-Lab-OS 可以通过 CLI 接入 Isaac worker,edge 运行时持有 physics backend,Query API 能从 `physics_live:isaac` 查询物理状态,并且能在 4090 上从 LabUtopia Isaac 场景生成真实 PNG 画面。 + +## 4090 运行环境 + +- Host:`ubuntu@172.20.0.39` +- GPU:NVIDIA GeForce RTX 4090,约 24GB 显存 +- Edge 环境:`/home/ubuntu/miniforge3/envs/unilab` +- Isaac 环境:`/home/ubuntu/miniforge3/envs/matterix` +- 远端测试副本:`/tmp/Uni-Lab-OS-phase2-c3c5` +- Isaac worker:`http://127.0.0.1:8092` +- Query gRPC:`127.0.0.1:50052` +- FastAPI:`0.0.0.0:8002` +- LabUtopia scene:`/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd` + +注意:4090 上 `8091` 可能已有其他 Isaac demo 占用,本轮验收使用 `8092`。 + +## 已实现能力 + +### C1/C2:Physics Contract 与 Isaac Bridge 基础 + +实现内容: + +- 扩展 `PhysicsBackend` 协议,新增 `load_scene(scene_path)` 和 `render(camera, width, height)`。 +- 新增 fake physics backend,用于本地测试、runtime 接线测试和虚拟设备测试。 +- 新增 Isaac HTTP bridge,把 edge 侧 physics 调用转成 worker RPC。 +- 新增 JSON RPC 协议编码/解码,支持 success/error 响应。 + +关键文件: + +- `unilabos/sim/physics_backend.py` +- `unilabos/sim/context.py` +- `unilabos/sim/runtime.py` +- `unilabos/sim/backends/fake_physics.py` +- `unilabos/sim/backends/isaac_bridge.py` +- `unilabos/sim/backends/isaac/protocol.py` + +### C3:Isaac Worker 独立运行 + +实现内容: + +- 新增独立 Isaac worker,可在 `matterix` 环境中运行。 +- Worker 暴露 HTTP `/health` 和 `/rpc`。 +- 支持 `reset`、`step`、`load_scene`、`get_observation`、`set_command`、`attach_rigid_body`、`get_joint_states`、`apply_wrench`、`render`。 +- 真实 Isaac API 调用通过主线程队列执行,避免 HTTP worker 线程直接调用 Kit/Usd 导致卡死。 +- 使用 replicator RGB annotator 生成真实 PNG,不再把 fallback 当作验收通过。 +- `load_scene/reset` 会清理 render cache,避免旧 stage 的 render product 失效。 + +关键文件: + +- `unilabos/sim/backends/isaac/worker.py` +- `unilabos/sim/backends/isaac/worker_http.py` +- `scripts/smoke_isaac_worker.py` +- `tests/sim/backends/test_isaac_worker_cli.py` +- `tests/sim/backends/test_isaac_worker_http.py` +- `tests/sim/backends/test_isaac_worker_protocol.py` +- `tests/sim/backends/test_isaac_worker_smoke_script.py` + +### C4:CLI、RuntimeContext、HAL/虚拟设备接线 + +实现内容: + +- CLI 新增: + - `--physics none|fake|isaac` + - `--physics_endpoint` + - `--physics_scene` + - `--physics_timeout` +- Backend 启动时自动构建 physics backend,并写入 `RuntimeContext`。 +- `RuntimeContext` 记录 physics 名称、endpoint、scene、timeout。 +- `build_physics_backend()` 统一创建 fake/isaac backend。 +- UR HAL sim 模式默认读取 `RuntimeContext.physics`。 +- 虚拟多通阀动作会 dispatch 到 physics backend。 +- 支持本地 graph + `fastapi` only 的离线启动,不再强制要求 AK/SK;如果启用 websocket 或远程资源,仍需要云端凭证。 + +关键文件: + +- `unilabos/app/main.py` +- `unilabos/app/backend.py` +- `unilabos/sim/backends/factory.py` +- `unilabos/sim/device_physics.py` +- `unilabos/hal/adapters/ur_adapter.py` +- `unilabos/devices/virtual/virtual_multiway_valve.py` +- `tests/sim/test_cli_runtime.py` +- `tests/sim/test_backend_physics_configuration.py` +- `tests/sim/test_device_physics.py` +- `tests/sim/test_virtual_device_clock.py` +- `tests/queries/test_ur_adapter.py` + +### C5:端到端 Query 与真画面验收 + +实现内容: + +- 新增 `PhysicsLiveSource`,从 `RuntimeContext.physics.get_observation(target)` 转成 Query API 的 `State` 和 `Pose`。 +- 支持 observation 中的 `pose` dict,也支持 UR 风格 `tcp_pose/tool_pose`。 +- Query startup 中接入 physics source,顺序为: + 1. ROS live source + 2. Physics live source + 3. LabUtopia static source +- 新增 C5 smoke 脚本,通过 gRPC query state/pose,再通过 Isaac bridge render PNG。 +- Smoke 脚本要求完整 PNG signature + IEND,fallback 数据不能通过验收。 + +关键文件: + +- `unilabos/queries/physics_live_source.py` +- `unilabos/queries/__init__.py` +- `unilabos/ros/main_slave_run.py` +- `scripts/smoke_sim_isaac_edge.py` +- `tests/queries/test_physics_live_source.py` +- `tests/integration/test_edge_query_wiring.py` +- `tests/integration/test_smoke_sim_isaac_edge_script.py` + +## 4090 验收结果 + +### 目标测试 + +命令: + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-phase2-c3c5 \ + python -m pytest \ + tests/sim/backends/test_isaac_worker_protocol.py \ + tests/sim/backends/test_isaac_worker_http.py \ + tests/sim/backends/test_isaac_worker_cli.py \ + tests/sim/backends/test_isaac_worker_smoke_script.py \ + tests/sim/backends/test_factory.py \ + tests/sim/test_cli_runtime.py \ + tests/sim/test_runtime_configuration.py \ + tests/sim/test_backend_physics_configuration.py \ + tests/sim/test_device_physics.py \ + tests/sim/test_virtual_device_clock.py \ + tests/queries/test_ur_adapter.py \ + tests/queries/test_physics_live_source.py \ + tests/integration/test_smoke_sim_isaac_edge_script.py \ + tests/integration/test_edge_query_wiring.py \ + -q' +``` + +结果: + +```text +49 passed, 1 warning +``` + +### C5 端到端 smoke + +命令: + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + /home/ubuntu/miniforge3/bin/conda run -n unilab env PYTHONPATH=. \ + python scripts/smoke_sim_isaac_edge.py \ + --grpc 127.0.0.1:50052 \ + --physics-endpoint http://127.0.0.1:8092 \ + --state-target /World \ + --pose-target /World \ + --camera /World/Camera \ + --physics-timeout-s 300 \ + --out /tmp/labutopia-c5-e2e.png + file /tmp/labutopia-c5-e2e.png + ls -lh /tmp/labutopia-c5-e2e.png +' +``` + +结果要点: + +- `state.source`:`physics_live:isaac` +- `pose.source`:`physics_live:isaac` +- PNG:`/tmp/labutopia-c5-e2e.png` +- 文件类型:`PNG image data, 640 x 480, 8-bit/color RGBA, non-interlaced` +- 文件大小:约 `313K` + +### 相关全量回归 + +命令: + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-phase2-c3c5 \ + python -m pytest tests/sim tests/queries tests/integration -q' +``` + +结果: + +```text +151 passed, 2 warnings +``` + +### Diff 检查 + +命令: + +```bash +git diff --check +``` + +结果:通过,无 trailing whitespace 等格式问题。 + +## 录屏演示命令 + +### 1. 展示 GPU 和项目路径 + +```bash +hostname +nvidia-smi +cd /tmp/Uni-Lab-OS-phase2-c3c5 +/home/ubuntu/miniforge3/bin/conda run -n unilab python --version +``` + +### 2. 展示 Isaac worker + +```bash +ss -ltnp "( sport = :8092 )" +curl -sS http://127.0.0.1:8092/health +``` + +### 3. 展示 Edge 和 Query API + +```bash +ss -ltnp "( sport = :50052 or sport = :8002 )" +tail -n 80 /tmp/unilab_edge_c5_50052.log +``` + +重点看: + +```text +Runtime mode initialized: mode=sim ... physics=isaac ... +Query API gRPC server started at :50052 +``` + +### 4. 展示 query 物理态和真 PNG + +```bash +/home/ubuntu/miniforge3/bin/conda run -n unilab env PYTHONPATH=. \ + python scripts/smoke_sim_isaac_edge.py \ + --grpc 127.0.0.1:50052 \ + --physics-endpoint http://127.0.0.1:8092 \ + --state-target /World \ + --pose-target /World \ + --camera /World/Camera \ + --physics-timeout-s 300 \ + --out /tmp/labutopia-c5-e2e-recording.png +file /tmp/labutopia-c5-e2e-recording.png +ls -lh /tmp/labutopia-c5-e2e-recording.png +``` + +### 5. 展示测试回归 + +```bash +/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-phase2-c3c5 \ + python -m pytest tests/sim tests/queries tests/integration -q +``` + +## 当前运行状态 + +截至本轮验收结束,4090 上保留了以下进程用于继续录屏或人工检查: + +- Isaac worker:`127.0.0.1:8092` +- Query gRPC:`127.0.0.1:50052` +- FastAPI:`0.0.0.0:8002` + +PNG 验收文件: + +- `/tmp/labutopia-phase2-c3c5.png` +- `/tmp/labutopia-c5-e2e.png` + +## 注意事项 + +- 不要使用 `pkill -f "unilabos.sim.backends.isaac.worker"`,该模式可能匹配并杀掉自己的 SSH shell。需要停止 worker 时,先用 `pgrep -af "[u]nilabos.sim.backends.isaac.worker.*8092"` 找精确 PID。 +- `8091` 可能被旧 Isaac demo 占用;本轮使用 `8092`。 +- 本地 graph + `--app_bridges fastapi` + 非 websocket 可以离线启动,不需要 AK/SK。 +- 如果启用 websocket、远程资源或云同步,仍需要有效 AK/SK。 +- 本地 macOS 默认 `python3` 是 3.9,缺少本仓库 Python 3.11 依赖;正式验证使用 4090 的 `unilab` conda 环境。 +- Worker 的 fallback render 只用于调试,不作为 C5 验收通过条件;smoke 脚本要求完整 PNG。 + +## 后续可继续增强 + +- 把更多 Isaac prim/device 映射成 Uni-Lab-OS 设备状态,而不仅是 `/World` 级别 observation。 +- 增加更完整的 joint state、contact、wrench、rigid body 真实物理读写。 +- 把 camera path、resolution、scene 作为标准 demo 配置管理。 +- 为 worker 增加更清晰的日志和 render 错误诊断。 +- 将当前工作树按 C3/C4/C5 拆分 commit,便于后续 PR review。 diff --git a/docs/superpowers/plans/2026-06-02-phase2-isaac-c1-c2.md b/docs/superpowers/plans/2026-06-02-phase2-isaac-c1-c2.md new file mode 100644 index 00000000..bd1bef08 --- /dev/null +++ b/docs/superpowers/plans/2026-06-02-phase2-isaac-c1-c2.md @@ -0,0 +1,634 @@ +# Phase 2 Isaac C1 C2 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the C1/C2 foundation for Route A by adding an in-process fake physics backend and an HTTP/JSON Isaac bridge client that implements the Uni-Lab-OS `PhysicsBackend` contract. + +**Architecture:** C1 extends the existing `PhysicsBackend` protocol with scene loading and rendering, stores physics configuration on `RuntimeContext`, and adds a deterministic `FakePhysicsBackend` for unit and CLI wiring tests. C2 adds a small JSON protocol module plus `IsaacBridgeBackend`, which serializes every backend operation into HTTP requests so the edge process can talk to an Isaac worker without importing Isaac or ROS dependencies. + +**Tech Stack:** Python 3.11, `typing.Protocol`, stdlib `urllib.request`, stdlib `json`, pytest. + +--- + +## File Structure + +- Create `unilabos/sim/backends/__init__.py` + - Exposes physics backend implementations without importing Isaac-specific code. +- Create `unilabos/sim/backends/fake_physics.py` + - Deterministic in-process `PhysicsBackend` for C1 tests and future `--physics fake`. +- Create `unilabos/sim/backends/isaac/__init__.py` + - Package marker for Isaac IPC files. +- Create `unilabos/sim/backends/isaac/protocol.py` + - Pure-Python HTTP/JSON request and response contract. +- Create `unilabos/sim/backends/isaac_bridge.py` + - Edge-side `PhysicsBackend` client that calls a worker endpoint over HTTP. +- Create `tests/sim/backends/test_fake_physics.py` + - Covers fake backend state, scene loading, command recording, observations, joint states, callbacks, and render bytes. +- Create `tests/sim/backends/test_isaac_protocol.py` + - Covers JSON encoding and response decoding behavior. +- Create `tests/sim/backends/test_isaac_bridge.py` + - Covers HTTP RPC methods against a local mock server. +- Modify `unilabos/sim/physics_backend.py` + - Add `load_scene(scene_path: str) -> None` and `render(camera: str, width: int, height: int) -> bytes`. +- Modify `unilabos/sim/context.py` + - Add `physics_backend_name`, `physics_endpoint`, and `physics_scene` to `RuntimeContext`. +- Modify `tests/sim/test_physics_backend.py` + - Update the existing runtime protocol test double to satisfy the extended contract. +- Modify `tests/sim/test_context_and_clock.py` + - Cover the new `RuntimeContext` physics configuration fields. + +## Task 1: Extend PhysicsBackend Contract + +**Files:** +- Modify: `unilabos/sim/physics_backend.py` +- Modify: `tests/sim/test_physics_backend.py` + +- [ ] **Step 1: Write the failing protocol test** + +Add these methods to `FakePhysics` in `tests/sim/test_physics_backend.py` and assert runtime protocol compatibility: + +```python + def load_scene(self, scene_path: str) -> None: + self.scene_path = scene_path + + def render(self, camera: str, width: int, height: int) -> bytes: + return f"{camera}:{width}x{height}".encode() + + +def test_physics_backend_scene_and_render_contract(): + backend = FakePhysics() + backend.load_scene("/tmp/lab.usd") + assert backend.scene_path == "/tmp/lab.usd" + assert backend.render("/World/Camera", 320, 240) == b"/World/Camera:320x240" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/sim/test_physics_backend.py::test_physics_backend_scene_and_render_contract -q` + +Expected: FAIL until `PhysicsBackend` declares `load_scene` and `render`, or runtime compatibility fails for a class missing those methods. + +- [ ] **Step 3: Extend the protocol** + +Add this to `PhysicsBackend`: + +```python + def load_scene(self, scene_path: str) -> None: + ... + + def render(self, camera: str, width: int, height: int) -> bytes: + ... +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/sim/test_physics_backend.py -q` + +Expected: PASS. + +- [ ] **Step 5: Commit point** + +Commit message: `feat(sim): extend physics backend contract` + +## Task 2: Add RuntimeContext Physics Configuration + +**Files:** +- Modify: `unilabos/sim/context.py` +- Modify: `tests/sim/test_context_and_clock.py` + +- [ ] **Step 1: Write the failing context test** + +Add this test to `tests/sim/test_context_and_clock.py`: + +```python +def test_runtime_context_stores_physics_configuration(): + ctx = RuntimeContext( + mode="sim", + physics_backend_name="isaac", + physics_endpoint="http://127.0.0.1:8091", + physics_scene="/tmp/lab.usd", + ) + + assert ctx.physics_backend_name == "isaac" + assert ctx.physics_endpoint == "http://127.0.0.1:8091" + assert ctx.physics_scene == "/tmp/lab.usd" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/sim/test_context_and_clock.py::test_runtime_context_stores_physics_configuration -q` + +Expected: FAIL with unexpected keyword argument before fields are added. + +- [ ] **Step 3: Add dataclass fields** + +Add these fields to `RuntimeContext`: + +```python + physics_backend_name: str = "none" + physics_endpoint: Optional[str] = None + physics_scene: Optional[str] = None +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/sim/test_context_and_clock.py::test_runtime_context_stores_physics_configuration -q` + +Expected: PASS. + +- [ ] **Step 5: Commit point** + +Commit message: `feat(sim): record physics backend configuration` + +## Task 3: Add FakePhysicsBackend + +**Files:** +- Create: `unilabos/sim/backends/__init__.py` +- Create: `unilabos/sim/backends/fake_physics.py` +- Create: `tests/sim/backends/test_fake_physics.py` + +- [ ] **Step 1: Write failing fake backend tests** + +Create `tests/sim/backends/test_fake_physics.py`: + +```python +from unilabos.sim.backends.fake_physics import FakePhysicsBackend +from unilabos.sim.physics_backend import PhysicsBackend + + +def test_fake_backend_satisfies_physics_protocol(): + assert isinstance(FakePhysicsBackend(), PhysicsBackend) + + +def test_fake_backend_records_scene_commands_and_steps(): + backend = FakePhysicsBackend() + backend.load_scene("/tmp/lab.usd") + backend.set_command("arm", {"type": "move_j", "joint_positions": [1.0, 2.0]}) + backend.step(0.05) + + assert backend.scene_path == "/tmp/lab.usd" + assert backend.commands["arm"] == {"type": "move_j", "joint_positions": [1.0, 2.0]} + assert backend.sim_time == 0.05 + assert backend.get_observation("arm")["last_command"]["type"] == "move_j" + + +def test_fake_backend_tracks_joint_states_and_rigid_bodies(): + backend = FakePhysicsBackend() + backend.set_joint_states("arm", {"joint_1": 1.25, "joint_2": -0.5}) + body_id = backend.attach_rigid_body("beaker", "beaker.usd", {"xyz": [0, 0, 0]}) + + assert body_id == "beaker" + assert backend.get_joint_states("arm") == {"joint_1": 1.25, "joint_2": -0.5} + assert backend.get_observation("beaker")["asset_path"] == "beaker.usd" + + +def test_fake_backend_render_returns_png_like_bytes(): + backend = FakePhysicsBackend() + image = backend.render("/World/Camera", 320, 240) + + assert image.startswith(b"\x89PNG\r\n\x1a\n") + assert b"/World/Camera" in image + + +def test_fake_backend_contact_callback_receives_applied_wrench_event(): + backend = FakePhysicsBackend() + events = [] + backend.register_contact_callback(events.append) + + backend.apply_wrench("arm", {"force": [1, 0, 0]}) + + assert events == [{"type": "wrench", "body_id": "arm", "wrench": {"force": [1, 0, 0]}}] +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/sim/backends/test_fake_physics.py -q` + +Expected: FAIL with module import error before `FakePhysicsBackend` exists. + +- [ ] **Step 3: Implement fake backend** + +Create `unilabos/sim/backends/__init__.py`: + +```python +"""Simulation physics backend implementations.""" +``` + +Create `unilabos/sim/backends/fake_physics.py` with: + +```python +from __future__ import annotations + +from typing import Any, Callable + + +class FakePhysicsBackend: + name = "fake" + + def __init__(self) -> None: + self.scene_path: str | None = None + self.sim_time = 0.0 + self.commands: dict[str, dict[str, Any]] = {} + self.observations: dict[str, dict[str, Any]] = {} + self.joint_states: dict[str, dict[str, float]] = {} + self.rigid_bodies: dict[str, dict[str, Any]] = {} + self.wrenches: list[tuple[str, dict[str, Any]]] = [] + self._contact_callbacks: list[Callable[[dict[str, Any]], None]] = [] + + def reset(self) -> None: + self.sim_time = 0.0 + self.commands.clear() + self.observations.clear() + self.joint_states.clear() + self.wrenches.clear() + + def step(self, dt: float) -> None: + self.sim_time += float(dt) + + def load_scene(self, scene_path: str) -> None: + self.scene_path = str(scene_path) + + def get_observation(self, entity_id: str) -> dict[str, Any]: + observation = dict(self.observations.get(entity_id, {})) + if entity_id in self.commands: + observation["last_command"] = dict(self.commands[entity_id]) + if entity_id in self.joint_states: + observation["joint_positions"] = list(self.joint_states[entity_id].values()) + observation["joint_states"] = dict(self.joint_states[entity_id]) + if entity_id in self.rigid_bodies: + observation.update(self.rigid_bodies[entity_id]) + observation.setdefault("entity_id", entity_id) + observation.setdefault("sim_time", self.sim_time) + return observation + + def set_observation(self, entity_id: str, observation: dict[str, Any]) -> None: + self.observations[entity_id] = dict(observation) + + def set_command(self, entity_id: str, command: dict[str, Any]) -> None: + self.commands[entity_id] = dict(command) + + def attach_rigid_body(self, name: str, asset_path: str, pose: dict[str, Any]) -> str: + body_id = str(name) + self.rigid_bodies[body_id] = {"name": str(name), "asset_path": str(asset_path), "pose": dict(pose)} + return body_id + + def set_joint_states(self, body_id: str, joints: dict[str, float]) -> None: + self.joint_states[body_id] = {str(key): float(value) for key, value in joints.items()} + + def get_joint_states(self, body_id: str) -> dict[str, float]: + return dict(self.joint_states.get(body_id, {})) + + def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: + payload = dict(wrench) + self.wrenches.append((body_id, payload)) + event = {"type": "wrench", "body_id": body_id, "wrench": payload} + for callback in list(self._contact_callbacks): + callback(event) + + def register_contact_callback(self, callback: Callable[[dict[str, Any]], None]) -> None: + self._contact_callbacks.append(callback) + + def render(self, camera: str, width: int, height: int) -> bytes: + meta = f"fake-render camera={camera} width={int(width)} height={int(height)}".encode() + return b"\x89PNG\r\n\x1a\n" + meta +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest tests/sim/backends/test_fake_physics.py -q` + +Expected: PASS. + +- [ ] **Step 5: Commit point** + +Commit message: `feat(sim): add fake physics backend` + +## Task 4: Add Isaac HTTP Protocol Helpers + +**Files:** +- Create: `unilabos/sim/backends/isaac/__init__.py` +- Create: `unilabos/sim/backends/isaac/protocol.py` +- Create: `tests/sim/backends/test_isaac_protocol.py` + +- [ ] **Step 1: Write failing protocol tests** + +Create `tests/sim/backends/test_isaac_protocol.py`: + +```python +import pytest + +from unilabos.sim.backends.isaac.protocol import decode_response, encode_request + + +def test_encode_request_builds_compact_json_payload(): + payload = encode_request("set_command", {"entity_id": "arm", "command": {"type": "move_j"}}) + + assert payload == b'{"op":"set_command","args":{"entity_id":"arm","command":{"type":"move_j"}}}' + + +def test_decode_response_returns_result_for_ok_payload(): + assert decode_response(b'{"ok":true,"result":{"joint_1":1.0}}') == {"joint_1": 1.0} + + +def test_decode_response_raises_for_worker_error(): + with pytest.raises(RuntimeError, match="Isaac worker RPC failed: bad scene"): + decode_response(b'{"ok":false,"error":"bad scene"}') +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/sim/backends/test_isaac_protocol.py -q` + +Expected: FAIL with module import error before protocol helpers exist. + +- [ ] **Step 3: Implement protocol helpers** + +Create `unilabos/sim/backends/isaac/__init__.py`: + +```python +"""Isaac Sim bridge protocol package.""" +``` + +Create `unilabos/sim/backends/isaac/protocol.py`: + +```python +from __future__ import annotations + +import json +from typing import Any + + +def encode_request(op: str, args: dict[str, Any] | None = None) -> bytes: + payload = {"op": str(op), "args": dict(args or {})} + return json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def decode_response(data: bytes) -> Any: + payload = json.loads(data.decode("utf-8")) + if not payload.get("ok", False): + error = payload.get("error", "unknown error") + raise RuntimeError(f"Isaac worker RPC failed: {error}") + return payload.get("result") +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest tests/sim/backends/test_isaac_protocol.py -q` + +Expected: PASS. + +- [ ] **Step 5: Commit point** + +Commit message: `feat(sim): add isaac bridge JSON protocol` + +## Task 5: Add IsaacBridgeBackend HTTP Client + +**Files:** +- Create: `unilabos/sim/backends/isaac_bridge.py` +- Create: `tests/sim/backends/test_isaac_bridge.py` + +- [ ] **Step 1: Write failing HTTP bridge tests** + +Create `tests/sim/backends/test_isaac_bridge.py`: + +```python +from __future__ import annotations + +import base64 +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend +from unilabos.sim.physics_backend import PhysicsBackend + + +class _RpcHandler(BaseHTTPRequestHandler): + calls = [] + + def log_message(self, format, *args): + return + + def do_POST(self): + length = int(self.headers["Content-Length"]) + payload = json.loads(self.rfile.read(length).decode("utf-8")) + self.__class__.calls.append(payload) + op = payload["op"] + args = payload["args"] + if op == "get_observation": + result = {"entity_id": args["entity_id"], "tcp_pose": [1, 2, 3, 0, 0, 0]} + elif op == "get_joint_states": + result = {"joint_1": 1.0} + elif op == "attach_rigid_body": + result = "beaker" + elif op == "render": + result = {"encoding": "base64", "data": base64.b64encode(b"png-bytes").decode("ascii")} + else: + result = None + body = json.dumps({"ok": True, "result": result}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def _start_server(): + _RpcHandler.calls = [] + server = HTTPServer(("127.0.0.1", 0), _RpcHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://127.0.0.1:{server.server_port}" + + +def test_isaac_bridge_satisfies_physics_protocol(): + assert isinstance(IsaacBridgeBackend("http://127.0.0.1:9"), PhysicsBackend) + + +def test_isaac_bridge_forwards_backend_methods_over_http(): + server, endpoint = _start_server() + try: + backend = IsaacBridgeBackend(endpoint) + + backend.load_scene("/tmp/lab.usd") + backend.reset() + backend.step(0.05) + backend.set_command("arm", {"type": "move_j"}) + observation = backend.get_observation("arm") + joints = backend.get_joint_states("arm") + body_id = backend.attach_rigid_body("beaker", "beaker.usd", {"xyz": [0, 0, 0]}) + backend.apply_wrench("arm", {"force": [1, 0, 0]}) + image = backend.render("/World/Camera", 320, 240) + + assert observation["entity_id"] == "arm" + assert joints == {"joint_1": 1.0} + assert body_id == "beaker" + assert image == b"png-bytes" + assert [call["op"] for call in _RpcHandler.calls] == [ + "load_scene", + "reset", + "step", + "set_command", + "get_observation", + "get_joint_states", + "attach_rigid_body", + "apply_wrench", + "render", + ] + finally: + server.shutdown() +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `pytest tests/sim/backends/test_isaac_bridge.py -q` + +Expected: FAIL with module import error before `IsaacBridgeBackend` exists. + +- [ ] **Step 3: Implement HTTP bridge** + +Create `unilabos/sim/backends/isaac_bridge.py`: + +```python +from __future__ import annotations + +import base64 +from typing import Any, Callable +from urllib import request +from urllib.error import HTTPError, URLError + +from unilabos.sim.backends.isaac.protocol import decode_response, encode_request + + +class IsaacBridgeBackend: + name = "isaac" + + def __init__(self, endpoint: str, timeout: float = 5.0) -> None: + self.endpoint = endpoint.rstrip("/") + self.timeout = float(timeout) + + def _rpc(self, op: str, args: dict[str, Any] | None = None) -> Any: + req = request.Request( + f"{self.endpoint}/rpc", + data=encode_request(op, args), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(req, timeout=self.timeout) as response: + return decode_response(response.read()) + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Isaac worker HTTP {exc.code}: {detail}") from exc + except URLError as exc: + raise RuntimeError(f"Isaac worker unavailable at {self.endpoint}: {exc.reason}") from exc + + def reset(self) -> None: + self._rpc("reset") + + def step(self, dt: float) -> None: + self._rpc("step", {"dt": float(dt)}) + + def load_scene(self, scene_path: str) -> None: + self._rpc("load_scene", {"scene_path": str(scene_path)}) + + def get_observation(self, entity_id: str) -> dict[str, Any]: + return dict(self._rpc("get_observation", {"entity_id": str(entity_id)}) or {}) + + def set_command(self, entity_id: str, command: dict[str, Any]) -> None: + self._rpc("set_command", {"entity_id": str(entity_id), "command": dict(command)}) + + def attach_rigid_body(self, name: str, asset_path: str, pose: dict[str, Any]) -> str: + result = self._rpc( + "attach_rigid_body", + {"name": str(name), "asset_path": str(asset_path), "pose": dict(pose)}, + ) + return str(result) + + def get_joint_states(self, body_id: str) -> dict[str, float]: + result = self._rpc("get_joint_states", {"body_id": str(body_id)}) or {} + return {str(key): float(value) for key, value in dict(result).items()} + + def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: + self._rpc("apply_wrench", {"body_id": str(body_id), "wrench": dict(wrench)}) + + def register_contact_callback(self, callback: Callable[[dict[str, Any]], None]) -> None: + raise NotImplementedError("IsaacBridgeBackend does not support edge-side contact callbacks yet") + + def render(self, camera: str, width: int, height: int) -> bytes: + result = self._rpc("render", {"camera": str(camera), "width": int(width), "height": int(height)}) + if isinstance(result, dict) and result.get("encoding") == "base64": + return base64.b64decode(str(result.get("data", ""))) + if isinstance(result, str): + return base64.b64decode(result) + raise TypeError(f"Isaac render returned unsupported payload: {type(result).__name__}") +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `pytest tests/sim/backends/test_isaac_bridge.py -q` + +Expected: PASS. + +- [ ] **Step 5: Commit point** + +Commit message: `feat(sim): add isaac HTTP physics bridge` + +## Task 6: Run C1/C2 Regression Suite on 4090 + +**Files:** +- No source files modified in this task. + +- [ ] **Step 1: Sync current branch to 4090** + +Use git push/pull if the branch is remote-ready. If not, copy only these changed files to the matching 4090 checkout: + +```bash +docs/superpowers/plans/2026-06-02-phase2-isaac-c1-c2.md +unilabos/sim/physics_backend.py +unilabos/sim/context.py +unilabos/sim/backends/__init__.py +unilabos/sim/backends/fake_physics.py +unilabos/sim/backends/isaac/__init__.py +unilabos/sim/backends/isaac/protocol.py +unilabos/sim/backends/isaac_bridge.py +tests/sim/test_physics_backend.py +tests/sim/test_context_and_clock.py +tests/sim/backends/test_fake_physics.py +tests/sim/backends/test_isaac_protocol.py +tests/sim/backends/test_isaac_bridge.py +``` + +- [ ] **Step 2: Run targeted C1/C2 tests on 4090** + +Run in `conda activate unilab`: + +```bash +pytest tests/sim/test_physics_backend.py \ + tests/sim/test_context_and_clock.py \ + tests/sim/backends/test_fake_physics.py \ + tests/sim/backends/test_isaac_protocol.py \ + tests/sim/backends/test_isaac_bridge.py -q +``` + +Expected: PASS. + +- [ ] **Step 3: Run Phase 1/3 regression slice on 4090** + +Run: + +```bash +pytest tests/sim tests/queries tests/integration -q +``` + +Expected: PASS, preserving the previous 98-test baseline plus the new C1/C2 tests. + +- [ ] **Step 4: Commit point** + +Commit message: `test(sim): cover phase2 isaac bridge c1 c2` + +## Self Review + +- Spec coverage: C1 fake backend, protocol extension, runtime physics config, C2 JSON protocol, and C2 HTTP bridge are all covered by tasks. +- Placeholder scan: The plan contains concrete paths, commands, expected outcomes, and code snippets for each implementation step. +- Type consistency: The backend method names match `PhysicsBackend`: `reset`, `step`, `load_scene`, `get_observation`, `set_command`, `attach_rigid_body`, `get_joint_states`, `apply_wrench`, `register_contact_callback`, and `render`. diff --git a/docs/superpowers/plans/2026-06-02-phase2-isaac-c3-worker.md b/docs/superpowers/plans/2026-06-02-phase2-isaac-c3-worker.md new file mode 100644 index 00000000..54ad66d9 --- /dev/null +++ b/docs/superpowers/plans/2026-06-02-phase2-isaac-c3-worker.md @@ -0,0 +1,520 @@ +# Phase 2 Isaac C3 Worker 实施计划 + +> **给执行 agent 的要求:** 实施本计划时必须使用 `superpowers:subagent-driven-development` 或 `superpowers:executing-plans`,逐项执行并用 checkbox 记录状态。 + +**目标:** 新增一个可以在 4090 `matterix` 环境中独立运行的 Isaac worker,对外提供 C1/C2 已定义的 `/rpc` 物理后端协议。 + +**架构:** C3 只做 Isaac 端 worker,不接 edge。worker 分成两层:一层是无 Isaac 依赖的 HTTP/RPC 壳,能在普通 `unilab` 环境单测;另一层是 lazy import 的 Isaac controller,只在 `matterix` 环境启动 `SimulationApp`、加载 LabUtopia USD、step、读取 observation、render PNG。 + +**技术栈:** Python 3.11、`http.server`、`json`、C1/C2 的 `unilabos.sim.backends.isaac.protocol`、Isaac Sim、pytest。 + +--- + +## 当前事实 + +- C1/C2 已完成: + - `PhysicsBackend.load_scene()` / `render()` + - `FakePhysicsBackend` + - `IsaacBridgeBackend` + - `/rpc` JSON request 编码 +- 4090 上环境分工: + - `unilab`: 跑 edge 和普通 pytest + - `matterix`: 跑 Isaac / LabUtopia headless +- 4090 上已有参考 demo: + - `/home/ubuntu/isaac_roboarm_bridge/isaac_virtual_leader_server.py` + - 它使用 `ThreadingHTTPServer`、延迟 Isaac import、CLI 参数和 HTTP handler。 +- LabUtopia USD 已确认存在: + - `/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd` + - `/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_003/lab_003.usd` + - `/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/hard_task/lab_004.usd` +- 本地 repo 里有 `robot_assets/roboarm_chem_04`,但它是早期 kinematic query asset,不代表高保真动力学模型。C3 不能声称完成校准机器人动力学。 + +## 文件清单 + +- 修改 `unilabos/sim/backends/isaac/protocol.py` + - 增加 worker 端 helper:`decode_request()`、`encode_response()`、`encode_error()`。 +- 新增 `unilabos/sim/backends/isaac/worker_http.py` + - 无 Isaac 依赖的 `/health` 和 `/rpc` HTTP server。 +- 新增 `unilabos/sim/backends/isaac/worker.py` + - Isaac worker CLI 入口。 + - `IsaacWorkerState` 负责 `/rpc` op 分发。 + - `IsaacController` 负责 Isaac API,必须 lazy import。 +- 新增 `scripts/smoke_isaac_worker.py` + - 连接一个已经运行的 worker,load scene、step、query observation、render PNG。 +- 新增测试: + - `tests/sim/backends/test_isaac_worker_protocol.py` + - `tests/sim/backends/test_isaac_worker_http.py` + - `tests/sim/backends/test_isaac_worker_cli.py` + - `tests/sim/backends/test_isaac_worker_smoke_script.py` + +## Task 1: 增加 worker 端 JSON helper + +**文件:** +- 修改 `unilabos/sim/backends/isaac/protocol.py` +- 新增 `tests/sim/backends/test_isaac_worker_protocol.py` + +- [ ] **Step 1: 写失败测试** + +测试内容要覆盖: + +```python +def test_decode_request_reads_operation_and_args(): + op, args = decode_request(b'{"op":"step","args":{"dt":0.05}}') + assert op == "step" + assert args == {"dt": 0.05} + + +def test_decode_request_rejects_missing_operation(): + with pytest.raises(ValueError, match="RPC request missing op"): + decode_request(b'{"args":{}}') + + +def test_encode_response_matches_client_decode_shape(): + body = encode_response({"ok_value": 1}) + assert json.loads(body.decode("utf-8")) == {"ok": True, "result": {"ok_value": 1}} + + +def test_encode_error_matches_client_decode_shape(): + body = encode_error("bad scene") + assert json.loads(body.decode("utf-8")) == {"ok": False, "error": "bad scene"} +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_protocol.py -q +``` + +预期:失败,因为 helper 还不存在。 + +- [ ] **Step 3: 实现 helper** + +在 `protocol.py` 增加: + +```python +def decode_request(data: bytes) -> tuple[str, dict[str, Any]]: + payload = json.loads(data.decode("utf-8")) + op = payload.get("op") + if not op: + raise ValueError("RPC request missing op") + args = payload.get("args") or {} + if not isinstance(args, dict): + raise ValueError("RPC request args must be an object") + return str(op), dict(args) + + +def encode_response(result: Any = None) -> bytes: + return json.dumps({"ok": True, "result": result}, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def encode_error(error: str) -> bytes: + return json.dumps({"ok": False, "error": str(error)}, separators=(",", ":"), ensure_ascii=False).encode("utf-8") +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_protocol.py tests/sim/backends/test_isaac_protocol.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/backends/isaac/protocol.py tests/sim/backends/test_isaac_worker_protocol.py +git commit -m "feat(sim): add isaac worker protocol helpers" +``` + +## Task 2: 增加无 Isaac 依赖的 HTTP server 壳 + +**文件:** +- 新增 `unilabos/sim/backends/isaac/worker_http.py` +- 新增 `tests/sim/backends/test_isaac_worker_http.py` + +- [ ] **Step 1: 写失败测试** + +测试要启动本地 `ThreadingHTTPServer`,用 fake state 覆盖: + +- `GET /health` 返回 JSON。 +- `POST /rpc` 能 dispatch 到 `state.dispatch(op, args)`。 +- dispatch 抛异常时返回 HTTP 500 和 `{"ok": false, "error": "..."}`。 + +核心测试形态: + +```python +class FakeWorkerState: + def __init__(self): + self.calls = [] + + def health(self): + return {"ok": True, "backend": "fake_isaac_worker"} + + def dispatch(self, op, args): + self.calls.append((op, args)) + if op == "explode": + raise RuntimeError("boom") + return {"op": op, "args": args} +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_http.py -q +``` + +预期:失败,因为 `worker_http.py` 不存在。 + +- [ ] **Step 3: 实现 HTTP 壳** + +`worker_http.py` 必须只依赖 stdlib 和 `protocol.py`: + +```python +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def make_handler(worker_state: Any): + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def do_GET(self): + path = self.path.split("?", 1)[0] + if path == "/health": + self._write_json(worker_state.health(), status=200) + return + self.send_response(404) + self.end_headers() + + def do_POST(self): + path = self.path.split("?", 1)[0] + if path != "/rpc": + self.send_response(404) + self.end_headers() + return + try: + length = int(self.headers.get("Content-Length", "0") or "0") + op, args = decode_request(self.rfile.read(length)) + self._write_body(encode_response(worker_state.dispatch(op, args)), status=200) + except Exception as exc: + self._write_body(encode_error(str(exc)), status=500) +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_http.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/backends/isaac/worker_http.py tests/sim/backends/test_isaac_worker_http.py +git commit -m "feat(sim): add isaac worker HTTP shell" +``` + +## Task 3: 增加 Isaac worker CLI 和 lazy controller + +**文件:** +- 新增 `unilabos/sim/backends/isaac/worker.py` +- 新增 `tests/sim/backends/test_isaac_worker_cli.py` + +- [ ] **Step 1: 写失败测试** + +测试目标: + +- import `worker.py` 时不加载 `isaacsim` / `omni.usd`。 +- `parse_args([])` 默认值正确。 +- `IsaacWorkerState.dispatch()` 能把 op 分发给 controller。 + +关键断言: + +```python +def test_worker_import_does_not_import_isaac_modules(): + assert "isaacsim" not in sys.modules + assert "omni.usd" not in sys.modules + + +def test_worker_parse_args_defaults(): + args = worker.parse_args([]) + assert args.host == "127.0.0.1" + assert args.port == 8091 + assert args.headless is True + assert args.scene is None + assert args.camera == "/World/Camera" +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_cli.py -q +``` + +预期:失败,因为 `worker.py` 不存在。 + +- [ ] **Step 3: 实现 `worker.py`** + +CLI 参数: + +```python +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Uni-Lab-OS Isaac physics worker") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8091) + parser.add_argument("--scene", default=None) + parser.add_argument("--robot-prim", default=None) + parser.add_argument("--camera", default="/World/Camera") + parser.add_argument("--headless", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--warmup-steps", type=int, default=2) + return parser.parse_args(argv) +``` + +`IsaacWorkerState.dispatch()` 必须支持这些 op: + +- `reset` +- `step` +- `load_scene` +- `get_observation` +- `set_command` +- `attach_rigid_body` +- `get_joint_states` +- `apply_wrench` +- `render` + +`render` 返回给 HTTP client 时必须 base64 包装: + +```python +if op == "render": + image = self.controller.render(str(args["camera"]), int(args["width"]), int(args["height"])) + return {"encoding": "base64", "data": base64.b64encode(image).decode("ascii")} +``` + +`IsaacController` 要求: + +- 只在 `__init__()` 内部 import Isaac: + +```python +from isaacsim import SimulationApp +``` + +- `load_scene(scene_path)` 使用 `omni.usd.get_context().open_stage(scene_path)`。 +- `step(dt)` 至少调用 `self.app.update()`。 +- `set_command(entity_id, command)` 先记录命令,不要求 C3 完成真实 articulation 控制。 +- `get_observation(entity_id)` 返回至少: + +```python +{ + "entity_id": entity_id, + "scene_path": self.scene_path, + "last_command": self.commands.get(entity_id), + "source": "isaac_worker", +} +``` + +如果 prim 存在,再加入 pose。 + +- `render(camera, width, height)` 优先用 Isaac capture;如果 Isaac capture API 失败,允许返回 minimal fallback PNG,但必须在 observation 或日志里标出 `render_fallback="minimal_png"`。C5 会把“真实画面像素”作为更严格验收。 + +- [ ] **Step 4: 验证普通环境可导入** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_cli.py tests/sim/backends/test_isaac_worker_http.py -q +``` + +预期:通过,并且不需要 Isaac。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/backends/isaac/worker.py tests/sim/backends/test_isaac_worker_cli.py +git commit -m "feat(sim): add isaac worker CLI" +``` + +## Task 4: 增加 worker smoke client + +**文件:** +- 新增 `scripts/smoke_isaac_worker.py` +- 新增 `tests/sim/backends/test_isaac_worker_smoke_script.py` + +- [ ] **Step 1: 写失败测试** + +测试 `parse_args()`: + +```python +def test_smoke_script_parse_args_defaults(tmp_path): + args = smoke_isaac_worker.parse_args([ + "--endpoint", "http://127.0.0.1:8091", + "--out", str(tmp_path / "frame.png"), + ]) + + assert args.endpoint == "http://127.0.0.1:8091" + assert args.camera == "/World/Camera" + assert args.width == 640 + assert args.height == 480 +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_smoke_script.py -q +``` + +预期:失败,因为脚本不存在。 + +- [ ] **Step 3: 实现脚本** + +脚本使用 `IsaacBridgeBackend`: + +```python +backend = IsaacBridgeBackend(args.endpoint, timeout=30.0) +if args.scene: + backend.load_scene(args.scene) +backend.step(0.016) +observation = backend.get_observation(args.entity) +image = backend.render(args.camera, args.width, args.height) +Path(args.out).write_bytes(image) +``` + +参数: + +- `--endpoint` +- `--scene` +- `--entity` +- `--camera` +- `--width` +- `--height` +- `--out` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/backends/test_isaac_worker_smoke_script.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add scripts/smoke_isaac_worker.py tests/sim/backends/test_isaac_worker_smoke_script.py +git commit -m "test(sim): add isaac worker smoke client" +``` + +## Task 5: 在 4090 的 `matterix` 中验证 C3 + +**文件:** 不新增源文件,只执行验证。 + +- [ ] **Step 1: 同步到 4090 独立目录** + +不要覆盖 4090 上 dirty 的 `~/canonical/Uni-Lab-OS`。 + +```bash +ssh ubuntu@172.20.0.39 'rm -rf /tmp/Uni-Lab-OS-c3-worker && mkdir -p /tmp/Uni-Lab-OS-c3-worker' +rsync -a --exclude .git --exclude __pycache__ --exclude .pytest_cache ./ \ + ubuntu@172.20.0.39:/tmp/Uni-Lab-OS-c3-worker/ +``` + +- [ ] **Step 2: 在 `unilab` 跑 C3 单测** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c3-worker \ + python -m pytest tests/sim/backends/test_isaac_worker_protocol.py \ + tests/sim/backends/test_isaac_worker_http.py \ + tests/sim/backends/test_isaac_worker_cli.py \ + tests/sim/backends/test_isaac_worker_smoke_script.py -q' +``` + +预期:通过。 + +- [ ] **Step 3: 在 `matterix` 启动 worker** + +```bash +ssh ubuntu@172.20.0.39 ' + pkill -f "unilabos.sim.backends.isaac.worker" || true + cd /tmp/Uni-Lab-OS-c3-worker + nohup /home/ubuntu/miniforge3/bin/conda run -n matterix env PYTHONPATH=. \ + python -m unilabos.sim.backends.isaac.worker \ + --host 127.0.0.1 \ + --port 8091 \ + --headless \ + --scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --camera /World/Camera \ + > /tmp/isaac_worker_c3.log 2>&1 & + echo $! > /tmp/isaac_worker_c3.pid +' +``` + +Isaac 启动慢,等 60 秒后检查: + +```bash +ssh ubuntu@172.20.0.39 'sleep 60; tail -n 80 /tmp/isaac_worker_c3.log; ss -ltn | grep 8091' +``` + +预期:worker 进程仍在,`8091` 正在监听。 + +- [ ] **Step 4: 运行 smoke client** + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-c3-worker + /home/ubuntu/miniforge3/bin/conda run -n unilab env PYTHONPATH=. \ + python scripts/smoke_isaac_worker.py \ + --endpoint http://127.0.0.1:8091 \ + --scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --entity /World \ + --camera /World/Camera \ + --out /tmp/labutopia-c3-worker.png + file /tmp/labutopia-c3-worker.png + ls -lh /tmp/labutopia-c3-worker.png +' +``` + +预期: + +- smoke script 退出码为 0。 +- `/tmp/labutopia-c3-worker.png` 存在且非空。 +- `/tmp/isaac_worker_c3.log` 没有 Python traceback。 + +- [ ] **Step 5: 停 worker** + +```bash +ssh ubuntu@172.20.0.39 'kill $(cat /tmp/isaac_worker_c3.pid) || true' +``` + +- [ ] **Step 6: 跑回归切片** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c3-worker \ + python -m pytest tests/sim tests/queries tests/integration -q' +``` + +预期:通过。 + +- [ ] **Step 7: 提交点** + +```bash +git add . +git commit -m "test(sim): verify isaac worker on 4090" +``` + +## C3 验收标准 + +- worker HTTP 壳可以在无 Isaac 环境单测。 +- import `unilabos.sim.backends.isaac.worker` 不会加载 Isaac 模块。 +- 4090 `matterix` 能在 `127.0.0.1:8091` 启动 worker。 +- worker 能 load LabUtopia `lab_001.usd`。 +- smoke client 能 step、get observation、写 PNG。 +- 4090 回归 `pytest tests/sim tests/queries tests/integration -q` 保持通过。 + +## C3 不做什么 + +- 不接 `unilab --mode sim --physics isaac`,那是 C4。 +- 不要求 query API 返回 physics state,那是 C5。 +- 不做机器人校准控制、抓取、contact、attachment。 +- 不保证渲染像素语义完全正确;C3 只证明 worker render 管道能返回 image bytes。 diff --git a/docs/superpowers/plans/2026-06-02-phase2-isaac-c4-edge-integration.md b/docs/superpowers/plans/2026-06-02-phase2-isaac-c4-edge-integration.md new file mode 100644 index 00000000..4ecbe5c7 --- /dev/null +++ b/docs/superpowers/plans/2026-06-02-phase2-isaac-c4-edge-integration.md @@ -0,0 +1,695 @@ +# Phase 2 Isaac C4 Edge 集成实施计划 + +> **给执行 agent 的要求:** 实施本计划时必须使用 `superpowers:subagent-driven-development` 或 `superpowers:executing-plans`,逐项执行并用 checkbox 记录状态。 + +**目标:** 把 C1/C2 的 physics backend 接入 Uni-Lab-OS edge 启动流程,使 `unilab --mode sim --physics fake|isaac` 能初始化 `RuntimeContext.physics`,并让代表性的 HAL / 虚拟设备命令走 physics backend。 + +**架构:** C4 只做 edge 侧集成,不做 Isaac worker 本体。CLI 增加 physics 参数,`app/backend.py` 通过 factory 构造 backend,`RuntimeContext` 成为进程内 physics 的唯一入口。HAL 侧先接 `URHAL`,因为它已有 sim backend 契约;虚拟设备侧先接 `VirtualMultiwayValve` 作为 opt-in 样例,不一次性重写所有虚拟设备。 + +**技术栈:** Python 3.11、argparse、`RuntimeContext`、C1 `FakePhysicsBackend`、C2 `IsaacBridgeBackend`、pytest。 + +--- + +## 当前事实 + +- `unilabos/app/main.py` 已有: + - `--mode` + - `--sim_rate` + - `--sim_paused` + - `--disable_sim_services` + - query 相关参数 +- `unilabos/app/backend.py:start_backend()` 当前会调用 `configure_runtime()`,并把 query 配置写入 `_runtime_services.context`。 +- C1 已给 `RuntimeContext` 增加: + - `physics_backend_name` + - `physics_endpoint` + - `physics_scene` +- `URHAL` 已支持显式传入 `mode="sim", sim_backend=...`。 +- 当前 repo 没有 `VirtualArm` 类。虚拟设备主要是化学设备,因此 C4 不应该虚构机械臂虚拟设备。 +- C5 才负责 query API 返回 physics observation;C4 只负责把 physics backend 初始化和基本命令路由接通。 + +## 文件清单 + +- 修改 `unilabos/app/main.py` + - 新增 CLI 参数:`--physics`、`--physics_endpoint`、`--physics_scene`。 +- 新增 `unilabos/sim/backends/factory.py` + - 构造 `None`、`FakePhysicsBackend` 或 `IsaacBridgeBackend`。 +- 修改 `unilabos/sim/runtime.py` + - `configure_runtime()` 接收 physics 对象和配置字段。 +- 修改 `unilabos/app/backend.py` + - 在启动 backend 时构造并写入 `RuntimeContext.physics`。 +- 新增 `unilabos/sim/device_physics.py` + - 给虚拟设备用的小 helper,避免每个虚拟设备直接 import backend 类型。 +- 修改 `unilabos/hal/adapters/ur_adapter.py` + - sim 模式优先用显式 `sim_backend`,否则读 `get_runtime_context().physics`。 +- 修改 `unilabos/devices/virtual/virtual_multiway_valve.py` + - `set_position()` / `close()` 发送代表性 command 到 physics backend。 +- 修改或新增测试: + - `tests/sim/test_cli_runtime.py` + - `tests/sim/backends/test_factory.py` + - `tests/sim/test_runtime_configuration.py` + - `tests/sim/test_backend_physics_configuration.py` + - `tests/queries/test_ur_adapter.py` + - `tests/sim/test_device_physics.py` + - `tests/sim/test_virtual_device_clock.py` + +## Task 1: 增加 CLI physics 参数 + +**文件:** +- 修改 `unilabos/app/main.py` +- 修改 `tests/sim/test_cli_runtime.py` + +- [ ] **Step 1: 写失败测试** + +在 `tests/sim/test_cli_runtime.py` 增加: + +```python +def test_cli_physics_defaults_are_backward_compatible(): + args = build_argparser().parse_args([]) + + assert args.physics == "none" + assert args.physics_endpoint is None + assert args.physics_scene is None + + +def test_cli_accepts_isaac_physics_options(): + args = build_argparser().parse_args( + [ + "--mode", + "sim", + "--physics", + "isaac", + "--physics_endpoint", + "http://127.0.0.1:8091", + "--physics_scene", + "/tmp/lab.usd", + ] + ) + + assert args.physics == "isaac" + assert args.physics_endpoint == "http://127.0.0.1:8091" + assert args.physics_scene == "/tmp/lab.usd" +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/test_cli_runtime.py::test_cli_physics_defaults_are_backward_compatible \ + tests/sim/test_cli_runtime.py::test_cli_accepts_isaac_physics_options -q +``` + +预期:失败,因为参数还不存在。 + +- [ ] **Step 3: 实现 parser 参数** + +在 `--disable_sim_services` 后添加: + +```python +parser.add_argument( + "--physics", + choices=["none", "fake", "isaac"], + default="none", + help="Physics backend for sim mode: none, fake in-process backend, or Isaac HTTP bridge.", +) +parser.add_argument( + "--physics_endpoint", + type=str, + default=None, + help="Physics backend endpoint, required for --physics isaac.", +) +parser.add_argument( + "--physics_scene", + type=str, + default=None, + help="Scene path to load into the selected physics backend during startup.", +) +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/test_cli_runtime.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/app/main.py tests/sim/test_cli_runtime.py +git commit -m "feat(edge): add physics backend CLI options" +``` + +## Task 2: 增加 physics backend factory + +**文件:** +- 新增 `unilabos/sim/backends/factory.py` +- 新增 `tests/sim/backends/test_factory.py` + +- [ ] **Step 1: 写失败测试** + +测试覆盖: + +- `none` 返回 `None` +- `fake` 返回 `FakePhysicsBackend` +- `fake` 支持 `scene` 并调用 `load_scene` +- `isaac` 缺 endpoint 时抛错 +- `isaac` 有 endpoint 时返回 `IsaacBridgeBackend` +- 未知 backend 抛错 + +核心测试: + +```python +def test_factory_builds_fake_backend_and_loads_scene(): + backend = build_physics_backend("fake", endpoint=None, scene="/tmp/lab.usd") + + assert isinstance(backend, FakePhysicsBackend) + assert backend.scene_path == "/tmp/lab.usd" + + +def test_factory_requires_endpoint_for_isaac_backend(): + with pytest.raises(ValueError, match="--physics_endpoint is required"): + build_physics_backend("isaac", endpoint=None, scene=None) +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/backends/test_factory.py -q +``` + +预期:失败,因为 factory 不存在。 + +- [ ] **Step 3: 实现 factory** + +`unilabos/sim/backends/factory.py`: + +```python +def build_physics_backend( + name: str | None, + endpoint: str | None = None, + scene: str | None = None, +) -> PhysicsBackend | None: + backend_name = (name or "none").strip().lower() + if backend_name == "none": + return None + if backend_name == "fake": + from unilabos.sim.backends.fake_physics import FakePhysicsBackend + backend: PhysicsBackend = FakePhysicsBackend() + elif backend_name == "isaac": + if not endpoint: + raise ValueError("--physics_endpoint is required when --physics isaac") + from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend + backend = IsaacBridgeBackend(endpoint) + else: + raise ValueError(f"Unsupported physics backend: {name}") + + if scene: + backend.load_scene(scene) + return backend +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/backends/test_factory.py tests/sim/backends/test_fake_physics.py tests/sim/backends/test_isaac_bridge.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/backends/factory.py tests/sim/backends/test_factory.py +git commit -m "feat(sim): add physics backend factory" +``` + +## Task 3: 让 `configure_runtime()` 携带 physics + +**文件:** +- 修改 `unilabos/sim/runtime.py` +- 修改 `tests/sim/test_runtime_configuration.py` + +- [ ] **Step 1: 写失败测试** + +新增: + +```python +class DummyPhysics: + name = "dummy" + + +def test_configure_runtime_stores_physics_backend_and_config(): + physics = DummyPhysics() + + services = configure_runtime( + mode="sim", + physics=physics, + physics_backend_name="fake", + physics_endpoint="http://127.0.0.1:8091", + physics_scene="/tmp/lab.usd", + ) + + assert services.context.physics is physics + assert get_runtime_context().physics is physics + assert services.context.physics_backend_name == "fake" + assert services.context.physics_endpoint == "http://127.0.0.1:8091" + assert services.context.physics_scene == "/tmp/lab.usd" +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/test_runtime_configuration.py::test_configure_runtime_stores_physics_backend_and_config -q +``` + +预期:失败,因为 `configure_runtime()` 还不接收这些参数。 + +- [ ] **Step 3: 修改 `configure_runtime()`** + +新增参数: + +```python +physics=None +physics_backend_name: str = "none" +physics_endpoint: str | None = None +physics_scene: str | None = None +``` + +构造 `RuntimeContext` 时传入: + +```python +context = RuntimeContext( + mode=mode, + clock=clock, + sim_paused=sim_paused, + physics=physics, + physics_backend_name=physics_backend_name, + physics_endpoint=physics_endpoint, + physics_scene=physics_scene, +) +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/test_runtime_configuration.py tests/sim/test_context_and_clock.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/runtime.py tests/sim/test_runtime_configuration.py +git commit -m "feat(sim): pass physics backend through runtime configuration" +``` + +## Task 4: 在 `app/backend.py` 初始化 physics + +**文件:** +- 修改 `unilabos/app/backend.py` +- 新增 `tests/sim/test_backend_physics_configuration.py` + +- [ ] **Step 1: 写失败测试** + +新增 helper 测试,不启动 ROS thread: + +```python +def test_initialize_runtime_for_backend_builds_fake_physics(): + services = backend_mod._initialize_runtime_for_backend( + backend="ros", + kwargs={ + "mode": "sim", + "sim_rate": 10.0, + "sim_paused": True, + "physics": "fake", + "physics_endpoint": None, + "physics_scene": "/tmp/lab.usd", + "disable_sim_services": False, + "disable_query_api": False, + "query_grpc_port": 50051, + }, + ) + + assert isinstance(services.context.physics, FakePhysicsBackend) + assert services.context.physics_backend_name == "fake" + assert services.context.physics_scene == "/tmp/lab.usd" + assert services.context.sim_services_enabled is True + assert services.context.query_api_enabled is True +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/test_backend_physics_configuration.py -q +``` + +预期:失败,因为 `_initialize_runtime_for_backend()` 不存在。 + +- [ ] **Step 3: 抽出 runtime 初始化 helper** + +在 `unilabos/app/backend.py` 添加: + +```python +from unilabos.sim.backends.factory import build_physics_backend +``` + +新增: + +```python +def _initialize_runtime_for_backend(backend: str, kwargs: dict) -> RuntimeServices: + mode = kwargs.get("mode", "real") + sim_rate = kwargs.get("sim_rate", 1.0) + sim_paused = kwargs.get("sim_paused", False) + physics_name = kwargs.get("physics", "none") + physics_endpoint = kwargs.get("physics_endpoint") + physics_scene = kwargs.get("physics_scene") + physics = build_physics_backend(physics_name, endpoint=physics_endpoint, scene=physics_scene) + start_sim_services = backend == "ros" and not kwargs.get("disable_sim_services", False) + services = configure_runtime( + mode=mode, + sim_rate=sim_rate, + sim_paused=sim_paused, + start_ros_services=False, + physics=physics, + physics_backend_name=physics_name, + physics_endpoint=physics_endpoint, + physics_scene=physics_scene, + ) + services.context.sim_services_enabled = start_sim_services and mode in ("sim", "twin") + services.context.query_api_enabled = backend == "ros" and not kwargs.get("disable_query_api", False) + services.context.query_grpc_port = int(kwargs.get("query_grpc_port", 50051)) + services.context.query_labutopia_assets = kwargs.get("query_labutopia_assets") + services.context.query_labutopia_config = kwargs.get("query_labutopia_config") + services.context.query_labutopia_usd = kwargs.get("query_labutopia_usd") + return services +``` + +在 `start_backend()` 中用它替换原有 runtime 初始化块: + +```python +_runtime_services = _initialize_runtime_for_backend(backend, kwargs) +``` + +日志补充: + +```python +f"physics={_runtime_services.context.physics_backend_name}, " +f"physics_endpoint={_runtime_services.context.physics_endpoint}, " +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/test_backend_physics_configuration.py tests/sim/test_runtime_configuration.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/app/backend.py tests/sim/test_backend_physics_configuration.py +git commit -m "feat(edge): initialize physics backend at startup" +``` + +## Task 5: 让 URHAL sim 模式默认使用 RuntimeContext.physics + +**文件:** +- 修改 `unilabos/hal/adapters/ur_adapter.py` +- 修改 `tests/queries/test_ur_adapter.py` + +- [ ] **Step 1: 写失败测试** + +新增测试: + +```python +def test_sim_mode_defaults_to_runtime_physics_backend(): + _reset_for_test() + backend = FakeSimBackend() + init_runtime_context(RuntimeContext(mode="sim", physics=backend, physics_backend_name="fake")) + try: + hal = URHAL(host="sim", robot_id="ur5_runtime", mode="sim") + hal.move_j([0, 1, 2, 3, 4, 5], speed=0.4) + + assert backend.commands[-1] == ( + "ur5_runtime", + {"type": "move_j", "joint_positions": [0, 1, 2, 3, 4, 5], "speed": 0.4}, + ) + finally: + _reset_for_test() +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/queries/test_ur_adapter.py::URAdapterTest::test_sim_mode_defaults_to_runtime_physics_backend -q +``` + +预期:失败,因为当前 `URHAL` 需要显式 `sim_backend`。 + +- [ ] **Step 3: 实现 runtime 默认 backend** + +在 `URHAL` 增加: + +```python +def _active_sim_backend(self): + if self.sim_backend is not None: + return self.sim_backend + from unilabos.sim.context import get_runtime_context + + backend = get_runtime_context().physics + if backend is None: + raise RuntimeError("URHAL sim mode requires sim_backend or RuntimeContext.physics") + return backend +``` + +替换 `_sim_observation()` / `_sim_command()`: + +```python +def _sim_observation(self) -> dict[str, Any]: + return dict(self._active_sim_backend().get_observation(self.robot_id)) + + +def _sim_command(self, command: dict[str, Any]) -> None: + self._active_sim_backend().set_command(self.robot_id, command) +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/queries/test_ur_adapter.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/hal/adapters/ur_adapter.py tests/queries/test_ur_adapter.py +git commit -m "feat(hal): default UR sim mode to runtime physics" +``` + +## Task 6: 增加虚拟设备 physics dispatch helper + +**文件:** +- 新增 `unilabos/sim/device_physics.py` +- 新增 `tests/sim/test_device_physics.py` + +- [ ] **Step 1: 写失败测试** + +测试内容: + +```python +def test_dispatch_device_command_noops_without_physics(): + assert dispatch_device_command("valve", {"type": "set_position"}) is False + + +def test_dispatch_device_command_sends_to_runtime_physics(): + physics = RecordingPhysics() + init_runtime_context(RuntimeContext(mode="sim", physics=physics, physics_backend_name="fake")) + + assert dispatch_device_command("valve", {"type": "set_position", "position": 3}) is True + assert physics.commands == [("valve", {"type": "set_position", "position": 3})] +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/test_device_physics.py -q +``` + +预期:失败,因为 helper 不存在。 + +- [ ] **Step 3: 实现 helper** + +`unilabos/sim/device_physics.py`: + +```python +def dispatch_device_command(entity_id: str, command: dict[str, Any]) -> bool: + backend = get_runtime_context().physics + if backend is None: + return False + backend.set_command(str(entity_id), dict(command)) + return True +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/test_device_physics.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/sim/device_physics.py tests/sim/test_device_physics.py +git commit -m "feat(sim): add virtual device physics dispatch helper" +``` + +## Task 7: 将 `VirtualMultiwayValve` 作为代表性虚拟设备接入 physics + +**文件:** +- 修改 `unilabos/devices/virtual/virtual_multiway_valve.py` +- 修改 `tests/sim/test_virtual_device_clock.py` + +- [ ] **Step 1: 写失败测试** + +新增: + +```python +def test_virtual_multiway_valve_dispatches_position_to_physics(): + physics = FakePhysicsBackend() + init_runtime_context(RuntimeContext(mode="sim", clock=SimClock("sim", scale=100.0), physics=physics)) + valve = VirtualMultiwayValve(id="valve_a", positions=8) + + valve.set_position(3) + + assert physics.commands["valve_a"] == { + "type": "set_position", + "position": 3, + "device": "virtual_multiway_valve", + } +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/sim/test_virtual_device_clock.py::test_virtual_multiway_valve_dispatches_position_to_physics -q +``` + +预期:失败,因为 valve 还不会 dispatch physics command。 + +- [ ] **Step 3: 实现 opt-in dispatch** + +在 `VirtualMultiwayValve.__init__` 存稳定 id: + +```python +self.device_id = kwargs.get("device_id") or kwargs.get("id") or self.port +``` + +导入 helper: + +```python +from unilabos.sim.device_physics import dispatch_device_command +``` + +在 `set_position()` 设置 `_target_position` 后添加: + +```python +dispatch_device_command( + self.device_id, + {"type": "set_position", "position": pos, "device": "virtual_multiway_valve"}, +) +``` + +在 `close()` 中设置 closing 状态后添加: + +```python +dispatch_device_command( + self.device_id, + {"type": "close", "position": self._current_position, "device": "virtual_multiway_valve"}, +) +``` + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/sim/test_virtual_device_clock.py tests/sim/test_device_physics.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/devices/virtual/virtual_multiway_valve.py tests/sim/test_virtual_device_clock.py +git commit -m "feat(devices): route virtual valve commands to physics" +``` + +## Task 8: 在 4090 上跑 C4 回归 + +**文件:** 不新增源文件,只执行验证。 + +- [ ] **Step 1: 同步到 4090 独立目录** + +```bash +ssh ubuntu@172.20.0.39 'rm -rf /tmp/Uni-Lab-OS-c4-edge && mkdir -p /tmp/Uni-Lab-OS-c4-edge' +rsync -a --exclude .git --exclude __pycache__ --exclude .pytest_cache ./ \ + ubuntu@172.20.0.39:/tmp/Uni-Lab-OS-c4-edge/ +``` + +- [ ] **Step 2: 跑 C4 目标测试** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c4-edge \ + python -m pytest tests/sim/test_cli_runtime.py \ + tests/sim/backends/test_factory.py \ + tests/sim/test_runtime_configuration.py \ + tests/sim/test_backend_physics_configuration.py \ + tests/sim/test_device_physics.py \ + tests/sim/test_virtual_device_clock.py \ + tests/queries/test_ur_adapter.py -q' +``` + +预期:通过。 + +- [ ] **Step 3: 跑 Phase 1/3 回归切片** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c4-edge \ + python -m pytest tests/sim tests/queries tests/integration -q' +``` + +预期:通过。 + +- [ ] **Step 4: 提交点** + +```bash +git add . +git commit -m "test(edge): verify physics integration on 4090" +``` + +## C4 验收标准 + +- `build_argparser()` 接受: + - `--physics fake` + - `--physics isaac` + - `--physics_endpoint` + - `--physics_scene` +- `start_backend()` 通过 factory 初始化 `RuntimeContext.physics`。 +- `--physics fake --physics_scene /tmp/lab.usd` 能在 startup 测试里加载 fake scene。 +- `URHAL(mode="sim")` 不显式传 `sim_backend` 时能使用 `RuntimeContext.physics`。 +- `VirtualMultiwayValve` 在 physics 启用时能发送代表性 command,同时保留原有 sim-clock 行为。 +- 4090 上 `pytest tests/sim tests/queries tests/integration -q` 保持通过。 + +## C4 不做什么 + +- 不启动真实 Isaac worker;C3 已覆盖 worker 独立运行。 +- 不让 query API 返回 physics observation;那是 C5。 +- 不把所有虚拟设备一次性改成 physics-aware。 +- 不启用 Feetech 真实动作;`FeetechRoboArmHAL` 仍保持 read-only。 diff --git a/docs/superpowers/plans/2026-06-02-phase2-isaac-c5-e2e-render.md b/docs/superpowers/plans/2026-06-02-phase2-isaac-c5-e2e-render.md new file mode 100644 index 00000000..9d137634 --- /dev/null +++ b/docs/superpowers/plans/2026-06-02-phase2-isaac-c5-e2e-render.md @@ -0,0 +1,559 @@ +# Phase 2 Isaac C5 端到端与真画面验收实施计划 + +> **给执行 agent 的要求:** 实施本计划时必须使用 `superpowers:subagent-driven-development` 或 `superpowers:executing-plans`,逐项执行并用 checkbox 记录状态。 + +**目标:** 在 4090 上证明完整 Route A 链路:Isaac worker 在 `matterix` 中运行,Uni-Lab-OS edge 在 `unilab` 中以 `--physics isaac` 运行,query API 能看到 physics observation,并且能从 Isaac worker 取回 LabUtopia 场景 PNG。 + +**架构:** C5 不改 query transport。新增 `PhysicsLiveSource`,它按需读取 `RuntimeContext.physics.get_observation(target)` 并转成 `Pose` / `State`。edge 启动 query API 时把它插入现有 source 链路:`RosLiveSource` 优先,其次 `PhysicsLiveSource`,最后 LabUtopia 静态 source。验收通过一个 smoke script 检查 gRPC query 和 render PNG。 + +**技术栈:** Python 3.11、现有 `QueryEngine`、`RosLiveSource`、C2 `IsaacBridgeBackend`、C3 Isaac worker、ROS2/gRPC query service、4090 `unilab`、4090 `matterix`、pytest。 + +--- + +## 当前事实 + +- C1/C2 已有 physics contract 和 HTTP bridge。 +- C3 应提供 worker:`http://127.0.0.1:8092/rpc`;4090 上 `8091` 可能已有其他 Isaac demo 占用。 +- C4 应提供 edge CLI: + - `--mode sim` + - `--physics isaac` + - `--physics_endpoint` + - `--physics_scene` +- 现有 query startup 在 `unilabos/ros/main_slave_run.py:_start_query_services()`。 +- 现有 live source 顺序是 `RosLiveSource` 在最前面。 +- 4090 上 LabUtopia USD: + - `/home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd` +- 4090 上 asset cards 目录在: + - `/home/ubuntu/lab4090/projects/robo-unilabos-phase13/generated/` + - 最终跑 C5 前要重新确认具体使用哪个 asset-card 子目录。 +- 4090 的 `~/canonical/Uni-Lab-OS` 曾经是 dirty 状态。C5 验证继续使用 `/tmp` 干净副本或新 clone,不覆盖 canonical。 + +## 文件清单 + +- 新增 `unilabos/queries/physics_live_source.py` + - 把 physics observation 映射成 query `Pose` / `State`。 +- 修改 `unilabos/queries/__init__.py` + - 导出 `PhysicsLiveSource`。 +- 修改 `unilabos/ros/main_slave_run.py` + - runtime 有 physics 时,把 `PhysicsLiveSource` 加入 query source。 +- 新增 `tests/queries/test_physics_live_source.py` + - 覆盖 pose/state 映射。 +- 修改 `tests/integration/test_edge_query_wiring.py` + - 覆盖 query source 构造顺序。 +- 新增 `scripts/smoke_sim_isaac_edge.py` + - 通过 gRPC 查询 state/pose,并通过 `IsaacBridgeBackend.render()` 写 PNG。 +- 新增 `tests/integration/test_smoke_sim_isaac_edge_script.py` + - 覆盖 smoke script 参数解析和 PNG 判断 helper。 +- 新增 `docs/demo/phase2_isaac_e2e_4090.md` + - 记录 4090 端到端命令和期望证据。 + +## Task 1: 新增 PhysicsLiveSource + +**文件:** +- 新增 `unilabos/queries/physics_live_source.py` +- 修改 `unilabos/queries/__init__.py` +- 新增 `tests/queries/test_physics_live_source.py` + +- [ ] **Step 1: 写失败测试** + +新测试覆盖: + +- observation 中有 `pose` dict 时能转为 `Pose`。 +- observation 中有 UR 风格 `tcp_pose` 时能转为 `Pose`。 +- observation 能转为 `State`。 +- target 缺失时返回 `None`,让 query engine 继续 fallback 到静态 source。 +- frame mismatch 时返回 `None`。 + +核心测试: + +```python +class FakePhysics: + name = "fake" + + def __init__(self): + self.observations = { + "arm": { + "entity_id": "arm", + "pose": {"xyz": [0.1, 0.2, 0.3], "quat_xyzw": [0, 0, 0, 1], "frame_id": "world"}, + "joint_positions": [1.0, 2.0], + "joint_names": ["j1", "j2"], + }, + "tool": { + "entity_id": "tool", + "tcp_pose": [0.4, 0.5, 0.6, 0.0, 0.0, 0.0], + }, + } + + def get_observation(self, entity_id): + if entity_id not in self.observations: + raise KeyError(entity_id) + return dict(self.observations[entity_id]) +``` + +关键断言: + +```python +def test_physics_live_source_maps_pose_dict_to_query_pose(): + source = PhysicsLiveSource(FakePhysics()) + pose = source.query_pose("arm") + + assert pose.xyz == [0.1, 0.2, 0.3] + assert pose.quat_xyzw == [0, 0, 0, 1] + assert pose.frame_id == "world" + assert pose.source == "physics_live:fake" + + +def test_physics_live_source_maps_observation_to_state(): + source = PhysicsLiveSource(FakePhysics()) + state = source.query_state("arm") + + assert state.values["entity_id"] == "arm" + assert state.values["joint_positions"] == [1.0, 2.0] + assert state.values["joint_names"] == ["j1", "j2"] +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/queries/test_physics_live_source.py -q +``` + +预期:失败,因为 `PhysicsLiveSource` 不存在。 + +- [ ] **Step 3: 实现 `PhysicsLiveSource`** + +`unilabos/queries/physics_live_source.py`: + +```python +class PhysicsLiveSource: + name = "physics_live" + + def __init__(self, physics_backend: Any): + self.physics_backend = physics_backend + + @property + def _source_name(self) -> str: + return f"physics_live:{getattr(self.physics_backend, 'name', 'unknown')}" + + def _observation(self, target: str) -> Optional[dict[str, Any]]: + try: + return dict(self.physics_backend.get_observation(target)) + except Exception: + return None +``` + +`query_pose()` 规则: + +- 优先读 `obs["pose"]`: + - `xyz` + - `quat_xyzw` + - `frame_id` +- 如果没有 `pose`,尝试读 `tcp_pose` 或 `tool_pose`: + - 前 3 个数为 xyz + - 后 3 个 rotvec 转 quaternion +- 如果 frame 不匹配,返回 `None`。 + +`query_state()` 规则: + +```python +return State(name=target, values=obs, stamp=utc_timestamp(), source=self._source_name) +``` + +其他 query source 方法返回空: + +- `query_affordance()` 返回 `[]` +- `query_action_schema()` 返回 `None` +- `query_safety_zones()` 返回 `[]` + +在 `unilabos/queries/__init__.py` 导出: + +```python +from unilabos.queries.physics_live_source import PhysicsLiveSource +``` + +并加入 `__all__`。 + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/queries/test_physics_live_source.py tests/queries/test_ros_live_source.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add unilabos/queries/physics_live_source.py unilabos/queries/__init__.py tests/queries/test_physics_live_source.py +git commit -m "feat(query): add physics live source" +``` + +## Task 2: 在 query startup 中接入 PhysicsLiveSource + +**文件:** +- 修改 `unilabos/ros/main_slave_run.py` +- 修改 `tests/integration/test_edge_query_wiring.py` + +- [ ] **Step 1: 写失败 helper 测试** + +在 `tests/integration/test_edge_query_wiring.py` 新增: + +```python +def test_build_query_static_sources_puts_physics_before_labutopia(monkeypatch): + from unilabos.ros.main_slave_run import _build_query_static_sources + from unilabos.sim.backends.fake_physics import FakePhysicsBackend + from unilabos.sim.context import RuntimeContext + + class StaticSource: + name = "static" + + import unilabos.ros.main_slave_run as mod + monkeypatch.setattr(mod, "_build_labutopia_sources", lambda ctx: [StaticSource()]) + + sources = _build_query_static_sources(RuntimeContext(mode="sim", physics=FakePhysicsBackend())) + + assert sources[0].name == "physics_live" + assert sources[1].name == "static" +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/integration/test_edge_query_wiring.py::test_build_query_static_sources_puts_physics_before_labutopia -q +``` + +预期:失败,因为 `_build_query_static_sources()` 不存在。 + +- [ ] **Step 3: 抽出 source 构造 helper** + +在 `unilabos/ros/main_slave_run.py` 添加: + +```python +def _build_query_static_sources(ctx) -> list: + sources = [] + physics = getattr(ctx, "physics", None) + if physics is not None: + from unilabos.queries.physics_live_source import PhysicsLiveSource + + sources.append(PhysicsLiveSource(physics)) + sources.extend(_build_labutopia_sources(ctx)) + return sources +``` + +- [ ] **Step 4: 修改 `_start_query_services()`** + +把: + +```python +static_sources = _build_labutopia_sources(ctx) +``` + +替换为: + +```python +static_sources = _build_query_static_sources(ctx) +``` + +最终 query source 优先级: + +1. `RosLiveSource` +2. `PhysicsLiveSource` +3. LabUtopia USD / asset card / task config source + +- [ ] **Step 5: 验证通过** + +```bash +python -m pytest tests/integration/test_edge_query_wiring.py tests/queries/test_physics_live_source.py -q +``` + +预期:通过。 + +- [ ] **Step 6: 提交点** + +```bash +git add unilabos/ros/main_slave_run.py tests/integration/test_edge_query_wiring.py +git commit -m "feat(edge): include physics observations in query API" +``` + +## Task 3: 增加 edge + Isaac worker smoke script + +**文件:** +- 新增 `scripts/smoke_sim_isaac_edge.py` +- 新增 `tests/integration/test_smoke_sim_isaac_edge_script.py` + +- [ ] **Step 1: 写失败测试** + +测试参数解析和 PNG 判断: + +```python +def test_smoke_script_parse_args(tmp_path): + args = smoke_sim_isaac_edge.parse_args( + [ + "--grpc", + "127.0.0.1:50051", + "--physics-endpoint", + "http://127.0.0.1:8091", + "--state-target", + "arm", + "--pose-target", + "tool", + "--out", + str(tmp_path / "frame.png"), + ] + ) + + assert args.grpc == "127.0.0.1:50051" + assert args.physics_endpoint == "http://127.0.0.1:8091" + assert args.camera == "/World/Camera" + assert args.width == 640 + assert args.height == 480 + + +def test_validate_png_rejects_empty_payload(): + assert smoke_sim_isaac_edge.is_png_like(b"") is False + assert smoke_sim_isaac_edge.is_png_like(b"\x89PNG\r\n\x1a\npayload") is True +``` + +- [ ] **Step 2: 运行失败测试** + +```bash +python -m pytest tests/integration/test_smoke_sim_isaac_edge_script.py -q +``` + +预期:失败,因为脚本不存在。 + +- [ ] **Step 3: 实现 smoke script** + +`scripts/smoke_sim_isaac_edge.py` 参数: + +- `--grpc` +- `--physics-endpoint` +- `--state-target` +- `--pose-target` +- `--camera` +- `--width` +- `--height` +- `--out` +- `--poll-timeout-s` +- `--poll-interval-s` + +核心逻辑: + +```python +client = RoboUniLabOSRemote(grpc_transport(args.grpc)) +state, pose = _poll_query(client, args.state_target, args.pose_target, args.poll_timeout_s, args.poll_interval_s) + +physics = IsaacBridgeBackend(args.physics_endpoint, timeout=30.0) +image = physics.render(args.camera, args.width, args.height) +if not is_png_like(image): + raise RuntimeError(f"render payload is not PNG-like, got {len(image)} bytes") + +Path(args.out).write_bytes(image) +``` + +`_poll_query()` 要在 timeout 内反复尝试: + +- `client.query_state(state_target)` +- `client.query_pose(pose_target)` + +直到都成功,或 timeout 抛 `RuntimeError`。 + +- [ ] **Step 4: 验证通过** + +```bash +python -m pytest tests/integration/test_smoke_sim_isaac_edge_script.py -q +``` + +预期:通过。 + +- [ ] **Step 5: 提交点** + +```bash +git add scripts/smoke_sim_isaac_edge.py tests/integration/test_smoke_sim_isaac_edge_script.py +git commit -m "test(edge): add isaac e2e smoke script" +``` + +## Task 4: 写 4090 端到端 runbook + +**文件:** +- 新增 `docs/demo/phase2_isaac_e2e_4090.md` + +- [ ] **Step 1: 写 runbook** + +内容必须包含以下命令。 + +同步代码: + +```bash +ssh ubuntu@172.20.0.39 'rm -rf /tmp/Uni-Lab-OS-c5-e2e && mkdir -p /tmp/Uni-Lab-OS-c5-e2e' +rsync -a --exclude .git --exclude __pycache__ --exclude .pytest_cache ./ \ + ubuntu@172.20.0.39:/tmp/Uni-Lab-OS-c5-e2e/ +``` + +启动 Isaac worker: + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + nohup /home/ubuntu/miniforge3/bin/conda run -n matterix env PYTHONPATH=. \ + python -m unilabos.sim.backends.isaac.worker \ + --host 127.0.0.1 \ + --port 8092 \ + --headless \ + --scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --camera /World/Camera \ + --rpc-timeout-s 600 \ + > /tmp/isaac_worker_phase2_c3c5_8092.log 2>&1 & + echo $! > /tmp/isaac_worker_phase2_c3c5_8092.pid +' +``` + +不要使用 `pkill -f "unilabos.sim.backends.isaac.worker"`;该模式可能匹配并杀掉自己的 SSH shell。需要停止时先用 `pgrep -af "[u]nilabos.sim.backends.isaac.worker.*8092"` 查精确 PID。 + +启动 edge: + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + nohup /home/ubuntu/miniforge3/bin/conda run --no-capture-output -n unilab env PYTHONPATH=. \ + unilab --graph unilabos/test/experiments/mock_devices/mock_all.json \ + --config unilabos/config/example_config.py \ + --backend ros \ + --mode sim \ + --sim_rate 10 \ + --physics isaac \ + --physics_endpoint http://127.0.0.1:8092 \ + --physics_scene /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --physics_timeout 300 \ + --query_labutopia_usd /home/ubuntu/labsim/LabUtopia_repro/assets/chemistry_lab/lab_001/lab_001.usd \ + --query_grpc_port 50052 \ + --app_bridges fastapi \ + --visual disable \ + --skip_env_check \ + --disable_browser \ + --test_mode \ + --port 8002 \ + > /tmp/unilab_edge_c5_50052.log 2>&1 & + echo $! > /tmp/unilab_edge_c5_50052.pid +' +``` + +本地 graph + `--app_bridges fastapi` + 非 websocket 可以离线启动,不需要 `UNILAB_AK/UNILAB_SK`。 + +运行 smoke: + +```bash +ssh ubuntu@172.20.0.39 ' + cd /tmp/Uni-Lab-OS-phase2-c3c5 + /home/ubuntu/miniforge3/bin/conda run -n unilab env PYTHONPATH=. \ + python scripts/smoke_sim_isaac_edge.py \ + --grpc 127.0.0.1:50052 \ + --physics-endpoint http://127.0.0.1:8092 \ + --state-target /World \ + --pose-target /World \ + --camera /World/Camera \ + --physics-timeout-s 300 \ + --out /tmp/labutopia-c5-e2e.png + file /tmp/labutopia-c5-e2e.png + ls -lh /tmp/labutopia-c5-e2e.png +' +``` + +停止进程: + +```bash +ssh ubuntu@172.20.0.39 ' + kill $(cat /tmp/unilab_edge_c5_50052.pid) || true + pgrep -af "[u]nilabos.sim.backends.isaac.worker.*8092" +' +``` + +runbook 必须写明期望证据: + +- `ss -ltnp "( sport = :8092 )"` 能看到 worker。 +- `ss -ltnp "( sport = :50052 )"` 能看到 query gRPC。 +- smoke script 退出码为 0。 +- smoke 输出的 `state.source` 和 `pose.source` 是 `physics_live:isaac`。 +- `/tmp/labutopia-c5-e2e.png` 是 `PNG image data, 640 x 480, 8-bit/color RGBA`。 +- 最终回归通过。 + +- [ ] **Step 2: 提交点** + +```bash +git add docs/demo/phase2_isaac_e2e_4090.md +git commit -m "docs(sim): add isaac e2e 4090 runbook" +``` + +## Task 5: 执行 4090 端到端验证 + +**文件:** 不新增源文件,只执行验证。 + +- [ ] **Step 1: 跑目标测试** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c5-e2e \ + python -m pytest tests/queries/test_physics_live_source.py \ + tests/integration/test_smoke_sim_isaac_edge_script.py \ + tests/integration/test_edge_query_wiring.py -q' +``` + +预期:通过。 + +- [ ] **Step 2: 跑 worker + edge + smoke** + +严格按 `docs/demo/phase2_isaac_e2e_4090.md` 执行。 + +注意: + +- 本地 graph + fastapi-only 离线模式不需要 `UNILAB_AK` 和 `UNILAB_SK`。 +- 如果切回 websocket 或远程资源模式,仍然需要凭证;不要把 ak/sk 写进文档、commit message 或 PR 描述。 +- 如果首次启动 `unilab` 出现交互提示,先确认 `/tmp/Uni-Lab-OS-c5-e2e/unilabos_data` 是否存在;必要时使用交接文档中的 `echo Y | unilab ...` 方案。 + +预期:smoke 退出码为 0,并写出 `/tmp/labutopia-c5-e2e.png`。 + +- [ ] **Step 3: 跑最终回归** + +```bash +ssh ubuntu@172.20.0.39 \ + '/home/ubuntu/miniforge3/bin/conda run -n unilab --cwd /tmp/Uni-Lab-OS-c5-e2e \ + python -m pytest tests/sim tests/queries tests/integration -q' +``` + +预期:通过。 + +- [ ] **Step 4: 记录 PR 证据** + +PR 描述里记录: + +```text +4090 C5 validation: +- Worker: matterix, http://127.0.0.1:8092 +- Edge: unilab, --mode sim --physics isaac +- Query: gRPC 127.0.0.1:50052 +- Render output: /tmp/labutopia-c5-e2e.png, +- Regression: pytest tests/sim tests/queries tests/integration -q => passed +``` + +- [ ] **Step 5: 提交点** + +```bash +git add . +git commit -m "test(sim): validate isaac edge e2e render" +``` + +## C5 验收标准 + +- `PhysicsLiveSource` 能把 Isaac observation 映射成 query `Pose` 和 `State`。 +- `_start_query_services()` 在 `RuntimeContext.physics` 存在时包含 physics live source。 +- edge 能用 `--mode sim --physics isaac` 启动,gRPC query 保持在线。 +- smoke script 能通过 gRPC 查到 physics-backed state/pose。 +- smoke script 能从 Isaac worker 写出 PNG-like LabUtopia render。 +- 4090 最终回归 `pytest tests/sim tests/queries tests/integration -q` 通过。 + +## C5 不做什么 + +- 不做前端 VirtualLab Toolbar。 +- 不做连续帧 WebSocket streaming。 +- 不验证化学任务成功、抓取成功或 contact correctness。 +- 不声称完成校准机器人动力学。C5 的完成标准是“LabUtopia 场景可渲染 + query 可见 physics state”。 diff --git a/scripts/create_isaac_lab_layout.py b/scripts/create_isaac_lab_layout.py new file mode 100644 index 00000000..0faf18ec --- /dev/null +++ b/scripts/create_isaac_lab_layout.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from unilabos.sim.backends.isaac.lab_layout import ( + DEFAULT_BEAKER_USD, + DEFAULT_HOTPLATE_USD, + DEFAULT_ROBOARM_URDF, + DEFAULT_TABLE_USD, + central_island_layout, + layout_to_manifest, + render_builder_script, + validate_layout_assets, +) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Generate the Isaac builder for the RoboArm Chem 04 lab layout") + parser.add_argument("--layout", choices=["central-island"], default="central-island") + parser.add_argument("--builder-out", required=True) + parser.add_argument("--manifest-out", default=None) + parser.add_argument("--stage-out", required=True) + parser.add_argument("--robot-urdf", default=DEFAULT_ROBOARM_URDF) + parser.add_argument("--table-usd", default=DEFAULT_TABLE_USD) + parser.add_argument("--hotplate-usd", default=DEFAULT_HOTPLATE_USD) + parser.add_argument("--beaker-usd", default=DEFAULT_BEAKER_USD) + parser.add_argument("--check-assets", action="store_true") + return parser.parse_args(argv) + + +def _build_layout(args: argparse.Namespace): + if args.layout != "central-island": + raise ValueError(f"unsupported layout: {args.layout}") + return central_island_layout( + robot_urdf=args.robot_urdf, + table_usd=args.table_usd, + hotplate_usd=args.hotplate_usd, + beaker_usd=args.beaker_usd, + ) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + layout = _build_layout(args) + if args.check_assets: + missing = validate_layout_assets(layout) + if missing: + raise FileNotFoundError("Isaac 布局资产缺失: " + "; ".join(missing)) + + builder_path = Path(args.builder_out) + builder_path.parent.mkdir(parents=True, exist_ok=True) + builder_path.write_text(render_builder_script(layout, default_output_stage=args.stage_out), encoding="utf-8") + + if args.manifest_out: + manifest_path = Path(args.manifest_out) + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest = layout_to_manifest(layout, output_stage=args.stage_out) + manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + print( + json.dumps( + { + "layout": layout.name, + "builder": str(builder_path), + "manifest": str(args.manifest_out) if args.manifest_out else None, + "stage": str(args.stage_out), + "query_targets": layout.query_targets, + }, + ensure_ascii=False, + ), + flush=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_isaac_worker.py b/scripts/smoke_isaac_worker.py new file mode 100644 index 00000000..9350399a --- /dev/null +++ b/scripts/smoke_isaac_worker.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend + + +def is_png_like(data: bytes) -> bool: + return data.startswith(b"\x89PNG\r\n\x1a\n") and data.endswith(b"IEND\xaeB`\x82") + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Smoke test a running Isaac worker") + parser.add_argument("--endpoint", required=True) + parser.add_argument("--scene", default=None) + parser.add_argument("--entity", default="/World") + parser.add_argument("--camera", default="/World/Camera") + parser.add_argument("--width", type=int, default=640) + parser.add_argument("--height", type=int, default=480) + parser.add_argument("--timeout-s", type=float, default=120.0) + parser.add_argument("--out", required=True) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + backend = IsaacBridgeBackend(args.endpoint, timeout=args.timeout_s) + if args.scene: + backend.load_scene(args.scene) + backend.step(0.016) + observation = backend.get_observation(args.entity) + image = backend.render(args.camera, args.width, args.height) + if not is_png_like(image): + raise RuntimeError(f"render payload is not a complete PNG, got {len(image)} bytes") + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(image) + print({"observation": observation, "image_path": str(out), "bytes": len(image)}, flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_sim_isaac_edge.py b/scripts/smoke_sim_isaac_edge.py new file mode 100644 index 00000000..b89889a0 --- /dev/null +++ b/scripts/smoke_sim_isaac_edge.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Smoke test Uni-Lab-OS edge + Isaac worker") + parser.add_argument("--grpc", default="127.0.0.1:50051") + parser.add_argument("--physics-endpoint", required=True) + parser.add_argument("--state-target", required=True) + parser.add_argument("--pose-target", required=True) + parser.add_argument("--camera", default="/World/Camera") + parser.add_argument("--width", type=int, default=640) + parser.add_argument("--height", type=int, default=480) + parser.add_argument("--out", required=True) + parser.add_argument("--physics-timeout-s", type=float, default=120.0) + parser.add_argument("--poll-timeout-s", type=float, default=30.0) + parser.add_argument("--poll-interval-s", type=float, default=1.0) + return parser.parse_args(argv) + + +def is_png_like(data: bytes) -> bool: + return data.startswith(b"\x89PNG\r\n\x1a\n") and data.endswith(b"IEND\xaeB`\x82") + + +def _poll_query(client, state_target: str, pose_target: str, timeout_s: float, interval_s: float): + deadline = time.monotonic() + timeout_s + last_error = None + while time.monotonic() < deadline: + try: + state = client.query_state(state_target) + pose = client.query_pose(pose_target) + return state, pose + except Exception as exc: + last_error = exc + time.sleep(interval_s) + raise RuntimeError(f"query API did not return expected state/pose before timeout: {last_error}") + + +def main(argv: list[str] | None = None) -> int: + from unilabos_client import RoboUniLabOSRemote, grpc_transport + + args = parse_args(argv) + client = RoboUniLabOSRemote(grpc_transport(args.grpc)) + state, pose = _poll_query(client, args.state_target, args.pose_target, args.poll_timeout_s, args.poll_interval_s) + physics = IsaacBridgeBackend(args.physics_endpoint, timeout=args.physics_timeout_s) + image = physics.render(args.camera, args.width, args.height) + if not is_png_like(image): + raise RuntimeError(f"render payload is not PNG-like, got {len(image)} bytes") + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(image) + print(json.dumps({"state": state, "pose": pose, "image": str(out), "bytes": len(image)}, ensure_ascii=False), flush=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/test_edge_query_wiring.py b/tests/integration/test_edge_query_wiring.py index 0b855d7a..548ae9b4 100644 --- a/tests/integration/test_edge_query_wiring.py +++ b/tests/integration/test_edge_query_wiring.py @@ -12,6 +12,24 @@ rclpy = pytest.importorskip("rclpy") +def test_build_query_static_sources_puts_physics_before_labutopia(monkeypatch): + from unilabos.ros.main_slave_run import _build_query_static_sources + from unilabos.sim.backends.fake_physics import FakePhysicsBackend + from unilabos.sim.context import RuntimeContext + + class StaticSource: + name = "static" + + import unilabos.ros.main_slave_run as mod + + monkeypatch.setattr(mod, "_build_labutopia_sources", lambda ctx: [StaticSource()]) + + sources = _build_query_static_sources(RuntimeContext(mode="sim", physics=FakePhysicsBackend())) + + assert sources[0].name == "physics_live" + assert sources[1].name == "static" + + @pytest.mark.integration def test_edge_query_services_live_flow(ros_context): from rclpy.executors import SingleThreadedExecutor diff --git a/tests/integration/test_smoke_sim_isaac_edge_script.py b/tests/integration/test_smoke_sim_isaac_edge_script.py new file mode 100644 index 00000000..c5632307 --- /dev/null +++ b/tests/integration/test_smoke_sim_isaac_edge_script.py @@ -0,0 +1,31 @@ +from scripts import smoke_sim_isaac_edge + + +def test_smoke_script_parse_args(tmp_path): + args = smoke_sim_isaac_edge.parse_args( + [ + "--grpc", + "127.0.0.1:50051", + "--physics-endpoint", + "http://127.0.0.1:8091", + "--state-target", + "arm", + "--pose-target", + "tool", + "--out", + str(tmp_path / "frame.png"), + ] + ) + + assert args.grpc == "127.0.0.1:50051" + assert args.physics_endpoint == "http://127.0.0.1:8091" + assert args.camera == "/World/Camera" + assert args.width == 640 + assert args.height == 480 + assert args.physics_timeout_s == 120.0 + + +def test_validate_png_rejects_empty_payload(): + assert smoke_sim_isaac_edge.is_png_like(b"") is False + assert smoke_sim_isaac_edge.is_png_like(b"\x89PNG\r\n\x1a\npayload") is False + assert smoke_sim_isaac_edge.is_png_like(b"\x89PNG\r\n\x1a\npayloadIEND\xaeB`\x82") is True diff --git a/tests/queries/test_physics_live_source.py b/tests/queries/test_physics_live_source.py new file mode 100644 index 00000000..9f710f01 --- /dev/null +++ b/tests/queries/test_physics_live_source.py @@ -0,0 +1,68 @@ +from unilabos.queries.physics_live_source import PhysicsLiveSource + + +class FakePhysics: + name = "fake" + + def __init__(self): + self.observations = { + "arm": { + "entity_id": "arm", + "pose": {"xyz": [0.1, 0.2, 0.3], "quat_xyzw": [0, 0, 0, 1], "frame_id": "world"}, + "joint_positions": [1.0, 2.0], + "joint_names": ["j1", "j2"], + }, + "tool": { + "entity_id": "tool", + "tcp_pose": [0.4, 0.5, 0.6, 0.0, 0.0, 0.0], + }, + } + + def get_observation(self, entity_id): + if entity_id not in self.observations: + raise KeyError(entity_id) + return dict(self.observations[entity_id]) + + +def test_physics_live_source_maps_pose_dict_to_query_pose(): + source = PhysicsLiveSource(FakePhysics()) + + pose = source.query_pose("arm") + + assert pose.xyz == [0.1, 0.2, 0.3] + assert pose.quat_xyzw == [0, 0, 0, 1] + assert pose.frame_id == "world" + assert pose.source == "physics_live:fake" + + +def test_physics_live_source_maps_tcp_pose_rotvec_to_pose(): + source = PhysicsLiveSource(FakePhysics()) + + pose = source.query_pose("tool") + + assert pose.xyz == [0.4, 0.5, 0.6] + assert pose.quat_xyzw == [0.0, 0.0, 0.0, 1.0] + + +def test_physics_live_source_maps_observation_to_state(): + source = PhysicsLiveSource(FakePhysics()) + + state = source.query_state("arm") + + assert state.values["entity_id"] == "arm" + assert state.values["joint_positions"] == [1.0, 2.0] + assert state.values["joint_names"] == ["j1", "j2"] + assert state.source == "physics_live:fake" + + +def test_physics_live_source_missing_target_returns_none(): + source = PhysicsLiveSource(FakePhysics()) + + assert source.query_pose("missing") is None + assert source.query_state("missing") is None + + +def test_physics_live_source_frame_mismatch_returns_none(): + source = PhysicsLiveSource(FakePhysics()) + + assert source.query_pose("arm", frame="base_link") is None diff --git a/tests/queries/test_ur_adapter.py b/tests/queries/test_ur_adapter.py index ecb00081..63f71826 100644 --- a/tests/queries/test_ur_adapter.py +++ b/tests/queries/test_ur_adapter.py @@ -2,6 +2,7 @@ from unilabos.hal.adapters.ur_adapter import URHAL from unilabos.queries.models import Pose +from unilabos.sim.context import RuntimeContext, _reset_for_test, init_runtime_context class FakeRTDEControl: @@ -72,6 +73,25 @@ def test_sim_mode_uses_physics_backend_contract(self): backend.commands[-1], ) + def test_sim_mode_defaults_to_runtime_physics_backend(self): + _reset_for_test() + backend = FakeSimBackend() + init_runtime_context(RuntimeContext(mode="sim", physics=backend, physics_backend_name="fake")) + try: + hal = URHAL(host="sim", robot_id="ur5_runtime", mode="sim") + + hal.move_j([0, 1, 2, 3, 4, 5], speed=0.4) + + self.assertEqual( + ( + "ur5_runtime", + {"type": "move_j", "joint_positions": [0, 1, 2, 3, 4, 5], "speed": 0.4}, + ), + backend.commands[-1], + ) + finally: + _reset_for_test() + if __name__ == "__main__": unittest.main() diff --git a/tests/sim/backends/test_create_isaac_lab_layout_script.py b/tests/sim/backends/test_create_isaac_lab_layout_script.py new file mode 100644 index 00000000..28dd9d0b --- /dev/null +++ b/tests/sim/backends/test_create_isaac_lab_layout_script.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from scripts import create_isaac_lab_layout + + +def test_parse_args_defaults(tmp_path): + args = create_isaac_lab_layout.parse_args( + [ + "--builder-out", + str(tmp_path / "builder.py"), + "--manifest-out", + str(tmp_path / "manifest.json"), + "--stage-out", + "/tmp/roboarm_lab_a.usda", + ] + ) + + assert args.layout == "central-island" + assert args.builder_out == str(tmp_path / "builder.py") + assert args.manifest_out == str(tmp_path / "manifest.json") + assert args.stage_out == "/tmp/roboarm_lab_a.usda" + assert args.check_assets is False + + +def test_main_writes_builder_and_manifest(tmp_path): + builder = tmp_path / "builder.py" + manifest = tmp_path / "manifest.json" + + result = create_isaac_lab_layout.main( + [ + "--builder-out", + str(builder), + "--manifest-out", + str(manifest), + "--stage-out", + "/tmp/roboarm_lab_a.usda", + ] + ) + + assert result == 0 + assert builder.exists() + assert manifest.exists() + assert "URDFParseAndImportFile" in builder.read_text() + manifest_payload = json.loads(Path(manifest).read_text()) + assert manifest_payload["layout"] == "roboarm_chem_04_central_island" + assert manifest_payload["query_targets"]["robot"] == "/World/Lab/RoboArmChem04" diff --git a/tests/sim/backends/test_factory.py b/tests/sim/backends/test_factory.py new file mode 100644 index 00000000..381b9cb1 --- /dev/null +++ b/tests/sim/backends/test_factory.py @@ -0,0 +1,34 @@ +import pytest + +from unilabos.sim.backends.factory import build_physics_backend +from unilabos.sim.backends.fake_physics import FakePhysicsBackend +from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend + + +def test_factory_returns_none_for_none_backend(): + assert build_physics_backend("none", endpoint=None, scene=None) is None + + +def test_factory_builds_fake_backend_and_loads_scene(): + backend = build_physics_backend("fake", endpoint=None, scene="/tmp/lab.usd") + + assert isinstance(backend, FakePhysicsBackend) + assert backend.scene_path == "/tmp/lab.usd" + + +def test_factory_requires_endpoint_for_isaac_backend(): + with pytest.raises(ValueError, match="--physics_endpoint is required"): + build_physics_backend("isaac", endpoint=None, scene=None) + + +def test_factory_builds_isaac_backend_without_calling_scene_when_absent(): + backend = build_physics_backend("isaac", endpoint="http://127.0.0.1:8091", scene=None, timeout=42.0) + + assert isinstance(backend, IsaacBridgeBackend) + assert backend.endpoint == "http://127.0.0.1:8091" + assert backend.timeout == 42.0 + + +def test_factory_rejects_unknown_backend(): + with pytest.raises(ValueError, match="Unsupported physics backend"): + build_physics_backend("mujoco", endpoint=None, scene=None) diff --git a/tests/sim/backends/test_fake_physics.py b/tests/sim/backends/test_fake_physics.py new file mode 100644 index 00000000..d3746836 --- /dev/null +++ b/tests/sim/backends/test_fake_physics.py @@ -0,0 +1,46 @@ +from unilabos.sim.backends.fake_physics import FakePhysicsBackend +from unilabos.sim.physics_backend import PhysicsBackend + + +def test_fake_backend_satisfies_physics_protocol(): + assert isinstance(FakePhysicsBackend(), PhysicsBackend) + + +def test_fake_backend_records_scene_commands_and_steps(): + backend = FakePhysicsBackend() + backend.load_scene("/tmp/lab.usd") + backend.set_command("arm", {"type": "move_j", "joint_positions": [1.0, 2.0]}) + backend.step(0.05) + + assert backend.scene_path == "/tmp/lab.usd" + assert backend.commands["arm"] == {"type": "move_j", "joint_positions": [1.0, 2.0]} + assert backend.sim_time == 0.05 + assert backend.get_observation("arm")["last_command"]["type"] == "move_j" + + +def test_fake_backend_tracks_joint_states_and_rigid_bodies(): + backend = FakePhysicsBackend() + backend.set_joint_states("arm", {"joint_1": 1.25, "joint_2": -0.5}) + body_id = backend.attach_rigid_body("beaker", "beaker.usd", {"xyz": [0, 0, 0]}) + + assert body_id == "beaker" + assert backend.get_joint_states("arm") == {"joint_1": 1.25, "joint_2": -0.5} + assert backend.get_observation("beaker")["asset_path"] == "beaker.usd" + + +def test_fake_backend_render_returns_png_like_bytes(): + backend = FakePhysicsBackend() + image = backend.render("/World/Camera", 320, 240) + + assert image.startswith(b"\x89PNG\r\n\x1a\n") + assert b"/World/Camera" in image + + +def test_fake_backend_contact_callback_receives_applied_wrench_event(): + backend = FakePhysicsBackend() + events = [] + backend.register_contact_callback(events.append) + + backend.apply_wrench("arm", {"force": [1, 0, 0]}) + + assert events == [{"type": "wrench", "body_id": "arm", "wrench": {"force": [1, 0, 0]}}] diff --git a/tests/sim/backends/test_isaac_bridge.py b/tests/sim/backends/test_isaac_bridge.py new file mode 100644 index 00000000..85649b76 --- /dev/null +++ b/tests/sim/backends/test_isaac_bridge.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import base64 +import json +import threading +from http.server import BaseHTTPRequestHandler, HTTPServer + +from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend +from unilabos.sim.physics_backend import PhysicsBackend + + +class _RpcHandler(BaseHTTPRequestHandler): + calls = [] + + def log_message(self, format, *args): + return + + def do_POST(self): + length = int(self.headers["Content-Length"]) + payload = json.loads(self.rfile.read(length).decode("utf-8")) + self.__class__.calls.append(payload) + op = payload["op"] + args = payload["args"] + if op == "get_observation": + result = {"entity_id": args["entity_id"], "tcp_pose": [1, 2, 3, 0, 0, 0]} + elif op == "get_joint_states": + result = {"joint_1": 1.0} + elif op == "attach_rigid_body": + result = "beaker" + elif op == "render": + result = {"encoding": "base64", "data": base64.b64encode(b"png-bytes").decode("ascii")} + else: + result = None + body = json.dumps({"ok": True, "result": result}).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def _start_server(): + _RpcHandler.calls = [] + server = HTTPServer(("127.0.0.1", 0), _RpcHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://127.0.0.1:{server.server_port}" + + +def test_isaac_bridge_satisfies_physics_protocol(): + assert isinstance(IsaacBridgeBackend("http://127.0.0.1:9"), PhysicsBackend) + + +def test_isaac_bridge_forwards_backend_methods_over_http(): + server, endpoint = _start_server() + try: + backend = IsaacBridgeBackend(endpoint) + + backend.load_scene("/tmp/lab.usd") + backend.reset() + backend.step(0.05) + backend.set_command("arm", {"type": "move_j"}) + observation = backend.get_observation("arm") + joints = backend.get_joint_states("arm") + body_id = backend.attach_rigid_body("beaker", "beaker.usd", {"xyz": [0, 0, 0]}) + backend.apply_wrench("arm", {"force": [1, 0, 0]}) + image = backend.render("/World/Camera", 320, 240) + + assert observation["entity_id"] == "arm" + assert joints == {"joint_1": 1.0} + assert body_id == "beaker" + assert image == b"png-bytes" + assert [call["op"] for call in _RpcHandler.calls] == [ + "load_scene", + "reset", + "step", + "set_command", + "get_observation", + "get_joint_states", + "attach_rigid_body", + "apply_wrench", + "render", + ] + finally: + server.shutdown() diff --git a/tests/sim/backends/test_isaac_lab_layout.py b/tests/sim/backends/test_isaac_lab_layout.py new file mode 100644 index 00000000..6fecb52c --- /dev/null +++ b/tests/sim/backends/test_isaac_lab_layout.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json + +from unilabos.sim.backends.isaac.lab_layout import ( + central_island_layout, + layout_to_manifest, + render_builder_script, +) + + +def test_central_island_layout_uses_roboarm_and_matterix_assets(): + layout = central_island_layout() + + assert layout.name == "roboarm_chem_04_central_island" + assert layout.query_targets["robot"] == "/World/Lab/RoboArmChem04" + assert layout.query_targets["hotplate"] == "/World/Lab/Hotplate" + assert layout.query_targets["beaker"] == "/World/Lab/Beaker500ml" + assert layout.camera.prim_path == "/World/Camera" + + placements = {placement.key: placement for placement in layout.placements} + assert placements["robot"].asset_kind == "urdf" + assert placements["robot"].asset_path.endswith("/roboarm_chem_04_query.urdf") + assert placements["table"].asset_path.endswith("/table-thorlabs-75x90/table.usda") + assert placements["hotplate"].asset_path.endswith("/hotplate_start_button/hotplate_start_button.usda") + assert placements["beaker"].asset_path.endswith("/beaker500ml/beaker-500ml.usda") + assert placements["transfer_deck"].asset_kind == "marker" + assert placements["robot"].translation == (0.0, 0.0, 0.82) + + +def test_layout_manifest_is_json_serializable_and_keeps_query_targets(): + layout = central_island_layout() + manifest = layout_to_manifest(layout, output_stage="/tmp/roboarm_lab_a.usda") + + encoded = json.dumps(manifest, ensure_ascii=False) + + assert "中央机械臂岛" in encoded + assert manifest["output_stage"] == "/tmp/roboarm_lab_a.usda" + assert manifest["camera"]["prim_path"] == "/World/Camera" + assert manifest["query_targets"]["robot"] == "/World/Lab/RoboArmChem04" + assert len(manifest["placements"]) >= 6 + + +def test_render_builder_script_contains_isaac_stage_camera_and_urdf_import(): + layout = central_island_layout() + script = render_builder_script(layout, default_output_stage="/tmp/roboarm_lab_a.usda") + + header = script.split("def parse_args", 1)[0] + assert "LAYOUT = json.loads" in header + assert "SimulationApp" in script + assert "URDFCreateImportConfig" in script + assert "URDFParseAndImportFile" in script + assert '"dest_path": placement["prim_path"]' not in script + assert 'enable_extension("isaacsim.asset.importer.urdf")' in script + assert "sim_app.update()" in script + assert 'parser.add_argument("--kit-exec"' in script + assert 'os.environ.get("UNILABOS_ISAAC_KIT_EXEC")' in script + assert "os._exit" not in script + assert "def _kit_exec_requested()" in script + assert "if _kit_exec_requested():\n main()\n else:\n raise SystemExit(main())" in script + assert "post_quit()" in script + assert "/World/Lab/RoboArmChem04" in script + assert "/World/Lab/ThorlabsTable" in script + assert "/World/Lab/Hotplate" in script + assert "/World/Lab/Beaker500ml" in script + assert 'UsdGeom.Camera.Define(stage, "/World/Camera")' in script + assert "ctx.new_stage()" in script + assert "UsdGeom.XformOp.PrecisionDouble" in script + assert "stage.GetRootLayer().Export(str(output))" in script + assert 'stage.GetRootLayer().Save()' in script diff --git a/tests/sim/backends/test_isaac_protocol.py b/tests/sim/backends/test_isaac_protocol.py new file mode 100644 index 00000000..0ac56230 --- /dev/null +++ b/tests/sim/backends/test_isaac_protocol.py @@ -0,0 +1,18 @@ +import pytest + +from unilabos.sim.backends.isaac.protocol import decode_response, encode_request + + +def test_encode_request_builds_compact_json_payload(): + payload = encode_request("set_command", {"entity_id": "arm", "command": {"type": "move_j"}}) + + assert payload == b'{"op":"set_command","args":{"entity_id":"arm","command":{"type":"move_j"}}}' + + +def test_decode_response_returns_result_for_ok_payload(): + assert decode_response(b'{"ok":true,"result":{"joint_1":1.0}}') == {"joint_1": 1.0} + + +def test_decode_response_raises_for_worker_error(): + with pytest.raises(RuntimeError, match="Isaac worker RPC failed: bad scene"): + decode_response(b'{"ok":false,"error":"bad scene"}') diff --git a/tests/sim/backends/test_isaac_worker_cli.py b/tests/sim/backends/test_isaac_worker_cli.py new file mode 100644 index 00000000..7c8a1591 --- /dev/null +++ b/tests/sim/backends/test_isaac_worker_cli.py @@ -0,0 +1,77 @@ +import sys +import threading + +from unilabos.sim.backends.isaac import worker + + +def test_worker_import_does_not_import_isaac_modules(): + assert "isaacsim" not in sys.modules + assert "omni.usd" not in sys.modules + + +def test_encode_png_rgb_writes_complete_png(): + image = [ + [[255, 0, 0], [0, 255, 0]], + [[0, 0, 255], [255, 255, 255]], + ] + payload = worker.encode_png_rgb(image) + + assert payload.startswith(b"\x89PNG\r\n\x1a\n") + assert payload.endswith(b"IEND\xaeB`\x82") + + +def test_worker_parse_args_defaults(): + args = worker.parse_args([]) + + assert args.host == "127.0.0.1" + assert args.port == 8091 + assert args.headless is True + assert args.scene is None + assert args.camera == "/World/Camera" + assert args.rpc_timeout_s == 600.0 + + +def test_worker_state_dispatches_to_controller(): + class FakeController: + def __init__(self): + self.calls = [] + + def load_scene(self, scene_path): + self.calls.append(("load_scene", scene_path)) + + def step(self, dt): + self.calls.append(("step", dt)) + + def get_observation(self, entity_id): + return {"entity_id": entity_id} + + controller = FakeController() + state = worker.IsaacWorkerState(controller) + + state.dispatch("load_scene", {"scene_path": "/tmp/lab.usd"}) + state.dispatch("step", {"dt": 0.1}) + assert state.dispatch("get_observation", {"entity_id": "arm"}) == {"entity_id": "arm"} + assert controller.calls == [("load_scene", "/tmp/lab.usd"), ("step", 0.1)] + + +def test_worker_state_can_dispatch_controller_calls_on_main_thread(): + class FakeController: + def __init__(self): + self.calls = [] + + def step(self, dt): + self.calls.append(("step", dt)) + return {"stepped": dt} + + controller = FakeController() + state = worker.IsaacWorkerState(controller, dispatch_on_main_thread=True, rpc_timeout_s=1.0) + result = [] + + thread = threading.Thread(target=lambda: result.append(state.dispatch("step", {"dt": 0.25}))) + thread.start() + + assert state.process_next(timeout=1.0) is True + thread.join(timeout=1.0) + + assert result == [{"stepped": 0.25}] + assert controller.calls == [("step", 0.25)] diff --git a/tests/sim/backends/test_isaac_worker_http.py b/tests/sim/backends/test_isaac_worker_http.py new file mode 100644 index 00000000..f9c8c1ec --- /dev/null +++ b/tests/sim/backends/test_isaac_worker_http.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import json +import threading +from urllib import request +from urllib.error import HTTPError + +import pytest + +from unilabos.sim.backends.isaac.worker_http import ThreadingHTTPServer, make_handler + + +class FakeWorkerState: + def __init__(self): + self.calls = [] + + def health(self): + return {"ok": True, "backend": "fake_isaac_worker"} + + def dispatch(self, op, args): + self.calls.append((op, args)) + if op == "explode": + raise RuntimeError("boom") + return {"op": op, "args": args} + + +def _start(state): + server = ThreadingHTTPServer(("127.0.0.1", 0), make_handler(state)) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, f"http://127.0.0.1:{server.server_port}" + + +def _post(endpoint, payload): + req = request.Request( + f"{endpoint}/rpc", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with request.urlopen(req, timeout=2.0) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def test_worker_health_endpoint(): + server, endpoint = _start(FakeWorkerState()) + try: + with request.urlopen(f"{endpoint}/health", timeout=2.0) as resp: + assert json.loads(resp.read().decode("utf-8")) == {"ok": True, "backend": "fake_isaac_worker"} + finally: + server.shutdown() + + +def test_worker_rpc_dispatches_request(): + state = FakeWorkerState() + server, endpoint = _start(state) + try: + payload = _post(endpoint, {"op": "step", "args": {"dt": 0.05}}) + + assert payload == {"ok": True, "result": {"op": "step", "args": {"dt": 0.05}}} + assert state.calls == [("step", {"dt": 0.05})] + finally: + server.shutdown() + + +def test_worker_rpc_returns_json_error_for_exception(): + server, endpoint = _start(FakeWorkerState()) + try: + with pytest.raises(HTTPError) as exc: + _post(endpoint, {"op": "explode", "args": {}}) + + assert exc.value.code == 500 + body = json.loads(exc.value.read().decode("utf-8")) + assert body == {"ok": False, "error": "boom"} + finally: + server.shutdown() diff --git a/tests/sim/backends/test_isaac_worker_protocol.py b/tests/sim/backends/test_isaac_worker_protocol.py new file mode 100644 index 00000000..e0af322e --- /dev/null +++ b/tests/sim/backends/test_isaac_worker_protocol.py @@ -0,0 +1,34 @@ +import json + +import pytest + +from unilabos.sim.backends.isaac.protocol import decode_request, encode_error, encode_response + + +def test_decode_request_reads_operation_and_args(): + op, args = decode_request(b'{"op":"step","args":{"dt":0.05}}') + + assert op == "step" + assert args == {"dt": 0.05} + + +def test_decode_request_rejects_missing_operation(): + with pytest.raises(ValueError, match="RPC request missing op"): + decode_request(b'{"args":{}}') + + +def test_decode_request_rejects_non_object_args(): + with pytest.raises(ValueError, match="RPC request args must be an object"): + decode_request(b'{"op":"step","args":[1,2]}') + + +def test_encode_response_matches_client_decode_shape(): + body = encode_response({"ok_value": 1}) + + assert json.loads(body.decode("utf-8")) == {"ok": True, "result": {"ok_value": 1}} + + +def test_encode_error_matches_client_decode_shape(): + body = encode_error("bad scene") + + assert json.loads(body.decode("utf-8")) == {"ok": False, "error": "bad scene"} diff --git a/tests/sim/backends/test_isaac_worker_smoke_script.py b/tests/sim/backends/test_isaac_worker_smoke_script.py new file mode 100644 index 00000000..fb00a8f6 --- /dev/null +++ b/tests/sim/backends/test_isaac_worker_smoke_script.py @@ -0,0 +1,18 @@ +from scripts import smoke_isaac_worker + + +def test_smoke_script_parse_args_defaults(tmp_path): + args = smoke_isaac_worker.parse_args( + ["--endpoint", "http://127.0.0.1:8091", "--out", str(tmp_path / "frame.png")] + ) + + assert args.endpoint == "http://127.0.0.1:8091" + assert args.camera == "/World/Camera" + assert args.width == 640 + assert args.height == 480 + assert args.timeout_s == 120.0 + + +def test_smoke_script_rejects_incomplete_png_payload(): + assert smoke_isaac_worker.is_png_like(b"\x89PNG\r\n\x1a\nmetadata") is False + assert smoke_isaac_worker.is_png_like(b"\x89PNG\r\n\x1a\npayloadIEND\xaeB`\x82") is True diff --git a/tests/sim/test_backend_physics_configuration.py b/tests/sim/test_backend_physics_configuration.py new file mode 100644 index 00000000..81a00e4a --- /dev/null +++ b/tests/sim/test_backend_physics_configuration.py @@ -0,0 +1,39 @@ +from unilabos.app import backend as backend_mod +from unilabos.sim.backends.fake_physics import FakePhysicsBackend + + +def test_initialize_runtime_for_backend_builds_fake_physics(): + services = backend_mod._initialize_runtime_for_backend( + backend="ros", + kwargs={ + "mode": "sim", + "sim_rate": 10.0, + "sim_paused": True, + "physics": "fake", + "physics_endpoint": None, + "physics_scene": "/tmp/lab.usd", + "physics_timeout": 77.0, + "disable_sim_services": False, + "disable_query_api": False, + "query_grpc_port": 50051, + }, + ) + + assert isinstance(services.context.physics, FakePhysicsBackend) + assert services.context.physics_backend_name == "fake" + assert services.context.physics_scene == "/tmp/lab.usd" + assert services.context.physics_timeout == 77.0 + assert services.context.clock.scale == 10.0 + assert services.context.clock.paused is True + assert services.context.sim_services_enabled is True + assert services.context.query_api_enabled is True + + +def test_initialize_runtime_for_non_ros_keeps_query_api_off(): + services = backend_mod._initialize_runtime_for_backend( + backend="simple", + kwargs={"mode": "sim", "physics": "none", "disable_query_api": False}, + ) + + assert services.context.physics is None + assert services.context.query_api_enabled is False diff --git a/tests/sim/test_cli_runtime.py b/tests/sim/test_cli_runtime.py index b643f9c5..3a3c5605 100644 --- a/tests/sim/test_cli_runtime.py +++ b/tests/sim/test_cli_runtime.py @@ -1,4 +1,4 @@ -from unilabos.app.main import build_argparser +from unilabos.app.main import _can_start_without_cloud_auth, build_argparser def test_cli_defaults_are_backward_compatible(): @@ -18,3 +18,46 @@ def test_cli_accepts_sim_options(): def test_cli_can_disable_sim_ros_services(): args = build_argparser().parse_args(["--mode", "sim", "--disable_sim_services"]) assert args.disable_sim_services is True + + +def test_cli_physics_defaults_are_backward_compatible(): + args = build_argparser().parse_args([]) + + assert args.physics == "none" + assert args.physics_endpoint is None + assert args.physics_scene is None + assert args.physics_timeout == 120.0 + + +def test_cli_accepts_isaac_physics_options(): + args = build_argparser().parse_args( + [ + "--mode", + "sim", + "--physics", + "isaac", + "--physics_endpoint", + "http://127.0.0.1:8091", + "--physics_scene", + "/tmp/lab.usd", + "--physics_timeout", + "180", + ] + ) + + assert args.physics == "isaac" + assert args.physics_endpoint == "http://127.0.0.1:8091" + assert args.physics_scene == "/tmp/lab.usd" + assert args.physics_timeout == 180.0 + + +def test_local_graph_fastapi_can_start_without_cloud_auth(): + args = {"app_bridges": ["fastapi"], "use_remote_resource": False} + + assert _can_start_without_cloud_auth(args, "/tmp/mock_all.json") is True + + +def test_websocket_or_remote_resource_still_requires_cloud_auth(): + assert _can_start_without_cloud_auth({"app_bridges": ["websocket"], "use_remote_resource": False}, "/tmp/g.json") is False + assert _can_start_without_cloud_auth({"app_bridges": ["fastapi"], "use_remote_resource": True}, "/tmp/g.json") is False + assert _can_start_without_cloud_auth({"app_bridges": ["fastapi"], "use_remote_resource": False}, None) is False diff --git a/tests/sim/test_context_and_clock.py b/tests/sim/test_context_and_clock.py index b41b3ad1..63ce947b 100644 --- a/tests/sim/test_context_and_clock.py +++ b/tests/sim/test_context_and_clock.py @@ -30,6 +30,19 @@ def test_init_sets_context_and_pause(): assert ctx.clock.paused is True +def test_runtime_context_stores_physics_configuration(): + ctx = RuntimeContext( + mode="sim", + physics_backend_name="isaac", + physics_endpoint="http://127.0.0.1:8091", + physics_scene="/tmp/lab.usd", + ) + + assert ctx.physics_backend_name == "isaac" + assert ctx.physics_endpoint == "http://127.0.0.1:8091" + assert ctx.physics_scene == "/tmp/lab.usd" + + def test_sim_clock_sleep_scales_wall_time(): clock = SimClock("sim", scale=20.0) t0 = time.monotonic() diff --git a/tests/sim/test_device_physics.py b/tests/sim/test_device_physics.py new file mode 100644 index 00000000..f1652c5b --- /dev/null +++ b/tests/sim/test_device_physics.py @@ -0,0 +1,32 @@ +from unilabos.sim.context import RuntimeContext, _reset_for_test, init_runtime_context +from unilabos.sim.device_physics import dispatch_device_command + + +class RecordingPhysics: + name = "recording" + + def __init__(self): + self.commands = [] + + def set_command(self, entity_id, command): + self.commands.append((entity_id, command)) + + +def setup_function(): + _reset_for_test() + + +def teardown_function(): + _reset_for_test() + + +def test_dispatch_device_command_noops_without_physics(): + assert dispatch_device_command("valve", {"type": "set_position"}) is False + + +def test_dispatch_device_command_sends_to_runtime_physics(): + physics = RecordingPhysics() + init_runtime_context(RuntimeContext(mode="sim", physics=physics, physics_backend_name="fake")) + + assert dispatch_device_command("valve", {"type": "set_position", "position": 3}) is True + assert physics.commands == [("valve", {"type": "set_position", "position": 3})] diff --git a/tests/sim/test_physics_backend.py b/tests/sim/test_physics_backend.py index 0b3f01c6..5284a44e 100644 --- a/tests/sim/test_physics_backend.py +++ b/tests/sim/test_physics_backend.py @@ -10,6 +10,9 @@ def reset(self) -> None: def step(self, dt: float) -> None: self.dt = dt + def load_scene(self, scene_path: str) -> None: + self.scene_path = scene_path + def get_observation(self, entity_id: str): return {"entity_id": entity_id} @@ -28,6 +31,9 @@ def apply_wrench(self, body_id: str, wrench): def register_contact_callback(self, callback): self.contact_callback = callback + def render(self, camera: str, width: int, height: int) -> bytes: + return f"{camera}:{width}x{height}".encode() + def test_physics_backend_protocol_runtime_check(): assert isinstance(FakePhysics(), PhysicsBackend) @@ -39,3 +45,10 @@ def test_physics_backend_extended_contract_methods(): assert backend.get_joint_states("arm:body") == {"joint_1": 0.0} backend.apply_wrench("arm:body", {"force": [1, 0, 0]}) assert backend.wrench == ("arm:body", {"force": [1, 0, 0]}) + + +def test_physics_backend_scene_and_render_contract(): + backend = FakePhysics() + backend.load_scene("/tmp/lab.usd") + assert backend.scene_path == "/tmp/lab.usd" + assert backend.render("/World/Camera", 320, 240) == b"/World/Camera:320x240" diff --git a/tests/sim/test_runtime_configuration.py b/tests/sim/test_runtime_configuration.py index 381d0fd4..23aeb393 100644 --- a/tests/sim/test_runtime_configuration.py +++ b/tests/sim/test_runtime_configuration.py @@ -10,6 +10,10 @@ def teardown_function(): _reset_for_test() +class DummyPhysics: + name = "dummy" + + def test_configure_runtime_initializes_context_without_ros_services(): services = configure_runtime(mode="sim", sim_rate=25.0, sim_paused=True, start_ros_services=False) @@ -26,3 +30,23 @@ def test_configure_runtime_real_mode_keeps_sim_services_off(): assert services.context.mode == "real" assert services.clock_publisher is None assert services.clock_control is None + + +def test_configure_runtime_stores_physics_backend_and_config(): + physics = DummyPhysics() + + services = configure_runtime( + mode="sim", + physics=physics, + physics_backend_name="fake", + physics_endpoint="http://127.0.0.1:8091", + physics_scene="/tmp/lab.usd", + physics_timeout=45.0, + ) + + assert services.context.physics is physics + assert get_runtime_context().physics is physics + assert services.context.physics_backend_name == "fake" + assert services.context.physics_endpoint == "http://127.0.0.1:8091" + assert services.context.physics_scene == "/tmp/lab.usd" + assert services.context.physics_timeout == 45.0 diff --git a/tests/sim/test_virtual_device_clock.py b/tests/sim/test_virtual_device_clock.py index a5039fa0..394f8ea1 100644 --- a/tests/sim/test_virtual_device_clock.py +++ b/tests/sim/test_virtual_device_clock.py @@ -2,6 +2,7 @@ from unilabos.devices.virtual.virtual_gas_source import VirtualGasSource from unilabos.devices.virtual.virtual_multiway_valve import VirtualMultiwayValve +from unilabos.sim.backends.fake_physics import FakePhysicsBackend from unilabos.sim.clock import SimClock from unilabos.sim.context import RuntimeContext, _reset_for_test, init_runtime_context @@ -30,3 +31,17 @@ def test_virtual_multiway_valve_uses_sim_clock(): result = valve.set_position(8) assert time.monotonic() - t0 < 0.3 assert "8" in result + + +def test_virtual_multiway_valve_dispatches_position_to_physics(): + physics = FakePhysicsBackend() + init_runtime_context(RuntimeContext(mode="sim", clock=SimClock("sim", scale=100.0), physics=physics)) + valve = VirtualMultiwayValve(id="valve_a", positions=8) + + valve.set_position(3) + + assert physics.commands["valve_a"] == { + "type": "set_position", + "position": 3, + "device": "virtual_multiway_valve", + } diff --git a/unilabos/app/backend.py b/unilabos/app/backend.py index 120ee0be..44d2f48d 100644 --- a/unilabos/app/backend.py +++ b/unilabos/app/backend.py @@ -3,6 +3,7 @@ import threading from unilabos.resources.resource_tracker import ResourceTreeSet +from unilabos.sim.backends.factory import build_physics_backend from unilabos.sim.runtime import RuntimeServices, configure_runtime from unilabos.utils import logger @@ -10,6 +11,41 @@ _runtime_services: RuntimeServices | None = None +def _initialize_runtime_for_backend(backend: str, kwargs: dict) -> RuntimeServices: + mode = kwargs.get("mode", "real") + sim_rate = kwargs.get("sim_rate", 1.0) + sim_paused = kwargs.get("sim_paused", False) + physics_name = kwargs.get("physics", "none") + physics_endpoint = kwargs.get("physics_endpoint") + physics_scene = kwargs.get("physics_scene") + physics_timeout = float(kwargs.get("physics_timeout", 120.0)) + physics = build_physics_backend( + physics_name, + endpoint=physics_endpoint, + scene=physics_scene, + timeout=physics_timeout, + ) + start_sim_services = backend == "ros" and not kwargs.get("disable_sim_services", False) + services = configure_runtime( + mode=mode, + sim_rate=sim_rate, + sim_paused=sim_paused, + start_ros_services=False, + physics=physics, + physics_backend_name=physics_name, + physics_endpoint=physics_endpoint, + physics_scene=physics_scene, + physics_timeout=physics_timeout, + ) + services.context.sim_services_enabled = start_sim_services and mode in ("sim", "twin") + services.context.query_api_enabled = backend == "ros" and not kwargs.get("disable_query_api", False) + services.context.query_grpc_port = int(kwargs.get("query_grpc_port", 50051)) + services.context.query_labutopia_assets = kwargs.get("query_labutopia_assets") + services.context.query_labutopia_config = kwargs.get("query_labutopia_config") + services.context.query_labutopia_usd = kwargs.get("query_labutopia_usd") + return services + + # 根据选择的 backend 启动相应的功能 def start_backend( backend: str, @@ -25,29 +61,17 @@ def start_backend( **kwargs, ): global _runtime_services - mode = kwargs.get("mode", "real") - sim_rate = kwargs.get("sim_rate", 1.0) - sim_paused = kwargs.get("sim_paused", False) - start_sim_services = backend == "ros" and not kwargs.get("disable_sim_services", False) - _runtime_services = configure_runtime( - mode=mode, - sim_rate=sim_rate, - sim_paused=sim_paused, - start_ros_services=False, - ) - _runtime_services.context.sim_services_enabled = start_sim_services and mode in ("sim", "twin") - _runtime_services.context.query_api_enabled = backend == "ros" and not kwargs.get("disable_query_api", False) - _runtime_services.context.query_grpc_port = int(kwargs.get("query_grpc_port", 50051)) - _runtime_services.context.query_labutopia_assets = kwargs.get("query_labutopia_assets") - _runtime_services.context.query_labutopia_config = kwargs.get("query_labutopia_config") - _runtime_services.context.query_labutopia_usd = kwargs.get("query_labutopia_usd") + _runtime_services = _initialize_runtime_for_backend(backend, kwargs) logger.info( "Runtime mode initialized: " - f"mode={mode}, sim_rate={_runtime_services.context.clock.scale}, " + f"mode={_runtime_services.context.mode}, sim_rate={_runtime_services.context.clock.scale}, " f"paused={_runtime_services.context.clock.paused}, " - f"sim_services={start_sim_services and mode in ('sim', 'twin')}, " + f"sim_services={_runtime_services.context.sim_services_enabled}, " f"query_api={_runtime_services.context.query_api_enabled}, " - f"grpc_port={_runtime_services.context.query_grpc_port}" + f"grpc_port={_runtime_services.context.query_grpc_port}, " + f"physics={_runtime_services.context.physics_backend_name}, " + f"physics_endpoint={_runtime_services.context.physics_endpoint}, " + f"physics_timeout={_runtime_services.context.physics_timeout}" ) if backend == "ros": diff --git a/unilabos/app/main.py b/unilabos/app/main.py index 1572ca78..74331e36 100644 --- a/unilabos/app/main.py +++ b/unilabos/app/main.py @@ -320,6 +320,30 @@ def build_argparser(): default=False, help="Do not auto-start /clock publisher and sim clock control ROS services.", ) + parser.add_argument( + "--physics", + choices=["none", "fake", "isaac"], + default="none", + help="Physics backend for sim mode: none, fake in-process backend, or Isaac HTTP bridge.", + ) + parser.add_argument( + "--physics_endpoint", + type=str, + default=None, + help="Physics backend endpoint, required for --physics isaac.", + ) + parser.add_argument( + "--physics_scene", + type=str, + default=None, + help="Scene path to load into the selected physics backend during startup.", + ) + parser.add_argument( + "--physics_timeout", + type=float, + default=120.0, + help="Physics backend RPC timeout in seconds.", + ) parser.add_argument( "--disable_query_api", action="store_true", @@ -419,6 +443,14 @@ def _load_graph_json_preview(file_path: str | None) -> Dict[str, Any] | None: return None +def _can_start_without_cloud_auth(args_dict: Dict[str, Any], graph_file_path: str | None) -> bool: + if graph_file_path is None: + return False + if args_dict.get("use_remote_resource", False): + return False + return "websocket" not in (args_dict.get("app_bridges") or []) + + def main(): """主函数""" # 解析命令行参数 @@ -695,22 +727,25 @@ def main(): print_status("工作流上传完成,程序退出", "info") os._exit(0) - if not BasicConfig.ak or not BasicConfig.sk: - print_status("后续运行必须拥有一个实验室,请前往 https://leap-lab.bohrium.com 注册实验室!", "warning") - os._exit(1) import networkx as nx import yaml graph: nx.Graph resource_tree_set: ResourceTreeSet resource_links: List[Dict[str, Any]] - request_startup_json = args_dict.get("_startup_json") - if request_startup_json is None: - request_startup_json = http_client.request_startup_json() - file_path = args_dict.get("_graph_file_path") if file_path is None: file_path = _resolve_graph_file_path(args_dict.get("graph") or BasicConfig.startup_json_path) + can_start_without_auth = _can_start_without_cloud_auth(args_dict, file_path) + if not BasicConfig.ak or not BasicConfig.sk: + if not can_start_without_auth: + print_status("后续运行必须拥有一个实验室,请前往 https://leap-lab.bohrium.com 注册实验室!", "warning") + os._exit(1) + print_status("未提供 ak/sk,使用本地 graph 和非 websocket bridge 进入离线启动模式", "warning") + + request_startup_json = args_dict.get("_startup_json") + if request_startup_json is None and BasicConfig.ak and BasicConfig.sk: + request_startup_json = http_client.request_startup_json() if file_path is None: if not request_startup_json: print_status( diff --git a/unilabos/devices/virtual/virtual_multiway_valve.py b/unilabos/devices/virtual/virtual_multiway_valve.py index c5c05d99..b5e76841 100644 --- a/unilabos/devices/virtual/virtual_multiway_valve.py +++ b/unilabos/devices/virtual/virtual_multiway_valve.py @@ -3,6 +3,7 @@ from unilabos.registry.decorators import topic_config from unilabos.sim.clock import sim_sleep_sync +from unilabos.sim.device_physics import dispatch_device_command class VirtualMultiwayValve: @@ -11,6 +12,7 @@ class VirtualMultiwayValve: """ def __init__(self, port: str = "VIRTUAL", positions: int = 8, **kwargs): self.port = port + self.device_id = kwargs.get("device_id") or kwargs.get("id") or self.port self.max_positions = positions # 1-8号位 self.total_positions = positions + 1 # 0-8号位,共9个位置 @@ -107,6 +109,10 @@ def set_position(self, command: Union[int, str]): self._status = "Busy" self._valve_state = "Moving" self._target_position = pos + dispatch_device_command( + self.device_id, + {"type": "set_position", "position": pos, "device": "virtual_multiway_valve"}, + ) # 模拟阀门切换时间 switch_time = abs(self._current_position - pos) * 0.5 # 每个位置0.5秒 @@ -163,6 +169,10 @@ def close(self): self._status = "Busy" self._valve_state = "Closing" + dispatch_device_command( + self.device_id, + {"type": "close", "position": self._current_position, "device": "virtual_multiway_valve"}, + ) sim_sleep_sync(0.5) # 可以选择保持当前位置或设置特殊关闭状态 diff --git a/unilabos/hal/adapters/ur_adapter.py b/unilabos/hal/adapters/ur_adapter.py index 792a88db..327f98ed 100644 --- a/unilabos/hal/adapters/ur_adapter.py +++ b/unilabos/hal/adapters/ur_adapter.py @@ -139,11 +139,17 @@ def close_gripper(self) -> None: self.gripper.close() def _sim_observation(self) -> dict[str, Any]: - if self.sim_backend is None: - raise RuntimeError("URHAL sim mode requires sim_backend") - return dict(self.sim_backend.get_observation(self.robot_id)) + return dict(self._active_sim_backend().get_observation(self.robot_id)) def _sim_command(self, command: dict[str, Any]) -> None: - if self.sim_backend is None: - raise RuntimeError("URHAL sim mode requires sim_backend") - self.sim_backend.set_command(self.robot_id, command) + self._active_sim_backend().set_command(self.robot_id, command) + + def _active_sim_backend(self): + if self.sim_backend is not None: + return self.sim_backend + from unilabos.sim.context import get_runtime_context + + backend = get_runtime_context().physics + if backend is None: + raise RuntimeError("URHAL sim mode requires sim_backend or RuntimeContext.physics") + return backend diff --git a/unilabos/queries/__init__.py b/unilabos/queries/__init__.py index 91de2739..844588f6 100644 --- a/unilabos/queries/__init__.py +++ b/unilabos/queries/__init__.py @@ -2,6 +2,7 @@ from unilabos.queries.action_schema import ActionSchemaRegistry, query_action_schema from unilabos.queries.engine import QueryEngine, QueryNotFound from unilabos.queries.models import ActionSchema, Pose, QueryAffordance, SafetyZone, State, VerificationResult +from unilabos.queries.physics_live_source import PhysicsLiveSource from unilabos.queries.resource_map_source import ResourceMapSource from unilabos.queries.ros_live_source import RosLiveSource, build_live_query_engine from unilabos.queries.robot_asset import ( @@ -18,6 +19,7 @@ "ActionSchemaRegistry", "ActionCatalogSource", "Pose", + "PhysicsLiveSource", "QueryAffordance", "QueryEngine", "QueryNotFound", diff --git a/unilabos/queries/physics_live_source.py b/unilabos/queries/physics_live_source.py new file mode 100644 index 00000000..ea406e88 --- /dev/null +++ b/unilabos/queries/physics_live_source.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +import math +from typing import Any, Optional + +from unilabos.queries.models import ActionSchema, Pose, QueryAffordance, SafetyZone, State, utc_timestamp + + +def _rotvec_to_quat_xyzw(rotvec: list[float]) -> list[float]: + angle = math.sqrt(sum(float(item) * float(item) for item in rotvec)) + if angle == 0.0: + return [0.0, 0.0, 0.0, 1.0] + axis = [float(item) / angle for item in rotvec] + half = angle / 2.0 + scale = math.sin(half) + return [axis[0] * scale, axis[1] * scale, axis[2] * scale, math.cos(half)] + + +class PhysicsLiveSource: + name = "physics_live" + + def __init__(self, physics_backend: Any): + self.physics_backend = physics_backend + + @property + def _source_name(self) -> str: + return f"physics_live:{getattr(self.physics_backend, 'name', 'unknown')}" + + def _observation(self, target: str) -> Optional[dict[str, Any]]: + try: + return dict(self.physics_backend.get_observation(target)) + except Exception: + return None + + def query_pose(self, target: str, frame: Optional[str] = None) -> Optional[Pose]: + obs = self._observation(target) + if not obs: + return None + pose_payload = obs.get("pose") + if isinstance(pose_payload, dict): + frame_id = str(pose_payload.get("frame_id") or obs.get("frame_id") or "world") + if frame is not None and frame_id != frame: + return None + return Pose( + xyz=[float(item) for item in pose_payload.get("xyz", [0.0, 0.0, 0.0])], + quat_xyzw=[float(item) for item in pose_payload.get("quat_xyzw", [0.0, 0.0, 0.0, 1.0])], + frame_id=frame_id, + stamp=utc_timestamp(), + source=self._source_name, + metadata={"target": target}, + ) + tcp_pose = obs.get("tcp_pose") or obs.get("tool_pose") + if tcp_pose is None: + return None + values = [float(item) for item in tcp_pose] + if len(values) < 6: + return None + frame_id = str(obs.get("frame_id") or "world") + if frame is not None and frame_id != frame: + return None + return Pose( + xyz=values[:3], + quat_xyzw=_rotvec_to_quat_xyzw(values[3:6]), + frame_id=frame_id, + stamp=utc_timestamp(), + source=self._source_name, + metadata={"target": target, "pose_format": "tcp_pose"}, + ) + + def query_state(self, target: str) -> Optional[State]: + obs = self._observation(target) + if not obs: + return None + return State(name=target, values=obs, stamp=utc_timestamp(), source=self._source_name) + + def query_affordance(self, target: str, kind: Optional[str] = None) -> list[QueryAffordance]: + return [] + + def query_action_schema(self, action: str) -> Optional[ActionSchema]: + return None + + def query_safety_zones(self) -> list[SafetyZone]: + return [] diff --git a/unilabos/ros/main_slave_run.py b/unilabos/ros/main_slave_run.py index 7a7a6bbb..56225a05 100644 --- a/unilabos/ros/main_slave_run.py +++ b/unilabos/ros/main_slave_run.py @@ -102,6 +102,17 @@ def _build_labutopia_sources(ctx) -> list: return sources +def _build_query_static_sources(ctx) -> list: + sources = [] + physics = getattr(ctx, "physics", None) + if physics is not None: + from unilabos.queries.physics_live_source import PhysicsLiveSource + + sources.append(PhysicsLiveSource(physics)) + sources.extend(_build_labutopia_sources(ctx)) + return sources + + def _start_query_services(executor) -> None: """启动 Robo-UniLabOS 信息层对外暴露:ROS2 /unilabos/query + gRPC :50051。 @@ -120,7 +131,7 @@ def _start_query_services(executor) -> None: from unilabos.api.ros2_query_service import QueryServiceNode from unilabos.queries.ros_live_source import build_live_query_engine - static_sources = _build_labutopia_sources(ctx) + static_sources = _build_query_static_sources(ctx) live, engine = build_live_query_engine(static_sources=static_sources) service = QueryService(engine) diff --git a/unilabos/sim/backends/__init__.py b/unilabos/sim/backends/__init__.py new file mode 100644 index 00000000..b814704e --- /dev/null +++ b/unilabos/sim/backends/__init__.py @@ -0,0 +1 @@ +"""Simulation physics backend implementations.""" diff --git a/unilabos/sim/backends/factory.py b/unilabos/sim/backends/factory.py new file mode 100644 index 00000000..7dc1a819 --- /dev/null +++ b/unilabos/sim/backends/factory.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from unilabos.sim.physics_backend import PhysicsBackend + + +def build_physics_backend( + name: str | None, + endpoint: str | None = None, + scene: str | None = None, + timeout: float = 120.0, +) -> PhysicsBackend | None: + backend_name = (name or "none").strip().lower() + if backend_name == "none": + return None + if backend_name == "fake": + from unilabos.sim.backends.fake_physics import FakePhysicsBackend + + backend: PhysicsBackend = FakePhysicsBackend() + elif backend_name == "isaac": + if not endpoint: + raise ValueError("--physics_endpoint is required when --physics isaac") + from unilabos.sim.backends.isaac_bridge import IsaacBridgeBackend + + backend = IsaacBridgeBackend(endpoint, timeout=timeout) + else: + raise ValueError(f"Unsupported physics backend: {name}") + + if scene: + backend.load_scene(scene) + return backend diff --git a/unilabos/sim/backends/fake_physics.py b/unilabos/sim/backends/fake_physics.py new file mode 100644 index 00000000..7f6587b7 --- /dev/null +++ b/unilabos/sim/backends/fake_physics.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from typing import Any, Callable + + +class FakePhysicsBackend: + name = "fake" + + def __init__(self) -> None: + self.scene_path: str | None = None + self.sim_time = 0.0 + self.commands: dict[str, dict[str, Any]] = {} + self.observations: dict[str, dict[str, Any]] = {} + self.joint_states: dict[str, dict[str, float]] = {} + self.rigid_bodies: dict[str, dict[str, Any]] = {} + self.wrenches: list[tuple[str, dict[str, Any]]] = [] + self._contact_callbacks: list[Callable[[dict[str, Any]], None]] = [] + + def reset(self) -> None: + self.sim_time = 0.0 + self.commands.clear() + self.observations.clear() + self.joint_states.clear() + self.wrenches.clear() + + def step(self, dt: float) -> None: + self.sim_time += float(dt) + + def load_scene(self, scene_path: str) -> None: + self.scene_path = str(scene_path) + + def get_observation(self, entity_id: str) -> dict[str, Any]: + observation = dict(self.observations.get(entity_id, {})) + if entity_id in self.commands: + observation["last_command"] = dict(self.commands[entity_id]) + if entity_id in self.joint_states: + observation["joint_positions"] = list(self.joint_states[entity_id].values()) + observation["joint_states"] = dict(self.joint_states[entity_id]) + if entity_id in self.rigid_bodies: + observation.update(self.rigid_bodies[entity_id]) + observation.setdefault("entity_id", entity_id) + observation.setdefault("sim_time", self.sim_time) + return observation + + def set_observation(self, entity_id: str, observation: dict[str, Any]) -> None: + self.observations[entity_id] = dict(observation) + + def set_command(self, entity_id: str, command: dict[str, Any]) -> None: + self.commands[entity_id] = dict(command) + + def attach_rigid_body(self, name: str, asset_path: str, pose: dict[str, Any]) -> str: + body_id = str(name) + self.rigid_bodies[body_id] = {"name": str(name), "asset_path": str(asset_path), "pose": dict(pose)} + return body_id + + def set_joint_states(self, body_id: str, joints: dict[str, float]) -> None: + self.joint_states[body_id] = {str(key): float(value) for key, value in joints.items()} + + def get_joint_states(self, body_id: str) -> dict[str, float]: + return dict(self.joint_states.get(body_id, {})) + + def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: + payload = dict(wrench) + self.wrenches.append((body_id, payload)) + event = {"type": "wrench", "body_id": body_id, "wrench": payload} + for callback in list(self._contact_callbacks): + callback(event) + + def register_contact_callback(self, callback: Callable[[dict[str, Any]], None]) -> None: + self._contact_callbacks.append(callback) + + def render(self, camera: str, width: int, height: int) -> bytes: + meta = f"fake-render camera={camera} width={int(width)} height={int(height)}".encode() + return b"\x89PNG\r\n\x1a\n" + meta diff --git a/unilabos/sim/backends/isaac/__init__.py b/unilabos/sim/backends/isaac/__init__.py new file mode 100644 index 00000000..64942b82 --- /dev/null +++ b/unilabos/sim/backends/isaac/__init__.py @@ -0,0 +1 @@ +"""Isaac Sim bridge protocol package.""" diff --git a/unilabos/sim/backends/isaac/lab_layout.py b/unilabos/sim/backends/isaac/lab_layout.py new file mode 100644 index 00000000..12f55dfb --- /dev/null +++ b/unilabos/sim/backends/isaac/lab_layout.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + + +DEFAULT_ROBOARM_URDF = ( + "/home/ubuntu/canonical/Uni-Lab-OS/robot_assets/roboarm_chem_04/urdf/roboarm_chem_04_query.urdf" +) +DEFAULT_TABLE_USD = ( + "/home/ubuntu/Matterix/source/matterix_assets/data/infrastructure/tables/" + "table-thorlabs-75x90/table.usda" +) +DEFAULT_HOTPLATE_USD = ( + "/home/ubuntu/Matterix/source/matterix_assets/data/instruments/" + "hotplate_start_button/hotplate_start_button.usda" +) +DEFAULT_BEAKER_USD = ( + "/home/ubuntu/Matterix/source/matterix_assets/data/labware/beaker500ml/beaker-500ml.usda" +) + + +Vector3 = tuple[float, float, float] + + +@dataclass(frozen=True) +class Placement: + key: str + label: str + prim_path: str + asset_kind: str + asset_path: str | None + translation: Vector3 + rotation_xyz: Vector3 = (0.0, 0.0, 0.0) + scale: Vector3 = (1.0, 1.0, 1.0) + color: Vector3 | None = None + note: str = "" + + +@dataclass(frozen=True) +class CameraSpec: + prim_path: str + translation: Vector3 + rotation_xyz: Vector3 + focal_length: float = 28.0 + + +@dataclass(frozen=True) +class IsaacLabLayout: + name: str + display_name: str + meters_per_unit: float + up_axis: str + placements: tuple[Placement, ...] + camera: CameraSpec + query_targets: dict[str, str] + notes: tuple[str, ...] = () + + +def central_island_layout( + *, + robot_urdf: str = DEFAULT_ROBOARM_URDF, + table_usd: str = DEFAULT_TABLE_USD, + hotplate_usd: str = DEFAULT_HOTPLATE_USD, + beaker_usd: str = DEFAULT_BEAKER_USD, +) -> IsaacLabLayout: + """返回 RoboArm Chem 04 中央机械臂岛布局。""" + placements = ( + Placement( + key="table", + label="Thorlabs 75x90 主桌", + prim_path="/World/Lab/ThorlabsTable", + asset_kind="usd", + asset_path=table_usd, + translation=(0.0, 0.0, 0.0), + note="主桌中心对齐世界原点。", + ), + Placement( + key="robot", + label="RoboArm Chem 04", + prim_path="/World/Lab/RoboArmChem04", + asset_kind="urdf", + asset_path=robot_urdf, + translation=(0.0, 0.0, 0.82), + note="机械臂底座放在主桌中央,后续可基于 URDF 实测高度微调。", + ), + Placement( + key="hotplate", + label="Hotplate", + prim_path="/World/Lab/Hotplate", + asset_kind="usd", + asset_path=hotplate_usd, + translation=(-0.26, 0.12, 0.86), + scale=(1.0, 1.0, 1.0), + note="左前反应加热区。", + ), + Placement( + key="beaker", + label="Beaker 500ml", + prim_path="/World/Lab/Beaker500ml", + asset_kind="usd", + asset_path=beaker_usd, + translation=(-0.24, -0.08, 0.88), + scale=(1.0, 1.0, 1.0), + note="烧杯放在左前侧,便于 query pose 和渲染验收。", + ), + Placement( + key="reagent_tray", + label="Reagent Tray", + prim_path="/World/Lab/ReagentTray", + asset_kind="marker", + asset_path=None, + translation=(0.28, 0.12, 0.865), + scale=(0.18, 0.12, 0.025), + color=(0.22, 0.58, 0.32), + note="试剂/耗材暂用可视 marker,等实际资产确定后替换。", + ), + Placement( + key="instrument_slot", + label="Instrument Slot", + prim_path="/World/Lab/InstrumentSlot", + asset_kind="marker", + asset_path=None, + translation=(0.28, -0.08, 0.865), + scale=(0.20, 0.12, 0.025), + color=(0.82, 0.55, 0.18), + note="右侧仪器占位,后续替换为泵、阀、天平或光谱仪 USD。", + ), + Placement( + key="transfer_deck", + label="Transfer Deck", + prim_path="/World/Lab/TransferDeck", + asset_kind="marker", + asset_path=None, + translation=(0.0, -0.28, 0.865), + scale=(0.22, 0.10, 0.02), + color=(0.06, 0.45, 0.43), + note="前侧转运区,适合录屏时展示目标点和后续放置动作。", + ), + ) + return IsaacLabLayout( + name="roboarm_chem_04_central_island", + display_name="中央机械臂岛", + meters_per_unit=1.0, + up_axis="Z", + placements=placements, + camera=CameraSpec( + prim_path="/World/Camera", + translation=(1.35, -1.65, 1.65), + rotation_xyz=(60.0, 0.0, 39.0), + ), + query_targets={ + "robot": "/World/Lab/RoboArmChem04", + "table": "/World/Lab/ThorlabsTable", + "hotplate": "/World/Lab/Hotplate", + "beaker": "/World/Lab/Beaker500ml", + "transfer_deck": "/World/Lab/TransferDeck", + }, + notes=( + "首版优先展示真实 PNG 渲染、query 物理态和 Uni-Lab-OS 到 Isaac worker 的闭环。", + "试剂托盘、仪器位、转运区先用 marker 占位,便于录屏说明和后续替换真实 USD。", + ), + ) + + +def layout_to_manifest(layout: IsaacLabLayout, *, output_stage: str) -> dict[str, Any]: + return { + "layout": layout.name, + "display_name": layout.display_name, + "output_stage": str(output_stage), + "meters_per_unit": layout.meters_per_unit, + "up_axis": layout.up_axis, + "camera": _camera_to_dict(layout.camera), + "query_targets": dict(layout.query_targets), + "placements": [_placement_to_dict(placement) for placement in layout.placements], + "notes": list(layout.notes), + } + + +def validate_layout_assets( + layout: IsaacLabLayout, + *, + exists: Callable[[str], bool] | None = None, +) -> list[str]: + exists = exists or (lambda path: Path(path).exists()) + missing: list[str] = [] + for placement in layout.placements: + if placement.asset_kind == "marker": + continue + if not placement.asset_path or not exists(placement.asset_path): + missing.append(f"{placement.key}: {placement.asset_path}") + return missing + + +def render_builder_script(layout: IsaacLabLayout, *, default_output_stage: str) -> str: + manifest = layout_to_manifest(layout, output_stage=default_output_stage) + manifest_json = json.dumps(manifest, indent=2, ensure_ascii=False) + return f'''#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +LAYOUT = json.loads(r"""{manifest_json}""") + + +def _kit_exec_requested() -> bool: + return os.environ.get("UNILABOS_ISAAC_KIT_EXEC") == "1" or "--kit-exec" in sys.argv + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Build the RoboArm Chem 04 central island Isaac stage") + parser.add_argument("--out", default=LAYOUT["output_stage"]) + parser.add_argument("--headless", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--kit-exec", action="store_true", help="run inside an already started Isaac Kit --exec session") + parser.add_argument("--warmup-steps", type=int, default=8) + return parser.parse_args() + + +class _KitExecAppAdapter: + def update(self): + import omni.kit.app + + omni.kit.app.get_app().update() + + +def _vec3(values): + from pxr import Gf + + return Gf.Vec3d(float(values[0]), float(values[1]), float(values[2])) + + +def _resolve_package_urdf(source: Path) -> Path: + text = source.read_text(encoding="utf-8") + if "package://" not in text: + return source + package_root = source.parents[1] + package_name = package_root.name + resolved = text.replace(f"package://{{package_name}}/", str(package_root) + "/") + resolved = resolved.replace("package://", str(package_root) + "/") + out = Path("/tmp/roboarm_chem_04_resolved.urdf") + out.write_text(resolved, encoding="utf-8") + return out + + +def _set_xform(prim, placement): + from pxr import Gf, UsdGeom + + xform = UsdGeom.Xformable(prim) + xform.ClearXformOpOrder() + xform.AddTranslateOp(UsdGeom.XformOp.PrecisionDouble).Set(_vec3(placement["translation"])) + xform.AddRotateXYZOp(UsdGeom.XformOp.PrecisionDouble).Set(Gf.Vec3d(*[float(v) for v in placement["rotation_xyz"]])) + xform.AddScaleOp(UsdGeom.XformOp.PrecisionDouble).Set(Gf.Vec3d(*[float(v) for v in placement["scale"]])) + + +def _define_usd_reference(stage, placement): + from pxr import UsdGeom + + prim = UsdGeom.Xform.Define(stage, placement["prim_path"]).GetPrim() + prim.GetReferences().AddReference(placement["asset_path"]) + _set_xform(prim, placement) + return prim + + +def _define_marker(stage, placement): + from pxr import Gf, UsdGeom + + prim = UsdGeom.Cube.Define(stage, placement["prim_path"]).GetPrim() + UsdGeom.Cube(prim).CreateSizeAttr(1.0) + color = placement.get("color") or [0.2, 0.6, 0.55] + UsdGeom.Gprim(prim).CreateDisplayColorAttr([Gf.Vec3f(*[float(v) for v in color])]) + _set_xform(prim, placement) + return prim + + +def _enable_urdf_extension(sim_app): + try: + try: + from isaacsim.core.utils.extensions import enable_extension + except Exception: + from omni.isaac.core.utils.extensions import enable_extension + + enable_extension("isaacsim.asset.importer.urdf") + sim_app.update() + except Exception as exc: + print(f"[lab layout] could not enable URDF importer: {{exc}}", flush=True) + + +def _move_imported_prim(stage, imported_path: str | None, target_path: str) -> str | None: + if not imported_path or imported_path == target_path: + return imported_path + prim = stage.GetPrimAtPath(str(imported_path)) + if not prim or not prim.IsValid(): + return imported_path + try: + import omni.kit.commands + + omni.kit.commands.execute("MovePrim", path_from=str(imported_path), path_to=str(target_path)) + moved = stage.GetPrimAtPath(target_path) + if moved and moved.IsValid(): + return target_path + except Exception as exc: + print(f"[lab layout] imported URDF prim move skipped: {{exc}}", flush=True) + return imported_path + + +def _import_urdf(stage, placement, sim_app): + import omni.kit.commands + + _enable_urdf_extension(sim_app) + source = Path(placement["asset_path"]).expanduser() + if not source.exists(): + raise FileNotFoundError(f"URDF asset not found: {{source}}") + resolved = _resolve_package_urdf(source) + status, import_config = omni.kit.commands.execute("URDFCreateImportConfig") + if not status: + raise RuntimeError("URDFCreateImportConfig failed") + for name, value in {{ + "merge_fixed_joints": False, + "fix_base": True, + "make_default_prim": False, + "self_collision": False, + "import_inertia_tensor": True, + "create_physics_scene": True, + }}.items(): + method = f"set_{{name}}" + try: + if hasattr(import_config, method): + getattr(import_config, method)(value) + elif hasattr(import_config, name): + setattr(import_config, name, value) + except Exception: + pass + kwargs = {{ + "urdf_path": str(resolved), + "import_config": import_config, + }} + imported_path = None + result = omni.kit.commands.execute("URDFParseAndImportFile", **kwargs) + sim_app.update() + if isinstance(result, tuple): + status = bool(result[0]) + if len(result) > 1: + imported_path = str(result[1]) + else: + status = bool(result) + if isinstance(result, str): + imported_path = result + if not status: + raise RuntimeError(f"URDFParseAndImportFile failed for {{placement['asset_path']}}") + _move_imported_prim(stage, imported_path, placement["prim_path"]) + prim = stage.GetPrimAtPath(placement["prim_path"]) + if not prim or not prim.IsValid(): + prim = stage.DefinePrim(placement["prim_path"], "Xform") + _set_xform(prim, placement) + return prim + + +def _define_camera(stage, camera): + from pxr import Gf, UsdGeom + + camera_prim = UsdGeom.Camera.Define(stage, "/World/Camera") + camera_prim.GetFocalLengthAttr().Set(float(camera["focal_length"])) + prim = camera_prim.GetPrim() + xform = UsdGeom.Xformable(prim) + xform.ClearXformOpOrder() + xform.AddTranslateOp().Set(_vec3(camera["translation"])) + xform.AddRotateXYZOp().Set(Gf.Vec3f(*[float(v) for v in camera["rotation_xyz"]])) + return prim + + +def _define_lighting(stage): + from pxr import UsdLux + + dome = UsdLux.DomeLight.Define(stage, "/World/Lights/Dome") + dome.CreateIntensityAttr(450.0) + distant = UsdLux.DistantLight.Define(stage, "/World/Lights/Key") + distant.CreateIntensityAttr(650.0) + distant.CreateAngleAttr(0.35) + + +def build_stage(output_stage: str, sim_app) -> dict: + from pxr import UsdGeom + import omni.usd + + output = Path(output_stage) + output.parent.mkdir(parents=True, exist_ok=True) + ctx = omni.usd.get_context() + ctx.new_stage() + sim_app.update() + stage = ctx.get_stage() + if stage is None: + raise RuntimeError("Isaac USD context did not create a stage") + UsdGeom.SetStageUpAxis(stage, UsdGeom.Tokens.z) + UsdGeom.SetStageMetersPerUnit(stage, float(LAYOUT["meters_per_unit"])) + world = UsdGeom.Xform.Define(stage, "/World") + stage.SetDefaultPrim(world.GetPrim()) + UsdGeom.Xform.Define(stage, "/World/Lab") + UsdGeom.Xform.Define(stage, "/World/Lights") + _define_lighting(stage) + _define_camera(stage, LAYOUT["camera"]) + for placement in LAYOUT["placements"]: + if placement["asset_kind"] == "usd": + _define_usd_reference(stage, placement) + elif placement["asset_kind"] == "urdf": + _import_urdf(stage, placement, sim_app) + elif placement["asset_kind"] == "marker": + _define_marker(stage, placement) + else: + raise ValueError(f"unknown asset kind: {{placement['asset_kind']}}") + stage.GetRootLayer().Export(str(output)) + try: + stage.GetRootLayer().Save() + except Exception: + pass + return {{ + "stage": str(output), + "camera": LAYOUT["camera"]["prim_path"], + "query_targets": LAYOUT["query_targets"], + }} + + +def main() -> int: + args = parse_args() + if args.kit_exec or _kit_exec_requested(): + app = _KitExecAppAdapter() + exit_code = 0 + try: + result = build_stage(args.out, app) + for _ in range(max(0, int(args.warmup_steps))): + app.update() + print(json.dumps(result, ensure_ascii=False), flush=True) + except Exception: + import traceback + + exit_code = 1 + traceback.print_exc() + finally: + try: + import omni.kit.app + + omni.kit.app.get_app().post_quit() + except Exception: + pass + return exit_code + + from isaacsim import SimulationApp + + app = SimulationApp({{"headless": bool(args.headless)}}) + try: + result = build_stage(args.out, app) + for _ in range(max(0, int(args.warmup_steps))): + app.update() + print(json.dumps(result, ensure_ascii=False), flush=True) + finally: + app.close() + return 0 + + +if __name__ == "__main__": + if _kit_exec_requested(): + main() + else: + raise SystemExit(main()) +''' + + +def _placement_to_dict(placement: Placement) -> dict[str, Any]: + payload: dict[str, Any] = { + "key": placement.key, + "label": placement.label, + "prim_path": placement.prim_path, + "asset_kind": placement.asset_kind, + "asset_path": placement.asset_path, + "translation": list(placement.translation), + "rotation_xyz": list(placement.rotation_xyz), + "scale": list(placement.scale), + "note": placement.note, + } + if placement.color is not None: + payload["color"] = list(placement.color) + return payload + + +def _camera_to_dict(camera: CameraSpec) -> dict[str, Any]: + return { + "prim_path": camera.prim_path, + "translation": list(camera.translation), + "rotation_xyz": list(camera.rotation_xyz), + "focal_length": camera.focal_length, + } diff --git a/unilabos/sim/backends/isaac/protocol.py b/unilabos/sim/backends/isaac/protocol.py new file mode 100644 index 00000000..7de708c7 --- /dev/null +++ b/unilabos/sim/backends/isaac/protocol.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import json +from typing import Any + + +def encode_request(op: str, args: dict[str, Any] | None = None) -> bytes: + payload = {"op": str(op), "args": dict(args or {})} + return json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def decode_request(data: bytes) -> tuple[str, dict[str, Any]]: + payload = json.loads(data.decode("utf-8")) + op = payload.get("op") + if not op: + raise ValueError("RPC request missing op") + args = payload.get("args") or {} + if not isinstance(args, dict): + raise ValueError("RPC request args must be an object") + return str(op), dict(args) + + +def encode_response(result: Any = None) -> bytes: + return json.dumps({"ok": True, "result": result}, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def encode_error(error: str) -> bytes: + return json.dumps({"ok": False, "error": str(error)}, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + + +def decode_response(data: bytes) -> Any: + payload = json.loads(data.decode("utf-8")) + if not payload.get("ok", False): + error = payload.get("error", "unknown error") + raise RuntimeError(f"Isaac worker RPC failed: {error}") + return payload.get("result") diff --git a/unilabos/sim/backends/isaac/worker.py b/unilabos/sim/backends/isaac/worker.py new file mode 100644 index 00000000..a7c9d418 --- /dev/null +++ b/unilabos/sim/backends/isaac/worker.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import argparse +import base64 +import queue +import struct +import threading +import zlib +from dataclasses import dataclass, field +from typing import Any + +from unilabos.sim.backends.isaac.worker_http import ThreadingHTTPServer, make_handler + + +def _png_chunk(tag: bytes, payload: bytes) -> bytes: + checksum = zlib.crc32(tag) + checksum = zlib.crc32(payload, checksum) & 0xFFFFFFFF + return struct.pack(">I", len(payload)) + tag + payload + struct.pack(">I", checksum) + + +def encode_png_rgb(image: Any) -> bytes: + import numpy as np + + array = np.asarray(image) + if array.ndim == 2: + array = np.repeat(array[:, :, None], 3, axis=2) + if array.ndim != 3 or array.shape[2] < 3: + raise ValueError(f"RGB image must have shape HxWxC with C>=3, got {array.shape}") + if array.dtype != np.uint8: + if np.issubdtype(array.dtype, np.floating) and float(np.nanmax(array)) <= 1.0: + array = array * 255.0 + array = np.clip(array, 0, 255).astype(np.uint8) + channels = 4 if array.shape[2] >= 4 else 3 + color_type = 6 if channels == 4 else 2 + array = np.ascontiguousarray(array[:, :, :channels]) + height, width = int(array.shape[0]), int(array.shape[1]) + raw = b"".join(b"\x00" + array[row].tobytes() for row in range(height)) + header = struct.pack(">IIBBBBB", width, height, 8, color_type, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", header) + + _png_chunk(b"IDAT", zlib.compress(raw)) + + _png_chunk(b"IEND", b"") + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Uni-Lab-OS Isaac physics worker") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8091) + parser.add_argument("--scene", default=None) + parser.add_argument("--robot-prim", default=None) + parser.add_argument("--camera", default="/World/Camera") + parser.add_argument("--headless", action=argparse.BooleanOptionalAction, default=True) + parser.add_argument("--warmup-steps", type=int, default=2) + parser.add_argument("--rpc-timeout-s", type=float, default=600.0) + return parser.parse_args(argv) + + +@dataclass +class _WorkerJob: + op: str + args: dict[str, Any] + event: threading.Event = field(default_factory=threading.Event) + result: Any = None + error: BaseException | None = None + + +class IsaacWorkerState: + def __init__( + self, + controller: Any, + *, + dispatch_on_main_thread: bool = False, + rpc_timeout_s: float = 600.0, + ): + self.controller = controller + self.rpc_timeout_s = float(rpc_timeout_s) + self._jobs: queue.Queue[_WorkerJob] | None = queue.Queue() if dispatch_on_main_thread else None + + def health(self) -> dict[str, Any]: + pending = self._jobs.qsize() if self._jobs is not None else 0 + return {"ok": True, "backend": "isaac", "controller": type(self.controller).__name__, "pending": pending} + + def dispatch(self, op: str, args: dict[str, Any]) -> Any: + if self._jobs is None: + return self._dispatch_direct(op, args) + + job = _WorkerJob(op=op, args=dict(args)) + self._jobs.put(job) + if not job.event.wait(self.rpc_timeout_s): + raise TimeoutError(f"Isaac worker op timed out waiting for main thread: {op}") + if job.error is not None: + raise job.error + return job.result + + def process_next(self, timeout: float = 0.05) -> bool: + if self._jobs is None: + return False + try: + job = self._jobs.get(timeout=timeout) + except queue.Empty: + return False + try: + job.result = self._dispatch_direct(job.op, job.args) + except BaseException as exc: + job.error = exc + finally: + job.event.set() + self._jobs.task_done() + return True + + def _dispatch_direct(self, op: str, args: dict[str, Any]) -> Any: + if op == "reset": + return self.controller.reset() + if op == "step": + return self.controller.step(float(args.get("dt", 0.0))) + if op == "load_scene": + return self.controller.load_scene(str(args["scene_path"])) + if op == "get_observation": + return self.controller.get_observation(str(args["entity_id"])) + if op == "set_command": + return self.controller.set_command(str(args["entity_id"]), dict(args.get("command") or {})) + if op == "attach_rigid_body": + return self.controller.attach_rigid_body( + str(args["name"]), + str(args["asset_path"]), + dict(args.get("pose") or {}), + ) + if op == "get_joint_states": + return self.controller.get_joint_states(str(args["body_id"])) + if op == "apply_wrench": + return self.controller.apply_wrench(str(args["body_id"]), dict(args.get("wrench") or {})) + if op == "render": + image = self.controller.render(str(args["camera"]), int(args["width"]), int(args["height"])) + return {"encoding": "base64", "data": base64.b64encode(image).decode("ascii")} + raise ValueError(f"Unsupported Isaac worker op: {op}") + + +class IsaacController: + def __init__(self, headless: bool, camera: str, robot_prim: str | None, warmup_steps: int = 2): + from isaacsim import SimulationApp + + self.app = SimulationApp({"headless": bool(headless)}) + self.camera = camera + self.robot_prim = robot_prim + self.scene_path: str | None = None + self.commands: dict[str, dict[str, Any]] = {} + self.observations: dict[str, dict[str, Any]] = {} + self.joint_states: dict[str, dict[str, float]] = {} + self.rigid_bodies: dict[str, dict[str, Any]] = {} + self.wrenches: list[tuple[str, dict[str, Any]]] = [] + self.render_fallback: str | None = None + self.render_error: str | None = None + self._stage = None + self._last_dt = 0.0 + self._rgb_annotators: dict[tuple[str, int, int], Any] = {} + for _ in range(max(0, int(warmup_steps))): + self.app.update() + + def reset(self) -> None: + self.commands.clear() + self.observations.clear() + self.joint_states.clear() + self.wrenches.clear() + self._rgb_annotators.clear() + self.app.update() + + def step(self, dt: float) -> None: + self._last_dt = float(dt) + self.app.update() + + def load_scene(self, scene_path: str) -> None: + import omni.usd + + self.scene_path = str(scene_path) + self._rgb_annotators.clear() + omni.usd.get_context().open_stage(self.scene_path) + for _ in range(2): + self.app.update() + self._stage = omni.usd.get_context().get_stage() + + def get_observation(self, entity_id: str) -> dict[str, Any]: + observation = dict(self.observations.get(entity_id, {})) + observation.setdefault("entity_id", entity_id) + observation.setdefault("scene_path", self.scene_path) + observation.setdefault("source", "isaac_worker") + observation.setdefault("last_dt", self._last_dt) + if entity_id in self.commands: + observation["last_command"] = dict(self.commands[entity_id]) + if entity_id in self.joint_states: + observation["joint_states"] = dict(self.joint_states[entity_id]) + observation["joint_names"] = list(self.joint_states[entity_id].keys()) + observation["joint_positions"] = list(self.joint_states[entity_id].values()) + if entity_id in self.rigid_bodies: + observation.update(self.rigid_bodies[entity_id]) + prim_pose = self._query_prim_pose(entity_id) + if prim_pose is not None: + observation["pose"] = prim_pose + if self.render_fallback is not None: + observation["render_fallback"] = self.render_fallback + if self.render_error is not None: + observation["render_error"] = self.render_error + return observation + + def set_command(self, entity_id: str, command: dict[str, Any]) -> None: + self.commands[entity_id] = dict(command) + joints = command.get("joint_positions") or command.get("q") + if isinstance(joints, list): + self.joint_states[entity_id] = {f"joint_{index + 1}": float(value) for index, value in enumerate(joints)} + self.app.update() + + def attach_rigid_body(self, name: str, asset_path: str, pose: dict[str, Any]) -> str: + body_id = str(name) + self.rigid_bodies[body_id] = {"name": body_id, "asset_path": str(asset_path), "pose": dict(pose)} + return body_id + + def get_joint_states(self, body_id: str) -> dict[str, float]: + return dict(self.joint_states.get(body_id, {})) + + def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: + self.wrenches.append((str(body_id), dict(wrench))) + + def idle(self) -> None: + self.app.update() + + def render(self, camera: str, width: int, height: int) -> bytes: + image = self._render_with_isaac(camera, width, height) + if image is not None: + self.render_fallback = None + self.render_error = None + return image + self.render_fallback = "minimal_png" + meta = f"isaac-worker-render camera={camera} width={int(width)} height={int(height)} scene={self.scene_path}".encode() + return b"\x89PNG\r\n\x1a\n" + meta + + def close(self) -> None: + self.app.close() + + def _query_prim_pose(self, entity_id: str) -> dict[str, Any] | None: + if self._stage is None or not entity_id.startswith("/"): + return None + try: + from pxr import Gf, UsdGeom + + prim = self._stage.GetPrimAtPath(entity_id) + if not prim or not prim.IsValid(): + return None + matrix = UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(0.0) + translation = matrix.ExtractTranslation() + quat = Gf.Transform(matrix).GetRotation().GetQuat() + imaginary = quat.GetImaginary() + return { + "xyz": [float(translation[0]), float(translation[1]), float(translation[2])], + "quat_xyzw": [float(imaginary[0]), float(imaginary[1]), float(imaginary[2]), float(quat.GetReal())], + "frame_id": "world", + } + except Exception: + return None + + def _render_with_isaac(self, camera: str, width: int, height: int) -> bytes | None: + image = self._render_with_replicator(camera, width, height) + if image is not None: + return image + return self._render_with_viewport(camera, width, height) + + def _render_with_replicator(self, camera: str, width: int, height: int) -> bytes | None: + try: + import omni.replicator.core as rep + except Exception as exc: + self.render_error = f"replicator unavailable: {exc}" + return None + + try: + self._ensure_camera(camera) + key = (str(camera), int(width), int(height)) + if key not in self._rgb_annotators: + render_product = rep.create.render_product(str(camera), (int(width), int(height)), force_new=True) + annotator = rep.AnnotatorRegistry.get_annotator("rgb") + annotator.attach([render_product]) + self._rgb_annotators[key] = (render_product, annotator) + _, annotator = self._rgb_annotators[key] + data = None + for _ in range(4): + rep.orchestrator.step() + self.app.update() + data = annotator.get_data() + if data is not None: + if isinstance(data, dict): + data = data.get("data") if data.get("data") is not None else data.get("rgb") + shape = getattr(data, "shape", None) + if shape is not None and len(shape) >= 2 and int(shape[0]) > 0 and int(shape[1]) > 0: + break + if data is None: + self.render_error = "replicator rgb annotator returned no data" + return None + return encode_png_rgb(data) + except Exception as exc: + self.render_error = f"replicator render failed: {exc}" + return None + + def _render_with_viewport(self, camera: str, width: int, height: int) -> bytes | None: + try: + import omni.kit.viewport.utility + from omni.kit.viewport.utility import capture_viewport_to_buffer + except Exception as exc: + self.render_error = f"viewport capture unavailable: {exc}" + return None + + try: + self._ensure_camera(camera) + viewport = omni.kit.viewport.utility.get_active_viewport() + if viewport is None: + self.render_error = "active viewport unavailable" + return None + viewport.camera_path = camera + viewport.resolution = (int(width), int(height)) + self.app.update() + capture = capture_viewport_to_buffer(viewport) + data = getattr(capture, "data", None) + if isinstance(data, bytes): + return data + except Exception as exc: + self.render_error = f"viewport render failed: {exc}" + return None + return None + + def _ensure_camera(self, camera: str) -> None: + if self._stage is None: + return + try: + from pxr import Gf, UsdGeom + + prim = self._stage.GetPrimAtPath(camera) + if prim and prim.IsValid() and prim.IsA(UsdGeom.Camera): + return + camera_prim = UsdGeom.Camera.Define(self._stage, camera) + camera_prim.GetFocalLengthAttr().Set(24.0) + xform = UsdGeom.Xformable(camera_prim.GetPrim()) + xform.ClearXformOpOrder() + xform.AddTranslateOp().Set(Gf.Vec3d(2.0, -3.0, 2.0)) + xform.AddRotateXYZOp().Set(Gf.Vec3f(60.0, 0.0, 35.0)) + self.app.update() + except Exception as exc: + self.render_error = f"camera setup failed: {exc}" + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + controller = IsaacController( + headless=args.headless, + camera=args.camera, + robot_prim=args.robot_prim, + warmup_steps=args.warmup_steps, + ) + if args.scene: + controller.load_scene(args.scene) + state = IsaacWorkerState( + controller, + dispatch_on_main_thread=True, + rpc_timeout_s=args.rpc_timeout_s, + ) + server = ThreadingHTTPServer((args.host, args.port), make_handler(state)) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + print(f"[isaac worker] serving http://{args.host}:{args.port}/rpc", flush=True) + try: + while True: + processed = state.process_next(timeout=0.05) + if not processed: + controller.idle() + except KeyboardInterrupt: + pass + finally: + server.shutdown() + server.server_close() + server_thread.join(timeout=2.0) + controller.close() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/unilabos/sim/backends/isaac/worker_http.py b/unilabos/sim/backends/isaac/worker_http.py new file mode 100644 index 00000000..5ca6ee96 --- /dev/null +++ b/unilabos/sim/backends/isaac/worker_http.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from socketserver import ThreadingMixIn +from typing import Any + +from unilabos.sim.backends.isaac.protocol import decode_request, encode_error, encode_response + + +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): + daemon_threads = True + allow_reuse_address = True + + +def make_handler(worker_state: Any): + class Handler(BaseHTTPRequestHandler): + def log_message(self, fmt, *args): + return + + def do_GET(self): + path = self.path.split("?", 1)[0] + if path == "/health": + self._write_json(worker_state.health(), status=200) + return + self.send_response(404) + self.end_headers() + + def do_POST(self): + path = self.path.split("?", 1)[0] + if path != "/rpc": + self.send_response(404) + self.end_headers() + return + try: + length = int(self.headers.get("Content-Length", "0") or "0") + op, args = decode_request(self.rfile.read(length)) + self._write_body(encode_response(worker_state.dispatch(op, args)), status=200) + except Exception as exc: + self._write_body(encode_error(str(exc)), status=500) + + def _write_json(self, payload: dict[str, Any], status: int) -> None: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + self._write_body(body, status=status) + + def _write_body(self, body: bytes, status: int) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + return Handler diff --git a/unilabos/sim/backends/isaac_bridge.py b/unilabos/sim/backends/isaac_bridge.py new file mode 100644 index 00000000..d1938bf6 --- /dev/null +++ b/unilabos/sim/backends/isaac_bridge.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import base64 +from typing import Any, Callable +from urllib import request +from urllib.error import HTTPError, URLError + +from unilabos.sim.backends.isaac.protocol import decode_response, encode_request + + +class IsaacBridgeBackend: + name = "isaac" + + def __init__(self, endpoint: str, timeout: float = 5.0) -> None: + self.endpoint = endpoint.rstrip("/") + self.timeout = float(timeout) + + def _rpc(self, op: str, args: dict[str, Any] | None = None) -> Any: + req = request.Request( + f"{self.endpoint}/rpc", + data=encode_request(op, args), + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with request.urlopen(req, timeout=self.timeout) as response: + return decode_response(response.read()) + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"Isaac worker HTTP {exc.code}: {detail}") from exc + except URLError as exc: + raise RuntimeError(f"Isaac worker unavailable at {self.endpoint}: {exc.reason}") from exc + except TimeoutError as exc: + raise RuntimeError(f"Isaac worker timed out after {self.timeout:.1f}s during {op}") from exc + + def reset(self) -> None: + self._rpc("reset") + + def step(self, dt: float) -> None: + self._rpc("step", {"dt": float(dt)}) + + def load_scene(self, scene_path: str) -> None: + self._rpc("load_scene", {"scene_path": str(scene_path)}) + + def get_observation(self, entity_id: str) -> dict[str, Any]: + return dict(self._rpc("get_observation", {"entity_id": str(entity_id)}) or {}) + + def set_command(self, entity_id: str, command: dict[str, Any]) -> None: + self._rpc("set_command", {"entity_id": str(entity_id), "command": dict(command)}) + + def attach_rigid_body(self, name: str, asset_path: str, pose: dict[str, Any]) -> str: + result = self._rpc( + "attach_rigid_body", + {"name": str(name), "asset_path": str(asset_path), "pose": dict(pose)}, + ) + return str(result) + + def get_joint_states(self, body_id: str) -> dict[str, float]: + result = self._rpc("get_joint_states", {"body_id": str(body_id)}) or {} + return {str(key): float(value) for key, value in dict(result).items()} + + def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: + self._rpc("apply_wrench", {"body_id": str(body_id), "wrench": dict(wrench)}) + + def register_contact_callback(self, callback: Callable[[dict[str, Any]], None]) -> None: + raise NotImplementedError("IsaacBridgeBackend does not support edge-side contact callbacks yet") + + def render(self, camera: str, width: int, height: int) -> bytes: + result = self._rpc("render", {"camera": str(camera), "width": int(width), "height": int(height)}) + if isinstance(result, dict) and result.get("encoding") == "base64": + return base64.b64decode(str(result.get("data", ""))) + if isinstance(result, str): + return base64.b64decode(result) + raise TypeError(f"Isaac render returned unsupported payload: {type(result).__name__}") diff --git a/unilabos/sim/context.py b/unilabos/sim/context.py index 8ffe3178..5ec8c929 100644 --- a/unilabos/sim/context.py +++ b/unilabos/sim/context.py @@ -17,6 +17,10 @@ class RuntimeContext: mode: RuntimeMode = "real" clock: SimClock = field(default_factory=lambda: SimClock("real")) physics: Optional[PhysicsBackend] = None + physics_backend_name: str = "none" + physics_endpoint: Optional[str] = None + physics_scene: Optional[str] = None + physics_timeout: float = 120.0 missing_sim_policy_default: MissingSimPolicy = "stub" sim_paused: bool = False sim_services_enabled: bool = True diff --git a/unilabos/sim/device_physics.py b/unilabos/sim/device_physics.py new file mode 100644 index 00000000..148686bd --- /dev/null +++ b/unilabos/sim/device_physics.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Any + +from unilabos.sim.context import get_runtime_context + + +def dispatch_device_command(entity_id: str, command: dict[str, Any]) -> bool: + backend = get_runtime_context().physics + if backend is None: + return False + backend.set_command(str(entity_id), dict(command)) + return True diff --git a/unilabos/sim/physics_backend.py b/unilabos/sim/physics_backend.py index e2f1432c..4bc90b9c 100644 --- a/unilabos/sim/physics_backend.py +++ b/unilabos/sim/physics_backend.py @@ -21,6 +21,9 @@ def reset(self) -> None: def step(self, dt: float) -> None: ... + def load_scene(self, scene_path: str) -> None: + ... + def get_observation(self, entity_id: str) -> dict[str, Any]: ... @@ -38,3 +41,6 @@ def apply_wrench(self, body_id: str, wrench: dict[str, Any]) -> None: def register_contact_callback(self, callback: Callable[[dict[str, Any]], None]) -> None: ... + + def render(self, camera: str, width: int, height: int) -> bytes: + ... diff --git a/unilabos/sim/runtime.py b/unilabos/sim/runtime.py index a40a87a0..7ef87188 100644 --- a/unilabos/sim/runtime.py +++ b/unilabos/sim/runtime.py @@ -26,9 +26,23 @@ def configure_runtime( sim_rate: float = 1.0, sim_paused: bool = False, start_ros_services: bool = False, + physics=None, + physics_backend_name: str = "none", + physics_endpoint: str | None = None, + physics_scene: str | None = None, + physics_timeout: float = 120.0, ) -> RuntimeServices: clock = SimClock(mode=mode, scale=sim_rate) - context = RuntimeContext(mode=mode, clock=clock, sim_paused=sim_paused) + context = RuntimeContext( + mode=mode, + clock=clock, + sim_paused=sim_paused, + physics=physics, + physics_backend_name=physics_backend_name, + physics_endpoint=physics_endpoint, + physics_scene=physics_scene, + physics_timeout=physics_timeout, + ) init_runtime_context(context) services = RuntimeServices(context=context) if start_ros_services and mode in ("sim", "twin"):