diff --git a/.gitattributes b/.gitattributes index c4ccd1f825..b4ea3b5a3b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,3 +18,6 @@ *.foxe filter=lfs diff=lfs merge=lfs -text binary docs/capabilities/memory/assets/** filter=lfs diff=lfs merge=lfs -text docs/capabilities/memory/assets/.gitattributes -filter -diff -merge text +# DimSim scene data and agent model +misc/DimSim/public/sims/*.json filter=lfs diff=lfs merge=lfs -text +misc/DimSim/public/agent-model/*.glb filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/dimsim-check.yml b/.github/workflows/dimsim-check.yml new file mode 100644 index 0000000000..372f919db0 --- /dev/null +++ b/.github/workflows/dimsim-check.yml @@ -0,0 +1,58 @@ +name: dimsim-check + +on: + push: + branches: [main] + paths: + - 'misc/DimSim/**' + - '.github/workflows/dimsim-check.yml' + pull_request: + paths: + - 'misc/DimSim/**' + - '.github/workflows/dimsim-check.yml' + +permissions: {} + +jobs: + check: + timeout-minutes: 15 + runs-on: ubuntu-latest + permissions: + contents: read + defaults: + run: + working-directory: misc/DimSim + steps: + - uses: actions/checkout@v6 + with: + lfs: true + + - uses: actions/setup-node@v4 + with: + node-version: 20 + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - name: Install npm deps + run: npm ci + + - name: Build frontend + run: npm run build + + - name: Type-check CLI + run: cd dimos-cli && deno check cli.ts mod.ts setup.ts + + - name: Validate scenes template + run: | + for scene in $(jq -r '.scenes | keys[]' scenes.template.json); do + DESC=$(jq -r ".scenes.\"$scene\".description" scenes.template.json) + SIZE=$(jq -r ".scenes.\"$scene\".size" scenes.template.json) + if [ "$DESC" = "null" ] || [ "$SIZE" = "null" ]; then + echo "::error::Scene '$scene' missing description or size in scenes.template.json" + exit 1 + fi + echo " $scene: $DESC (${SIZE} bytes)" + done + echo "Template valid." diff --git a/.gitignore b/.gitignore index 7e12f67569..efb34af7d0 100644 --- a/.gitignore +++ b/.gitignore @@ -39,8 +39,8 @@ __pycache__ # node env (used by devcontainers cli) node_modules -package.json -package-lock.json +/package.json +/package-lock.json # Ignore build artifacts dist/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9cb8099080..7591ab08f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -39,11 +39,13 @@ repos: args: [--fix=lf] exclude: \.patch$ # keep patch files byte-identical to upstream diffs - id: check-json + exclude: ^misc/DimSim/public/sims/ # LFS-tracked - id: check-toml - id: check-yaml - id: pretty-format-json name: format json args: [ --autofix, --no-sort-keys ] + exclude: ^misc/DimSim/public/sims/ # LFS-tracked - repo: https://github.com/editorconfig-checker/editorconfig-checker.python rev: 3.4.1 diff --git a/dimos/simulation/dimsim/dimsim_process.py b/dimos/simulation/dimsim/dimsim_process.py index ead774c677..67803dcf89 100644 --- a/dimos/simulation/dimsim/dimsim_process.py +++ b/dimos/simulation/dimsim/dimsim_process.py @@ -19,7 +19,6 @@ import time from typing import IO -from dimos.constants import STATE_DIR from dimos.core.global_config import GlobalConfig from dimos.simulation.dimsim.deno_utils import ensure_deno, ensure_playwright_chromium from dimos.utils.logging_config import setup_logger @@ -28,8 +27,7 @@ _VIDEO_RATE = 50 _LIDAR_RATE = 1000 -_DIMSIM_REPO_URL = "https://github.com/paul-nechifor/DimSim.git" -_DIMSIM_REPO_BRANCH = "run-from-repo" +_DIMSIM_DIR = Path(__file__).resolve().parents[3] / "misc" / "DimSim" class DimSimProcess: @@ -39,8 +37,7 @@ def __init__(self, global_config: GlobalConfig) -> None: def start(self) -> None: deno_path = ensure_deno() - repo_dir = _ensure_repo() - base_cmd = _deno_cmd(deno_path, repo_dir) + base_cmd = _deno_cmd(deno_path, _DIMSIM_DIR) scene = self.global_config.dimsim_scene port = self.global_config.dimsim_port @@ -126,28 +123,6 @@ def _kill_port_holder(port: int) -> None: logger.warning(f"Failed to check/kill port {port}: {e}") -def _ensure_repo() -> Path: - repo_dir = STATE_DIR / "dimsim_repo" - if (repo_dir / ".git").exists(): - return repo_dir - STATE_DIR.mkdir(parents=True, exist_ok=True) - logger.info(f"Cloning DimSim into {repo_dir}") - subprocess.run( - [ - "git", - "clone", - "--depth", - "1", - "--branch", - _DIMSIM_REPO_BRANCH, - _DIMSIM_REPO_URL, - str(repo_dir), - ], - check=True, - ) - return repo_dir - - def _deno_cmd(deno_path: str, repo_dir: Path) -> list[str]: cli_ts = repo_dir / "dimos-cli" / "cli.ts" return [deno_path, "run", "--allow-all", "--unstable-net", str(cli_ts)] diff --git a/misc/DimSim/.gitignore b/misc/DimSim/.gitignore new file mode 100644 index 0000000000..ce85a96daa --- /dev/null +++ b/misc/DimSim/.gitignore @@ -0,0 +1,7 @@ +node_modules/ +*.tar.gz +dist/ +.DS_Store +**/.DS_Store +.deno/ +scenes.json diff --git a/misc/DimSim/README.md b/misc/DimSim/README.md new file mode 100644 index 0000000000..4df77dd82e --- /dev/null +++ b/misc/DimSim/README.md @@ -0,0 +1,55 @@ +# DimSim + +Standalone 3D simulation runner for SimStudio scenes. Load a scene, spawn AI agents, run tasks — with full sensor support (RGB-D, LiDAR). + +## Setup + +```bash +npm install # installs everything (frontend + backend) +``` + +## Run + +Terminal 1: +```bash +npm run server # Node.js VLM backend on :8000 +``` + +Terminal 2: +```bash +npm run dev # Frontend on :5173 +``` + +## Architecture + +``` +DimSim/ +├── index.html ← Sim-mode UI (scene dropdown + full sensor controls) +├── server.js ← VLM backend (Express + OpenAI SDK) +├── src/ +│ ├── main.js ← Entry point (imports engine.js) +│ ├── engine.js ← Full SimStudio engine (synced via copy-sources.sh) +│ ├── style.css ← Synced from SimStudio +│ ├── AiAvatar.js ← Agent class (synced) +│ └── ai/ ← VLM modules (synced) +├── public/ +│ ├── sims/ ← Scene JSON files + manifest.json +│ └── agent-model/ ← Robot GLB models +├── vlm-server/ +│ └── asset-library.json ← Persisted asset library data +├── copy-sources.sh ← Sync engine from SimStudio +└── update-sims.sh ← Rebuild scene manifest +``` + +## Sync from SimStudio + +```bash +npm run sync +``` + +## Add/remove scenes + +Drop `.json` files in `public/sims/`, then: +```bash +npm run update-sims +``` diff --git a/misc/DimSim/deno.lock b/misc/DimSim/deno.lock new file mode 100644 index 0000000000..d1b33f132e --- /dev/null +++ b/misc/DimSim/deno.lock @@ -0,0 +1,122 @@ +{ + "version": "5", + "specifiers": { + "jsr:@antim/dimsim@*": "0.1.3", + "jsr:@antim/dimsim@0.1.28": "0.1.28", + "jsr:@antim/dimsim@0.1.29": "0.1.29", + "jsr:@dimos/lcm@*": "0.2.0", + "jsr:@dimos/lcm@0.2.0": "0.2.0", + "jsr:@dimos/msgs@0.1.4": "0.1.4", + "jsr:@dimos/msgs@~0.1.4": "0.1.4", + "jsr:@std/assert@*": "1.0.19", + "jsr:@std/cli@^1.0.28": "1.0.28", + "jsr:@std/encoding@^1.0.10": "1.0.10", + "jsr:@std/fmt@^1.0.9": "1.0.9", + "jsr:@std/fs@^1.0.23": "1.0.23", + "jsr:@std/html@^1.0.5": "1.0.5", + "jsr:@std/http@1": "1.0.25", + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/media-types@^1.1.0": "1.1.0", + "jsr:@std/net@^1.0.6": "1.0.6", + "jsr:@std/path@1": "1.1.4", + "jsr:@std/path@^1.1.4": "1.1.4", + "jsr:@std/streams@^1.0.17": "1.0.17" + }, + "jsr": { + "@antim/dimsim@0.1.3": { + "integrity": "b145e83a4545ba03004ccfcd0c14afad3cb9af3258f8174b0f0a8cc64b9949da", + "dependencies": [ + "jsr:@std/http", + "jsr:@std/path@1" + ] + }, + "@antim/dimsim@0.1.28": { + "integrity": "ad194c3c478a403d31b9320df6fe14d52783a26fa6c8c25a6ea57863926513a4", + "dependencies": [ + "jsr:@dimos/msgs@~0.1.4", + "jsr:@std/http", + "jsr:@std/path@1" + ] + }, + "@antim/dimsim@0.1.29": { + "integrity": "b8d6c2edc8bf82c6b83ac01fa8cc24f16332aadbf8b1e46733bcdf9c7d9e77b6", + "dependencies": [ + "jsr:@dimos/msgs@~0.1.4", + "jsr:@std/http", + "jsr:@std/path@1" + ] + }, + "@dimos/lcm@0.2.0": { + "integrity": "03399f5e4800f28a0c294981e0210d784232fc65a57707de19052ad805bd5fea" + }, + "@dimos/msgs@0.1.4": { + "integrity": "564bc30b4bc41a562c296c257a15055283ca0cbd66d0627991ede5295832d0c4" + }, + "@std/assert@1.0.19": { + "integrity": "eaada96ee120cb980bc47e040f82814d786fe8162ecc53c91d8df60b8755991e", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/cli@1.0.28": { + "integrity": "74ef9b976db59ca6b23a5283469c9072be6276853807a83ec6c7ce412135c70a" + }, + "@std/encoding@1.0.10": { + "integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1" + }, + "@std/fmt@1.0.9": { + "integrity": "2487343e8899fb2be5d0e3d35013e54477ada198854e52dd05ed0422eddcabe0" + }, + "@std/fs@1.0.23": { + "integrity": "3ecbae4ce4fee03b180fa710caff36bb5adb66631c46a6460aaad49515565a37" + }, + "@std/html@1.0.5": { + "integrity": "4e2d693f474cae8c16a920fa5e15a3b72267b94b84667f11a50c6dd1cb18d35e" + }, + "@std/http@1.0.25": { + "integrity": "577b4252290af1097132812b339fffdd55fb0f4aeb98ff11bdbf67998aa17193", + "dependencies": [ + "jsr:@std/cli", + "jsr:@std/encoding", + "jsr:@std/fmt", + "jsr:@std/fs", + "jsr:@std/html", + "jsr:@std/media-types", + "jsr:@std/net", + "jsr:@std/path@^1.1.4", + "jsr:@std/streams" + ] + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/media-types@1.1.0": { + "integrity": "c9d093f0c05c3512932b330e3cc1fe1d627b301db33a4c2c2185c02471d6eaa4" + }, + "@std/net@1.0.6": { + "integrity": "110735f93e95bb9feb95790a8b1d1bf69ec0dc74f3f97a00a76ea5efea25500c" + }, + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/streams@1.0.17": { + "integrity": "7859f3d9deed83cf4b41f19223d4a67661b3d3819e9fc117698f493bf5992140" + } + }, + "workspace": { + "packageJson": { + "dependencies": [ + "npm:@dimforge/rapier3d-compat@0.14", + "npm:@sparkjsdev/spark@latest", + "npm:cors@^2.8.5", + "npm:express@^4.21.0", + "npm:openai@^4.77.0", + "npm:three@0.168", + "npm:vite@^5.4.10" + ] + } + } +} diff --git a/misc/DimSim/dimos-cli/README.md b/misc/DimSim/dimos-cli/README.md new file mode 100644 index 0000000000..613e5d6b78 --- /dev/null +++ b/misc/DimSim/dimos-cli/README.md @@ -0,0 +1,58 @@ +# DimSim + +3D simulation environment for the [dimos](https://github.com/dimensionalOS/dimos) robotics stack. + +Browser-based Three.js + Rapier simulator with LCM transport, sensor publishing (RGB, depth, LiDAR, odometry), and an eval harness for automated testing of navigation and perception pipelines. + +## Install + +```sh +deno install -gAf --unstable-net jsr:@antim/dimsim +``` + +## Setup + +Download core assets (~22 MB) and install a scene: + +```sh +dimsim setup +dimsim scene install apt +``` + +## Run + +Start the dev server and open the URL it prints: + +```sh +dimsim dev --scene apt +``` + +Run headless evals in CI: + +```sh +dimsim eval --headless --env apt --workflow reach-vase +``` + +## Programmatic API + +```ts +import { startBridgeServer } from "@antim/dimsim"; + +startBridgeServer({ port: 8090, distDir: "./dist", scene: "apt" }); +``` + +## Commands + +| Command | Description | +|---------|-------------| +| `dimsim setup` | Download core assets | +| `dimsim scene install ` | Install a scene | +| `dimsim scene list` | List available and installed scenes | +| `dimsim scene remove ` | Remove a scene | +| `dimsim dev [--scene ]` | Dev server (open browser manually) | +| `dimsim eval --headless` | Run eval workflows in CI | +| `dimsim agent` | Launch dimos Python agent | + +## License + +MIT diff --git a/misc/DimSim/dimos-cli/agent.py b/misc/DimSim/dimos-cli/agent.py new file mode 100644 index 0000000000..ea68949272 --- /dev/null +++ b/misc/DimSim/dimos-cli/agent.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +DimSim Agent — runs the dimos nav + agent stack connected to DimSim via LCM. + +DimSim acts as the robot (like simplerobot.py but richer): + - Publishes: /odom, /color_image, /lidar, /depth_image + - Subscribes: /cmd_vel + +This script runs the dimos brain that processes those sensors and sends commands. + +Usage (run with dimos venv): + ../dimos/.venv/bin/python dimos-cli/agent.py + ../dimos/.venv/bin/python dimos-cli/agent.py --nav-only # no LLM agent, just exploration +""" + +import argparse + +from dimos.core.blueprints import autoconnect +from dimos.core.transport import JpegLcmTransport, LCMTransport +from dimos.mapping.costmapper import cost_mapper +from dimos.mapping.voxels import voxel_mapper +from dimos.msgs.geometry_msgs import PoseStamped, Twist +from dimos.msgs.sensor_msgs import Image, PointCloud2 +from dimos.navigation.frontier_exploration import wavefront_frontier_explorer +from dimos.navigation.replanning_a_star.module import replanning_a_star_planner +from dimos.protocol.service.lcmservice import autoconf + +# LCM transports — same channels DimSim publishes/subscribes on. +_transports = { + ("color_image", Image): JpegLcmTransport("/color_image", Image), + ("odom", PoseStamped): LCMTransport("/odom", PoseStamped), + ("cmd_vel", Twist): LCMTransport("/cmd_vel", Twist), + ("lidar", PointCloud2): LCMTransport("/lidar", PointCloud2), +} + +# Navigation stack: LiDAR → voxels → costmap → frontier explorer → path planner +nav = ( + autoconnect( + voxel_mapper(voxel_size=0.1), + cost_mapper(algo="simple"), + replanning_a_star_planner(), + wavefront_frontier_explorer(), + ) + .transports(_transports) + .global_config(n_dask_workers=6, robot_model="dimsim") +) + + +def build_agentic(): + """Full agentic: nav + spatial memory + LLM agent + skills.""" + from dimos.agents.agent import llm_agent + from dimos.agents.cli.human import human_input + from dimos.agents.cli.web import web_input + from dimos.agents.skills.navigation import navigation_skill + from dimos.agents.skills.speak_skill import speak_skill + from dimos.perception.spatial_perception import spatial_memory + from dimos.utils.monitoring import utilization + + return autoconnect( + nav, + spatial_memory(), + utilization(), + llm_agent(), + human_input(), + navigation_skill(), + web_input(), + speak_skill(), + ).global_config(n_dask_workers=8) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="DimSim dimos agent") + parser.add_argument( + "--nav-only", + action="store_true", + help="Run nav stack only (no LLM agent)", + ) + args = parser.parse_args() + + autoconf() + + blueprint = nav if args.nav_only else build_agentic() + coordinator = blueprint.build() + + print("DimSim agent running.") + print(" Subscribing: /odom, /color_image, /lidar") + print(" Publishing: /cmd_vel") + print(" Ctrl+C to exit") + + coordinator.loop() diff --git a/misc/DimSim/dimos-cli/bridge/lidar.ts b/misc/DimSim/dimos-cli/bridge/lidar.ts new file mode 100644 index 0000000000..275a0af1f5 --- /dev/null +++ b/misc/DimSim/dimos-cli/bridge/lidar.ts @@ -0,0 +1,284 @@ +/** + * Server-side LiDAR raycasting using Rapier physics world snapshot. + * + * Runs 20K Fibonacci-sphere raycasts at 5 Hz on the Deno bridge server, + * encodes PointCloud2 via @dimos/msgs, and publishes directly to LCM — + * no WebSocket hop needed. + */ + +import { + sensor_msgs, + std_msgs, +} from "@dimos/msgs"; +import type { LCM } from "../vendor/lcm/lcm.ts"; + +// -- Lidar constants (must match engine.js) ----------------------------------- +const NUM_POINTS = 15000; +const MIN_RANGE = 0.1; +const MAX_RANGE = 4; +const V_MIN_RAD = (-30 * Math.PI) / 180; +const V_MAX_RAD = (15 * Math.PI) / 180; +const RATE_MS = 100; // 10 Hz + +const CH_LIDAR = "/lidar#sensor_msgs.PointCloud2"; + +// Agent capsule geometry → lidar mount offset (must match engine.js) +const DEFAULT_HALF_HEIGHT = 0.25; +const DEFAULT_RADIUS = 0.12; +const DEFAULT_LIDAR_MOUNT = 0.35; + +/** Subset of EmbodimentConfig relevant to lidar mount offset. */ +export interface LidarEmbodimentConfig { + radius?: number; + halfHeight?: number; + lidarMountHeight?: number; +} + +// -- _lidarToCamQuat: transforms FLU (x=forward, y=left, z=up) → Three.js camera-local -- +// Matches engine.js _lidarToCamQuat derived from rotation matrix: +// FLU x(forward) → cam -z, FLU y(left) → cam -x, FLU z(up) → cam +y +// Quaternion: (0.5, -0.5, -0.5, -0.5) +const L2C_QX = 0.5, L2C_QY = -0.5, L2C_QZ = -0.5, L2C_QW = -0.5; + +// -- Pre-compute Fibonacci sphere ray directions (pre-rotated to camera-local) - +// Directions are computed in FLU frame then rotated by _lidarToCamQuat so that +// only the agent's yaw quaternion is needed at scan time (cam-local → world). +const fibDirs = (() => { + const golden = (1 + Math.sqrt(5)) / 2; + const zMin = Math.sin(V_MIN_RAD); + const zMax = Math.sin(V_MAX_RAD); + const dirs = new Float32Array(NUM_POINTS * 3); + for (let i = 0; i < NUM_POINTS; i++) { + const z = zMin + (zMax - zMin) * (i + 0.5) / NUM_POINTS; + const r = Math.sqrt(1 - z * z); + const phi = (2 * Math.PI * i) / golden; + const fx = r * Math.cos(phi); // FLU x + const fy = r * Math.sin(phi); // FLU y + const fz = z; // FLU z + + // Rotate FLU → camera-local using _lidarToCamQuat + const tx = 2 * (L2C_QY * fz - L2C_QZ * fy); + const ty = 2 * (L2C_QZ * fx - L2C_QX * fz); + const tz = 2 * (L2C_QX * fy - L2C_QY * fx); + dirs[i * 3 + 0] = fx + L2C_QW * tx + (L2C_QY * tz - L2C_QZ * ty); + dirs[i * 3 + 1] = fy + L2C_QW * ty + (L2C_QZ * tx - L2C_QX * tz); + dirs[i * 3 + 2] = fz + L2C_QW * tz + (L2C_QX * ty - L2C_QY * tx); + } + return dirs; +})(); + +// -- Quaternion rotation helper (q * v) --------------------------------------- +function rotateByQuat( + vx: number, vy: number, vz: number, + qx: number, qy: number, qz: number, qw: number, +): [number, number, number] { + // t = 2 * cross(q.xyz, v) + const tx = 2 * (qy * vz - qz * vy); + const ty = 2 * (qz * vx - qx * vz); + const tz = 2 * (qx * vy - qy * vx); + // result = v + qw * t + cross(q.xyz, t) + return [ + vx + qw * tx + (qy * tz - qz * ty), + vy + qw * ty + (qz * tx - qx * tz), + vz + qw * tz + (qx * ty - qy * tx), + ]; +} + +// -- ServerLidar -------------------------------------------------------------- + +export class ServerLidar { + private lcm: LCM; + private world: any; // RAPIER.World + private RAPIER: any; + private sentSeqs: Set; // echo filter shared with bridge server + private timer: ReturnType | null = null; + private scanCount = 0; + private logN = 0; + private publishing = false; // busy guard — skip scan if previous publish still in flight + private excludeBody: any = null; // rigid body to exclude from raycasting (agent's own colliders) + private lidarYOffset: number; + + // Current robot pose (Three.js Y-up world frame) + private px = 0; + private py = 0; + private pz = 0; + private qx = 0; + private qy = 0; + private qz = 0; + private qw = 1; + private hasPose = false; + + private ray: any; // Reusable Ray object (avoids 20k allocations per scan) + + constructor(lcm: LCM, rapierWorld: any, RAPIER: any, sentSeqs: Set, embodiment?: LidarEmbodimentConfig) { + this.lcm = lcm; + this.world = rapierWorld; + this.RAPIER = RAPIER; + this.sentSeqs = sentSeqs; + this.ray = new RAPIER.Ray({ x: 0, y: 0, z: 0 }, { x: 0, y: 0, z: 1 }); + + const halfH = embodiment?.halfHeight ?? DEFAULT_HALF_HEIGHT; + const radius = embodiment?.radius ?? DEFAULT_RADIUS; + const mount = embodiment?.lidarMountHeight ?? DEFAULT_LIDAR_MOUNT; + this.lidarYOffset = mount - (halfH + radius); + + // Step once with zero dt to initialize the query pipeline after snapshot restore. + // queryPipeline.update() crashes on restored worlds (WASM type mismatch), + // but world.step() internally updates the pipeline correctly. + this.world.step(); + } + + /** Reconfigure lidar mount offset after embodiment change. */ + reconfigure(embodiment: LidarEmbodimentConfig): void { + const halfH = embodiment.halfHeight ?? DEFAULT_HALF_HEIGHT; + const radius = embodiment.radius ?? DEFAULT_RADIUS; + const mount = embodiment.lidarMountHeight ?? DEFAULT_LIDAR_MOUNT; + this.lidarYOffset = mount - (halfH + radius); + console.log(`[lidar] reconfigured: lidarYOffset=${this.lidarYOffset.toFixed(3)}`); + } + + /** Set rigid body to exclude from raycasting (agent's own capsule). */ + setExcludeBody(body: any): void { + this.excludeBody = body; + } + + /** Update robot pose. Position is capsule center (odom); we apply lidar mount offset internally. */ + updatePose(x: number, y: number, z: number, qx: number, qy: number, qz: number, qw: number): void { + this.px = x; + this.py = y + this.lidarYOffset; // capsule center → lidar mount height + this.pz = z; + this.qx = qx; + this.qy = qy; + this.qz = qz; + this.qw = qw; + this.hasPose = true; + } + + start(): void { + if (this.timer) return; + // quiet + this.timer = setInterval(() => this._scan(), RATE_MS); + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + private _scan(): void { + if (!this.hasPose || this.publishing) return; + this.publishing = true; + this._doScan().catch((e) => { + console.warn("[lidar] publish error (dropped frame):", e?.message || e); + }).finally(() => { this.publishing = false; }); + } + + private async _doScan(): Promise { + this.scanCount++; + const jitterAngle = this.scanCount * 2.399963; // golden angle per scan + const cosJ = Math.cos(jitterAngle); + const sinJ = Math.sin(jitterAngle); + + const RAPIER = this.RAPIER; + const world = this.world; + + // Pre-allocate output buffers + const worldPts = new Float32Array(NUM_POINTS * 3); + const intensity = new Float32Array(NUM_POINTS); + let n = 0; + + const ox = this.px, oy = this.py, oz = this.pz; + const rqx = this.qx, rqy = this.qy, rqz = this.qz, rqw = this.qw; + + for (let i = 0; i < NUM_POINTS; i++) { + // Fibonacci direction (pre-rotated to camera-local) with per-scan golden angle jitter. + // In FLU frame, jitter rotates around Z (up). After lidarToCamQuat, FLU Z → cam Y, + // so jitter must rotate around camera-local Y axis. + const fx = fibDirs[i * 3 + 0], fy = fibDirs[i * 3 + 1], fz = fibDirs[i * 3 + 2]; + const lx = fx * cosJ + fz * sinJ; + const ly = fy; + const lz = -fx * sinJ + fz * cosJ; + + // Rotate local direction by robot quaternion → world direction + const [dx, dy, dz] = rotateByQuat(lx, ly, lz, rqx, rqy, rqz, rqw); + const len = Math.sqrt(dx * dx + dy * dy + dz * dz); + if (len < 1e-8) continue; + const nx = dx / len, ny = dy / len, nz = dz / len; + + // Reuse ray object — avoids 20k allocations per scan + this.ray.origin.x = ox; this.ray.origin.y = oy; this.ray.origin.z = oz; + this.ray.dir.x = nx; this.ray.dir.y = ny; this.ray.dir.z = nz; + // Use world.castRay (not queryPipeline.castRayAndGetNormal) — + // the pipeline API crashes on restored snapshot worlds. + // Exclude agent's own rigid body so lidar doesn't hit its own colliders. + const hit = world.castRay( + this.ray, MAX_RANGE, false, + undefined, undefined, undefined, + this.excludeBody, + ); + + if (!hit) continue; + const toi = hit.timeOfImpact ?? 0; + if (toi < MIN_RANGE || toi > MAX_RANGE) continue; + + worldPts[n * 3 + 0] = ox + nx * toi; + worldPts[n * 3 + 1] = oy + ny * toi; + worldPts[n * 3 + 2] = oz + nz * toi; + intensity[n] = 1.0 / (1.0 + 0.02 * toi * toi); + n++; + } + + if (n === 0) return; + + this.logN++; + // scan logging removed — too noisy + + // Encode PointCloud2: Y-up → Z-up (ROS) cyclic permutation x→y, y→z, z→x + const pointStep = 16; + const buf = new ArrayBuffer(n * pointStep); + const view = new DataView(buf); + + for (let i = 0; i < n; i++) { + const off = i * pointStep; + const tx = worldPts[i * 3 + 0]; + const ty = worldPts[i * 3 + 1]; + const tz = worldPts[i * 3 + 2]; + view.setFloat32(off, tz, true); // ROS x = Three.js z + view.setFloat32(off + 4, tx, true); // ROS y = Three.js x + view.setFloat32(off + 8, ty, true); // ROS z = Three.js y + view.setFloat32(off + 12, intensity[i], true); + } + + const now = Date.now(); + const header = new std_msgs.Header({ + stamp: new std_msgs.Time({ sec: Math.floor(now / 1000), nsec: (now % 1000) * 1_000_000 }), + frame_id: "world", + }); + + const msg = new sensor_msgs.PointCloud2({ + header, + height: 1, + width: n, + fields_length: 4, + fields: [ + new sensor_msgs.PointField({ name: "x", offset: 0, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "y", offset: 4, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "z", offset: 8, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "intensity", offset: 12, datatype: 7, count: 1 }), + ], + is_bigendian: false, + point_step: pointStep, + row_step: n * pointStep, + data_length: n * pointStep, + data: new Uint8Array(buf), + is_dense: true, + }); + + // Mark seq for echo filtering (prevent server re-forwarding to browser WS) + this.sentSeqs.add(this.lcm.getNextSeq()); + // Publish directly to LCM — no WS hop (await so buffer pressure is felt) + await this.lcm.publish(CH_LIDAR, msg); + } +} diff --git a/misc/DimSim/dimos-cli/bridge/physics.ts b/misc/DimSim/dimos-cli/bridge/physics.ts new file mode 100644 index 0000000000..e902de3fc9 --- /dev/null +++ b/misc/DimSim/dimos-cli/bridge/physics.ts @@ -0,0 +1,385 @@ +/** + * Server-side agent physics (Deno/Rapier). + * + * Runs the agent's kinematic character controller at a fixed timestep on the + * server, eliminating the browser from the control loop: + * + * Python cmd_vel → LCM → Deno (physics step) → LCM odom → Python + * ↓ + * WS position → Browser (render only) + * + * The browser no longer integrates cmd_vel or steps physics — it just receives + * position updates and moves the visual avatar. + */ + +import { geometry_msgs, std_msgs } from "@dimos/msgs"; + +import type { LCM } from "../vendor/lcm/lcm.ts"; + +// -- Agent dimensions (must match AiAvatar.js / engine.js) -------------------- +const DEFAULT_AGENT_RADIUS = 0.12; +const DEFAULT_AGENT_HALF_HEIGHT = 0.25; +const CONTROLLER_OFFSET = 0.05; + +// -- Physics constants -------------------------------------------------------- +const PHYSICS_HZ = 50; +const PHYSICS_DT = 1.0 / PHYSICS_HZ; +const DEFAULT_GRAVITY_Y = -9.81; +const DEFAULT_SPEED_SCALE = 3.0; // Multiplier for cmd_vel (linear + angular) +const DEFAULT_TURN_SCALE = 3.0; +const DEFAULT_MAX_ALTITUDE = 50; + +/** Embodiment configuration passed from SceneClient / control channel. */ +export interface EmbodimentConfig { + radius?: number; + halfHeight?: number; + lidarMountHeight?: number; + embodimentType?: string; // "ground" | "drone" + maxSpeed?: number; + turnRate?: number; + gravity?: number; + maxStepHeight?: number; + groundSnapDist?: number; + maxSlopeAngle?: number; + friction?: number; + maxAltitude?: number; +} + +const CH_ODOM = "/odom#geometry_msgs.PoseStamped"; +const CH_CMD_VEL = "/cmd_vel#geometry_msgs.Twist"; +const CMD_VEL_TIMEOUT_MS = 500; + +// -- ServerPhysics ------------------------------------------------------------ + +export class ServerPhysics { + private lcm: LCM; + private world: any; // RAPIER.World + private RAPIER: any; + private sentSeqs: Set; + + private body: any; + private collider: any; + private spineCollider: any; + private controller: any; + private timer: ReturnType | null = null; + + // Embodiment params + private embodimentType: string; + private speedScale: number; + private turnScale: number; + private gravity: number; + private maxAltitude: number; + private agentRadius: number; + private agentHalfHeight: number; + private friction: number; + private maxStepHeight: number; + private groundSnapDist: number; + private maxSlopeAngle: number; + + // Agent state + private yaw = 0; + private seq = 0; + + // cmd_vel (ROS frame: x=fwd, z=yaw) + private linX = 0; // forward + private linY = 0; // lateral + private linZ = 0; // vertical + private angZ = 0; // yaw rotation + private cmdVelStamp = 0; + + // Callback to send position to browser + private onPoseUpdate: ((x: number, y: number, z: number, yaw: number) => void) | null = null; + + constructor( + lcm: LCM, + rapierWorld: any, + RAPIER: any, + sentSeqs: Set, + embodiment?: EmbodimentConfig, + ) { + this.lcm = lcm; + this.world = rapierWorld; + this.RAPIER = RAPIER; + this.sentSeqs = sentSeqs; + + // Apply embodiment config with defaults + this.embodimentType = embodiment?.embodimentType ?? "ground"; + this.speedScale = embodiment?.maxSpeed ?? DEFAULT_SPEED_SCALE; + this.turnScale = embodiment?.turnRate ?? DEFAULT_TURN_SCALE; + this.gravity = embodiment?.gravity ?? DEFAULT_GRAVITY_Y; + this.maxAltitude = embodiment?.maxAltitude ?? DEFAULT_MAX_ALTITUDE; + this.agentRadius = embodiment?.radius ?? DEFAULT_AGENT_RADIUS; + this.agentHalfHeight = embodiment?.halfHeight ?? DEFAULT_AGENT_HALF_HEIGHT; + this.friction = embodiment?.friction ?? 0.8; + this.maxStepHeight = embodiment?.maxStepHeight ?? 0.25; + this.groundSnapDist = embodiment?.groundSnapDist ?? 0.5; + this.maxSlopeAngle = embodiment?.maxSlopeAngle ?? 45; + + this._createBodyAndColliders(); + + // Count colliders to verify world integrity + let colliderCount = 0; + this.world.colliders.forEach(() => { colliderCount++; }); + // Quiet init — only log on error or reconfigure + } + + private _createBodyAndColliders(): void { + const RAPIER = this.RAPIER; + + // Create agent body (kinematic position-based, like AiAvatar) + this.body = this.world.createRigidBody( + RAPIER.RigidBodyDesc.kinematicPositionBased().setTranslation(0, 3, 0), + ); + + // Main capsule collider + this.collider = this.world.createCollider( + RAPIER.ColliderDesc.capsule(this.agentHalfHeight, this.agentRadius) + .setFriction(this.friction), + this.body, + ); + + // Spine collider (horizontal, behind body center — matches AiAvatar) + const spineHalfLen = Math.max(this.agentRadius * 1.2, 0.13); + const spineRadius = Math.max(this.agentRadius * 0.62, 0.07); + const spineOffsetBack = Math.max( + this.agentRadius * 2.2, + spineHalfLen + spineRadius + 0.02, + ); + const spineOffsetY = Math.max(this.agentHalfHeight * 0.35, 0.08); + this.spineCollider = this.world.createCollider( + RAPIER.ColliderDesc.capsule(spineHalfLen, spineRadius) + .setFriction(this.friction) + .setTranslation(0, spineOffsetY, -spineOffsetBack) + .setRotation({ + x: Math.SQRT1_2, + y: 0, + z: 0, + w: Math.SQRT1_2, + }), + this.body, + ); + + // Character controller + this.controller = this.world.createCharacterController(CONTROLLER_OFFSET); + this.controller.enableAutostep(this.maxStepHeight, 0.15, true); + this.controller.enableSnapToGround(this.groundSnapDist); + this.controller.setSlideEnabled(true); + this.controller.setMaxSlopeClimbAngle((this.maxSlopeAngle * Math.PI) / 180); + this.controller.setMinSlopeSlideAngle((75 * Math.PI) / 180); + } + + /** Reconfigure physics with new embodiment params (e.g. after set_embodiment). */ + reconfigure(embodiment: EmbodimentConfig): void { + // Save current position and yaw + const pos = this.body.translation(); + const savedYaw = this.yaw; + + // Update params + this.embodimentType = embodiment.embodimentType ?? this.embodimentType; + this.speedScale = embodiment.maxSpeed ?? this.speedScale; + this.turnScale = embodiment.turnRate ?? this.turnScale; + this.gravity = embodiment.gravity ?? this.gravity; + this.maxAltitude = embodiment.maxAltitude ?? this.maxAltitude; + this.agentRadius = embodiment.radius ?? this.agentRadius; + this.agentHalfHeight = embodiment.halfHeight ?? this.agentHalfHeight; + this.friction = embodiment.friction ?? this.friction; + this.maxStepHeight = embodiment.maxStepHeight ?? this.maxStepHeight; + this.groundSnapDist = embodiment.groundSnapDist ?? this.groundSnapDist; + this.maxSlopeAngle = embodiment.maxSlopeAngle ?? this.maxSlopeAngle; + + // Remove old colliders and body + if (this.spineCollider) this.world.removeCollider(this.spineCollider, false); + if (this.collider) this.world.removeCollider(this.collider, false); + if (this.body) this.world.removeRigidBody(this.body); + + // Recreate with new params + this._createBodyAndColliders(); + + // Restore position and yaw + this.body.setNextKinematicTranslation({ x: pos.x, y: pos.y, z: pos.z }); + this.yaw = savedYaw; + this.world.step(); + + console.log(`[physics] reconfigured: type=${this.embodimentType} radius=${this.agentRadius} halfHeight=${this.agentHalfHeight} speed=${this.speedScale} gravity=${this.gravity}`); + } + + /** Set spawn position (Three.js Y-up). */ + setPosition(x: number, y: number, z: number): void { + this.body.setNextKinematicTranslation({ x, y, z }); + this.world.step(); // apply immediately + // quiet + } + + /** Set callback for browser position sync. */ + setOnPoseUpdate( + cb: (x: number, y: number, z: number, yaw: number) => void, + ): void { + this.onPoseUpdate = cb; + } + + /** Handle incoming cmd_vel (ROS frame). */ + handleCmdVel(twist: any): void { + this.linX = twist.linear.x; // forward (ROS +x) + this.linY = twist.linear.y; // lateral + this.linZ = twist.linear.z; // vertical + this.angZ = twist.angular.z; // yaw (ROS +z = rotate left) + this.cmdVelStamp = Date.now(); + } + + /** Subscribe to cmd_vel on LCM. */ + subscribeCmdVel(): void { + this.lcm.subscribe(CH_CMD_VEL, geometry_msgs.Twist, (msg: any) => { + this.handleCmdVel(msg.data); + }); + // quiet + } + + /** Start fixed-rate physics stepping + odom publish. */ + start(): void { + if (this.timer) return; + this.subscribeCmdVel(); + this.timer = setInterval(() => this._step(), 1000 / PHYSICS_HZ); + // quiet + } + + stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = null; + } + } + + /** Get current position in Three.js Y-up frame. */ + getPosition(): { x: number; y: number; z: number } { + return this.body.translation(); + } + + /** Get the agent's rigid body (for lidar exclusion). */ + getBody(): any { + return this.body; + } + + getYaw(): number { + return this.yaw; + } + + private _step(): void { + // Safety timeout — zero velocity if no cmd_vel received recently + const hasVel = Date.now() - this.cmdVelStamp < CMD_VEL_TIMEOUT_MS; + const linX = hasVel ? this.linX * this.speedScale : 0; + const linY = hasVel ? this.linY * this.speedScale : 0; + const linZ = hasVel ? this.linZ * this.speedScale : 0; + const angZ = hasVel ? this.angZ * this.turnScale : 0; + + // Integrate yaw (ROS angZ → Three.js Y rotation) + // ROS +z yaw = CCW from above = Three.js +Y rotation + this.yaw += angZ * PHYSICS_DT; + + const pos = this.body.translation(); + const cosY = Math.cos(this.yaw); + const sinY = Math.sin(this.yaw); + + let newPos: { x: number; y: number; z: number }; + + if (this.embodimentType === "drone") { + // Drone: 6DoF movement, no gravity, altitude clamping + const fwd = linX; + const lat = linY; + const vert = linZ; // ROS z = vertical for drone + const desired = { + x: (fwd * sinY + lat * cosY) * PHYSICS_DT, + y: vert * PHYSICS_DT, + z: (fwd * cosY - lat * sinY) * PHYSICS_DT, + }; + + this.controller.computeColliderMovement( + this.collider, + desired, + this.RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + ); + const m = this.controller.computedMovement(); + newPos = { + x: pos.x + m.x, + y: Math.min(pos.y + m.y, this.maxAltitude), + z: pos.z + m.z, + }; + } else { + // Ground robot: gravity, collision-aware + const fwd = linX; + const desired = { + x: (fwd * sinY) * PHYSICS_DT, + y: this.gravity * PHYSICS_DT * PHYSICS_DT * 0.5, // gravity + z: (fwd * cosY) * PHYSICS_DT, + }; + + this.controller.computeColliderMovement( + this.collider, + desired, + this.RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + ); + const m = this.controller.computedMovement(); + newPos = { + x: pos.x + m.x, + y: pos.y + m.y, + z: pos.z + m.z, + }; + } + + this.body.setNextKinematicTranslation(newPos); + + // Step world to apply kinematic translation (needed for next computeColliderMovement) + this.world.step(); + + // Publish odom to LCM (Three.js Y-up → ROS Z-up) + this._publishOdom(newPos); + + // Debug: log first few steps + // step logging removed — too noisy for dimos subprocess output + + // Notify browser for visual sync + if (this.onPoseUpdate) { + this.onPoseUpdate(newPos.x, newPos.y, newPos.z, this.yaw); + } + } + + private _publishOdom(pos: { x: number; y: number; z: number }): void { + // Three.js Y-up → ROS Z-up: (x,y,z) → (z,x,y) + const rosX = pos.z; + const rosY = pos.x; + const rosZ = pos.y; + + // Yaw quaternion (Three.js Y-axis → ROS Z-axis) + const qw = Math.cos(this.yaw / 2); + const qRosZ = Math.sin(this.yaw / 2); // rotation about ROS Z + + const now = Date.now(); + + const header = new std_msgs.Header({ + seq: this.seq++, + stamp: new std_msgs.Time({ sec: Math.floor(now / 1000), nsec: (now % 1000) * 1_000_000 }), + frame_id: "world", + }); + + const pose = new geometry_msgs.Pose(); + pose.position = new geometry_msgs.Point(); + pose.position.x = rosX; + pose.position.y = rosY; + pose.position.z = rosZ; + pose.orientation = new geometry_msgs.Quaternion(); + pose.orientation.x = 0; + pose.orientation.y = 0; + pose.orientation.z = qRosZ; + pose.orientation.w = qw; + + const odom = new geometry_msgs.PoseStamped(); + odom.header = header; + odom.pose = pose; + + try { + this.sentSeqs.add(this.lcm.getNextSeq()); + this.lcm.publishRaw(CH_ODOM, odom.encode()).catch(() => {}); + } catch (e: unknown) { + if (this.seq <= 3) console.warn("[physics] odom publish error:", e); + } + } +} diff --git a/misc/DimSim/dimos-cli/bridge/server.ts b/misc/DimSim/dimos-cli/bridge/server.ts new file mode 100644 index 0000000000..6824a2ee55 --- /dev/null +++ b/misc/DimSim/dimos-cli/bridge/server.ts @@ -0,0 +1,349 @@ +#!/usr/bin/env -S deno run --allow-net --allow-read --unstable-net + +/** + * DimSim Bridge Server + * + * - One control WebSocket plus multiple sensor WebSockets. + * Separate TCP streams so large sensor data never blocks real-time odom + * or other sensor streams. + * - LCM multicast relay (WS ↔ LCM) + * - Per-channel isolation for multi-page parallel evals + * - Static file server for the pre-built DimSim frontend (dist/) + * - Uses vendored LCM transport with joinMulticastV4 fix + */ + +import { LCM } from "../vendor/lcm/lcm.ts"; +import { decodePacket } from "../vendor/lcm/transport.ts"; +import { MAGIC_SHORT, SHORT_HEADER_SIZE } from "../vendor/lcm/types.ts"; +import { serveDir } from "@std/http/file-server"; +import { ServerLidar } from "./lidar.ts"; +import { ServerPhysics } from "./physics.ts"; +import { geometry_msgs } from "@dimos/msgs"; + +// Magic prefix for Rapier world snapshot (ASCII "DSSN") +const SNAPSHOT_MAGIC = 0x4453534E; +const DEFAULT_LCM_PORT = 7667; +const DEFAULT_LCM_HOST = "239.255.76.67"; + +export interface BridgeServerOptions { + port: number; + distDir: string; + scene?: string; + evalOnly?: boolean; + headless?: boolean; + channels?: string[]; + lcmBasePort?: number; + sensorRates?: Record; + sensorEnable?: Record; + cameraFov?: number; +} + +/** Per-channel state: each channel gets its own LCM, physics, lidar, and WS client sets. */ +interface ChannelState { + name: string; + controlClients: Set; + activeControlClient: WebSocket | null; + sensorClients: Set; + lcm: LCM | null; + sentSeqs: Set; + serverLidar: ServerLidar | null; + serverPhysics: ServerPhysics | null; + embodiment: Record | null; +} + +export async function startBridgeServer(options: BridgeServerOptions) { + const { + port, distDir, scene, + evalOnly = false, headless = false, + channels, lcmBasePort = DEFAULT_LCM_PORT, + sensorRates, sensorEnable, cameraFov, + } = options; + + // Build channel list: if channels provided, use them; otherwise single default + const channelNames = channels && channels.length > 0 + ? channels + : [""]; // empty string = default (backward compat, no channel routing) + + const channelMap = new Map(); + + for (let i = 0; i < channelNames.length; i++) { + const name = channelNames[i]; + const lcmPort = lcmBasePort + i; + const lcmUrl = `udpm://${DEFAULT_LCM_HOST}:${lcmPort}?ttl=0`; + const state: ChannelState = { + name, + controlClients: new Set(), + activeControlClient: null, + sensorClients: new Set(), + lcm: null, + sentSeqs: new Set(), + serverLidar: null, + serverPhysics: null, + embodiment: null, + }; + + if (!evalOnly) { + state.lcm = new LCM(lcmUrl); + await state.lcm.start(); + console.log(`[bridge] channel "${name || "default"}" LCM on ${lcmUrl}`); + + // LCM → WS: forward external packets to this channel's active control client + state.lcm.subscribePacket((packet: Uint8Array) => { + if (packet.length < 8) return; + const view = new DataView(packet.buffer, packet.byteOffset, packet.byteLength); + const magic = view.getUint32(0, false); + if (magic !== MAGIC_SHORT) return; + + const seq = view.getUint32(4, false); + if (state.sentSeqs.has(seq)) { + state.sentSeqs.delete(seq); + return; + } + if (state.sentSeqs.size > 1000) state.sentSeqs.clear(); + + const copy = packet.slice(); + const client = state.activeControlClient; + if (client && client.readyState === WebSocket.OPEN) client.send(copy); + }); + } + + channelMap.set(name, state); + } + + /** Resolve channel from WS query param. Falls back to default ("") if not found. */ + function resolveChannel(channelParam: string | null): ChannelState { + if (channelParam && channelMap.has(channelParam)) { + return channelMap.get(channelParam)!; + } + // If no channel param or unknown, use default (first channel) + return channelMap.values().next().value!; + } + + // -- Server-side init from Rapier snapshot ---------------------------------- + async function initServerSystems( + chState: ChannelState, + snapshot: Uint8Array, + spawnPos?: { x: number; y: number; z: number }, + ): Promise { + if (chState.serverLidar) { chState.serverLidar.stop(); chState.serverLidar = null; } + if (chState.serverPhysics) { chState.serverPhysics.stop(); chState.serverPhysics = null; } + if (!chState.lcm) return; + + try { + const RAPIER = await import("@dimforge/rapier3d-compat"); + await RAPIER.init(); + const world = RAPIER.World.restoreSnapshot(snapshot); + if (!world) { console.error(`[bridge:${chState.name || "default"}] failed to restore Rapier snapshot`); return; } + + const bodiesToRemove: any[] = []; + world.bodies.forEach((body: any) => { + if (!body.isFixed()) bodiesToRemove.push(body.handle); + }); + for (const handle of bodiesToRemove) { + world.removeRigidBody(world.getRigidBody(handle)); + } + console.log(`[bridge:${chState.name || "default"}] Rapier snapshot restored (removed ${bodiesToRemove.length} non-fixed bodies)`); + + chState.serverPhysics = new ServerPhysics(chState.lcm, world, RAPIER, chState.sentSeqs, chState.embodiment ?? undefined); + if (spawnPos) { + chState.serverPhysics.setPosition(spawnPos.x, spawnPos.y, spawnPos.z); + } + + chState.serverLidar = new ServerLidar(chState.lcm, world, RAPIER, chState.sentSeqs, chState.embodiment ?? undefined); + chState.serverLidar.setExcludeBody(chState.serverPhysics.getBody()); + + chState.serverPhysics.setOnPoseUpdate((x, y, z, yaw) => { + const qw = Math.cos(yaw / 2); + const qy = Math.sin(yaw / 2); + chState.serverLidar!.updatePose(x, y, z, 0, qy, 0, qw); + + const msg = JSON.stringify({ type: "pose", x, y, z, yaw }); + const client = chState.activeControlClient; + if (client && client.readyState === WebSocket.OPEN) { + try { client.send(msg); } catch { /* ignore */ } + } + }); + + chState.serverPhysics.start(); + chState.serverLidar.start(); + } catch (e) { + console.error(`[bridge:${chState.name || "default"}] server systems init error:`, e); + } + } + + // ── HTTP + WebSocket server ───────────────────────────────────────────── + Deno.serve({ port }, async (req: Request) => { + const url = new URL(req.url); + + if (req.headers.get("upgrade") === "websocket") { + const { socket, response } = Deno.upgradeWebSocket(req); + socket.binaryType = "arraybuffer"; + const ch = url.searchParams.get("ch") || "control"; + const channelParam = url.searchParams.get("channel"); + const chState = resolveChannel(channelParam); + const isSensor = ch !== "control"; + const logPrefix = `[bridge:${chState.name || "default"}]`; + + if (isSensor) { + // ── SENSOR WebSocket ────────────────────────────────────────── + socket.onopen = () => { chState.sensorClients.add(socket); }; + socket.onclose = () => { chState.sensorClients.delete(socket); }; + socket.onerror = () => chState.sensorClients.delete(socket); + + let _sensorLogN = 0; + socket.onmessage = (event: MessageEvent) => { + if (!(event.data instanceof ArrayBuffer) || !chState.lcm) return; + const packet = new Uint8Array(event.data); + + // Check for Rapier snapshot + if (packet.length > 4) { + const dv = new DataView(packet.buffer, packet.byteOffset); + const magic = dv.getUint32(0, false); + + if (magic === 0x44535332) { // "DSS2" + const sx = dv.getFloat32(4, true); + const sy = dv.getFloat32(8, true); + const sz = dv.getFloat32(12, true); + const snapshot = packet.slice(16); + const spawnPos = { x: sx, y: sy, z: sz }; + console.log(`${logPrefix} Rapier snapshot received (${(snapshot.byteLength / 1024).toFixed(0)}KB) spawn=(${sx.toFixed(1)},${sy.toFixed(1)},${sz.toFixed(1)})`); + initServerSystems(chState, snapshot, spawnPos); + return; + } + + if (magic === SNAPSHOT_MAGIC) { // "DSSN" + const snapshot = packet.slice(4); + console.log(`${logPrefix} Rapier snapshot received (${(snapshot.byteLength / 1024).toFixed(0)}KB) [legacy, no spawn]`); + initServerSystems(chState, snapshot); + return; + } + } + + try { + const decoded = decodePacket(packet); + if (decoded && decoded.type === "small") { + _sensorLogN++; + if (_sensorLogN === 1 || _sensorLogN % 1000 === 0) { + const chName = decoded.channel.split("#")[0].replace("/", ""); + // quiet — sensor relay logging removed + } + chState.sentSeqs.add(chState.lcm.getNextSeq()); + chState.lcm.publishRaw(decoded.channel, decoded.data).catch(() => {}); + } + } catch { /* ignore */ } + }; + } else { + // ── CONTROL WebSocket ───────────────────────────────────────── + socket.onopen = () => { + if (!chState.activeControlClient || chState.activeControlClient.readyState !== WebSocket.OPEN) { + chState.activeControlClient = socket; + } + chState.controlClients.add(socket); + // quiet + }; + socket.onerror = () => chState.controlClients.delete(socket); + + let _odomLogN = 0; + + socket.onclose = () => { + chState.controlClients.delete(socket); + if (chState.activeControlClient === socket) chState.activeControlClient = null; + // quiet + }; + + socket.onmessage = (event: MessageEvent) => { + // Text messages: handle special types, relay the rest + if (typeof event.data === "string") { + try { + const msg = JSON.parse(event.data); + + // -- Embodiment config: store & reconfigure running systems -- + if (msg.type === "embodimentConfig") { + chState.embodiment = msg.config ?? msg; + console.log(`${logPrefix} embodiment config stored:`, JSON.stringify(chState.embodiment)); + if (chState.serverPhysics) chState.serverPhysics.reconfigure(chState.embodiment as any); + if (chState.serverLidar) chState.serverLidar.reconfigure(chState.embodiment as any); + // fall through to relay to browser + } + + // -- Teleport: reposition physics agent, don't relay -- + if (msg.type === "teleport") { + if (chState.serverPhysics && msg.x != null && msg.y != null && msg.z != null) { + chState.serverPhysics.setPosition(msg.x, msg.y, msg.z); + console.log(`${logPrefix} teleport to (${msg.x},${msg.y},${msg.z})`); + } + return; // don't relay teleport commands + } + + // -- Physics collider add/remove: forward to Rapier world -- + if (msg.type === "physicsColliderAdd" || msg.type === "physicsColliderRemove") { + // These are handled by the browser's physics; just relay + } + } catch { /* not JSON, relay as-is */ } + + for (const client of chState.controlClients) { + if (client !== socket && client.readyState === WebSocket.OPEN) { + try { client.send(event.data); } catch { /* ignore */ } + } + } + return; + } + if (!(event.data instanceof ArrayBuffer) || !chState.lcm) return; + if (chState.activeControlClient !== socket) return; + const packet = new Uint8Array(event.data); + try { + const decoded = decodePacket(packet); + if (decoded && decoded.type === "small") { + _odomLogN++; + + if (chState.serverPhysics && decoded.channel === "/odom#geometry_msgs.PoseStamped") { + return; + } + + chState.sentSeqs.add(chState.lcm.getNextSeq()); + chState.lcm.publishRaw(decoded.channel, decoded.data).catch(() => {}); + } + } catch { /* ignore */ } + }; + } + + return response; + } + + if (url.pathname === "/" || url.pathname === "/index.html") { + try { + let html = await Deno.readTextFile(`${distDir}/index.html`); + const ratesJs = sensorRates ? `window.__dimosSensorRates=${JSON.stringify(sensorRates)};` : ""; + const enableJs = sensorEnable ? `window.__dimosSensorEnable=${JSON.stringify(sensorEnable)};` : ""; + const fovJs = cameraFov ? `window.__dimosCameraFov=${cameraFov};` : ""; + const inject = ``; + html = html.replace("", `${inject}\n`); + return new Response(html, { headers: { "content-type": "text/html; charset=utf-8" } }); + } catch { + return new Response("index.html not found", { status: 404 }); + } + } + + return serveDir(req, { fsRoot: distDir, quiet: true }); + }); + + const channelInfo = channelNames.length > 1 + ? ` (${channelNames.length} channels: ${channelNames.join(", ")})` + : ""; + console.log(`[bridge] :${port}${evalOnly ? " (eval-only)" : " (LCM bridge)"}${channelInfo}`); + + // Run all LCM instances + const lcmInstances = [...channelMap.values()].map(s => s.lcm).filter(Boolean) as LCM[]; + if (lcmInstances.length > 0) { + await Promise.all(lcmInstances.map(l => l.run())); + } else { + await new Promise(() => {}); + } +} + +if (import.meta.main) { + const distDir = new URL("../../dist", import.meta.url).pathname; + const scene = Deno.args.find((_a: string, i: number, arr: string[]) => arr[i - 1] === "--scene") || "apt"; + const port = parseInt(Deno.args.find((_a: string, i: number, arr: string[]) => arr[i - 1] === "--port") || "8090"); + await startBridgeServer({ port, distDir, scene }); +} diff --git a/misc/DimSim/dimos-cli/cli.ts b/misc/DimSim/dimos-cli/cli.ts new file mode 100644 index 0000000000..af712b218e --- /dev/null +++ b/misc/DimSim/dimos-cli/cli.ts @@ -0,0 +1,873 @@ +#!/usr/bin/env -S deno run --allow-all --unstable-net + +/** + * DimSim CLI — 3D simulation, eval runner, dev server, and scene manager. + * + * Usage: + * dimsim setup Download core assets + * dimsim scene install Download a scene + * dimsim scene list List scenes + * dimsim scene remove Remove a scene + * dimsim dev [--scene ] [--port ] Dev server + browser + * dimsim eval create Interactive eval wizard + * dimsim eval [--headless] [--parallel N] [--render gpu] Headless CI evals + * dimsim agent [--nav-only] dimos Python agent + */ + +import { resolve, dirname, fromFileUrl } from "@std/path"; +import { startBridgeServer } from "./bridge/server.ts"; +import { launchHeadless, launchMultiPage, type RenderMode } from "./headless/launcher.ts"; +import { runEvals, runEvalsMultiPage, collectWorkflows, toJunitXml, type EvalResult } from "./eval/runner.ts"; +import { getDimsimHome, getDistDir, setup, sceneInstall, sceneList, sceneRemove } from "./setup.ts"; +import { loadSceneIndex, findObject, suggestObjects } from "./eval/scene-index.ts"; +import { buildEval } from "./eval/builder.ts"; + +// Detect compiled binary: Deno.execPath() won't contain "deno" when compiled. +// When compiled or installed from JSR, local source paths don't exist. +const IS_COMPILED = !Deno.execPath().toLowerCase().includes("deno"); +const IS_REMOTE = IS_COMPILED || !import.meta.url.startsWith("file:"); + +const CLI_DIR = IS_REMOTE ? null : dirname(fromFileUrl(import.meta.url)); +const PROJECT_DIR = CLI_DIR ? resolve(CLI_DIR, "..") : null; +const LOCAL_DIST_DIR = PROJECT_DIR ? resolve(PROJECT_DIR, "dist") : null; +const EVALS_DIR = PROJECT_DIR ? resolve(PROJECT_DIR, "evals") : `${getDimsimHome()}/evals`; +const DIMOS_VENV = PROJECT_DIR ? resolve(PROJECT_DIR, "../dimos/.venv/bin/python") : null; +const AGENT_PY = CLI_DIR ? resolve(CLI_DIR, "agent.py") : null; + +/** + * Build dist/ from the repo's own sources using Deno's npm compat. + * Everything needed is already in-tree: src/ (engine), public/ (scenes, + * agent-model, logo). Vite bundles src/ and copies public/ verbatim, so + * no assets need to be downloaded from GitHub releases, and npm/Node are + * not required — Deno runs Vite directly. + * + * --no-lock keeps the repo's deno.lock (which tracks only JSR deps for + * the CLI) from being polluted with the frontend's npm dep graph. + */ +async function tryBuildFromSource( + projectDir: string, + distDir: string, +): Promise { + let viteSpec = "npm:vite@^5"; + try { + const pkg = JSON.parse(await Deno.readTextFile(`${projectDir}/package.json`)); + const v = pkg.devDependencies?.vite ?? pkg.dependencies?.vite; + if (!v) return false; + viteSpec = `npm:vite@${v}`; + } catch { + return false; + } + + // node_modules/ — install from package.json if missing. Vite resolves + // bare imports (three, rapier, etc.) via node_modules at build time. + try { + await Deno.stat(`${projectDir}/node_modules`); + } catch { + console.log(`[dimsim] node_modules/ not found — running 'deno install' (one-time)...`); + const install = new Deno.Command(Deno.execPath(), { + args: ["install", "--no-lock"], cwd: projectDir, + stdout: "inherit", stderr: "inherit", + }).spawn(); + const s = await install.status; + if (!s.success) { + console.error(`[dimsim] deno install failed (exit ${s.code}).`); + return false; + } + } + + console.log(`[dimsim] Building frontend with Vite...`); + const build = new Deno.Command(Deno.execPath(), { + args: ["run", "-A", "--no-lock", viteSpec, "build"], + cwd: projectDir, + stdout: "inherit", stderr: "inherit", + }).spawn(); + const bs = await build.status; + if (!bs.success) { + console.error(`[dimsim] vite build failed (exit ${bs.code}).`); + return false; + } + + try { + await Deno.stat(`${distDir}/index.html`); + return true; + } catch { + console.error(`[dimsim] Build completed but ${distDir}/index.html is missing.`); + return false; + } +} + +/** Resolve distDir: use local dist/ if it exists (dev), else ~/.dimsim/dist/ (installed). */ +async function resolveDistDir(): Promise { + // Check local dist/ (only in dev mode, running from source) + if (LOCAL_DIST_DIR && PROJECT_DIR) { + try { + await Deno.stat(`${LOCAL_DIST_DIR}/index.html`); + return LOCAL_DIST_DIR; + } catch { /* not found — try to build from repo sources */ } + + if (await tryBuildFromSource(PROJECT_DIR, LOCAL_DIST_DIR)) { + return LOCAL_DIST_DIR; + } + } + + const installed = getDistDir(); + try { + await Deno.stat(`${installed}/index.html`); + return installed; + } catch { /* not found */ } + + console.error(`[dimsim] No dist/ found.`); + console.error(`[dimsim] Run 'dimsim setup' to download core assets.`); + Deno.exit(1); + throw new Error("unreachable"); // for TS flow analysis — Deno.exit is never +} + +function printUsage() { + console.log(` +DimSim CLI — 3D simulation + eval harness for dimos + +Commands: + dimsim setup Download core assets (~40MB) + dimsim scene install Download a scene + dimsim scene list List available + installed scenes + dimsim scene remove Remove a local scene + dimsim dev [options] Dev server (open browser, optional eval) + dimsim eval list List installed eval workflows + dimsim eval create Interactive eval builder wizard + dimsim eval [options] Run eval workflows (headless CI) + dimsim list objects [options] List scene objects (eval targets) + dimsim build eval [options] Generate eval from validated target + dimsim agent [options] Launch dimos Python agent + +Setup: + --local Use local archive instead of downloading + +Dev: + --scene Scene to load (default: apt) + --port Server port (default: 8090) + --headless Launch headless browser (no GUI) + --render gpu|cpu Render mode for headless (default: gpu) + --channels Number of parallel browser pages (multi-instance) + --eval Run eval after browser connects + --env Environment filter + --image-rate Image publish interval in ms (default: 500 = 2 Hz) + --lidar-rate LiDAR publish interval in ms (default: 200 = 5 Hz) + --odom-rate Odom publish interval in ms (default: 20 = 50 Hz) + --no-depth Disable depth image publishing + --camera-fov Camera FOV in degrees (default: 80) + +Eval: + --connect Connect to existing bridge (use with dimos) + --headless Headless Chromium (required for CI) + --parallel N parallel browser pages (default: 1) + --render gpu|cpu gpu = Metal/ANGLE, cpu = SwiftShader (default: cpu) + --env Filter to environment + --workflow Filter to workflow + --output json|junit Output format (default: json) + --port Bridge port (default: 8090) + --timeout Engine init timeout (default: auto) + +List Objects: + --scene Scene to inspect (required) + --search Filter objects by name + +Build Eval: + --scene Scene name (required) + --target Target object name (required, validated) + --threshold Distance threshold (default: 2.0) + --timeout Timeout in seconds (default: 60) + --task Agent prompt (default: auto from target) + --name Eval name (default: slugified target) + --env Manifest environment (default: scene name) + +Agent: + --nav-only Nav stack only (no LLM agent) + --venv Python venv path (default: ../dimos/.venv/bin/python) + +Environment: + DIMSIM_HOME Override data dir (default: ~/.dimsim) +`); +} + +function parseArgs(args: string[]) { + const opts: Record = {}; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg.startsWith("--")) { + const key = arg.slice(2); + const next = args[i + 1]; + if (next && !next.startsWith("--")) { + opts[key] = next; + i++; + } else { + opts[key] = true; + } + } + } + return opts; +} + +async function main() { + const subcommand = Deno.args[0]; + const opts = parseArgs(Deno.args.slice(1)); + + if (!subcommand || subcommand === "help" || subcommand === "--help") { + printUsage(); + Deno.exit(0); + } + + if (subcommand === "--version" || subcommand === "version") { + if (IS_COMPILED) { + // Version is read from the embedded deno.json at compile time + try { + const text = await Deno.readTextFile(new URL("./deno.json", import.meta.url)); + console.log(JSON.parse(text).version); + } catch { + console.log("0.1.31"); // fallback — updated at release time + } + } else { + const metaUrl = new URL("./deno.json", import.meta.url); + try { + const resp = await fetch(metaUrl); + const meta = await resp.json(); + console.log(meta.version); + } catch { + console.log("unknown"); + } + } + Deno.exit(0); + } + + const port = parseInt(opts.port as string) || 8090; + + // ── Setup ─────────────────────────────────────────────────────────── + if (subcommand === "setup") { + const local = opts.local; + if (local === true) { + console.error("[dimsim] --local requires a path: dimsim setup --local ./dimsim-core-v0.1.0.tar.gz"); + Deno.exit(1); + } + await setup(local as string | undefined); + Deno.exit(0); + } + + // ── Scene management ──────────────────────────────────────────────── + if (subcommand === "scene") { + const action = Deno.args[1]; + const name = Deno.args[2]; + const sceneOpts = parseArgs(Deno.args.slice(2)); + + if (action === "install" && name) { + const local = sceneOpts.local; + if (local === true) { + console.error("[dimsim] --local requires a path: dimsim scene install apt --local ./scene-apt-v0.1.0.tar.gz"); + Deno.exit(1); + } + await sceneInstall(name, local as string | undefined); + } else if (action === "list") { + await sceneList(); + } else if (action === "remove" && name) { + await sceneRemove(name); + } else { + console.log("Usage:"); + console.log(" dimsim scene install [--local ]"); + console.log(" dimsim scene list"); + console.log(" dimsim scene remove "); + } + Deno.exit(0); + } + + // ── List objects ──────────────────────────────────────────────────── + if (subcommand === "list") { + const what = Deno.args[1]; + if (what === "objects") { + const listOpts = parseArgs(Deno.args.slice(2)); + const sceneName = listOpts.scene as string; + if (!sceneName) { + console.error("[dimsim] --scene is required. Example: dimsim list objects --scene apt"); + Deno.exit(1); + } + + const distDir = await resolveDistDir(); + const scenePath = `${distDir}/sims/${sceneName}.json`; + try { + await Deno.stat(scenePath); + } catch { + console.error(`[dimsim] Scene "${sceneName}" not found at ${scenePath}`); + console.error(`[dimsim] Run 'dimsim scene install ${sceneName}' first.`); + Deno.exit(1); + } + + const index = loadSceneIndex(scenePath, sceneName); + const search = listOpts.search as string | undefined; + + let filtered = index.objects; + if (search) { + const lower = search.toLowerCase(); + filtered = index.objects.filter( + (o) => o.title.toLowerCase().includes(lower) || o.id.toLowerCase().includes(lower), + ); + console.log(`\nObjects matching "${search}" in scene "${sceneName}" (${filtered.length}):\n`); + } else { + console.log(`\nObjects in scene "${sceneName}" (${filtered.length} titled assets):\n`); + } + + if (filtered.length === 0) { + console.log(" (none)"); + } else { + const maxTitle = Math.min(45, Math.max(...filtered.map((o) => o.title.length))); + for (const obj of filtered) { + const t = obj.title.padEnd(maxTitle); + console.log(` ${t} (${obj.position.x}, ${obj.position.y}, ${obj.position.z})`); + } + } + console.log(); + Deno.exit(0); + } + console.log("Usage: dimsim list objects --scene [--search ]"); + Deno.exit(1); + } + + // ── Build eval ───────────────────────────────────────────────────── + if (subcommand === "build") { + const what = Deno.args[1]; + if (what === "eval") { + const buildOpts = parseArgs(Deno.args.slice(2)); + const sceneName = buildOpts.scene as string; + const target = buildOpts.target as string; + + if (!sceneName || !target) { + console.error("[dimsim] --scene and --target are required."); + console.error("Example: dimsim build eval --scene apt --target television"); + Deno.exit(1); + } + + const distDir = await resolveDistDir(); + const scenePath = `${distDir}/sims/${sceneName}.json`; + try { + await Deno.stat(scenePath); + } catch { + console.error(`[dimsim] Scene "${sceneName}" not found at ${scenePath}`); + console.error(`[dimsim] Run 'dimsim scene install ${sceneName}' first.`); + Deno.exit(1); + } + + try { + const result = buildEval({ + scenePath, + sceneName, + target, + threshold: buildOpts.threshold ? parseFloat(buildOpts.threshold as string) : undefined, + timeout: buildOpts.timeout ? parseInt(buildOpts.timeout as string) : undefined, + task: buildOpts.task as string | undefined, + name: buildOpts.name as string | undefined, + env: buildOpts.env as string | undefined, + evalsDir: EVALS_DIR, + }); + + console.log(`\nCreated eval: ${result.filePath}`); + console.log(` Task: "${result.task}"`); + console.log(` Target: ${result.targetTitle} (${result.targetPosition.x}, ${result.targetPosition.y}, ${result.targetPosition.z})`); + console.log(` Threshold: ${result.threshold}m`); + console.log(` Timeout: ${result.timeout}s`); + console.log(`\nRun: dimsim eval --connect --env ${result.env} --workflow ${result.workflowName}\n`); + } catch (err: any) { + console.error(`[dimsim] ${err.message}`); + Deno.exit(1); + } + Deno.exit(0); + } + console.log("Usage: dimsim build eval --scene --target [options]"); + Deno.exit(1); + } + + // ── Dev ───────────────────────────────────────────────────────────── + if (subcommand === "dev") { + const distDir = await resolveDistDir(); + const scene = (opts.scene as string) || "apt"; + const headless = opts.headless === true; + const render = ((opts.render as string) === "cpu" ? "cpu" : "gpu") as RenderMode; + const numChannels = Math.max(1, parseInt(opts.channels as string) || 1); + const evalWorkflow = opts.eval as string | undefined; + + // Sensor publish rates (ms) — overrides browser defaults + const sensorRates: Record = {}; + if (opts["image-rate"]) sensorRates.images = parseInt(opts["image-rate"] as string); + if (opts["lidar-rate"]) sensorRates.lidar = parseInt(opts["lidar-rate"] as string); + if (opts["odom-rate"]) sensorRates.odom = parseInt(opts["odom-rate"] as string); + + // Sensor enable/disable (depth only — color and lidar are essential) + const sensorEnable: Record = {}; + if (opts["no-depth"] === true) sensorEnable.depth = false; + + // Camera FOV + const cameraFov = opts["camera-fov"] ? parseInt(opts["camera-fov"] as string) : undefined; + + // Build channel list for multi-instance mode + const channels = numChannels > 1 + ? Array.from({ length: numChannels }, (_, i) => `page-${i}`) + : undefined; + + console.log(`[dimsim] Dev mode — scene: ${scene}, port: ${port}${headless ? " (headless)" : ""}${channels ? ` (${numChannels} channels)` : ""}`); + console.log(`[dimsim] Serving from: ${distDir}`); + + // LCM bridge is always active in dev mode (unlike eval --headless which disables it) + startBridgeServer({ + port, distDir, scene, headless, channels, + sensorRates: Object.keys(sensorRates).length > 0 ? sensorRates : undefined, + sensorEnable: Object.keys(sensorEnable).length > 0 ? sensorEnable : undefined, + cameraFov, + }); + + if (headless) { + if (channels) { + // Multi-page mode: open N browser pages in one Chromium instance + console.log(`[dimsim] Launching headless browser with ${numChannels} pages...`); + const url = `http://localhost:${port}`; + await launchMultiPage({ url, numPages: numChannels, render, timeout: 120_000 }); + await new Promise((r) => setTimeout(r, 3000)); + console.log(`[dimsim] ${numChannels} headless pages ready. LCM bridge active.`); + } else { + console.log("[dimsim] Launching headless browser..."); + const url = `http://localhost:${port}`; + // CPU rendering with SwiftShader is slow — scene + agent init takes + // ~27s on CI Macs. Allow 90s by default; override via env var. + const headlessTimeout = parseInt( + Deno.env.get("DIMSIM_HEADLESS_TIMEOUT") || "90000", + ); + await launchHeadless({ url, timeout: headlessTimeout, render }); + await new Promise((r) => setTimeout(r, 3000)); + console.log("[dimsim] Headless browser ready. LCM bridge active."); + } + } else { + console.log(`[dimsim] Open http://localhost:${port} in your browser`); + } + + if (evalWorkflow) { + console.log(`[dimsim] Eval workflow: ${evalWorkflow}`); + console.log("[dimsim] Waiting for browser to connect and load scene...\n"); + + const wsUrl = `ws://localhost:${port}`; + const manifestPath = resolve(EVALS_DIR, "manifest.json"); + + const results = await runEvals({ + wsUrl, + manifestPath, + filterEnv: opts.env as string, + filterWorkflow: evalWorkflow, + outputFormat: "json", + }); + + const passed = results.filter((r) => r.pass).length; + const failed = results.length - passed; + console.log(`\n[dimsim] Eval done: ${passed} passed, ${failed} failed`); + + // Stay alive in dev mode (don't exit like headless eval does) + console.log("[dimsim] Eval complete. Server still running. Press Ctrl+C to stop."); + } else { + console.log("[dimsim] Press Ctrl+C to stop."); + } + + // Keep alive + await new Promise(() => {}); + } + + // ── Agent ─────────────────────────────────────────────────────────── + if (subcommand === "agent") { + if (IS_REMOTE && !opts.venv) { + console.error(`[dimsim] Agent mode requires a local dimos install.`); + console.error(`[dimsim] Pass --venv /path/to/python`); + Deno.exit(1); + } + const pythonBin = (opts.venv as string) || DIMOS_VENV!; + const navOnly = opts["nav-only"] === true; + + if (IS_REMOTE && !AGENT_PY) { + console.error(`[dimsim] Agent mode is only available when running from source.`); + Deno.exit(1); + } + + // Verify python exists + try { + await Deno.stat(pythonBin); + } catch { + console.error(`[dimsim] dimos venv not found at: ${pythonBin}`); + console.error(`[dimsim] Install dimos first, or pass --venv /path/to/python`); + Deno.exit(1); + } + + const cmd = [pythonBin, AGENT_PY!]; + if (navOnly) cmd.push("--nav-only"); + + console.log(`[dimsim] Starting dimos agent${navOnly ? " (nav-only)" : ""}...`); + console.log(`[dimsim] Python: ${pythonBin}`); + + const proc = new Deno.Command(cmd[0], { + args: cmd.slice(1), + stdin: "inherit", + stdout: "inherit", + stderr: "inherit", + env: { ...Deno.env.toObject() }, + }).spawn(); + + const status = await proc.status; + Deno.exit(status.code); + } + + // ── Eval list ─────────────────────────────────────────────────────── + if (subcommand === "eval" && Deno.args[1] === "list") { + const evalsDir = EVALS_DIR; + const envs: Map = new Map(); + + try { + for await (const entry of Deno.readDir(evalsDir)) { + if (!entry.isDirectory) continue; + const workflows: string[] = []; + for await (const file of Deno.readDir(`${evalsDir}/${entry.name}`)) { + if (file.name.endsWith(".json") && file.name !== "manifest.json") { + workflows.push(file.name.replace(".json", "")); + } + } + if (workflows.length > 0) { + workflows.sort(); + envs.set(entry.name, workflows); + } + } + } catch { + console.log("\nNo evals installed. Run 'dimsim setup' or 'dimsim eval create' first.\n"); + Deno.exit(0); + } + + if (envs.size === 0) { + console.log("\nNo eval workflows found.\n"); + Deno.exit(0); + } + + const sorted = [...envs.entries()].sort((a, b) => a[0].localeCompare(b[0])); + let total = 0; + console.log(""); + for (const [env, workflows] of sorted) { + console.log(` \x1b[1m${env}\x1b[0m \x1b[2m(${workflows.length})\x1b[0m`); + for (const w of workflows) { + console.log(` ${w}`); + total++; + } + } + console.log(`\n \x1b[2m${total} workflow(s) across ${envs.size} environment(s)\x1b[0m\n`); + Deno.exit(0); + } + + // ── Eval create (interactive wizard) ───────────────────────────────── + if (subcommand === "eval" && Deno.args[1] === "create") { + // ANSI helpers + const c = { + bold: (s: string) => `\x1b[1m${s}\x1b[0m`, + cyan: (s: string) => `\x1b[36m${s}\x1b[0m`, + green: (s: string) => `\x1b[32m${s}\x1b[0m`, + yellow: (s: string) => `\x1b[33m${s}\x1b[0m`, + red: (s: string) => `\x1b[31m${s}\x1b[0m`, + dim: (s: string) => `\x1b[2m${s}\x1b[0m`, + }; + + const distDir = await resolveDistDir(); + const simsDir = `${distDir}/sims`; + + // ── 1. Pick scene ────────────────────────────────────────────────── + const installed: string[] = []; + try { + for await (const entry of Deno.readDir(simsDir)) { + if (entry.name.endsWith(".json") && entry.name !== "manifest.json") { + installed.push(entry.name.replace(".json", "")); + } + } + } catch { /* no sims */ } + installed.sort(); + + if (installed.length === 0) { + console.error(c.red("No scenes installed. Run 'dimsim scene install ' first.")); + Deno.exit(1); + } + + console.log(`\n${c.bold(" Create Eval Workflow")}\n`); + console.log(c.cyan(" Installed scenes:")); + installed.forEach((s, i) => console.log(` ${c.dim(`${i + 1}.`)} ${s}`)); + + let sceneName = ""; + let scenePath = ""; + while (true) { + const input = prompt(`\n ${c.cyan("Scene")} ${c.dim(`[${installed[0]}]`)}:`) || installed[0]; + const resolved = installed.includes(input) + ? input + : installed[parseInt(input) - 1]; + if (resolved) { + sceneName = resolved; + scenePath = `${simsDir}/${sceneName}.json`; + try { + await Deno.stat(scenePath); + console.log(` ${c.green("→")} ${sceneName}`); + break; + } catch { /* fall through */ } + } + console.log(c.yellow(` "${input}" not found. Pick a number or name from the list above.`)); + } + + // ── 2. Pick rubric ───────────────────────────────────────────────── + const rubricChoices = [ + { key: "objectDistance", label: "objectDistance", desc: "agent must reach a target object" }, + { key: "llmJudge", label: "llmJudge", desc: "VLM judges success from screenshots" }, + { key: "groundTruth", label: "groundTruth", desc: "check spatial ground truth conditions" }, + ]; + + console.log(`\n${c.cyan(" Rubric types:")}`); + rubricChoices.forEach((r, i) => + console.log(` ${c.dim(`${i + 1}.`)} ${c.bold(r.label)} ${c.dim(`— ${r.desc}`)}`) + ); + + let rubric = ""; + while (true) { + const input = prompt(`\n ${c.cyan("Rubric")} ${c.dim("[1]")}:`) || "1"; + const byNum = rubricChoices[parseInt(input) - 1]; + const byName = rubricChoices.find((r) => r.key === input); + const match = byNum || byName; + if (match) { + rubric = match.key; + console.log(` ${c.green("→")} ${match.label}`); + break; + } + console.log(c.yellow(` Invalid choice. Enter 1-3 or a rubric name.`)); + } + + // ── 3. Pick target object (objectDistance needs it) ───────────────── + const needsTarget = rubric === "objectDistance"; + const index = loadSceneIndex(scenePath, sceneName); + let target = ""; + let matchedObj: ReturnType = null; + + if (needsTarget) { + console.log(`\n${c.cyan(` Objects in "${sceneName}"`)} ${c.dim(`(${index.objects.length})`)}:`); + const sample = index.objects.slice(0, 20); + for (const obj of sample) { + console.log(` ${obj.title}`); + } + if (index.objects.length > 20) { + console.log(c.dim(` ... and ${index.objects.length - 20} more (dimsim list objects --scene ${sceneName})`)); + } + + while (true) { + const input = prompt(`\n ${c.cyan("Target object")}:`); + if (!input) { + console.log(c.yellow(" Target is required for objectDistance rubric.")); + continue; + } + matchedObj = findObject(input, index); + if (matchedObj) { + target = input; + console.log(` ${c.green("→")} "${matchedObj.title}" at (${matchedObj.position.x}, ${matchedObj.position.y}, ${matchedObj.position.z})`); + break; + } + const suggestions = suggestObjects(input, index); + if (suggestions.length > 0) { + console.log(c.yellow(` No match for "${input}". Similar: ${suggestions.join(", ")}`)); + } else { + console.log(c.yellow(` No match for "${input}". Try 'dimsim list objects --scene ${sceneName}'.`)); + } + } + } + + // ── 4. Task prompt ───────────────────────────────────────────────── + const defaultTask = needsTarget && matchedObj + ? `Go to the ${matchedObj.title}` + : ""; + let task = ""; + while (true) { + const suffix = defaultTask ? ` ${c.dim(`[${defaultTask}]`)}` : ""; + const input = prompt(`\n ${c.cyan("Task prompt")}${suffix}:`) || defaultTask; + if (input) { + task = input; + break; + } + console.log(c.yellow(" Task prompt is required.")); + } + + // ── 5. Rubric-specific config ────────────────────────────────────── + let threshold = 2.0; + let llmPrompt = ""; + + if (rubric === "objectDistance") { + while (true) { + const input = prompt(` ${c.cyan("Distance threshold")} ${c.dim("[2.0m]")}:`) || "2.0"; + const val = parseFloat(input); + if (!isNaN(val) && val > 0) { + threshold = val; + break; + } + console.log(c.yellow(" Enter a positive number (meters).")); + } + } else if (rubric === "llmJudge") { + const defaultJudge = `Did the agent successfully complete: ${task}?`; + llmPrompt = prompt(` ${c.cyan("LLM judge prompt")} ${c.dim(`[${defaultJudge}]`)}:`) || defaultJudge; + } + + // ── 6. Eval name ─────────────────────────────────────────────────── + const slug = (s: string) => s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); + const defaultName = target ? slug(target) : slug(task.slice(0, 40)); + const name = prompt(` ${c.cyan("Eval name")} ${c.dim(`[${defaultName}]`)}:`) || defaultName; + + // ── 7. Timeout ───────────────────────────────────────────────────── + let timeout = 60; + while (true) { + const input = prompt(` ${c.cyan("Timeout")} ${c.dim("[60s]")}:`) || "60"; + const val = parseInt(input); + if (!isNaN(val) && val > 0) { + timeout = val; + break; + } + console.log(c.yellow(" Enter a positive number (seconds).")); + } + + // ── Build & write ────────────────────────────────────────────────── + const successCriteria: Record = {}; + if (rubric === "objectDistance") { + successCriteria.objectDistance = { object: "agent", target, thresholdM: threshold }; + } else if (rubric === "llmJudge") { + successCriteria.llmJudge = { prompt: llmPrompt }; + } else if (rubric === "groundTruth") { + successCriteria.groundTruth = {}; + } + + const env = sceneName; + const workflow = { + name, + environment: env, + task, + startPose: { x: 0, y: 0.5, z: 3, yaw: 0 }, + timeoutSec: timeout, + successCriteria, + }; + + const envDir = `${EVALS_DIR}/${env}`; + try { Deno.mkdirSync(envDir, { recursive: true }); } catch { /* exists */ } + const filePath = `${envDir}/${name}.json`; + Deno.writeTextFileSync(filePath, JSON.stringify(workflow, null, 2) + "\n"); + + console.log(`\n ${c.green("Created:")} ${filePath}`); + console.log(`\n ${c.cyan("Run it:")}`); + console.log(` dimsim eval --connect --env ${env} --workflow ${name}`); + console.log(` dimsim eval --headless --env ${env} --workflow ${name}\n`); + Deno.exit(0); + } + + // ── Eval ──────────────────────────────────────────────────────────── + if (subcommand === "eval") { + const connectMode = opts.connect === true; + const outputFormat = (opts.output as string) === "junit" ? "junit" : "json"; + const manifestPath = resolve(EVALS_DIR, "manifest.json"); + + // --connect mode: just run the eval runner against an existing bridge + if (connectMode) { + const wsUrl = `ws://localhost:${port}`; + console.log(`[dimsim] Connecting to existing bridge at ${wsUrl}...`); + + const results = await runEvals({ + wsUrl, + manifestPath, + filterEnv: opts.env as string, + filterWorkflow: opts.workflow as string, + outputFormat: outputFormat as "json" | "junit", + }); + + const passed = results.filter((r) => r.pass).length; + const failed = results.length - passed; + console.log(`\n[dimsim] Done: ${passed} passed, ${failed} failed, ${results.length} total`); + Deno.exit(failed > 0 ? 1 : 0); + } + + const distDir = await resolveDistDir(); + const headless = opts.headless === true; + const scene = (opts.scene as string) || (opts.env as string) || "apt"; + const parallel = Math.max(1, parseInt(opts.parallel as string) || 1); + const render = ((opts.render as string) === "gpu" ? "gpu" : "cpu") as RenderMode; + const defaultTimeout = render === "cpu" ? 120000 : 30000; + const timeout = parseInt(opts.timeout as string) || defaultTimeout; + + if (headless && parallel > 1) { + const allWorkflows = collectWorkflows( + manifestPath, + opts.env as string, + opts.workflow as string, + ); + + if (allWorkflows.length === 0) { + console.log("[dimsim] No workflows match filter criteria."); + Deno.exit(0); + } + + const numPages = Math.min(parallel, allWorkflows.length); + console.log(`[dimsim] Multi-page eval — ${allWorkflows.length} workflows across ${numPages} page(s)`); + + startBridgeServer({ port, distDir, scene, evalOnly: true }); + await new Promise((r) => setTimeout(r, 500)); + + const url = `http://localhost:${port}`; + const instance = await launchMultiPage({ url, numPages, timeout, render }); + await new Promise((r) => setTimeout(r, 2000)); + + const allResults = await runEvalsMultiPage({ + wsUrl: `ws://localhost:${port}`, + manifestPath, + channels: instance.channels, + filterEnv: opts.env as string, + filterWorkflow: opts.workflow as string, + }); + + await instance.close(); + + if (outputFormat === "junit") { + console.log(toJunitXml(allResults)); + } else { + console.log(JSON.stringify(allResults, null, 2)); + } + + const passed = allResults.filter((r) => r.pass).length; + const failed = allResults.length - passed; + console.log(`\n[dimsim] Done: ${passed} passed, ${failed} failed, ${allResults.length} total`); + Deno.exit(failed > 0 ? 1 : 0); + } + + // -- Single worker eval (sequential) ----------------------------------- + console.log(`[dimsim] Eval mode — headless: ${headless}, port: ${port}`); + + startBridgeServer({ port, distDir, scene, evalOnly: headless }); + await new Promise((r) => setTimeout(r, 500)); + + const url = `http://localhost:${port}`; + + if (headless) { + console.log("[dimsim] Launching headless browser..."); + const instance = await launchHeadless({ url, timeout, render }); + await new Promise((r) => setTimeout(r, 3000)); + + const results = await runEvals({ + wsUrl: `ws://localhost:${port}`, + manifestPath, + filterEnv: opts.env as string, + filterWorkflow: opts.workflow as string, + outputFormat: outputFormat as "json" | "junit", + }); + + await instance.close(); + + const failed = results.filter((r) => !r.pass).length; + Deno.exit(failed > 0 ? 1 : 0); + } else { + console.log(`[dimsim] Open ${url} in your browser to start evals`); + console.log("[dimsim] Press Ctrl+C to stop."); + await new Promise(() => {}); + } + } + + printUsage(); + Deno.exit(1); +} + +main(); diff --git a/misc/DimSim/dimos-cli/deno.json b/misc/DimSim/dimos-cli/deno.json new file mode 100644 index 0000000000..a2c8e98d6f --- /dev/null +++ b/misc/DimSim/dimos-cli/deno.json @@ -0,0 +1,37 @@ +{ + "name": "@antim/dimsim", + "version": "0.2.2", + "description": "3D simulation environment for the dimos robotics stack. Browser-based Three.js + Rapier sim with LCM transport, sensor publishing, and eval harness.", + "license": "MIT", + "exports": { + ".": "./cli.ts", + "./mod": "./mod.ts" + }, + "publish": { + "include": [ + "README.md", + "cli.ts", + "mod.ts", + "bridge/", + "eval/", + "headless/", + "vendor/", + "setup.ts" + ], + "exclude": [ + "test/" + ] + }, + "tasks": { + "start": "deno run --allow-net --allow-read --unstable-net bridge/server.ts", + "eval": "deno run --allow-all cli.ts eval" + }, + "imports": { + "@dimos/lcm": "./vendor/lcm/mod.ts", + "@dimos/msgs": "jsr:@dimos/msgs@^0.1.4", + "@std/path": "jsr:@std/path@^1", + "@std/http": "jsr:@std/http@^1", + "playwright": "npm:playwright@^1.58", + "@dimforge/rapier3d-compat": "npm:@dimforge/rapier3d-compat@^0.14.0" + } +} diff --git a/misc/DimSim/dimos-cli/deno.lock b/misc/DimSim/dimos-cli/deno.lock new file mode 100644 index 0000000000..7cbc0c61ac --- /dev/null +++ b/misc/DimSim/dimos-cli/deno.lock @@ -0,0 +1,117 @@ +{ + "version": "5", + "specifiers": { + "jsr:@antim/dimsim@*": "0.1.27", + "jsr:@dimos/msgs@~0.1.4": "0.1.4", + "jsr:@std/cli@^1.0.27": "1.0.27", + "jsr:@std/encoding@^1.0.10": "1.0.10", + "jsr:@std/fmt@^1.0.9": "1.0.9", + "jsr:@std/fs@^1.0.22": "1.0.22", + "jsr:@std/html@^1.0.5": "1.0.5", + "jsr:@std/http@1": "1.0.24", + "jsr:@std/internal@^1.0.12": "1.0.12", + "jsr:@std/media-types@^1.1.0": "1.1.0", + "jsr:@std/net@^1.0.6": "1.0.6", + "jsr:@std/path@1": "1.1.4", + "jsr:@std/path@^1.1.4": "1.1.4", + "jsr:@std/streams@^1.0.17": "1.0.17", + "npm:@dimforge/rapier3d-compat@0.14": "0.14.0", + "npm:playwright@*": "1.58.2", + "npm:playwright@^1.58.0": "1.58.2" + }, + "jsr": { + "@antim/dimsim@0.1.27": { + "integrity": "fc7f8f3aaeb0e06b06aaf35e157308f656be344db434a10bb573343c43a4bbbb", + "dependencies": [ + "jsr:@dimos/msgs", + "jsr:@std/http", + "jsr:@std/path@1", + "npm:@dimforge/rapier3d-compat", + "npm:playwright@^1.58.0" + ] + }, + "@dimos/msgs@0.1.4": { + "integrity": "564bc30b4bc41a562c296c257a15055283ca0cbd66d0627991ede5295832d0c4" + }, + "@std/cli@1.0.27": { + "integrity": "eba97edd0891871a7410e835dd94b3c260c709cca5983df2689c25a71fbe04de" + }, + "@std/encoding@1.0.10": { + "integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1" + }, + "@std/fmt@1.0.9": { + "integrity": "2487343e8899fb2be5d0e3d35013e54477ada198854e52dd05ed0422eddcabe0" + }, + "@std/fs@1.0.22": { + "integrity": "de0f277a58a867147a8a01bc1b181d0dfa80bfddba8c9cf2bacd6747bcec9308" + }, + "@std/html@1.0.5": { + "integrity": "4e2d693f474cae8c16a920fa5e15a3b72267b94b84667f11a50c6dd1cb18d35e" + }, + "@std/http@1.0.24": { + "integrity": "4dd59afd7cfd6e2e96e175b67a5a829b449ae55f08575721ec691e5d85d886d4", + "dependencies": [ + "jsr:@std/cli", + "jsr:@std/encoding", + "jsr:@std/fmt", + "jsr:@std/fs", + "jsr:@std/html", + "jsr:@std/media-types", + "jsr:@std/net", + "jsr:@std/path@^1.1.4", + "jsr:@std/streams" + ] + }, + "@std/internal@1.0.12": { + "integrity": "972a634fd5bc34b242024402972cd5143eac68d8dffaca5eaa4dba30ce17b027" + }, + "@std/media-types@1.1.0": { + "integrity": "c9d093f0c05c3512932b330e3cc1fe1d627b301db33a4c2c2185c02471d6eaa4" + }, + "@std/net@1.0.6": { + "integrity": "110735f93e95bb9feb95790a8b1d1bf69ec0dc74f3f97a00a76ea5efea25500c" + }, + "@std/path@1.1.4": { + "integrity": "1d2d43f39efb1b42f0b1882a25486647cb851481862dc7313390b2bb044314b5", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/streams@1.0.17": { + "integrity": "7859f3d9deed83cf4b41f19223d4a67661b3d3819e9fc117698f493bf5992140" + } + }, + "npm": { + "@dimforge/rapier3d-compat@0.14.0": { + "integrity": "sha512-/uHrUzS+CRQ+NQrrJCEDUkhwHlNsAAexbNXgbN9sHY+GwR+SFFAFrxRr8Llf5/AJZzqiLANdQIfJ63Cw4gJVqw==" + }, + "fsevents@2.3.2": { + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "os": ["darwin"], + "scripts": true + }, + "playwright-core@1.58.2": { + "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "bin": true + }, + "playwright@1.58.2": { + "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "dependencies": [ + "playwright-core" + ], + "optionalDependencies": [ + "fsevents" + ], + "bin": true + } + }, + "workspace": { + "dependencies": [ + "jsr:@dimos/msgs@~0.1.4", + "jsr:@std/http@1", + "jsr:@std/path@1", + "npm:@dimforge/rapier3d-compat@0.14", + "npm:playwright@^1.58.0" + ] + } +} diff --git a/misc/DimSim/dimos-cli/eval/builder.ts b/misc/DimSim/dimos-cli/eval/builder.ts new file mode 100644 index 0000000000..99888aa528 --- /dev/null +++ b/misc/DimSim/dimos-cli/eval/builder.ts @@ -0,0 +1,123 @@ +/** + * Eval Builder — generate workflow JSON from validated inputs. + * + * An eval = rubric + target + timeout. This module validates the target + * exists in the scene, generates the workflow JSON, and updates the manifest. + */ + +import { loadSceneIndex, findObject, findAllObjects, suggestObjects } from "./scene-index.ts"; + +export interface BuildEvalOptions { + scenePath: string; + sceneName: string; + target: string; + threshold?: number; + timeout?: number; + task?: string; + name?: string; + env?: string; + evalsDir: string; +} + +export interface BuildResult { + filePath: string; + workflowName: string; + task: string; + targetTitle: string; + targetPosition: { x: number; y: number; z: number }; + threshold: number; + timeout: number; + env: string; +} + +function slugify(s: string): string { + return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, ""); +} + +export function buildEval(opts: BuildEvalOptions): BuildResult { + const index = loadSceneIndex(opts.scenePath, opts.sceneName); + + // Validate target exists in scene + const match = findObject(opts.target, index); + if (!match) { + const suggestions = suggestObjects(opts.target, index); + let msg = `No asset matching "${opts.target}" in scene "${opts.sceneName}".`; + if (suggestions.length > 0) { + msg += `\nSimilar: ${suggestions.join(", ")}`; + } + msg += `\nHint: dimsim list objects --scene ${opts.sceneName}`; + throw new Error(msg); + } + + // Warn about duplicates + const allMatches = findAllObjects(opts.target, index); + if (allMatches.length > 1) { + console.warn( + `[build] Warning: "${opts.target}" matches ${allMatches.length} objects. ` + + `Using first: "${match.title}" at (${match.position.x}, ${match.position.y}, ${match.position.z})` + ); + } + + const threshold = opts.threshold ?? 2.0; + const timeout = opts.timeout ?? 60; + const env = opts.env || opts.sceneName; + const workflowName = opts.name || slugify(opts.target); + const task = opts.task || `Go to the ${match.title}`; + + const workflow = { + name: workflowName, + environment: env, + task, + startPose: { x: 0, y: 0.5, z: 3, yaw: 0 }, + timeoutSec: timeout, + successCriteria: { + objectDistance: { + object: "agent", + target: opts.target, + thresholdM: threshold, + }, + }, + }; + + // Write workflow JSON + const envDir = `${opts.evalsDir}/${env}`; + try { Deno.mkdirSync(envDir, { recursive: true }); } catch { /* exists */ } + const filePath = `${envDir}/${workflowName}.json`; + Deno.writeTextFileSync(filePath, JSON.stringify(workflow, null, 2) + "\n"); + + // Update manifest + updateManifest(`${opts.evalsDir}/manifest.json`, env, opts.sceneName, workflowName); + + return { + filePath, + workflowName, + task, + targetTitle: match.title, + targetPosition: match.position, + threshold, + timeout, + env, + }; +} + +function updateManifest(manifestPath: string, env: string, scene: string, workflowName: string): void { + let manifest: { version: string; environments: { name: string; scene: string; workflows: string[] }[] }; + + try { + manifest = JSON.parse(Deno.readTextFileSync(manifestPath)); + } catch { + manifest = { version: "1.0", environments: [] }; + } + + let envEntry = manifest.environments.find((e) => e.name === env); + if (!envEntry) { + envEntry = { name: env, scene, workflows: [] }; + manifest.environments.push(envEntry); + } + + if (!envEntry.workflows.includes(workflowName)) { + envEntry.workflows.push(workflowName); + } + + Deno.writeTextFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n"); +} diff --git a/misc/DimSim/dimos-cli/eval/runner.ts b/misc/DimSim/dimos-cli/eval/runner.ts new file mode 100644 index 0000000000..b7dbd8d39e --- /dev/null +++ b/misc/DimSim/dimos-cli/eval/runner.ts @@ -0,0 +1,389 @@ +/** + * Eval Runner — Deno-side orchestrator that drives eval workflows. + * + * Connects to the bridge server via WebSocket and sends commands to the + * browser's EvalHarness: load environment, start workflow, collect results. + * + * Two modes: + * runEvals() — Sequential: one page, one workflow at a time + * runEvalsMultiPage() — Parallel: N pages in one browser, channel-routed + */ + +export interface EvalResult { + name: string; + environment: string; + reason: string; + durationMs: number; + rubricScores: Record; + pass: boolean; +} + +export interface RunEvalOptions { + wsUrl: string; + manifestPath: string; + filterEnv?: string; + filterWorkflow?: string; + /** Run only these specific workflow names (overrides filterWorkflow). */ + filterWorkflows?: string[]; + outputFormat?: "json" | "junit"; +} + +export interface WorkflowEntry { + env: string; + scene: string; + workflowPath: string; + workflowName: string; +} + +/** Collect workflows from manifest, applying filters. */ +export function collectWorkflows(manifestPath: string, filterEnv?: string, filterWorkflow?: string, filterWorkflows?: string[]): WorkflowEntry[] { + const manifestText = Deno.readTextFileSync(manifestPath); + const manifest = JSON.parse(manifestText); + const result: WorkflowEntry[] = []; + + for (const env of manifest.environments) { + if (filterEnv && env.name !== filterEnv) continue; + for (const wfName of env.workflows) { + if (filterWorkflows) { + if (!filterWorkflows.includes(wfName)) continue; + } else if (filterWorkflow && wfName !== filterWorkflow) { + continue; + } + const dir = new URL(`../../evals/${env.name}/`, import.meta.url).pathname; + result.push({ + env: env.name, + scene: env.scene, + workflowPath: `${dir}${wfName}.json`, + workflowName: wfName, + }); + } + } + return result; +} + +export async function runEvals(options: RunEvalOptions): Promise { + const { wsUrl, manifestPath, filterEnv, filterWorkflow, filterWorkflows, outputFormat } = options; + + const workflowsToRun = collectWorkflows(manifestPath, filterEnv, filterWorkflow, filterWorkflows); + + if (workflowsToRun.length === 0) { + console.log("[runner] No workflows match filter criteria."); + return []; + } + + console.log(`[runner] Running ${workflowsToRun.length} workflow(s)...`); + + // Connect to bridge WebSocket + const ws = new WebSocket(wsUrl); + ws.binaryType = "arraybuffer"; + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("WebSocket connect timeout")), 30000); + ws.onopen = () => { clearTimeout(timeout); resolve(); }; + ws.onerror = (e) => { clearTimeout(timeout); reject(new Error(`WebSocket connection failed: ${e}`)); }; + }); + + console.log("[runner] Connected to bridge"); + + // Helper: send command and wait for response + function sendAndWait(cmd: Record, responseType: string, timeoutMs = 60000): Promise> { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error(`Timeout waiting for ${responseType}`)), timeoutMs); + + const handler = (event: MessageEvent) => { + if (typeof event.data !== "string") return; + try { + const msg = JSON.parse(event.data); + if (msg.type === responseType) { + clearTimeout(timeout); + ws.removeEventListener("message", handler); + resolve(msg); + } + } catch { /* not JSON */ } + }; + + ws.addEventListener("message", handler); + ws.send(JSON.stringify(cmd)); + }); + } + + // Wait for the browser eval harness to be alive (ping/pong handshake) + console.log("[runner] Waiting for browser eval harness..."); + const harnessTimeout = 60000; + const harnessStart = Date.now(); + let harnessReady = false; + while (Date.now() - harnessStart < harnessTimeout) { + try { + await sendAndWait({ type: "ping" }, "pong", 3000); + harnessReady = true; + break; + } catch { + // No response yet — browser not connected or harness not initialized + await new Promise((r) => setTimeout(r, 1000)); + } + } + if (!harnessReady) { + console.error("[runner] Timeout waiting for browser eval harness. Is the browser open?"); + ws.close(); + return []; + } + console.log("[runner] Browser eval harness connected!"); + + const results: EvalResult[] = []; + let currentScene = ""; + + for (const wf of workflowsToRun) { + // Load environment if different from current + if (wf.scene !== currentScene) { + console.log(`[runner] Loading environment: ${wf.env} (scene: ${wf.scene})`); + await sendAndWait({ type: "loadEnv", scene: wf.scene }, "envReady", 120000); + currentScene = wf.scene; + // Wait for physics to settle + await new Promise((r) => setTimeout(r, 2000)); + } + + // Load workflow definition + const wfText = await Deno.readTextFile(wf.workflowPath); + const workflow = JSON.parse(wfText); + + console.log(`[runner] Starting workflow: ${wf.workflowName} — "${workflow.task}"`); + + // Start workflow and wait for completion + const timeoutMs = (workflow.timeoutSec || 120) * 1000 + 30000; // +30s buffer for slow renderers + const result = await sendAndWait( + { type: "startWorkflow", workflow }, + "workflowComplete", + timeoutMs, + ) as Record; + + const scores = result.rubricScores as Record || {}; + const allPass = Object.values(scores).every((s) => s.pass !== false); + + const evalResult: EvalResult = { + name: wf.workflowName, + environment: wf.env, + reason: result.reason as string, + durationMs: result.durationMs as number, + rubricScores: scores, + pass: allPass, + }; + + results.push(evalResult); + + const status = allPass ? "PASS" : "FAIL"; + console.log(`[runner] ${status}: ${wf.workflowName} (${evalResult.durationMs}ms)`); + } + + ws.close(); + + // Output results + if (outputFormat === "junit") { + const xml = toJunitXml(results); + console.log(xml); + } else { + console.log(JSON.stringify(results, null, 2)); + } + + // Summary + const passed = results.filter((r) => r.pass).length; + const failed = results.length - passed; + console.log(`\n[runner] Done: ${passed} passed, ${failed} failed, ${results.length} total`); + + return results; +} + +// ── Multi-page parallel runner ──────────────────────────────────────────── + +export interface RunEvalsMultiPageOptions { + wsUrl: string; + manifestPath: string; + /** Channel IDs matching the browser pages (e.g. ["page-0", "page-1"]) */ + channels: string[]; + filterEnv?: string; + filterWorkflow?: string; +} + +/** + * Run eval workflows in parallel across multiple browser pages within a + * single browser instance. One WebSocket connection to the bridge; commands + * are routed to pages via a `channel` field that each page's EvalHarness + * filters on. + */ +export async function runEvalsMultiPage(options: RunEvalsMultiPageOptions): Promise { + const { wsUrl, manifestPath, channels, filterEnv, filterWorkflow } = options; + const numPages = channels.length; + + const allWorkflows = collectWorkflows(manifestPath, filterEnv, filterWorkflow); + if (allWorkflows.length === 0) { + console.log("[runner] No workflows match filter criteria."); + return []; + } + + console.log(`[runner] Multi-page: ${allWorkflows.length} workflow(s) across ${numPages} page(s)`); + + // Connect to bridge + const ws = new WebSocket(wsUrl); + ws.binaryType = "arraybuffer"; + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error("WebSocket connect timeout")), 30000); + ws.onopen = () => { clearTimeout(timeout); resolve(); }; + ws.onerror = (e) => { clearTimeout(timeout); reject(new Error(`WebSocket connection failed: ${e}`)); }; + }); + + console.log("[runner] Connected to bridge"); + + // Channel-aware sendAndWait: matches on BOTH message type AND channel + function sendAndWait( + cmd: Record, + responseType: string, + channel: string, + timeoutMs = 60000, + ): Promise> { + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`Timeout waiting for ${responseType} on ${channel}`)), + timeoutMs, + ); + + const handler = (event: MessageEvent) => { + if (typeof event.data !== "string") return; + try { + const msg = JSON.parse(event.data); + if (msg.type === responseType && msg.channel === channel) { + clearTimeout(timer); + ws.removeEventListener("message", handler); + resolve(msg); + } + } catch { /* not JSON */ } + }; + + ws.addEventListener("message", handler); + ws.send(JSON.stringify({ ...cmd, channel })); + }); + } + + // Wait for all pages to be ready (ping/pong per channel) + console.log("[runner] Waiting for all pages to be ready..."); + for (const ch of channels) { + const start = Date.now(); + let ready = false; + while (Date.now() - start < 60000) { + try { + await sendAndWait({ type: "ping" }, "pong", ch, 3000); + ready = true; + break; + } catch { + await new Promise((r) => setTimeout(r, 1000)); + } + } + if (!ready) { + console.error(`[runner] Timeout waiting for page ${ch}`); + ws.close(); + return []; + } + console.log(`[runner] Page ${ch} ready`); + } + + // Distribute workflows round-robin across pages + const batches: WorkflowEntry[][] = Array.from({ length: numPages }, () => []); + allWorkflows.forEach((wf, i) => batches[i % numPages].push(wf)); + + // Run each page's batch concurrently + const pagePromises = batches.map(async (batch, i) => { + const ch = channels[i]; + const tag = `[${ch}]`; + const pageResults: EvalResult[] = []; + let currentScene = ""; + + for (const wf of batch) { + try { + // Load scene if needed + if (wf.scene !== currentScene) { + console.log(`${tag} Loading environment: ${wf.env} (scene: ${wf.scene})`); + await sendAndWait({ type: "loadEnv", scene: wf.scene }, "envReady", ch, 120000); + currentScene = wf.scene; + await new Promise((r) => setTimeout(r, 2000)); + } + + const wfText = await Deno.readTextFile(wf.workflowPath); + const workflow = JSON.parse(wfText); + + console.log(`${tag} Starting: ${wf.workflowName} — "${workflow.task}"`); + + const timeoutMs = (workflow.timeoutSec || 120) * 1000 + 30000; + const result = await sendAndWait( + { type: "startWorkflow", workflow }, + "workflowComplete", + ch, + timeoutMs, + ) as Record; + + const scores = result.rubricScores as Record || {}; + const allPass = Object.values(scores).every((s) => s.pass !== false); + + const evalResult: EvalResult = { + name: wf.workflowName, + environment: wf.env, + reason: result.reason as string, + durationMs: result.durationMs as number, + rubricScores: scores, + pass: allPass, + }; + pageResults.push(evalResult); + + const status = allPass ? "PASS" : "FAIL"; + console.log(`${tag} ${status}: ${wf.workflowName} (${evalResult.durationMs}ms)`); + } catch (err) { + console.error(`${tag} Error on ${wf.workflowName}: ${err}`); + pageResults.push({ + name: wf.workflowName, + environment: wf.env, + reason: `page error: ${err}`, + durationMs: 0, + rubricScores: {}, + pass: false, + }); + } + } + + return pageResults; + }); + + const allResults = (await Promise.all(pagePromises)).flat(); + ws.close(); + + // Summary + const passed = allResults.filter((r) => r.pass).length; + const failed = allResults.length - passed; + console.log(`\n[runner] Done: ${passed} passed, ${failed} failed, ${allResults.length} total`); + + return allResults; +} + +// ── JUnit XML output ────────────────────────────────────────────────────── + +export function toJunitXml(results: EvalResult[]): string { + const totalTime = results.reduce((s, r) => s + r.durationMs, 0) / 1000; + const failures = results.filter((r) => !r.pass).length; + + let xml = `\n`; + xml += `\n`; + xml += ` \n`; + + for (const r of results) { + const time = (r.durationMs / 1000).toFixed(1); + xml += ` \n`; + } else { + xml += `>\n`; + xml += ` ${JSON.stringify(r.rubricScores)}\n`; + xml += ` \n`; + } + } + + xml += ` \n`; + xml += `\n`; + return xml; +} diff --git a/misc/DimSim/dimos-cli/eval/scene-index.ts b/misc/DimSim/dimos-cli/eval/scene-index.ts new file mode 100644 index 0000000000..e27d2caec3 --- /dev/null +++ b/misc/DimSim/dimos-cli/eval/scene-index.ts @@ -0,0 +1,107 @@ +/** + * Scene Index — parse scene JSON and search for objects by name. + * + * Uses the same substring matching logic as the browser-side rubric + * (`_findObject` in rubrics.ts) so validation matches scoring exactly. + */ + +export interface SceneObject { + title: string; + id: string; + position: { x: number; y: number; z: number }; +} + +export interface SceneIndex { + sceneName: string; + objects: SceneObject[]; +} + +/** + * Load a scene JSON and extract all titled assets with positions. + * Skips assets without titles (structural delta patches, etc). + */ +export function loadSceneIndex(scenePath: string, sceneName: string): SceneIndex { + const text = Deno.readTextFileSync(scenePath); + const json = JSON.parse(text); + const objects: SceneObject[] = []; + + if (Array.isArray(json.assets)) { + for (const asset of json.assets) { + const title = asset.title; + if (!title || typeof title !== "string") continue; + const pos = asset.transform?.position || asset.transform || {}; + objects.push({ + title: title.trim(), + id: asset.id || "", + position: { + x: Math.round((pos.x || 0) * 10) / 10, + y: Math.round((pos.y || 0) * 10) / 10, + z: Math.round((pos.z || 0) * 10) / 10, + }, + }); + } + } + + // Sort by title for display + objects.sort((a, b) => a.title.localeCompare(b.title)); + return { sceneName, objects }; +} + +/** + * Find an object by name — same case-insensitive substring match as the rubric. + * Returns the first match (same as rubric behavior). + */ +export function findObject(searchTerm: string, index: SceneIndex): SceneObject | null { + const lower = searchTerm.toLowerCase(); + for (const obj of index.objects) { + if (obj.title.toLowerCase().includes(lower) || obj.id.toLowerCase().includes(lower)) { + return obj; + } + } + return null; +} + +/** + * Find ALL objects matching the search term (for duplicate warnings). + */ +export function findAllObjects(searchTerm: string, index: SceneIndex): SceneObject[] { + const lower = searchTerm.toLowerCase(); + return index.objects.filter( + (obj) => obj.title.toLowerCase().includes(lower) || obj.id.toLowerCase().includes(lower), + ); +} + +/** + * Suggest similar objects when search fails (simple substring overlap). + */ +export function suggestObjects(searchTerm: string, index: SceneIndex, limit = 5): string[] { + const lower = searchTerm.toLowerCase(); + const scored: { title: string; score: number }[] = []; + + for (const obj of index.objects) { + const t = obj.title.toLowerCase(); + // Score: longest common substring length + let best = 0; + for (let len = 1; len <= lower.length; len++) { + for (let start = 0; start + len <= lower.length; start++) { + if (t.includes(lower.substring(start, start + len))) { + best = len; + } + } + } + if (best > 0) scored.push({ title: obj.title, score: best }); + } + + scored.sort((a, b) => b.score - a.score); + // Deduplicate titles + const seen = new Set(); + const result: string[] = []; + for (const s of scored) { + if (!seen.has(s.title)) { + seen.add(s.title); + result.push(s.title); + if (result.length >= limit) break; + } + } + return result; +} diff --git a/misc/DimSim/dimos-cli/headless/launcher.ts b/misc/DimSim/dimos-cli/headless/launcher.ts new file mode 100644 index 0000000000..0a9a6c8875 --- /dev/null +++ b/misc/DimSim/dimos-cli/headless/launcher.ts @@ -0,0 +1,204 @@ +/** + * Headless Launcher — Playwright-based headless Chromium for CI/CD evals. + * + * Rendering modes: + * gpu — Metal/ANGLE (macOS, fast, max ~3 parallel pages) + * cpu — SwiftShader (Linux CI, no GPU needed, sequential only on <16 cores) + */ + +import { chromium, type Browser, type Page } from "playwright"; + +export type RenderMode = "gpu" | "cpu"; + +export interface LaunchOptions { + url: string; + timeout?: number; + render?: RenderMode; +} + +export interface HeadlessInstance { + browser: Browser; + page: Page; + close: () => Promise; +} + +export interface MultiPageInstance { + browser: Browser; + pages: Page[]; + channels: string[]; + close: () => Promise; +} + +export interface MultiPageOptions { + url: string; + numPages: number; + timeout?: number; + render?: RenderMode; +} + +// ── Chrome flags per render mode ────────────────────────────────────────── + +const GPU_ARGS = [ + "--headless=new", + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-features=SkiaGraphite", + "--enable-webgl", + "--enable-webgl2", + "--ignore-gpu-blocklist", + "--enable-gpu", + "--use-gl=angle", + "--use-angle=metal", + "--in-process-gpu", + "--disable-gpu-sandbox", + // Prevent Chrome from throttling timers in headless/background mode + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", +]; + +const CPU_ARGS = [ + "--headless=new", + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-features=SkiaGraphite", + "--enable-webgl", + "--enable-webgl2", + "--use-gl=angle", + "--use-angle=swiftshader", + "--enable-unsafe-swiftshader", + "--disable-gpu", + // Prevent Chrome from throttling timers in headless/background mode + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", +]; + +// Default: bundled Chromium (works on Linux + macOS in CPU mode with SwiftShader). +// Set DIMSIM_CHROME_CHANNEL=chrome to use system Google Chrome (needed for hardware +// WebGL on macOS — bundled Chromium ships without the full Metal/ANGLE GPU stack). +const LAUNCH_CHANNEL = Deno.env.get("DIMSIM_CHROME_CHANNEL") || undefined; + +// ── Helpers ─────────────────────────────────────────────────────────────── + +/** Filter noisy browser console output — only forward errors, warnings, and eval/bridge logs. */ +function hookPageConsole(page: Page, tag: string): void { + const verbose = Deno.env.get("DIMSIM_VERBOSE") === "1"; + page.on("console", (msg) => { + const type = msg.type(); + const text = msg.text(); + if (!verbose) { + if (text.includes("Texture marked for update") || text.includes("Failed to load resource") || + text.includes("GPU stall due to ReadPixels") || text.includes("Automatic fallback to software WebGL") || + text.includes("GroupMarkerNotSet")) return; + } + if (type === "error") console.error(`${tag} ${text}`); + else if (type === "warning") console.warn(`${tag} ${text}`); + else if (verbose || text.startsWith("[eval]") || text.startsWith("[DimosBridge]")) { + console.log(`${tag} ${text}`); + } + }); +} + +function getViewport(render: RenderMode) { + // CPU mode: tiny viewport — SwiftShader renders every pixel on CPU + return render === "cpu" + ? { width: 320, height: 240 } + : { width: 1280, height: 720 }; +} + +// ── Single-page launcher ───────────────────────────────────────────────── + +export async function launchHeadless(options: LaunchOptions): Promise { + const { url, timeout = 30000, render = "cpu" } = options; + const args = render === "gpu" ? GPU_ARGS : CPU_ARGS; + + console.log(`[headless] Launching: render=${render}`); + + const browser = await chromium.launch({ + headless: false, // --headless=new passed via args (Playwright's built-in headless uses old mode) + channel: LAUNCH_CHANNEL, + args, + }); + + const context = await browser.newContext({ viewport: getViewport(render), deviceScaleFactor: 1 }); + const page = await context.newPage(); + hookPageConsole(page, "[browser]"); + + // Set default timeout so waitForFunction picks it up (its third arg is + // options, not the second — passing {timeout} as second silently uses + // Playwright's 30s default). + context.setDefaultTimeout(timeout); + page.setDefaultTimeout(timeout); + + await page.goto(url, { waitUntil: "load", timeout }); + await page.waitForFunction( + () => typeof (window as unknown as Record).__dimosBridge !== "undefined", + undefined, + { timeout }, + ); + + console.log("[headless] Engine ready."); + + return { + browser, + page, + close: async () => { + await browser.close(); + console.log("[headless] Browser closed."); + }, + }; +} + +// ── Multi-page launcher (single browser, N tabs) ──────────────────────── + +export async function launchMultiPage(options: MultiPageOptions): Promise { + const { url, numPages, timeout = 120_000, render = "cpu" } = options; + const args = render === "gpu" ? GPU_ARGS : CPU_ARGS; + const viewport = getViewport(render); + + console.log(`[headless] Multi-page: ${numPages} pages, render=${render}, timeout=${timeout}ms`); + + const browser = await chromium.launch({ headless: false, channel: LAUNCH_CHANNEL, args }); + + const pages: Page[] = []; + const channels: string[] = []; + + for (let i = 0; i < numPages; i++) { + const channel = `page-${i}`; + channels.push(channel); + + const context = await browser.newContext({ viewport, deviceScaleFactor: 1 }); + context.setDefaultTimeout(timeout); + const page = await context.newPage(); + page.setDefaultTimeout(timeout); + hookPageConsole(page, `[page-${i}]`); + + const pageUrl = `${url}?channel=${channel}`; + console.log(`[headless] Page ${i}: loading...`); + await page.goto(pageUrl, { waitUntil: "load", timeout }); + await page.waitForFunction( + () => typeof (window as unknown as Record).__dimosBridge !== "undefined", + undefined, + { timeout }, + ); + console.log(`[headless] Page ${i}: ready`); + + pages.push(page); + + // Stagger launches to avoid GPU/CPU contention during scene load + if (i < numPages - 1) await new Promise((r) => setTimeout(r, 5000)); + } + + console.log(`[headless] All ${numPages} pages ready.`); + + return { + browser, + pages, + channels, + close: async () => { + await browser.close(); + console.log("[headless] Browser closed."); + }, + }; +} diff --git a/misc/DimSim/dimos-cli/mod.ts b/misc/DimSim/dimos-cli/mod.ts new file mode 100644 index 0000000000..ff096eae93 --- /dev/null +++ b/misc/DimSim/dimos-cli/mod.ts @@ -0,0 +1,86 @@ +/** + * @module + * + * **DimSim** — 3D simulation environment for the + * [dimos](https://github.com/dimensionalOS/dimos) robotics stack. + * + * Provides a browser-based Three.js + Rapier simulator with LCM transport, + * sensor publishing (RGB, depth, LiDAR, odometry), and an eval harness for + * automated testing of navigation and perception pipelines. + * + * ## Install + * + * ```sh + * deno install -gAf --unstable-net jsr:@antim/dimsim + * ``` + * + * ## Setup + * + * Download core assets (~22 MB) and install a scene: + * + * ```sh + * dimsim setup + * dimsim scene install apt + * ``` + * + * ## Run + * + * Start the dev server and open the URL it prints: + * + * ```sh + * dimsim dev --scene apt + * ``` + * + * Run headless evals in CI: + * + * ```sh + * dimsim eval --headless --env apt --workflow reach-vase + * ``` + * + * ## Programmatic API + * + * ```ts + * import { startBridgeServer } from "@antim/dimsim"; + * + * startBridgeServer({ port: 8090, distDir: "./dist", scene: "apt" }); + * ``` + */ + +/** Start the WebSocket bridge server that relays LCM packets between the browser and external agents. */ +export { startBridgeServer } from "./bridge/server.ts"; + +/** Launch a single headless Chromium page pointed at the sim. */ +export { launchHeadless } from "./headless/launcher.ts"; + +/** Launch multiple headless pages for parallel eval workflows. */ +export { launchMultiPage } from "./headless/launcher.ts"; + +/** Run eval workflows sequentially against a connected browser. */ +export { runEvals } from "./eval/runner.ts"; + +/** Run eval workflows distributed across multiple browser pages. */ +export { runEvalsMultiPage } from "./eval/runner.ts"; + +/** Collect workflow definitions from the manifest, optionally filtered by env/workflow name. */ +export { collectWorkflows } from "./eval/runner.ts"; + +/** Convert eval results to JUnit XML format for CI reporting. */ +export { toJunitXml } from "./eval/runner.ts"; + +/** Download and extract core DimSim assets to ~/.dimsim/. */ +export { setup } from "./setup.ts"; + +/** Download and install a scene by name from the registry. */ +export { sceneInstall } from "./setup.ts"; + +/** List installed and available scenes. */ +export { sceneList } from "./setup.ts"; + +/** Remove a locally installed scene. */ +export { sceneRemove } from "./setup.ts"; + +/** Get the DimSim home directory path (~/.dimsim or DIMSIM_HOME). */ +export { getDimsimHome } from "./setup.ts"; + +/** Get the path to the dist directory containing built frontend assets. */ +export { getDistDir } from "./setup.ts"; diff --git a/misc/DimSim/dimos-cli/run-eval.ts b/misc/DimSim/dimos-cli/run-eval.ts new file mode 100644 index 0000000000..9e44621203 --- /dev/null +++ b/misc/DimSim/dimos-cli/run-eval.ts @@ -0,0 +1,40 @@ +#!/usr/bin/env -S deno run --allow-all --unstable-net +/** + * Run eval against an already-running bridge server. + * Usage: deno run --allow-all --unstable-net dimos-cli/run-eval.ts [--workflow reach-vase] [--port 8090] + */ +import { resolve, dirname, fromFileUrl } from "@std/path"; +import { runEvals } from "./eval/runner.ts"; + +const CLI_DIR = dirname(fromFileUrl(import.meta.url)); +const EVALS_DIR = resolve(CLI_DIR, "../evals"); + +const args = Deno.args; +let port = 8090; +let workflow: string | undefined; +let env: string | undefined; + +for (let i = 0; i < args.length; i++) { + if (args[i] === "--port" && args[i + 1]) port = parseInt(args[++i]); + if (args[i] === "--workflow" && args[i + 1]) workflow = args[++i]; + if (args[i] === "--env" && args[i + 1]) env = args[++i]; +} + +const wsUrl = `ws://localhost:${port}`; +const manifestPath = resolve(EVALS_DIR, "manifest.json"); + +console.log(`[eval] Connecting to bridge at ${wsUrl}`); +console.log(`[eval] Workflow: ${workflow || "all"}, Env: ${env || "all"}`); + +const results = await runEvals({ + wsUrl, + manifestPath, + filterEnv: env, + filterWorkflow: workflow, + outputFormat: "json", +}); + +const passed = results.filter((r) => r.pass).length; +const failed = results.length - passed; +console.log(`\n[eval] Done: ${passed} passed, ${failed} failed`); +Deno.exit(failed > 0 ? 1 : 0); diff --git a/misc/DimSim/dimos-cli/setup.ts b/misc/DimSim/dimos-cli/setup.ts new file mode 100644 index 0000000000..8bd2665204 --- /dev/null +++ b/misc/DimSim/dimos-cli/setup.ts @@ -0,0 +1,357 @@ +/** + * DimSim Setup — Downloads core assets and scenes from GitHub Releases. + * + * Local data stored at ~/.dimsim/ (override with DIMSIM_HOME env var). + * + * ~/.dimsim/ + * ├── dist/ (core frontend: index.html, assets/, agent-model/) + * │ └── sims/ (downloaded scene JSON files) + * │ └── apt.json + * └── evals/ (eval workflows) + */ + +const GITHUB_REPO = "Antim-Labs/DimSim"; +const RELEASES_API = `https://api.github.com/repos/${GITHUB_REPO}/releases/latest`; + +export function getDimsimHome(): string { + return ( + Deno.env.get("DIMSIM_HOME") || + `${Deno.env.get("HOME")}/.dimsim` + ); +} + +export function getDistDir(): string { + return `${getDimsimHome()}/dist`; +} + +// ── Registry ──────────────────────────────────────────────────────────── + +interface SceneEntry { + url: string; + description: string; + size: number; +} + +interface Registry { + version: string; + coreUrl: string; + evalsUrl?: string; + scenes: Record; +} + +async function fetchRegistry(localPath?: string): Promise { + if (localPath) { + return JSON.parse(await Deno.readTextFile(localPath)); + } + + // Fetch latest release from GitHub API, find registry.json asset + const resp = await fetch(RELEASES_API, { + headers: { Accept: "application/vnd.github.v3+json" }, + }); + if (!resp.ok) throw new Error(`Failed to fetch latest release: ${resp.status}`); + const release = await resp.json(); + + const asset = release.assets?.find( + (a: { name: string }) => a.name === "registry.json", + ); + if (!asset) { + throw new Error("registry.json not found in latest GitHub release"); + } + + const regResp = await fetch(asset.browser_download_url); + if (!regResp.ok) throw new Error(`Failed to download registry: ${regResp.status}`); + return regResp.json(); +} + +// ── Download with progress ────────────────────────────────────────────── + +async function download(url: string, dest: string): Promise { + console.log(` Downloading ${url}`); + const resp = await fetch(url); + if (!resp.ok) throw new Error(`Download failed: ${resp.status} ${url}`); + + const total = parseInt(resp.headers.get("content-length") || "0"); + const reader = resp.body!.getReader(); + const chunks: Uint8Array[] = []; + let received = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + received += value.length; + if (total > 0) { + const pct = ((received / total) * 100).toFixed(0); + const mb = (received / 1e6).toFixed(1); + Deno.stderr.writeSync( + new TextEncoder().encode(`\r ${mb} MB / ${(total / 1e6).toFixed(1)} MB (${pct}%)`) + ); + } + } + Deno.stderr.writeSync(new TextEncoder().encode("\n")); + + // Concatenate chunks into a single Uint8Array + const totalLength = chunks.reduce((sum, c) => sum + c.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + await Deno.writeFile(dest, result); +} + +// ── Extract tar.gz ────────────────────────────────────────────────────── + +async function extractTarGz(archive: string, destDir: string): Promise { + await Deno.mkdir(destDir, { recursive: true }); + const proc = new Deno.Command("tar", { + args: ["-xzf", archive, "-C", destDir], + stdout: "inherit", + stderr: "inherit", + }).spawn(); + const status = await proc.status; + if (!status.success) throw new Error(`tar extract failed (exit ${status.code})`); +} + +/** Extract a single gzipped file (not a tar, just gzip). */ +async function extractGz(archive: string, destFile: string): Promise { + const parentDir = destFile.substring(0, destFile.lastIndexOf("/")); + await Deno.mkdir(parentDir, { recursive: true }); + const proc = new Deno.Command("sh", { + args: ["-c", `gunzip -c "${archive}" > "${destFile}"`], + stdout: "inherit", + stderr: "inherit", + }).spawn(); + const status = await proc.status; + if (!status.success) throw new Error(`gunzip failed (exit ${status.code})`); +} + +// ── Version tracking ──────────────────────────────────────────────────── + +interface VersionInfo { + core?: string; + scenes?: Record; +} + +function versionPath(): string { + return `${getDimsimHome()}/version.json`; +} + +async function readVersionInfo(): Promise { + try { + return JSON.parse(await Deno.readTextFile(versionPath())); + } catch { + return {}; + } +} + +async function writeVersionInfo(info: VersionInfo): Promise { + await Deno.writeTextFile(versionPath(), JSON.stringify(info, null, 2)); +} + +// ── Public API ────────────────────────────────────────────────────────── + +export async function setup(localArchive?: string): Promise { + const home = getDimsimHome(); + const distDir = getDistDir(); + + console.log(`[dimsim] Setting up in ${home}`); + await Deno.mkdir(home, { recursive: true }); + + let registry: Registry | null = null; + + if (localArchive) { + console.log(`[dimsim] Extracting core from local archive: ${localArchive}`); + await extractTarGz(localArchive, distDir); + } else { + registry = await fetchRegistry(); + const local = await readVersionInfo(); + + if (local.core === registry.version) { + console.log(`[dimsim] Core already up-to-date (v${registry.version})`); + } else { + if (local.core) { + console.log(`[dimsim] Updating core: v${local.core} → v${registry.version}`); + } + const tmpFile = `${home}/core-download.tar.gz`; + await download(registry.coreUrl, tmpFile); + console.log(`[dimsim] Extracting core assets...`); + await extractTarGz(tmpFile, distDir); + await Deno.remove(tmpFile); + + // Write updated version + local.core = registry.version; + await writeVersionInfo(local); + } + } + + // Ensure sims directory exists + await Deno.mkdir(`${distDir}/sims`, { recursive: true }); + + // Create empty manifest if none exists + const manifestPath = `${distDir}/sims/manifest.json`; + try { + await Deno.stat(manifestPath); + } catch { + await Deno.writeTextFile(manifestPath, JSON.stringify([], null, 2)); + } + + // Install evals to ~/.dimsim/evals/ + if (registry?.evalsUrl) { + const evalsDir = `${home}/evals`; + const evalsVerFile = `${home}/evals-version`; + let installedEvalsVer: string | null = null; + try { + installedEvalsVer = (await Deno.readTextFile(evalsVerFile)).trim(); + } catch { /* not installed */ } + + if (installedEvalsVer === registry.version) { + console.log(`[dimsim] Evals already up-to-date (v${registry.version})`); + } else { + const tmpFile = `${home}/evals-download.tar.gz`; + console.log(`[dimsim] Downloading evals...`); + await download(registry.evalsUrl, tmpFile); + console.log(`[dimsim] Extracting evals...`); + await extractTarGz(tmpFile, evalsDir); + await Deno.remove(tmpFile); + await Deno.writeTextFile(evalsVerFile, registry.version); + } + } + + console.log(`[dimsim] Core setup complete.`); + console.log(`[dimsim] Install a scene: dimsim scene install apt`); +} + +export async function sceneInstall( + name: string, + localArchive?: string, +): Promise { + const home = getDimsimHome(); + const distDir = getDistDir(); + const simsDir = `${distDir}/sims`; + const destFile = `${simsDir}/${name}.json`; + + // Check core is set up + try { + await Deno.stat(distDir); + } catch { + console.error(`[dimsim] Core not installed. Run 'dimsim setup' first.`); + Deno.exit(1); + } + + await Deno.mkdir(simsDir, { recursive: true }); + + if (localArchive) { + console.log(`[dimsim] Installing scene '${name}' from local: ${localArchive}`); + if (localArchive.endsWith(".json")) { + await Deno.copyFile(localArchive, destFile); + } else { + await extractGz(localArchive, destFile); + } + } else { + const registry = await fetchRegistry(); + const entry = registry.scenes[name]; + if (!entry) { + console.error(`[dimsim] Scene '${name}' not found. Available:`); + for (const [k, v] of Object.entries(registry.scenes)) { + console.error(` ${k} — ${v.description}`); + } + Deno.exit(1); + } + + const local = await readVersionInfo(); + const localSceneVer = local.scenes?.[name]; + + if (localSceneVer === registry.version) { + console.log(`[dimsim] Scene '${name}' already up-to-date (v${registry.version})`); + return; + } + + if (localSceneVer) { + console.log(`[dimsim] Updating scene '${name}': v${localSceneVer} → v${registry.version}`); + } + const tmpFile = `${home}/${name}-download.gz`; + console.log(`[dimsim] Downloading scene '${name}' (${(entry.size / 1e6).toFixed(1)} MB)...`); + await download(entry.url, tmpFile); + console.log(`[dimsim] Extracting...`); + await extractGz(tmpFile, destFile); + await Deno.remove(tmpFile); + + // Write updated version + if (!local.scenes) local.scenes = {}; + local.scenes[name] = registry.version; + await writeVersionInfo(local); + } + + // Update local manifest + const manifestPath = `${simsDir}/manifest.json`; + let manifest: string[] = []; + try { + manifest = JSON.parse(await Deno.readTextFile(manifestPath)); + } catch { /* empty */ } + if (!manifest.includes(name)) { + manifest.push(name); + await Deno.writeTextFile(manifestPath, JSON.stringify(manifest, null, 2)); + } + + console.log(`[dimsim] Scene '${name}' installed.`); +} + +export async function sceneList(): Promise { + const simsDir = `${getDistDir()}/sims`; + + // Local scenes + const installed: string[] = []; + try { + for await (const entry of Deno.readDir(simsDir)) { + if (entry.name.endsWith(".json") && entry.name !== "manifest.json") { + installed.push(entry.name.replace(".json", "")); + } + } + } catch { /* no sims dir */ } + + // Remote scenes + let registry: Registry | null = null; + try { + registry = await fetchRegistry(); + } catch { + console.log("[dimsim] Could not fetch remote registry."); + } + + console.log("\nInstalled scenes:"); + if (installed.length === 0) { + console.log(" (none)"); + } else { + for (const s of installed) console.log(` * ${s}`); + } + + if (registry) { + console.log("\nAvailable scenes:"); + for (const [name, entry] of Object.entries(registry.scenes)) { + const status = installed.includes(name) ? " (installed)" : ""; + console.log(` ${name} — ${entry.description} (${(entry.size / 1e6).toFixed(1)} MB)${status}`); + } + } + console.log(); +} + +export async function sceneRemove(name: string): Promise { + const simsDir = `${getDistDir()}/sims`; + const scenePath = `${simsDir}/${name}.json`; + try { + await Deno.remove(scenePath); + console.log(`[dimsim] Scene '${name}' removed.`); + } catch { + console.error(`[dimsim] Scene '${name}' not found locally.`); + Deno.exit(1); + } + + // Update manifest + const manifestPath = `${simsDir}/manifest.json`; + try { + const manifest: string[] = JSON.parse(await Deno.readTextFile(manifestPath)); + const filtered = manifest.filter((s) => s !== name); + await Deno.writeTextFile(manifestPath, JSON.stringify(filtered, null, 2)); + } catch { /* ok */ } +} diff --git a/misc/DimSim/dimos-cli/test/diagnose_costmap.py b/misc/DimSim/dimos-cli/test/diagnose_costmap.py new file mode 100644 index 0000000000..5323a355af --- /dev/null +++ b/misc/DimSim/dimos-cli/test/diagnose_costmap.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Diagnostic: subscribe to /global_costmap and /odom via LCM, print costmap stats. + +Run with the dimos venv while the agent + bridge are running: + cd DimSim + ../dimos/.venv/bin/python dimos-cli/test/diagnose_costmap.py + +Prints: cell counts (FREE/OCCUPIED/UNKNOWN), robot grid position, +and whether frontier-eligible cells exist. +""" + +import sys +import time + +import numpy as np + +try: + import lcm as lcm_lib +except ImportError: + print("ERROR: 'lcm' Python package not found. Run with dimos venv.") + sys.exit(1) + +from dimos_lcm.geometry_msgs.PoseStamped import PoseStamped as LCMPoseStamped +from dimos_lcm.nav_msgs.OccupancyGrid import OccupancyGrid as LCMOccupancyGrid + + +class CostmapDiagnostic: + def __init__(self): + self.lc = lcm_lib.LCM("udpm://239.255.76.67:7667?ttl=1") + self.latest_costmap = None + self.latest_odom = None + self.costmap_count = 0 + self.odom_count = 0 + + def _on_costmap(self, channel, data): + try: + msg = LCMOccupancyGrid.lcm_decode(data) + self.latest_costmap = msg + self.costmap_count += 1 + except Exception as e: + print(f"Costmap decode error: {e}") + + def _on_odom(self, channel, data): + try: + msg = LCMPoseStamped.lcm_decode(data) + self.latest_odom = msg + self.odom_count += 1 + except Exception as e: + print(f"Odom decode error: {e}") + + def analyze(self): + if self.latest_costmap is None: + print("No costmap received yet.") + return + + msg = self.latest_costmap + w = msg.info.width + h = msg.info.height + res = msg.info.resolution + ox = msg.info.origin.position.x + oy = msg.info.origin.position.y + + grid = np.array(msg.data, dtype=np.int8).reshape(h, w) + + free_count = int(np.sum(grid == 0)) + occupied_count = int(np.sum(grid == 100)) + unknown_count = int(np.sum(grid == -1)) + other_count = int(np.sum((grid != 0) & (grid != 100) & (grid != -1))) + total = w * h + + print(f"\n{'=' * 60}") + print(f"COSTMAP #{self.costmap_count} ({w}x{h} cells, {res:.3f} m/cell)") + print(f"Origin: ({ox:.2f}, {oy:.2f})") + print(f"World extent: X=[{ox:.2f}, {ox + w * res:.2f}] Y=[{oy:.2f}, {oy + h * res:.2f}]") + print(f" FREE (0): {free_count:>8} ({100 * free_count / total:.1f}%)") + print(f" OCCUPIED (100): {occupied_count:>8} ({100 * occupied_count / total:.1f}%)") + print(f" UNKNOWN (-1): {unknown_count:>8} ({100 * unknown_count / total:.1f}%)") + print(f" OTHER (1-99): {other_count:>8} ({100 * other_count / total:.1f}%)") + + if other_count > 0: + mask = (grid != 0) & (grid != 100) & (grid != -1) + other_vals = grid[mask] + unique, counts = np.unique(other_vals, return_counts=True) + print(" Other cost distribution (top 10):") + for v, c in sorted(zip(unique, counts, strict=False), key=lambda x: -x[1])[:10]: + print(f" cost={v:>4}: {c} cells") + + # Check robot position + if self.latest_odom is not None: + rx = self.latest_odom.pose.position.x + ry = self.latest_odom.pose.position.y + rz = self.latest_odom.pose.position.z + gx = int((rx - ox) / res) + gy = int((ry - oy) / res) + print(f"\nRobot world pos: ({rx:.2f}, {ry:.2f}, {rz:.2f})") + print(f"Robot grid cell: ({gx}, {gy})") + if 0 <= gx < w and 0 <= gy < h: + print(f"Robot cell cost: {grid[gy, gx]}") + r = 5 + region = grid[max(0, gy - r) : gy + r + 1, max(0, gx - r) : gx + r + 1] + rfree = int(np.sum(region == 0)) + rocc = int(np.sum(region == 100)) + runk = int(np.sum(region == -1)) + roth = int(np.sum((region != 0) & (region != 100) & (region != -1))) + print(f"11x11 neighborhood: FREE={rfree} OCC={rocc} UNK={runk} OTHER={roth}") + else: + print("WARNING: Robot is OUTSIDE costmap bounds!") + else: + print(f"\nNo odom received (count={self.odom_count})") + + # Frontier analysis + unk_mask = grid == -1 + free_mask = grid == 0 + occ_mask = grid >= 100 + + from scipy import ndimage + + kernel = np.ones((3, 3)) + free_dilated = ndimage.binary_dilation(free_mask, structure=kernel) + occ_dilated = ndimage.binary_dilation(occ_mask, structure=kernel) + + candidates = unk_mask & free_dilated + unknown_near_free = int(np.sum(candidates)) + frontier_eligible = candidates & ~occ_dilated + frontier_count = int(np.sum(frontier_eligible)) + frontier_blocked = unknown_near_free - frontier_count + + print("\nFRONTIER ANALYSIS (pre-inflation):") + print(f" Unknown cells adjacent to free: {unknown_near_free}") + print(f" ...blocked by adjacent occupied: {frontier_blocked}") + print(f" VALID FRONTIER CELLS: {frontier_count}") + + # Simulate inflation effect + if frontier_count > 0: + inflate_radius = 0.25 # meters, default in frontier explorer + cell_radius = int(np.ceil(inflate_radius / res)) + y, x = np.ogrid[-cell_radius : cell_radius + 1, -cell_radius : cell_radius + 1] + inflate_kernel = (x**2 + y**2 <= cell_radius**2).astype(np.uint8) + inflated_occ = ndimage.binary_dilation(occ_mask, structure=inflate_kernel) + inflated_occ_dilated = ndimage.binary_dilation(inflated_occ, structure=kernel) + frontier_after_inflate = candidates & ~inflated_occ_dilated + print("\n After 0.25m inflation:") + print( + f" VALID FRONTIER CELLS: {int(np.sum(frontier_after_inflate))}" + ) + + if frontier_count == 0 and unknown_near_free > 0: + print(f"\n *** DIAGNOSIS: ALL {unknown_near_free} unknown-near-free cells") + print(" are also adjacent to occupied cells. Obstacles border every") + print(" free/unknown boundary. The height_cost algorithm may produce") + print(" high-gradient costs at edges, or the LiDAR sees obstacles") + print(" exactly at the boundary of observed space.") + elif frontier_count == 0 and free_count == 0: + print("\n *** DIAGNOSIS: No FREE (cost=0) cells at all!") + print(" The height_cost algorithm is not seeing flat ground.") + print(" Check if LiDAR produces ground-hitting points (Z<0.1 in robotics frame).") + elif frontier_count == 0 and unknown_near_free == 0 and free_count > 0: + print("\n *** DIAGNOSIS: FREE cells exist but no UNKNOWN cells border them.") + print(" The free space is fully enclosed by occupied/other-cost cells.") + + def run(self): + self.lc.subscribe("/global_costmap#nav_msgs.OccupancyGrid", self._on_costmap) + self.lc.subscribe("/odom#geometry_msgs.PoseStamped", self._on_odom) + + print("Listening on LCM for /global_costmap and /odom...") + print("Start the dimos agent + bridge in other terminals. Ctrl+C to stop.\n") + + try: + last_print = 0 + while True: + self.lc.handle_timeout(500) + now = time.time() + if now - last_print > 3.0: + print(f"\n--- msgs: costmap={self.costmap_count}, odom={self.odom_count} ---") + self.analyze() + last_print = now + except KeyboardInterrupt: + print("\nDone.") + + +if __name__ == "__main__": + CostmapDiagnostic().run() diff --git a/misc/DimSim/dimos-cli/test/dimos_integration.py b/misc/DimSim/dimos-cli/test/dimos_integration.py new file mode 100755 index 0000000000..98ceb90364 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/dimos_integration.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +DimSim ↔ dimos Integration Test (UDP Multicast) + +Validates end-to-end connectivity between DimSim (browser sim) and dimos +(Python robotics stack) via LCM UDP multicast through the bridge server. + +Data flow: + Python ──UDP multicast──▶ Bridge Server ──WebSocket──▶ Browser (DimSim) + Python ◀──UDP multicast── Bridge Server ◀──WebSocket── Browser (DimSim) + +This script: + 1. Joins the LCM multicast group (239.255.76.67:7667) + 2. Publishes /cmd_vel Twist commands as LCM packets via UDP multicast → agent moves + 3. Listens for /odom, /camera/image, /camera/depth, /lidar/points on multicast + 4. Reports what it receives; SUCCESS when all 4 channels are live + +Prerequisites: + 1. Start DimSim bridge: + ~/.deno/bin/deno run --allow-all --unstable-net dimos-cli/cli.ts dev + 2. Open http://localhost:8090 in Chrome (scene must load) + 3. Run this script from the dimos venv: + /path/to/dimos/.venv/bin/python dimos-cli/test/dimos_integration.py + +Options: + --timeout N Timeout in seconds (default: 30) + --rate N cmd_vel publish rate in Hz (default: 10) +""" + +import argparse +import socket +import struct +import sys +import threading +import time + +# dimos message types for encoding cmd_vel +from dimos.msgs.geometry_msgs import Twist, Vector3 + +# -- LCM constants ------------------------------------------------------------ +LCM_MAGIC = 0x4C433032 # "LC02" in ASCII / big-endian +MCAST_GRP = "239.255.76.67" +MCAST_PORT = 7667 +_seq = 0 + +# -- LCM packet codec (matches @dimos/msgs encodePacket / decodePacket) -------- + + +def encode_lcm_packet(channel: str, payload: bytes) -> bytes: + """Encode an LCM binary packet (same format as @dimos/msgs encodePacket).""" + global _seq + ch_bytes = channel.encode("utf-8") + buf = struct.pack(">II", LCM_MAGIC, _seq) + ch_bytes + b"\x00" + payload + _seq += 1 + return buf + + +def decode_lcm_packet(data: bytes) -> tuple[str, bytes]: + """Decode an LCM packet → (channel, payload). Raises ValueError on bad packet.""" + if len(data) < 9: + raise ValueError("Packet too short") + magic = struct.unpack_from(">I", data, 0)[0] + if magic != LCM_MAGIC: + raise ValueError(f"Bad magic: 0x{magic:08x}") + null_pos = data.index(0, 8) + channel = data[8:null_pos].decode("utf-8") + payload = data[null_pos + 1 :] + return channel, payload + + +# -- Channel names (must match DimSim's dimosBridge.ts) ------------------------ +CH_CMD_VEL = "/cmd_vel#geometry_msgs.Twist" +CH_ODOM = "/odom#geometry_msgs.PoseStamped" +CH_IMAGE = "/camera/image#sensor_msgs.Image" +CH_DEPTH = "/camera/depth#sensor_msgs.Image" +CH_LIDAR = "/lidar/points#sensor_msgs.PointCloud2" + + +def create_mcast_recv_socket() -> socket.socket: + """Create a UDP socket joined to the LCM multicast group for receiving.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except AttributeError: + pass + sock.bind(("", MCAST_PORT)) + mreq = struct.pack("4s4s", socket.inet_aton(MCAST_GRP), socket.inet_aton("0.0.0.0")) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 0) + sock.settimeout(1.0) + return sock + + +def create_mcast_send_socket() -> socket.socket: + """Create a UDP socket for sending to the LCM multicast group.""" + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_LOOP, 0) + return sock + + +def main(): + parser = argparse.ArgumentParser(description="DimSim ↔ dimos integration test (UDP multicast)") + parser.add_argument("--timeout", type=int, default=30, help="Timeout in seconds") + parser.add_argument("--rate", type=int, default=10, help="cmd_vel publish rate (Hz)") + args = parser.parse_args() + + received = {"odom": 0, "image": 0, "depth": 0, "lidar": 0} + tick = 0 + success = False + running = True + + recv_sock = create_mcast_recv_socket() + send_sock = create_mcast_send_socket() + + print(f"[integration] LCM multicast {MCAST_GRP}:{MCAST_PORT}") + print(f"[integration] Publishing /cmd_vel at {args.rate} Hz") + print("[integration] Listening for sensor data on multicast") + print(f"[integration] Timeout: {args.timeout}s\n") + + # -- Receive thread -------------------------------------------------------- + def recv_loop(): + while running: + try: + data, addr = recv_sock.recvfrom(65536) + except TimeoutError: + continue + except OSError: + break + + try: + channel, payload = decode_lcm_packet(data) + except (ValueError, IndexError): + continue + + if "/odom" in channel: + received["odom"] += 1 + if received["odom"] <= 3 or received["odom"] % 10 == 0: + print(f"[integration] Got odom #{received['odom']} ({len(payload)}B)") + elif "/camera/image" in channel: + received["image"] += 1 + if received["image"] <= 3 or received["image"] % 10 == 0: + print(f"[integration] Got RGB #{received['image']} ({len(payload)}B)") + elif "/camera/depth" in channel: + received["depth"] += 1 + if received["depth"] <= 3 or received["depth"] % 10 == 0: + print(f"[integration] Got depth #{received['depth']} ({len(payload)}B)") + elif "/lidar/points" in channel: + received["lidar"] += 1 + if received["lidar"] <= 3 or received["lidar"] % 10 == 0: + print(f"[integration] Got LiDAR #{received['lidar']} ({len(payload)}B)") + + recv_thread = threading.Thread(target=recv_loop, daemon=True) + recv_thread.start() + + # -- Send loop ------------------------------------------------------------- + interval = 1.0 / args.rate + start_time = time.time() + + try: + while time.time() - start_time < args.timeout: + # Build Twist — Three.js identity: z=forward, y=yaw + twist = Twist( + linear=Vector3(0, 0, 0.5), + angular=Vector3(0, 0.3, 0), + ) + payload = twist.lcm_encode() + packet = encode_lcm_packet(CH_CMD_VEL, payload) + send_sock.sendto(packet, (MCAST_GRP, MCAST_PORT)) + tick += 1 + + if tick <= 3 or tick % 20 == 0: + print(f"[integration] Sent cmd_vel #{tick}") + + # Status check every 5s + elapsed = time.time() - start_time + if tick > 1 and (tick % (args.rate * 5) == 0): + print( + f"\n[integration] STATUS ({elapsed:.0f}s): " + f"cmd_sent={tick} odom={received['odom']} " + f"rgb={received['image']} depth={received['depth']} " + f"lidar={received['lidar']}" + ) + + if all(v > 0 for v in received.values()): + success = True + print("\n========================================") + print(" SUCCESS: All channels working!") + print(" DimSim ↔ dimos LCM multicast verified.") + print("========================================\n") + break + + if received["odom"] == 0 and elapsed > 10: + print("[integration] No sensor data on multicast. Check:") + print(" 1. Bridge running with vendored @dimos/lcm (joinMulticastV4)") + print(" 2. Browser open at localhost:8090 with scene loaded") + print() + + time.sleep(interval) + + if not success: + print(f"\n[integration] TIMEOUT after {args.timeout}s") + print( + f"[integration] Final: cmd_sent={tick} odom={received['odom']} " + f"rgb={received['image']} depth={received['depth']} " + f"lidar={received['lidar']}" + ) + + except KeyboardInterrupt: + print("\n[integration] Interrupted by user") + + finally: + running = False + # Send zero velocity (safety stop) + try: + stop_twist = Twist() + stop_pkt = encode_lcm_packet(CH_CMD_VEL, stop_twist.lcm_encode()) + send_sock.sendto(stop_pkt, (MCAST_GRP, MCAST_PORT)) + except Exception: + pass + + recv_sock.close() + send_sock.close() + print("[integration] Done.") + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/misc/DimSim/dimos-cli/test/lcm_cross_test.py b/misc/DimSim/dimos-cli/test/lcm_cross_test.py new file mode 100644 index 0000000000..d1694f0d31 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/lcm_cross_test.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Quick test: can Python receive LCM messages from Deno?""" + +import sys +import time + +sys.path.insert(0, "/Users/viswajitnair/Desktop/4Wall.nosync/Dimensional/dimos") +import lcm + +received = {"count": 0} + + +def handler(channel, data): + received["count"] += 1 + print(f"[py] Got message #{received['count']} on {channel} ({len(data)} bytes)") + + +lc = lcm.LCM("udpm://239.255.76.67:7667?ttl=0") +lc.subscribe(".*lcm_cross_test.*", handler) + +print("[py] Listening for LCM messages on /lcm_cross_test...") +print("[py] Waiting 15s for Deno publisher...\n") + +deadline = time.time() + 15 +while time.time() < deadline: + lc.handle_timeout(500) + if received["count"] >= 3: + print(f"\n[py] SUCCESS: received {received['count']} messages from Deno!") + sys.exit(0) + +print(f"\n[py] FAIL: only received {received['count']} messages") +sys.exit(1) diff --git a/misc/DimSim/dimos-cli/test/lcm_cross_test.ts b/misc/DimSim/dimos-cli/test/lcm_cross_test.ts new file mode 100644 index 0000000000..fde302dbc0 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/lcm_cross_test.ts @@ -0,0 +1,37 @@ +#!/usr/bin/env -S deno run --allow-all --unstable-net +/** + * Quick test: can Deno send LCM messages that Python receives? + * Publishes raw packets on /lcm_cross_test channel. + */ +import { LCM } from "@dimos/lcm"; +import { geometry_msgs } from "@dimos/msgs"; + +const lcm = new LCM(); +await lcm.start(); + +// Also subscribe to see if we get Python's messages +lcm.subscribe("/lcm_cross_test", geometry_msgs.Twist, (msg: any) => { + console.log(`[deno] Got message back:`, msg.data?.linear); +}); + +console.log("[deno] Using patched @dimos/lcm with joinMulticastV4"); +console.log("[deno] Publishing test messages on /lcm_cross_test...\n"); + +let count = 0; +const interval = setInterval(async () => { + count++; + const twist = new geometry_msgs.Twist({ + linear: new geometry_msgs.Vector3({ x: count, y: 0, z: 0 }), + angular: new geometry_msgs.Vector3({ x: 0, y: 0, z: 0 }), + }); + await lcm.publish("/lcm_cross_test", twist); + console.log(`[deno] Sent message #${count}`); + + if (count >= 10) { + clearInterval(interval); + console.log("\n[deno] Done publishing. Waiting 2s for responses..."); + setTimeout(() => Deno.exit(0), 2000); + } +}, 500); + +await lcm.run(); diff --git a/misc/DimSim/dimos-cli/test/loopback.ts b/misc/DimSim/dimos-cli/test/loopback.ts new file mode 100644 index 0000000000..5c7392cee3 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/loopback.ts @@ -0,0 +1,120 @@ +#!/usr/bin/env -S deno run --allow-all --unstable-net + +/** + * Loopback Test — Verify bridge + DimSim sensor pipeline end-to-end. + * + * Connects to the bridge server's WebSocket (same as the browser does) and: + * 1. Publishes /cmd_vel Twist packets (simulates dimos nav stack sending velocity commands) + * 2. Listens for /odom, /camera/image, /camera/depth, /lidar/points packets back + * 3. Reports what it receives + * + * Usage: + * 1. Start bridge: deno run --allow-all --unstable-net dimos-cli/cli.ts dev + * 2. Open http://localhost:8090 in Chrome + * 3. Run this: deno run --allow-all --unstable-net dimos-cli/test/loopback.ts + */ + +import { encodePacket, decodePacket, geometry_msgs, std_msgs } from "@dimos/msgs"; + +const WS_URL = Deno.args.find((_a, i, arr) => arr[i - 1] === "--ws") || "ws://localhost:8090"; + +console.log(`[loopback] Connecting to bridge at ${WS_URL}...`); + +const ws = new WebSocket(WS_URL); +ws.binaryType = "arraybuffer"; + +const received = { odom: 0, image: 0, depth: 0, lidar: 0 }; +let tick = 0; +let cmdInterval: ReturnType | null = null; + +ws.onopen = () => { + console.log("[loopback] Connected to bridge WebSocket"); + + // Publish cmd_vel at 10Hz (agent walks forward while slowly turning) + cmdInterval = setInterval(() => { + const twist = new geometry_msgs.Twist({ + linear: new geometry_msgs.Vector3({ x: 0, y: 0, z: 0.5 }), // forward 0.5 m/s + angular: new geometry_msgs.Vector3({ x: 0, y: 0.3, z: 0 }), // turn 0.3 rad/s + }); + + const packet = encodePacket("/cmd_vel#geometry_msgs.Twist", twist); + ws.send(packet); + tick++; + + if (tick <= 3 || tick % 20 === 0) { + console.log(`[loopback] Sent cmd_vel #${tick}: linZ=0.5 angY=0.3`); + } + }, 100); +}; + +ws.onmessage = (event: MessageEvent) => { + if (!(event.data instanceof ArrayBuffer)) return; + + try { + const { channel, data } = decodePacket(new Uint8Array(event.data)); + + if (channel.includes("/odom")) { + received.odom++; + if (received.odom <= 3 || received.odom % 10 === 0) { + const pos = data.pose?.position; + const posStr = pos ? `x=${pos.x.toFixed(2)} y=${pos.y.toFixed(2)} z=${pos.z.toFixed(2)}` : "?"; + console.log(`[loopback] Got odom #${received.odom}: ${posStr}`); + } + } else if (channel.includes("/camera/image")) { + received.image++; + if (received.image <= 3 || received.image % 10 === 0) { + console.log(`[loopback] Got RGB #${received.image}`); + } + } else if (channel.includes("/camera/depth")) { + received.depth++; + if (received.depth <= 3 || received.depth % 10 === 0) { + console.log(`[loopback] Got depth #${received.depth}`); + } + } else if (channel.includes("/lidar/points")) { + received.lidar++; + if (received.lidar <= 3 || received.lidar % 10 === 0) { + console.log(`[loopback] Got LiDAR #${received.lidar}`); + } + } + } catch { + // not a valid LCM packet + } +}; + +ws.onerror = (e) => { + console.error("[loopback] WebSocket error:", e); +}; + +ws.onclose = () => { + console.log("[loopback] WebSocket closed"); + if (cmdInterval) clearInterval(cmdInterval); +}; + +// Status report every 5s +const statusInterval = setInterval(() => { + console.log(`[loopback] STATUS: cmd_sent=${tick} odom=${received.odom} rgb=${received.image} depth=${received.depth} lidar=${received.lidar}`); + if (received.odom > 0 && received.image > 0 && received.depth > 0 && received.lidar > 0) { + console.log("\n[loopback] SUCCESS: All channels working! (odom + RGB + depth + LiDAR)"); + cleanup(0); + } +}, 5000); + +// Timeout after 60s +setTimeout(() => { + console.log("\n[loopback] TIMEOUT after 60s"); + console.log(`[loopback] Final: cmd_sent=${tick} odom=${received.odom} rgb=${received.image} depth=${received.depth} lidar=${received.lidar}`); + if (received.image === 0 && received.depth === 0 && received.lidar === 0) { + console.log("[loopback] No sensor data received. Make sure:"); + console.log(" 1. Bridge server is running: deno run --allow-all dimos-cli/cli.ts dev"); + console.log(" 2. Browser is open at http://localhost:8090"); + console.log(" 3. DimSim loaded the scene (check browser console for [dimos] logs)"); + } + cleanup(1); +}, 60000); + +function cleanup(code: number) { + if (cmdInterval) clearInterval(cmdInterval); + clearInterval(statusInterval); + ws.close(); + Deno.exit(code); +} diff --git a/misc/DimSim/dimos-cli/test/rubrics_test.ts b/misc/DimSim/dimos-cli/test/rubrics_test.ts new file mode 100644 index 0000000000..9d40c90679 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/rubrics_test.ts @@ -0,0 +1,164 @@ +/** + * Unit tests for eval rubrics — pure scoring functions, no browser needed. + * + * cd DimSim && deno test dimos-cli/test/rubrics_test.ts + */ + +import { assertEquals, assertAlmostEquals } from "jsr:@std/assert"; +// Import from source — rubrics are pure TS, no DOM deps +import { + scoreObjectDistance, + scoreRadiusContains, + type SceneState, +} from "../../src/dimos/rubrics.ts"; + +// -- helpers ------------------------------------------------------------------ + +function mkScene(assets: { title: string; x: number; y: number; z: number }[], agentPos?: { x: number; y: number; z: number }): SceneState { + return { + assets: assets.map((a) => ({ + title: a.title, + transform: { x: a.x, y: a.y, z: a.z }, + })), + agentPos, + }; +} + +// -- objectDistance (existing, sanity check) ----------------------------------- + +Deno.test("objectDistance: pass when agent is close to target", () => { + const scene = mkScene([{ title: "Television", x: 5, y: 0, z: 3 }], { x: 5, y: 0, z: 3.3 }); + const result = scoreObjectDistance({ object: "agent", target: "television", thresholdM: 0.5 }, scene); + assertEquals(result.pass, true); +}); + +Deno.test("objectDistance: fail when agent is far from target", () => { + const scene = mkScene([{ title: "Television", x: 5, y: 0, z: 3 }], { x: 0, y: 0, z: 0 }); + const result = scoreObjectDistance({ object: "agent", target: "television", thresholdM: 2.0 }, scene); + assertEquals(result.pass, false); +}); + +Deno.test("objectDistance: fail when target not found", () => { + const scene = mkScene([], { x: 0, y: 0, z: 0 }); + const result = scoreObjectDistance({ object: "agent", target: "nonexistent", thresholdM: 1.0 }, scene); + assertEquals(result.pass, false); + assertEquals(result.distanceM, Infinity); +}); + +// -- radiusContains ----------------------------------------------------------- + +Deno.test("radiusContains: pass when agent is within centroid radius", () => { + // Kitchen objects form a triangle around (5, 0, 5) + const scene = mkScene( + [ + { title: "Refrigerator", x: 4, y: 0, z: 4 }, + { title: "Stove", x: 6, y: 0, z: 4 }, + { title: "Sink", x: 5, y: 0, z: 7 }, + ], + { x: 5, y: 0, z: 5 }, // agent near centroid + ); + const result = scoreRadiusContains( + { targets: ["refrigerator", "stove", "sink"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, true); + assertEquals(result.foundTargets.length, 3); + assertEquals(result.missingTargets.length, 0); + // Centroid should be (5, 0, 5) + assertAlmostEquals(result.centroid.x, 5, 0.01); + assertAlmostEquals(result.centroid.z, 5, 0.01); +}); + +Deno.test("radiusContains: fail when agent is far from centroid", () => { + const scene = mkScene( + [ + { title: "Refrigerator", x: 4, y: 0, z: 4 }, + { title: "Stove", x: 6, y: 0, z: 4 }, + { title: "Sink", x: 5, y: 0, z: 7 }, + ], + { x: 20, y: 0, z: 20 }, // agent far away + ); + const result = scoreRadiusContains( + { targets: ["refrigerator", "stove", "sink"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, false); + assertEquals(result.foundTargets.length, 3); +}); + +Deno.test("radiusContains: partial match — 2 of 3 targets found, still scores", () => { + const scene = mkScene( + [ + { title: "Refrigerator", x: 4, y: 0, z: 4 }, + { title: "Stove", x: 6, y: 0, z: 4 }, + // Sink is missing + ], + { x: 5, y: 0, z: 4 }, // agent at centroid of found targets + ); + const result = scoreRadiusContains( + { targets: ["refrigerator", "stove", "sink"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, true); + assertEquals(result.foundTargets.length, 2); + assertEquals(result.missingTargets, ["sink"]); + // Centroid should be (5, 0, 4) + assertAlmostEquals(result.centroid.x, 5, 0.01); + assertAlmostEquals(result.centroid.z, 4, 0.01); +}); + +Deno.test("radiusContains: fail when no targets found", () => { + const scene = mkScene([], { x: 0, y: 0, z: 0 }); + const result = scoreRadiusContains( + { targets: ["refrigerator", "stove"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, false); + assertEquals(result.distanceM, Infinity); + assertEquals(result.missingTargets.length, 2); +}); + +Deno.test("radiusContains: fail when agent position not available", () => { + const scene = mkScene([{ title: "Refrigerator", x: 4, y: 0, z: 4 }]); + const result = scoreRadiusContains( + { targets: ["refrigerator"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, false); +}); + +Deno.test("radiusContains: single target degrades to point distance", () => { + const scene = mkScene( + [{ title: "Bed", x: 10, y: 0, z: 10 }], + { x: 10, y: 0, z: 11 }, // 1m away + ); + const result = scoreRadiusContains( + { targets: ["bed"], radiusM: 2.0 }, + scene, + ); + assertEquals(result.pass, true); + assertAlmostEquals(result.distanceM, 1.0, 0.01); + assertEquals(result.foundTargets, ["bed"]); +}); + +Deno.test("radiusContains: exact threshold boundary", () => { + const scene = mkScene( + [ + { title: "A", x: 0, y: 0, z: 0 }, + { title: "B", x: 2, y: 0, z: 0 }, + ], + { x: 1, y: 0, z: 3 }, // centroid at (1,0,0), agent at (1,0,3) → dist = 3.0 + ); + const result = scoreRadiusContains( + { targets: ["a", "b"], radiusM: 3.0 }, + scene, + ); + assertEquals(result.pass, true); // exactly at boundary + assertAlmostEquals(result.distanceM, 3.0, 0.01); +}); + +Deno.test("radiusContains: empty targets array fails", () => { + const scene = mkScene([], { x: 0, y: 0, z: 0 }); + const result = scoreRadiusContains({ targets: [], radiusM: 3.0 }, scene); + assertEquals(result.pass, false); +}); diff --git a/misc/DimSim/dimos-cli/test/scene_editor_test.py b/misc/DimSim/dimos-cli/test/scene_editor_test.py new file mode 100644 index 0000000000..33466b426d --- /dev/null +++ b/misc/DimSim/dimos-cli/test/scene_editor_test.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Integration test for SceneEditor — script execution engine. + +Requires dimsim running headless on port 8090: + DIMSIM_HEADLESS=1 dimsim dev + +Then: + python dimos-cli/test/scene_editor_test.py +""" + +import json +import sys +import time +import uuid + +import websocket + +PORT = 8090 +WS_URL = f"ws://localhost:{PORT}?ch=control" + + +def send_exec(ws: websocket.WebSocket, code: str, timeout: float = 10) -> dict: + """Send an exec command and wait for the execResult.""" + msg_id = str(uuid.uuid4())[:8] + ws.send(json.dumps({"type": "exec", "id": msg_id, "code": code})) + ws.settimeout(timeout) + deadline = time.time() + timeout + while time.time() < deadline: + raw = ws.recv() + if isinstance(raw, bytes): + continue + msg = json.loads(raw) + if msg.get("type") == "execResult" and msg.get("id") == msg_id: + return msg + raise TimeoutError(f"No execResult for {msg_id} after {timeout}s") + + +def wait_for_scene(ws: websocket.WebSocket, timeout: float = 60) -> bool: + """Wait until sceneEditor is responding (browser has loaded).""" + deadline = time.time() + timeout + while time.time() < deadline: + try: + result = send_exec(ws, "return 'ready'", timeout=5) + if result.get("success") and result.get("result") == "ready": + return True + except Exception: + time.sleep(2) + return False + + +def test_basic_exec(ws): + """Test: basic JS evaluation returns a value.""" + print(" [1] Basic exec: return 1 + 1") + r = send_exec(ws, "return 1 + 1") + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"] == 2, f"expected 2, got {r['result']}" + print(f" PASS — result: {r['result']}") + + +def test_scene_access(ws): + """Test: can access scene.children.""" + print(" [2] Scene access: scene.children.length") + r = send_exec(ws, "return scene.children.length") + assert r["success"], f"exec failed: {r.get('error')}" + assert isinstance(r["result"], int) and r["result"] > 0, f"unexpected: {r['result']}" + print(f" PASS — scene has {r['result']} children") + + +def test_three_access(ws): + """Test: THREE namespace available, can create geometry.""" + print(" [3] THREE access: create Vector3") + r = send_exec(ws, "const v = new THREE.Vector3(1, 2, 3); return {x: v.x, y: v.y, z: v.z}") + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"] == {"x": 1, "y": 2, "z": 3}, f"unexpected: {r['result']}" + print(f" PASS — Vector3: {r['result']}") + + +def test_add_primitive(ws): + """Test: add a red box to the scene via script.""" + print(" [4] Add primitive: red box at (3, 1, 3)") + code = """ +const geo = new THREE.BoxGeometry(1, 1, 1); +const mat = new THREE.MeshStandardMaterial({color: 0xff0000}); +const mesh = new THREE.Mesh(geo, mat); +mesh.name = "test-red-box"; +mesh.position.set(3, 1, 3); +scene.add(mesh); +return {name: mesh.name, pos: {x: mesh.position.x, y: mesh.position.y, z: mesh.position.z}} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-red-box", f"unexpected: {r['result']}" + print(f" PASS — added: {r['result']}") + + # Verify it's in the scene + r2 = send_exec(ws, 'return scene.getObjectByName("test-red-box") !== null') + assert r2["success"] and r2["result"] is True, "Box not found in scene" + print(" PASS — verified in scene") + + +def test_load_gltf(ws): + """Test: load robot.glb via loadGLTF helper.""" + print(" [5] Load GLTF: /agent-model/robot.glb") + code = """ +const gltf = await loadGLTF('/agent-model/robot.glb'); +gltf.scene.name = "test-loaded-robot"; +gltf.scene.position.set(5, 1, 5); +gltf.scene.scale.set(2, 2, 2); +scene.add(gltf.scene); +return {name: gltf.scene.name, childCount: gltf.scene.children.length} +""" + r = send_exec(ws, code, timeout=15) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-loaded-robot", f"unexpected: {r['result']}" + print(f" PASS — loaded: {r['result']}") + + # Verify it's in the scene + r2 = send_exec(ws, 'return scene.getObjectByName("test-loaded-robot") !== null') + assert r2["success"] and r2["result"] is True, "Loaded robot not found in scene" + print(" PASS — verified in scene") + + +def test_error_handling(ws): + """Test: syntax/runtime errors returned gracefully.""" + print(" [6] Error handling: bad code") + r = send_exec(ws, "this is not valid javascript!!!") + assert not r["success"], "Expected failure" + assert "error" in r, "Expected error field" + print(f" PASS — error caught: {r['error'][:60]}") + + +def test_async_exec(ws): + """Test: top-level await works.""" + print(" [7] Async exec: await Promise") + code = """ +const val = await new Promise(resolve => setTimeout(() => resolve(42), 100)); +return val +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"] == 42, f"expected 42, got {r['result']}" + print(f" PASS — async result: {r['result']}") + + +def test_agent_access(ws): + """Test: can read agent position.""" + print(" [8] Agent access: getPosition") + r = send_exec(ws, "const p = agent.getPosition(); return {x: p[0], y: p[1], z: p[2]}") + assert r["success"], f"exec failed: {r.get('error')}" + assert "x" in r["result"], f"unexpected: {r['result']}" + print( + f" PASS — agent at ({r['result']['x']:.2f}, {r['result']['y']:.2f}, {r['result']['z']:.2f})" + ) + + +def test_add_light(ws): + """Test: add a point light to the scene.""" + print(" [9] Add light: PointLight at (0, 5, 0)") + code = """ +const light = new THREE.PointLight(0xffff00, 2, 50); +light.name = "test-point-light"; +light.position.set(0, 5, 0); +scene.add(light); +return {name: light.name, color: light.color.getHex(), intensity: light.intensity} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-point-light", f"unexpected: {r['result']}" + assert r["result"]["intensity"] == 2, f"unexpected intensity: {r['result']}" + print(f" PASS — added: {r['result']}") + + # Verify in scene + r2 = send_exec(ws, 'return scene.getObjectByName("test-point-light") !== null') + assert r2["success"] and r2["result"] is True, "Light not found in scene" + print(" PASS — verified in scene") + + +def test_modify_object(ws): + """Test: move an existing object (the red box from test_add_primitive).""" + print(" [10] Modify object: move test-red-box to (7, 2, 7)") + code = """ +const box = scene.getObjectByName("test-red-box"); +if (!box) return {error: "box not found"}; +box.position.set(7, 2, 7); +box.scale.set(2, 2, 2); +box.material.color.setHex(0x00ff00); +return { + name: box.name, + pos: {x: box.position.x, y: box.position.y, z: box.position.z}, + scale: {x: box.scale.x, y: box.scale.y, z: box.scale.z}, + color: box.material.color.getHex() +} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["pos"] == {"x": 7, "y": 2, "z": 7}, f"position wrong: {r['result']}" + assert r["result"]["scale"] == {"x": 2, "y": 2, "z": 2}, f"scale wrong: {r['result']}" + assert r["result"]["color"] == 0x00FF00, f"color wrong: {r['result']}" + print(f" PASS — modified: {r['result']}") + + +def test_remove_object(ws): + """Test: remove the test-red-box we added earlier.""" + print(" [11] Remove object: remove test-red-box") + remove_code = """ +const box = scene.getObjectByName("test-red-box"); +if (!box) return {error: "box not found"}; +if (box.geometry) box.geometry.dispose(); +if (box.material) box.material.dispose(); +box.name = ""; +scene.remove(box); +return {removed: "test-red-box"} +""" + r = send_exec(ws, remove_code) + assert r["success"], f"remove failed: {r.get('error')}" + print(f" PASS — removed: {r['result']}") + # Note: verification that test-red-box is gone happens in test_query_scene (test 12) + + +def test_query_scene(ws): + """Test: query scene objects by traversal.""" + print(" [12] Query scene: list named objects") + code = """ +const named = []; +scene.traverse(obj => { + if (obj.name && obj.name.startsWith("test-")) { + named.push({name: obj.name, type: obj.type}); + } +}); +return named +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + names = [o["name"] for o in r["result"]] + assert "test-point-light" in names, f"Light not found: {names}" + assert "test-loaded-robot" in names, f"Robot not found: {names}" + assert "test-red-box" not in names, f"Removed box still found: {names}" + print(f" PASS — found {len(r['result'])} test objects: {names}") + + +def test_add_sphere(ws): + """Test: add a sphere primitive (second geometry type).""" + print(" [13] Add primitive: blue sphere at (-3, 1.5, 0)") + code = """ +const geo = new THREE.SphereGeometry(0.75, 32, 32); +const mat = new THREE.MeshStandardMaterial({color: 0x0088ff, metalness: 0.3, roughness: 0.4}); +const mesh = new THREE.Mesh(geo, mat); +mesh.name = "test-blue-sphere"; +mesh.position.set(-3, 1.5, 0); +scene.add(mesh); +return {name: mesh.name, pos: {x: mesh.position.x, y: mesh.position.y, z: mesh.position.z}} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-blue-sphere", f"unexpected: {r['result']}" + print(f" PASS — added: {r['result']}") + + +def test_add_directional_light(ws): + """Test: add a directional light with shadow.""" + print(" [14] Add light: DirectionalLight") + code = """ +const dlight = new THREE.DirectionalLight(0xffffff, 1.5); +dlight.name = "test-dir-light"; +dlight.position.set(10, 10, 10); +dlight.castShadow = true; +scene.add(dlight); +return {name: dlight.name, intensity: dlight.intensity, castShadow: dlight.castShadow} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-dir-light", f"unexpected: {r['result']}" + assert r["result"]["castShadow"] is True, "Shadow not enabled" + print(f" PASS — added: {r['result']}") + + +def test_add_collider_box(ws): + """Test: add a box collider to a mesh (explicit shape).""" + print(" [15] Physics: addCollider (box)") + code = """ +const geo = new THREE.BoxGeometry(1, 1, 1); +const mat = new THREE.MeshStandardMaterial({color: 0xff8800}); +const mesh = new THREE.Mesh(geo, mat); +mesh.name = "test-physics-box"; +mesh.position.set(0, 1, 0); +scene.add(mesh); +const info = addCollider(mesh, "box"); +return info +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["shape"] == "box", f"unexpected shape: {r['result']}" + assert "uuid" in r["result"], f"no uuid: {r['result']}" + print(f" PASS — collider: {r['result']}") + + +def test_add_collider_sphere(ws): + """Test: add a sphere collider to a mesh.""" + print(" [16] Physics: addCollider (sphere)") + code = """ +const mesh = scene.getObjectByName("test-blue-sphere"); +if (!mesh) return {error: "sphere not found"}; +const info = addCollider(mesh, "sphere"); +return info +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["shape"] == "sphere", f"unexpected shape: {r['result']}" + print(f" PASS — collider: {r['result']}") + + +def test_remove_collider(ws): + """Test: remove a previously added collider.""" + print(" [17] Physics: removeCollider") + code = """ +const mesh = scene.getObjectByName("test-physics-box"); +if (!mesh) return {error: "box not found"}; +const removed = removeCollider(mesh); +return {removed} +""" + r = send_exec(ws, code) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["removed"] is True, f"collider not removed: {r['result']}" + print(f" PASS — removed: {r['result']}") + + # Verify double-remove returns false + r2 = send_exec( + ws, + """ +const mesh = scene.getObjectByName("test-physics-box"); +return {removed: removeCollider(mesh)} +""", + ) + assert r2["success"] and r2["result"]["removed"] is False, "Double remove should return false" + print(" PASS — double remove returns false") + + +def test_add_collider_trimesh(ws): + """Test: add a trimesh collider to the loaded robot.""" + print(" [18] Physics: addCollider (trimesh)") + code = """ +const robot = scene.getObjectByName("test-loaded-robot"); +if (!robot) return {error: "robot not found"}; +const info = addCollider(robot, "trimesh"); +return info +""" + r = send_exec(ws, code, timeout=15) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["shape"] == "trimesh", f"unexpected shape: {r['result']}" + print(f" PASS — collider: {r['result']}") + + +def test_add_npc(ws): + """Test: addNPC with walk animation.""" + print(" [19] NPC: addNPC (Soldier, Walk)") + code = """ +const npc = await addNPC({ + url: '/local-assets/Soldier.glb', + name: 'test-npc-soldier', + position: { x: 5, y: 0, z: 5 }, + rotation: Math.PI / 4, + scale: 1.0, + animation: 'Walk', + collider: true, +}); +return npc +""" + r = send_exec(ws, code, timeout=15) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-npc-soldier", f"unexpected: {r['result']}" + assert "Walk" in r["result"]["animations"], f"no Walk anim: {r['result']}" + assert r["result"]["activeAnimation"] == "Walk", f"wrong anim: {r['result']}" + assert r["result"]["collider"] is not None, "no collider" + print(f" PASS — NPC: {r['result']['name']}, anims: {r['result']['animations']}") + + +def test_add_npc_idle(ws): + """Test: addNPC with idle animation (by index).""" + print(" [20] NPC: addNPC (Soldier, Idle by index)") + code = """ +const npc = await addNPC({ + url: '/local-assets/Soldier.glb', + name: 'test-npc-idle', + position: { x: -5, y: 0, z: -5 }, + animation: 0, +}); +return npc +""" + r = send_exec(ws, code, timeout=15) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["name"] == "test-npc-idle", f"unexpected: {r['result']}" + assert r["result"]["activeAnimation"] == "Idle", f"wrong anim: {r['result']}" + print(f" PASS — NPC idle: {r['result']['activeAnimation']}") + + +def test_remove_npc(ws): + """Test: removeNPC removes NPC and cleans up.""" + print(" [21] NPC: removeNPC") + r = send_exec( + ws, + """ +removeNPC('test-npc-idle'); +// Check immediately in same exec — name was cleared by removeNPC +const npcs = []; +scene.traverse(obj => { if (obj.name === 'test-npc-idle') npcs.push(obj.name); }); +return { removed: true, remaining: npcs.length } +""", + ) + assert r["success"], f"exec failed: {r.get('error')}" + assert r["result"]["remaining"] == 0, f"NPC still found: {r['result']}" + print(f" PASS — removed and verified: {r['result']}") + + +def test_embodiment_config(ws): + """Test: embodiment config is accessible from scene.""" + print(" [22] Embodiment: config loaded") + r = send_exec(ws, "return window.currentEmbodiment || null") + assert r["success"], f"exec failed: {r.get('error')}" + cfg = r["result"] + if cfg is None: + print(" SKIP — not in dimos mode (embodiment only set in dimos boot)") + return + assert "radius" in cfg, f"no radius: {cfg}" + assert "halfHeight" in cfg, f"no halfHeight: {cfg}" + assert "type" in cfg, f"no type: {cfg}" + print( + f" PASS — embodiment: type={cfg['type']} radius={cfg['radius']} halfHeight={cfg['halfHeight']}" + ) + + +def main(): + print(f"Connecting to {WS_URL}...") + ws = websocket.WebSocket() + ws.connect(WS_URL) + + print("Waiting for scene to load...") + if not wait_for_scene(ws, timeout=90): + print("FAIL: scene not ready after 90s") + sys.exit(1) + print("Scene ready.\n") + + tests = [ + test_basic_exec, + test_scene_access, + test_three_access, + test_add_primitive, + test_load_gltf, + test_error_handling, + test_async_exec, + test_agent_access, + test_add_light, + test_modify_object, + test_remove_object, + test_query_scene, + test_add_sphere, + test_add_directional_light, + test_add_collider_box, + test_add_collider_sphere, + test_remove_collider, + test_add_collider_trimesh, + test_add_npc, + test_add_npc_idle, + test_remove_npc, + test_embodiment_config, + ] + + passed = 0 + failed = 0 + for test in tests: + try: + test(ws) + passed += 1 + except Exception as e: + print(f" FAIL — {e}") + failed += 1 + + ws.close() + print(f"\n{'=' * 50}") + print(f"Results: {passed} passed, {failed} failed out of {len(tests)}") + sys.exit(1 if failed else 0) + + +if __name__ == "__main__": + main() diff --git a/misc/DimSim/dimos-cli/test/smoke.ts b/misc/DimSim/dimos-cli/test/smoke.ts new file mode 100644 index 0000000000..636d5b7410 --- /dev/null +++ b/misc/DimSim/dimos-cli/test/smoke.ts @@ -0,0 +1,75 @@ +#!/usr/bin/env -S deno run --allow-all --unstable-net + +/** + * Smoke Test — Verify DimSim sensor data flows over LCM. + * + * 1. Subscribes to /camera/image, /camera/depth, /lidar/points + * 2. Publishes test /odom messages + * 3. Waits for all sensor types to be received + * 4. Exits 0 on success, 1 on timeout + * + * Requires: DimSim running in dimos mode + bridge server active. + */ + +import { LCM } from "@dimos/lcm"; +import { geometry_msgs, sensor_msgs, std_msgs } from "@dimos/msgs"; + +const TIMEOUT_MS = 30000; + +const lcm = new LCM(); +await lcm.start(); + +let receivedImage = false; +let receivedDepth = false; +let receivedLidar = false; + +lcm.subscribe("/camera/image", sensor_msgs.Image, (msg: { data: { width: number; height: number; encoding: string } }) => { + console.log(`[smoke] Got RGB: ${msg.data.width}x${msg.data.height} enc=${msg.data.encoding}`); + receivedImage = true; +}); + +lcm.subscribe("/camera/depth", sensor_msgs.Image, (msg: { data: { width: number; height: number; encoding: string } }) => { + console.log(`[smoke] Got depth: ${msg.data.width}x${msg.data.height} enc=${msg.data.encoding}`); + receivedDepth = true; +}); + +lcm.subscribe("/lidar/points", sensor_msgs.PointCloud2, (msg: { data: { width: number; point_step: number } }) => { + console.log(`[smoke] Got LiDAR: ${msg.data.width} points, ${msg.data.point_step} bytes/pt`); + receivedLidar = true; +}); + +// Publish test odom at 10 Hz to drive the agent +const odomInterval = setInterval(async () => { + const pose = new geometry_msgs.PoseStamped({ + header: new std_msgs.Header({ + stamp: new std_msgs.Time({ sec: 0, nsec: 0 }), + frame_id: "map", + }), + pose: new geometry_msgs.Pose({ + position: new geometry_msgs.Point({ x: 0, y: 0.5, z: 0 }), + orientation: new geometry_msgs.Quaternion({ x: 0, y: 0, z: 0, w: 1 }), + }), + }); + await lcm.publish("/odom", pose); +}, 100); + +// Poll for results +const deadline = Date.now() + TIMEOUT_MS; +const checkInterval = setInterval(() => { + if (receivedImage && receivedDepth && receivedLidar) { + clearInterval(odomInterval); + clearInterval(checkInterval); + console.log("\n[smoke] PASS: All sensor types received"); + Deno.exit(0); + } + if (Date.now() > deadline) { + clearInterval(odomInterval); + clearInterval(checkInterval); + console.error("\n[smoke] FAIL: Timeout. Missing sensors:", { + image: receivedImage, + depth: receivedDepth, + lidar: receivedLidar, + }); + Deno.exit(1); + } +}, 500); diff --git a/misc/DimSim/dimos-cli/vendor/lcm/lcm.ts b/misc/DimSim/dimos-cli/vendor/lcm/lcm.ts new file mode 100644 index 0000000000..f4aa7b2c96 --- /dev/null +++ b/misc/DimSim/dimos-cli/vendor/lcm/lcm.ts @@ -0,0 +1,236 @@ +// LCM Main Class - Pure TypeScript Implementation (vendored from @dimos/lcm@0.2.0) + +import type { + LCMOptions, + LCMMessage, + MessageClass, + ParsedUrl, + Subscription, + SubscriptionHandler, + PacketHandler, + PacketSubscription, +} from "./types.ts"; +import { MAX_SMALL_MESSAGE, SHORT_HEADER_SIZE } from "./types.ts"; +import { parseUrl } from "./url.ts"; +import { + UdpMulticastSocket, + FragmentReassembler, + encodeSmallMessage, + encodeFragmentedMessage, + decodePacket, +} from "./transport.ts"; + +const textEncoder = new TextEncoder(); + +export class LCM { + private readonly config: ParsedUrl; + private socket: UdpMulticastSocket | null = null; + private reassembler = new FragmentReassembler(); + private subscriptions: Subscription[] = []; + private packetSubscriptions: PacketSubscription[] = []; + private sequenceNumber = 0; + private running = false; + private messageQueue: LCMMessage[] = []; + + constructor(url?: string); + constructor(options?: LCMOptions); + constructor(urlOrOptions?: string | LCMOptions) { + if (typeof urlOrOptions === "string") { + this.config = parseUrl(urlOrOptions); + } else { + this.config = parseUrl(urlOrOptions?.url); + if (urlOrOptions?.ttl !== undefined) { + this.config.ttl = urlOrOptions.ttl; + } + if (urlOrOptions?.iface !== undefined) { + this.config.iface = urlOrOptions.iface; + } + } + } + + async start(): Promise { + if (this.running) return; + this.socket = new UdpMulticastSocket(this.config); + this.running = true; + await this.socket.listen((data, _addr) => { + this.handlePacket(data); + }); + } + + stop(): void { + this.running = false; + if (this.socket) { + this.socket.close(); + this.socket = null; + } + } + + subscribeRaw(channelPattern: string, handler: SubscriptionHandler): () => void { + const pattern = this.channelToRegex(channelPattern); + const subscription: Subscription = { + channel: channelPattern, + pattern, + handler: handler as SubscriptionHandler, + }; + this.subscriptions.push(subscription); + return () => { + const idx = this.subscriptions.indexOf(subscription); + if (idx !== -1) this.subscriptions.splice(idx, 1); + }; + } + + subscribePacket(handler: PacketHandler): () => void; + subscribePacket(channelPattern: string, handler: PacketHandler): () => void; + subscribePacket(patternOrHandler: string | PacketHandler, maybeHandler?: PacketHandler): () => void { + const pattern = typeof patternOrHandler === "string" + ? this.channelToRegex(patternOrHandler) + : null; + const handler = typeof patternOrHandler === "function" + ? patternOrHandler + : maybeHandler!; + const subscription: PacketSubscription = { pattern, handler }; + this.packetSubscriptions.push(subscription); + return () => { + const idx = this.packetSubscriptions.indexOf(subscription); + if (idx !== -1) this.packetSubscriptions.splice(idx, 1); + }; + } + + subscribe(channel: string, msgClass: MessageClass, handler: SubscriptionHandler): () => void { + const typeName = (msgClass as unknown as { _NAME: string })._NAME; + const fullChannel = channel.includes("#") ? channel : `${channel}#${typeName}`; + const pattern = this.channelToRegex(fullChannel); + const subscription: Subscription = { + channel: fullChannel, + pattern, + handler: handler as SubscriptionHandler, + msgClass: msgClass as MessageClass, + }; + this.subscriptions.push(subscription); + return () => { + const idx = this.subscriptions.indexOf(subscription); + if (idx !== -1) this.subscriptions.splice(idx, 1); + }; + } + + async publishRaw(channel: string, data: Uint8Array): Promise { + if (!this.socket) throw new Error("LCM not started. Call start() first."); + const channelBytes = textEncoder.encode(channel); + const totalSize = SHORT_HEADER_SIZE + channelBytes.length + 1 + data.length; + const seq = this.sequenceNumber++; + if (totalSize <= MAX_SMALL_MESSAGE) { + const packet = encodeSmallMessage(channel, data, seq); + await this.socket.send(packet); + } else { + const fragments = encodeFragmentedMessage(channel, data, seq); + for (const fragment of fragments) { + await this.socket.send(fragment); + } + } + } + + async publish(channel: string, msg: T): Promise { + const data = msg.encode(); + const typeName = (msg.constructor as unknown as { _NAME?: string })._NAME; + const fullChannel = typeName && !channel.includes("#") + ? `${channel}#${typeName}` + : channel; + await this.publishRaw(fullChannel, data); + } + + async publishPacket(packet: Uint8Array): Promise { + if (!this.socket) throw new Error("LCM not started. Call start() first."); + await this.socket.send(packet); + } + + handle(timeoutMs: number = 0): number { + const messages = this.messageQueue.splice(0); + for (const msg of messages) { + this.dispatchMessage(msg); + } + return messages.length; + } + + async handleAsync(timeoutMs: number = 100): Promise { + const startTime = Date.now(); + while (this.messageQueue.length === 0) { + if (timeoutMs >= 0 && Date.now() - startTime >= timeoutMs) break; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + return this.handle(); + } + + async run(callback?: () => void | Promise): Promise { + while (this.running) { + await this.handleAsync(100); + if (callback) await callback(); + } + } + + private channelToRegex(pattern: string): RegExp { + const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, "\\$&"); + const regexStr = "^" + escaped.replace(/\*/g, ".*") + "$"; + return new RegExp(regexStr); + } + + private handlePacket(data: Uint8Array): void { + const decoded = decodePacket(data); + if (!decoded) return; + + const channel = decoded.type === "small" ? decoded.channel : decoded.channel; + + if (channel) { + for (const sub of this.packetSubscriptions) { + if (!sub.pattern || sub.pattern.test(channel)) { + try { sub.handler(data); } catch (e) { console.error(`Error in raw packet handler:`, e); } + } + } + } + + if (decoded.type === "small") { + this.queueMessage(decoded.channel, decoded.data); + } else { + const complete = this.reassembler.processFragment(decoded); + if (complete) this.queueMessage(complete.channel, complete.data); + } + } + + private queueMessage(channel: string, data: Uint8Array): void { + const msg: LCMMessage = { + channel, + data: new Uint8Array(data), + timestamp: Date.now(), + }; + this.messageQueue.push(msg); + } + + private dispatchMessage(msg: LCMMessage): void { + for (const sub of this.subscriptions) { + if (sub.pattern.test(msg.channel)) { + try { + if (sub.msgClass) { + const decoded = sub.msgClass.decode(msg.data); + sub.handler({ channel: msg.channel, data: decoded, timestamp: msg.timestamp }); + } else { + sub.handler(msg); + } + } catch (e) { + console.error(`Error in subscription handler for ${msg.channel}:`, e); + } + } + } + } + + getConfig(): ParsedUrl { + return { ...this.config }; + } + + isRunning(): boolean { + return this.running; + } + + /** Peek at the next sequence number (for echo filtering). */ + getNextSeq(): number { + return this.sequenceNumber; + } +} diff --git a/misc/DimSim/dimos-cli/vendor/lcm/mod.ts b/misc/DimSim/dimos-cli/vendor/lcm/mod.ts new file mode 100644 index 0000000000..f1ad2a5bd1 --- /dev/null +++ b/misc/DimSim/dimos-cli/vendor/lcm/mod.ts @@ -0,0 +1,6 @@ +// LCM Pure TypeScript Implementation (vendored from @dimos/lcm@0.2.0) +// FIX: Added joinMulticastV4() in transport.ts + +export { LCM } from "./lcm.ts"; +export type { LCMOptions, LCMMessage, MessageClass, ParsedUrl, PacketHandler } from "./types.ts"; +export { parseUrl, DEFAULT_MULTICAST_GROUP, DEFAULT_PORT } from "./url.ts"; diff --git a/misc/DimSim/dimos-cli/vendor/lcm/transport.ts b/misc/DimSim/dimos-cli/vendor/lcm/transport.ts new file mode 100644 index 0000000000..b98a6bc866 --- /dev/null +++ b/misc/DimSim/dimos-cli/vendor/lcm/transport.ts @@ -0,0 +1,352 @@ +// LCM UDP Multicast Transport (vendored from @dimos/lcm@0.2.0) +// FIX: Added joinMulticastV4() call in UdpMulticastSocket.listen() + +import { + MAGIC_SHORT, + MAGIC_LONG, + MAX_SMALL_MESSAGE, + SHORT_HEADER_SIZE, + FRAGMENT_HEADER_SIZE, +} from "./types.ts"; +import type { ParsedUrl } from "./types.ts"; + +const textEncoder = new TextEncoder(); +const textDecoder = new TextDecoder(); + +/** Encode a small LCM message (fits in single UDP packet) */ +export function encodeSmallMessage( + channel: string, + data: Uint8Array, + sequenceNumber: number +): Uint8Array { + const channelBytes = textEncoder.encode(channel); + const totalSize = SHORT_HEADER_SIZE + channelBytes.length + 1 + data.length; + + if (totalSize > MAX_SMALL_MESSAGE) { + throw new Error(`Message too large for small message format: ${totalSize} > ${MAX_SMALL_MESSAGE}`); + } + + const buffer = new Uint8Array(totalSize); + const view = new DataView(buffer.buffer); + + let offset = 0; + + // Magic number (big-endian) + view.setUint32(offset, MAGIC_SHORT, false); + offset += 4; + + // Sequence number (big-endian) + view.setUint32(offset, sequenceNumber, false); + offset += 4; + + // Channel name (null-terminated) + buffer.set(channelBytes, offset); + offset += channelBytes.length; + buffer[offset] = 0; // null terminator + offset += 1; + + // Payload + buffer.set(data, offset); + + return buffer; +} + +/** Encode a fragmented LCM message (requires multiple UDP packets) */ +export function encodeFragmentedMessage( + channel: string, + data: Uint8Array, + sequenceNumber: number, + maxFragmentSize: number = 65000 +): Uint8Array[] { + const channelBytes = textEncoder.encode(channel); + const payloadSize = data.length; + + const firstFragmentPayloadSpace = maxFragmentSize - FRAGMENT_HEADER_SIZE - channelBytes.length - 1; + const subsequentFragmentPayloadSpace = maxFragmentSize - FRAGMENT_HEADER_SIZE; + + let numFragments = 1; + let remainingBytes = payloadSize - Math.min(payloadSize, firstFragmentPayloadSpace); + if (remainingBytes > 0) { + numFragments += Math.ceil(remainingBytes / subsequentFragmentPayloadSpace); + } + + const fragments: Uint8Array[] = []; + let payloadOffset = 0; + + for (let fragmentNum = 0; fragmentNum < numFragments; fragmentNum++) { + const isFirst = fragmentNum === 0; + const headerSize = FRAGMENT_HEADER_SIZE; + const channelSize = isFirst ? channelBytes.length + 1 : 0; + + const maxPayloadForThisFragment = isFirst + ? firstFragmentPayloadSpace + : subsequentFragmentPayloadSpace; + + const payloadForThisFragment = Math.min( + maxPayloadForThisFragment, + payloadSize - payloadOffset + ); + + const fragmentSize = headerSize + channelSize + payloadForThisFragment; + const fragment = new Uint8Array(fragmentSize); + const view = new DataView(fragment.buffer); + + let offset = 0; + + view.setUint32(offset, MAGIC_LONG, false); + offset += 4; + view.setUint32(offset, sequenceNumber, false); + offset += 4; + view.setUint32(offset, payloadSize, false); + offset += 4; + view.setUint32(offset, payloadOffset, false); + offset += 4; + view.setUint16(offset, fragmentNum, false); + offset += 2; + view.setUint16(offset, numFragments, false); + offset += 2; + + if (isFirst) { + fragment.set(channelBytes, offset); + offset += channelBytes.length; + fragment[offset] = 0; + offset += 1; + } + + fragment.set(data.subarray(payloadOffset, payloadOffset + payloadForThisFragment), offset); + payloadOffset += payloadForThisFragment; + fragments.push(fragment); + } + + return fragments; +} + +/** Decoded small message */ +export interface DecodedSmallMessage { + type: "small"; + channel: string; + data: Uint8Array; + sequenceNumber: number; +} + +/** Decoded fragment */ +export interface DecodedFragment { + type: "fragment"; + sequenceNumber: number; + payloadSize: number; + fragmentOffset: number; + fragmentNumber: number; + numFragments: number; + channel?: string; + data: Uint8Array; +} + +/** Decode a received UDP packet */ +export function decodePacket(packet: Uint8Array): DecodedSmallMessage | DecodedFragment | null { + if (packet.length < SHORT_HEADER_SIZE) { + return null; + } + + const view = new DataView(packet.buffer, packet.byteOffset, packet.byteLength); + const magic = view.getUint32(0, false); + + if (magic === MAGIC_SHORT) { + return decodeSmallPacket(packet, view); + } else if (magic === MAGIC_LONG) { + return decodeFragmentPacket(packet, view); + } + + return null; +} + +function decodeSmallPacket(packet: Uint8Array, view: DataView): DecodedSmallMessage | null { + const sequenceNumber = view.getUint32(4, false); + + let channelEnd = SHORT_HEADER_SIZE; + while (channelEnd < packet.length && packet[channelEnd] !== 0) { + channelEnd++; + } + + if (channelEnd >= packet.length) { + return null; + } + + const channel = textDecoder.decode(packet.subarray(SHORT_HEADER_SIZE, channelEnd)); + const data = packet.subarray(channelEnd + 1); + + return { type: "small", channel, data, sequenceNumber }; +} + +function decodeFragmentPacket(packet: Uint8Array, view: DataView): DecodedFragment | null { + if (packet.length < FRAGMENT_HEADER_SIZE) { + return null; + } + + const sequenceNumber = view.getUint32(4, false); + const payloadSize = view.getUint32(8, false); + const fragmentOffset = view.getUint32(12, false); + const fragmentNumber = view.getUint16(16, false); + const numFragments = view.getUint16(18, false); + + let offset = FRAGMENT_HEADER_SIZE; + let channel: string | undefined; + + if (fragmentNumber === 0) { + let channelEnd = offset; + while (channelEnd < packet.length && packet[channelEnd] !== 0) { + channelEnd++; + } + if (channelEnd >= packet.length) { + return null; + } + channel = textDecoder.decode(packet.subarray(offset, channelEnd)); + offset = channelEnd + 1; + } + + const data = packet.subarray(offset); + + return { type: "fragment", sequenceNumber, payloadSize, fragmentOffset, fragmentNumber, numFragments, channel, data }; +} + +/** Fragment reassembler for handling large messages */ +export class FragmentReassembler { + private pending = new Map; + buffer: Uint8Array; + lastActivity: number; + }>(); + + private timeoutMs: number; + + constructor(timeoutMs: number = 5000) { + this.timeoutMs = timeoutMs; + } + + processFragment(fragment: DecodedFragment): { channel: string; data: Uint8Array } | null { + const now = Date.now(); + this.cleanup(now); + + let entry = this.pending.get(fragment.sequenceNumber); + + if (!entry) { + if (fragment.fragmentNumber !== 0 || !fragment.channel) { + return null; + } + + entry = { + channel: fragment.channel, + payloadSize: fragment.payloadSize, + numFragments: fragment.numFragments, + receivedFragments: new Set(), + buffer: new Uint8Array(fragment.payloadSize), + lastActivity: now, + }; + this.pending.set(fragment.sequenceNumber, entry); + } + + if (fragment.fragmentOffset + fragment.data.length > entry.buffer.length) { + // Fragment doesn't fit — corrupted or mismatched packet, discard. + this.pending.delete(fragment.sequenceNumber); + return null; + } + entry.buffer.set(fragment.data, fragment.fragmentOffset); + entry.receivedFragments.add(fragment.fragmentNumber); + entry.lastActivity = now; + + if (entry.receivedFragments.size === entry.numFragments) { + this.pending.delete(fragment.sequenceNumber); + return { channel: entry.channel, data: entry.buffer }; + } + + return null; + } + + private cleanup(now: number): void { + for (const [seq, entry] of this.pending) { + if (now - entry.lastActivity > this.timeoutMs) { + this.pending.delete(seq); + } + } + } +} + +/** UDP Multicast socket wrapper for Deno */ +export class UdpMulticastSocket { + private socket: Deno.DatagramConn | null = null; + private readonly config: ParsedUrl; + private running = false; + + constructor(config: ParsedUrl) { + this.config = config; + } + + /** Start listening for multicast messages */ + async listen(onMessage: (data: Uint8Array, addr: Deno.NetAddr) => void): Promise { + // reuseAddress allows multiple processes to bind to the same multicast port + this.socket = Deno.listenDatagram({ + port: this.config.port, + transport: "udp", + hostname: "0.0.0.0", + reuseAddress: true, + }); + + // FIX: Join the multicast group and enable loopback for local IPC + const membership = await this.socket.joinMulticastV4(this.config.host, this.config.iface ?? "0.0.0.0"); + membership.setLoopback(true); + if (this.config.ttl !== undefined) { + membership.setTTL(this.config.ttl); + } + + this.running = true; + + // Read loop + (async () => { + try { + while (this.running && this.socket) { + const [data, addr] = await this.socket.receive(); + if (addr.transport === "udp") { + onMessage(data, addr); + } + } + } catch (e) { + if (this.running) { + console.error("UDP receive error:", e); + } + } + })(); + } + + /** Send a UDP packet to the multicast group */ + async send(data: Uint8Array): Promise { + if (!this.socket) { + // Create a socket for sending if we don't have one + this.socket = Deno.listenDatagram({ + port: 0, // Ephemeral port for sending + transport: "udp", + hostname: "0.0.0.0", + }); + } + + await this.socket.send(data, { + transport: "udp", + hostname: this.config.host, + port: this.config.port, + }); + } + + /** Close the socket */ + close(): void { + this.running = false; + if (this.socket) { + try { + this.socket.close(); + } catch { + // Ignore close errors + } + this.socket = null; + } + } +} diff --git a/misc/DimSim/dimos-cli/vendor/lcm/types.ts b/misc/DimSim/dimos-cli/vendor/lcm/types.ts new file mode 100644 index 0000000000..2f4e1a14b3 --- /dev/null +++ b/misc/DimSim/dimos-cli/vendor/lcm/types.ts @@ -0,0 +1,62 @@ +// LCM Type Definitions (vendored from @dimos/lcm@0.2.0) + +/** LCM message as received */ +export interface LCMMessage { + channel: string; + data: T; + timestamp: number; +} + +/** Interface for LCM message classes (generated types) */ +export interface MessageClass { + readonly _HASH: bigint; + readonly _NAME: string; + decode(data: Uint8Array): T; + new (init?: Partial): T & { encode(): Uint8Array }; +} + +/** Subscription handler function */ +export type SubscriptionHandler = (msg: LCMMessage) => void; + +/** Packet handler function (for raw UDP packets) */ +export type PacketHandler = (packet: Uint8Array) => void; + +/** LCM configuration options */ +export interface LCMOptions { + /** LCM URL (e.g., "udpm://239.255.76.67:7667?ttl=1") */ + url?: string; + /** Multicast TTL (time-to-live) */ + ttl?: number; + /** Network interface to bind to */ + iface?: string; +} + +/** Parsed LCM URL */ +export interface ParsedUrl { + scheme: string; + host: string; + port: number; + ttl: number; + iface?: string; +} + +/** Internal subscription record */ +export interface Subscription { + channel: string; + pattern: RegExp; + handler: SubscriptionHandler; + msgClass?: MessageClass; +} + +/** Internal packet subscription record */ +export interface PacketSubscription { + pattern: RegExp | null; // null = match all + handler: PacketHandler; +} + +// LCM Protocol constants +export const MAGIC_SHORT = 0x4c433032; // "LC02" +export const MAGIC_LONG = 0x4c433033; // "LC03" +export const MAX_SMALL_MESSAGE = 65535; +export const SHORT_HEADER_SIZE = 8; +export const FRAGMENT_HEADER_SIZE = 20; diff --git a/misc/DimSim/dimos-cli/vendor/lcm/url.ts b/misc/DimSim/dimos-cli/vendor/lcm/url.ts new file mode 100644 index 0000000000..ecad329e02 --- /dev/null +++ b/misc/DimSim/dimos-cli/vendor/lcm/url.ts @@ -0,0 +1,61 @@ +// LCM URL Parser (vendored from @dimos/lcm@0.2.0) + +import type { ParsedUrl } from "./types.ts"; + +export const DEFAULT_MULTICAST_GROUP = "239.255.76.67"; +export const DEFAULT_PORT = 7667; +export const DEFAULT_TTL = 0; + +/** + * Parse an LCM URL into its components. + * + * Supported formats: + * - "udpm://239.255.76.67:7667?ttl=1" + * - "udpm://239.255.76.67:7667" + * - "udpm://" (uses defaults) + * - "" (uses defaults) + */ +export function parseUrl(url?: string): ParsedUrl { + if (!url || url === "" || url === "udpm://") { + return { + scheme: "udpm", + host: DEFAULT_MULTICAST_GROUP, + port: DEFAULT_PORT, + ttl: DEFAULT_TTL, + }; + } + + const match = url.match(/^(\w+):\/\/([^:/?#]+)?(?::(\d+))?(?:\?(.*))?$/); + if (!match) { + throw new Error(`Invalid LCM URL: ${url}`); + } + + const [, scheme, host, portStr, queryStr] = match; + + if (scheme !== "udpm") { + throw new Error(`Unsupported LCM scheme: ${scheme} (only "udpm" is supported)`); + } + + const port = portStr ? parseInt(portStr, 10) : DEFAULT_PORT; + + // Parse query parameters + let ttl = DEFAULT_TTL; + let iface: string | undefined; + + if (queryStr) { + const params = new URLSearchParams(queryStr); + const ttlStr = params.get("ttl"); + if (ttlStr) { + ttl = parseInt(ttlStr, 10); + } + iface = params.get("iface") ?? undefined; + } + + return { + scheme, + host: host || DEFAULT_MULTICAST_GROUP, + port, + ttl, + iface, + }; +} diff --git a/misc/DimSim/docker/ci-test/Dockerfile b/misc/DimSim/docker/ci-test/Dockerfile new file mode 100644 index 0000000000..c0b51c7c17 --- /dev/null +++ b/misc/DimSim/docker/ci-test/Dockerfile @@ -0,0 +1,37 @@ +FROM debian:bookworm-slim + +# System deps: Playwright browser deps + multicast tools +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl unzip ca-certificates iproute2 procps \ + fonts-liberation libnss3 libatk-bridge2.0-0 \ + libdrm2 libxcomposite1 libxdamage1 libxrandr2 libgbm1 \ + libpango-1.0-0 libcairo2 libasound2 libxshmfence1 \ + libx11-xcb1 libxcb1 libxext6 libxfixes3 libxi6 \ + libxrender1 libxtst6 libglib2.0-0 libdbus-1-3 \ + && rm -rf /var/lib/apt/lists/* + +# Install Deno +RUN curl -fsSL https://deno.land/install.sh | sh +ENV DENO_DIR=/root/.deno +ENV PATH="/root/.deno/bin:$PATH" + +WORKDIR /app + +# DimSim — pre-built frontend + CLI tooling +COPY DimSim/dist/ ./DimSim/dist/ +COPY DimSim/dimos-cli/ ./DimSim/dimos-cli/ +COPY DimSim/evals/ ./DimSim/evals/ + +# dimos — TS language interop controller +COPY dimos/examples/language-interop/ts/ ./dimos/examples/language-interop/ts/ + +# Entrypoint +COPY DimSim/docker/ci-test/run-test.sh ./run-test.sh +RUN chmod +x run-test.sh + +# Pre-cache Deno deps and install Playwright's Chromium +RUN cd DimSim && deno cache --unstable-net dimos-cli/cli.ts || true +RUN cd /app/dimos/examples/language-interop/ts && deno cache --unstable-net main_custom_multicast.ts || true +RUN cd DimSim && deno run -A npm:playwright install --with-deps chromium || true + +ENTRYPOINT ["./run-test.sh"] diff --git a/misc/DimSim/docker/ci-test/build-and-test.sh b/misc/DimSim/docker/ci-test/build-and-test.sh new file mode 100755 index 0000000000..68b7c39e2b --- /dev/null +++ b/misc/DimSim/docker/ci-test/build-and-test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Navigate to Dimensional/ (parent of DimSim/ and dimos/) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +DIMENSIONAL_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +echo "Build context: $DIMENSIONAL_DIR" +echo "Building CI test image..." + +docker build \ + -f "$SCRIPT_DIR/Dockerfile" \ + -t dimsim-ci-test \ + "$DIMENSIONAL_DIR" + +echo "" +echo "Running integration test..." +docker run --rm \ + --cap-add=NET_ADMIN \ + dimsim-ci-test diff --git a/misc/DimSim/docker/ci-test/run-test.sh b/misc/DimSim/docker/ci-test/run-test.sh new file mode 100755 index 0000000000..66ee716030 --- /dev/null +++ b/misc/DimSim/docker/ci-test/run-test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== DimSim + dimos Integration Test ===" + +# --- Multicast setup --- +echo "Setting up multicast routing..." +ip link set lo multicast on 2>/dev/null || true +ip route add 224.0.0.0/4 dev lo 2>/dev/null || true + +# --- Start bridge server (DimSim) --- +echo "Starting DimSim bridge server..." +cd /app/DimSim +deno run --allow-all --unstable-net \ + dimos-cli/cli.ts dev --scene apt --port 8090 --headless --render cpu \ + &> /tmp/bridge.log & +BRIDGE_PID=$! +cd /app + +# Wait for HTTP to be ready +echo "Waiting for bridge..." +for i in $(seq 1 30); do + if curl -s http://localhost:8090 > /dev/null 2>&1; then + echo "Bridge ready (${i}s)" + break + fi + if [ "$i" -eq 30 ]; then + echo "FAIL: Bridge never started" + cat /tmp/bridge.log + exit 1 + fi + sleep 1 +done + +# Wait for headless browser to load scene + start publishing +# SwiftShader CPU rendering is very slow — needs more time +echo "Waiting for scene load (30s)..." +sleep 30 + +# --- Run dimos TS controller --- +echo "Starting dimos TS controller (main_custom_multicast.ts)..." +echo "Will timeout after 60s if no data received." + +# Create sensor output dir so the controller doesn't crash +mkdir -p /app/sensor_output + +cd /app/dimos/examples/language-interop/ts +timeout 60 deno run --allow-net --allow-write --unstable-net \ + main_custom_multicast.ts &> /tmp/controller.log & +CTRL_PID=$! +cd /app + +# Monitor: wait for 3+ odom messages (poll controller output) +PASS=false +for i in $(seq 1 60); do + ODOM_COUNT="$(grep -c '\[recv\] odom' /tmp/controller.log 2>/dev/null)" || ODOM_COUNT=0 + if [ "$ODOM_COUNT" -ge 3 ]; then + PASS=true + break + fi + sleep 1 +done + +# Cleanup +kill $CTRL_PID 2>/dev/null || true +kill $BRIDGE_PID 2>/dev/null || true + +echo "" +echo "--- Controller output ---" +cat /tmp/controller.log + +if $PASS; then + echo "" + echo "=== TEST PASSED — dimos ↔ DimSim multicast integration works ===" + exit 0 +else + echo "" + echo "=== TEST FAILED — no odom data received ===" + echo "--- Bridge logs ---" + tail -50 /tmp/bridge.log + exit 1 +fi diff --git a/misc/DimSim/docker/cli-test/Dockerfile b/misc/DimSim/docker/cli-test/Dockerfile new file mode 100644 index 0000000000..0525a640d9 --- /dev/null +++ b/misc/DimSim/docker/cli-test/Dockerfile @@ -0,0 +1,23 @@ +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl unzip ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install Deno +RUN curl -fsSL https://deno.land/install.sh | sh +ENV PATH="/root/.deno/bin:$PATH" + +WORKDIR /app + +# Install CLI from JSR (the real install path) +RUN deno install -gAf --name dimsim --allow-all --unstable-net jsr:@antim/dimsim + +# Copy release tarballs for local setup test +COPY dimsim-core-v0.1.0.tar.gz ./ +COPY scene-apt-v0.1.0.tar.gz ./ + +COPY docker/cli-test/test.sh ./test.sh +RUN chmod +x test.sh + +ENTRYPOINT ["./test.sh"] diff --git a/misc/DimSim/docker/cli-test/test.sh b/misc/DimSim/docker/cli-test/test.sh new file mode 100755 index 0000000000..5736049857 --- /dev/null +++ b/misc/DimSim/docker/cli-test/test.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== DimSim CLI Install Test ===" +echo "" + +# 1. Verify CLI is installed +echo "--- Step 1: Verify CLI installed ---" +which dimsim +dimsim help 2>&1 | head -5 +echo "" + +# 2. Setup core assets +echo "--- Step 2: dimsim setup ---" +dimsim setup --local /app/dimsim-core-v0.1.0.tar.gz +echo "" + +# 3. Verify core installed +echo "--- Step 3: Verify core ---" +ls -la ~/.dimsim/dist/ +echo "" + +# 4. Install apt scene +echo "--- Step 4: dimsim scene install apt ---" +dimsim scene install apt --local /app/scene-apt-v0.1.0.tar.gz +echo "" + +# 5. List scenes +echo "--- Step 5: dimsim scene list ---" +dimsim scene list +echo "" + +# 6. Verify scene installed +echo "--- Step 6: Verify scene files ---" +ls -lh ~/.dimsim/dist/sims/ +echo "" + +# 7. Start dev server briefly and verify it responds +echo "--- Step 7: dimsim dev (quick test) ---" +dimsim dev --scene apt --port 8090 & +DEV_PID=$! + +# Wait for server to start +for i in $(seq 1 15); do + if curl -s http://localhost:8090 > /dev/null 2>&1; then + echo "Server responding after ${i}s" + break + fi + if [ "$i" -eq 15 ]; then + echo "FAIL: Server never started" + kill $DEV_PID 2>/dev/null || true + exit 1 + fi + sleep 1 +done + +# Verify HTML response contains DimSim +RESP=$(curl -s http://localhost:8090) +if echo "$RESP" | grep -q "dimosMode"; then + echo "HTML contains dimosMode injection — correct!" +else + echo "FAIL: HTML missing dimosMode" + kill $DEV_PID 2>/dev/null || true + exit 1 +fi + +# Verify scene asset is served +STATUS=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8090/sims/apt.json) +if [ "$STATUS" = "200" ]; then + echo "Scene apt.json served — HTTP 200" +else + echo "FAIL: Scene not served (HTTP $STATUS)" + kill $DEV_PID 2>/dev/null || true + exit 1 +fi + +kill $DEV_PID 2>/dev/null || true + +echo "" +echo "=== ALL TESTS PASSED ===" diff --git a/misc/DimSim/evals/apt/go-to-couch.json b/misc/DimSim/evals/apt/go-to-couch.json new file mode 100644 index 0000000000..74d4535487 --- /dev/null +++ b/misc/DimSim/evals/apt/go-to-couch.json @@ -0,0 +1,19 @@ +{ + "name": "go-to-couch", + "environment": "apt", + "task": "Go to the couch", + "startPose": { + "x": 0, + "y": 0.5, + "z": 3, + "yaw": 0 + }, + "timeoutSec": 30, + "successCriteria": { + "objectDistance": { + "object": "agent", + "target": "sectional", + "thresholdM": 2.0 + } + } +} diff --git a/misc/DimSim/evals/apt/go-to-kitchen.json b/misc/DimSim/evals/apt/go-to-kitchen.json new file mode 100644 index 0000000000..7e68734583 --- /dev/null +++ b/misc/DimSim/evals/apt/go-to-kitchen.json @@ -0,0 +1,19 @@ +{ + "name": "go-to-kitchen", + "environment": "apt", + "task": "Go to the kitchen", + "startPose": { + "x": 0, + "y": 0.5, + "z": 3, + "yaw": 0 + }, + "timeoutSec": 30, + "successCriteria": { + "objectDistance": { + "object": "agent", + "target": "refrigerator", + "thresholdM": 3.0 + } + } +} diff --git a/misc/DimSim/evals/apt/go-to-tv.json b/misc/DimSim/evals/apt/go-to-tv.json new file mode 100644 index 0000000000..ba07f96167 --- /dev/null +++ b/misc/DimSim/evals/apt/go-to-tv.json @@ -0,0 +1,19 @@ +{ + "name": "go-to-tv", + "environment": "apt", + "task": "Go to the TV", + "startPose": { + "x": 0, + "y": 0.5, + "z": 3, + "yaw": 0 + }, + "timeoutSec": 30, + "successCriteria": { + "objectDistance": { + "object": "agent", + "target": "television", + "thresholdM": 2.0 + } + } +} diff --git a/misc/DimSim/evals/apt/television.json b/misc/DimSim/evals/apt/television.json new file mode 100644 index 0000000000..450375e524 --- /dev/null +++ b/misc/DimSim/evals/apt/television.json @@ -0,0 +1,19 @@ +{ + "name": "television", + "environment": "apt", + "task": "Go to the Television", + "startPose": { + "x": 0, + "y": 0.5, + "z": 3, + "yaw": 0 + }, + "timeoutSec": 60, + "successCriteria": { + "objectDistance": { + "object": "agent", + "target": "television", + "thresholdM": 2 + } + } +} diff --git a/misc/DimSim/evals/manifest.json b/misc/DimSim/evals/manifest.json new file mode 100644 index 0000000000..8e34d570b5 --- /dev/null +++ b/misc/DimSim/evals/manifest.json @@ -0,0 +1,15 @@ +{ + "version": "1.0", + "environments": [ + { + "name": "apt", + "scene": "apt", + "workflows": [ + "go-to-tv", + "go-to-couch", + "go-to-kitchen", + "television" + ] + } + ] +} diff --git a/misc/DimSim/index.html b/misc/DimSim/index.html new file mode 100644 index 0000000000..9d4f457b75 --- /dev/null +++ b/misc/DimSim/index.html @@ -0,0 +1,212 @@ + + + + + + DimSim + + + + + + + + + + +
+ +
+ + + + + + + +
+
+ DimSim + +
+ +
+ +
+
+ + +
+
No scene loaded
+ + +
+ + + +
+
+ Sensor Debug +
+ + +
+
+ +
+
+ + +
+
+
+ Min + + 0.2m +
+
+ Max + + 12.0m +
+
+
+ +
+
+ +
+
+ + +
+
+
+ + +
+ Agent Vision +
+
RGB
+ Agent POV +
Depth
+ +
LiDAR
+ +
+
+ +
Waiting...
+
+
+
+ Request Details +
+
No request yet
+
+ Prompt +

+              
+
+ Context +

+              
+
+ Raw Output +

+              
+
+
+
+ Activity Log +
+
+
+ + +
+ + + +
+ + +
+ + + +
+ + + + + +
+ WASD move + E interact + B spawn + G ghost +
+ +
+ + + + + + diff --git a/misc/DimSim/package-lock.json b/misc/DimSim/package-lock.json new file mode 100644 index 0000000000..3478389ba2 --- /dev/null +++ b/misc/DimSim/package-lock.json @@ -0,0 +1,2120 @@ +{ + "name": "dimsim", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dimsim", + "version": "1.0.0", + "dependencies": { + "@dimforge/rapier3d-compat": "^0.14.0", + "@sparkjsdev/spark": "latest", + "cors": "^2.8.5", + "express": "^4.21.0", + "openai": "^4.77.0", + "three": "^0.168.0" + }, + "devDependencies": { + "vite": "^5.4.10" + } + }, + "node_modules/@dimforge/rapier3d-compat": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.14.0.tgz", + "integrity": "sha512-/uHrUzS+CRQ+NQrrJCEDUkhwHlNsAAexbNXgbN9sHY+GwR+SFFAFrxRr8Llf5/AJZzqiLANdQIfJ63Cw4gJVqw==", + "license": "Apache-2.0" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.57.1.tgz", + "integrity": "sha512-A6ehUVSiSaaliTxai040ZpZ2zTevHYbvu/lDoeAteHI8QnaosIzm4qwtezfRg1jOYaUmnzLX1AOD6Z+UJjtifg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.57.1.tgz", + "integrity": "sha512-dQaAddCY9YgkFHZcFNS/606Exo8vcLHwArFZ7vxXq4rigo2bb494/xKMMwRRQW6ug7Js6yXmBZhSBRuBvCCQ3w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.57.1.tgz", + "integrity": "sha512-crNPrwJOrRxagUYeMn/DZwqN88SDmwaJ8Cvi/TN1HnWBU7GwknckyosC2gd0IqYRsHDEnXf328o9/HC6OkPgOg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.57.1.tgz", + "integrity": "sha512-Ji8g8ChVbKrhFtig5QBV7iMaJrGtpHelkB3lsaKzadFBe58gmjfGXAOfI5FV0lYMH8wiqsxKQ1C9B0YTRXVy4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.57.1.tgz", + "integrity": "sha512-R+/WwhsjmwodAcz65guCGFRkMb4gKWTcIeLy60JJQbXrJ97BOXHxnkPFrP+YwFlaS0m+uWJTstrUA9o+UchFug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.57.1.tgz", + "integrity": "sha512-IEQTCHeiTOnAUC3IDQdzRAGj3jOAYNr9kBguI7MQAAZK3caezRrg0GxAb6Hchg4lxdZEI5Oq3iov/w/hnFWY9Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.57.1.tgz", + "integrity": "sha512-F8sWbhZ7tyuEfsmOxwc2giKDQzN3+kuBLPwwZGyVkLlKGdV1nvnNwYD0fKQ8+XS6hp9nY7B+ZeK01EBUE7aHaw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.57.1.tgz", + "integrity": "sha512-rGfNUfn0GIeXtBP1wL5MnzSj98+PZe/AXaGBCRmT0ts80lU5CATYGxXukeTX39XBKsxzFpEeK+Mrp9faXOlmrw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.57.1.tgz", + "integrity": "sha512-MMtej3YHWeg/0klK2Qodf3yrNzz6CGjo2UntLvk2RSPlhzgLvYEB3frRvbEF2wRKh1Z2fDIg9KRPe1fawv7C+g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.57.1.tgz", + "integrity": "sha512-1a/qhaaOXhqXGpMFMET9VqwZakkljWHLmZOX48R0I/YLbhdxr1m4gtG1Hq7++VhVUmf+L3sTAf9op4JlhQ5u1Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.57.1.tgz", + "integrity": "sha512-QWO6RQTZ/cqYtJMtxhkRkidoNGXc7ERPbZN7dVW5SdURuLeVU7lwKMpo18XdcmpWYd0qsP1bwKPf7DNSUinhvA==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.57.1.tgz", + "integrity": "sha512-xpObYIf+8gprgWaPP32xiN5RVTi/s5FCR+XMXSKmhfoJjrpRAjCuuqQXyxUa/eJTdAE6eJ+KDKaoEqjZQxh3Gw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.57.1.tgz", + "integrity": "sha512-4BrCgrpZo4hvzMDKRqEaW1zeecScDCR+2nZ86ATLhAoJ5FQ+lbHVD3ttKe74/c7tNT9c6F2viwB3ufwp01Oh2w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.57.1.tgz", + "integrity": "sha512-NOlUuzesGauESAyEYFSe3QTUguL+lvrN1HtwEEsU2rOwdUDeTMJdO5dUYl/2hKf9jWydJrO9OL/XSSf65R5+Xw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.57.1.tgz", + "integrity": "sha512-ptA88htVp0AwUUqhVghwDIKlvJMD/fmL/wrQj99PRHFRAG6Z5nbWoWG4o81Nt9FT+IuqUQi+L31ZKAFeJ5Is+A==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.57.1.tgz", + "integrity": "sha512-S51t7aMMTNdmAMPpBg7OOsTdn4tySRQvklmL3RpDRyknk87+Sp3xaumlatU+ppQ+5raY7sSTcC2beGgvhENfuw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.57.1.tgz", + "integrity": "sha512-Bl00OFnVFkL82FHbEqy3k5CUCKH6OEJL54KCyx2oqsmZnFTR8IoNqBF+mjQVcRCT5sB6yOvK8A37LNm/kPJiZg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.57.1.tgz", + "integrity": "sha512-ABca4ceT4N+Tv/GtotnWAeXZUZuM/9AQyCyKYyKnpk4yoA7QIAuBt6Hkgpw8kActYlew2mvckXkvx0FfoInnLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.57.1.tgz", + "integrity": "sha512-HFps0JeGtuOR2convgRRkHCekD7j+gdAuXM+/i6kGzQtFhlCtQkpwtNzkNj6QhCDp7DRJ7+qC/1Vg2jt5iSOFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.57.1.tgz", + "integrity": "sha512-H+hXEv9gdVQuDTgnqD+SQffoWoc0Of59AStSzTEj/feWTBAnSfSD3+Dql1ZruJQxmykT/JVY0dE8Ka7z0DH1hw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.57.1.tgz", + "integrity": "sha512-4wYoDpNg6o/oPximyc/NG+mYUejZrCU2q+2w6YZqrAs2UcNUChIZXjtafAiiZSUc7On8v5NyNj34Kzj/Ltk6dQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.57.1.tgz", + "integrity": "sha512-O54mtsV/6LW3P8qdTcamQmuC990HDfR71lo44oZMZlXU4tzLrbvTii87Ni9opq60ds0YzuAlEr/GNwuNluZyMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.57.1.tgz", + "integrity": "sha512-P3dLS+IerxCT/7D2q2FYcRdWRl22dNbrbBEtxdWhXrfIMPP9lQhb5h4Du04mdl5Woq05jVCDPCMF7Ub0NAjIew==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.57.1.tgz", + "integrity": "sha512-VMBH2eOOaKGtIJYleXsi2B8CPVADrh+TyNxJ4mWPnKfLB/DBUmzW+5m1xUrcwWoMfSLagIRpjUFeW5CO5hyciQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.57.1.tgz", + "integrity": "sha512-mxRFDdHIWRxg3UfIIAwCm6NzvxG0jDX/wBN6KsQFTvKFqqg9vTrWUE68qEjHt19A5wwx5X5aUi2zuZT7YR0jrA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@sparkjsdev/spark": { + "version": "0.1.10", + "resolved": "https://registry.npmjs.org/@sparkjsdev/spark/-/spark-0.1.10.tgz", + "integrity": "sha512-CiijdZQuj7KPDUqIZPiEqyUkJCYo1JqR05vq/V+ElxMwqR7L70ZuZDyIKcasjZHSiPB8pGRMH8HZGqUKO9aRPQ==", + "license": "MIT", + "dependencies": { + "fflate": "^0.8.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", + "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/cors": { + "version": "2.8.6", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.6.tgz", + "integrity": "sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==", + "license": "MIT", + "dependencies": { + "object-assign": "^4", + "vary": "^1" + }, + "engines": { + "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/openai": { + "version": "4.104.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.104.0.tgz", + "integrity": "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.6", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.6.tgz", + "integrity": "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.2.tgz", + "integrity": "sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/rollup": { + "version": "4.57.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.57.1.tgz", + "integrity": "sha512-oQL6lgK3e2QZeQ7gcgIkS2YZPg5slw37hYufJ3edKlfQSGGm8ICoxswK15ntSzF/a8+h7ekRy7k7oWc3BQ7y8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.8" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.57.1", + "@rollup/rollup-android-arm64": "4.57.1", + "@rollup/rollup-darwin-arm64": "4.57.1", + "@rollup/rollup-darwin-x64": "4.57.1", + "@rollup/rollup-freebsd-arm64": "4.57.1", + "@rollup/rollup-freebsd-x64": "4.57.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.57.1", + "@rollup/rollup-linux-arm-musleabihf": "4.57.1", + "@rollup/rollup-linux-arm64-gnu": "4.57.1", + "@rollup/rollup-linux-arm64-musl": "4.57.1", + "@rollup/rollup-linux-loong64-gnu": "4.57.1", + "@rollup/rollup-linux-loong64-musl": "4.57.1", + "@rollup/rollup-linux-ppc64-gnu": "4.57.1", + "@rollup/rollup-linux-ppc64-musl": "4.57.1", + "@rollup/rollup-linux-riscv64-gnu": "4.57.1", + "@rollup/rollup-linux-riscv64-musl": "4.57.1", + "@rollup/rollup-linux-s390x-gnu": "4.57.1", + "@rollup/rollup-linux-x64-gnu": "4.57.1", + "@rollup/rollup-linux-x64-musl": "4.57.1", + "@rollup/rollup-openbsd-x64": "4.57.1", + "@rollup/rollup-openharmony-arm64": "4.57.1", + "@rollup/rollup-win32-arm64-msvc": "4.57.1", + "@rollup/rollup-win32-ia32-msvc": "4.57.1", + "@rollup/rollup-win32-x64-gnu": "4.57.1", + "@rollup/rollup-win32-x64-msvc": "4.57.1", + "fsevents": "~2.3.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/three": { + "version": "0.168.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.168.0.tgz", + "integrity": "sha512-6m6jXtDwMJEK/GGMbAOTSAmxNdzKvvBzgd7q8bE/7Tr6m7PaBh5kKLrN7faWtlglXbzj7sVba48Idwx+NRsZXw==", + "license": "MIT" + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + } + } +} diff --git a/misc/DimSim/package.json b/misc/DimSim/package.json new file mode 100644 index 0000000000..abdc9269cf --- /dev/null +++ b/misc/DimSim/package.json @@ -0,0 +1,30 @@ +{ + "name": "dimsim", + "private": true, + "version": "1.0.0", + "type": "module", + "description": "Standalone 3D simulation runner for SimStudio scenes", + "scripts": { + "dev": "vite --force", + "build": "vite build", + "preview": "vite preview", + "server": "node server.js", + "sync": "bash copy-sources.sh", + "parity:check": "bash check-parity.sh", + "update-sims": "bash update-sims.sh", + "dimos": "deno run --allow-all dimos-cli/cli.ts eval", + "dimos:headless": "deno run --allow-all dimos-cli/cli.ts eval --headless", + "dimos:dev": "deno run --allow-all dimos-cli/cli.ts dev" + }, + "dependencies": { + "@sparkjsdev/spark": "latest", + "@dimforge/rapier3d-compat": "^0.14.0", + "three": "^0.168.0", + "express": "^4.21.0", + "cors": "^2.8.5", + "openai": "^4.77.0" + }, + "devDependencies": { + "vite": "^5.4.10" + } +} diff --git a/misc/DimSim/public/agent-model/robot.glb b/misc/DimSim/public/agent-model/robot.glb new file mode 100644 index 0000000000..79296152a2 --- /dev/null +++ b/misc/DimSim/public/agent-model/robot.glb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:761813170c01429ab9215ad4e37b95a768437fc0c6aa1d643f5938a672263322 +size 55746312 diff --git a/misc/DimSim/public/logo.svg b/misc/DimSim/public/logo.svg new file mode 100644 index 0000000000..d64b96adfc --- /dev/null +++ b/misc/DimSim/public/logo.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/misc/DimSim/public/sims/apt.json b/misc/DimSim/public/sims/apt.json new file mode 100644 index 0000000000..fe9be506f7 --- /dev/null +++ b/misc/DimSim/public/sims/apt.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44d3dd75c18a55f02cf2e221f89769ee84f0607ffcc266b31fe9d2f4bc0ac214 +size 102228474 diff --git a/misc/DimSim/public/sims/apt_.json b/misc/DimSim/public/sims/apt_.json new file mode 100644 index 0000000000..2cd08d4d7b --- /dev/null +++ b/misc/DimSim/public/sims/apt_.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ee06fcd4ca045e069828733d767367c72fab8c2c3b6fb6b8a60d3b81759888c +size 102258137 diff --git a/misc/DimSim/public/sims/empty.json b/misc/DimSim/public/sims/empty.json new file mode 100644 index 0000000000..c620dc13fb --- /dev/null +++ b/misc/DimSim/public/sims/empty.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ce082c5e3c7e88342b1781b494e9b2a9bb8bf080590c75b1d3924604e15e52 +size 3100 diff --git a/misc/DimSim/public/sims/manifest.json b/misc/DimSim/public/sims/manifest.json new file mode 100644 index 0000000000..903a9c41d6 --- /dev/null +++ b/misc/DimSim/public/sims/manifest.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3050b1e85fb6d72fb444b51adb171205ca560eddaf9316ffce74895e003a33d +size 12 diff --git a/misc/DimSim/scenes.template.json b/misc/DimSim/scenes.template.json new file mode 100644 index 0000000000..2bf30efd2f --- /dev/null +++ b/misc/DimSim/scenes.template.json @@ -0,0 +1,9 @@ +{ + "scenes": { + "apt": { + "description": "Modern apartment interior", + "size": 71000000 + } + }, + "evalsUrl": "" +} diff --git a/misc/DimSim/scripts/package-release.sh b/misc/DimSim/scripts/package-release.sh new file mode 100755 index 0000000000..b823f82e7e --- /dev/null +++ b/misc/DimSim/scripts/package-release.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Creates release artifacts for DimSim distribution. +# dimsim-core-v{VERSION}.tar.gz — frontend bundle (no scenes) +# scene-apt-v{VERSION}.tar.gz — apt scene (gzipped JSON) + +cd "$(dirname "$0")/.." + +VERSION=${1:-"0.1.0"} + +echo "Packaging DimSim v${VERSION}..." + +# Verify dist/ exists +if [ ! -f "dist/index.html" ]; then + echo "Error: dist/ not found. Run 'npm run build' first." + exit 1 +fi + +# Core: everything in dist/ except sims/ +echo "Packaging core assets..." +tar -czf "dimsim-core-v${VERSION}.tar.gz" \ + -C dist \ + --exclude='sims' \ + . + +echo "Packaging apt scene..." +gzip -c dist/sims/apt.json > "scene-apt-v${VERSION}.tar.gz" + +echo "Packaging evals..." +tar -czf "dimsim-evals-v${VERSION}.tar.gz" \ + -C evals \ + . + +echo "" +echo "Release artifacts:" +ls -lh "dimsim-core-v${VERSION}.tar.gz" "scene-apt-v${VERSION}.tar.gz" "dimsim-evals-v${VERSION}.tar.gz" + +echo "" +echo "Upload to GitHub Release:" +echo " gh release create v${VERSION} dimsim-core-v${VERSION}.tar.gz scene-apt-v${VERSION}.tar.gz dimsim-evals-v${VERSION}.tar.gz" diff --git a/misc/DimSim/scripts/profile-live.sh b/misc/DimSim/scripts/profile-live.sh new file mode 100755 index 0000000000..8bb187c933 --- /dev/null +++ b/misc/DimSim/scripts/profile-live.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Simple profiler: one-line-per-sample, appends to terminal (no clearing). +# Usage: bash scripts/profile-live.sh [interval_seconds] + +INTERVAL=${1:-3} + +printf "%-6s %8s %8s %8s %8s %8s %8s %8s %8s %s\n" \ + "TIME" "PY_CPU%" "PY_MB" "DENO_CPU%" "DENO_MB" "SIM_UI%" "SIM_UI_MB" "GPU_CPU%" "GPU_MB" "LOAD" +printf "%-6s %8s %8s %8s %8s %8s %8s %8s %8s %s\n" \ + "------" "--------" "--------" "--------" "--------" "--------" "--------" "--------" "--------" "--------" + +while true; do + py_cpu=0; py_mem=0 + while IFS= read -r line; do + c=$(echo "$line" | awk '{print $1}'); m=$(echo "$line" | awk '{print $2}') + py_cpu=$(awk "BEGIN{print $py_cpu + $c}") + py_mem=$(awk "BEGIN{print $py_mem + $m}") + done < <(ps -eo %cpu,rss,command | grep -i '[p]ython.*dimos' 2>/dev/null) + + deno_cpu=0; deno_mem=0 + while IFS= read -r line; do + c=$(echo "$line" | awk '{print $1}'); m=$(echo "$line" | awk '{print $2}') + deno_cpu=$(awk "BEGIN{print $deno_cpu + $c}") + deno_mem=$(awk "BEGIN{print $deno_mem + $m}") + done < <(ps -eo %cpu,rss,command | grep -E '[d]eno|[d]imsim' 2>/dev/null | grep -v grep) + + # Find browser PIDs connected to bridge port 8090 (exclude deno server itself) + chrome_cpu=0; chrome_mem=0 + while IFS= read -r pid; do + [ -z "$pid" ] && continue + while IFS= read -r line; do + c=$(echo "$line" | awk '{print $1}'); m=$(echo "$line" | awk '{print $2}') + chrome_cpu=$(awk "BEGIN{print $chrome_cpu + $c}") + chrome_mem=$(awk "BEGIN{print $chrome_mem + $m}") + done < <(ps -p "$pid" -o %cpu,rss 2>/dev/null | tail -n +2) + done < <(lsof -i :8090 2>/dev/null | grep -v 'deno\|LISTEN' | awk 'NR>1{print $2}' | sort -u) + + # GPU processes — macOS Metal/WindowServer GPU usage + gpu_cpu=0; gpu_mem=0 + while IFS= read -r line; do + c=$(echo "$line" | awk '{print $1}'); m=$(echo "$line" | awk '{print $2}') + gpu_cpu=$(awk "BEGIN{print $gpu_cpu + $c}") + gpu_mem=$(awk "BEGIN{print $gpu_mem + $m}") + done < <(ps -eo %cpu,rss,command | grep -E '[W]indowServer|[G]PU.*Driver|com\.apple\.gpu' 2>/dev/null) + + py_mb=$(awk "BEGIN{printf \"%.0f\", $py_mem/1024}") + deno_mb=$(awk "BEGIN{printf \"%.0f\", $deno_mem/1024}") + chrome_mb=$(awk "BEGIN{printf \"%.0f\", $chrome_mem/1024}") + gpu_mb=$(awk "BEGIN{printf \"%.0f\", $gpu_mem/1024}") + load=$(sysctl -n vm.loadavg 2>/dev/null | awk '{print $2}' || echo "?") + + printf "%-6s %8.1f %6sMB %8.1f %6sMB %8.1f %6sMB %8.1f %6sMB %s\n" \ + "$(date '+%H:%M:%S')" "$py_cpu" "$py_mb" "$deno_cpu" "$deno_mb" "$chrome_cpu" "$chrome_mb" "$gpu_cpu" "$gpu_mb" "$load" + + sleep "$INTERVAL" +done diff --git a/misc/DimSim/scripts/speed-test.py b/misc/DimSim/scripts/speed-test.py new file mode 100644 index 0000000000..4ce143a8b6 --- /dev/null +++ b/misc/DimSim/scripts/speed-test.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Speed test: send constant cmd_vel for N seconds, measure actual displacement via odom. + +Usage (from dimos venv): + python DimSim/scripts/speed-test.py [--speed 0.5] [--duration 5] + +Requires: dimos environment with LCM working. +Run while DimSim bridge is active (browser tab open). +""" + +import argparse +import math +import struct +import threading +import time + +# LCM multicast constants +LCM_ADDR = "239.255.76.67" +LCM_PORT = 7667 +MAGIC = 0x4C433032 + +# ── Minimal LCM encode/decode (no dimos import needed) ───────────────────── + + +def _encode_lcm_packet(channel: str, payload: bytes, seq: int) -> bytes: + ch_bytes = channel.encode("utf-8") + header = struct.pack(">II", MAGIC, seq) + return header + ch_bytes + b"\x00" + payload + + +def _decode_lcm_packet(data: bytes): + if len(data) < 8: + return None, None + magic, seq = struct.unpack(">II", data[:8]) + if magic != MAGIC: + return None, None + rest = data[8:] + null_idx = rest.index(0) + channel = rest[:null_idx].decode("utf-8") + payload = rest[null_idx + 1 :] + return channel, payload + + +# ── Twist encoding (geometry_msgs.Twist LCM) ─────────────────────────────── +# Layout: hash(8) + 6 x float64 (linear.x,y,z, angular.x,y,z) + + +def encode_twist( + linear_x=0.0, linear_y=0.0, linear_z=0.0, angular_x=0.0, angular_y=0.0, angular_z=0.0 +) -> bytes: + from dimos.msgs.geometry_msgs import Twist, Vector3 + + t = Twist( + linear=Vector3(x=linear_x, y=linear_y, z=linear_z), + angular=Vector3(x=angular_x, y=angular_y, z=angular_z), + ) + return t.lcm_encode() + + +# ── PoseStamped decoding ─────────────────────────────────────────────────── + + +def decode_pose_stamped(data: bytes): + """Decode geometry_msgs.PoseStamped using dimos LCM decoder.""" + from dimos.msgs.geometry_msgs import PoseStamped + + msg = PoseStamped.lcm_decode(data) + p = msg.position + q = msg.orientation + return p.x, p.y, p.z, q.x, q.y, q.z, q.w + + +# ── Main ──────────────────────────────────────────────────────────────────── + + +def main(): + import socket + + parser = argparse.ArgumentParser(description="DimSim speed test") + parser.add_argument("--speed", type=float, default=0.5, help="linear.x m/s (default 0.5)") + parser.add_argument("--duration", type=float, default=5.0, help="seconds to drive (default 5)") + args = parser.parse_args() + + speed = args.speed + duration = args.duration + + # ── Setup UDP multicast socket ── + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + sock.bind(("", LCM_PORT)) + + mreq = struct.pack("4s4s", socket.inet_aton(LCM_ADDR), socket.inet_aton("0.0.0.0")) + sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + sock.settimeout(0.1) + + # Send socket (separate so we can send without receiving our own) + send_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) + send_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 1) + + # ── Collect initial odom ── + print("Waiting for odom...") + odom_channel = "/odom#geometry_msgs.PoseStamped" + start_pose = None + + for _ in range(200): # up to 20s + try: + data, _ = sock.recvfrom(65535) + except TimeoutError: + continue + ch, payload = _decode_lcm_packet(data) + if ch == odom_channel: + start_pose = decode_pose_stamped(payload) + break + + if start_pose is None: + print("ERROR: No odom received. Is the bridge running?") + return + + sx, sy, sz = start_pose[0], start_pose[1], start_pose[2] + print(f"Start pose: ({sx:.3f}, {sy:.3f}, {sz:.3f})") + print(f"Sending cmd_vel: linear.x={speed} m/s for {duration}s") + print(f"Expected displacement: {speed * duration:.2f}m") + print() + + # ── Drive: send cmd_vel at 10 Hz, collect odom ── + seq = 1000 + stop = threading.Event() + poses = [] + + def send_cmd_vel(): + nonlocal seq + while not stop.is_set(): + twist_data = encode_twist(linear_x=speed) + packet = _encode_lcm_packet("/cmd_vel#geometry_msgs.Twist", twist_data, seq) + send_sock.sendto(packet, (LCM_ADDR, LCM_PORT)) + seq += 1 + time.sleep(0.1) # 10 Hz, matches planner rate + + sender = threading.Thread(target=send_cmd_vel, daemon=True) + sender.start() + + t0 = time.time() + while time.time() - t0 < duration: + try: + data, _ = sock.recvfrom(65535) + except TimeoutError: + continue + ch, payload = _decode_lcm_packet(data) + if ch == odom_channel: + pose = decode_pose_stamped(payload) + elapsed = time.time() - t0 + poses.append((elapsed, pose)) + + # Stop sending, send zero velocity + stop.set() + sender.join() + for _ in range(5): + twist_data = encode_twist() + packet = _encode_lcm_packet("/cmd_vel#geometry_msgs.Twist", twist_data, seq) + send_sock.sendto(packet, (LCM_ADDR, LCM_PORT)) + seq += 1 + time.sleep(0.05) + + # ── Results ── + if not poses: + print("ERROR: No odom received during test") + return + + last_pose = poses[-1][1] + ex, ey, ez = last_pose[0], last_pose[1], last_pose[2] + dx, dy, dz = ex - sx, ey - sy, ez - sz + dist = math.sqrt(dx * dx + dy * dy + dz * dz) + expected = speed * duration + ratio = dist / expected if expected > 0 else 0 + + print(f"{'─' * 50}") + print(f"End pose: ({ex:.3f}, {ey:.3f}, {ez:.3f})") + print(f"Displacement: {dist:.3f}m") + print(f"Expected: {expected:.3f}m") + print( + f"Ratio: {ratio:.2f}x {'(OK)' if 0.85 < ratio < 1.15 else '(SLOW!)' if ratio < 0.85 else '(FAST!)'}" + ) + print(f"Odom samples: {len(poses)} ({len(poses) / duration:.0f} Hz)") + print(f"{'─' * 50}") + + # Show velocity over time + print("\nVelocity trace (sampled):") + prev_t, prev_p = 0, start_pose + for i, (t, p) in enumerate(poses): + if i % max(1, len(poses) // 10) != 0: + continue + dt = t - prev_t + if dt > 0: + d = math.sqrt( + (p[0] - prev_p[0]) ** 2 + (p[1] - prev_p[1]) ** 2 + (p[2] - prev_p[2]) ** 2 + ) + v = d / dt + print(f" t={t:5.2f}s v={v:.3f} m/s pos=({p[0]:.3f}, {p[1]:.3f}, {p[2]:.3f})") + prev_t, prev_p = t, p + + sock.close() + send_sock.close() + + +if __name__ == "__main__": + main() diff --git a/misc/DimSim/server.js b/misc/DimSim/server.js new file mode 100644 index 0000000000..9e45ee4663 --- /dev/null +++ b/misc/DimSim/server.js @@ -0,0 +1,174 @@ +import express from "express"; +import cors from "cors"; +import OpenAI from "openai"; +import { createHash } from "crypto"; +import { readFileSync, writeFileSync, existsSync } from "fs"; +import { fileURLToPath } from "url"; +import { dirname, join } from "path"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +// ── API Keys (from environment variables) ─────────────────────────────────── +const OPENAI_API_KEY = process.env.OPENAI_API_KEY || ""; +const GEMINI_API_KEY = process.env.GEMINI_API_KEY || ""; +const GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta/openai/"; +const VLM_MIN_INTERVAL_MS = 900; +const VLM_MAX_RETRIES = 4; +let _lastVlmAt = 0; + +// ── Clients ───────────────────────────────────────────────────────────────── +function getClient(model) { + if (model.startsWith("gemini")) { + return new OpenAI({ apiKey: GEMINI_API_KEY, baseURL: GEMINI_BASE_URL }); + } + return new OpenAI({ apiKey: OPENAI_API_KEY }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function paceVlmRequests() { + const now = Date.now(); + const waitMs = Math.max(0, _lastVlmAt + VLM_MIN_INTERVAL_MS - now); + if (waitMs > 0) await sleep(waitMs); + _lastVlmAt = Date.now(); +} + +async function requestWithRetry(runRequest) { + let attempt = 0; + let lastErr = null; + while (attempt < VLM_MAX_RETRIES) { + try { + await paceVlmRequests(); + return await runRequest(); + } catch (err) { + lastErr = err; + const status = Number(err?.status || err?.statusCode || 0); + const retryAfterHeader = + Number(err?.headers?.["retry-after"] || err?.response?.headers?.get?.("retry-after") || 0) || 0; + const shouldRetry = status === 429 || status === 503 || status === 504; + attempt += 1; + if (!shouldRetry || attempt >= VLM_MAX_RETRIES) break; + const backoff = Math.min(6000, 600 * 2 ** (attempt - 1)); + const retryAfterMs = retryAfterHeader > 0 ? retryAfterHeader * 1000 : 0; + const jitterMs = Math.floor(Math.random() * 250); + await sleep(Math.max(backoff, retryAfterMs) + jitterMs); + } + } + throw lastErr; +} + +// ── Asset library file ────────────────────────────────────────────────────── +const ASSET_LIBRARY_FILE = join(__dirname, "vlm-server", "asset-library.json"); + +// ── App ───────────────────────────────────────────────────────────────────── +const app = express(); +app.use(cors()); +app.use(express.json({ limit: "100mb" })); + +// ── POST /vlm/decision ────────────────────────────────────────────────────── +app.post("/vlm/decision", async (req, res) => { + try { + const { model, prompt, imageBase64, context, messages: reqMessages, max_tokens } = req.body; + const client = getClient(model); + + let messages = reqMessages; + if (!messages) { + const userContent = imageBase64 + ? [ + { type: "text", text: `Context:\n${JSON.stringify(context || {})}` }, + { type: "image_url", image_url: { url: `data:image/jpeg;base64,${imageBase64}` } }, + ] + : `Context:\n${JSON.stringify(context || {})}`; + messages = [ + { role: "system", content: prompt || "You are an AI agent. Output JSON only." }, + { role: "user", content: userContent }, + ]; + } + + const maxTok = max_tokens || 16384; + const params = { model, messages, temperature: 0.3 }; + if (model.startsWith("gemini")) { + params.max_tokens = maxTok; + } else { + params.max_completion_tokens = maxTok; + } + + const response = await requestWithRetry(() => client.chat.completions.create(params)); + const text = response.choices?.[0]?.message?.content || ""; + res.json({ raw: text }); + } catch (err) { + const status = Number(err?.status || err?.statusCode || 500); + const safeStatus = Number.isFinite(status) && status >= 400 && status <= 599 ? status : 500; + const msg = err?.message || "VLM request failed"; + console.error("[/vlm/decision]", safeStatus, msg); + res.status(safeStatus).json({ detail: msg }); + } +}); + +// ── POST /vlm/generate-image ──────────────────────────────────────────────── +app.post("/vlm/generate-image", async (req, res) => { + try { + const { prompt, size = "1024x1024", quality = "medium" } = req.body; + const client = new OpenAI({ apiKey: OPENAI_API_KEY }); + const fullPrompt = `Seamless tileable texture for 3D rendering, top-down flat view, no perspective: ${prompt}`; + + let lastError = null; + for (const modelName of ["gpt-image-1-mini", "gpt-image-1"]) { + try { + const response = await client.images.generate({ model: modelName, prompt: fullPrompt, size, quality, n: 1 }); + const b64 = response.data?.[0]?.b64_json; + if (b64) return res.json({ b64, dataUrl: `data:image/png;base64,${b64}`, model: modelName }); + } catch (err) { + lastError = err.message; + continue; + } + } + + const hash = createHash("sha256").update(fullPrompt).digest("hex"); + const h = parseInt(hash.slice(0, 8), 16); + const hue = h % 360; + const hue2 = (hue + 34) % 360; + const svg = ` + + +`; + res.json({ b64: null, dataUrl: "data:image/svg+xml;charset=utf-8," + encodeURIComponent(svg), fallback: true, error: lastError }); + } catch (err) { + console.error("[/vlm/generate-image]", err.message); + res.status(500).json({ detail: err.message }); + } +}); + +// ── GET /vlm/asset-library ────────────────────────────────────────────────── +app.get("/vlm/asset-library", (req, res) => { + try { + if (existsSync(ASSET_LIBRARY_FILE)) { + const data = JSON.parse(readFileSync(ASSET_LIBRARY_FILE, "utf-8")); + if (Array.isArray(data)) return res.json({ assets: data }); + } + } catch (err) { + console.error("[asset-library] read failed:", err.message); + } + res.json({ assets: [] }); +}); + +// ── POST /vlm/asset-library ───────────────────────────────────────────────── +app.post("/vlm/asset-library", (req, res) => { + try { + const { assets } = req.body; + writeFileSync(ASSET_LIBRARY_FILE, JSON.stringify(assets), "utf-8"); + res.json({ ok: true, count: assets?.length || 0 }); + } catch (err) { + console.error("[asset-library] write failed:", err.message); + res.status(500).json({ detail: err.message }); + } +}); + +// ── Start ─────────────────────────────────────────────────────────────────── +const PORT = process.env.PORT || 8000; +app.listen(PORT, "127.0.0.1", () => { + console.log(`DimSim VLM server running on http://127.0.0.1:${PORT}`); +}); diff --git a/misc/DimSim/src/AiAvatar.js b/misc/DimSim/src/AiAvatar.js new file mode 100644 index 0000000000..cd1075ae6d --- /dev/null +++ b/misc/DimSim/src/AiAvatar.js @@ -0,0 +1,1507 @@ +import * as THREE from "three"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; + +/** + * AiAvatar (SimStudio) + * - Vanilla THREE (no React) + * - Uses Rapier KinematicCharacterController for collision-aware movement + * - "Awareness" of tagged areas via `getTags()` and a sensing radius + * - Persistent memory stored in localStorage per (worldKey, agentId) + */ +export class AiAvatar { + constructor({ + id = null, + scene, + rapierWorld, + RAPIER, + getWorldKey, + getTags, + getPlayerPosition, + avatarUrl = "/avatars/kai.glb", + senseRadius = 3.0, + walkSpeed = 2.0, + vlm = null, + headless = false, + }) { + this.id = id || `ai-${Math.random().toString(36).slice(2, 8)}`; + this.scene = scene; + this.rapierWorld = rapierWorld; + this.RAPIER = RAPIER; + this.getWorldKey = getWorldKey || (() => "default"); + this.getTags = getTags || (() => []); + this.getPlayerPosition = getPlayerPosition || (() => [0, 0, 0]); + this.avatarUrl = avatarUrl; + this.senseRadius = senseRadius; + this.walkSpeed = walkSpeed; + this.vlm = vlm; // { enabled, endpoint, model, buildPrompt, actions, captureBase64, decideEverySteps, stepMeters } + this.headless = headless; // Skip visual rendering in headless mode, keep colliders + + // AI dimensions (match the smaller player so the agent fits in tight interiors too). + this.radius = 0.12; + this.halfHeight = 0.25; + + // Look pitch (radians). Used for agent POV capture to allow looking up/down. + // Kept separate from group rotation (yaw) so walking remains on XZ. + this.pitch = 0; + + this.group = new THREE.Group(); + this.group.name = `AiAvatar:${this.id}`; + this.scene.add(this.group); + + // Fallback visual (if GLB fails) + this.fallback = new THREE.Mesh( + new THREE.CapsuleGeometry(this.radius * 0.9, this.halfHeight * 2.0, 6, 10), + new THREE.MeshStandardMaterial({ color: 0x8cffc1, roughness: 0.65 }) + ); + this.fallback.castShadow = false; + this.fallback.receiveShadow = false; + this.group.add(this.fallback); + + // Facing indicator (always present so direction is obvious even with capsule / GLB). + // Cone points along +Z (our forward convention in this project). + this._facing = new THREE.Mesh( + new THREE.ConeGeometry(this.radius * 0.45, this.radius * 1.35, 14), + new THREE.MeshStandardMaterial({ color: 0xffc36a, roughness: 0.45, metalness: 0.05 }) + ); + this._facing.rotation.x = Math.PI / 2; + this._facing.position.set(0, this.halfHeight * 0.15, this.radius * 1.1); + this._facing.castShadow = false; + this._facing.receiveShadow = false; + this._facing.renderOrder = 10; + this.group.add(this._facing); + + // Thought label (simple sprite) + this._labelCanvas = document.createElement("canvas"); + this._labelCanvas.width = 512; + this._labelCanvas.height = 512; + this._labelCtx = this._labelCanvas.getContext("2d"); + this._labelTex = new THREE.CanvasTexture(this._labelCanvas); + this._labelSprite = new THREE.Sprite( + new THREE.SpriteMaterial({ + map: this._labelTex, + transparent: true, + depthWrite: false, + depthTest: false, // always visible (splats/voxels can otherwise cover it) + toneMapped: false, + }) + ); + this._labelSprite.scale.set(2.0, 1.0, 1); + this._labelSprite.position.set(0, this.halfHeight * 2 + this.radius + 0.8, 0); + this._labelSprite.renderOrder = 5000; // after SparkRenderer (999) + this.group.add(this._labelSprite); + this._setThought(""); + this._lastDecisionBubbleAt = 0; + + this._target = null; // THREE.Vector3 + this._state = "IDLE"; // IDLE | WALK | INSPECT + this._stateUntil = 0; + this._nearbyTagId = null; + this._lastInspectAtByTagId = {}; + this._inspectCooldownMs = 8000; // don't repeatedly "stop" while standing inside a big tag radius + + // VLM action plan + this._plan = null; // { type, ... } + this._planRemaining = 0; + this._stepCounter = 0; + this._nextDecisionAt = 0; + this._decisionJitterMs = (Math.random() * 350) | 0; + this._vlmInFlight = null; + this._pendingDecision = null; + + // Short-term trace for the current task run (fed back to the VLM). + this._taskStartedAt = 0; + this._trace = []; // [{t,type,msg,data?}] + this._traceLimit = 20; + // Prior model outputs for the current task run (fed back as assistant messages). + this._vlmAssistantHistory = []; // string[] + this._vlmAssistantHistoryLimit = 15; + this._moveStepAcc = 0; + this._turnAcc = 0; + + // Task state tracking + this._startPosition = null; // {x,y,z} - where agent was when task started + this._currentSubgoal = ""; // Current sub-goal from VLM reasoning + this._discoveredItems = []; // Items/assets found during exploration + + // Rapier body/collider/controller + const radius = this.radius; + const halfHeight = this.halfHeight; + this.body = this.rapierWorld.createRigidBody( + this.RAPIER.RigidBodyDesc.kinematicPositionBased().setTranslation(0, 3, 0) + ); + this.collider = this.rapierWorld.createCollider( + this.RAPIER.ColliderDesc.capsule(halfHeight, radius).setFriction(0.8), + this.body + ); + // Fake robodog body volume: add a horizontal spine capsule to reduce rear clipping. + // Movement still uses the vertical capsule for stable character-controller behavior. + this._spineHalfLen = Math.max(radius * 1.2, 0.13); + this._spineRadius = Math.max(radius * 0.62, 0.07); + // Keep fake volume mostly behind body center so it doesn't "push from the air" in front. + this._spineOffsetBack = Math.max(radius * 2.2, this._spineHalfLen + this._spineRadius + 0.02); + this._spineOffsetY = Math.max(this.halfHeight * 0.35, 0.08); + this.spineCollider = this.rapierWorld.createCollider( + this.RAPIER.ColliderDesc.capsule(this._spineHalfLen, this._spineRadius) + .setFriction(0.8) + .setTranslation(0, this._spineOffsetY, -this._spineOffsetBack) + // Rotate local +Y capsule axis to +Z (dog spine direction at yaw=0). + .setRotation({ x: Math.SQRT1_2, y: 0, z: 0, w: Math.SQRT1_2 }), + this.body + ); + this.boxCollider = null; // Box collider created when GLB loads (matches model dimensions) + this.controller = this.rapierWorld.createCharacterController(0.05); + this.controller.enableAutostep(0.25, 0.15, true); + this.controller.enableSnapToGround(0.5); + this.controller.setSlideEnabled(true); + this.controller.setMaxSlopeClimbAngle((45 * Math.PI) / 180); + this.controller.setMinSlopeSlideAngle((75 * Math.PI) / 180); + + // Persistent memory + this.memory = this._loadMemory(); + + // Best-effort: load GLB visual + this._loadGLB(); + } + + dispose() { + try { + this.scene.remove(this.group); + } catch {} + try { + if (this.boxCollider) this.rapierWorld.removeCollider(this.boxCollider, true); + if (this.spineCollider) this.rapierWorld.removeCollider(this.spineCollider, true); + this.rapierWorld.removeCollider(this.collider, true); + this.rapierWorld.removeRigidBody(this.body); + } catch {} + } + + setPosition(x, y, z) { + this.body.setTranslation({ x, y, z }, true); + this.body.setLinvel({ x: 0, y: 0, z: 0 }, true); + } + + getPosition() { + const p = this.body.translation(); + return [p.x, p.y, p.z]; + } + + update(dt, nowMs) { + if (!this.rapierWorld || !this.body) return; + const now = nowMs ?? Date.now(); + + if (this.mixer) this.mixer.update(dt); + + // Sense tags in radius and update memory. + const p = this.body.translation(); + const tags = this.getTags() || []; + const nearby = []; + for (const t of tags) { + if (!t?.position) continue; + const dx = t.position.x - p.x; + const dy = t.position.y - p.y; + const dz = t.position.z - p.z; + const d = Math.sqrt(dx * dx + dy * dy + dz * dz); + const r = Math.max(Number(t.radius ?? 1.5), this.senseRadius); + if (d <= r) nearby.push({ tag: t, dist: d }); + } + nearby.sort((a, b) => a.dist - b.dist); + + if (nearby.length > 0) { + const top = nearby[0].tag; + this._rememberTag(top); + const topId = top.id || null; + const title = top.title || "(untitled)"; + // Don't continuously overwrite model-output bubble while the VLM is active. + const taskActive = !!this.vlm?.getTask?.()?.active; + if (!this.vlm?.enabled && !taskActive && now - this._lastDecisionBubbleAt > 1200) { + this._setThought(`I see: ${title}`); + } + + // Only enter INSPECT when we newly encounter a tag OR the cooldown elapsed. + // Without this, a large tag radius causes the agent to repeatedly re-enter INSPECT + // and look "stuck" near the tag forever. + const lastAt = (topId && this._lastInspectAtByTagId[topId]) || 0; + const cooldownOk = !topId || now - lastAt > this._inspectCooldownMs; + const newlyEntered = topId && topId !== this._nearbyTagId; + this._nearbyTagId = topId; + if (this._state !== "INSPECT" && (newlyEntered || cooldownOk)) { + if (topId) this._lastInspectAtByTagId[topId] = now; + this._state = "INSPECT"; + this._stateUntil = now + 900; + this._target = null; + } + } else { + this._nearbyTagId = null; + } + + if (this._state === "INSPECT") { + if (now > this._stateUntil) { + this._state = "IDLE"; + } + // When inspecting, gently look toward the closest tag in full 3D (pitch + yaw). + try { + const tags = this.getTags?.() || []; + const p = this.body.translation(); + let best = null; + let bestD = Infinity; + for (const t of tags) { + if (!t?.position) continue; + const dx = t.position.x - p.x; + const dy = t.position.y - (p.y + this.halfHeight); // approximate eye height + const dz = t.position.z - p.z; + const d = Math.hypot(dx, dy, dz); + if (d < bestD) { + bestD = d; + best = { dx, dy, dz }; + } + } + if (best) { + // Yaw toward target (XZ) and pitch toward target (Y). + this.group.rotation.y = Math.atan2(best.dx, best.dz); + const horiz = Math.hypot(best.dx, best.dz) || 1; + const desiredPitch = Math.atan2(best.dy, horiz); + const maxPitch = (85 * Math.PI) / 180; + // Smooth pitch change to avoid snapping. + const alpha = Math.min(1, dt * 8); + const clamped = Math.max(-maxPitch, Math.min(maxPitch, desiredPitch)); + this.pitch = this.pitch + (clamped - this.pitch) * alpha; + } + } catch { + // ignore + } + this._syncVisual(); + this._publishGlobals(); + return; + } + + // If VLM mode is enabled, it drives the movement plan. + if (this.vlm?.enabled) { + this._vlmUpdate(dt, now, nearby); + if (!this._plan) this._applyIdleGravity(dt); + this._syncVisual(); + this._publishGlobals(); + return; + } + + // Editor helper mode: hold position when not actively tasked. + // This prevents idle spawned agents from random-walking before assignment. + if (this.vlm?.holdPositionWhenIdle) { + const task = this.vlm?.getTask?.(); + if (!task?.active || !this.vlm?.enabled) { + this._state = "IDLE"; + this._target = null; + this._setThought(""); + this._applyIdleGravity(dt); + this._syncVisual(); + this._publishGlobals(); + return; + } + } + + // Pick a new wander target if needed. + if (!this._target || this._state === "IDLE") { + this._target = this._pickWanderTarget(p); + this._state = "WALK"; + } + + // Move toward target on XZ plane with collision sliding. + const dx = this._target.x - p.x; + const dz = this._target.z - p.z; + const dist = Math.sqrt(dx * dx + dz * dz); + if (dist < 0.4) { + this._state = "IDLE"; + this._target = null; + this._setThought(""); + this._syncVisual(); + this._publishGlobals(); + return; + } + + const vx = (dx / (dist || 1)) * this.walkSpeed; + const vz = (dz / (dist || 1)) * this.walkSpeed; + const desired = { x: vx * dt, y: -2.0 * dt, z: vz * dt }; // small down force to keep grounded + + // Query pipeline is updated by rapierWorld.step() in the main loop + const m = this._computeConservativeMovement(desired); + const mx = m.x, my = m.y, mz = m.z; + this.body.setNextKinematicTranslation({ x: p.x + mx, y: p.y + my, z: p.z + mz }); + + // Face direction. + const yaw = Math.atan2(vx, vz); + this.group.rotation.y = yaw; + // While walking, ease pitch back to level. + { + const alpha = Math.min(1, dt * 6); + this.pitch = this.pitch + (0 - this.pitch) * alpha; + } + + this._syncVisual(); + this._publishGlobals(); + } + + // --- internals --- + _vlmUpdate(dt, now, nearby) { + const task = this.vlm?.getTask?.(); + // If there is no active instruction, do not call the VLM (keeps UI quiet and enforces one-task-at-a-time). + if (!task?.active) return; + + // Reset per-task trace if a new task started. + const startedAt = Number(task.startedAt || 0); + if (startedAt && startedAt !== this._taskStartedAt) { + this._taskStartedAt = startedAt; + this._trace = []; + this._vlmAssistantHistory = []; + this._stepCounter = 0; + this._plan = null; + this._planRemaining = 0; + this._moveStepAcc = 0; + this._turnAcc = 0; + this._currentSubgoal = ""; + this._discoveredItems = []; + // Record start position for "return to start" navigation + const [sx, sy, sz] = this.getPosition?.() || [0, 0, 0]; + this._startPosition = { x: sx, y: sy, z: sz }; + this._tracePush(now, "TASK_START", { instruction: String(task.instruction || ""), startPos: this._startPosition }); + } + + // Apply any resolved decision. + if (this._pendingDecision) { + const d = this._pendingDecision; + this._pendingDecision = null; + this._applyVlmDecision(d); + } + + // Execute current plan if any. + if (this._plan) { + const done = this._stepPlan(dt); + if (!done) return; + // Plan completed. + { + const [x, y, z] = this.getPosition?.() || [0, 0, 0]; + const yaw = this.group?.rotation?.y ?? 0; + this._tracePush(now, "PLAN_DONE", { plan: this._plan, pose: { x, y, z, yaw } }); + } + this._plan = null; + this._planRemaining = 0; + } + + // Decide periodically (every N executed steps) or when idle. + const decideEverySteps = Number(this.vlm.decideEverySteps ?? 4); + if (now < this._nextDecisionAt) return; + if (this._vlmInFlight) return; + + // Kick off async decision (non-blocking). + this._vlmInFlight = this._requestVlmDecision(now, nearby) + .then((d) => { + this._pendingDecision = d; + }) + .catch((e) => { + // If it fails, wait a bit then try again. + this._nextDecisionAt = now + 2000; + this._setThought(`VLM error`); + try { + this.vlm?.onError?.(e); + } catch {} + console.warn("VLM decision failed:", e); + }) + .finally(() => { + this._vlmInFlight = null; + }); + + // Rate limit: even if we render at 60fps, don't spam. + this._nextDecisionAt = now + Math.max(500, decideEverySteps * 250) + this._decisionJitterMs; + } + + async _requestVlmDecision(now, nearby) { + // Increment step counter once per VLM decision + this._stepCounter++; + + const capture = this.vlm.captureBase64; + const prompt = this.vlm.buildPrompt?.({ actions: this.vlm.actions }) ?? ""; + const model = this.vlm.getModel?.() || this.vlm.model; + const endpoint = this.vlm.endpoint; + + const imageBase64 = await capture(this); + if (!imageBase64) throw new Error("No image."); + try { + this.vlm.onCapture?.(imageBase64); + } catch {} + + const [ax, ay, az] = this.getPosition?.() || [0, 0, 0]; + const yaw = this.group?.rotation?.y ?? 0; + const pitch = typeof this.pitch === "number" ? this.pitch : 0; + + const pose = { x: ax.toFixed(2), y: ay.toFixed(2), z: az.toFixed(2), yaw: (yaw * 180 / Math.PI).toFixed(0) + "°", pitch: (pitch * 180 / Math.PI).toFixed(0) + "°" }; + + // Get nearby assets with simplified info (filter out held items - they're not "nearby") + const nearbyAssets = (this.vlm?.getNearbyAssets?.(this) || []) + .filter(a => !a.isHeld) // Don't show held items in nearby list + .map((a) => ({ + id: a.id, + name: a.title || a.id, + description: a.notes || "", + distance: a.dist.toFixed(1) + "m", + lookingAt: a.isLookedAt, + state: a.currentStateName || a.currentState, + canDo: (a.actions || []).map((x) => x.label).filter(Boolean), + pickable: a.pickable || false, + })); + const nearbyPrimitives = (this.vlm?.getNearbyPrimitives?.(this) || []).map((p) => ({ + id: p.id, + name: p.name || p.id, + distance: p.dist.toFixed(1) + "m", + lookingAt: !!p.isLookedAt, + type: p.type || "primitive", + })); + const assetLibraryNames = (this.vlm?.getAssetLibraryNames?.() || []).slice(0, 12); + const isEditorMode = !!this.vlm?.isEditorMode?.(); + + // Get what the agent is currently holding + const heldAsset = this.vlm?.getHeldAsset?.(this); + const recentGeneratedAssets = (this.vlm?.getRecentGeneratedAssets?.(this) || []).slice(0, 8); + + // Get nearby tags/locations + const nearbyLocations = (nearby || []).slice(0, 6).map((x) => ({ + id: x.tag?.id, + name: x.tag?.title || "unknown", + distance: x.dist.toFixed(1) + "m", + })); + + // Build condensed context for user message + const task = this.vlm?.getTask?.(); + const taskInstruction = String(task?.instruction || "No active task"); + + // Chat-style message history: system prompt + prior assistant outputs + current user (context + image). + const assistantMsgs = (this._vlmAssistantHistory || []) + .slice(-this._vlmAssistantHistoryLimit) + .filter((s) => typeof s === "string" && s.trim().length > 0) + .map((s) => ({ role: "assistant", content: s })); + + // Build concise user message + const lines = [ + `TASK: ${taskInstruction}`, + // `${this._stepCounter}${this._stepCounter === 1 ? 'st' : this._stepCounter === 2 ? 'nd' : this._stepCounter === 3 ? 'rd' : 'th'} ACTION`, + `POSITION: ${pose.x}, ${pose.y}, ${pose.z} | facing ${pose.yaw} yaw, ${pose.pitch} pitch`, + ]; + + if (this._startPosition) { + lines.push(`START POSITION: ${this._startPosition.x.toFixed(1)}, ${this._startPosition.y.toFixed(1)}, ${this._startPosition.z.toFixed(1)}`); + } + + // Show what the agent is currently holding + if (heldAsset) { + lines.push(`\nHOLDING: ${heldAsset.title || heldAsset.id} [id: ${heldAsset.id}]`); + lines.push(` → Use DROP to place it in front of you`); + } + + if (nearbyAssets.length > 0) { + lines.push(`\nNEARBY OBJECTS:`); + for (const a of nearbyAssets) { + const looking = a.lookingAt ? " [LOOKING AT]" : ""; + const pickable = a.pickable ? " [pickable]" : ""; + const portal = a.isPortal ? ` [PORTAL → ${a.destinationWorld || "?"}]` : ""; + const actions = a.canDo.length > 0 ? ` → can: ${a.canDo.join(", ")}` : ""; + // Include the asset ID so the VLM can use it for INTERACT/PICK_UP actions + lines.push(` • ${a.name} [id: ${a.id}] (${a.distance})${looking}${pickable}${portal} - ${a.state}${actions}`); + } + } + if (nearbyPrimitives.length > 0) { + lines.push(`\nNEARBY PRIMITIVES:`); + for (const p of nearbyPrimitives) { + const looking = p.lookingAt ? " [LOOKING AT]" : ""; + lines.push(` • ${p.name} [id: ${p.id}] (${p.distance})${looking} - ${p.type}`); + } + } + if (recentGeneratedAssets.length > 0) { + lines.push(`\nRECENT GENERATED ASSETS (transform these by ID even if not in nearby list):`); + for (const a of recentGeneratedAssets) { + const name = String(a?.name || "generated-asset"); + const id = String(a?.id || ""); + if (!id) continue; + lines.push(` • ${name} [id: ${id}]`); + } + } + + if (nearbyLocations.length > 0) { + lines.push(`\nNEARBY LOCATIONS:`); + for (const loc of nearbyLocations) { + lines.push(` • ${loc.name} (${loc.distance}) [id: ${loc.id}]`); + } + } + lines.push(`\nEDITOR MODE: ${isEditorMode ? "ON" : "OFF"}`); + if (isEditorMode && assetLibraryNames.length > 0) { + lines.push(`ASSET LIBRARY: ${assetLibraryNames.join(", ")}`); + } + + const userText = lines.join("\n"); + + // Keep context object for backwards compatibility but simplified + const context = { + task: { instruction: taskInstruction, active: !!task?.active }, + pose: { x: ax, y: ay, z: az, yaw, pitch }, + step: this._stepCounter, + startPosition: this._startPosition, + editorMode: isEditorMode, + nearbyAssets, + nearbyPrimitives, + assetLibraryNames, + nearbyLocations, + heldAsset: heldAsset ? { id: heldAsset.id, title: heldAsset.title } : null, + }; + + const messages = [ + { role: "system", content: prompt }, + ...assistantMsgs, + { + role: "user", + content: [ + { type: "text", text: userText }, + { type: "image_url", image_url: { url: `data:image/jpeg;base64,${imageBase64}` } }, + ], + }, + ]; + + try { + this.vlm?.onRequest?.({ endpoint, model, prompt, context, imageBase64, messages }); + } catch {} + + const res = await this.vlm.request({ + endpoint, + model, + prompt, + imageBase64, + context, + messages, + }); + + // Server returns {raw: "..."} or direct JSON object. + const raw = res?.raw ?? res; + // Persist raw model output into per-task assistant history for next turn context. + try { + const rawStr = typeof raw === "string" ? raw : JSON.stringify(raw); + this._vlmAssistantHistory.push(rawStr); + if (this._vlmAssistantHistory.length > this._vlmAssistantHistoryLimit) { + this._vlmAssistantHistory.splice(0, this._vlmAssistantHistory.length - this._vlmAssistantHistoryLimit); + } + } catch { + // ignore + } + const parsed = typeof raw === "string" ? safeParseJson(raw) : raw; + if (!parsed || typeof parsed !== "object") throw new Error("Invalid VLM output."); + // Update speech bubble immediately on each model turn (before action application). + try { + const responseBubble = this._extractBubbleTextFromModelOutput(parsed, raw); + if (responseBubble) { + this._setThought(responseBubble); + this._lastDecisionBubbleAt = Date.now(); + } + } catch {} + try { + this.vlm?.onResponse?.({ raw: typeof raw === "string" ? raw : JSON.stringify(raw), parsed }); + } catch {} + return parsed; + } + + _applyVlmDecision(decision) { + // Extract reasoning/thinking for logs and bubble display. + const paramsForBubble = decision?.params && typeof decision.params === "object" ? decision.params : {}; + const thinking = decision.thinking || decision.thought || decision.reasoning || ""; + const observation = + decision.observation || + decision.obs || + decision.perception || + paramsForBubble.observation || + ""; + const subgoal = decision.currentSubgoal || ""; + + // Update current sub-goal tracking + if (subgoal) this._currentSubgoal = subgoal; + + // Display observation bubble (prioritize what the agent sees, not chain-of-thought). + // If observation is absent, show a concise action status so the bubble still updates each turn. + const actionPreview = String(decision?.action || "").trim(); + const paramPreview = Object.entries(paramsForBubble) + .slice(0, 2) + .map(([k, v]) => `${k}=${typeof v === "string" ? v : JSON.stringify(v)}`) + .join(", "); + const fallbackStatus = actionPreview + ? `Action: ${actionPreview}${paramPreview ? ` (${paramPreview})` : ""}` + : subgoal || ""; + const displayText = String(observation || fallbackStatus || "").trim(); + this._setThought(displayText); + this._lastDecisionBubbleAt = Date.now(); + + try { + this.vlm?.onDecision?.(decision); + } catch {} + + const action = String(decision.action || "").trim().toUpperCase(); + const params = decision.params && typeof decision.params === "object" ? decision.params : {}; + + const [x, y, z] = this.getPosition?.() || [0, 0, 0]; + const yaw = this.group?.rotation?.y ?? 0; + this._tracePush(Date.now(), "ACTION", { + action, + params, + thinking: thinking.slice(0, 100), + subgoal, + pose: { x: x.toFixed(2), y: y.toFixed(2), z: z.toFixed(2), yaw: (yaw * 180 / Math.PI).toFixed(0) } + }); + + // Normalize bounds helpers + const clampInt = (v, a, b, d) => { + const n = Math.floor(Number(v)); + return Number.isFinite(n) ? Math.max(a, Math.min(b, n)) : d; + }; + const clampNum = (v, a, b, d) => { + const n = Number(v); + return Number.isFinite(n) ? Math.max(a, Math.min(b, n)) : d; + }; + + // === DONE / FINISH_TASK === + if (action === "DONE" || action === "FINISH_TASK") { + const summary = String(params.summary || "Task completed"); + const confidence = clampNum(params.confidence, 0, 1, 1); + try { + this.vlm?.onTaskFinished?.({ summary, confidence }); + } catch {} + this._tracePush(Date.now(), "TASK_DONE", { summary, confidence }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + + // === THINK - pause to reason === + if (action === "THINK") { + const thought = String(params.thought || thinking || "thinking..."); + // THINK action still surfaces as a visible status, but never truncated. + this._setThought(thought); + this._tracePush(Date.now(), "THINK", { thought }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + + // === INTERACT / INTERACT_ASSET === + if (action === "INTERACT" || action === "INTERACT_ASSET") { + const assetId = String(params.assetId || ""); + // Support both actionLabel (new) and actionId (old) + const actionLabel = String(params.actionLabel || params.actionId || ""); + + console.log(`[AI INTERACT] Raw params: assetId="${assetId}", actionLabel="${actionLabel}"`); + + // Find matching action by label + const nearbyAssets = this.vlm?.getNearbyAssets?.(this) || []; + console.log(`[AI INTERACT] Nearby assets:`, nearbyAssets.map(a => ({ + id: a.id, + title: a.title, + dist: a.dist?.toFixed(2), + isLookedAt: a.isLookedAt, + currentState: a.currentState, + actions: a.actions?.map(act => `${act.id}:${act.label}(${act.from}->${act.to})`) + }))); + + const targetAsset = nearbyAssets.find((a) => a.id === assetId); + let actionId = actionLabel; + + if (targetAsset && targetAsset.actions) { + console.log(`[AI INTERACT] Target asset found: "${targetAsset.title}", looking for action: "${actionLabel}"`); + console.log(`[AI INTERACT] Available actions:`, targetAsset.actions); + + const matchingAction = targetAsset.actions.find( + (a) => a.label?.toLowerCase() === actionLabel.toLowerCase() || a.id === actionLabel + ); + if (matchingAction) { + console.log(`[AI INTERACT] Matched action: "${matchingAction.id}" (label: "${matchingAction.label}")`); + actionId = matchingAction.id; + } else { + console.warn(`[AI INTERACT] No matching action found for label "${actionLabel}"`); + } + } else { + console.warn(`[AI INTERACT] Target asset "${assetId}" not found in nearby assets`); + } + + console.log(`[AI INTERACT] Final actionId to use: "${actionId}"`); + this._tracePush(Date.now(), "INTERACT", { assetId, actionLabel, actionId }); + + Promise.resolve() + .then(() => this.vlm?.interactAsset?.({ agent: this, assetId, actionId })) + .then((res) => { + if (res?.ok) { + this._tracePush(Date.now(), "INTERACT_OK", { assetId, actionLabel }); + this._discoveredItems.push({ id: assetId, interacted: true, at: Date.now() }); + } else { + this._tracePush(Date.now(), "INTERACT_FAIL", { assetId, actionLabel, reason: res?.reason }); + } + }) + .catch((e) => { + this._tracePush(Date.now(), "INTERACT_FAIL", { assetId, actionLabel, reason: String(e?.message || e) }); + }); + + this._plan = { type: "WAIT" }; + this._planRemaining = 0.6; + return; + } + + // === PICK_UP === + if (action === "PICK_UP") { + const assetId = String(params.assetId || ""); + console.log(`[AI PICK_UP] Attempting to pick up: assetId="${assetId}"`); + + this._tracePush(Date.now(), "PICK_UP", { assetId }); + + Promise.resolve() + .then(() => this.vlm?.pickUpAsset?.({ agent: this, assetId })) + .then((res) => { + if (res?.ok) { + this._tracePush(Date.now(), "PICK_UP_OK", { assetId }); + console.log(`[AI PICK_UP] Successfully picked up: ${assetId}`); + } else { + this._tracePush(Date.now(), "PICK_UP_FAIL", { assetId, reason: res?.reason }); + console.warn(`[AI PICK_UP] Failed: ${res?.reason}`); + } + }) + .catch((e) => { + this._tracePush(Date.now(), "PICK_UP_FAIL", { assetId, reason: String(e?.message || e) }); + }); + + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + + // === DROP === + if (action === "DROP") { + console.log(`[AI DROP] Attempting to drop held item`); + + this._tracePush(Date.now(), "DROP", {}); + + Promise.resolve() + .then(() => this.vlm?.dropAsset?.({ agent: this })) + .then((res) => { + if (res?.ok) { + this._tracePush(Date.now(), "DROP_OK", { assetId: res.assetId }); + console.log(`[AI DROP] Successfully dropped: ${res.assetId}`); + } else { + this._tracePush(Date.now(), "DROP_FAIL", { reason: res?.reason }); + console.warn(`[AI DROP] Failed: ${res?.reason}`); + } + }) + .catch((e) => { + this._tracePush(Date.now(), "DROP_FAIL", { reason: String(e?.message || e) }); + }); + + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + + // === TURN_LEFT / TURN_RIGHT === + if (action === "TURN_LEFT" || action === "TURN_RIGHT") { + const deg = clampNum(params.degrees, 15, 90, 30); + this._plan = { type: "TURN", dir: action === "TURN_LEFT" ? 1 : -1, radians: (deg * Math.PI) / 180 }; + this._planRemaining = this._plan.radians; + this._turnAcc = 0; + return; + } + + // === LOOK_UP / LOOK_DOWN / PITCH_UP / PITCH_DOWN === + if (action === "LOOK_UP" || action === "PITCH_UP") { + const deg = clampNum(params.degrees, 10, 60, 20); + this._plan = { type: "PITCH", dir: 1, radians: (deg * Math.PI) / 180 }; + this._planRemaining = this._plan.radians; + return; + } + if (action === "LOOK_DOWN" || action === "PITCH_DOWN") { + const deg = clampNum(params.degrees, 10, 60, 20); + this._plan = { type: "PITCH", dir: -1, radians: (deg * Math.PI) / 180 }; + this._planRemaining = this._plan.radians; + return; + } + + // === EDITOR: CREATE_PRIMITIVE === + if (action === "CREATE_PRIMITIVE") { + if (!this.vlm?.isEditorMode?.()) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: "not-edit-mode" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + const shape = String(params.shape || "box").toLowerCase(); + this._tracePush(Date.now(), "EDIT_CREATE_PRIMITIVE", { shape }); + Promise.resolve() + .then(() => this.vlm?.createPrimitiveInEditor?.({ agent: this, shape })) + .then((res) => { + if (!res?.ok) this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: res?.reason || "create-failed" }); + }) + .catch((e) => { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: String(e?.message || e) }); + }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.45; + return; + } + + // === EDITOR: SPAWN_LIBRARY_ASSET === + if (action === "SPAWN_LIBRARY_ASSET") { + if (!this.vlm?.isEditorMode?.()) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: "not-edit-mode" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + const assetName = String(params.assetName || "").trim(); + this._tracePush(Date.now(), "EDIT_SPAWN_LIBRARY_ASSET", { assetName }); + Promise.resolve() + .then(() => this.vlm?.spawnLibraryAssetInEditor?.({ agent: this, assetName })) + .then((res) => { + if (!res?.ok) this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: res?.reason || "spawn-failed" }); + }) + .catch((e) => { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: String(e?.message || e) }); + }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.55; + return; + } + + // === EDITOR: TRANSFORM_OBJECT === + if (action === "TRANSFORM_OBJECT") { + if (!this.vlm?.isEditorMode?.()) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: "not-edit-mode" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + const targetType = String(params.targetType || "").toLowerCase(); + const targetId = String(params.targetId || ""); + const editParams = { + targetType, + targetId, + moveX: Number(params.moveX) || 0, + moveY: Number(params.moveY) || 0, + moveZ: Number(params.moveZ) || 0, + rotateYDeg: Number(params.rotateYDeg) || 0, + scaleMul: Number(params.scaleMul) || 1, + setPositionX: Number.isFinite(Number(params.setPositionX)) ? Number(params.setPositionX) : undefined, + setPositionY: Number.isFinite(Number(params.setPositionY)) ? Number(params.setPositionY) : undefined, + setPositionZ: Number.isFinite(Number(params.setPositionZ)) ? Number(params.setPositionZ) : undefined, + setRotationYDeg: Number.isFinite(Number(params.setRotationYDeg)) ? Number(params.setRotationYDeg) : undefined, + setScaleX: Number.isFinite(Number(params.setScaleX)) ? Number(params.setScaleX) : undefined, + setScaleY: Number.isFinite(Number(params.setScaleY)) ? Number(params.setScaleY) : undefined, + setScaleZ: Number.isFinite(Number(params.setScaleZ)) ? Number(params.setScaleZ) : undefined, + snapToCrosshair: params.snapToCrosshair === true, + }; + this._tracePush(Date.now(), "EDIT_TRANSFORM_OBJECT", editParams); + Promise.resolve() + .then(() => this.vlm?.transformObjectInEditor?.({ ...editParams, agent: this })) + .then((res) => { + if (!res?.ok) this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: res?.reason || "transform-failed" }); + }) + .catch((e) => { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: String(e?.message || e) }); + }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.45; + return; + } + + // === EDITOR: GENERATE_ASSET === + if (action === "GENERATE_ASSET") { + if (!this.vlm?.isEditorMode?.()) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: "not-edit-mode" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + const prompt = String(params.prompt || "").trim(); + if (!prompt) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: "missing-prompt" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + const placeNow = params.placeNow !== false; + this._tracePush(Date.now(), "EDIT_GENERATE_ASSET", { prompt: prompt.slice(0, 120), placeNow }); + Promise.resolve() + .then(() => this.vlm?.generateAssetInEditor?.({ agent: this, prompt, placeNow })) + .then((res) => { + if (!res?.ok) { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: res?.reason || "asset-generate-failed" }); + return; + } + this._tracePush(Date.now(), "EDIT_ASSET_READY", { + action, + assetName: String(res?.assetName || ""), + assetId: String(res?.assetId || ""), + reused: !!res?.reused, + placed: !!res?.placed, + }); + const labelId = res?.assetId ? ` [id: ${res.assetId}]` : ""; + const kind = res?.reused ? "Reused" : "Generated"; + this._setThought(`${kind}: ${String(res?.assetName || "asset")}${labelId}. Next: transform to fit.`); + }) + .catch((e) => { + this._tracePush(Date.now(), "EDIT_FAIL", { action, reason: String(e?.message || e) }); + }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.8; + return; + } + + // === GOTO_LOCATION / GOTO_TAG === + if (action === "GOTO_LOCATION" || action === "GOTO_TAG") { + const locId = String(params.locationId || params.tagId || ""); + + // Special case: "start" returns to start position + if (locId === "start" && this._startPosition) { + this._plan = { type: "GOTO", x: this._startPosition.x, z: this._startPosition.z }; + this._planRemaining = 999; + this._tracePush(Date.now(), "GOTO_START", {}); + return; + } + + // Find tag by ID + const tags = this.getTags?.() || []; + const t = tags.find((x) => x?.id === locId); + if (t?.position) { + this._plan = { type: "GOTO", x: t.position.x, z: t.position.z }; + this._planRemaining = 999; + return; + } + + // Tag not found - wait briefly + this._tracePush(Date.now(), "GOTO_FAIL", { locId, reason: "not found" }); + this._plan = { type: "WAIT" }; + this._planRemaining = 0.5; + return; + } + + // === MOVEMENT === + const isMove = ["MOVE_FORWARD", "MOVE_BACKWARD", "STRAFE_LEFT", "STRAFE_RIGHT", "MOVE_UP", "MOVE_DOWN"].includes(action); + if (isMove) { + const steps = clampInt(params.steps, 1, 8, 2); + this._plan = { type: "MOVE", dir: action, steps }; + this._planRemaining = steps; + this._moveStepAcc = 0; + return; + } + + // === WAIT (explicit or fallback) === + const secs = clampNum(params.seconds, 0.3, 3, 0.5); + this._plan = { type: "WAIT" }; + this._planRemaining = secs; + } + + _stepPlan(dt) { + if (!this._plan) return true; + const p = this.body.translation(); + + if (this._plan.type === "WAIT") { + this._planRemaining -= dt; + return this._planRemaining <= 0; + } + + if (this._plan.type === "TURN") { + const turnRate = 2.4; // rad/sec + const dYaw = Math.min(this._planRemaining, turnRate * dt); + this.group.rotation.y += dYaw * this._plan.dir; + this._planRemaining -= dYaw; + this._turnAcc += dYaw; + if (this._turnAcc >= Math.PI / 6) { + this._tracePush(Date.now(), "TURN_PROGRESS", { degrees: Math.round((this._turnAcc * 180) / Math.PI) }); + this._turnAcc = 0; + } + return this._planRemaining <= 0; + } + + if (this._plan.type === "PITCH") { + const pitchRate = 2.6; // rad/sec + const d = Math.min(this._planRemaining, pitchRate * dt); + const before = this.pitch || 0; + const maxPitch = (85 * Math.PI) / 180; + let after = before + d * this._plan.dir; + after = Math.max(-maxPitch, Math.min(maxPitch, after)); + this.pitch = after; + const applied = Math.abs(after - before); + this._planRemaining -= applied; + // If we hit the clamp and couldn't apply, consider the action done. + if (applied <= 1e-6) this._planRemaining = 0; + return this._planRemaining <= 0; + } + + // Compute basis from yaw. + const yaw = this.group.rotation.y; + const forward = new THREE.Vector3(Math.sin(yaw), 0, Math.cos(yaw)); + const right = new THREE.Vector3(forward.z, 0, -forward.x); + + if (this._plan.type === "GOTO") { + const dx = this._plan.x - p.x; + const dz = this._plan.z - p.z; + const dist = Math.sqrt(dx * dx + dz * dz); + if (dist < 0.35) return true; + const vx = (dx / (dist || 1)) * this.walkSpeed; + const vz = (dz / (dist || 1)) * this.walkSpeed; + const desired = { x: vx * dt, y: -2.0 * dt, z: vz * dt }; + const m = this._computeConservativeMovement(desired); + const mx = m.x, my = m.y, mz = m.z; + this.body.setNextKinematicTranslation({ x: p.x + mx, y: p.y + my, z: p.z + mz }); + this.group.rotation.y = Math.atan2(vx, vz); + return false; + } + + if (this._plan.type === "MOVE") { + const stepMeters = Number(this.vlm.stepMeters ?? 0.35); + const dir = this._plan.dir; + let v = new THREE.Vector3(); + if (dir === "MOVE_FORWARD") v.copy(forward); + if (dir === "MOVE_BACKWARD") v.copy(forward).multiplyScalar(-1); + if (dir === "STRAFE_LEFT") v.copy(right).multiplyScalar(-1); + if (dir === "STRAFE_RIGHT") v.copy(right); + if (dir === "MOVE_UP") v.set(0, 1, 0); + if (dir === "MOVE_DOWN") v.set(0, -1, 0); + + // Consume "steps" by distance traveled (approx). + const speed = this.walkSpeed; + const distThisFrame = speed * dt; + const verticalMove = dir === "MOVE_UP" || dir === "MOVE_DOWN"; + const desired = verticalMove + ? { x: 0, y: v.y * speed * dt, z: 0 } + : { x: v.x * speed * dt, y: -2.0 * dt, z: v.z * speed * dt }; + const m = this._computeConservativeMovement(desired); + const mx = m.x, my = m.y, mz = m.z; + this.body.setNextKinematicTranslation({ x: p.x + mx, y: p.y + my, z: p.z + mz }); + + // decrement steps by fraction of stepMeters + this._moveStepAcc += distThisFrame / Math.max(0.05, stepMeters); + if (this._moveStepAcc >= 1) { + const whole = Math.floor(this._moveStepAcc); + this._moveStepAcc -= whole; + this._tracePush(Date.now(), "MOVE_PROGRESS", { steps: whole, dir }); + } + this._planRemaining -= distThisFrame / Math.max(0.05, stepMeters); + if (this._planRemaining <= 0) { + // Plan completed - step counter is incremented per VLM decision, not here + return true; + } + return false; + } + + return true; + } + + _tracePush(t, type, data) { + const msg = (() => { + if (type === "TASK_START") return `task: ${String(data?.instruction || "").slice(0, 120)}`; + if (type === "DECISION") return `${data?.action || ""}`; + if (type === "PLAN_SET") return `${data?.plan?.type || ""}`; + if (type === "PLAN_DONE") return `${data?.plan?.type || ""}`; + if (type === "MOVE_PROGRESS") return `${data?.dir || ""} +${data?.steps || 0} step`; + if (type === "TURN_PROGRESS") return `turn ~${data?.degrees || 0}°`; + if (type === "FINISH_TASK") return `finish: ${String(data?.summary || "").slice(0, 120)}`; + if (type === "EDIT_ASSET_READY") return `asset ready: ${String(data?.assetName || data?.assetId || "").slice(0, 120)}`; + return ""; + })(); + this._trace.push({ t, type, msg, data }); + if (this._trace.length > this._traceLimit) this._trace.splice(0, this._trace.length - this._traceLimit); + } + + _pickWanderTarget(p) { + // Simple random walk around current position; if world has tags, bias toward them sometimes. + const tags = this.getTags() || []; + if (tags.length > 0 && Math.random() < 0.35) { + const t = tags[(Math.random() * tags.length) | 0]; + if (t?.position) return new THREE.Vector3(t.position.x, p.y, t.position.z); + } + const r = 6 + Math.random() * 10; + const a = Math.random() * Math.PI * 2; + return new THREE.Vector3(p.x + Math.cos(a) * r, p.y, p.z + Math.sin(a) * r); + } + + _applyIdleGravity(dt) { + try { + const p = this.body.translation(); + const desired = { x: 0, y: -15.0 * dt, z: 0 }; + const m = this._computeConservativeMovement(desired); + const my = m.y; + this.body.setNextKinematicTranslation({ x: p.x, y: p.y + my, z: p.z }); + } catch {} + } + + _computeConservativeMovement(desired) { + const flags = this.RAPIER.QueryFilterFlags.EXCLUDE_SENSORS; + this.controller.computeColliderMovement(this.collider, desired, flags); + const mm = this.controller.computedMovement(); + const main = { x: mm.x, y: mm.y, z: mm.z }; + if (!this.spineCollider) return main; + + // Keep the rear fake body aligned before querying its allowed movement. + this._syncSpineCollider(); + this.controller.computeColliderMovement(this.spineCollider, desired, flags); + const ms = this.controller.computedMovement(); + const spine = { x: ms.x, y: ms.y, z: ms.z }; + + // Conservative merge: use whichever collider allows less displacement per axis. + const towardZero = (a, b) => (Math.abs(a) <= Math.abs(b) ? a : b); + return { + x: towardZero(main.x, spine.x), + y: towardZero(main.y, spine.y), + z: towardZero(main.z, spine.z), + }; + } + + _syncVisual() { + const p = this.body.translation(); + this.group.position.set(p.x, p.y, p.z); + this._syncSpineCollider(); + } + + _syncSpineCollider() { + if (!this.spineCollider) return; + const p = this.body?.translation?.(); + if (!p) return; + const yaw = this.group?.rotation?.y ?? 0; + // Keep horizontal capsule aligned with model yaw and shifted toward rear torso. + const xOff = -Math.sin(yaw) * this._spineOffsetBack; + const zOff = -Math.cos(yaw) * this._spineOffsetBack; + const q = new THREE.Quaternion().setFromEuler(new THREE.Euler(Math.PI / 2, yaw, 0, "YXZ")); + try { + // Prefer parent-relative APIs when available (stable for colliders attached to a rigid body). + if (typeof this.spineCollider.setTranslationWrtParent === "function") { + this.spineCollider.setTranslationWrtParent({ x: xOff, y: this._spineOffsetY, z: zOff }); + } else { + this.spineCollider.setTranslation({ x: p.x + xOff, y: p.y + this._spineOffsetY, z: p.z + zOff }); + } + if (typeof this.spineCollider.setRotationWrtParent === "function") { + this.spineCollider.setRotationWrtParent({ x: q.x, y: q.y, z: q.z, w: q.w }); + } else { + this.spineCollider.setRotation({ x: q.x, y: q.y, z: q.z, w: q.w }); + } + } catch { + // ignore physics updates if collider is unavailable this frame + } + } + + _loadGLB() { + if (!this.avatarUrl) return; + const loader = new GLTFLoader(); + const urls = Array.isArray(this.avatarUrl) ? this.avatarUrl : [this.avatarUrl]; + const tryLoad = (index) => { + if (index >= urls.length) { + console.log(`[AiAvatar] No avatar model found, using capsule fallback.`); + return; + } + loader.load( + urls[index], + (gltf) => { this._applyGLB(gltf); }, + undefined, + () => { tryLoad(index + 1); } + ); + }; + tryLoad(0); + } + + _applyGLB(gltf) { + { + // Replace fallback with GLB + try { + this.group.remove(this.fallback); + } catch {} + // Also hide the facing indicator since the model shows direction + try { + this._facing.visible = false; + } catch {} + + this.model = gltf.scene; + + // Auto-fit the model to our agent dimensions + const bbox = new THREE.Box3().setFromObject(this.model); + const size = bbox.getSize(new THREE.Vector3()); + const center = bbox.getCenter(new THREE.Vector3()); + + // Target height: roughly capsule height (halfHeight * 2 + radius * 2) + const targetHeight = this.halfHeight * 2 + this.radius * 2; + const scaleFactor = targetHeight / (size.y || 1); + // Y-squash: the current GLB has no rig, so we can't pose it into a + // proper crouch. Compress Y instead so the robot reads as a low-slung + // quadruped rather than a tall upright one. Purely cosmetic — camera + // POV (GO2_CAMERA_HEIGHT in engine.js) is the accurate signal. + this.model.scale.set(scaleFactor, scaleFactor * 0.6, scaleFactor); + + // Re-center: the group origin is at the physics body center, + // which sits at (halfHeight + radius) above the ground. + // We need to offset the model down so its feet touch the ground. + bbox.setFromObject(this.model); + const newCenter = bbox.getCenter(new THREE.Vector3()); + const newMin = bbox.min; + this.model.position.x -= newCenter.x; + this.model.position.z -= newCenter.z; + // Offset down by the body's center height so feet are on the floor + this.model.position.y = -newMin.y - (this.halfHeight + this.radius); + // Keep the model centered on the group origin so yaw rotation pivots + // around the body's geometric center (between the legs), not the head. + + // The agent's forward convention is +Z (yaw=0 looks along +Z). + // This robot model already faces +Z, so no rotation needed. + + // Create box collider matching the scaled and positioned model dimensions + bbox.setFromObject(this.model); + const finalSize = bbox.getSize(new THREE.Vector3()); + const finalCenter = bbox.getCenter(new THREE.Vector3()); + + // Remove old box collider if it exists + if (this.boxCollider) { + this.rapierWorld.removeCollider(this.boxCollider, true); + this.boxCollider = null; + } + + // Create box collider centered on the model's actual bounding box + const boxHalfExtents = { + x: finalSize.x / 2, + y: finalSize.y / 2, + z: finalSize.z / 2 + }; + this.boxCollider = this.rapierWorld.createCollider( + this.RAPIER.ColliderDesc.cuboid(boxHalfExtents.x, boxHalfExtents.y, boxHalfExtents.z) + .setFriction(0.8) + .setTranslation(finalCenter.x, finalCenter.y, finalCenter.z), + this.body + ); + + console.log(`[AI] Created box collider: ${finalSize.x.toFixed(2)}x${finalSize.y.toFixed(2)}x${finalSize.z.toFixed(2)} at offset (${finalCenter.x.toFixed(2)}, ${finalCenter.y.toFixed(2)}, ${finalCenter.z.toFixed(2)})`); + + // Headless mode: skip visual rendering (save memory/GPU), keep collider + if (this.headless) { + console.log(`[AI] Headless mode: skipping visual model rendering`); + // Don't add model to scene, don't set up animations + // Model is loaded only to calculate box collider dimensions + this.model = null; // Release reference to free memory + return; + } + + // Normal mode: add visual model + // Real meshes receive shadows but don't cast (too expensive) + this.model.traverse((m) => { + if (m.isMesh) { + m.castShadow = false; + m.receiveShadow = true; + } + }); + + // No shadow proxy box: avoid cube-like shadow blockers around the robot. + + this.group.add(this.model); + + if (gltf.animations?.length) { + this.mixer = new THREE.AnimationMixer(this.model); + this._actions = {}; + for (const clip of gltf.animations) { + this._actions[clip.name] = this.mixer.clipAction(clip); + } + // Try common idle animation names + const idle = this._actions["idle"] || this._actions["Idle"] || this._actions["Idle_A"]; + if (idle) idle.play(); + else this.mixer.clipAction(gltf.animations[0]).play(); + } + + console.log(`[AI] Loaded robot model: ${size.x.toFixed(2)}x${size.y.toFixed(2)}x${size.z.toFixed(2)}, scale=${scaleFactor.toFixed(3)}`); + } + } + + _setThought(text) { + const ctx = this._labelCtx; + if (!ctx) return; + if (this.vlm?.showSpeechBubbleInScene === false) { + ctx.clearRect(0, 0, this._labelCanvas.width, this._labelCanvas.height); + this._labelTex.needsUpdate = true; + this._labelSprite.visible = false; + return; + } + if (!text) { + ctx.clearRect(0, 0, this._labelCanvas.width, this._labelCanvas.height); + this._labelSprite.scale.set(2.0, 1.0, 1); + this._labelTex.needsUpdate = true; + this._labelSprite.visible = false; + return; + } + + const bubbleX = 20; + const bubbleY = 20; + const bubbleW = 472; + const bubbleH = 472; + const padX = 24; + const padY = 20; + const maxTextH = bubbleH - padY * 2; + + // Keep canvas size fixed to avoid stale visual remnants from resizing. + if (this._labelCanvas.width !== 512) this._labelCanvas.width = 512; + if (this._labelCanvas.height !== 512) this._labelCanvas.height = 512; + + // Fit all text within fixed bubble by reducing font size as needed. + let fontSize = 30; + let lineHeight = 36; + let lines = []; + while (fontSize >= 12) { + ctx.font = `bold ${fontSize}px ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto`; + lineHeight = Math.round(fontSize * 1.18); + lines = wrapTextLines(ctx, text, bubbleW - padX * 2); + const usedH = lines.length * lineHeight; + if (usedH <= maxTextH) break; + fontSize -= 2; + } + + ctx.clearRect(0, 0, this._labelCanvas.width, this._labelCanvas.height); + this._labelSprite.visible = true; + // bubble + ctx.fillStyle = "rgba(0,0,0,0.65)"; + ctx.strokeStyle = "rgba(255,255,255,0.25)"; + roundRect(ctx, bubbleX, bubbleY, bubbleW, bubbleH, 18); + ctx.fill(); + ctx.stroke(); + ctx.fillStyle = "rgba(255,255,255,0.92)"; + ctx.font = `bold ${fontSize}px ui-sans-serif, system-ui, -apple-system, Segoe UI, Roboto`; + ctx.textBaseline = "top"; + const textX = bubbleX + padX; + let textY = bubbleY + padY; + for (const line of lines) { + ctx.fillText(line, textX, textY); + textY += lineHeight; + } + this._labelSprite.scale.set(2.0, 2.0, 1); + this._labelTex.needsUpdate = true; + } + + _extractBubbleTextFromModelOutput(parsed, raw) { + const p = parsed && typeof parsed === "object" ? parsed : {}; + const obs = + p.observation || + p.obs || + p.perception || + p.sceneObservation || + p.visualObservation || + p.params?.observation || + ""; + if (typeof obs === "string" && obs.trim()) return obs.trim(); + const action = typeof p.action === "string" ? p.action.trim() : ""; + if (action) return `Action: ${action}`; + + const rawText = typeof raw === "string" ? raw : ""; + if (rawText) { + const m = rawText.match(/"observation"\s*:\s*"([^"]+)"/i); + if (m?.[1]) return m[1]; + } + return ""; + } + + _memoryKey() { + return `sparkWorldAiMemory:${this.getWorldKey()}:${this.id}`; + } + + _loadMemory() { + try { + const raw = localStorage.getItem(this._memoryKey()); + return raw ? JSON.parse(raw) : { seenTags: {} }; + } catch { + return { seenTags: {} }; + } + } + + _saveMemory() { + try { + localStorage.setItem(this._memoryKey(), JSON.stringify(this.memory)); + } catch { + // ignore + } + } + + _rememberTag(tag) { + if (!tag?.id) return; + const now = Date.now(); + if (!this.memory.seenTags) this.memory.seenTags = {}; + const entry = this.memory.seenTags[tag.id] || { + firstSeen: now, + lastSeen: now, + count: 0, + title: tag.title || "", + notes: tag.notes || "", + }; + entry.lastSeen = now; + entry.count = (entry.count || 0) + 1; + entry.title = tag.title || entry.title; + entry.notes = tag.notes || entry.notes; + this.memory.seenTags[tag.id] = entry; + // Throttle writes a bit + if (!this._nextMemSave || now > this._nextMemSave) { + this._nextMemSave = now + 1200; + this._saveMemory(); + } + } + + _publishGlobals() { + if (typeof window === "undefined") return; + if (!window.__aiAgentPositions) window.__aiAgentPositions = {}; + const [x, y, z] = this.getPosition(); + window.__aiAgentPositions[this.id] = { x, y, z }; + } +} + +function safeParseJson(text) { + try { + return JSON.parse(text); + } catch { + // Attempt to extract the first JSON object. + const s = String(text || ""); + const i = s.indexOf("{"); + const j = s.lastIndexOf("}"); + if (i !== -1 && j !== -1 && j > i) { + try { + return JSON.parse(s.slice(i, j + 1)); + } catch {} + } + return null; + } +} + +function roundRect(ctx, x, y, w, h, r) { + const rr = Math.min(r, w / 2, h / 2); + ctx.beginPath(); + ctx.moveTo(x + rr, y); + ctx.arcTo(x + w, y, x + w, y + h, rr); + ctx.arcTo(x + w, y + h, x, y + h, rr); + ctx.arcTo(x, y + h, x, y, rr); + ctx.arcTo(x, y, x + w, y, rr); + ctx.closePath(); +} + +function wrapTextLines(ctx, text, maxWidth) { + const words = String(text).replace(/\s+/g, " ").trim().split(" "); + const lines = []; + let line = ""; + for (const rawWord of words) { + const word = rawWord || ""; + const test = line ? `${line} ${word}` : word; + if (ctx.measureText(test).width <= maxWidth || !line) { + // If single word is too long, hard-break it. + if (!line && ctx.measureText(word).width > maxWidth) { + let chunk = ""; + for (const ch of word) { + const next = chunk + ch; + if (ctx.measureText(next).width > maxWidth && chunk) { + lines.push(chunk); + chunk = ch; + } else { + chunk = next; + } + } + line = chunk; + } else { + line = test; + } + } else { + lines.push(line); + line = word; + } + } + if (line) lines.push(line); + return lines.length ? lines : [""]; +} diff --git a/misc/DimSim/src/ai/modelConfig.js b/misc/DimSim/src/ai/modelConfig.js new file mode 100644 index 0000000000..faf73b0dd7 --- /dev/null +++ b/misc/DimSim/src/ai/modelConfig.js @@ -0,0 +1,14 @@ +// Single source of truth for SimStudio model selection. +// Update these two values only. +export const MODEL_CONFIG = { + // Used by editor-mode spawned agents AND Scene Builder (vibeCreator) + editorMode: "gemini-3-flash-preview", + // Used by sim mode task agent + simMode: "gemini-robotics-er-1.5-preview", +}; + +// model: "gemini-3.1-pro-preview", +// model: "gpt-4o", +// model: "gpt-4.1-2025-04-14", // OpenAI GPT-4.1 +// model: "gemini-3-flash-preview", // Google Gemini Flash +// model: "gemini-robotics-er-1.5-preview", diff --git a/misc/DimSim/src/ai/sim/vlmActions.js b/misc/DimSim/src/ai/sim/vlmActions.js new file mode 100644 index 0000000000..7b1460b6e3 --- /dev/null +++ b/misc/DimSim/src/ai/sim/vlmActions.js @@ -0,0 +1,109 @@ +import { MODEL_CONFIG } from "../modelConfig.js"; + +// Sim-only action surface for SimStudio/DimSim parity. +// Keep this list free of editor/build actions. + +export const ACTIONS = [ + // === MOVEMENT === + { + id: "MOVE_FORWARD", + description: "Move forward. Use for approaching things you see.", + params: { steps: "integer 1-5" }, + }, + { + id: "MOVE_BACKWARD", + description: "Move backward. Use to back away or reposition.", + params: { steps: "integer 1-3" }, + }, + { + id: "STRAFE_LEFT", + description: "Sidestep left without turning.", + params: { steps: "integer 1-3" }, + }, + { + id: "STRAFE_RIGHT", + description: "Sidestep right without turning.", + params: { steps: "integer 1-3" }, + }, + { + id: "MOVE_UP", + description: "Move upward (float up) for vertical repositioning in sim.", + params: { steps: "integer 1-3" }, + }, + { + id: "MOVE_DOWN", + description: "Move downward (float down) for vertical repositioning in sim.", + params: { steps: "integer 1-3" }, + }, + + // === LOOKING/TURNING === + { + id: "TURN_LEFT", + description: "Turn body left (yaw). Use to see what's to your left or explore new directions.", + params: { degrees: "number 30-90" }, + }, + { + id: "TURN_RIGHT", + description: "Turn body right (yaw). Use to see what's to your right or explore new directions.", + params: { degrees: "number 30-90" }, + }, + { + id: "LOOK_UP", + description: "Tilt view upward. Use to see shelves, ceilings, tall objects.", + params: { degrees: "number 15-45" }, + }, + { + id: "LOOK_DOWN", + description: "Tilt view downward. Use to see floor, low objects, items on ground.", + params: { degrees: "number 15-45" }, + }, + + // === NAVIGATION === + { + id: "GOTO_LOCATION", + description: "Navigate toward a known location. Use 'start' to return to where you began.", + params: { locationId: "string (tag id from nearbyLocations, or 'start')" }, + }, + + // === INTERACTION === + { + id: "INTERACT", + description: "Interact with an object. REQUIRES: object in NEARBY OBJECTS list AND distance < 1.5m AND object should be in your field of vision. Use the EXACT assetId from [id: ...] brackets!", + params: { assetId: "string (EXACT id from [id: xxx] in NEARBY OBJECTS)", actionLabel: "string (from can: list)" }, + }, + + // === PICK UP / DROP === + { + id: "PICK_UP", + description: "Pick up a pickable object. REQUIRES: object marked [pickable] in NEARBY OBJECTS AND distance < 1.5m AND you're not already holding something AND object should be in your field of vision.", + params: { assetId: "string (EXACT id from [id: xxx] in NEARBY OBJECTS)" }, + }, + { + id: "DROP", + description: "Drop the object you're currently holding. Places it in front of you.", + params: {}, + }, + + // === META === + { + id: "THINK", + description: "Pause to reason about your situation. Use when stuck or need to reconsider your approach.", + params: { thought: "string (your reasoning)" }, + }, + { + id: "DONE", + description: "Task is complete. Only use when you've achieved the goal.", + params: { summary: "string (what you accomplished)" }, + }, +]; + +export const DEFAULTS = { + model: MODEL_CONFIG.simMode, + decideEverySteps: 6, + stepMeters: 0.4, + maxToiMeters: 50, +}; +// model: "gpt-4o", +// model: "gemini-3.1-pro-preview", +// model: "gemini-3-flash-preview", // Google Gemini Flash +// model: "gemini-robotics-er-1.5-preview", diff --git a/misc/DimSim/src/ai/sim/vlmPrompt.js b/misc/DimSim/src/ai/sim/vlmPrompt.js new file mode 100644 index 0000000000..e7725cad7e --- /dev/null +++ b/misc/DimSim/src/ai/sim/vlmPrompt.js @@ -0,0 +1,80 @@ +import { ACTIONS } from "./vlmActions.js"; + +export function buildPrompt({ actions = ACTIONS } = {}) { + const actionList = actions + .map((a) => ` ${a.id}: ${a.description}${a.params && Object.keys(a.params).length ? ` | params: ${JSON.stringify(a.params)}` : ""}`) + .join("\n"); + + return `You are an embodied AI agent navigating a 3D environment. You see through a first-person camera and receive ONE screenshot per decision. After each action completes, you'll get a NEW screenshot showing the result. + +## Your Actions +${actionList} + +## Decision Process + +For each screenshot: +1. **OBSERVE**: What do you see? Be specific about objects, their positions, and distances. +2. **THINK**: How does this relate to your goal? What should you do next? +3. **ACT**: Choose ONE action. + +## Exploration Strategy + +Since you only see one frame at a time: +- **Turn incrementally** (30-90°) to survey your surroundings +- **Move toward** interesting objects or unexplored areas you see +- **Look up/down** if you need to see shelves, floors, or tall objects +- **Remember** what you've seen in previous frames (your history is provided) + +## Task Decomposition + +Break complex tasks into steps: +EXAMPLE: +- "Find the book" → Turn to look around → Move toward bookshelf → Look at books +- "Go to kitchen" → Look for doorways → Navigate through them → Identify kitchen + +## Interaction Rules + +To interact with an object: +1. It must appear in "NEARBY OBJECTS" list +2. It must be within 1.5 meters (check the distance in parentheses) +3. It must be visible in your Field of Vision. +4. Use INTERACT with: + - assetId: the EXACT ID shown in [id: xxx] brackets - copy it exactly! + - actionLabel: one of the actions from the "can:" list + +Example: If you see "Fridge [id: 73799fa3d397c-19b5c3d31fb] (0.8m) - Closed → can: Open" +Then use: {"action": "INTERACT", "params": {"assetId": "73799fa3d397c-19b5c3d31fb", "actionLabel": "Open"}} + +## Pick Up / Drop Rules + +Some objects are marked [pickable] - you can carry them: +- Use PICK_UP with the assetId to grab it (must be within 1.5m, can only hold ONE item) +- Use DROP to place the held item in front of you +- When holding something, it shows in "HOLDING:" at the top of the context + +Example pickup: {"action": "PICK_UP", "params": {"assetId": "73799fa3d397c-19b5c3d31fb"}} +Example drop: {"action": "DROP", "params": {}} + +**CRITICAL**: +- The assetId must be EXACTLY as shown in [id: ...] - don't make up IDs! +- If no objects appear in NEARBY OBJECTS, move around to find them +- If distance > 1.5m, move closer first +- If interaction fails, try moving forward 1-2 steps and try again + +Hard constraints: +- If object IDs are missing/unclear, do NOT guess; reorient until the target appears in nearby lists. +- If IDs are missing, navigate yourself (MOVE/TURN/LOOK) until IDs are visible. +- Do not claim completion unless the final screenshot visibly matches the task intent. + +## Output Format + +Return ONLY valid JSON: +{ + "observation": "What I see in this screenshot", + "thinking": "My reasoning about what to do", + "action": "ACTION_NAME", + "params": { ... } +} + +No markdown. No extra text. JSON only.`; +} diff --git a/misc/DimSim/src/ai/visionCapture.js b/misc/DimSim/src/ai/visionCapture.js new file mode 100644 index 0000000000..63d5215d94 --- /dev/null +++ b/misc/DimSim/src/ai/visionCapture.js @@ -0,0 +1,267 @@ +import * as THREE from "three"; + +// ============================================================================= +// AGENT POV CAPTURE SYSTEM +// ============================================================================= +// +// For Gaussian splats to render correctly, the SparkRenderer needs to sort splats +// based on the camera position. This sorting happens during the render call. +// +// ARCHITECTURE: +// Instead of capturing mid-frame, we use a request/callback system: +// 1. Agent requests a capture -> we queue the request +// 2. Main render loop sees pending request +// 3. Main loop renders from AGENT's POV first (splats get sorted for agent) +// 4. Capture the result +// 5. Then render from PLAYER's POV (splats get re-sorted for player) +// +// This ensures the agent always gets a properly rendered frame. + +const _lampByAgent = new WeakMap(); + +// Pending capture requests: Map +const _pendingCaptures = new Map(); + +// Track if we have a pending capture +export function hasPendingCapture() { + return _pendingCaptures.size > 0; +} + +// Get all pending captures +export function getPendingCaptures() { + return Array.from(_pendingCaptures.values()); +} + +// Clear a pending capture after it's processed +export function clearPendingCapture(agentId) { + _pendingCaptures.delete(agentId); +} + +/** + * Request a capture - returns a promise that resolves when the capture is complete. + * The actual capture is performed by the main render loop via processPendingCaptures(). + */ +export function requestAgentCapture({ + agent, + renderer, + scene, + mainCamera, + width = 960, // Wider for more human-like FOV + height = 432, // ~2.2:1 aspect ratio (wider than 16:9) + eyeHeight = 0.55, + jpegQuality = 0.75, + fov = 80, // Wider vertical FOV for more peripheral vision + near = 0.05, + far = 2000, + headLamp = null, + preRender = null, + renderFn = null, // optional: (renderer, scene, camera) => void — renders active view mode +}) { + return new Promise((resolve, reject) => { + if (!agent) { + reject(new Error("No agent provided")); + return; + } + + const agentId = agent.id || "default"; + + // Store the capture request + _pendingCaptures.set(agentId, { + agent, + renderer, + scene, + mainCamera, + width, + height, + eyeHeight, + jpegQuality, + fov, + near, + far, + headLamp, + preRender, + renderFn, + resolve, + reject, + }); + }); +} + +/** + * Process all pending captures - called from the main render loop. + * This renders from each agent's POV and captures the result. + * + * We do a "warm-up" render first to trigger splat sorting, wait for the + * renderer to complete, then do the actual capture render. + */ +export async function processPendingCaptures() { + const captures = getPendingCaptures(); + if (captures.length === 0) return; + + for (const capture of captures) { + try { + const base64 = await performCaptureWithDelay(capture); + capture.resolve(base64); + } catch (e) { + capture.reject(e); + } finally { + clearPendingCapture(capture.agent.id || "default"); + } + } +} + +/** + * Perform capture with a delay to allow SparkRenderer to sort splats. + * 1. First render triggers splat sorting for agent's viewpoint + * 2. Wait 500ms for GPU sorting to complete + * 3. Second render captures with properly sorted splats + */ +async function performCaptureWithDelay(captureParams) { + const { + agent, + renderer, + scene, + mainCamera, + width, + height, + eyeHeight, + fov, + near, + far, + headLamp, + preRender, + jpegQuality, + } = captureParams; + + if (!agent || !renderer || !scene || !mainCamera) return null; + + let lamp = null; + + // Calculate agent's eye position and direction + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch); + const sp = Math.sin(pitch); + const forward = new THREE.Vector3(Math.sin(yaw) * cp, sp, Math.cos(yaw) * cp); + const eyeY = ay + eyeHeight; + + // Use a dedicated offscreen camera so user view never switches. + const captureCamera = new THREE.PerspectiveCamera(fov, width / height, near, far); + captureCamera.position.set(ax, eyeY, az); + captureCamera.lookAt(ax + forward.x, eyeY + forward.y, az + forward.z); + captureCamera.updateProjectionMatrix(); + captureCamera.updateMatrixWorld(true); + + const prevTarget = renderer.getRenderTarget?.() || null; + const captureTarget = new THREE.WebGLRenderTarget(width, height, { + minFilter: THREE.LinearFilter, + magFilter: THREE.LinearFilter, + format: THREE.RGBAFormat, + depthBuffer: true, + stencilBuffer: false, + }); + + // Optional capture-only fill light. Keep null for strict view parity. + if (headLamp && typeof headLamp === "object") { + lamp = _lampByAgent.get(agent); + if (!lamp) { + lamp = new THREE.PointLight(0xffffff, headLamp.intensity, headLamp.distance, headLamp.decay); + _lampByAgent.set(agent, lamp); + } + const fwdN = forward.clone().normalize(); + const up = new THREE.Vector3(0, 1, 0); + const off = headLamp.offset || { x: 0, y: 1.0, z: 0.6 }; + const right = new THREE.Vector3().crossVectors(fwdN, up).normalize(); + const lampPos = new THREE.Vector3(ax, eyeY, az) + .addScaledVector(right, off.x) + .addScaledVector(up, off.y) + .addScaledVector(fwdN, off.z); + lamp.position.copy(lampPos); + scene.add(lamp); + } + + // Call preRender callback + let cleanup = null; + if (typeof preRender === "function") { + try { + cleanup = preRender(captureCamera) || null; + } catch { + // ignore + } + } + + const { renderFn } = captureParams; + const doRender = () => { + renderer.setRenderTarget(captureTarget); + if (typeof renderFn === "function") { + // Intentionally pass captureCamera; renderFn should avoid mutating player camera. + renderFn(renderer, scene, captureCamera); + } else { + renderer.render(scene, captureCamera); + } + renderer.setRenderTarget(null); + }; + + // FIRST RENDER: Trigger splat sorting for agent's viewpoint + doRender(); + + // Give Spark sorting a short moment to settle. + await new Promise(resolve => setTimeout(resolve, 180)); + + // SECOND RENDER: Capture with properly sorted splats + doRender(); + + // Read pixels from offscreen target and encode JPEG. + const raw = new Uint8Array(width * height * 4); + renderer.readRenderTargetPixels(captureTarget, 0, 0, width, height, raw); + const flipped = new Uint8ClampedArray(width * height * 4); + const rowBytes = width * 4; + for (let y = 0; y < height; y++) { + const srcY = height - 1 - y; + const srcOff = srcY * rowBytes; + const dstOff = y * rowBytes; + // Match on-screen output transform: convert linear RT pixels to sRGB. + for (let i = 0; i < rowBytes; i += 4) { + const r = raw[srcOff + i + 0] / 255; + const g = raw[srcOff + i + 1] / 255; + const b = raw[srcOff + i + 2] / 255; + const a = raw[srcOff + i + 3]; + const toSrgb = (x) => (x <= 0.0031308 ? 12.92 * x : 1.055 * Math.pow(x, 1 / 2.4) - 0.055); + flipped[dstOff + i + 0] = Math.max(0, Math.min(255, Math.round(toSrgb(r) * 255))); + flipped[dstOff + i + 1] = Math.max(0, Math.min(255, Math.round(toSrgb(g) * 255))); + flipped[dstOff + i + 2] = Math.max(0, Math.min(255, Math.round(toSrgb(b) * 255))); + flipped[dstOff + i + 3] = a; + } + } + const cvs = document.createElement("canvas"); + cvs.width = width; + cvs.height = height; + const ctx = cvs.getContext("2d"); + if (!ctx) return null; + ctx.putImageData(new ImageData(flipped, width, height), 0, 0); + const dataUrl = cvs.toDataURL("image/jpeg", jpegQuality); + + // Remove optional capture lamp + if (lamp) scene.remove(lamp); + + // Call cleanup + if (typeof cleanup === "function") { + try { + cleanup(); + } catch { + // ignore + } + } + + renderer.setRenderTarget(prevTarget); + captureTarget.dispose(); + + const idx = dataUrl.indexOf("base64,"); + return idx !== -1 ? dataUrl.slice(idx + "base64,".length) : null; +} + +// Legacy function for backwards compatibility - now just calls requestAgentCapture +export async function captureAgentPovBase64(params) { + return requestAgentCapture(params); +} diff --git a/misc/DimSim/src/ai/vlmClient.js b/misc/DimSim/src/ai/vlmClient.js new file mode 100644 index 0000000000..84b5ee0bbe --- /dev/null +++ b/misc/DimSim/src/ai/vlmClient.js @@ -0,0 +1,36 @@ +// Browser client: talks to the SimStudio VLM backend server +// so your OpenAI key never ships to the browser. + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export async function requestVlmDecision({ endpoint, model, prompt, imageBase64, context, messages }) { + const payload = JSON.stringify({ model, prompt, imageBase64, context, messages }); + const maxAttempts = 6; + let lastError = null; + + for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { + const res = await fetch(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: payload, + }); + if (res.ok) return await res.json(); + + const text = await res.text().catch(() => ""); + const isRetryable = res.status === 429 || res.status === 503 || res.status === 504; + if (!isRetryable || attempt >= maxAttempts) { + throw new Error(`VLM request failed (${res.status}): ${text || res.statusText}`); + } + + lastError = new Error(`retryable status ${res.status}`); + const retryAfter = Number(res.headers.get("retry-after") || 0); + const retryAfterMs = retryAfter > 0 ? retryAfter * 1000 : 0; + const backoffMs = Math.min(12000, 700 * 2 ** (attempt - 1)); + const jitterMs = Math.floor(Math.random() * 350); + await sleep(Math.max(retryAfterMs, backoffMs) + jitterMs); + } + + throw lastError || new Error("VLM request failed"); +} diff --git a/misc/DimSim/src/dimos/dimosBridge.ts b/misc/DimSim/src/dimos/dimosBridge.ts new file mode 100644 index 0000000000..1b5a4ada8b --- /dev/null +++ b/misc/DimSim/src/dimos/dimosBridge.ts @@ -0,0 +1,557 @@ +/** + * DimosBridge — Browser-side WebSocket client for dimos integration. + * + * Uses separate WebSocket connections so large sensor streams do not block + * each other or real-time odom/cmd_vel: + * wsControl → /odom, /cmd_vel (tiny packets, real-time) + * wsSensors → /lidar + snapshots + * wsRgb → /color_image + * wsDepth → /depth_image + * + * All messages are LCM-encoded binary packets using @dimos/msgs, sent over + * WebSocket to the bridge server which relays them to/from dimos via LCM/UDP. + */ + +// @ts-ignore — CDN import (runs in browser, no Deno/Node type resolution) +import { + encodePacket, + decodePacket, + geometry_msgs, + sensor_msgs, + std_msgs, +} from "https://esm.sh/jsr/@dimos/msgs@0.1.4"; + +// -- Channels ---------------------------------------------------------------- +const CH_CMD_VEL = "/cmd_vel#geometry_msgs.Twist"; +const CH_ODOM = "/odom#geometry_msgs.PoseStamped"; +const CH_IMAGE = "/color_image#sensor_msgs.Image"; +const CH_DEPTH = "/depth_image#sensor_msgs.Image"; +const CH_LIDAR = "/lidar#sensor_msgs.PointCloud2"; + +// -- Default publish rates (ms) ---------------------------------------------- +const DEFAULT_RATES: PublishRates = { odom: 20, lidar: 200, images: 200 }; // 50 Hz odom, 5 Hz lidar, 5 Hz images +const CMD_VEL_TIMEOUT_MS = 500; +const BRIDGE_DEBUG = false; +const LIDAR_POINT_STEP = 16; + +// -- Types -------------------------------------------------------------------- + +export interface PublishRates { odom: number; lidar: number; images: number; } +export interface SensorEnable { depth: boolean; } + +export interface RgbFrame { + data: Uint8Array; + width: number; + height: number; +} + +export interface DepthFrame { + data: Float32Array; + width: number; + height: number; +} + +export interface LidarFrame { + numPoints: number; + points: Float32Array; // N*3 interleaved XYZ + intensity?: Float32Array; // N +} + +export interface OdomPose { + x: number; y: number; z: number; + qx: number; qy: number; qz: number; qw: number; +} + +export interface SensorSources { + captureRgb: () => RgbFrame | null; + captureDepth: () => DepthFrame | null; + captureLidar: () => LidarFrame | null; + getOdomPose: () => OdomPose | null; +} + +export type FrameTransform = "identity" | "ros"; + +export interface DimosBridgeOptions { + wsUrl?: string; + agent: any; + sensorSources: SensorSources; + rates?: Partial; + sensorEnable?: Partial; + frameTransform?: FrameTransform; +} + +export class DimosBridge { + wsUrl: string; + agent: any; + sensors: SensorSources; + rates: PublishRates; + sensorEnable: SensorEnable; + frameTransform: FrameTransform; + + // Separate sockets so depth backlog does not starve RGB. + wsControl: WebSocket | null; // odom + cmd_vel (tiny, real-time) + wsSensors: WebSocket | null; // lidar + snapshots + wsRgb: WebSocket | null; // color image + wsDepth: WebSocket | null; // depth image + + // Keep legacy .ws alias pointing to control for compatibility + get ws(): WebSocket | null { return this.wsControl; } + + _timers: Record>; + _dirty: { odom: boolean; lidar: boolean; images: boolean }; + _rafId: number | null; + _connected: boolean; + + _cmdVel: { linX: number; linY: number; linZ: number; angX: number; angY: number; angZ: number } | null; + _cmdVelStamp: number; + _serverLidar: boolean; + _lidarBuf: ArrayBuffer; + _lidarView: DataView; + _lidarCapacityPoints: number; + _pc2Fields: any[]; + + constructor({ wsUrl, agent, sensorSources, rates, sensorEnable, frameTransform }: DimosBridgeOptions) { + const protocol = location.protocol === "https:" ? "wss:" : "ws:"; + this.wsUrl = wsUrl || `${protocol}//${location.host}`; + this.agent = agent; + this.sensors = sensorSources; + this.rates = { ...DEFAULT_RATES, ...rates }; + this.sensorEnable = { depth: true, ...sensorEnable }; + this.frameTransform = frameTransform || "ros"; + this.wsControl = null; + this.wsSensors = null; + this.wsRgb = null; + this.wsDepth = null; + this._timers = {}; + this._dirty = { odom: false, lidar: false, images: false }; + this._rafId = null; + this._connected = false; + this._cmdVel = null; + this._cmdVelStamp = 0; + this._serverLidar = false; + this._lidarBuf = new ArrayBuffer(0); + this._lidarView = new DataView(this._lidarBuf); + this._lidarCapacityPoints = 0; + this._pc2Fields = [ + new sensor_msgs.PointField({ name: "x", offset: 0, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "y", offset: 4, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "z", offset: 8, datatype: 7, count: 1 }), + new sensor_msgs.PointField({ name: "intensity", offset: 12, datatype: 7, count: 1 }), + ]; + } + + connect(): void { + // Read channel from URL param (for multi-page parallel evals) + const channel = new URLSearchParams(location.search).get("channel") || ""; + const channelSuffix = channel ? `&channel=${channel}` : ""; + + // Control socket: odom out, cmd_vel in + this.wsControl = new WebSocket(this.wsUrl + "?ch=control" + channelSuffix); + this.wsControl.binaryType = "arraybuffer"; + + this.wsControl.onopen = () => { + console.log("[DimosBridge] control WS connected"); + this._connected = true; + this._startPublishing(); + }; + + this.wsControl.onmessage = (event: MessageEvent) => { + // Text messages: server-side physics pose updates + embodiment config + if (typeof event.data === "string") { + try { + const msg = JSON.parse(event.data); + if (msg.type === "pose") { + this._handleServerPose(msg.x, msg.y, msg.z, msg.yaw); + } else if (msg.type === "embodimentConfig") { + this._handleEmbodimentConfig(msg); + } + } catch {} + return; + } + // Binary messages: LCM packets (cmd_vel relay) + if (!(event.data instanceof ArrayBuffer)) return; + try { + const raw = new Uint8Array(event.data); + const { channel, data } = decodePacket(raw); + this._handlePacket(channel, data); + } catch {} + }; + + this.wsControl.onclose = () => { + console.log("[DimosBridge] control WS disconnected, reconnecting in 2s..."); + this._connected = false; + this._stopPublishing(); + setTimeout(() => this.connect(), 2000); + }; + + this.wsControl.onerror = () => {}; + + // Sensor socket: lidar + snapshots out (no incoming expected) + this.wsSensors = new WebSocket(this.wsUrl + "?ch=sensors" + channelSuffix); + this.wsSensors.binaryType = "arraybuffer"; + + this.wsSensors.onopen = () => { + console.log("[DimosBridge] sensor WS connected"); + }; + + this.wsSensors.onclose = () => { + console.log("[DimosBridge] sensor WS disconnected"); + }; + + this.wsSensors.onerror = () => {}; + + this.wsRgb = new WebSocket(this.wsUrl + "?ch=rgb" + channelSuffix); + this.wsRgb.binaryType = "arraybuffer"; + this.wsRgb.onclose = () => { + console.log("[DimosBridge] RGB WS disconnected"); + }; + this.wsRgb.onerror = () => {}; + + this.wsDepth = new WebSocket(this.wsUrl + "?ch=depth" + channelSuffix); + this.wsDepth.binaryType = "arraybuffer"; + this.wsDepth.onclose = () => { + console.log("[DimosBridge] depth WS disconnected"); + }; + this.wsDepth.onerror = () => {}; + } + + // -- Incoming packets ------------------------------------------------------- + + _handlePacket(channel: string, data: any): void { + if (channel === CH_CMD_VEL) { + this._handleCmdVel(data); + } + } + + _handleCmdVel(twist: any): void { + const lin = twist.linear; + const ang = twist.angular; + + let linX: number, linY: number, linZ: number; + let angX: number, angY: number, angZ: number; + + if (this.frameTransform === "ros") { + // ROS → Three.js: inverse of the cyclic permutation (x→y, y→z, z→x) + linX = lin.y; + linY = lin.z; + linZ = lin.x; + angX = ang.y; + angY = ang.z; + angZ = ang.x; + } else { + linX = lin.x; linY = lin.y; linZ = lin.z; + angX = ang.x; angY = ang.y; angZ = ang.z; + } + + this._cmdVel = { linX, linY, linZ, angX, angY, angZ }; + this._cmdVelStamp = Date.now(); + } + + /** Get current velocity, auto-zeroing after CMD_VEL_TIMEOUT_MS (safety stop). */ + getCmdVel(): { linX: number; linY: number; linZ: number; angX: number; angY: number; angZ: number } { + if (!this._cmdVel || Date.now() - this._cmdVelStamp > CMD_VEL_TIMEOUT_MS) { + return { linX: 0, linY: 0, linZ: 0, angX: 0, angY: 0, angZ: 0 }; + } + return this._cmdVel; + } + + /** Handle server-side physics pose update (Three.js Y-up frame). */ + _handleServerPose(x: number, y: number, z: number, yaw: number): void { + if (!this.agent) return; + // Move the agent body to the server-authoritative position + if (this.agent.body) { + this.agent.body.setNextKinematicTranslation({ x, y, z }); + } + if (this.agent.group) { + this.agent.group.rotation.y = yaw; + } + // Update engine's _dimosYaw for sensor capture / odom pose reading + if ((window as any).__dimosSetYaw) { + (window as any).__dimosSetYaw(yaw); + } + // Store for odom/sensor capture + this._serverPose = { x, y, z, yaw }; + } + + _serverPose: { x: number; y: number; z: number; yaw: number } | null = null; + + _handleEmbodimentConfig(msg: any): void { + console.log("[DimosBridge] embodiment config received:", msg.embodimentType || "quadruped"); + // Apply config to engine globals (dimensions, speed, type) + if ((window as any).applyEmbodiment) { + (window as any).applyEmbodiment(msg); + } + // Swap the agent's avatar model if avatarUrl changed + if (this.agent && msg.avatarUrl) { + const urls = Array.isArray(msg.avatarUrl) ? msg.avatarUrl : [msg.avatarUrl]; + this.agent.avatarUrl = urls; + // Update dimensions on the agent so _applyGLB auto-fits correctly + if (msg.radius != null) this.agent.radius = msg.radius; + if (msg.halfHeight != null) this.agent.halfHeight = msg.halfHeight; + // Remove current model and reload + if (this.agent.model) { + this.agent.group.remove(this.agent.model); + this.agent.model = null; + } + this.agent._loadGLB(); + } + } + + // -- Outgoing sensor data --------------------------------------------------- + + sceneReady = false; + + _startPublishing(): void { + // No lidar timer — server-side lidar handles it via LCM directly. + // Images default 5 Hz (configurable via rates.images) + if (this.rates.images > 0) { + this._timers["images"] = setInterval(() => this._publishImages(), this.rates.images); + } + } + + _makeHeader(frameId: string): any { + const now = Date.now(); + return new std_msgs.Header({ + stamp: new std_msgs.Time({ sec: Math.floor(now / 1000), nsec: (now % 1000) * 1_000_000 }), + frame_id: frameId, + }); + } + + _publishOdom(): void { + if (!this._isControlSocketOpen()) return; + this._publishOdomSync(this._makeHeader("world")); + } + + _publishLidar(): void { + if (!this._isSocketOpen(this.wsSensors)) return; + this._publishLidarSync(this._makeHeader("world")); + } + + _publishImages(): void { + if (!this._isSocketOpen(this.wsRgb) && !this._isSocketOpen(this.wsDepth)) return; + const camHeader = this._makeHeader("camera_optical"); + if (this._isSocketOpen(this.wsRgb)) this._publishRgbSync(camHeader); + if (this.sensorEnable.depth && this._isSocketOpen(this.wsDepth)) this._publishDepthSync(camHeader); + } + + // -- Odom ------------------------------------------------------------------- + + _odomDbgN = 0; + + _publishOdomSync(header: any): void { + try { + const pose = this.sensors.getOdomPose(); + if (!pose) return; + + this._odomDbgN++; + + // Three.js (Y-up) → ROS (Z-up) cyclic permutation: x→y, y→z, z→x + const rosQx = pose.qz; + const rosQy = pose.qx; + const rosQz = pose.qy; + const rosQw = pose.qw; + + const q = new geometry_msgs.Quaternion(); + q.x = rosQx; q.y = rosQy; q.z = rosQz; q.w = rosQw; + const pt = new geometry_msgs.Point(); + pt.x = pose.z; pt.y = pose.x; pt.z = pose.y; + const p = new geometry_msgs.Pose(); + p.position = pt; + p.orientation = q; + + header.seq = this._odomDbgN; + const odomMsg = new geometry_msgs.PoseStamped(); + odomMsg.header = header; + odomMsg.pose = p; + + if (BRIDGE_DEBUG && (this._odomDbgN <= 3 || this._odomDbgN % 100 === 0)) { + console.log(`[odom TX seq=${this._odomDbgN}] qz=${rosQz.toFixed(4)} qw=${rosQw.toFixed(4)}`); + } + + // Send on CONTROL socket (not sensor socket) + this._sendControl(CH_ODOM, odomMsg); + } catch (e) { + console.warn("[DimosBridge] odom publish error:", e); + } + } + + _stopPublishing(): void { + for (const k of Object.keys(this._timers)) clearInterval(this._timers[k]); + this._timers = {}; + if (this._rafId) cancelAnimationFrame(this._rafId); + this._rafId = null; + } + + /** Send on the control WebSocket (odom, small real-time data) */ + _sendControl(channel: string, msg: any): void { + const ws = this.wsControl; + if (!ws || ws.readyState !== WebSocket.OPEN) return; + ws.send(encodePacket(channel, msg)); + } + + /** Send on a sensor WebSocket (images, lidar — large data). */ + _sendSensor(ws: WebSocket | null, channel: string, msg: any): void { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + ws.send(encodePacket(channel, msg)); + } + + /** Legacy _send — routes to appropriate socket based on channel */ + _send(channel: string, msg: any): void { + if (channel === CH_ODOM) { + this._sendControl(channel, msg); + } else if (channel === CH_IMAGE) { + this._sendSensor(this.wsRgb, channel, msg); + } else if (channel === CH_DEPTH) { + this._sendSensor(this.wsDepth, channel, msg); + } else { + this._sendSensor(this.wsSensors, channel, msg); + } + } + + // -- RGB -------------------------------------------------------------------- + + _publishRgbSync(header: any): void { + try { + if (!this._isSocketOpen(this.wsRgb)) return; + const frame = this.sensors.captureRgb(); + if (!frame) return; + + this._sendSensor(this.wsRgb, CH_IMAGE, new sensor_msgs.Image({ + header, + height: frame.height, + width: frame.width, + encoding: "jpeg", + is_bigendian: 0, + step: 0, // not applicable for compressed format + data_length: frame.data.length, + data: frame.data, + })); + } catch (e) { + console.warn("[DimosBridge] RGB publish error:", e); + } + } + + // -- Depth ------------------------------------------------------------------ + + _depthU16: Uint16Array | null = null; + + _publishDepthSync(header: any): void { + try { + if (!this._isSocketOpen(this.wsDepth)) return; + const frame = this.sensors.captureDepth(); + if (!frame) return; + + // Quantize float32 meters → uint16 millimeters (0–65.535m range, 1mm precision) + const n = frame.data.length; + if (!this._depthU16 || this._depthU16.length !== n) { + this._depthU16 = new Uint16Array(n); + } + const f32 = frame.data; + const u16 = this._depthU16; + for (let i = 0; i < n; i++) { + const mm = f32[i] * 1000; + u16[i] = mm > 65535 ? 65535 : mm < 0 ? 0 : mm; + } + const depthBytes = new Uint8Array(u16.buffer, u16.byteOffset, u16.byteLength); + + this._sendSensor(this.wsDepth, CH_DEPTH, new sensor_msgs.Image({ + header, + height: frame.height, + width: frame.width, + encoding: "16UC1", + is_bigendian: 0, + step: frame.width * 2, + data_length: depthBytes.length, + data: depthBytes, + })); + } catch (e) { + console.warn("[DimosBridge] depth publish error:", e); + } + } + + // -- LiDAR ------------------------------------------------------------------ + + _lidarDbgN = 0; + _publishLidarSync(header: any): void { + try { + if (!this._isSocketOpen(this.wsSensors)) return; + const frame = this.sensors.captureLidar(); + this._lidarDbgN++; + if (BRIDGE_DEBUG && (this._lidarDbgN <= 3 || this._lidarDbgN % 100 === 0)) { + console.log(`[DimosBridge] lidar #${this._lidarDbgN}: ${frame ? frame.numPoints : 'null'} pts, sensorWS=${this.wsSensors?.readyState}`); + } + if (!frame) return; + + const numPoints = frame.numPoints || 0; + if (numPoints === 0) return; + + this._ensureLidarCapacity(numPoints); + const pointStep = LIDAR_POINT_STEP; + const view = this._lidarView; + const pts = frame.points; + const intensity = frame.intensity; + + // Points are Three.js world-frame (Y-up). + // Convert to ROS world-frame (Z-up): cyclic permutation x→y, y→z, z→x + for (let i = 0; i < numPoints; i++) { + const off = i * pointStep; + const tx = pts[i * 3 + 0], ty = pts[i * 3 + 1], tz = pts[i * 3 + 2]; + view.setFloat32(off, tz, true); // ROS x = Three.js z + view.setFloat32(off + 4, tx, true); // ROS y = Three.js x + view.setFloat32(off + 8, ty, true); // ROS z = Three.js y + view.setFloat32(off + 12, intensity ? intensity[i] : 1.0, true); + } + + this._sendSensor(this.wsSensors, CH_LIDAR, new sensor_msgs.PointCloud2({ + header, + height: 1, + width: numPoints, + fields_length: this._pc2Fields.length, + fields: this._pc2Fields, + is_bigendian: 0, + point_step: pointStep, + row_step: numPoints * pointStep, + data_length: numPoints * pointStep, + data: new Uint8Array(this._lidarBuf, 0, numPoints * pointStep), + is_dense: 1, + })); + } catch (e) { + console.warn("[DimosBridge] LiDAR publish error:", e); + } + } + + /** Send a JSON command on the control WebSocket (used by EvalHarness). */ + sendCommand(cmd: Record): void { + const ws = this.wsControl; + if (ws && ws.readyState === WebSocket.OPEN) { + ws.send(JSON.stringify(cmd)); + } + } + + dispose(): void { + this._stopPublishing(); + if (this.wsControl) { this.wsControl.onclose = null; this.wsControl.close(); } + if (this.wsSensors) { this.wsSensors.onclose = null; this.wsSensors.close(); } + if (this.wsRgb) { this.wsRgb.onclose = null; this.wsRgb.close(); } + if (this.wsDepth) { this.wsDepth.onclose = null; this.wsDepth.close(); } + this.wsControl = null; + this.wsSensors = null; + this.wsRgb = null; + this.wsDepth = null; + } + + _isControlSocketOpen(): boolean { + return !!this.wsControl && this.wsControl.readyState === WebSocket.OPEN; + } + + _isSocketOpen(ws: WebSocket | null): boolean { + return !!ws && ws.readyState === WebSocket.OPEN; + } + + _ensureLidarCapacity(numPoints: number): void { + if (numPoints <= this._lidarCapacityPoints) return; + this._lidarCapacityPoints = numPoints; + this._lidarBuf = new ArrayBuffer(numPoints * LIDAR_POINT_STEP); + this._lidarView = new DataView(this._lidarBuf); + } +} diff --git a/misc/DimSim/src/dimos/evalHarness.ts b/misc/DimSim/src/dimos/evalHarness.ts new file mode 100644 index 0000000000..a77f6d9248 --- /dev/null +++ b/misc/DimSim/src/dimos/evalHarness.ts @@ -0,0 +1,227 @@ +/** + * EvalHarness — Browser-side eval orchestrator. + * + * Receives commands from the Deno eval runner (via WebSocket text messages), + * runs eval workflow, scores objectDistance rubric at timeout, returns result. + */ + +import { + scoreObjectDistance, + scoreRadiusContains, + type SceneState, + type ObjectDistanceCriteria, + type RadiusContainsCriteria, +} from "./rubrics.ts"; +import type { DimosBridge } from "./dimosBridge.ts"; + +export interface AgentPose { x: number; y: number; z: number; yaw: number; pitch: number; } +export interface StartPose { x?: number; y?: number; z?: number; yaw?: number; } + +export interface SuccessCriteria { + objectDistance?: ObjectDistanceCriteria; + radiusContains?: RadiusContainsCriteria; +} + +export interface Workflow { + name: string; + task: string; + environment?: string; + startPose?: StartPose; + timeoutSec?: number; + successCriteria?: SuccessCriteria; +} + +export interface EvalHarnessOptions { + bridge: DimosBridge; + getSceneState: () => SceneState; + getAgentPose: () => AgentPose | null; + channel?: string; +} + +declare global { + interface Window { __dimosAgent?: any; } +} + +export class EvalHarness { + bridge: DimosBridge; + getSceneState: () => SceneState; + getAgentPose: () => AgentPose | null; + channel: string; + + _workflow: Workflow | null = null; + _startTime = 0; + _timeoutTimer: ReturnType | null = null; + _overlay: HTMLDivElement | null = null; + + constructor({ bridge, getSceneState, getAgentPose, channel }: EvalHarnessOptions) { + this.bridge = bridge; + this.getSceneState = getSceneState; + this.getAgentPose = getAgentPose; + this.channel = channel || ""; + this._hookBridgeMessages(); + } + + _hookBridgeMessages(): void { + const origConnect = this.bridge.connect.bind(this.bridge); + this.bridge.connect = () => { + origConnect(); + setTimeout(() => { + const ws = this.bridge.ws; + if (ws) this._patchWsOnMessage(ws); + }, 100); + }; + const ws = this.bridge.ws; + if (ws) this._patchWsOnMessage(ws); + } + + _patchWsOnMessage(ws: WebSocket): void { + const origOnMessage = ws.onmessage; + const evalTypes = new Set(["startWorkflow", "stopWorkflow", "loadEnv", "ping"]); + ws.onmessage = (event: MessageEvent) => { + if (typeof event.data === "string") { + try { + const cmd = JSON.parse(event.data); + if (cmd.type && evalTypes.has(cmd.type)) { + this._handleCommand(cmd); + return; + } + } catch { /* not JSON, pass through */ } + // Pass all non-eval text messages through to origOnMessage + if (origOnMessage) (origOnMessage as (e: MessageEvent) => void).call(ws, event); + return; + } + if (origOnMessage) (origOnMessage as (e: MessageEvent) => void).call(ws, event); + }; + } + + _send(cmd: Record): void { + // Tag outgoing messages with channel for multi-page routing + if (this.channel) cmd.channel = this.channel; + this.bridge.sendCommand(cmd); + } + + async _handleCommand(cmd: { type: string; channel?: string; workflow?: Workflow; [k: string]: any }): Promise { + // Channel filtering: if cmd has a channel and it doesn't match ours, ignore + if (this.channel && cmd.channel && cmd.channel !== this.channel) return; + console.log("[eval] command:", cmd.type); + switch (cmd.type) { + case "startWorkflow": + await this._startWorkflow(cmd.workflow!); + break; + case "stopWorkflow": + await this._stopWorkflow("runner-requested"); + break; + case "loadEnv": + // Scene is already loaded in --connect mode, just ack + this._send({ type: "envReady", scene: cmd.scene }); + break; + case "ping": + this._send({ type: "pong", ts: Date.now() }); + break; + default: + break; + } + } + + async _startWorkflow(workflow: Workflow): Promise { + this._workflow = workflow; + this._startTime = Date.now(); + + console.log(`[eval] starting: ${workflow.name} — "${workflow.task}"`); + + if (workflow.startPose) { + const p = workflow.startPose; + const agent = window.__dimosAgent; + if (agent) { + agent.setPosition(p.x ?? 0, p.y ?? 0.5, p.z ?? 0); + if (p.yaw !== undefined) agent.group.rotation.y = (p.yaw * Math.PI) / 180; + } + } + + const timeoutMs = (workflow.timeoutSec || 120) * 1000; + this._timeoutTimer = setTimeout(() => this._stopWorkflow("timeout"), timeoutMs); + + this._showOverlay(workflow.task, workflow.timeoutSec || 120); + this._send({ type: "workflowStarted", name: workflow.name }); + } + + async _stopWorkflow(reason: string): Promise { + if (!this._workflow) return; + if (this._timeoutTimer) clearTimeout(this._timeoutTimer); + this._timeoutTimer = null; + + console.log(`[eval] stopped: ${this._workflow.name} (${reason})`); + + const sceneState = this.getSceneState(); + const agentPose = this.getAgentPose(); + if (agentPose) { + sceneState.agentPos = { x: agentPose.x, y: agentPose.y, z: agentPose.z }; + } + + const criteria = this._workflow.successCriteria || {}; + const scores: Record = {}; + if (criteria.objectDistance) { + scores.objectDistance = scoreObjectDistance(criteria.objectDistance, sceneState); + } + if (criteria.radiusContains) { + scores.radiusContains = scoreRadiusContains(criteria.radiusContains, sceneState); + } + + const pass = Object.values(scores).every((s: any) => s.pass !== false); + const od = scores.objectDistance; + this._showResult(pass, od ? od.details : reason); + + const result = { + type: "workflowComplete", + name: this._workflow.name, + environment: this._workflow.environment, + reason, + durationMs: Date.now() - this._startTime, + rubricScores: scores, + }; + console.log("[eval] result:", result); + this._send(result); + this._workflow = null; + } + + // -- UI overlay -------------------------------------------------------------- + + _showOverlay(task: string, timeoutSec: number): void { + if (this._overlay) this._overlay.remove(); + const el = document.createElement("div"); + el.style.cssText = "position:fixed;top:16px;left:50%;transform:translateX(-50%);z-index:99999;background:rgba(0,0,0,0.85);color:#fff;font:14px/1.5 monospace;padding:12px 24px;border-radius:10px;text-align:center;pointer-events:none;"; + const taskEl = document.createElement("div"); + taskEl.style.cssText = "color:#4fc3f7;font-size:16px;font-weight:bold;margin-bottom:4px;"; + taskEl.textContent = `EVAL: ${task}`; + const timerEl = document.createElement("div"); + timerEl.style.cssText = "color:#aaa;font-size:13px;"; + el.appendChild(taskEl); + el.appendChild(timerEl); + document.body.appendChild(el); + this._overlay = el; + + let remaining = timeoutSec; + timerEl.textContent = `${remaining}s remaining`; + const interval = setInterval(() => { + remaining--; + if (remaining <= 0 || !this._workflow) { clearInterval(interval); return; } + timerEl.textContent = `${remaining}s remaining`; + }, 1000); + } + + _showResult(pass: boolean, details: string): void { + if (this._overlay) this._overlay.remove(); + const el = document.createElement("div"); + const bg = pass ? "rgba(46,125,50,0.9)" : "rgba(198,40,40,0.9)"; + el.style.cssText = `position:fixed;top:16px;left:50%;transform:translateX(-50%);z-index:99999;background:${bg};color:#fff;font:14px/1.5 monospace;padding:12px 24px;border-radius:10px;text-align:center;pointer-events:none;`; + el.textContent = `${pass ? "PASS" : "FAIL"}: ${details}`; + document.body.appendChild(el); + this._overlay = el; + setTimeout(() => { if (this._overlay === el) { el.remove(); this._overlay = null; } }, 5000); + } + + dispose(): void { + if (this._timeoutTimer) clearTimeout(this._timeoutTimer); + if (this._overlay) { this._overlay.remove(); this._overlay = null; } + } +} diff --git a/misc/DimSim/src/dimos/rubrics.ts b/misc/DimSim/src/dimos/rubrics.ts new file mode 100644 index 0000000000..fef455a42d --- /dev/null +++ b/misc/DimSim/src/dimos/rubrics.ts @@ -0,0 +1,156 @@ +/** + * Eval Rubrics — deterministic scoring for eval workflows. + * + * Rubrics: + * objectDistance — Euclidean distance from agent to target bbox surface + * radiusContains — agent is within radius of centroid computed from multiple targets + */ + +export interface Vec3 { x: number; y: number; z: number; } + +export interface AssetEntry { + title?: string; + id?: string; + transform?: { x?: number; y?: number; z?: number }; + _bbox?: { w: number; h: number; d: number }; +} + +export interface SceneState { + assets?: AssetEntry[]; + agentPos?: Vec3; +} + +export interface ObjectDistanceCriteria { + object: string; + target: string; + thresholdM?: number; +} + +export interface ObjectDistanceResult { + pass: boolean; + distanceM: number; + details: string; +} + +export function scoreObjectDistance(criteria: ObjectDistanceCriteria, sceneState: SceneState): ObjectDistanceResult { + const { target: targetName, thresholdM = 0.5 } = criteria; + + if (!sceneState.agentPos) { + return { pass: false, distanceM: Infinity, details: "Agent position not available" }; + } + + const targetHit = _findTarget(targetName, sceneState); + if (!targetHit) { + return { pass: false, distanceM: Infinity, details: `Target "${targetName}" not found in scene` }; + } + + const dist = _distToSurface(sceneState.agentPos, targetHit.pos, targetHit.bbox); + + return { + pass: dist <= thresholdM, + distanceM: Math.round(dist * 1000) / 1000, + details: `agent is ${dist.toFixed(3)}m from "${targetName}" surface (threshold: ${thresholdM}m)`, + }; +} + +// -- radiusContains ----------------------------------------------------------- + +export interface RadiusContainsCriteria { + object?: string; // defaults to "agent" + targets: string[]; // scene objects whose centroid defines the region + radiusM?: number; // max distance from centroid (default 3.0) +} + +export interface RadiusContainsResult { + pass: boolean; + distanceM: number; + centroid: Vec3; + foundTargets: string[]; + missingTargets: string[]; + details: string; +} + +export function scoreRadiusContains(criteria: RadiusContainsCriteria, sceneState: SceneState): RadiusContainsResult { + const { targets, radiusM = 3.0 } = criteria; + + const fail = (details: string): RadiusContainsResult => ({ + pass: false, distanceM: Infinity, centroid: { x: 0, y: 0, z: 0 }, + foundTargets: [], missingTargets: targets, details, + }); + + if (!sceneState.agentPos) return fail("Agent position not available"); + if (!targets || targets.length === 0) return fail("No targets specified"); + + const found: { name: string; pos: Vec3 }[] = []; + const missing: string[] = []; + for (const name of targets) { + const hit = _findTarget(name, sceneState); + if (hit) found.push({ name, pos: hit.pos }); + else missing.push(name); + } + + if (found.length < 2 && found.length < targets.length) { + // Need at least 2 targets found, or all if fewer than 2 specified + if (found.length === 0) return fail(`No targets found: ${missing.join(", ")}`); + } + + // Compute centroid of found targets + const centroid: Vec3 = { x: 0, y: 0, z: 0 }; + for (const f of found) { + centroid.x += f.pos.x; + centroid.y += f.pos.y; + centroid.z += f.pos.z; + } + centroid.x /= found.length; + centroid.y /= found.length; + centroid.z /= found.length; + + const dx = sceneState.agentPos.x - centroid.x; + const dy = sceneState.agentPos.y - centroid.y; + const dz = sceneState.agentPos.z - centroid.z; + const dist = Math.round(Math.sqrt(dx * dx + dy * dy + dz * dz) * 1000) / 1000; + + const foundNames = found.map((f) => f.name); + const pass = dist <= radiusM; + const missingNote = missing.length > 0 ? ` (missing: ${missing.join(", ")})` : ""; + + return { + pass, + distanceM: dist, + centroid, + foundTargets: foundNames, + missingTargets: missing, + details: `agent is ${dist.toFixed(3)}m from centroid of [${foundNames.join(", ")}]${missingNote} (radius: ${radiusM}m)`, + }; +} + +// -- helpers ------------------------------------------------------------------ + +function _distToSurface(from: Vec3, center: Vec3, bbox?: { w: number; h: number; d: number }): number { + if (!bbox) { + const dx = from.x - center.x, dy = from.y - center.y, dz = from.z - center.z; + return Math.sqrt(dx * dx + dy * dy + dz * dz); + } + const hw = bbox.w / 2, hh = bbox.h / 2, hd = bbox.d / 2; + const cx = Math.max(center.x - hw, Math.min(from.x, center.x + hw)); + const cy = Math.max(center.y - hh, Math.min(from.y, center.y + hh)); + const cz = Math.max(center.z - hd, Math.min(from.z, center.z + hd)); + const dx = from.x - cx, dy = from.y - cy, dz = from.z - cz; + return Math.sqrt(dx * dx + dy * dy + dz * dz); +} + +function _findTarget(name: string, sceneState: SceneState): { pos: Vec3; bbox?: { w: number; h: number; d: number } } | null { + const lower = name.toLowerCase(); + if (!sceneState.assets) return null; + for (const asset of sceneState.assets) { + if (asset.title?.toLowerCase().includes(lower) || asset.id?.toLowerCase().includes(lower)) { + if (asset.transform) { + return { + pos: { x: asset.transform.x || 0, y: asset.transform.y || 0, z: asset.transform.z || 0 }, + bbox: asset._bbox, + }; + } + } + } + return null; +} diff --git a/misc/DimSim/src/dimos/sceneEditor.ts b/misc/DimSim/src/dimos/sceneEditor.ts new file mode 100644 index 0000000000..e6d7cc1ca9 --- /dev/null +++ b/misc/DimSim/src/dimos/sceneEditor.ts @@ -0,0 +1,452 @@ +/** + * SceneEditor — Browser-side script execution engine. + * + * Receives {type: "exec", code, id?} commands via the DimosBridge control WS, + * evaluates user JS with full Three.js + Rapier globals exposed, and returns + * {type: "execResult", id, success, result?, error?}. + * + * Must NOT modify engine.js — hooks into DimosBridge WS the same way EvalHarness does. + */ + +import type { DimosBridge } from "./dimosBridge.ts"; + +export interface SceneEditorGlobals { + scene: any; // THREE.Scene + THREE: any; // Three.js namespace + RAPIER: any; // Rapier namespace (may be null until ensureRapierLoaded) + rapierWorld: any; // Rapier.World (may be null) + worldBody: any; // Fixed RigidBody for static colliders + renderer: any; // THREE.WebGLRenderer + camera: any; // THREE.PerspectiveCamera + agent: any; // Player agent (has getPosition, setPosition, group) + assets: any[]; // Scene assets array + assetsGroup: any; // THREE.Group containing loaded asset meshes + gltfLoader: any; // THREE GLTFLoader instance +} + +export interface SceneEditorOptions { + bridge: DimosBridge; + globals: SceneEditorGlobals; + channel?: string; +} + +// AsyncFunction constructor — allows top-level await in user scripts +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + +export class SceneEditor { + bridge: DimosBridge; + globals: SceneEditorGlobals; + channel: string; + + constructor({ bridge, globals, channel }: SceneEditorOptions) { + this.bridge = bridge; + this.globals = globals; + this.channel = channel || ""; + this._hookBridgeMessages(); + } + + _hookBridgeMessages(): void { + const origConnect = this.bridge.connect.bind(this.bridge); + this.bridge.connect = () => { + origConnect(); + setTimeout(() => { + const ws = this.bridge.ws; + if (ws) this._patchWsOnMessage(ws); + }, 100); + }; + const ws = this.bridge.ws; + if (ws) this._patchWsOnMessage(ws); + } + + _patchWsOnMessage(ws: WebSocket): void { + const origOnMessage = ws.onmessage; + ws.onmessage = (event: MessageEvent) => { + if (typeof event.data === "string") { + try { + const cmd = JSON.parse(event.data); + if (cmd.type === "exec" || cmd.type === "loadScript") { + this._handleCommand(cmd); + return; + } + } catch { /* not JSON or not for us */ } + } + // Pass through to existing handlers (EvalHarness, DimosBridge) + if (origOnMessage) (origOnMessage as (e: MessageEvent) => void).call(ws, event); + }; + } + + _send(msg: Record): void { + if (this.channel) msg.channel = this.channel; + this.bridge.sendCommand(msg); + } + + async _handleCommand(cmd: { type: string; code?: string; url?: string; id?: string; channel?: string }): Promise { + if (this.channel && cmd.channel && cmd.channel !== this.channel) return; + + if (cmd.type === "exec" && cmd.code) { + await this._execCode(cmd.code, cmd.id); + } else if (cmd.type === "loadScript" && cmd.url) { + await this._loadScript(cmd.url, cmd.id); + } + } + + // Track colliders created by addCollider so removeCollider can clean up + _colliderMap: Map = new Map(); // mesh.uuid → Rapier collider + // Track dynamic rigid bodies (body + mesh ref for position sync) + _dynamicBodies: Map = new Map(); + // Track NPC mixers for animation updates + _npcMixers: Map = new Map(); // npc name → THREE.AnimationMixer + _npcAnimFrame: number | null = null; + _npcClock: any = null; // THREE.Clock + + async _execCode(code: string, id?: string): Promise { + console.log(`[sceneEditor] exec${id ? ` (${id})` : ""}:`, code.slice(0, 100)); + try { + const g = this.globals; + const colliderMap = this._colliderMap; + + // loadGLTF: convenience async helper for loading GLTF/GLB models + const loadGLTF = (url: string): Promise => + new Promise((resolve, reject) => + g.gltfLoader.load(url, resolve, undefined, reject), + ); + + // addCollider: create a physics collider for a mesh/group + // shape: "trimesh" (default) | "box" | "sphere" + // opts.dynamic: if true, creates a dynamic rigid body (responds to gravity/collisions) + // opts.mass: mass in kg (default 1.0, only for dynamic) + // opts.restitution: bounciness 0-1 (default 0.3, only for dynamic) + // Creates collider browser-side AND sends command to server (for lidar/physics) + const sendPhysics = this._send.bind(this); + const dynamicBodies = this._dynamicBodies; + const selfRef = this; + const addCollider = (obj: any, shapeOrOpts?: string | { shape?: string; dynamic?: boolean; mass?: number; restitution?: number }): any => { + let shape = "trimesh"; + let dynamic = false; + let mass = 1.0; + let restitution = 0.3; + if (typeof shapeOrOpts === "string") { + shape = shapeOrOpts; + } else if (shapeOrOpts) { + shape = shapeOrOpts.shape || "trimesh"; + dynamic = !!shapeOrOpts.dynamic; + mass = shapeOrOpts.mass ?? 1.0; + restitution = shapeOrOpts.restitution ?? 0.3; + } + + // Remove existing collider if any + removeCollider(obj); + + const bbox = new g.THREE.Box3().setFromObject(obj); + const size = new g.THREE.Vector3(); + const center = new g.THREE.Vector3(); + bbox.getSize(size); + bbox.getCenter(center); + + const clamp = (v: number) => Math.max(v, 0.001); + + // Build server-side descriptor (shape-agnostic) + const serverDesc: any = { + shape, + position: { x: center.x, y: center.y, z: center.z }, + }; + + if (shape === "sphere") { + const r = clamp(Math.max(size.x, size.y, size.z) / 2); + serverDesc.radius = r; + } else if (shape === "trimesh") { + const verts: number[] = []; + const indices: number[] = []; + let vertBase = 0; + obj.traverse((m: any) => { + if (!m.isMesh) return; + const geom = m.geometry; + const posAttr = geom?.attributes?.position; + if (!posAttr) return; + const tmpPos = new g.THREE.Vector3(); + for (let i = 0; i < posAttr.count; i++) { + tmpPos.fromBufferAttribute(posAttr, i).applyMatrix4(m.matrixWorld); + verts.push(tmpPos.x, tmpPos.y, tmpPos.z); + } + if (geom.index) { + for (let i = 0; i < geom.index.count; i++) indices.push(geom.index.getX(i) + vertBase); + } else { + for (let i = 0; i < posAttr.count; i++) indices.push(vertBase + i); + } + vertBase += posAttr.count; + }); + if (verts.length < 9 || indices.length < 3) throw new Error("Not enough geometry for trimesh"); + serverDesc.vertices = Array.from(verts); + serverDesc.indices = Array.from(indices); + } else { + // box (default) + serverDesc.halfExtents = { + x: clamp(size.x / 2), y: clamp(size.y / 2), z: clamp(size.z / 2), + }; + } + + // Browser-side collider (for standalone / non-dimos mode) + if (g.RAPIER && g.rapierWorld) { + let desc: any; + if (shape === "sphere") { + desc = g.RAPIER.ColliderDesc.ball(serverDesc.radius); + if (!dynamic) desc.setTranslation(center.x, center.y, center.z); + } else if (shape === "trimesh") { + desc = g.RAPIER.ColliderDesc.trimesh( + new Float32Array(serverDesc.vertices), new Uint32Array(serverDesc.indices) + ); + } else { + const h = serverDesc.halfExtents; + desc = g.RAPIER.ColliderDesc.cuboid(h.x, h.y, h.z); + if (!dynamic) desc.setTranslation(center.x, center.y, center.z); + } + desc.setFriction(0.9); + desc.setRestitution(restitution); + + if (dynamic && shape !== "trimesh") { + // Dynamic: create rigid body + attach collider + const bodyDesc = g.RAPIER.RigidBodyDesc.dynamic() + .setTranslation(center.x, center.y, center.z); + const body = g.rapierWorld.createRigidBody(bodyDesc); + body.setAdditionalMass(mass); + const collider = g.rapierWorld.createCollider(desc, body); + colliderMap.set(obj.uuid, collider); + dynamicBodies.set(obj.uuid, { body, mesh: obj }); + selfRef._ensureDynamicSyncLoop(); + } else { + // Static: collider with no parent body + const collider = g.rapierWorld.createCollider(desc); + colliderMap.set(obj.uuid, collider); + } + } + + // Server-side collider (for lidar + dimos physics) + serverDesc.dynamic = dynamic; + if (dynamic) { serverDesc.mass = mass; serverDesc.restitution = restitution; } + sendPhysics({ type: "physicsColliderAdd", uuid: obj.uuid, desc: serverDesc }); + + return { shape, dynamic, uuid: obj.uuid, size: { x: +size.x.toFixed(3), y: +size.y.toFixed(3), z: +size.z.toFixed(3) } }; + }; + + // removeCollider: remove collider browser-side + server-side + const removeCollider = (obj: any): boolean => { + const existing = colliderMap.get(obj.uuid); + if (existing) { + try { + g.rapierWorld?.removeCollider(existing, true); + } catch { /* already removed */ } + colliderMap.delete(obj.uuid); + } + // Clean up dynamic rigid body if any + const dynEntry = dynamicBodies.get(obj.uuid); + if (dynEntry) { + try { g.rapierWorld?.removeRigidBody(dynEntry.body); } catch { /* already removed */ } + dynamicBodies.delete(obj.uuid); + } + // Always tell server to remove (even if browser didn't have it) + sendPhysics({ type: "physicsColliderRemove", uuid: obj.uuid }); + return !!existing; + }; + + // addNPC: load an animated GLTF character, place it, and play an animation + const npcMixers = this._npcMixers; + const self = this; + const addNPC = async (opts: { + url: string; + name?: string; + position?: { x: number; y: number; z: number }; + rotation?: number; // yaw in radians + scale?: number; + animation?: string | number; // clip name or index (default: 0) + collider?: boolean; // add trimesh collider (default: true) + }): Promise => { + const gltf = await loadGLTF(opts.url); + const model = gltf.scene; + const npcName = opts.name || `npc-${Date.now().toString(36)}`; + model.name = npcName; + if (opts.position) model.position.set(opts.position.x, opts.position.y, opts.position.z); + if (opts.rotation != null) model.rotation.y = opts.rotation; + if (opts.scale != null) model.scale.setScalar(opts.scale); + model.traverse((child: any) => { if (child.isMesh) { child.castShadow = true; child.receiveShadow = true; } }); + g.scene.add(model); + + // Set up animation + let activeClipName = ""; + const clipNames: string[] = []; + if (gltf.animations && gltf.animations.length > 0) { + const mixer = new g.THREE.AnimationMixer(model); + npcMixers.set(npcName, mixer); + + for (const clip of gltf.animations) clipNames.push(clip.name); + // Store clips on model so they can be switched at runtime + model.animations = gltf.animations; + + // Select animation clip + let clipIndex = 0; + if (typeof opts.animation === "string") { + const idx = gltf.animations.findIndex((c: any) => c.name.toLowerCase().includes(opts.animation!.toString().toLowerCase())); + if (idx >= 0) clipIndex = idx; + } else if (typeof opts.animation === "number") { + clipIndex = Math.min(opts.animation, gltf.animations.length - 1); + } + + const clip = gltf.animations[clipIndex]; + activeClipName = clip.name; + const action = mixer.clipAction(clip); + action.play(); + + // Start the animation loop if not already running + self._ensureNpcAnimLoop(); + } + + // Add collider (default: trimesh) + let colliderInfo = null; + if (opts.collider !== false) { + colliderInfo = addCollider(model, "trimesh"); + } + + return { + name: npcName, + animations: clipNames, + activeAnimation: activeClipName, + collider: colliderInfo, + }; + }; + + // removeNPC: remove an NPC from scene and clean up its mixer + const removeNPC = (name: string): boolean => { + const obj = g.scene.getObjectByName(name); + if (!obj) return false; + // Stop animation mixer + const mixer = npcMixers.get(name); + if (mixer) { mixer.stopAllAction(); npcMixers.delete(name); } + // Remove collider + removeCollider(obj); + // Clear name before removal so getObjectByName won't find stale refs + obj.name = ""; + // Remove from scene + obj.traverse((child: any) => { + if (child.isMesh) { child.geometry?.dispose(); child.material?.dispose(); } + }); + g.scene.remove(obj); + return true; + }; + + // autoScale: detect cm/m mismatch and normalize model to scene scale. + // Heuristic: if bounding box exceeds targetMaxDim (default 50m) in any axis, + // assume the model is in centimeters and scale by 0.01. For intermediate cases + // (10-50m), scale proportionally so the largest dimension equals targetMaxDim. + // Returns the scale factor applied (1.0 if no change). + const autoScale = (obj: any, targetMaxDim = 50): number => { + const bbox = new g.THREE.Box3().setFromObject(obj); + const size = new g.THREE.Vector3(); + bbox.getSize(size); + const maxDim = Math.max(size.x, size.y, size.z); + if (maxDim <= 0.001) return 1.0; // degenerate + let scaleFactor = 1.0; + if (maxDim > targetMaxDim * 2) { + // Very large — likely centimeters (100x off) + scaleFactor = 0.01; + } else if (maxDim > targetMaxDim) { + // Moderately large — scale down proportionally + scaleFactor = targetMaxDim / maxDim; + } + if (scaleFactor !== 1.0) { + obj.scale.multiplyScalar(scaleFactor); + obj.updateMatrixWorld(true); + console.log(`[sceneEditor] autoScale: ${maxDim.toFixed(1)}m → ${(maxDim * scaleFactor).toFixed(1)}m (×${scaleFactor.toFixed(4)})`); + } + return scaleFactor; + }; + + const fn = new AsyncFunction( + "scene", "THREE", "RAPIER", "rapierWorld", "renderer", "camera", + "agent", "playerBody", "assets", "assetsGroup", + "loadGLTF", "addCollider", "removeCollider", "addNPC", "removeNPC", "autoScale", + code, + ); + const result = await fn( + g.scene, g.THREE, g.RAPIER, g.rapierWorld, g.renderer, g.camera, + g.agent, g.agent, g.assets, g.assetsGroup, + loadGLTF, addCollider, removeCollider, addNPC, removeNPC, autoScale, + ); + this._send({ type: "execResult", id, success: true, result: _serialize(result) }); + } catch (err: any) { + console.error("[sceneEditor] exec error:", err); + this._send({ type: "execResult", id, success: false, error: String(err) }); + } + } + + async _loadScript(url: string, id?: string): Promise { + console.log(`[sceneEditor] loadScript${id ? ` (${id})` : ""}:`, url); + try { + const resp = await fetch(url); + if (!resp.ok) throw new Error(`HTTP ${resp.status}: ${resp.statusText}`); + const code = await resp.text(); + await this._execCode(code, id); + } catch (err: any) { + console.error("[sceneEditor] loadScript error:", err); + this._send({ type: "execResult", id, success: false, error: String(err) }); + } + } + + _ensureNpcAnimLoop(): void { + if (this._npcAnimFrame != null) return; + this._startUpdateLoop(); + } + + _ensureDynamicSyncLoop(): void { + if (this._npcAnimFrame != null) return; + this._startUpdateLoop(); + } + + /** Shared rAF loop for NPC animations + dynamic body position sync. */ + _startUpdateLoop(): void { + if (this._npcAnimFrame != null) return; + if (!this._npcClock) { + this._npcClock = new this.globals.THREE.Clock(); + } + const tick = () => { + const hasWork = this._npcMixers.size > 0 || this._dynamicBodies.size > 0; + if (!hasWork) { + this._npcAnimFrame = null; + return; + } + const dt = this._npcClock.getDelta(); + + // Update NPC animations + for (const mixer of this._npcMixers.values()) { + mixer.update(dt); + } + + // Sync dynamic body positions → Three.js meshes + for (const { body, mesh } of this._dynamicBodies.values()) { + const t = body.translation(); + const r = body.rotation(); + mesh.position.set(t.x, t.y, t.z); + mesh.quaternion.set(r.x, r.y, r.z, r.w); + } + + this._npcAnimFrame = requestAnimationFrame(tick); + }; + this._npcAnimFrame = requestAnimationFrame(tick); + } + + dispose(): void { + if (this._npcAnimFrame != null) cancelAnimationFrame(this._npcAnimFrame); + for (const mixer of this._npcMixers.values()) mixer.stopAllAction(); + this._npcMixers.clear(); + this._dynamicBodies.clear(); + } +} + +/** Safely serialize a return value for JSON transport. */ +function _serialize(val: any): any { + if (val === undefined || val === null) return val; + if (typeof val === "number" || typeof val === "string" || typeof val === "boolean") return val; + if (Array.isArray(val)) return val.map(_serialize); + // Three.js objects have .toJSON() but it's huge — just return type + id + if (val.isObject3D) return { _type: "Object3D", type: val.type, name: val.name, uuid: val.uuid }; + if (val.isMesh) return { _type: "Mesh", name: val.name, uuid: val.uuid }; + try { return JSON.parse(JSON.stringify(val)); } catch { return String(val); } +} diff --git a/misc/DimSim/src/engine.js b/misc/DimSim/src/engine.js new file mode 100644 index 0000000000..2f19688c03 --- /dev/null +++ b/misc/DimSim/src/engine.js @@ -0,0 +1,7763 @@ +import "./style.css"; + +import * as THREE from "three"; +import { PointerLockControls } from "three/examples/jsm/controls/PointerLockControls.js"; +import { AiAvatar } from "./AiAvatar.js"; +import { ACTIONS as SIM_VLM_ACTIONS, DEFAULTS as SIM_VLM_DEFAULTS } from "./ai/sim/vlmActions.js"; +import { buildPrompt as buildSimVlmPrompt } from "./ai/sim/vlmPrompt.js"; +import { MODEL_CONFIG } from "./ai/modelConfig.js"; +import { requestVlmDecision } from "./ai/vlmClient.js"; +import { captureAgentPovBase64, processPendingCaptures, hasPendingCapture } from "./ai/visionCapture.js"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; +import { RoomEnvironment } from "three/examples/jsm/environments/RoomEnvironment.js"; +import { RoundedBoxGeometry } from "three/examples/jsm/geometries/RoundedBoxGeometry.js"; + +const ACTIVE_VLM_ACTIONS = SIM_VLM_ACTIONS; +const ACTIVE_VLM_DEFAULTS = SIM_VLM_DEFAULTS; +const buildActiveVlmPrompt = () => buildSimVlmPrompt({ actions: ACTIVE_VLM_ACTIONS }); +const resolveActiveVlmModel = () => MODEL_CONFIG.simMode; + +let threeRendererRef = null; +let threeSceneRef = null; +let RAPIER = null; +let _rapierInitPromise = null; +let rapierWorld = null; +let worldBody = null; +let playerBody = null; +let playerCollider = null; +let flyMode = true; +let ghostMode = true; +let characterController = null; +let _rapierStepFaultCount = 0; +let walkVerticalVel = 0; +let aiAgents = []; + +// Track asset collider handles for cleanup +const _assetColliderHandles = new Map(); + +// Player dimensions (tuned smaller so you can fit inside tighter splat/glb interiors). +const PLAYER_RADIUS = 0.12; +const PLAYER_HALF_HEIGHT = 0.25; +const PLAYER_EYE_HEIGHT = PLAYER_HALF_HEIGHT + PLAYER_RADIUS + 0.2; // camera above body origin +const LIDAR_MOUNT_HEIGHT = 0.35; // Go2 lidar mount height above ground +// Real Go2 front camera height above ground in its home/operating crouch pose. +// Used for agent-POV captures so the rendered view matches what the hardware +// camera would see (low, not eye-level). ~0.30 m matches the MuJoCo go2.xml +// home keyframe (thigh 0.9, calf -1.8) which the Go2 actually stands in. +const GO2_CAMERA_HEIGHT = 0.30; +// Go2 front RGB-D camera is mounted on the front of the head, forward of the +// body center. Offsetting places the camera origin outside the robot mesh so +// POV captures don't render the inside of the body. +const GO2_CAMERA_FORWARD = 0.18; + +const canvas = document.getElementById("c"); +const statusEl = document.getElementById("status"); +const resetBtn = document.getElementById("reset"); +const assetsListEl = document.getElementById("assets-list"); // null in sim-only +const overlayEl = document.getElementById("overlay"); +const simPanelCollapseBtn = document.getElementById("sim-panel-collapse"); +const simPanelOpenBtn = document.getElementById("sim-panel-open"); +const statusSimEl = document.getElementById("status-sim"); +const spawnAiBtn = document.getElementById("spawn-ai"); +const agentPanelEl = document.getElementById("agent-panel"); +const agentLastEl = document.getElementById("agent-last"); +const agentObservationEl = document.getElementById("agent-observation"); +const agentShotImgEl = document.getElementById("agent-shot-img"); +const agentReqMetaEl = document.getElementById("agent-req-meta"); +const agentReqPromptEl = document.getElementById("agent-req-prompt"); +const agentReqContextEl = document.getElementById("agent-req-context"); +const agentRespRawEl = document.getElementById("agent-resp-raw"); +const agentLogEl = document.getElementById("agent-log"); +const agentTaskStatusEl = document.getElementById("agent-task-status"); +const agentTaskInputEl = document.getElementById("agent-task-input"); +const agentTaskStartBtn = document.getElementById("agent-task-start"); +const agentTaskEndBtn = document.getElementById("agent-task-end"); +const simCameraModeToggleBtn = document.getElementById("sim-camera-toggle"); +const simViewRgbdBtn = document.getElementById("sim-view-rgbd"); +const simViewLidarBtn = document.getElementById("sim-view-lidar"); +const simViewCompareBtn = document.getElementById("sim-view-compare"); +const simRgbdGrayBtn = document.getElementById("sim-rgbd-gray"); +const simRgbdColormapBtn = document.getElementById("sim-rgbd-colormap"); +const simRgbdAutoRangeBtn = document.getElementById("sim-rgbd-auto-range"); +const simRgbdNoiseBtn = document.getElementById("sim-rgbd-noise"); +const simRgbdSpeckleBtn = document.getElementById("sim-rgbd-speckle"); +const simRgbdMinEl = document.getElementById("sim-rgbd-min"); +const simRgbdMaxEl = document.getElementById("sim-rgbd-max"); +const simRgbdMinValEl = document.getElementById("sim-rgbd-min-val"); +const simRgbdMaxValEl = document.getElementById("sim-rgbd-max-val"); +const simRgbdPcOverlayBtn = document.getElementById("sim-rgbd-pc-overlay"); +const simLidarColorRangeBtn = document.getElementById("sim-lidar-color-range"); +const simLidarOrderedDebugBtn = document.getElementById("sim-lidar-ordered-debug"); +const simLidarNoiseBtn = document.getElementById("sim-lidar-noise"); +const simLidarMultiReturnBtn = document.getElementById("sim-lidar-multireturn"); + +// ── dimos integration mode ────────────────────────────────────────────────── +// Activated via ?dimos=1 URL param or window.__dimosMode (injected by Deno bridge server). +// When active: internal VLM loop disabled, agent pose driven by external /odom, +// sensor data (RGB, depth, LiDAR) published as LCM packets via WebSocket bridge. +const _dimosParams = new URLSearchParams(window.location.search); +const dimosMode = _dimosParams.get("dimos") === "1" || window.__dimosMode === true; +if (dimosMode) document.body.classList.add("dimos-mode"); +const dimosScene = _dimosParams.get("scene") || window.__dimosScene || null; +let simSensorViewMode = "rgb"; // "rgb" | "rgbd" | "lidar" +let simCompareView = false; // show RGB + RGB-D + LiDAR side-by-side +let simPanelCollapsed = false; +let simUserCameraMode = localStorage.getItem("sparkWorldSimCameraMode") === "user" ? "user" : "agent"; +let rgbdVizMode = "colormap"; // "colormap" | "gray" +let rgbdAutoRange = true; +let rgbdRangeMinM = 0.2; +let rgbdRangeMaxM = 12.0; +let rgbdNoiseEnabled = false; +let rgbdSpeckleEnabled = false; +let rgbdPcOverlayOnLidar = false; +let lidarColorByRange = false; // false = intensity grayscale (realistic default) +let lidarOrderedDebugView = false; // false=unordered 3D cloud, true=ordered rings debug +let lidarNoiseEnabled = false; // deterministic range noise + dropouts +let lidarMultiReturnMode = "strongest"; // "strongest" | "last" +let worldKey = localStorage.getItem("sparkWorldLastWorldKey") ?? "default"; + +function clampNum(v, min, max) { + const n = Number(v); + if (!Number.isFinite(n)) return min; + return Math.min(max, Math.max(min, n)); +} + +function normalizeHexColor(value, fallback) { + try { + return `#${new THREE.Color(value).getHexString()}`; + } catch { + return fallback; + } +} + +function createDefaultSceneSettings() { + return { + sky: { + enabled: false, + topColor: "#7aa9ff", + horizonColor: "#cfe5ff", + bottomColor: "#f4f8ff", + brightness: 1.0, + softness: 1.35, + sunStrength: 0.18, + sunHeight: 0.45, + }, + }; +} + +function normalizeSceneSettings(raw) { + const defaults = createDefaultSceneSettings(); + const src = raw && typeof raw === "object" ? raw : {}; + const srcSky = src.sky && typeof src.sky === "object" ? src.sky : {}; + return { + sky: { + enabled: !!srcSky.enabled, + topColor: normalizeHexColor(srcSky.topColor, defaults.sky.topColor), + horizonColor: normalizeHexColor(srcSky.horizonColor, defaults.sky.horizonColor), + bottomColor: normalizeHexColor(srcSky.bottomColor, defaults.sky.bottomColor), + brightness: clampNum(srcSky.brightness, 0.2, 2.0), + softness: clampNum(srcSky.softness, 0.2, 3.0), + sunStrength: clampNum(srcSky.sunStrength, 0.0, 1.0), + sunHeight: clampNum(srcSky.sunHeight, -0.2, 1.0), + }, + }; +} + +function serializeSceneSettings() { + return normalizeSceneSettings(sceneSettings); +} + +let sceneSettings = createDefaultSceneSettings(); +let tags = []; +let selectedTagId = null; +let draftTag = null; // tag being edited/created +const tagsGroup = new THREE.Group(); +tagsGroup.name = "tagsGroup"; + +// Assets (Edit mode) +let assets = []; // [{id,title,notes,states:[{id,name,glbName,dataBase64,interactions:[{id,label,to}]}],currentStateId,actions:[{id,label,from,to}],transform:{...}, _colliderHandle?}] +let selectedAssetId = null; +const assetsGroup = new THREE.Group(); +assetsGroup.name = "assetsGroup"; +const gltfLoader = new GLTFLoader(); + +// ============================================================================= +// BLOB SHADOW – lightweight planar shadow for GLB assets (no shadow maps needed) +// ============================================================================= +// Procedural radial-gradient texture (created once, shared by all blob shadows) +let _blobShadowTexture = null; +let _blobShadowGeometry = null; + +function getBlobShadowTexture() { + if (_blobShadowTexture) return _blobShadowTexture; + const size = 128; + const canvas = document.createElement("canvas"); + canvas.width = size; + canvas.height = size; + // Use a GRAYSCALE gradient: white = opaque shadow, black = transparent. + // This texture will be used as an alphaMap (only the luminance/R channel matters). + const ctx = canvas.getContext("2d"); + const gradient = ctx.createRadialGradient(size / 2, size / 2, 0, size / 2, size / 2, size / 2); + gradient.addColorStop(0, "#ffffff"); // center: fully opaque + gradient.addColorStop(0.35, "#cccccc"); + gradient.addColorStop(0.65, "#444444"); + gradient.addColorStop(1, "#000000"); // edge: fully transparent + ctx.fillStyle = gradient; + ctx.fillRect(0, 0, size, size); + _blobShadowTexture = new THREE.CanvasTexture(canvas); + _blobShadowTexture.needsUpdate = true; + return _blobShadowTexture; +} + +function getBlobShadowGeometry() { + if (_blobShadowGeometry) return _blobShadowGeometry; + _blobShadowGeometry = new THREE.PlaneGeometry(1, 1); + // Rotate so the plane lies flat on the XZ ground plane (face up) + _blobShadowGeometry.rotateX(-Math.PI / 2); + return _blobShadowGeometry; +} + +// Create a blob shadow mesh sized to an asset's footprint. +// Returns a Mesh that should be added as a child of the asset root. +// `opts` = { opacity, scale, stretch, rotationDeg, offsetX, offsetY, offsetZ } +function createBlobShadow(assetId, footprintX, footprintZ, localGroundY, opts) { + const o = opts || {}; + const userScale = o.scale ?? 1.0; + const userOpacity = o.opacity ?? 0.5; + const stretch = o.stretch ?? 1.0; // >1 elongates X, <1 elongates Z + const rotDeg = o.rotationDeg ?? 0; // rotation around Y in degrees + const offsetX = o.offsetX ?? 0; + const offsetY = o.offsetY ?? 0; + const offsetZ = o.offsetZ ?? 0; + + // Base diameter from asset footprint, then apply user scale + const baseDiameter = Math.max(footprintX, footprintZ) * 1.1; + const d = baseDiameter * userScale; + if (d < 0.04) return null; + + const mat = new THREE.MeshBasicMaterial({ + color: 0x000000, + alphaMap: getBlobShadowTexture(), + transparent: true, + depthWrite: false, + depthTest: true, + opacity: userOpacity, + side: THREE.DoubleSide, + // Use ONLY constant depth bias. Slope-based factor causes the blob to + // appear to slide as the camera angle changes while moving. + polygonOffset: true, + polygonOffsetFactor: 0, + polygonOffsetUnits: -300, + }); + const mesh = new THREE.Mesh(getBlobShadowGeometry(), mat); + // stretch > 1 makes the X axis wider; Z axis is inversely narrower to + // keep the overall area roughly constant. + const sx = d * stretch; + const sz = d / stretch; + mesh.scale.set(sx, 1, sz); + // Raise slightly so it stays on/just above floor. + mesh.position.set(offsetX, localGroundY + 0.08 + offsetY, offsetZ); + // The shared geometry is already rotated to lie on XZ. An additional Y + // rotation spins the ellipse around the vertical axis. + mesh.rotation.y = (rotDeg * Math.PI) / 180; + mesh.renderOrder = 1000; + mesh.castShadow = false; + mesh.receiveShadow = false; + mesh.name = `blobShadow:${assetId}`; + mesh.userData.isBlobShadow = true; + mesh.userData._baseDiameter = baseDiameter; + mesh.userData._baseLocalY = localGroundY + 0.08; + return mesh; +} + +// ============================================================================= +// PRIMITIVES (Level Editor) – lightweight parametric shapes +// ============================================================================= +let primitives = []; // [{id, type, name, dimensions:{...}, transform:{position,rotation,scale}, material:{color,roughness,metalness,textureDataUrl}, physics:bool, _colliderHandle?}] +let selectedPrimitiveId = null; +const _assetBumpVelocities = new Map(); // assetId -> THREE.Vector3 +const _playerPosPrevForBump = new THREE.Vector3(); +let _playerPosPrevForBumpValid = false; +const _agentPosPrevForBump = new Map(); // agentId -> THREE.Vector3 +let _lastBumpSaveAt = 0; +let _lastBumpColliderSyncAt = 0; +const primitivesGroup = new THREE.Group(); +primitivesGroup.name = "primitivesGroup"; + +const PRIMITIVE_DEFAULTS = { + box: { + width: 1, + height: 1, + depth: 1, + edgeRadius: 0, + edgeSegments: 4, + widthSegments: 1, + heightSegments: 1, + depthSegments: 1, + }, + sphere: { + radius: 0.5, + widthSegments: 32, + heightSegments: 16, + phiStartDeg: 0, + phiLengthDeg: 360, + thetaStartDeg: 0, + thetaLengthDeg: 180, + }, + cylinder: { radiusTop: 0.5, radiusBottom: 0.5, height: 1, radialSegments: 32, heightSegments: 1, openEnded: 0 }, + cone: { radius: 0.5, height: 1, radialSegments: 32, heightSegments: 1, openEnded: 0 }, + torus: { radius: 0.5, tube: 0.15, radialSegments: 16, tubularSegments: 48, arcDeg: 360 }, + plane: { width: 2, height: 2, widthSegments: 1, heightSegments: 1 }, +}; + +const PRIMITIVE_DIM_CONFIG = { + width: { min: 0.05, max: 50, step: 0.05 }, + height: { min: 0.05, max: 50, step: 0.05 }, + depth: { min: 0.05, max: 50, step: 0.05 }, + radius: { min: 0.01, max: 20, step: 0.01 }, + radiusTop: { min: 0.01, max: 20, step: 0.01 }, + radiusBottom: { min: 0.01, max: 20, step: 0.01 }, + tube: { min: 0.01, max: 10, step: 0.01 }, + edgeRadius: { min: 0, max: 2.5, step: 0.01 }, + edgeSegments: { min: 1, max: 12, step: 1, integer: true }, + widthSegments: { min: 1, max: 128, step: 1, integer: true }, + heightSegments: { min: 1, max: 128, step: 1, integer: true }, + depthSegments: { min: 1, max: 128, step: 1, integer: true }, + radialSegments: { min: 3, max: 128, step: 1, integer: true }, + tubularSegments: { min: 3, max: 256, step: 1, integer: true }, + phiStartDeg: { min: 0, max: 360, step: 1 }, + phiLengthDeg: { min: 1, max: 360, step: 1 }, + thetaStartDeg: { min: 0, max: 180, step: 1 }, + thetaLengthDeg: { min: 1, max: 180, step: 1 }, + arcDeg: { min: 1, max: 360, step: 1 }, + openEnded: { min: 0, max: 1, step: 1, integer: true }, +}; + +function formatPrimitiveDimValue(key, value) { + if (PRIMITIVE_DIM_CONFIG[key]?.integer) return String(Math.round(value)); + if (key.endsWith("Deg")) return `${Math.round(value)}°`; + if (key === "openEnded") return value >= 0.5 ? "Yes" : "No"; + return Number(value).toFixed(2); +} + +const PRIMITIVE_DIM_LABELS = { + edgeRadius: "Roundness", + edgeSegments: "Round Detail", + widthSegments: "Detail X", + heightSegments: "Detail Y", + depthSegments: "Detail Z", + radialSegments: "Circle Detail", + tubularSegments: "Ring Detail", + phiStartDeg: "Horizontal Cut Start", + phiLengthDeg: "Horizontal Fill", + thetaStartDeg: "Vertical Cut Start", + thetaLengthDeg: "Vertical Fill", + arcDeg: "Ring Opening", + openEnded: "Open Ends", + radiusTop: "Top Radius", + radiusBottom: "Bottom Radius", +}; + +function getPrimitiveDimLabel(key) { + return PRIMITIVE_DIM_LABELS[key] || key.replace(/([A-Z])/g, " $1").replace(/^./, (s) => s.toUpperCase()); +} + +// ============================================================================= +// EDITOR LIGHTS – user-placed lights with full control +// ============================================================================= +let editorLights = []; // [{id, type, name, color, intensity, position:{x,y,z}, target:{x,y,z}, distance, angle, penumbra, castShadow, _lightObj?, _helperObj?}] +let groups = []; // [{id, name, children:[primId,...], pickable?}] +const lightsGroup = new THREE.Group(); +lightsGroup.name = "lightsGroup"; +const _assetRaycaster = new THREE.Raycaster(); +const _agentAssetRaycaster = new THREE.Raycaster(); +const _tmpV1 = new THREE.Vector3(); +const _tmpV2 = new THREE.Vector3(); +const _tmpV3 = new THREE.Vector3(); + +// Agent camera follow mode (first-person POV) +let agentCameraFollow = false; +let _agentFollowInitialized = false; + +// Agent task state — per-agent tasks for parallel execution. +let agentTask = { + active: false, + instruction: "", + startedAt: 0, + finishedAt: 0, + finishedReason: "", + lastSummary: "", +}; +const _agentTasks = new Map(); // agentId -> { active, instruction, startedAt, finishedAt, finishedReason, lastSummary } + +function _getAgentTask(agentId) { + return _agentTasks.get(agentId) || agentTask; +} + +function _setAgentTask(agentId, task) { + _agentTasks.set(agentId, task); + // Keep global agentTask in sync with the most recent active task (for UI compat) + if (task.active) { + agentTask = { ...task }; + } +} +let selectedAgentInspectorId = null; +const agentInspectorStateById = new Map(); // id -> { shot, request, response } +let agentCameraFollowId = null; +let agentUiSelectedLabelEl = null; +let agentUiSpawnBtn = null; +let agentUiFollowBtn = null; +let agentUiStopBtn = null; +let agentUiRemoveBtn = null; +let agentUiTaskInputEl = null; +let agentUiTaskRunBtn = null; +let agentTaskTargetId = null; +let agentBadgeLayerEl = null; +const agentBadgeElsById = new Map(); +const MAX_AGENT_COUNT = 4; + +// ============================================================================= +// WORLD MANIFEST & LOADING +// ============================================================================= + +// Helper to normalize asset schema (backward compat) +// This function ensures all asset properties are properly loaded including states, interactions, and actions +function normalizeAssetSchema(raw) { + // Ensure states exist + if (!raw.states || raw.states.length === 0) { + raw.states = [{ + id: crypto.randomUUID(), + name: "default", + glbName: raw.glbName || "", + dataBase64: raw.dataBase64 || "", + interactions: [], + }]; + raw.currentStateId = raw.states[0].id; + } + + // Ensure each state has interactions array + for (const state of raw.states) { + if (!Array.isArray(state.interactions)) { + state.interactions = []; + } + } + + // Build the normalized asset object + const normalized = { + id: raw.id ?? crypto.randomUUID(), + title: raw.title ?? "", + notes: raw.notes ?? "", + states: raw.states, + currentStateId: raw.currentStateId ?? raw.states[0]?.id, + actions: [], // Will be populated below + transform: raw.transform ?? { position: { x: 0, y: 0, z: 0 }, rotation: { x: 0, y: 0, z: 0 }, scale: { x: 1, y: 1, z: 1 } }, + pickable: raw.pickable ?? false, // Can be picked up and moved + castShadow: raw.castShadow ?? false, + receiveShadow: raw.receiveShadow ?? false, + blobShadow: raw.blobShadow ?? null, // { opacity, scale, stretch, rotationDeg, offsetX, offsetY, offsetZ } + }; + + // Copy actions if they exist in raw data + if (Array.isArray(raw.actions) && raw.actions.length > 0) { + normalized.actions = raw.actions.map(act => ({ + id: act.id, + label: act.label || "toggle", + from: act.from, + to: act.to, + })); + } else { + // Backfill actions from state interactions if no actions array exists + for (const state of normalized.states) { + for (const interaction of state.interactions || []) { + if (interaction.to && interaction.to !== state.id) { + normalized.actions.push({ + id: interaction.id || `act_${state.id}_${interaction.to}`, + label: interaction.label || "toggle", + from: state.id, + to: interaction.to, + }); + } + } + } + } + + // Also backfill interactions from actions if any state is missing them + if (normalized.actions.length > 0) { + const actionsByFrom = new Map(); + for (const act of normalized.actions) { + if (!actionsByFrom.has(act.from)) actionsByFrom.set(act.from, []); + actionsByFrom.get(act.from).push({ id: act.id, label: act.label, to: act.to }); + } + for (const state of normalized.states) { + if (!state.interactions || state.interactions.length === 0) { + state.interactions = actionsByFrom.get(state.id) || []; + } + } + } + + return normalized; +} + +const renderer = new THREE.WebGLRenderer({ + canvas, + // SparkRenderer docs recommend antialias:false for splats (MSAA doesn't help splats and can hurt) + antialias: false, + powerPreference: "high-performance", + // Required for reading pixels from the canvas (agent POV capture) + preserveDrawingBuffer: true, +}); +renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2)); +renderer.setSize(window.innerWidth, window.innerHeight, false); +renderer.outputColorSpace = THREE.SRGBColorSpace; +renderer.toneMapping = THREE.ACESFilmicToneMapping; +renderer.toneMappingExposure = 1.1; + +// Shadows: OFF by default. Enabled dynamically only when a light actually casts shadows. +// BasicShadowMap is fully deterministic (no PCF/stochastic filtering). +renderer.shadowMap.enabled = false; +renderer.shadowMap.type = THREE.BasicShadowMap; +renderer.shadowMap.autoUpdate = false; // we control when shadow maps update + +const clock = new THREE.Clock(); +const scene = new THREE.Scene(); +scene.background = new THREE.Color(0x06070a); + +// Image-based lighting for PBR GLBs. This dramatically improves "too dark" assets. +try { + const pmrem = new THREE.PMREMGenerator(renderer); + scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture; + pmrem.dispose(); +} catch { + // ignore +} + +// Make renderer/scene available to SparkRenderer initialization after dynamic import. +threeRendererRef = renderer; +threeSceneRef = scene; + +const camera = new THREE.PerspectiveCamera( + 65, + window.innerWidth / window.innerHeight, + 0.05, + 2000 +); +camera.position.set(0, 1.7, 4); + +// Lighting for non-splat geometry (assets/avatars). +// Splats are mostly self-lit visually; GLB assets need strong, stable fill to avoid looking black. +const ambientLight = new THREE.AmbientLight(0xffffff, 0.65); +scene.add(ambientLight); + +const hemi = new THREE.HemisphereLight(0xffffff, 0x223344, 0.85); +hemi.position.set(0, 10, 0); +scene.add(hemi); + +const dir = new THREE.DirectionalLight(0xffffff, 1.6); +dir.position.set(8, 14, 6); +dir.castShadow = false; // off by default; user enables via Scene Lighting panel +dir.shadow.mapSize.width = 512; +dir.shadow.mapSize.height = 512; +dir.shadow.camera.near = 0.5; +dir.shadow.camera.far = 40; +dir.shadow.camera.left = -15; +dir.shadow.camera.right = 15; +dir.shadow.camera.top = 15; +dir.shadow.camera.bottom = -15; +dir.shadow.bias = -0.003; +scene.add(dir); + +// Headlamp-style light attached to the camera so assets are visible wherever they are placed. +const headLamp = new THREE.PointLight(0xffffff, 1.4, 26, 1.5); +headLamp.position.set(0, 1.0, 0.6); +camera.add(headLamp); + +// Lightweight procedural sky dome (single draw call). This is intentionally +// simple so it remains cheap for scale/headless workloads. +const skyUniforms = { + uTop: { value: new THREE.Color("#7aa9ff") }, + uHorizon: { value: new THREE.Color("#cfe5ff") }, + uBottom: { value: new THREE.Color("#f4f8ff") }, + uBrightness: { value: 1.0 }, + uSoftness: { value: 1.35 }, + uSunStrength: { value: 0.18 }, + uSunDir: { value: new THREE.Vector3(0, 0.45, -1).normalize() }, +}; +const skyDome = new THREE.Mesh( + new THREE.SphereGeometry(220, 24, 16), + new THREE.ShaderMaterial({ + uniforms: skyUniforms, + side: THREE.BackSide, + depthWrite: false, + vertexShader: ` + varying vec3 vWorldDir; + void main() { + vec4 worldPos = modelMatrix * vec4(position, 1.0); + vWorldDir = normalize(worldPos.xyz - cameraPosition); + gl_Position = projectionMatrix * viewMatrix * worldPos; + } + `, + fragmentShader: ` + varying vec3 vWorldDir; + uniform vec3 uTop; + uniform vec3 uHorizon; + uniform vec3 uBottom; + uniform float uBrightness; + uniform float uSoftness; + uniform float uSunStrength; + uniform vec3 uSunDir; + void main() { + float h = clamp(vWorldDir.y * 0.5 + 0.5, 0.0, 1.0); + float shaped = pow(h, max(0.15, uSoftness)); + vec3 col = mix(uBottom, uHorizon, smoothstep(0.0, 0.55, shaped)); + col = mix(col, uTop, smoothstep(0.45, 1.0, shaped)); + float sun = pow(max(dot(normalize(vWorldDir), normalize(uSunDir)), 0.0), 220.0); + col += vec3(1.0, 0.92, 0.78) * sun * uSunStrength; + gl_FragColor = vec4(col * uBrightness, 1.0); + } + `, + }) +); +skyDome.frustumCulled = false; +skyDome.renderOrder = -1000; +skyDome.visible = false; +scene.add(skyDome); + + +// Registry of built-in scene lights so the editor can expose them +const sceneLights = [ + { id: "_ambient", label: "Ambient", obj: ambientLight, type: "ambient" }, + { id: "_hemi", label: "Hemisphere", obj: hemi, type: "hemisphere" }, + { id: "_dir", label: "Directional", obj: dir, type: "directional" }, + { id: "_headlamp", label: "Head Lamp", obj: headLamp, type: "point" }, + { id: "_sky", label: "Sky", obj: skyDome, type: "sky" }, +]; +scene.add(camera); + +// Avatar: simple capsule that follows the first-person camera. +const avatar = new THREE.Mesh( + new THREE.CapsuleGeometry(PLAYER_RADIUS * 0.8, PLAYER_HALF_HEIGHT * 2.0, 6, 12), + new THREE.MeshStandardMaterial({ color: 0x7cc4ff, roughness: 0.5 }) +); +avatar.castShadow = false; +avatar.receiveShadow = false; +avatar.visible = false; // always hidden; physics capsule handles collision +scene.add(avatar); +scene.add(tagsGroup); +scene.add(assetsGroup); +scene.add(primitivesGroup); +scene.add(lightsGroup); + + +// ----------------------------------------------------------------------------- +// Sim sensor view modes (deterministic + lightweight) +// ----------------------------------------------------------------------------- +const DEFAULT_SCENE_BG = new THREE.Color(0x06070a); +const RGBD_BG = new THREE.Color(0x000000); +function applySceneSkySettings() { + const s = normalizeSceneSettings(sceneSettings).sky; + sceneSettings.sky = s; + skyUniforms.uTop.value.set(s.topColor); + skyUniforms.uHorizon.value.set(s.horizonColor); + skyUniforms.uBottom.value.set(s.bottomColor); + skyUniforms.uBrightness.value = s.brightness; + skyUniforms.uSoftness.value = s.softness; + skyUniforms.uSunStrength.value = s.sunStrength; + skyUniforms.uSunDir.value.set(0, s.sunHeight, -1).normalize(); +} +function applySceneRgbBackground() { + if (sceneSettings.sky.enabled) { + skyDome.visible = true; + scene.background = null; + } else { + skyDome.visible = false; + scene.background = DEFAULT_SCENE_BG; + } +} +applySceneSkySettings(); +// RGB-D visualization range tuned for indoor robotics scenes (meters). +const RGBD_MIN_DEPTH_M = 0.2; +const RGBD_MAX_DEPTH_M = 12.0; +const RGBD_AUTO_PERCENTILE_LOW = 0.05; +const RGBD_AUTO_PERCENTILE_HIGH = 0.95; +const RGBD_AUTO_RANGE_UPDATE_MS = 250; +const RGBD_AUTO_RANGE_SMOOTH = 0.2; +const RGBD_CLEAR_ALPHA = 1.0; +rgbdRangeMinM = RGBD_MIN_DEPTH_M; +rgbdRangeMaxM = RGBD_MAX_DEPTH_M; +const _rgbdSize = new THREE.Vector2( + Math.max(1, Math.floor(window.innerWidth * renderer.getPixelRatio())), + Math.max(1, Math.floor(window.innerHeight * renderer.getPixelRatio())) +); +const rgbdDepthTarget = new THREE.WebGLRenderTarget(_rgbdSize.x, _rgbdSize.y, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: THREE.RGBAFormat, + type: THREE.UnsignedByteType, + depthBuffer: true, + stencilBuffer: false, +}); +rgbdDepthTarget.texture.generateMipmaps = false; +rgbdDepthTarget.depthTexture = new THREE.DepthTexture(_rgbdSize.x, _rgbdSize.y, THREE.UnsignedIntType); +rgbdDepthTarget.depthTexture.minFilter = THREE.NearestFilter; +rgbdDepthTarget.depthTexture.magFilter = THREE.NearestFilter; +rgbdDepthTarget.depthTexture.generateMipmaps = false; +const RGBD_PC_OVERLAY_RT_W = 192; +const RGBD_PC_OVERLAY_RT_H = 108; +const rgbdOverlayDepthTarget = new THREE.WebGLRenderTarget(RGBD_PC_OVERLAY_RT_W, RGBD_PC_OVERLAY_RT_H, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: THREE.RGBAFormat, + type: THREE.UnsignedByteType, + depthBuffer: true, + stencilBuffer: false, +}); +rgbdOverlayDepthTarget.texture.generateMipmaps = false; +rgbdOverlayDepthTarget.depthTexture = new THREE.DepthTexture(RGBD_PC_OVERLAY_RT_W, RGBD_PC_OVERLAY_RT_H, THREE.UnsignedIntType); +rgbdOverlayDepthTarget.depthTexture.minFilter = THREE.NearestFilter; +rgbdOverlayDepthTarget.depthTexture.magFilter = THREE.NearestFilter; +rgbdOverlayDepthTarget.depthTexture.generateMipmaps = false; + +// RGB-D debug material (planar forward-axis depth from view-space z). +// Kept only for debugging and no longer used as default RGB-D output. +const rgbdPlanarDepthDebugMaterial = new THREE.ShaderMaterial({ + uniforms: { + uMinDepth: { value: RGBD_MIN_DEPTH_M }, + uMaxDepth: { value: RGBD_MAX_DEPTH_M }, + }, + vertexShader: ` + varying float vLinearDepth; + void main() { + vec4 mv = modelViewMatrix * vec4(position, 1.0); + vLinearDepth = -mv.z; + gl_Position = projectionMatrix * mv; + } + `, + fragmentShader: ` + varying float vLinearDepth; + uniform float uMinDepth; + uniform float uMaxDepth; + void main() { + // Blend linear + inverse depth for strong near-range sensitivity while + // preserving metric ordering (deterministic, no auto-exposure). + float d = clamp(vLinearDepth, uMinDepth, uMaxDepth); + float lin = (d - uMinDepth) / max(0.0001, (uMaxDepth - uMinDepth)); // 0 near, 1 far + float inv = (1.0 / d - 1.0 / uMaxDepth) / max(0.0001, (1.0 / uMinDepth - 1.0 / uMaxDepth)); // 1 near, 0 far + float t = clamp(0.35 * (1.0 - lin) + 0.65 * inv, 0.0, 1.0); // near -> 1, far -> 0 + + // High-contrast pseudo-color ramp (near cyan/green, far orange/red) + vec3 nearC = vec3(0.05, 0.98, 0.98); + vec3 midC = vec3(0.40, 0.95, 0.10); + vec3 farC = vec3(0.98, 0.15, 0.05); + vec3 c = (t > 0.5) ? mix(midC, nearC, (t - 0.5) * 2.0) : mix(farC, midC, t * 2.0); + gl_FragColor = vec4(c, 1.0); + } + `, +}); +rgbdPlanarDepthDebugMaterial.toneMapped = false; + +// Fullscreen passes: +// 1) reconstruct metric camera-space Z into a float render target +// 2) visualize that metric depth for display +const rgbdPostCamera = new THREE.OrthographicCamera(-1, 1, 1, -1, 0, 1); +const rgbdMetricUsesR32F = renderer.capabilities.isWebGL2 && !!renderer.extensions.get("EXT_color_buffer_float"); +const rgbdMetricTargetType = rgbdMetricUsesR32F ? THREE.FloatType : THREE.HalfFloatType; +const rgbdMetricTarget = new THREE.WebGLRenderTarget(_rgbdSize.x, _rgbdSize.y, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: rgbdMetricUsesR32F ? THREE.RedFormat : THREE.RGBAFormat, + type: rgbdMetricTargetType, + depthBuffer: false, + stencilBuffer: false, +}); +if (rgbdMetricUsesR32F) rgbdMetricTarget.texture.internalFormat = "R32F"; +rgbdMetricTarget.texture.generateMipmaps = false; +const rgbdOverlayMetricTarget = new THREE.WebGLRenderTarget(RGBD_PC_OVERLAY_RT_W, RGBD_PC_OVERLAY_RT_H, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: rgbdMetricUsesR32F ? THREE.RedFormat : THREE.RGBAFormat, + type: rgbdMetricTargetType, + depthBuffer: false, + stencilBuffer: false, +}); +if (rgbdMetricUsesR32F) rgbdOverlayMetricTarget.texture.internalFormat = "R32F"; +rgbdOverlayMetricTarget.texture.generateMipmaps = false; + +const rgbdMetricScene = new THREE.Scene(); +const rgbdMetricMaterial = new THREE.ShaderMaterial({ + uniforms: { + uDepthTex: { value: rgbdDepthTarget.depthTexture }, + uNear: { value: camera.near }, + uFar: { value: camera.far }, + uMinDepth: { value: rgbdRangeMinM }, + uMaxDepth: { value: rgbdRangeMaxM }, + uNoiseEnabled: { value: 0.0 }, + uSpeckleEnabled: { value: 0.0 }, + }, + vertexShader: ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = vec4(position.xy, 0.0, 1.0); + } + `, + fragmentShader: ` + varying vec2 vUv; + uniform sampler2D uDepthTex; + uniform float uNear; + uniform float uFar; + uniform float uMinDepth; + uniform float uMaxDepth; + uniform float uNoiseEnabled; + uniform float uSpeckleEnabled; + + // Perspective depth [0,1] -> view-space z (negative in front of camera). + float perspectiveDepthToViewZ(const in float depth, const in float near, const in float far) { + return (near * far) / ((far - near) * depth - far); + } + + float hash12(vec2 p) { + vec3 p3 = fract(vec3(p.xyx) * 0.1031); + p3 += dot(p3, p3.yzx + 33.33); + return fract((p3.x + p3.y) * p3.z); + } + + void main() { + float depth01 = texture2D(uDepthTex, vUv).x; + // No geometry hit: treat as max range. + if (depth01 >= 0.999999) { + gl_FragColor = vec4(uMaxDepth, uMaxDepth, uMaxDepth, 1.0); + return; + } + + float viewZ = perspectiveDepthToViewZ(depth01, uNear, uFar); + float zMetric = -viewZ; // camera-space Z in meters (robotics back-projection convention) + float d = clamp(zMetric, uMinDepth, uMaxDepth); + + if (uNoiseEnabled > 0.5) { + float span = max(0.0001, uMaxDepth - uMinDepth); + float t = clamp((d - uMinDepth) / span, 0.0, 1.0); + // Quantization: ~1mm near, up to ~8mm far (indoors). + float q = mix(0.001, 0.008, t * t); + d = floor(d / q + 0.5) * q; + + // Dropouts: more likely on edges and farther range. + float edge = clamp(length(vec2(dFdx(depth01), dFdy(depth01))) * 250.0, 0.0, 1.0); + float pDrop = 0.01 + 0.08 * t * t + 0.18 * edge; + float u = hash12(vUv * vec2(4096.0, 4096.0)); + if (u < pDrop) { + gl_FragColor = vec4(uMaxDepth, uMaxDepth, uMaxDepth, 1.0); + return; + } + + // Optional speckle noise (small multiplicative perturbation). + if (uSpeckleEnabled > 0.5) { + float n = hash12(vUv * vec2(8192.0, 8192.0) + vec2(17.3, 9.1)) - 0.5; + float amp = 0.002 + 0.01 * t; // 2mm near -> 12mm far + d = clamp(d + n * amp, uMinDepth, uMaxDepth); + } + } + + gl_FragColor = vec4(d, d, d, 1.0); + } + `, + depthTest: false, + depthWrite: false, +}); +rgbdMetricMaterial.toneMapped = false; +const rgbdMetricQuad = new THREE.Mesh(new THREE.PlaneGeometry(2, 2), rgbdMetricMaterial); +rgbdMetricScene.add(rgbdMetricQuad); + +const rgbdVizScene = new THREE.Scene(); +const rgbdVizMaterial = new THREE.ShaderMaterial({ + uniforms: { + uMetricDepthTex: { value: rgbdMetricTarget.texture }, + uMinDepth: { value: rgbdRangeMinM }, + uMaxDepth: { value: rgbdRangeMaxM }, + uGrayMode: { value: 0.0 }, + }, + vertexShader: ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = vec4(position.xy, 0.0, 1.0); + } + `, + fragmentShader: ` + varying vec2 vUv; + uniform sampler2D uMetricDepthTex; + uniform float uMinDepth; + uniform float uMaxDepth; + uniform float uGrayMode; + void main() { + float d = texture2D(uMetricDepthTex, vUv).r; + d = clamp(d, uMinDepth, uMaxDepth); + float lin = (d - uMinDepth) / max(0.0001, (uMaxDepth - uMinDepth)); // 0 near, 1 far + if (uGrayMode > 0.5) { + float g = 1.0 - lin; + gl_FragColor = vec4(g, g, g, 1.0); + return; + } + float inv = (1.0 / d - 1.0 / uMaxDepth) / max(0.0001, (1.0 / uMinDepth - 1.0 / uMaxDepth)); // 1 near, 0 far + float t = clamp(0.35 * (1.0 - lin) + 0.65 * inv, 0.0, 1.0); // near -> 1, far -> 0 + vec3 nearC = vec3(0.05, 0.98, 0.98); + vec3 midC = vec3(0.40, 0.95, 0.10); + vec3 farC = vec3(0.98, 0.15, 0.05); + vec3 c = (t > 0.5) ? mix(midC, nearC, (t - 0.5) * 2.0) : mix(farC, midC, t * 2.0); + gl_FragColor = vec4(c, 1.0); + } + `, + depthTest: false, + depthWrite: false, +}); +rgbdVizMaterial.toneMapped = false; +const rgbdVizQuad = new THREE.Mesh(new THREE.PlaneGeometry(2, 2), rgbdVizMaterial); +rgbdVizScene.add(rgbdVizQuad); +let _savedOverrideMaterial = null; + +function resizeRgbdTargets() { + const w = Math.max(1, Math.floor(window.innerWidth * renderer.getPixelRatio())); + const h = Math.max(1, Math.floor(window.innerHeight * renderer.getPixelRatio())); + rgbdDepthTarget.setSize(w, h); + rgbdMetricTarget.setSize(w, h); + if (rgbdDepthTarget.depthTexture) { + rgbdDepthTarget.depthTexture.image.width = w; + rgbdDepthTarget.depthTexture.image.height = h; + rgbdDepthTarget.depthTexture.needsUpdate = true; + } +} + +let _rgbdNearFarAsserted = false; +let _rgbdLastAutoRangeMs = 0; + +function updateRgbdRangeLabels() { + if (simRgbdMinValEl) simRgbdMinValEl.textContent = `${rgbdRangeMinM.toFixed(1)}m`; + if (simRgbdMaxValEl) simRgbdMaxValEl.textContent = `${rgbdRangeMaxM.toFixed(1)}m`; +} + +function setRgbdRange(minD, maxD) { + const lo = Math.max(0.05, Math.min(minD, maxD - 0.05)); + const hi = Math.max(lo + 0.05, maxD); + rgbdRangeMinM = lo; + rgbdRangeMaxM = hi; + rgbdMetricMaterial.uniforms.uMinDepth.value = lo; + rgbdMetricMaterial.uniforms.uMaxDepth.value = hi; + rgbdVizMaterial.uniforms.uMinDepth.value = lo; + rgbdVizMaterial.uniforms.uMaxDepth.value = hi; + if (simRgbdMinEl) simRgbdMinEl.value = lo.toFixed(1); + if (simRgbdMaxEl) simRgbdMaxEl.value = hi.toFixed(1); + updateRgbdRangeLabels(); +} + +setRgbdRange(RGBD_MIN_DEPTH_M, RGBD_MAX_DEPTH_M); + +function percentileFromSorted(sorted, p) { + if (!sorted.length) return 0; + const idx = Math.min(sorted.length - 1, Math.max(0, Math.floor(p * (sorted.length - 1)))); + return sorted[idx]; +} + +function updateRgbdAutoRangeFromMetricTarget() { + const now = performance.now(); + if (now - _rgbdLastAutoRangeMs < RGBD_AUTO_RANGE_UPDATE_MS) return; + _rgbdLastAutoRangeMs = now; + const depth = readRgbdMetricDepthFrameMeters(); + if (!depth || depth.length === 0) return; + const samples = []; + const stride = Math.max(1, Math.floor(depth.length / 5000)); + for (let i = 0; i < depth.length; i += stride) { + const d = depth[i]; + if (!Number.isFinite(d)) continue; + if (d <= RGBD_MIN_DEPTH_M || d >= RGBD_MAX_DEPTH_M) continue; + samples.push(d); + } + if (samples.length < 32) return; + samples.sort((a, b) => a - b); + const p05 = percentileFromSorted(samples, RGBD_AUTO_PERCENTILE_LOW); + const p95 = percentileFromSorted(samples, RGBD_AUTO_PERCENTILE_HIGH); + const targetMin = Math.max(RGBD_MIN_DEPTH_M, Math.min(p05, p95 - 0.1)); + const targetMax = Math.min(RGBD_MAX_DEPTH_M, Math.max(p95, targetMin + 0.1)); + const smoothMin = rgbdRangeMinM + (targetMin - rgbdRangeMinM) * RGBD_AUTO_RANGE_SMOOTH; + const smoothMax = rgbdRangeMaxM + (targetMax - rgbdRangeMaxM) * RGBD_AUTO_RANGE_SMOOTH; + setRgbdRange(smoothMin, smoothMax); +} + +function renderRgbdView(enableAutoRange = true) { + renderRgbdMetricPassOffscreen(); + + if (enableAutoRange && rgbdAutoRange) updateRgbdAutoRangeFromMetricTarget(); + rgbdVizMaterial.uniforms.uGrayMode.value = rgbdVizMode === "gray" ? 1.0 : 0.0; + + // Pass 3: visualize metric depth target. + renderer.setRenderTarget(null); + renderer.setClearColor(RGBD_BG, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(rgbdVizScene, rgbdPostCamera); +} + +function renderRgbdMetricPassOffscreen(overrideCamera) { + const cam = overrideCamera || camera; + rgbdMetricMaterial.uniforms.uNear.value = cam.near; + rgbdMetricMaterial.uniforms.uFar.value = cam.far; + rgbdMetricMaterial.uniforms.uNoiseEnabled.value = rgbdNoiseEnabled ? 1.0 : 0.0; + rgbdMetricMaterial.uniforms.uSpeckleEnabled.value = rgbdSpeckleEnabled ? 1.0 : 0.0; + if (!_rgbdNearFarAsserted && !overrideCamera) { + console.assert( + Math.abs(rgbdMetricMaterial.uniforms.uNear.value - camera.near) < 1e-9 && + Math.abs(rgbdMetricMaterial.uniforms.uFar.value - camera.far) < 1e-9, + "[RGB-D] Reconstruction near/far must match active camera near/far." + ); + _rgbdNearFarAsserted = true; + } + + // Ensure depth pass sees scene geometry, not lidar/overlay debug points. + const savedOverride = scene.overrideMaterial; + const savedAssets = assetsGroup.visible; + const savedPrims = primitivesGroup.visible; + const savedLights = lightsGroup.visible; + const savedTags = tagsGroup.visible; + const savedLidarViz = lidarVizGroup.visible; + const savedRgbdPc = rgbdPcOverlayGroup.visible; + + scene.overrideMaterial = null; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + + renderer.setRenderTarget(rgbdDepthTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(scene, cam); + + renderer.setRenderTarget(rgbdMetricTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(rgbdMetricScene, rgbdPostCamera); + + scene.overrideMaterial = savedOverride; + assetsGroup.visible = savedAssets; + primitivesGroup.visible = savedPrims; + lightsGroup.visible = savedLights; + tagsGroup.visible = savedTags; + lidarVizGroup.visible = savedLidarViz; + rgbdPcOverlayGroup.visible = savedRgbdPc; +} + +function halfToFloat(h) { + const s = (h & 0x8000) >> 15; + const e = (h & 0x7c00) >> 10; + const f = h & 0x03ff; + if (e === 0) return (s ? -1 : 1) * Math.pow(2, -14) * (f / 1024); + if (e === 31) return f ? NaN : ((s ? -1 : 1) * Infinity); + return (s ? -1 : 1) * Math.pow(2, e - 15) * (1 + f / 1024); +} + +function readRgbdMetricDepthFrameMeters() { + const w = rgbdMetricTarget.width; + const h = rgbdMetricTarget.height; + if (!w || !h) return null; + + if (rgbdMetricUsesR32F) { + const depth = new Float32Array(w * h); + renderer.readRenderTargetPixels(rgbdMetricTarget, 0, 0, w, h, depth); + return depth; + } + + if (rgbdMetricTarget.texture.type === THREE.FloatType) { + const raw = new Float32Array(w * h * 4); + renderer.readRenderTargetPixels(rgbdMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = raw[i * 4 + 0]; + return depth; + } + + // Half-float fallback (WebGL1 / constrained platforms) + const raw = new Uint16Array(w * h * 4); + renderer.readRenderTargetPixels(rgbdMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = halfToFloat(raw[i * 4 + 0]); + return depth; +} + +function readRgbdOverlayMetricDepthFrameMeters() { + const w = rgbdOverlayMetricTarget.width; + const h = rgbdOverlayMetricTarget.height; + if (!w || !h) return null; + if (rgbdMetricUsesR32F) { + const depth = new Float32Array(w * h); + renderer.readRenderTargetPixels(rgbdOverlayMetricTarget, 0, 0, w, h, depth); + return depth; + } + if (rgbdOverlayMetricTarget.texture.type === THREE.FloatType) { + const raw = new Float32Array(w * h * 4); + renderer.readRenderTargetPixels(rgbdOverlayMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = raw[i * 4 + 0]; + return depth; + } + const raw = new Uint16Array(w * h * 4); + renderer.readRenderTargetPixels(rgbdOverlayMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = halfToFloat(raw[i * 4 + 0]); + return depth; +} + +function updateRgbdPcOverlayCloud(force = false) { + if (!rgbdPcOverlayOnLidar || simSensorViewMode !== "lidar" || lidarOrderedDebugView) { + _rgbdPcGeom.setDrawRange(0, 0); + _rgbdPcGeom.attributes.position.needsUpdate = true; + _rgbdPcGeom.attributes.color.needsUpdate = true; + _rgbdPcOverlayLastCount = 0; + _rgbdPcOverlayLastPose = null; + _rgbdPcOverlayDirty = false; + rgbdPcOverlayGroup.visible = false; + return; + } + if (!force && !_rgbdPcOverlayDirty) return; + const now = performance.now(); + const curPos = camera.getWorldPosition(new THREE.Vector3()); + const curQuat = camera.getWorldQuaternion(new THREE.Quaternion()); + if (_rgbdPcOverlayLastPose) { + const dp = curPos.distanceTo(_rgbdPcOverlayLastPose.pos); + const da = THREE.MathUtils.radToDeg(curQuat.angleTo(_rgbdPcOverlayLastPose.quat)); + if (!force && dp < RGBD_PC_OVERLAY_MIN_TRANSLATION_M && da < RGBD_PC_OVERLAY_MIN_ROT_DEG) return; + } + _rgbdPcOverlayLastUpdateMs = now; + _rgbdPcOverlayLastPose = { pos: curPos.clone(), quat: curQuat.clone() }; + + const savedDepthTex = rgbdMetricMaterial.uniforms.uDepthTex.value; + + // Low-res depth+metric pass for overlay to avoid expensive full-res readback stalls. + const savedOverride = scene.overrideMaterial; + const savedAssets = assetsGroup.visible; + const savedPrims = primitivesGroup.visible; + const savedLights = lightsGroup.visible; + const savedTags = tagsGroup.visible; + const savedLidarViz = lidarVizGroup.visible; + const savedRgbdPc = rgbdPcOverlayGroup.visible; + scene.overrideMaterial = null; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + + renderer.setRenderTarget(rgbdOverlayDepthTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(scene, camera); + rgbdMetricMaterial.uniforms.uDepthTex.value = rgbdOverlayDepthTarget.depthTexture; + renderer.setRenderTarget(rgbdOverlayMetricTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(rgbdMetricScene, rgbdPostCamera); + rgbdMetricMaterial.uniforms.uDepthTex.value = savedDepthTex; + + scene.overrideMaterial = savedOverride; + assetsGroup.visible = savedAssets; + primitivesGroup.visible = savedPrims; + lightsGroup.visible = savedLights; + tagsGroup.visible = savedTags; + lidarVizGroup.visible = savedLidarViz; + rgbdPcOverlayGroup.visible = savedRgbdPc; + + const depth = readRgbdOverlayMetricDepthFrameMeters(); + if (!depth) { + rgbdPcOverlayGroup.visible = false; + return; + } + const w = rgbdOverlayMetricTarget.width; + const h = rgbdOverlayMetricTarget.height; + if (!w || !h) return; + + const tanHalfY = Math.tan(THREE.MathUtils.degToRad(camera.fov * 0.5)); + const fy = 0.5 * h / Math.max(1e-6, tanHalfY); + const fx = fy * camera.aspect; + const cx = (w - 1) * 0.5; + const cy = (h - 1) * 0.5; + const targetCount = Math.min(RGBD_PC_OVERLAY_MAX_POINTS, Math.floor(w * h)); + const stride = Math.max(1, Math.floor(Math.sqrt((w * h) / Math.max(1, targetCount)))); + const pCam = new THREE.Vector3(); + const pWorld = new THREE.Vector3(); + + let n = 0; + for (let py = 0; py < h; py += stride) { + const v = h - 1 - py; // flip Y because render-target readback is bottom-up + for (let px = 0; px < w; px += stride) { + if (n >= RGBD_PC_OVERLAY_MAX_POINTS) break; + const d = depth[py * w + px]; + if (!Number.isFinite(d) || d <= RGBD_MIN_DEPTH_M || d >= RGBD_MAX_DEPTH_M) continue; + const x = ((px - cx) / fx) * d; + const y = -((v - cy) / fy) * d; + const z = -d; // camera forward is -Z in three.js camera coordinates + pCam.set(x, y, z); + pWorld.copy(pCam).applyMatrix4(camera.matrixWorld); + + _rgbdPcPosArray[n * 3 + 0] = pWorld.x; + _rgbdPcPosArray[n * 3 + 1] = pWorld.y; + _rgbdPcPosArray[n * 3 + 2] = pWorld.z; + + _rgbdPcColArray[n * 3 + 0] = 0.10; + _rgbdPcColArray[n * 3 + 1] = 1.00; + _rgbdPcColArray[n * 3 + 2] = 0.25; + n++; + } + if (n >= RGBD_PC_OVERLAY_MAX_POINTS) break; + } + + _rgbdPcGeom.setDrawRange(0, n); + _rgbdPcGeom.attributes.position.needsUpdate = true; + _rgbdPcGeom.attributes.color.needsUpdate = true; + _rgbdPcOverlayLastCount = n; + _rgbdPcOverlayDirty = false; + rgbdPcOverlayGroup.visible = rgbdPcOverlayOnLidar && simSensorViewMode === "lidar" && !lidarOrderedDebugView && n > 0; +} + +// ----------------------------------------------------------------------------- +// RoboVal standardized LiDAR schema + sensor model +// ----------------------------------------------------------------------------- +// We use lidar->world pose convention for pose_T_world_lidar (T_w_l). +// i.e. p_world = T_w_l * p_lidar +// Livox Mid-360 sensor model (non-repetitive Fibonacci scan pattern) +const LIDAR_SCAN_DURATION_S = 0.1; // 10 Hz scan rate +const LIDAR_NUM_POINTS = 10000; // points per scan +const LIDAR_MAX_POINTS = LIDAR_NUM_POINTS; +const LIDAR_MIN_RANGE_M = 0.1; // Mid-360: 0.1m min +const LIDAR_MAX_RANGE_M = 5; +const LIDAR_V_MIN_RAD = THREE.MathUtils.degToRad(-30); // sees ground ~0.6m from robot +const LIDAR_V_MAX_RAD = THREE.MathUtils.degToRad(15); // 15° up avoids ceiling, focuses rays on walls/ground +// Legacy constants kept for browser UI range image (not used by dimos path) +const LIDAR_NUM_RINGS = 1; +const LIDAR_RANGE_IMAGE_W = 1; +let _lidarScanCount = 0; + +// Pre-compute Fibonacci sphere ray directions (uniform sampling on spherical cap) +const _fibLidarDirs = (() => { + const golden = (1 + Math.sqrt(5)) / 2; + const zMin = Math.sin(LIDAR_V_MIN_RAD); // sin(-7°) ≈ -0.122 + const zMax = Math.sin(LIDAR_V_MAX_RAD); // sin(52°) ≈ 0.788 + const dirs = new Float32Array(LIDAR_NUM_POINTS * 3); + for (let i = 0; i < LIDAR_NUM_POINTS; i++) { + const z = zMin + (zMax - zMin) * (i + 0.5) / LIDAR_NUM_POINTS; + const r = Math.sqrt(1 - z * z); + const phi = 2 * Math.PI * i / golden; + dirs[i * 3 + 0] = r * Math.cos(phi); // x (forward in FLU) + dirs[i * 3 + 1] = r * Math.sin(phi); // y (left in FLU) + dirs[i * 3 + 2] = z; // z (up in FLU) + } + return dirs; +})(); +const LIDAR_ACCUM_FRAMES = 50; +const LIDAR_STATS_INTERVAL_MS = 1500; +const LIDAR_ACCUM_MIN_TRANSLATION_M = 0.08; +const LIDAR_ACCUM_MIN_ROT_DEG = 1.5; +const LIDAR_ACCUM_REFRESH_S = 2.0; + +// Lidar frame uses FLU convention: +// x=forward, y=left, z=up (right-handed). Camera local is x=right, y=up, z=back. +const _lidarToCamQuat = (() => { + const m = new THREE.Matrix4().set( + 0, -1, 0, 0, + 0, 0, 1, 0, + -1, 0, 0, 0, + 0, 0, 0, 1 + ); + return new THREE.Quaternion().setFromRotationMatrix(m); +})(); + +// Pose history for deskew (camera used as lidar pose proxy) +const _lidarPoseHistory = []; // [{stampNs, pos:Vector3, quat:Quaternion}] +const LIDAR_POSE_HISTORY_NS = 2_000_000_000; // keep ~2s history +let _lidarLastStatsMs = 0; +let _lidarUseKnownGoodDebugCloud = false; + +function nowNs() { + // Use unix epoch in ns consistently (browser clock based). + return Math.floor(performance.timeOrigin * 1e6 + performance.now() * 1e6); +} + +function pushLidarPoseSample(stampNs = nowNs()) { + let pos, quat; + const dimosAgent = dimosMode && window.__dimosAgent; + if (dimosAgent) { + // In dimos mode, sample from the agent's body position + orientation. + // getPosition() returns capsule center (~0.37m above ground), so subtract + // capsule half-extent to get ground level, then add mount height. + const [ax, ay, az] = dimosAgent.getPosition?.() || [0, 0, 0]; + const groundY = ay - (PLAYER_HALF_HEIGHT + PLAYER_RADIUS); + const lidarY = groundY + LIDAR_MOUNT_HEIGHT; + pos = new THREE.Vector3(ax, lidarY, az); + const yaw = window.__dimosYaw ?? dimosAgent.group?.rotation?.y ?? 0; + const agentQuat = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), yaw); + quat = agentQuat.multiply(_lidarToCamQuat); + } else { + pos = camera.getWorldPosition(new THREE.Vector3()); + const camQuat = camera.getWorldQuaternion(new THREE.Quaternion()); + quat = camQuat.clone().multiply(_lidarToCamQuat); + } + _lidarPoseHistory.push({ stampNs, pos, quat }); + const minNs = stampNs - LIDAR_POSE_HISTORY_NS; + while (_lidarPoseHistory.length > 2 && _lidarPoseHistory[0].stampNs < minNs) { + _lidarPoseHistory.shift(); + } +} + +function getLidarPoseAtNs(stampNs) { + if (_lidarPoseHistory.length === 0) { + const camQuat = camera.getWorldQuaternion(new THREE.Quaternion()); + return { + pos: camera.getWorldPosition(new THREE.Vector3()), + quat: camQuat.multiply(_lidarToCamQuat), + }; + } + if (_lidarPoseHistory.length === 1) { + return { + pos: _lidarPoseHistory[0].pos.clone(), + quat: _lidarPoseHistory[0].quat.clone(), + }; + } + // Find bounding samples + let i1 = 0; + while (i1 < _lidarPoseHistory.length && _lidarPoseHistory[i1].stampNs < stampNs) i1++; + if (i1 <= 0) { + return { + pos: _lidarPoseHistory[0].pos.clone(), + quat: _lidarPoseHistory[0].quat.clone(), + }; + } + if (i1 >= _lidarPoseHistory.length) { + const last = _lidarPoseHistory[_lidarPoseHistory.length - 1]; + return { pos: last.pos.clone(), quat: last.quat.clone() }; + } + const a = _lidarPoseHistory[i1 - 1]; + const b = _lidarPoseHistory[i1]; + const alpha = (stampNs - a.stampNs) / Math.max(1, b.stampNs - a.stampNs); + const pos = a.pos.clone().lerp(b.pos, alpha); + const quat = a.quat.clone().slerp(b.quat, alpha); + return { pos, quat }; +} + +function composeTwlFlat64(pos, quat) { + const m = new THREE.Matrix4().compose(pos, quat, new THREE.Vector3(1, 1, 1)); + const e = m.elements; + // Return row-major 4x4 flattened float64 (explicitly for stable downstream use) + return new Float64Array([ + e[0], e[4], e[8], e[12], + e[1], e[5], e[9], e[13], + e[2], e[6], e[10], e[14], + e[3], e[7], e[11], e[15], + ]); +} + +function twlInverseMatrix(pos, quat) { + const twl = new THREE.Matrix4().compose(pos, quat, new THREE.Vector3(1, 1, 1)); + return twl.clone().invert(); +} + +function lidarVerticalAngleForRing(ring) { + if (LIDAR_NUM_RINGS === 1) return 0; + const t = ring / (LIDAR_NUM_RINGS - 1); + return LIDAR_V_MIN_RAD + (LIDAR_V_MAX_RAD - LIDAR_V_MIN_RAD) * t; +} + +function makeRoboValLidarFrame({ + frameId, + stampNs, + points, + intensity, + ring, + t, + hasRing, + hasPerPointTime, + scanDurationS, + poseTWorldLidar, +}) { + // RoboValLidarFrame schema (used across sim/export/eval) + return { + frame_id: frameId, + stamp_ns: stampNs, + points, // Float32Array length N*3 (xyz meters, lidar frame) + intensity, // Float32Array length N + ring, // Uint16Array length N + t, // Float32Array length N (seconds from start of scan) + has_ring: hasRing, + has_per_point_time: hasPerPointTime, + scan_duration_s: scanDurationS, + pose_T_world_lidar: poseTWorldLidar, // Float64Array length 16, row-major + }; +} + +// ROS2 PointField datatype constants: +// INT8=1, UINT8=2, INT16=3, UINT16=4, INT32=5, UINT32=6, FLOAT32=7, FLOAT64=8 +function to_pointcloud2(frame) { + const n = Math.floor((frame.points?.length || 0) / 3); + const pointStep = 22; // x,y,z,float32(12) + intensity,float32(4) + ring,uint16(2) + t,float32(4) + const data = new Uint8Array(n * pointStep); + const dv = new DataView(data.buffer); + for (let i = 0; i < n; i++) { + const o = i * pointStep; + dv.setFloat32(o + 0, frame.points[i * 3 + 0], true); + dv.setFloat32(o + 4, frame.points[i * 3 + 1], true); + dv.setFloat32(o + 8, frame.points[i * 3 + 2], true); + dv.setFloat32(o + 12, frame.intensity[i] ?? 0, true); + dv.setUint16(o + 16, frame.ring[i] ?? 0, true); + dv.setFloat32(o + 18, frame.t[i] ?? 0, true); + } + return { + header: { + frame_id: frame.frame_id, + stamp: { + sec: Math.floor(frame.stamp_ns / 1e9), + nanosec: Math.floor(frame.stamp_ns % 1e9), + }, + }, + height: 1, + width: n, + fields: [ + { name: "x", offset: 0, datatype: 7, count: 1 }, + { name: "y", offset: 4, datatype: 7, count: 1 }, + { name: "z", offset: 8, datatype: 7, count: 1 }, + { name: "intensity", offset: 12, datatype: 7, count: 1 }, + { name: "ring", offset: 16, datatype: 4, count: 1 }, + { name: "t", offset: 18, datatype: 7, count: 1 }, + ], + is_bigendian: false, + point_step: pointStep, + row_step: pointStep * n, + data, + is_dense: true, + }; +} + +function toNpyBytes(typedArray, shape, descr) { + // NPY v1.0 + const magic = new Uint8Array([0x93, 0x4e, 0x55, 0x4d, 0x50, 0x59, 0x01, 0x00]); + const shapeStr = `(${shape.join(", ")}${shape.length === 1 ? "," : ""})`; + let header = `{'descr': '${descr}', 'fortran_order': False, 'shape': ${shapeStr}, }`; + // Pad so (magic+2-byte-len+header+\n) % 16 == 0 + const preamble = 10; + const base = preamble + header.length + 1; + const pad = (16 - (base % 16)) % 16; + header = header + " ".repeat(pad) + "\n"; + const headerBytes = new TextEncoder().encode(header); + const out = new Uint8Array(magic.length + 2 + headerBytes.length + typedArray.byteLength); + out.set(magic, 0); + const dv = new DataView(out.buffer); + dv.setUint16(magic.length, headerBytes.length, true); + out.set(headerBytes, magic.length + 2); + out.set(new Uint8Array(typedArray.buffer, typedArray.byteOffset, typedArray.byteLength), magic.length + 2 + headerBytes.length); + return out; +} + +function makeZipStore(entries) { + // Uncompressed ZIP (store) writer for deterministic byte output ordering. + const enc = new TextEncoder(); + const localParts = []; + const centralParts = []; + let offset = 0; + const files = []; + const crcTable = (() => { + const t = new Uint32Array(256); + for (let i = 0; i < 256; i++) { + let c = i; + for (let k = 0; k < 8; k++) c = (c & 1) ? (0xedb88320 ^ (c >>> 1)) : (c >>> 1); + t[i] = c >>> 0; + } + return t; + })(); + const crc32 = (u8) => { + let c = 0xffffffff; + for (let i = 0; i < u8.length; i++) c = crcTable[(c ^ u8[i]) & 0xff] ^ (c >>> 8); + return (c ^ 0xffffffff) >>> 0; + }; + + for (const e of entries) { + const nameBytes = enc.encode(e.name); + const data = e.data; + const crc = crc32(data); + const lfh = new Uint8Array(30 + nameBytes.length); + const dv = new DataView(lfh.buffer); + dv.setUint32(0, 0x04034b50, true); + dv.setUint16(4, 20, true); + dv.setUint16(6, 0, true); + dv.setUint16(8, 0, true); // store + dv.setUint16(10, 0, true); + dv.setUint16(12, 0, true); + dv.setUint32(14, crc, true); + dv.setUint32(18, data.length, true); + dv.setUint32(22, data.length, true); + dv.setUint16(26, nameBytes.length, true); + dv.setUint16(28, 0, true); + lfh.set(nameBytes, 30); + localParts.push(lfh, data); + files.push({ nameBytes, crc, size: data.length, offset }); + offset += lfh.length + data.length; + } + + let centralSize = 0; + for (const f of files) { + const cfh = new Uint8Array(46 + f.nameBytes.length); + const dv = new DataView(cfh.buffer); + dv.setUint32(0, 0x02014b50, true); + dv.setUint16(4, 20, true); + dv.setUint16(6, 20, true); + dv.setUint16(8, 0, true); + dv.setUint16(10, 0, true); + dv.setUint16(12, 0, true); + dv.setUint16(14, 0, true); + dv.setUint32(16, f.crc, true); + dv.setUint32(20, f.size, true); + dv.setUint32(24, f.size, true); + dv.setUint16(28, f.nameBytes.length, true); + dv.setUint16(30, 0, true); + dv.setUint16(32, 0, true); + dv.setUint16(34, 0, true); + dv.setUint16(36, 0, true); + dv.setUint32(38, 0, true); + dv.setUint32(42, f.offset, true); + cfh.set(f.nameBytes, 46); + centralParts.push(cfh); + centralSize += cfh.length; + } + + const eocd = new Uint8Array(22); + const dvE = new DataView(eocd.buffer); + dvE.setUint32(0, 0x06054b50, true); + dvE.setUint16(4, 0, true); + dvE.setUint16(6, 0, true); + dvE.setUint16(8, files.length, true); + dvE.setUint16(10, files.length, true); + dvE.setUint32(12, centralSize, true); + dvE.setUint32(16, offset, true); + dvE.setUint16(20, 0, true); + + return new Blob([...localParts, ...centralParts, eocd], { type: "application/zip" }); +} + +function frameToNpzBlob(frame, rangeImage = null) { + const n = Math.floor((frame.points?.length || 0) / 3); + const xyz = toNpyBytes(frame.points, [n, 3], " URL.revokeObjectURL(a1.href), 500); + + const a2 = document.createElement("a"); + a2.href = URL.createObjectURL(deskBlob); + a2.download = `${base}_lidar_deskewed.npz`; + document.body.appendChild(a2); + a2.click(); + a2.remove(); + setTimeout(() => URL.revokeObjectURL(a2.href), 500); + + if (rangeImage) { + const rBlob = makeZipStore([ + { name: "range.npy", data: toNpyBytes(rangeImage.range, [rangeImage.H, rangeImage.W], " URL.revokeObjectURL(a3.href), 500); + } +} + +const lidarVizGroup = new THREE.Group(); +lidarVizGroup.name = "lidarVizGroup"; +lidarVizGroup.visible = false; +const LIDAR_VIZ_MAX_POINTS = LIDAR_MAX_POINTS * LIDAR_ACCUM_FRAMES; +const _lidarPosArray = new Float32Array(LIDAR_VIZ_MAX_POINTS * 3); +const _lidarColArray = new Float32Array(LIDAR_VIZ_MAX_POINTS * 3); +const _lidarAccumFrames = []; // [{pos: Float32Array, col: Float32Array}] +let _lidarLastAccumPose = null; // {pos:Vector3, quat:Quaternion, stampNs:number} +const _lidarGeom = new THREE.BufferGeometry(); +_lidarGeom.setAttribute("position", new THREE.BufferAttribute(_lidarPosArray, 3)); +_lidarGeom.setAttribute("color", new THREE.BufferAttribute(_lidarColArray, 3)); +_lidarGeom.setDrawRange(0, 0); +const _lidarMat = new THREE.PointsMaterial({ + color: 0xffffff, + vertexColors: true, + size: 0.03, + sizeAttenuation: true, + depthTest: true, + transparent: false, +}); +const _lidarPoints = new THREE.Points(_lidarGeom, _lidarMat); +_lidarPoints.frustumCulled = false; // point cloud covers entire scene; never cull +console.assert(_lidarPoints.isPoints === true, "[LiDAR] Visualization must use THREE.Points"); +lidarVizGroup.add(_lidarPoints); +scene.add(lidarVizGroup); +let _lidarLastNonZeroDrawCount = 0; +const rgbdPcOverlayGroup = new THREE.Group(); +rgbdPcOverlayGroup.name = "rgbdPcOverlayGroup"; +rgbdPcOverlayGroup.visible = false; +const RGBD_PC_OVERLAY_MAX_POINTS = 12000; +const RGBD_PC_OVERLAY_MIN_TRANSLATION_M = 0.15; +const RGBD_PC_OVERLAY_MIN_ROT_DEG = 4.0; +const _rgbdPcPosArray = new Float32Array(RGBD_PC_OVERLAY_MAX_POINTS * 3); +const _rgbdPcColArray = new Float32Array(RGBD_PC_OVERLAY_MAX_POINTS * 3); +const _rgbdPcGeom = new THREE.BufferGeometry(); +_rgbdPcGeom.setAttribute("position", new THREE.BufferAttribute(_rgbdPcPosArray, 3)); +_rgbdPcGeom.setAttribute("color", new THREE.BufferAttribute(_rgbdPcColArray, 3)); +_rgbdPcGeom.setDrawRange(0, 0); +const _rgbdPcMat = new THREE.PointsMaterial({ + color: 0x00ff4f, + vertexColors: true, + size: 3.0, + sizeAttenuation: false, + depthTest: false, + depthWrite: false, + blending: THREE.AdditiveBlending, + transparent: true, + opacity: 1.0, +}); +const _rgbdPcPoints = new THREE.Points(_rgbdPcGeom, _rgbdPcMat); +_rgbdPcPoints.frustumCulled = false; // overlay covers entire scene; never cull +console.assert(_rgbdPcPoints.isPoints === true, "[RGB-D overlay] Visualization must use THREE.Points"); +_rgbdPcPoints.renderOrder = 2000; +rgbdPcOverlayGroup.add(_rgbdPcPoints); +scene.add(rgbdPcOverlayGroup); +let _rgbdPcOverlayLastUpdateMs = 0; +let _rgbdPcOverlayLastPose = null; +let _rgbdPcOverlayLastCount = 0; +let _rgbdPcOverlayDirty = false; + +let _lidarScanState = null; // incremental scan state (processed across frames) + +function updateSimSensorButtons() { + if (simViewCompareBtn) simViewCompareBtn.classList.toggle("active", simCompareView); + if (simViewRgbdBtn) simViewRgbdBtn.classList.toggle("active", simSensorViewMode === "rgbd" && !simCompareView); + if (simRgbdGrayBtn) simRgbdGrayBtn.classList.toggle("active", rgbdVizMode === "gray"); + if (simRgbdColormapBtn) simRgbdColormapBtn.classList.toggle("active", rgbdVizMode === "colormap"); + if (simRgbdAutoRangeBtn) simRgbdAutoRangeBtn.classList.toggle("active", rgbdAutoRange); + if (simRgbdNoiseBtn) simRgbdNoiseBtn.classList.toggle("active", rgbdNoiseEnabled); + if (simRgbdSpeckleBtn) simRgbdSpeckleBtn.classList.toggle("active", rgbdSpeckleEnabled); + if (simRgbdPcOverlayBtn) simRgbdPcOverlayBtn.classList.toggle("active", rgbdPcOverlayOnLidar); + if (simRgbdMinEl) simRgbdMinEl.disabled = rgbdAutoRange; + if (simRgbdMaxEl) simRgbdMaxEl.disabled = rgbdAutoRange; + if (simViewLidarBtn) simViewLidarBtn.classList.toggle("active", simSensorViewMode === "lidar" && !lidarOrderedDebugView && !simCompareView); + if (simLidarColorRangeBtn) simLidarColorRangeBtn.classList.toggle("active", lidarColorByRange); + if (simLidarOrderedDebugBtn) simLidarOrderedDebugBtn.classList.toggle("active", lidarOrderedDebugView); + if (simLidarNoiseBtn) simLidarNoiseBtn.classList.toggle("active", lidarNoiseEnabled); + if (simLidarMultiReturnBtn) { + simLidarMultiReturnBtn.classList.toggle("active", lidarMultiReturnMode === "last"); + simLidarMultiReturnBtn.textContent = lidarMultiReturnMode === "last" ? "LiDAR: Last Return" : "LiDAR: Strongest"; + } + updateRgbdRangeLabels(); +} + +function applySimPanelCollapsedState() { + if (!overlayEl || !agentPanelEl) return; + const shouldCollapse = simPanelCollapsed; + overlayEl.classList.toggle("sim-panel-collapsed", shouldCollapse); + agentPanelEl.classList.toggle("hidden", shouldCollapse); + simPanelOpenBtn?.classList.toggle("hidden", !shouldCollapse); +} + +function lidarRangeColor01(t) { + // Deterministic near->far gradient: cyan -> green -> yellow -> red + const x = Math.min(1, Math.max(0, t)); + if (x < 0.33) { + const u = x / 0.33; + return [0.05 + 0.35 * u, 0.98, 0.98 - 0.88 * u]; + } + if (x < 0.66) { + const u = (x - 0.33) / 0.33; + return [0.40 + 0.58 * u, 0.95 - 0.15 * u, 0.10 * (1.0 - u)]; + } + const u = (x - 0.66) / 0.34; + return [0.98, 0.80 - 0.65 * u, 0.02 + 0.03 * (1.0 - u)]; +} + +function lidarHash01(seed) { + let x = seed | 0; + x ^= x >>> 16; + x = Math.imul(x, 0x7feb352d); + x ^= x >>> 15; + x = Math.imul(x, 0x846ca68b); + x ^= x >>> 16; + return (x >>> 0) / 4294967296; +} + +function lidarGaussianNoise(seedBase) { + // Deterministic approx N(0,1) from 6 uniforms (CLT). + let s = 0; + for (let i = 0; i < 6; i++) { + s += lidarHash01(seedBase + i * 2654435761); + } + return s - 3.0; +} + +function applyLidarRealityModel(toi, incidence, scanSeed, vi, hi) { + let outRange = toi; + let dropped = false; + + if (lidarNoiseEnabled) { + // Indoor-friendly deterministic noise profile (meters). + const sigma = 0.004 + 0.0015 * Math.max(0, toi); // ~4mm near, grows with range + const n = lidarGaussianNoise(scanSeed ^ (vi * 73856093) ^ (hi * 19349663)); + outRange = Math.max(LIDAR_MIN_RANGE_M, Math.min(LIDAR_MAX_RANGE_M, outRange + sigma * n)); + + const tr = Math.min(1, Math.max(0, toi / LIDAR_MAX_RANGE_M)); + const dropoutP = 0.005 + 0.04 * tr * tr; // deterministic, stronger at longer range + const u = lidarHash01(scanSeed ^ (vi * 83492791) ^ (hi * 2654435761)); + if (u < dropoutP) dropped = true; + } + + // Multi-return knob for future lidar profiles. + // With a single physics hit, "last" is approximated as a slight farther-biased return. + if (!dropped && lidarMultiReturnMode === "last") { + const weakSurface = 1.0 - Math.max(0, Math.min(1, incidence)); + const tail = 0.015 * weakSurface; // up to 1.5 cm + outRange = Math.min(LIDAR_MAX_RANGE_M, outRange + tail); + } + + return { range: outRange, dropped }; +} + +function buildKnownGoodDebugCloud() { + // Deterministic 1m cube grid centered 2m in front of camera. + const center = new THREE.Vector3(0, 0, -2).applyMatrix4(camera.matrixWorld); + const step = 0.1; // 11^3 ~= 1331 points + const points = []; + const colors = []; + for (let x = -0.5; x <= 0.5001; x += step) { + for (let y = -0.5; y <= 0.5001; y += step) { + for (let z = -0.5; z <= 0.5001; z += step) { + points.push(center.x + x, center.y + y, center.z + z); + colors.push(0.15 + (x + 0.5) * 0.7, 0.25 + (y + 0.5) * 0.6, 0.95 - (z + 0.5) * 0.5); + } + } + } + return { + pos: new Float32Array(points), + col: new Float32Array(colors), + }; +} + +function logLidarFrameStats(points, n, ring) { + const now = performance.now(); + if (now - _lidarLastStatsMs < LIDAR_STATS_INTERVAL_MS) return; + _lidarLastStatsMs = now; + if (!n) { + console.info("[LiDAR stats]", { n_points: 0, nan_inf_pct: 0 }); + return; + } + let minX = Infinity, minY = Infinity, minZ = Infinity; + let maxX = -Infinity, maxY = -Infinity, maxZ = -Infinity; + let ringMin = Infinity; + let ringMax = -Infinity; + let bad = 0; + const yQuant = new Set(); + for (let i = 0; i < n; i++) { + const x = points[i * 3 + 0]; + const y = points[i * 3 + 1]; + const z = points[i * 3 + 2]; + if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) { + bad++; + continue; + } + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + if (z < minZ) minZ = z; + if (z > maxZ) maxZ = z; + yQuant.add(Math.round(y * 1000)); + const rr = ring[i]; + if (rr < ringMin) ringMin = rr; + if (rr > ringMax) ringMax = rr; + } + console.info("[LiDAR stats]", { + n_points: n, + min: { x: minX, y: minY, z: minZ }, + max: { x: maxX, y: maxY, z: maxZ }, + nan_inf_pct: (100 * bad) / n, + unique_y_mm: yQuant.size, + rings_configured: LIDAR_NUM_RINGS, + ring_min: Number.isFinite(ringMin) ? ringMin : 0, + ring_max: Number.isFinite(ringMax) ? ringMax : 0, + }); +} + +function shouldAppendAccumFrame(refPose, stampNs) { + if (!_lidarLastAccumPose) return true; + const dtS = (stampNs - _lidarLastAccumPose.stampNs) / 1e9; + if (dtS >= LIDAR_ACCUM_REFRESH_S) return true; + const dp = refPose.pos.distanceTo(_lidarLastAccumPose.pos); + if (dp >= LIDAR_ACCUM_MIN_TRANSLATION_M) return true; + const ang = THREE.MathUtils.radToDeg(refPose.quat.angleTo(_lidarLastAccumPose.quat)); + if (ang >= LIDAR_ACCUM_MIN_ROT_DEG) return true; + return false; +} + +function resetLidarScanState() { + _lidarScanState = null; +} + +function updateLidarPointCloud() { + if (!rapierWorld || !RAPIER || (simSensorViewMode !== "lidar" && !dimosMode)) return; + + if (_lidarUseKnownGoodDebugCloud) { + resetLidarScanState(); + const dbg = buildKnownGoodDebugCloud(); + const nDbg = Math.min(LIDAR_VIZ_MAX_POINTS, Math.floor(dbg.pos.length / 3)); + _lidarPosArray.set(dbg.pos.subarray(0, nDbg * 3), 0); + _lidarColArray.set(dbg.col.subarray(0, nDbg * 3), 0); + _lidarGeom.setDrawRange(0, nDbg); + _lidarGeom.attributes.position.needsUpdate = true; + _lidarGeom.attributes.color.needsUpdate = true; + lidarVizGroup.position.set(0, 0, 0); + lidarVizGroup.quaternion.identity(); + lidarVizGroup.scale.set(1, 1, 1); + return; + } + + // Build set of collider handles to exclude from lidar raycasts. + // Excludes player collider and ALL AI agent colliders (lidar origin is inside them). + // In dimos mode, also explicitly exclude the active dimos agent body/colliders. + const _lidarExcludeHandles = new Set(); + const _lidarHostAgent = dimosMode ? window.__dimosAgent : null; + const _lidarExcludeRigidBodyHandle = _lidarHostAgent?.body?.handle; + if (playerCollider) _lidarExcludeHandles.add(playerCollider.handle); + if (_lidarHostAgent?.collider?.handle != null) _lidarExcludeHandles.add(_lidarHostAgent.collider.handle); + if (_lidarHostAgent?.spineCollider?.handle != null) _lidarExcludeHandles.add(_lidarHostAgent.spineCollider.handle); + for (const a of aiAgents) { + if (a?.collider) _lidarExcludeHandles.add(a.collider.handle); + if (a?.spineCollider) _lidarExcludeHandles.add(a.spineCollider.handle); + } + + // Livox Mid-360 style: Fibonacci sphere sampling, incremental over ~0.1s wall-clock. + const N = LIDAR_NUM_POINTS; + const scanDurationS = LIDAR_SCAN_DURATION_S; + const scanDurationNs = Math.floor(scanDurationS * 1e9); + if (!_lidarScanState) { + const scanStartNs = nowNs(); + _lidarScanCount++; + const jitterAngle = _lidarScanCount * 2.399963; // golden angle rotation per scan + const rangeImg = new Float32Array(LIDAR_NUM_RINGS * LIDAR_RANGE_IMAGE_W); + const intenImg = new Float32Array(LIDAR_NUM_RINGS * LIDAR_RANGE_IMAGE_W); + const ringIdxImg = new Uint16Array(LIDAR_NUM_RINGS * LIDAR_RANGE_IMAGE_W); + _lidarScanState = { + scanStartNs, + scanDurationS, + scanDurationNs, + scanSeed: (scanStartNs / 1e6) | 0, + cosJitter: Math.cos(jitterAngle), + sinJitter: Math.sin(jitterAngle), + nextIdx: 0, + n: 0, + rawPts: new Float32Array(LIDAR_MAX_POINTS * 3), + deskPts: new Float32Array(LIDAR_MAX_POINTS * 3), + intensity: new Float32Array(LIDAR_MAX_POINTS), + ring: new Uint16Array(LIDAR_MAX_POINTS), + tArr: new Float32Array(LIDAR_MAX_POINTS), + worldPts: new Float32Array(LIDAR_MAX_POINTS * 3), + colArray: new Float32Array(LIDAR_MAX_POINTS * 3), + rangeImg, + intenImg, + ringIdxImg, + }; + } + const st = _lidarScanState; + const dirLocal = new THREE.Vector3(); + const dirWorld = new THREE.Vector3(); + const pWorld = new THREE.Vector3(); + const pRawLocal = new THREE.Vector3(); + const elapsedNs = Math.max(0, nowNs() - st.scanStartNs); + const progress = Math.min(1, elapsedNs / Math.max(1, st.scanDurationNs)); + let targetIdx = Math.floor(progress * N); + targetIdx = Math.max(targetIdx, Math.min(N, st.nextIdx + 1)); + if (elapsedNs >= st.scanDurationNs) targetIdx = N; + + const cosJ = st.cosJitter, sinJ = st.sinJitter; + + for (let i = st.nextIdx; i < targetIdx; i++) { + { + if (st.n >= LIDAR_MAX_POINTS) break; + // Fibonacci direction with per-scan golden-angle rotation around Z (non-repetitive) + const fx = _fibLidarDirs[i * 3 + 0], fy = _fibLidarDirs[i * 3 + 1], fz = _fibLidarDirs[i * 3 + 2]; + const tSec = (i / Math.max(1, N - 1)) * scanDurationS; + const stampNs = st.scanStartNs + Math.floor(tSec * 1e9); + const pose = getLidarPoseAtNs(stampNs); + const w2lNow = twlInverseMatrix(pose.pos, pose.quat); + const origin = pose.pos; + + // Fibonacci direction rotated by per-scan golden angle (FLU frame) + dirLocal.set(fx * cosJ - fy * sinJ, fx * sinJ + fy * cosJ, fz); + dirWorld.copy(dirLocal).applyQuaternion(pose.quat).normalize(); + const ray = new RAPIER.Ray( + { x: origin.x, y: origin.y, z: origin.z }, + { x: dirWorld.x, y: dirWorld.y, z: dirWorld.z } + ); + let hit = null; + let singleExcludeHandle = undefined; + // Defensive retry: if a self-collider slips through, recast while excluding it. + // Keeps scans alive even if exclusion bookkeeping is briefly stale. + for (let castAttempt = 0; castAttempt < 4; castAttempt++) { + hit = rapierWorld.queryPipeline.castRayAndGetNormal( + rapierWorld.bodies, + rapierWorld.colliders, + ray, + LIDAR_MAX_RANGE_M, + false, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + singleExcludeHandle, + _lidarExcludeRigidBodyHandle, + (h) => !_lidarExcludeHandles.has(h) + ); + const hitHandle = hit?.colliderHandle; + if (!hit || hitHandle == null || !_lidarExcludeHandles.has(hitHandle)) break; + singleExcludeHandle = hitHandle; + } + let toi = hit ? (hit.toi ?? hit.timeOfImpact ?? 0) : Infinity; + const hitNormal = hit?.normal || null; + + // Ground-truth-style lidar: no-return beams are omitted. + if (!Number.isFinite(toi) || toi > LIDAR_MAX_RANGE_M || toi < LIDAR_MIN_RANGE_M) continue; + + const nx = hitNormal?.x ?? 0; + const ny = hitNormal?.y ?? 0; + const nz = hitNormal?.z ?? 1; + const incidence = hitNormal ? Math.max(0, -(dirWorld.x * nx + dirWorld.y * ny + dirWorld.z * nz)) : 0.7; + const reality = applyLidarRealityModel(toi, incidence, st.scanSeed, i & 0xff, i >> 8); + if (reality.dropped) continue; + toi = reality.range; + + pWorld.set( + origin.x + dirWorld.x * toi, + origin.y + dirWorld.y * toi, + origin.z + dirWorld.z * toi + ); + pRawLocal.copy(pWorld).applyMatrix4(w2lNow); + + st.rawPts[st.n * 3 + 0] = pRawLocal.x; + st.rawPts[st.n * 3 + 1] = pRawLocal.y; + st.rawPts[st.n * 3 + 2] = pRawLocal.z; + st.worldPts[st.n * 3 + 0] = pWorld.x; + st.worldPts[st.n * 3 + 1] = pWorld.y; + st.worldPts[st.n * 3 + 2] = pWorld.z; + + st.ring[st.n] = 0; + st.tArr[st.n] = tSec; + + const atten = 1.0 / (1.0 + 0.02 * toi * toi); + const I = Math.max(0.06, Math.min(1.0, incidence * atten)); + st.intensity[st.n] = I; + const tr = Math.min(1, Math.max(0, toi / LIDAR_MAX_RANGE_M)); + const depthShade = 1.0 - 0.35 * tr; // cheap EDL-like darkening by depth/range + + if (lidarColorByRange) { + const [r, g, b] = lidarRangeColor01(tr); + st.colArray[st.n * 3 + 0] = r * depthShade; + st.colArray[st.n * 3 + 1] = g * depthShade; + st.colArray[st.n * 3 + 2] = b * depthShade; + } else { + // Intensity-like grayscale (closer to raw LiDAR semantics) + const g = I * depthShade; + st.colArray[st.n * 3 + 0] = g; + st.colArray[st.n * 3 + 1] = g; + st.colArray[st.n * 3 + 2] = g; + } + st.n++; + } + } + st.nextIdx = targetIdx; + if (st.nextIdx < N) { + // Keep LiDAR visible while a scan is still being built. + // If we don't have accumulated frames yet, show the partial current scan. + if (!lidarOrderedDebugView && _lidarAccumFrames.length === 0 && st.n > 0) { + _lidarPosArray.set(st.worldPts.subarray(0, st.n * 3), 0); + _lidarColArray.set(st.colArray.subarray(0, st.n * 3), 0); + _lidarGeom.setDrawRange(0, st.n); + if (st.n > 0) _lidarLastNonZeroDrawCount = st.n; + _lidarGeom.attributes.position.needsUpdate = true; + _lidarGeom.attributes.color.needsUpdate = true; + lidarVizGroup.position.set(0, 0, 0); + lidarVizGroup.quaternion.identity(); + lidarVizGroup.scale.set(1, 1, 1); + } + return; // scan still in progress + } + + const scanEndNs = st.scanStartNs + st.scanDurationNs; + const refPose = getLidarPoseAtNs(scanEndNs); + const refTwlFlat = composeTwlFlat64(refPose.pos, refPose.quat); + const refW2L = twlInverseMatrix(refPose.pos, refPose.quat); + const pDeskLocal = new THREE.Vector3(); + for (let i = 0; i < st.n; i++) { + pDeskLocal.set( + st.worldPts[i * 3 + 0], + st.worldPts[i * 3 + 1], + st.worldPts[i * 3 + 2] + ).applyMatrix4(refW2L); + st.deskPts[i * 3 + 0] = pDeskLocal.x; + st.deskPts[i * 3 + 1] = pDeskLocal.y; + st.deskPts[i * 3 + 2] = pDeskLocal.z; + } + + logLidarFrameStats(st.worldPts, st.n, st.ring); + + const rawFrame = makeRoboValLidarFrame({ + frameId: "lidar", + stampNs: scanEndNs, + points: st.rawPts.subarray(0, st.n * 3), + intensity: st.intensity.subarray(0, st.n), + ring: st.ring.subarray(0, st.n), + t: st.tArr.subarray(0, st.n), + hasRing: true, + hasPerPointTime: true, + scanDurationS, + poseTWorldLidar: refTwlFlat, + }); + const deskewedFrame = makeRoboValLidarFrame({ + frameId: "lidar", + stampNs: scanEndNs, + points: st.deskPts.subarray(0, st.n * 3), + intensity: st.intensity.subarray(0, st.n), + ring: st.ring.subarray(0, st.n), + t: st.tArr.subarray(0, st.n), + hasRing: true, + hasPerPointTime: true, + scanDurationS, + poseTWorldLidar: refTwlFlat, + }); + const sensorModelMeta = { + range_min_m: LIDAR_MIN_RANGE_M, + range_max_m: LIDAR_MAX_RANGE_M, + noise_enabled: lidarNoiseEnabled, + multi_return_mode: lidarMultiReturnMode, + ordered_render_debug: lidarOrderedDebugView, + deskewed: true, + }; + rawFrame.sensor_model = sensorModelMeta; + deskewedFrame.sensor_model = sensorModelMeta; + const rangeImage = { + H: LIDAR_NUM_RINGS, + W: LIDAR_RANGE_IMAGE_W, + range: st.rangeImg, + intensity: st.intenImg, + ring_index: st.ringIdxImg, + metadata: { + azimuth_convention: "col increases with azimuth in lidar FLU frame", + binning: "uniform azimuth bins", + num_rings: LIDAR_NUM_RINGS, + num_azimuth_bins: LIDAR_RANGE_IMAGE_W, + sensor_model: sensorModelMeta, + visualization_mode: lidarOrderedDebugView ? "single_sweep_ordered" : "accumulated_unordered", + accumulation: { + max_frames: LIDAR_ACCUM_FRAMES, + min_translation_m: LIDAR_ACCUM_MIN_TRANSLATION_M, + min_rotation_deg: LIDAR_ACCUM_MIN_ROT_DEG, + refresh_s: LIDAR_ACCUM_REFRESH_S, + }, + }, + }; + _lidarLatestRawFrame = rawFrame; + _lidarLatestDeskewedFrame = deskewedFrame; + _lidarLatestRangeImage = rangeImage; + // Save world-frame points for dimos bridge (Three.js Y-up coords). + // The bridge's cyclic permutation correctly converts these to ROS Z-up. + _lidarLatestWorldPts = st.worldPts.slice(0, st.n * 3); + _lidarLatestLocalPts = st.deskPts.slice(0, st.n * 3); + _lidarLatestWorldIntensity = st.intensity.slice(0, st.n); + + // Default visualization: accumulated world-space point cloud (depth-tested). + if (!lidarOrderedDebugView) { + if (shouldAppendAccumFrame(refPose, scanEndNs)) { + const framePos = new Float32Array(st.n * 3); + const frameCol = new Float32Array(st.n * 3); + framePos.set(st.worldPts.subarray(0, st.n * 3)); + frameCol.set(st.colArray.subarray(0, st.n * 3)); + _lidarAccumFrames.push({ pos: framePos, col: frameCol }); + while (_lidarAccumFrames.length > LIDAR_ACCUM_FRAMES) _lidarAccumFrames.shift(); + _lidarLastAccumPose = { + pos: refPose.pos.clone(), + quat: refPose.quat.clone(), + stampNs: scanEndNs, + }; + } + + let out = 0; + const len = _lidarAccumFrames.length; + for (let fi = 0; fi < len && out < LIDAR_VIZ_MAX_POINTS; fi++) { + const f = _lidarAccumFrames[fi]; + const age01 = len <= 1 ? 0 : (len - 1 - fi) / (len - 1); // 1 old -> 0 newest + const fade = 1.0 - 0.7 * age01; + const fn = Math.floor(f.pos.length / 3); + for (let i = 0; i < fn && out < LIDAR_VIZ_MAX_POINTS; i++, out++) { + _lidarPosArray[out * 3 + 0] = f.pos[i * 3 + 0]; + _lidarPosArray[out * 3 + 1] = f.pos[i * 3 + 1]; + _lidarPosArray[out * 3 + 2] = f.pos[i * 3 + 2]; + _lidarColArray[out * 3 + 0] = Math.max(0, Math.min(1, f.col[i * 3 + 0] * fade)); + _lidarColArray[out * 3 + 1] = Math.max(0, Math.min(1, f.col[i * 3 + 1] * fade)); + _lidarColArray[out * 3 + 2] = Math.max(0, Math.min(1, f.col[i * 3 + 2] * fade)); + } + } + if (out > 0) { + _lidarGeom.setDrawRange(0, out); + _lidarLastNonZeroDrawCount = out; + } + lidarVizGroup.position.set(0, 0, 0); + lidarVizGroup.quaternion.identity(); + lidarVizGroup.scale.set(1, 1, 1); + } else { + // Debug visualization: ordered current-frame cloud in deskewed lidar frame. + _lidarAccumFrames.length = 0; + _lidarPosArray.set(st.deskPts.subarray(0, st.n * 3), 0); + _lidarGeom.setDrawRange(0, st.n); + if (st.n > 0) _lidarLastNonZeroDrawCount = st.n; + lidarVizGroup.position.copy(refPose.pos); + lidarVizGroup.quaternion.copy(refPose.quat); + lidarVizGroup.scale.set(1, 1, 1); + } + _lidarGeom.attributes.position.needsUpdate = true; + _lidarGeom.attributes.color.needsUpdate = true; + // Guard against intermittent empty frames causing visible flicker. + if (_lidarGeom.drawRange.count <= 0 && _lidarLastNonZeroDrawCount > 0) { + _lidarGeom.setDrawRange(0, _lidarLastNonZeroDrawCount); + } + if (_lidarAutoExport) { + writeLidarFrameFiles(rawFrame, deskewedFrame, rangeImage); + } + resetLidarScanState(); +} + +function applySimSensorViewMode() { + if (simSensorViewMode === "rgb") { + // Restore default rendering. + scene.overrideMaterial = _savedOverrideMaterial; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + _rgbdPcGeom.setDrawRange(0, 0); + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + applySceneRgbBackground(); + } else if (simSensorViewMode === "rgbd") { + // RGB-D mode: render scene depth to offscreen target, then post-process to + // metric camera-space Z visualization. Do not override scene materials. + _savedOverrideMaterial = null; + scene.overrideMaterial = null; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + _rgbdPcGeom.setDrawRange(0, 0); + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + skyDome.visible = false; + scene.background = RGBD_BG; + } else { + // LiDAR mode: hide scene visuals and render deterministic point cloud only. + _savedOverrideMaterial = null; + scene.overrideMaterial = null; + assetsGroup.visible = false; + primitivesGroup.visible = false; + lightsGroup.visible = false; + tagsGroup.visible = false; + lidarVizGroup.visible = true; + rgbdPcOverlayGroup.visible = rgbdPcOverlayOnLidar && _rgbdPcOverlayLastCount > 0; + skyDome.visible = false; + scene.background = RGBD_BG; + } + updateSimSensorButtons(); +} + +function setSimSensorViewMode(mode) { + const next = mode === "rgbd" || mode === "lidar" ? mode : "rgb"; + // Toggle behavior: clicking an already-active sensor mode returns to RGB. + simSensorViewMode = (simSensorViewMode === next && next !== "rgb") ? "rgb" : next; + applySimSensorViewMode(); + if (simSensorViewMode === "rgb") { + setStatus("RGB view"); + } else if (simSensorViewMode === "rgbd") { + setStatus(`RGB-D ${rgbdVizMode === "gray" ? "grayscale" : "colormap"} (${rgbdRangeMinM.toFixed(1)}-${rgbdRangeMaxM.toFixed(1)}m)`); + } else { + setStatus(lidarOrderedDebugView ? "LiDAR single sweep view" : "LiDAR accumulated 3D point cloud"); + } +} + +// Controls: pointer-lock look + WASD move. +const controls = new PointerLockControls(camera, document.body); +scene.add(controls.object); + + +const keys = { + forward: false, + backward: false, + left: false, + right: false, + up: false, + down: false, +}; + +function setStatus(msg) { + if (statusEl) statusEl.textContent = msg || ""; + if (statusSimEl) statusSimEl.textContent = msg || ""; +} + +function randId() { + return Math.random().toString(16).slice(2) + "-" + Date.now().toString(16); +} + +function escapeHtml(s) { + return String(s) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); +} + +function loadTagsForWorld() { + // Clean up old primitive colliders BEFORE replacing the arrays + for (const p of primitives) { + removePrimitiveCollider(p); + } + + try { + const rawState = localStorage.getItem("sparkWorldStateByWorld"); + const byWorld = rawState ? JSON.parse(rawState) : {}; + const state = byWorld[worldKey] || null; + if (state && typeof state === "object") { + tags = Array.isArray(state.tags) ? state.tags : []; + assets = Array.isArray(state.assets) ? state.assets.map(normalizeAsset).filter(Boolean) : []; + primitives = Array.isArray(state.primitives) ? state.primitives : []; + editorLights = Array.isArray(state.lights) ? state.lights : []; + groups = Array.isArray(state.groups) ? state.groups : []; + sceneSettings = normalizeSceneSettings(state.sceneSettings); + } else { + // Backwards compat: tags-only storage + const raw = localStorage.getItem("sparkWorldTagsByWorld"); + const byWorldOld = raw ? JSON.parse(raw) : {}; + tags = Array.isArray(byWorldOld[worldKey]) ? byWorldOld[worldKey] : []; + assets = []; + primitives = []; + editorLights = []; + groups = []; + sceneSettings = createDefaultSceneSettings(); + } + } catch { + tags = []; + assets = []; + primitives = []; + editorLights = []; + groups = []; + sceneSettings = createDefaultSceneSettings(); + } + selectedTagId = null; + draftTag = null; + selectedAssetId = null; + selectedPrimitiveId = null; + rebuildTagMarkers(); + renderTagsList(); + renderTagPanel(); + rebuildAssets(); + renderAssetsList(); + rebuildAllPrimitives(); + renderPrimitivesList(); + rebuildAllEditorLights(); + applySceneSkySettings(); + applySceneRgbBackground(); +} + +function saveTagsForWorld() { + try { + let rawState = localStorage.getItem("sparkWorldStateByWorld"); + let byWorld = {}; + + try { + byWorld = rawState ? JSON.parse(rawState) : {}; + } catch { + // Corrupted data, start fresh + console.warn("[SAVE] Corrupted localStorage data, clearing..."); + byWorld = {}; + } + + console.log(`[SAVE] Saving ${assets.length} assets for world: ${worldKey}`); + + // Only save lightweight metadata - NOT the full dataBase64 model data + // Only save state changes (currentStateId, transform) + const lightweightAssets = assets.map(a => { + // Regular assets: only save delta/metadata, not model data + return { + id: a.id, + currentStateId: a.currentStateId || a.currentState, + transform: a.transform, + pickable: a.pickable, + castShadow: a.castShadow ?? false, + receiveShadow: a.receiveShadow ?? false, + blobShadow: a.blobShadow || null, + _deltaOnly: true, + }; + }); + + // Save primitives — strip collider handles and large texture data URLs + // (textures are preserved in Export but too big for localStorage) + const savePrimitives = primitives.map((p) => { + const { _colliderHandle, ...rest } = p; + if (rest.material?.textureDataUrl) { + rest.material = { ...rest.material, textureDataUrl: null }; + } + return rest; + }); + + // Save lights (strip runtime objects) + const saveLights = editorLights.map((l) => { + const { _lightObj, _helperObj, _proxyObj, ...rest } = l; + return rest; + }); + + byWorld[worldKey] = { + tags, + assets: lightweightAssets, + primitives: savePrimitives, + lights: saveLights, + groups, + sceneSettings: serializeSceneSettings(), + }; + const dataStr = JSON.stringify(byWorld); + + // Check size before saving (localStorage limit is typically 5MB) + const sizeKB = (dataStr.length * 2) / 1024; // Rough estimate (UTF-16) + console.log(`[SAVE] Data size: ${sizeKB.toFixed(1)}KB`); + + localStorage.setItem("sparkWorldStateByWorld", dataStr); + localStorage.setItem("sparkWorldLastWorldKey", worldKey); + } catch (e) { + console.error("[SAVE] Failed to save world state:", e); + + // If quota exceeded, try clearing old data and retry + if (e.name === "QuotaExceededError") { + console.warn("[SAVE] Quota exceeded, clearing old world data..."); + try { + localStorage.removeItem("sparkWorldStateByWorld"); + // Retry with just current world and minimal data + const freshData = {}; + freshData[worldKey] = { + tags, + assets: [], + sceneSettings: serializeSceneSettings() + }; + localStorage.setItem("sparkWorldStateByWorld", JSON.stringify(freshData)); + console.log("[SAVE] Saved minimal data after clearing old data"); + } catch (e2) { + console.error("[SAVE] Still failed after clearing:", e2); + } + } + } +} + +// Clear all localStorage data for this app (useful for debugging) +function clearWorldStorage() { + localStorage.removeItem("sparkWorldStateByWorld"); + localStorage.removeItem("sparkWorldLastWorldKey"); + console.log("[STORAGE] Cleared all world storage"); +} +// Expose for debugging: window.clearWorldStorage = clearWorldStorage; + +function setWorldKey(key) { + worldKey = key || "default"; + localStorage.setItem("sparkWorldLastWorldKey", worldKey); + loadTagsForWorld(); +} + + +function getSelectedTag() { + return tags.find((t) => t.id === selectedTagId) ?? null; +} + +function renderTagPanel() { + // No-op in sim-only mode (editor tag panel not present) +} + +function renderTagsList() { + // No-op in sim-only mode (editor tags list not present) +} + +const markerGeom = new THREE.SphereGeometry(0.08, 12, 12); +const markerMat = new THREE.MeshBasicMaterial({ color: 0x7cc4ff }); +const markerMatActive = new THREE.MeshBasicMaterial({ color: 0xffd36e }); +const radiusGeom = new THREE.SphereGeometry(1, 20, 14); +const radiusMat = new THREE.MeshBasicMaterial({ + color: 0x7cc4ff, + transparent: true, + opacity: 0.08, + depthWrite: false, +}); + +function agentUiPush(event) { + const logs = [ + agentLogEl, + document.getElementById("edit-agent-log"), + ].filter(Boolean); + for (const log of logs) { + const el = document.createElement("div"); + el.className = "agent-log-item"; + el.textContent = event; + log.prepend(el); + // cap + while (log.children.length > 10) log.removeChild(log.lastChild); + } +} + +function agentUiSetLast(text) { + const value = text || ""; + if (agentLastEl) agentLastEl.textContent = value; + const editLast = document.getElementById("edit-agent-last"); + if (editLast) editLast.textContent = value; +} + +function agentUiSetShot(base64) { + if (!base64) return; + const src = `data:image/jpeg;base64,${base64}`; + if (agentShotImgEl) agentShotImgEl.src = src; + const editShot = document.getElementById("edit-agent-shot-img"); + if (editShot) editShot.src = src; +} + +function extractObservationText(parsed, raw) { + const p = parsed && typeof parsed === "object" ? parsed : {}; + const observation = + (typeof p.observation === "string" && p.observation) || + (typeof p.obs === "string" && p.obs) || + (typeof p.perception === "string" && p.perception) || + (typeof p.sceneObservation === "string" && p.sceneObservation) || + (typeof p.visualObservation === "string" && p.visualObservation) || + (typeof p.params?.observation === "string" && p.params.observation) || + ""; + if (observation.trim()) return observation.trim(); + + if (typeof raw === "string" && raw.trim()) { + const m = raw.match(/"observation"\s*:\s*"([^"]+)"/i); + if (m?.[1]) return m[1]; + } + return ""; +} + +function agentUiSetObservation(text) { + const value = String(text || "").trim(); + if (!agentObservationEl) return; + agentObservationEl.textContent = value || "No observation in latest response."; +} + +function agentUiSetRequest({ endpoint, model, prompt, context, imageBytes, messages }) { + const metaText = `endpoint: ${endpoint}\nmodel: ${model}\nimageBytes: ${imageBytes ?? "?"}\nworld: ${worldKey}`; + if (agentReqMetaEl) agentReqMetaEl.textContent = metaText; + const editMeta = document.getElementById("edit-agent-req-meta"); + if (editMeta) editMeta.textContent = metaText; + if (agentReqPromptEl) agentReqPromptEl.textContent = prompt || ""; + const editPrompt = document.getElementById("edit-agent-req-prompt"); + if (editPrompt) editPrompt.textContent = prompt || ""; + + // Format messages for display (only assistant and user messages, not system) + let contextText = ""; + if (messages && messages.length > 0) { + // Filter out system messages - only show assistant and user + const conversationMessages = messages.filter(msg => msg.role !== "system"); + if (conversationMessages.length > 0) { + contextText = conversationMessages.map((msg) => { + const role = msg.role.toUpperCase(); + let content = ""; + if (typeof msg.content === "string") { + content = msg.content; + } else if (Array.isArray(msg.content)) { + // Handle multimodal content (text + image) + content = msg.content.map(part => { + if (part.type === "text") return part.text; + if (part.type === "image_url") return "[IMAGE]"; + return JSON.stringify(part); + }).join("\n"); + } else { + content = JSON.stringify(msg.content, null, 2); + } + return `═══ ${role} ═══\n${content}`; + }).join("\n\n"); + } else { + contextText = "(No conversation history yet)"; + } + } else { + contextText = JSON.stringify(context ?? {}, null, 2); + } + if (agentReqContextEl) agentReqContextEl.textContent = contextText; + const editContext = document.getElementById("edit-agent-req-context"); + if (editContext) editContext.textContent = contextText; +} + +function agentUiSetResponse({ raw, parsed }) { + if (agentRespRawEl) agentRespRawEl.textContent = raw || ""; + const editRaw = document.getElementById("edit-agent-resp-raw"); + if (editRaw) editRaw.textContent = raw || ""; + if (agentLastEl) agentLastEl.textContent = JSON.stringify(parsed ?? {}, null, 2); + agentUiSetObservation(extractObservationText(parsed, raw)); + const editLast = document.getElementById("edit-agent-last"); + if (editLast) editLast.textContent = JSON.stringify(parsed ?? {}, null, 2); +} + +function clearAgentInspectorViews() { + if (agentShotImgEl) agentShotImgEl.removeAttribute("src"); + if (agentReqMetaEl) agentReqMetaEl.textContent = "No request yet"; + if (agentReqPromptEl) agentReqPromptEl.textContent = ""; + if (agentReqContextEl) agentReqContextEl.textContent = ""; + if (agentRespRawEl) agentRespRawEl.textContent = ""; + if (agentLastEl) agentLastEl.textContent = "Waiting..."; + if (agentObservationEl) agentObservationEl.textContent = "Waiting for first observation..."; + + const editShot = document.getElementById("edit-agent-shot-img"); + const editReqMeta = document.getElementById("edit-agent-req-meta"); + const editReqPrompt = document.getElementById("edit-agent-req-prompt"); + const editReqContext = document.getElementById("edit-agent-req-context"); + const editRespRaw = document.getElementById("edit-agent-resp-raw"); + const editLast = document.getElementById("edit-agent-last"); + if (editShot) editShot.removeAttribute("src"); + if (editReqMeta) editReqMeta.textContent = "No request yet"; + if (editReqPrompt) editReqPrompt.textContent = ""; + if (editReqContext) editReqContext.textContent = ""; + if (editRespRaw) editRespRaw.textContent = ""; + if (editLast) editLast.textContent = "Waiting..."; +} + +function showEditSpawnedAgentsTab() { + const btn = document.getElementById("vibe-tab-agents"); + btn?.click?.(); +} + +function getAgentById(id) { + const key = String(id || ""); + if (!key) return null; + return aiAgents.find((a) => a?.id === key) || null; +} + +function ensureAgentControlStrip() { + // Restrict spawned-agent controls to the right-panel "Spawned Agents" tab only. + const panelContent = document.getElementById("vibe-tab-agents-pane"); + if (!panelContent) return; + + let strip = document.getElementById("agent-control-strip"); + + // Re-parent strip if it ended up in the wrong panel after mode switch. + if (strip && strip.parentElement !== panelContent) { + strip.remove(); + strip = null; + agentUiSelectedLabelEl = null; + agentUiSpawnBtn = null; + agentUiFollowBtn = null; + agentUiStopBtn = null; + agentUiRemoveBtn = null; + agentUiTaskInputEl = null; + agentUiTaskRunBtn = null; + } + + if (agentUiSelectedLabelEl && agentUiFollowBtn && agentUiStopBtn && agentUiRemoveBtn) return; + + if (!strip) { + strip = document.createElement("div"); + strip.id = "agent-control-strip"; + strip.className = "agent-control-strip"; + strip.innerHTML = ` +
Selected: none
+
+ + + + +
+
+ + +
+ `; + panelContent.insertBefore(strip, panelContent.firstChild || null); + } + + agentUiSelectedLabelEl = document.getElementById("agent-selected-label"); + agentUiSpawnBtn = document.getElementById("agent-selected-spawn"); + agentUiFollowBtn = document.getElementById("agent-selected-follow"); + agentUiStopBtn = document.getElementById("agent-selected-stop"); + agentUiRemoveBtn = document.getElementById("agent-selected-remove"); + agentUiTaskInputEl = document.getElementById("agent-selected-task-input"); + agentUiTaskRunBtn = document.getElementById("agent-selected-task-run"); + + agentUiSpawnBtn?.addEventListener("click", () => { + void spawnOrMoveAiAtAim({ createNew: true, silent: false, ephemeral: false }).then(() => { + const newest = aiAgents[aiAgents.length - 1]; + if (newest?.id) selectAgentInspector(newest.id); + renderSelectedAgentControls(); + }); + showEditSpawnedAgentsTab(); + }); + agentUiFollowBtn?.addEventListener("click", () => { + const a = getAgentById(selectedAgentInspectorId); + if (!a) return; + if (agentCameraFollow && agentCameraFollowId === a.id) { + disableAgentCameraFollow(); + } else { + enableAgentCameraFollow(a.id); + } + renderSelectedAgentControls(); + }); + agentUiStopBtn?.addEventListener("click", () => { + const a = getAgentById(selectedAgentInspectorId); + if (!a) return; + stopAiAgent(a, "ui-stop"); + setStatus(`Stopped ${a.id}.`); + renderSelectedAgentControls(); + }); + agentUiRemoveBtn?.addEventListener("click", () => { + const a = getAgentById(selectedAgentInspectorId); + if (!a) return; + removeAiAgent(a, "ui-remove"); + setStatus(`Removed ${a.id}.`); + if (agentTask.active && aiAgents.length === 0) endAgentTask("all-agents-removed"); + renderSelectedAgentControls(); + }); + const runSelectedTask = () => { + const a = getAgentById(selectedAgentInspectorId); + if (!a) return; + const text = String(agentUiTaskInputEl?.value || "").trim(); + if (!text) return; + if (agentTask.active) endAgentTask("replace-task"); + void startAgentTask(text, { autoPool: false, targetAgentId: a.id }); + if (agentUiTaskInputEl) agentUiTaskInputEl.value = ""; + setStatus(`Running task on ${a.id}.`); + showEditSpawnedAgentsTab(); + }; + agentUiTaskRunBtn?.addEventListener("click", runSelectedTask); + agentUiTaskInputEl?.addEventListener("keydown", (e) => { + e.stopPropagation(); + if (e.key === "Enter") runSelectedTask(); + }); +} + +function renderSelectedAgentControls() { + ensureAgentControlStrip(); + if (!agentUiSelectedLabelEl || !agentUiFollowBtn || !agentUiStopBtn || !agentUiRemoveBtn) return; + const a = getAgentById(selectedAgentInspectorId); + const has = !!a; + agentUiSelectedLabelEl.textContent = has ? `Selected: ${a.id}` : "Selected: none"; + agentUiFollowBtn.disabled = !has; + agentUiStopBtn.disabled = !has; + agentUiRemoveBtn.disabled = !has; + agentUiFollowBtn.textContent = has && agentCameraFollow && agentCameraFollowId === a.id ? "Unfollow POV" : "Follow POV"; +} + +function getOrCreateAgentInspectorState(agentId) { + const id = String(agentId || ""); + if (!id) return { shot: "", request: null, response: null }; + if (!agentInspectorStateById.has(id)) { + agentInspectorStateById.set(id, { shot: "", request: null, response: null }); + } + return agentInspectorStateById.get(id); +} + +function renderAgentInspector(agentId = selectedAgentInspectorId) { + const id = String(agentId || ""); + if (!id) return; + const s = getOrCreateAgentInspectorState(id); + if (agentReqMetaEl) { + const base = s.request || { endpoint: "-", model: "-", prompt: "", context: {}, imageBytes: null, messages: [] }; + agentUiSetRequest(base); + agentReqMetaEl.textContent = `${agentReqMetaEl.textContent}\nagent: ${id}`; + const editMeta = document.getElementById("edit-agent-req-meta"); + if (editMeta) editMeta.textContent = `${editMeta.textContent}\nagent: ${id}`; + } + if (s.shot) agentUiSetShot(s.shot); + if (s.response) agentUiSetResponse(s.response); +} + +function selectAgentInspector(agentId) { + const id = String(agentId || ""); + if (!id) return; + selectedAgentInspectorId = id; + showEditSpawnedAgentsTab(); + // Force strip into correct panel on selection. + ensureAgentControlStrip(); + renderAgentInspector(id); + renderSelectedAgentControls(); + // Visual flash feedback. + const strip = document.getElementById("agent-control-strip"); + if (strip) { + strip.style.outline = "2px solid var(--accent-primary)"; + setTimeout(() => { strip.style.outline = ""; }, 600); + } +} + +function renderAgentTaskUi() { + ensureAgentControlStrip(); + const bar = document.getElementById("agent-command-bar"); + const hasAgent = aiAgents.length > 0; + + if (bar) bar.style.display = ""; + if (spawnAiBtn) spawnAiBtn.style.display = hasAgent ? "none" : ""; + + if (!agentTaskStatusEl || !agentTaskInputEl || !agentTaskStartBtn || !agentTaskEndBtn) return; + + if (!agentTask.active) { + agentTaskStatusEl.textContent = ""; + agentTaskInputEl.disabled = false; + agentTaskStartBtn.disabled = !hasAgent; + agentTaskEndBtn.disabled = true; + if (bar) bar.classList.remove("active"); + } else { + agentTaskStatusEl.textContent = "Running"; + agentTaskInputEl.disabled = true; + agentTaskStartBtn.disabled = true; + agentTaskEndBtn.disabled = false; + if (bar) bar.classList.add("active"); + } + updateSimCameraModeToggleUi(); + renderSelectedAgentControls(); +} + +function updateSimCameraModeToggleUi() { + if (!simCameraModeToggleBtn) return; + const isUserCam = simUserCameraMode === "user"; + simCameraModeToggleBtn.textContent = isUserCam ? "Camera: User" : "Camera: Agent"; + simCameraModeToggleBtn.classList.toggle("active", isUserCam); + simCameraModeToggleBtn.classList.toggle("tb-muted", !isUserCam); + simCameraModeToggleBtn.title = isUserCam + ? "Keep your user camera while the agent runs" + : "Follow the active agent while the task runs"; +} + +function enableAgentCameraFollow(agentId = selectedAgentInspectorId) { + if (aiAgents.length === 0) return; + const target = getAgentById(agentId) || aiAgents[0]; + if (!target) return; + agentCameraFollow = true; + agentCameraFollowId = target.id; + _agentFollowInitialized = false; + + // Unlock player controls so camera isn't fighting with pointer lock + controls?.unlock?.(); + + // Hide the player avatar + avatar.visible = false; + + // Hide crosshair and interaction hints during follow mode + const crosshair = document.getElementById("crosshair"); + if (crosshair) crosshair.style.display = "none"; + const hint = document.getElementById("interaction-hint"); + if (hint) hint.style.display = "none"; + + console.log("[AGENT CAM] Following agent"); + renderSelectedAgentControls(); +} + +function disableAgentCameraFollow() { + agentCameraFollow = false; + agentCameraFollowId = null; + + // Show all agent meshes again + for (const a of aiAgents) { + if (a?.group) a.group.visible = true; + } + + // Avatar mesh stays hidden (physics capsule still active) + + // Restore crosshair and interaction hints + const crosshair = document.getElementById("crosshair"); + if (crosshair) crosshair.style.display = ""; + const hint = document.getElementById("interaction-hint"); + if (hint) hint.style.display = ""; + + console.log("[AGENT CAM] Returning to player"); + renderSelectedAgentControls(); +} + +function updateAgentCameraFollow(dt) { + if (!agentCameraFollow || aiAgents.length === 0) return; + + const agent = getAgentById(agentCameraFollowId) || aiAgents[0]; + if (!agent) return; + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + + // Place camera at the real Go2 front-camera mount: GO2_CAMERA_HEIGHT above + // the ground and GO2_CAMERA_FORWARD along the agent's heading so the origin + // sits outside the body mesh (Go2's head-mounted RGB-D, not body-center). + const feetY = ay - ((agent.halfHeight || 0.25) + (agent.radius || 0.12)); + const eyeY = feetY + GO2_CAMERA_HEIGHT; + const eyeX = ax + Math.sin(yaw) * GO2_CAMERA_FORWARD; + const eyeZ = az + Math.cos(yaw) * GO2_CAMERA_FORWARD; + camera.position.set(eyeX, eyeY, eyeZ); + + // Compute forward direction exactly like visionCapture.js does + const cp = Math.cos(pitch); + const sp = Math.sin(pitch); + const fx = Math.sin(yaw) * cp; + const fy = sp; + const fz = Math.cos(yaw) * cp; + + // Use lookAt to match the VLM capture camera + camera.lookAt(eyeX + fx, eyeY + fy, eyeZ + fz); + + // Hide the agent's own mesh so it doesn't block the view + if (agent.group) agent.group.visible = false; +} + +async function startAgentTask(instruction, { autoPool = true, targetAgentId = null } = {}) { + const text = String(instruction || "").trim(); + if (!text) return; + + const now = Date.now(); + const taskState = { + active: true, + instruction: text, + startedAt: now, + finishedAt: 0, + finishedReason: "", + lastSummary: "", + }; + + // Determine which agents get this task + const target = targetAgentId ? getAgentById(targetAgentId) : null; + agentTaskTargetId = target?.id || null; + const targets = target ? [target] : aiAgents; + + for (const a of targets) { + _setAgentTask(a.id, { ...taskState }); + a._taskStartedAt = now; + if (a?.vlm) a.vlm.enabled = true; + } + + agentUiPush(`${new Date().toLocaleTimeString()}\nTASK START\n${text}${target ? ` [${target.id}]` : ` [${targets.length} agents]`}`); + renderAgentTaskUi(); + + if (simUserCameraMode === "agent") enableAgentCameraFollow(); +} + +function endAgentTask(reason = "manual", agentId = null) { + if (agentId) { + // End task for a specific agent + const task = _agentTasks.get(agentId); + if (task?.active) { + task.active = false; + task.finishedAt = Date.now(); + task.finishedReason = reason; + _agentTasks.set(agentId, task); + } + agentUiPush(`${new Date().toLocaleTimeString()}\nTASK END (${reason}) [${agentId}]`); + } else { + // End all tasks + for (const [id, task] of _agentTasks) { + if (task.active) { + task.active = false; + task.finishedAt = Date.now(); + task.finishedReason = reason; + } + } + agentTask.active = false; + agentTask.finishedAt = Date.now(); + agentTask.finishedReason = reason; + agentUiPush(`${new Date().toLocaleTimeString()}\nTASK END ALL (${reason})`); + } + agentTaskTargetId = null; + + // Check if any agent still has an active task + const anyActive = [..._agentTasks.values()].some((t) => t.active); + if (!anyActive) { + agentTask.active = false; + disableAgentCameraFollow(); + } + + renderAgentTaskUi(); + +} + +function rebuildTagMarkers() { + while (tagsGroup.children.length) tagsGroup.remove(tagsGroup.children[0]); + + for (const t of tags) { + if (!t.position) continue; + const m = new THREE.Mesh(markerGeom, t.id === selectedTagId ? markerMatActive : markerMat); + m.position.set(t.position.x, t.position.y, t.position.z); + m.userData.tagId = t.id; + m.renderOrder = 1000; + tagsGroup.add(m); + + const r = Number(t.radius ?? 1.5); + const shell = new THREE.Mesh(radiusGeom, radiusMat); + shell.position.copy(m.position); + shell.scale.setScalar(Math.max(0.01, r)); + shell.userData.tagId = t.id; + shell.userData.isRadius = true; + tagsGroup.add(shell); + } + + updateMarkerMaterials(); +} + +function updateMarkerMaterials() { + for (const child of tagsGroup.children) { + if (!child.isMesh) continue; + if (child.userData?.isRadius) continue; + child.material = child.userData.tagId === selectedTagId ? markerMatActive : markerMat; + } +} + +function renderAssetModal() { + // No-op in sim-only mode (asset upload modal not present) +} + +function base64FromArrayBuffer(buf) { + const bytes = new Uint8Array(buf); + let binary = ""; + const chunk = 0x8000; + for (let i = 0; i < bytes.length; i += chunk) { + binary += String.fromCharCode.apply(null, bytes.subarray(i, i + chunk)); + } + return btoa(binary); +} + +function arrayBufferFromBase64(base64) { + const bin = atob(base64); + const len = bin.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) bytes[i] = bin.charCodeAt(i); + return bytes.buffer; +} + +function normalizeAsset(a) { + if (!a || typeof a !== "object") return null; + // Ensure pickable property exists + if (typeof a.pickable !== "boolean") a.pickable = false; + if (typeof a.bumpable !== "boolean") a.bumpable = false; + if (!Number.isFinite(a.bumpResponse)) a.bumpResponse = 0.9; + if (!Number.isFinite(a.bumpDamping)) a.bumpDamping = 0.9; + // New schema: states is array + if (Array.isArray(a.states)) { + if (!a.currentStateId && a.states[0]?.id) a.currentStateId = a.states[0].id; + // Ensure interactions exist per state. + for (const s of a.states) { + if (!Array.isArray(s.interactions)) s.interactions = []; + } + // Backfill actions from interactions if missing. + if (!Array.isArray(a.actions) || a.actions.length === 0) { + a.actions = []; + for (const s of a.states) { + for (const it of s.interactions) { + a.actions.push({ id: it.id || `act_${s.id}_${it.to}`, label: it.label || "toggle", from: s.id, to: it.to }); + } + } + } else { + // Backfill interactions from actions if missing. + const byFrom = new Map(); + for (const act of a.actions) { + if (!byFrom.has(act.from)) byFrom.set(act.from, []); + byFrom.get(act.from).push({ id: act.id, label: act.label, to: act.to }); + } + for (const s of a.states) { + if (!s.interactions || s.interactions.length === 0) s.interactions = byFrom.get(s.id) || []; + } + } + return a; + } + // Old schema: states is {A,B} + if (a.states && typeof a.states === "object") { + const out = { + id: a.id, + title: a.title || "", + notes: a.notes || "", + states: [], + currentStateId: a.currentState || "A", + actions: Array.isArray(a.actions) ? a.actions : [], + transform: a.transform || null, + bumpable: a.bumpable === true, + bumpResponse: Number.isFinite(a.bumpResponse) ? a.bumpResponse : 0.9, + bumpDamping: Number.isFinite(a.bumpDamping) ? a.bumpDamping : 0.9, + }; + const A = a.states.A; + const B = a.states.B; + if (A) out.states.push({ id: "A", name: A.name || "stateA", glbName: A.glbName || "", dataBase64: A.dataBase64 || "", interactions: [] }); + if (B) out.states.push({ id: "B", name: B.name || "stateB", glbName: B.glbName || "", dataBase64: B.dataBase64 || "", interactions: [] }); + if (!out.currentStateId) out.currentStateId = out.states[0]?.id || "A"; + out.actions = Array.isArray(out.actions) ? out.actions : []; + // Backfill interactions from actions. + const byFrom = new Map(); + for (const act of out.actions) { + if (!byFrom.has(act.from)) byFrom.set(act.from, []); + byFrom.get(act.from).push({ id: act.id, label: act.label, to: act.to }); + } + for (const s of out.states) s.interactions = byFrom.get(s.id) || []; + return out; + } + return a; +} + +function getSelectedAsset() { + return assets.find((a) => a.id === selectedAssetId) || null; +} + +function renderAssetsList() { + if (!assetsListEl) return; + assetsListEl.innerHTML = ""; + for (const a of assets) { + const el = document.createElement("div"); + el.className = "tag-item" + (a.id === selectedAssetId ? " active" : ""); + const sId = a.currentStateId || a.currentState || "A"; + const stateObj = Array.isArray(a.states) ? a.states.find((s) => s.id === sId) : a.states?.[sId]; + const label = a.title || stateObj?.glbName || "(asset)"; + const kind = stateObj?.scene || stateObj?.shapeScene ? "shape" : "glb"; + el.innerHTML = `${escapeHtml(label)}${kind}`; + el.addEventListener("click", () => selectAsset(a.id)); + assetsListEl.appendChild(el); + } +} + +function selectAsset(id) { + selectedAssetId = id; + if (id) { + selectedPrimitiveId = null; + } + renderAssetsList(); +} + +function persistSelectedAssetTransform() { + const a = getSelectedAsset(); + if (!a) return; + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (!obj) return; + a.transform = { + position: { x: obj.position.x, y: obj.position.y, z: obj.position.z }, + rotation: { x: obj.rotation.x, y: obj.rotation.y, z: obj.rotation.z }, + scale: { x: obj.scale.x, y: obj.scale.y, z: obj.scale.z }, + }; + saveTagsForWorld(); + if (!a.bumpable) rebuildAssetCollider(a.id); +} + +function normalizeShapeStateScene(sceneLike) { + const raw = sceneLike || { tags: [], primitives: [], lights: [], groups: [] }; + return { + tags: Array.isArray(raw.tags) ? raw.tags : [], + primitives: Array.isArray(raw.primitives) ? raw.primitives : [], + lights: Array.isArray(raw.lights) ? raw.lights : [], + groups: Array.isArray(raw.groups) ? raw.groups : [], + }; +} + +function buildShapeStateRoot(state, assetId, fixedPivotCenter = null) { + const sceneState = normalizeShapeStateScene(state?.scene || state?.shapeScene); + const root = new THREE.Group(); + const primMap = new Map(); + for (const p of sceneState.primitives) { + const geom = createPrimitiveGeometry(p.type, p.dimensions || {}); + const mat = createPrimitiveMaterial(p.material || {}); + const mesh = new THREE.Mesh(geom, mat); + applyPrimitiveCutoutShader(mesh, p); + mesh.name = `assetPrim:${assetId}:${p.id || randId()}`; + mesh.userData.assetId = assetId; + mesh.userData.isAssetPrimitive = true; + mesh.castShadow = p.castShadow !== false; + mesh.receiveShadow = p.receiveShadow !== false; + const tr = p.transform || {}; + if (tr.position) mesh.position.set(tr.position.x || 0, tr.position.y || 0, tr.position.z || 0); + if (tr.rotation) mesh.rotation.set(tr.rotation.x || 0, tr.rotation.y || 0, tr.rotation.z || 0); + if (tr.scale) mesh.scale.set(tr.scale.x ?? 1, tr.scale.y ?? 1, tr.scale.z ?? 1); + root.add(mesh); + if (p.id) primMap.set(p.id, mesh); + } + for (const g of sceneState.groups || []) { + if (!Array.isArray(g.children) || g.children.length === 0) continue; + const subgroup = new THREE.Group(); + subgroup.name = `assetGroup:${assetId}:${g.id || randId()}`; + root.add(subgroup); + for (const cid of g.children) { + const child = primMap.get(cid); + if (!child) continue; + subgroup.add(child); + } + } + + // Re-center: move the pivot to the bounding-box center so the transform + // gizmo appears on the asset rather than at an arbitrary offset. + root.updateMatrixWorld(true); + const bbox = new THREE.Box3().setFromObject(root); + if (!bbox.isEmpty()) { + const autoCenter = bbox.getCenter(new THREE.Vector3()); + const center = fixedPivotCenter ? fixedPivotCenter.clone() : autoCenter; + for (const child of root.children) { + child.position.sub(center); + } + root.position.copy(center); + root.userData._pivotCenter = center.clone(); + } + + return root; +} + +function disposeShapeStateRoot(root) { + if (!root) return; + root.traverse((obj) => { + if (!obj?.isMesh) return; + obj.geometry?.dispose?.(); + disposePrimitiveMaterial(obj.material); + }); +} + +async function instantiateAsset(a) { + if (!a?.states) return; + const sId = a.currentStateId || a.currentState || (Array.isArray(a.states) ? a.states[0]?.id : "A"); + const state = Array.isArray(a.states) + ? a.states.find((s) => s.id === sId) || a.states[0] + : a.states[sId] || a.states.A; + let root = null; + if (state?.scene || state?.shapeScene) { + let fixedPivotCenter = null; + if (a._shapePivotCenter + && Number.isFinite(a._shapePivotCenter.x) + && Number.isFinite(a._shapePivotCenter.y) + && Number.isFinite(a._shapePivotCenter.z)) { + fixedPivotCenter = new THREE.Vector3(a._shapePivotCenter.x, a._shapePivotCenter.y, a._shapePivotCenter.z); + } else if (Array.isArray(a.states) && a.states.length > 0) { + const anchorState = a.states[0]; + const anchorRoot = buildShapeStateRoot(anchorState, `${a.id}:anchor`); + const anchorCenter = anchorRoot.userData?._pivotCenter; + if (anchorCenter) { + fixedPivotCenter = anchorCenter.clone(); + a._shapePivotCenter = { x: anchorCenter.x, y: anchorCenter.y, z: anchorCenter.z }; + } + disposeShapeStateRoot(anchorRoot); + } + root = buildShapeStateRoot(state, a.id, fixedPivotCenter); + const rootCenter = root.userData?._pivotCenter; + if (rootCenter && !a._shapePivotCenter) { + a._shapePivotCenter = { x: rootCenter.x, y: rootCenter.y, z: rootCenter.z }; + } + } else if (state?.dataBase64) { + const buf = arrayBufferFromBase64(state.dataBase64); + const url = URL.createObjectURL(new Blob([buf], { type: "model/gltf-binary" })); + const gltf = await new Promise((resolve, reject) => { + gltfLoader.load(url, (g) => resolve(g), undefined, (e) => reject(e)); + }); + URL.revokeObjectURL(url); + root = gltf.scene; + } else { + return; + } + root.name = `asset:${a.id}`; + const wantShadow = a.castShadow === true; // opt-in, default OFF + const wantReceive = a.receiveShadow === true; // opt-in, default OFF + + root.traverse((m) => { + if (m.isMesh) { + if (!m.userData?.isAssetPrimitive) m.castShadow = false; // GLB assets keep cheap shadow behavior + m.receiveShadow = wantReceive; + m.userData.assetId = a.id; + } + }); + + // Pre-compute local bounding sphere ONCE (cached — never call setFromObject again) + const bbox = new THREE.Box3().setFromObject(root); + const localSphere = new THREE.Sphere(); + bbox.getBoundingSphere(localSphere); + const localCenter = localSphere.center.clone(); + root.worldToLocal(localCenter); + root.userData._localSphereCenter = localCenter; + root.userData._localSphereRadius = Math.max(localSphere.radius, 0.2); + + // Blob shadow: a cheap flat gradient circle beneath the asset. + // Uses zero shadow-map resources — just a textured plane with transparency. + if (wantShadow) { + const bboxSize = bbox.getSize(new THREE.Vector3()); + const localGroundY = bbox.min.y + 0.005; + const blob = createBlobShadow(a.id, bboxSize.x, bboxSize.z, localGroundY, { + opacity: a.blobShadow?.opacity ?? 0.5, + scale: a.blobShadow?.scale ?? 1.0, + stretch: a.blobShadow?.stretch ?? 1.0, + rotationDeg: a.blobShadow?.rotationDeg ?? 0, + offsetX: a.blobShadow?.offsetX ?? 0, + offsetY: a.blobShadow?.offsetY ?? 0, + offsetZ: a.blobShadow?.offsetZ ?? 0, + }); + if (blob) root.add(blob); + } + + const tr = a.transform || {}; + if (tr.position) root.position.set(tr.position.x, tr.position.y, tr.position.z); + if (tr.rotation) root.rotation.set(tr.rotation.x, tr.rotation.y, tr.rotation.z); + if (tr.scale) root.scale.set(tr.scale.x, tr.scale.y, tr.scale.z); + assetsGroup.add(root); + await rebuildAssetCollider(a.id); +} + +async function setAssetState(assetId, nextState) { + const a = assets.find((x) => x.id === assetId); + if (!a) return; + const exists = Array.isArray(a.states) ? a.states.some((s) => s.id === nextState) : !!a.states?.[nextState]; + if (!exists) return; + a.currentStateId = nextState; + saveTagsForWorld(); + // Replace visual + const existing = assetsGroup.getObjectByName(`asset:${a.id}`); + if (existing?.parent) existing.parent.remove(existing); + await instantiateAsset(a); + renderAssetsList(); + selectAsset(a.id); +} + +async function applyAssetAction(assetId, actionId) { + const a = assets.find((x) => x.id === assetId); + if (!a) return false; + + const act = (a.actions || []).find((x) => x.id === actionId) || null; + if (!act) return false; + const cur = a.currentStateId || a.currentState || "A"; + if (cur !== act.from) return false; + await setAssetState(assetId, act.to); + return true; +} + +function getNearbyAssetsForAgent(agent, maxDist = 1.0) { + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch); + const sp = Math.sin(pitch); + + // Full 3D forward direction (with pitch) - used for raycasting + const forward3D = _tmpV1.set(Math.sin(yaw) * cp, sp, Math.cos(yaw) * cp).normalize(); + + // Horizontal-only forward direction (yaw only, no pitch) - used for "in front" check + // BUG FIX: Previously used forward3D which includes pitch, causing dot product to be + // artificially reduced when looking up/down (cos(pitch) scaling factor) + const forwardHoriz = new THREE.Vector3(Math.sin(yaw), 0, Math.cos(yaw)).normalize(); + + const eye = _tmpV2.set(ax, ay + PLAYER_EYE_HEIGHT * 0.9, az); + + const results = []; + for (const a of assets) { + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (!obj) continue; + + // Use cached sphere center (O(1) — no vertex traversal) + const _agentSphere = new THREE.Sphere(); + if (!getAssetWorldSphere(obj, _agentSphere)) continue; + const center = _agentSphere.center; + + const dx = center.x - ax; + const dy = center.y - ay; + const dz = center.z - az; + const dist = Math.sqrt(dx * dx + dy * dy + dz * dz); + if (dist > maxDist) continue; + + // Horizontal direction to object (for "in front" check) + const toHoriz = _tmpV3.set(dx, 0, dz); + const horizLen = toHoriz.length() || 1; + toHoriz.multiplyScalar(1 / horizLen); + + // BUG FIX: Use horizontal forward for horizontal "in front" check + // Threshold relaxed from 0.92 (~23°) to 0.7 (~45°) for better usability + const inFrontHoriz = forwardHoriz.dot(toHoriz) > 0.7; + + let isLookedAt = false; + + // Debug: log for specific asset checks + const debugThis = a.title?.toLowerCase().includes('bathtub') || a.title?.toLowerCase().includes('tub'); + + if (debugThis) { + console.log(`[RAYCAST DEBUG] Checking "${a.title}" (${a.id})`); + console.log(` Eye position:`, eye.toArray().map(v => v.toFixed(2))); + console.log(` Object center:`, [center.x.toFixed(2), center.y.toFixed(2), center.z.toFixed(2)]); + console.log(` Forward3D:`, forward3D.toArray().map(v => v.toFixed(2))); + console.log(` ForwardHoriz:`, [forwardHoriz.x.toFixed(2), forwardHoriz.y.toFixed(2), forwardHoriz.z.toFixed(2)]); + console.log(` ToHoriz:`, [toHoriz.x.toFixed(2), toHoriz.y.toFixed(2), toHoriz.z.toFixed(2)]); + const dotVal = forwardHoriz.dot(toHoriz); + console.log(` Horiz dot product:`, dotVal.toFixed(3), `(need > 0.7 for inFront, > 0.3 for proximity)`); + console.log(` inFrontHoriz (>0.7):`, inFrontHoriz); + console.log(` roughlyInFront (>0.3):`, dotVal > 0.3); + } + + // For interaction purposes, we use a very lenient "roughly in front" check + // The bounding box center can be off to the side for wide objects + const roughlyInFront = forwardHoriz.dot(toHoriz) > 0.3; // ~72° cone + + if (inFrontHoriz || roughlyInFront) { + // Cheap bounding-sphere ray test instead of expensive recursive mesh raycast + const objNode = assetsGroup.getObjectByName(`asset:${a.id}`); + if (objNode) { + const _tmpSphere = new THREE.Sphere(); + if (!getAssetWorldSphere(objNode, _tmpSphere)) { /* skip */ } + _tmpSphere.radius = Math.max(_tmpSphere.radius, 0.3); + + // Test look direction against bounding sphere + const lookRay = new THREE.Ray(eye, forward3D); + if (lookRay.intersectsSphere(_tmpSphere)) { + isLookedAt = true; + } + + // Also test toward center direction (catches pitch misalignment) + if (!isLookedAt) { + const toCenter = new THREE.Vector3( + center.x - eye.x, center.y - eye.y, center.z - eye.z + ).normalize(); + const lookAlignment = forward3D.dot(toCenter); + if (lookAlignment > 0.5) { + const centerRay = new THREE.Ray(eye, toCenter); + if (centerRay.intersectsSphere(_tmpSphere)) { + isLookedAt = true; + } + } + } + } + } + + // Method 3: If close enough and roughly in front, allow interaction even if raycast fails + // This handles cases where: + // - Mesh geometry doesn't raycast well + // - Wide objects have their center off to the side + if (!isLookedAt && dist < 1.5 && roughlyInFront) { + if (debugThis) { + console.log(` Method 3 (proximity fallback): dist=${dist.toFixed(2)}, roughlyInFront=${roughlyInFront}`); + } + isLookedAt = true; + } + + if (debugThis) { + console.log(` Final isLookedAt:`, isLookedAt); + } + + const stateKey = a.currentStateId || a.currentState || "A"; + const stateObj = Array.isArray(a.states) + ? a.states.find((s) => s.id === stateKey) + : a.states?.[stateKey]; + const stateName = stateObj?.name || stateKey; + const holdStatus = isAssetHeld(a.id); + results.push({ + id: a.id, + title: a.title || "", + notes: a.notes || "", + dist, + isLookedAt, + currentState: stateKey, + currentStateName: stateName, + actions: (a.actions || []).filter((x) => x.from === stateKey).map((x) => ({ id: x.id, label: x.label, from: x.from, to: x.to })), + pickable: a.pickable || false, + isHeld: holdStatus.held, + heldBy: holdStatus.by || null, + }); + } + + results.sort((a, b) => a.dist - b.dist); + return results.slice(0, 20); +} + +function getNearbyPrimitivesForAgent(agent, maxDist = 2.5) { + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch); + const sp = Math.sin(pitch); + const forward3D = _tmpV1.set(Math.sin(yaw) * cp, sp, Math.cos(yaw) * cp).normalize(); + const forwardHoriz = _tmpV2.set(Math.sin(yaw), 0, Math.cos(yaw)).normalize(); + const eye = _tmpV3.set(ax, ay + PLAYER_EYE_HEIGHT * 0.9, az); + + const out = []; + for (const p of primitives) { + const obj = primitivesGroup.getObjectByName(`prim:${p.id}`); + if (!obj) continue; + const center = obj.getWorldPosition(new THREE.Vector3()); + const dx = center.x - ax; + const dy = center.y - ay; + const dz = center.z - az; + const dist = Math.hypot(dx, dy, dz); + if (dist > maxDist) continue; + + const toObj = new THREE.Vector3(center.x - eye.x, center.y - eye.y, center.z - eye.z).normalize(); + const toHoriz = new THREE.Vector3(dx, 0, dz); + const horizLen = toHoriz.length() || 1; + toHoriz.multiplyScalar(1 / horizLen); + const lookAlignment = forward3D.dot(toObj); + const horizAlignment = forwardHoriz.dot(toHoriz); + const isLookedAt = lookAlignment > 0.82 || (dist < 1.6 && horizAlignment > 0.35); + + out.push({ + id: p.id, + name: p.name || p.type || "primitive", + type: p.type || "primitive", + dist, + isLookedAt, + }); + } + out.sort((a, b) => a.dist - b.dist); + return out.slice(0, 20); +} + + +async function agentInteractAsset({ agent, assetId, actionId }) { + console.log(`[INTERACT] Attempting interaction: assetId="${assetId}", actionId="${actionId}"`); + + const candidates = getNearbyAssetsForAgent(agent, 1.5); // Interaction distance + + // Debug: if no candidates, show what assets exist + if (candidates.length === 0) { + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + console.log(`[INTERACT] Agent position:`, [ax.toFixed(2), ay.toFixed(2), az.toFixed(2)]); + console.log(`[INTERACT] All assets in scene:`, assets.map(a => { + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (!obj) return { id: a.id, title: a.title, inScene: false }; + const _ds = new THREE.Sphere(); + getAssetWorldSphere(obj, _ds); + const center = _ds.center; + const dx = center.x - ax; + const dy = center.y - ay; + const dz = center.z - az; + const dist = Math.sqrt(dx * dx + dy * dy + dz * dz); + return { id: a.id, title: a.title, dist: dist.toFixed(2), center: [center.x.toFixed(2), center.y.toFixed(2), center.z.toFixed(2)] }; + })); + } + + console.log(`[INTERACT] Nearby candidates:`, candidates.map(c => ({ + id: c.id, + title: c.title, + dist: c.dist.toFixed(2), + isLookedAt: c.isLookedAt, + currentState: c.currentState, + actions: c.actions.map(a => `${a.id}:${a.label}`) + }))); + + const target = candidates.find((x) => x.id === assetId); + if (!target) { + console.warn(`[INTERACT] FAIL: Asset "${assetId}" not in nearby candidates`); + return { ok: false, reason: "not-nearby" }; + } + + console.log(`[INTERACT] Found target: dist=${target.dist.toFixed(2)}m, isLookedAt=${target.isLookedAt}, currentState=${target.currentState}`); + + if (!target.isLookedAt && target.dist > 1.2) { + console.warn(`[INTERACT] FAIL: Asset "${assetId}" not looked at (isLookedAt=false)`); + return { ok: false, reason: "not-looking" }; + } + if (!target.isLookedAt && target.dist <= 1.2) { + console.log(`[INTERACT] Allowing close-range interaction despite look mismatch (dist=${target.dist.toFixed(2)}m).`); + } + + // Check if the actionId exists in the target's available actions + const availableAction = target.actions.find(a => a.id === actionId); + if (!availableAction) { + console.warn(`[INTERACT] actionId "${actionId}" not in available actions:`, target.actions); + // Try to find by label instead + const byLabel = target.actions.find(a => a.label?.toLowerCase() === actionId?.toLowerCase()); + if (byLabel) { + console.log(`[INTERACT] Found action by label match: "${byLabel.id}"`); + actionId = byLabel.id; + } + } + + const ok = await applyAssetAction(assetId, actionId); + console.log(`[INTERACT] applyAssetAction result: ${ok}`); + + if (!ok) { + // Diagnose why it failed + const asset = assets.find(a => a.id === assetId); + if (asset) { + const curState = asset.currentStateId || asset.currentState || "A"; + const allActions = asset.actions || []; + const matchingAction = allActions.find(a => a.id === actionId); + console.warn(`[INTERACT] applyAssetAction FAILED diagnosis:`, { + currentState: curState, + requestedActionId: actionId, + actionFound: !!matchingAction, + actionFromState: matchingAction?.from, + actionToState: matchingAction?.to, + fromMatchesCurrent: matchingAction?.from === curState, + allActionIds: allActions.map(a => `${a.id}(${a.from}->${a.to})`) + }); + } + } + + return { ok, reason: ok ? "ok" : "invalid-action" }; +} + +// ============================================================================ +// PLAYER INTERACTION SYSTEM +// ============================================================================ +const PLAYER_INTERACT_DISTANCE = 1.5; // Max distance player can interact with assets +const _playerInteractRaycaster = new THREE.Raycaster(); +let _interactionPopup = null; +let _currentInteractableAsset = null; +let _crosshairInteractCycleIndex = 0; +let _crosshairInteractCycleSig = ""; +let _crosshairInteractCandidates = []; + +// ============================================================================ +// PICK UP / DROP SYSTEM +// ============================================================================ +let playerHeldAsset = null; // Asset ID currently held by player +let playerHeldGroupId = null; // Group ID currently held by player +const agentHeldAssets = new Map(); // Map - assets held by each agent + +/** + * Check if an asset is currently being held by anyone + */ +function isAssetHeld(assetId) { + if (playerHeldAsset === assetId) return { held: true, by: "player" }; + for (const [agentId, heldId] of agentHeldAssets.entries()) { + if (heldId === assetId) return { held: true, by: "agent", agentId }; + } + return { held: false }; +} + +function isGroupHeld(groupId) { + if (playerHeldGroupId === groupId) return { held: true, by: "player" }; + return { held: false }; +} + +function getGroupById(groupId) { + return groups.find((g) => g.id === groupId) || null; +} + +function getGroupCentroid(groupId) { + const g = getGroupById(groupId); + if (!g || !Array.isArray(g.children) || g.children.length === 0) return null; + let cx = 0, cy = 0, cz = 0, count = 0; + for (const cid of g.children) { + const p = primitives.find((x) => x.id === cid); + const pos = p?.transform?.position; + if (!pos) continue; + cx += pos.x || 0; + cy += pos.y || 0; + cz += pos.z || 0; + count++; + } + if (count === 0) return null; + return { x: cx / count, y: cy / count, z: cz / count }; +} + +function playerPickUpGroup(groupId) { + const g = getGroupById(groupId); + if (!g) return { ok: false, reason: "not-found" }; + if (!g.pickable) return { ok: false, reason: "not-pickable" }; + if (playerHeldAsset || playerHeldGroupId) return { ok: false, reason: "hands-full" }; + const holdStatus = isGroupHeld(groupId); + if (holdStatus.held) return { ok: false, reason: "already-held", by: holdStatus.by }; + + playerHeldGroupId = groupId; + for (const cid of g.children || []) { + const mesh = primitivesGroup.getObjectByName(`prim:${cid}`); + if (mesh) mesh.visible = false; + const prim = primitives.find((p) => p.id === cid); + if (prim) removePrimitiveCollider(prim); + } + setStatus(`Picked up group: ${g.name || "group"}`); + return { ok: true }; +} + +function playerDropGroup() { + if (!playerHeldGroupId) return { ok: false, reason: "not-holding" }; + const g = getGroupById(playerHeldGroupId); + if (!g) { + playerHeldGroupId = null; + return { ok: false, reason: "not-found" }; + } + const centroid = getGroupCentroid(g.id); + if (!centroid) { + playerHeldGroupId = null; + return { ok: false, reason: "invalid-group" }; + } + // Raycast from crosshair to find drop point + const dropRay = new THREE.Raycaster(); + dropRay.setFromCamera({ x: 0, y: 0 }, camera); + dropRay.far = 6; + const candidates = []; + // Exclude held group's own meshes + const heldChildSet = new Set(g.children || []); + primitivesGroup.traverse((c) => { + if (c.isMesh && !heldChildSet.has(c.name?.replace("prim:", ""))) candidates.push(c); + }); + assetsGroup.traverse((c) => { if (c.isMesh) candidates.push(c); }); + scene.traverse((c) => { + if (c.isMesh && !candidates.includes(c) && c.parent !== assetsGroup && c.parent !== primitivesGroup) candidates.push(c); + }); + const hits = dropRay.intersectObjects(candidates, false); + let dropPos; + if (hits.length > 0) { + dropPos = { x: hits[0].point.x, y: hits[0].point.y, z: hits[0].point.z }; + } else { + const forward = new THREE.Vector3(); + camera.getWorldDirection(forward); + dropPos = { + x: camera.position.x + forward.x * 1.5, + y: 0, + z: camera.position.z + forward.z * 1.5, + }; + } + const dx = dropPos.x - centroid.x; + const dz = dropPos.z - centroid.z; + for (const cid of g.children || []) { + const prim = primitives.find((p) => p.id === cid); + if (!prim?.transform?.position) continue; + prim.transform.position.x += dx; + prim.transform.position.z += dz; + const mesh = primitivesGroup.getObjectByName(`prim:${cid}`); + if (mesh) { + mesh.position.x = prim.transform.position.x; + mesh.position.y = prim.transform.position.y; + mesh.position.z = prim.transform.position.z; + mesh.visible = true; + } + rebuildPrimitiveColliderSync(prim); + } + const droppedId = playerHeldGroupId; + playerHeldGroupId = null; + saveTagsForWorld(); + setStatus(`Dropped group: ${g.name || "group"}`); + return { ok: true, groupId: droppedId }; +} + +/** + * Pick up an asset (for player) + */ +function playerPickUpAsset(assetId) { + const asset = assets.find(a => a.id === assetId); + if (!asset) return { ok: false, reason: "not-found" }; + if (!asset.pickable) return { ok: false, reason: "not-pickable" }; + + const holdStatus = isAssetHeld(assetId); + if (holdStatus.held) return { ok: false, reason: "already-held", by: holdStatus.by }; + + if (playerHeldAsset) return { ok: false, reason: "hands-full" }; + + playerHeldAsset = assetId; + + // Hide the asset from the scene (it's now "in hand") + const obj = assetsGroup.getObjectByName(`asset:${assetId}`); + if (obj) obj.visible = false; + + // Remove collider while held + removeAssetCollider(assetId); + + console.log(`[PICKUP] Player picked up: ${asset.title || assetId}`); + setStatus(`Picked up: ${asset.title || "item"}`); + return { ok: true }; +} + +/** + * Drop the held asset (for player) + */ +function playerDropAsset() { + if (!playerHeldAsset) return { ok: false, reason: "not-holding" }; + + const asset = assets.find(a => a.id === playerHeldAsset); + if (!asset) { + playerHeldAsset = null; + return { ok: false, reason: "not-found" }; + } + + // Raycast from crosshair to find where the player is looking + const dropRay = new THREE.Raycaster(); + dropRay.setFromCamera({ x: 0, y: 0 }, camera); + dropRay.far = 6; + // Collect all scene meshes except the held asset itself + const candidates = []; + primitivesGroup.traverse((c) => { if (c.isMesh) candidates.push(c); }); + assetsGroup.traverse((c) => { + if (c.isMesh && !c.name?.includes(playerHeldAsset)) candidates.push(c); + }); + // Also include splat / collision meshes if any + scene.traverse((c) => { + if (c.isMesh && !candidates.includes(c) && c.parent !== assetsGroup && c.parent !== primitivesGroup) candidates.push(c); + }); + const hits = dropRay.intersectObjects(candidates, false); + let dropPos; + if (hits.length > 0) { + // Place at the hit point + dropPos = hits[0].point.clone(); + } else { + // Fallback: fixed distance along look direction, at ground level + const forward = new THREE.Vector3(); + camera.getWorldDirection(forward); + const fallbackDist = 1.5; + dropPos = new THREE.Vector3( + camera.position.x + forward.x * fallbackDist, + 0, + camera.position.z + forward.z * fallbackDist + ); + } + + // Update asset transform + asset.transform.position = { x: dropPos.x, y: dropPos.y, z: dropPos.z }; + + // Show and reposition the asset — traverse to ensure all children are visible + const obj = assetsGroup.getObjectByName(`asset:${playerHeldAsset}`); + if (obj) { + obj.position.copy(dropPos); + obj.visible = true; + obj.traverse((child) => { child.visible = true; }); + } else { + // Object was lost — re-instantiate from asset data + console.warn(`[DROP] 3D object missing for ${playerHeldAsset}, re-instantiating...`); + instantiateAsset(asset); + } + + // Rebuild collider + rebuildAssetCollider(playerHeldAsset); + + console.log(`[DROP] Player dropped: ${asset.title || playerHeldAsset}`); + setStatus(`Dropped: ${asset.title || "item"}`); + + const droppedId = playerHeldAsset; + playerHeldAsset = null; + saveTagsForWorld(); + + return { ok: true, assetId: droppedId }; +} + +/** + * Pick up an asset (for AI agent) + */ +function agentPickUpAsset(agent, assetId) { + const agentId = agent.id || "default"; + const asset = assets.find(a => a.id === assetId); + + if (!asset) return { ok: false, reason: "not-found" }; + if (!asset.pickable) return { ok: false, reason: "not-pickable" }; + + const holdStatus = isAssetHeld(assetId); + if (holdStatus.held) return { ok: false, reason: "already-held", by: holdStatus.by }; + + if (agentHeldAssets.has(agentId)) return { ok: false, reason: "hands-full" }; + + // Check distance + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const obj = assetsGroup.getObjectByName(`asset:${assetId}`); + if (obj) { + const _pickSphere = new THREE.Sphere(); + getAssetWorldSphere(obj, _pickSphere); + const center = _pickSphere.center; + const dist = Math.sqrt( + Math.pow(center.x - ax, 2) + + Math.pow(center.y - ay, 2) + + Math.pow(center.z - az, 2) + ); + if (dist > 1.5) return { ok: false, reason: "too-far", dist }; + } + + agentHeldAssets.set(agentId, assetId); + + // Hide the asset from the scene + if (obj) obj.visible = false; + + // Remove collider while held + removeAssetCollider(assetId); + + console.log(`[PICKUP] Agent ${agentId} picked up: ${asset.title || assetId}`); + return { ok: true }; +} + +/** + * Drop the held asset (for AI agent) + */ +function agentDropAsset(agent) { + const agentId = agent.id || "default"; + const assetId = agentHeldAssets.get(agentId); + + if (!assetId) return { ok: false, reason: "not-holding" }; + + const asset = assets.find(a => a.id === assetId); + if (!asset) { + agentHeldAssets.delete(agentId); + return { ok: false, reason: "not-found" }; + } + + // Calculate drop position (in front of agent) + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const dropDist = 0.6; + + const dropPos = new THREE.Vector3( + ax + Math.sin(yaw) * dropDist, + ay + 0.1, // Slightly above ground + az + Math.cos(yaw) * dropDist + ); + + // Update asset transform + asset.transform.position = { x: dropPos.x, y: dropPos.y, z: dropPos.z }; + + // Show and reposition the asset + const obj = assetsGroup.getObjectByName(`asset:${assetId}`); + if (obj) { + obj.position.copy(dropPos); + obj.visible = true; + } + + // Rebuild collider + rebuildAssetCollider(assetId); + + console.log(`[DROP] Agent ${agentId} dropped: ${asset.title || assetId}`); + + agentHeldAssets.delete(agentId); + saveTagsForWorld(); + + return { ok: true, assetId }; +} + +/** + * Remove collider for an asset (when picked up) + */ +function removeAssetCollider(assetId) { + const handle = _assetColliderHandles.get(assetId); + if (handle != null && rapierWorld) { + const collider = rapierWorld.getCollider(handle); + if (collider) rapierWorld.removeCollider(collider, true); + _assetColliderHandles.delete(assetId); + } +} + +/** + * Get what the player is currently holding + */ +function getPlayerHeldAsset() { + if (!playerHeldAsset) return null; + return assets.find(a => a.id === playerHeldAsset) || null; +} + +/** + * Get what an agent is currently holding + */ +function getAgentHeldAsset(agent) { + const agentId = agent.id || "default"; + const assetId = agentHeldAssets.get(agentId); + if (!assetId) return null; + return assets.find(a => a.id === assetId) || null; +} + +/** + * Get the interactable asset at the player's crosshair (center of screen). + * Returns { asset, actions, dist, canPickUp } if found, or null if nothing interactable. + */ +const _hintRayOrigin = new THREE.Vector3(); +const _hintRayDir = new THREE.Vector3(); +const _hintTmpSphere = new THREE.Sphere(); +const _hintRay = new THREE.Ray(); +const _cachedSphereCenter = new THREE.Vector3(); + +// Get the world-space bounding sphere of an asset from its cached local data. +// This is O(1) — no vertex traversal, just one matrix-vector multiply. +function getAssetWorldSphere(obj, outSphere) { + const lc = obj.userData._localSphereCenter; + const lr = obj.userData._localSphereRadius; + if (lc && lr) { + _cachedSphereCenter.copy(lc); + obj.localToWorld(_cachedSphereCenter); + const scale = obj.matrixWorld.getMaxScaleOnAxis(); + outSphere.set(_cachedSphereCenter, lr * scale); + return true; + } + return false; +} + +function getInteractableAssetCandidatesAtCrosshair() { + if (!camera) return []; + camera.getWorldPosition(_hintRayOrigin); + camera.getWorldDirection(_hintRayDir); + _hintRay.set(_hintRayOrigin, _hintRayDir); + + const maxDist = PLAYER_INTERACT_DISTANCE + 0.8; + const candidates = []; + for (const child of assetsGroup.children) { + const aid = child.name?.startsWith("asset:") ? child.name.slice(6) : null; + if (!aid) continue; + if (!getAssetWorldSphere(child, _hintTmpSphere)) continue; + _hintTmpSphere.radius = Math.max(_hintTmpSphere.radius, 0.3); + const centerDist = _hintRayOrigin.distanceTo(_hintTmpSphere.center); + if (centerDist > maxDist + _hintTmpSphere.radius) continue; + const hitPoint = _hintRay.intersectSphere(_hintTmpSphere, _tmpV1); + if (!hitPoint) continue; + const d = _hintRayOrigin.distanceTo(hitPoint); + if (d > maxDist) continue; + const toCenter = _tmpV2.copy(_hintTmpSphere.center).sub(_hintRayOrigin).normalize(); + const aim = Math.max(0, _hintRayDir.dot(toCenter)); + const score = aim * 4.0 - d * 0.45; + candidates.push({ id: aid, dist: d, aim, score }); + } + candidates.sort((a, b) => (b.score - a.score) || (a.dist - b.dist)); + return candidates.slice(0, 6); +} + +function cycleInteractableTarget(step = 1) { + const candidates = getInteractableAssetCandidatesAtCrosshair(); + if (!Array.isArray(candidates) || candidates.length <= 1) return false; + const sig = candidates.map((c) => c.id).join("|"); + if (sig !== _crosshairInteractCycleSig) { + _crosshairInteractCycleSig = sig; + _crosshairInteractCycleIndex = 0; + } + const len = candidates.length; + _crosshairInteractCycleIndex = (_crosshairInteractCycleIndex + step + len) % len; + _crosshairInteractCandidates = candidates; + return true; +} + +function getInteractableAssetAtCrosshair() { + const candidates = getInteractableAssetCandidatesAtCrosshair(); + const sig = candidates.map((c) => c.id).join("|"); + if (sig !== _crosshairInteractCycleSig) { + _crosshairInteractCycleSig = sig; + _crosshairInteractCycleIndex = 0; + } + _crosshairInteractCandidates = candidates; + const primary = candidates[_crosshairInteractCycleIndex] || null; + + if (!primary) { + // Fallback: pickable grouped shape assets + _playerInteractRaycaster.setFromCamera({ x: 0, y: 0 }, camera); + const hits = _playerInteractRaycaster.intersectObjects(primitivesGroup.children, false); + for (const hit of hits) { + if (hit.distance > PLAYER_INTERACT_DISTANCE + 0.5) continue; + const name = hit.object?.name || ""; + const m = name.match(/^prim:(.+)$/); + if (!m) continue; + const primId = m[1]; + const g = groups.find((gr) => (gr.children || []).includes(primId) && gr.pickable); + if (!g) continue; + const canPickUp = !playerHeldAsset && !playerHeldGroupId && !isGroupHeld(g.id).held; + return { kind: "group", group: g, actions: [], dist: hit.distance, canPickUp }; + } + return null; + } + + const asset = assets.find((a) => a.id === primary.id); + if (!asset) return null; + + const currentState = asset.currentStateId || asset.currentState || "A"; + const actions = (asset.actions || []).filter((act) => act.from === currentState); + const holdStatus = isAssetHeld(primary.id); + const canPickUp = asset.pickable && !holdStatus.held && !playerHeldAsset && !playerHeldGroupId; + + if (actions.length === 0 && !canPickUp) return null; + + return { + kind: "asset", + asset, + actions, + dist: primary.dist, + canPickUp, + candidateIndex: _crosshairInteractCycleIndex, + candidateCount: candidates.length, + }; +} + +/** + * Create or get the interaction popup element + */ +function getInteractionPopup() { + if (_interactionPopup) return _interactionPopup; + + _interactionPopup = document.createElement("div"); + _interactionPopup.id = "interaction-popup"; + // Styles are now in CSS, just set display none initially + _interactionPopup.style.display = "none"; + document.body.appendChild(_interactionPopup); + return _interactionPopup; +} + +/** + * Show the interaction popup with available actions + */ +function showInteractionPopup(asset, actions) { + const popup = getInteractionPopup(); + + // Build popup content + const title = asset.title || "(asset)"; + const stateObj = Array.isArray(asset.states) + ? asset.states.find((s) => s.id === (asset.currentStateId || asset.currentState)) + : null; + const stateName = stateObj?.name || ""; + + let html = `
${escapeHtml(title)}${stateName ? ` · ${escapeHtml(stateName)}` : ""}
`; + + actions.forEach((act, idx) => { + html += ``; + }); + + html += `
Press 1-${actions.length} or click · Esc to cancel
`; + + popup.innerHTML = html; + popup.style.display = "flex"; + _currentInteractableAsset = { asset, actions }; + + // Add click handlers to buttons + popup.querySelectorAll(".interact-action-btn").forEach((btn) => { + btn.addEventListener("click", async (e) => { + e.stopPropagation(); + const actionId = btn.getAttribute("data-action-id"); + + // Hide popup first + hideInteractionPopup(); + + // Execute the action + if (actionId === "__PICK_UP__") { + playerPickUpAsset(asset.id); + } else { + await executePlayerInteraction(asset.id, actionId); + } + + // Re-lock pointer after a short delay (click events can re-lock) + setTimeout(() => { + try { + controls?.lock?.(); + } catch (err) { + // Ignore + } + }, 50); + }); + // Hover effects handled in CSS + }); +} + +/** + * Hide the interaction popup + */ +function hideInteractionPopup() { + if (_interactionPopup) { + _interactionPopup.style.display = "none"; + } + _currentInteractableAsset = null; +} + +/** + * Check if interaction popup is visible + */ +function isInteractionPopupVisible() { + return _interactionPopup?.style.display === "flex"; +} + +/** + * Execute a player interaction with an asset + */ +async function executePlayerInteraction(assetId, actionId) { + // Handle special pick up action + if (actionId === "__PICK_UP__") { + const result = playerPickUpAsset(assetId); + return result.ok; + } + + const asset = assets.find((a) => a.id === assetId); + if (!asset) { + setStatus("Asset not found."); + return false; + } + + const action = (asset.actions || []).find((a) => a.id === actionId); + if (!action) { + setStatus("Action not available."); + return false; + } + + const ok = await applyAssetAction(assetId, actionId); + if (ok) { + setStatus(`${action.label || "Interacted"}: ${asset.title || "asset"}`); + } else { + setStatus("Interaction failed."); + } + return ok; +} + +/** + * Handle player interaction attempt (click or E key) + */ +async function handlePlayerInteraction() { + // If popup is already showing, do nothing (let popup handle it) + if (isInteractionPopupVisible()) { + return; + } + + // First, check if player is holding something - pressing E drops it + if (playerHeldAsset) { + playerDropAsset(); + return; + } + if (playerHeldGroupId) { + playerDropGroup(); + return; + } + + const target = getInteractableAssetAtCrosshair(); + if (!target) { + // No interactable asset at crosshair + return; + } + + const { kind, asset, group, actions, dist, canPickUp } = target; + if (kind === "group") { + if (canPickUp) playerPickUpGroup(group.id); + return; + } + + // Build combined action list (regular actions + pick up if available) + const combinedActions = [...actions]; + if (canPickUp) { + combinedActions.push({ id: "__PICK_UP__", label: "Pick up", special: true }); + } + + if (combinedActions.length === 1) { + // Single action - execute immediately + if (combinedActions[0].id === "__PICK_UP__") { + playerPickUpAsset(asset.id); + } else { + await executePlayerInteraction(asset.id, combinedActions[0].id); + } + } else if (combinedActions.length > 1) { + // Multiple actions - show selection popup + // Temporarily unlock pointer to allow clicking popup + controls?.unlock?.(); + showInteractionPopup(asset, combinedActions); + } +} + +// ============================================================================ +// END PLAYER INTERACTION SYSTEM +// ============================================================================ + +function deleteSelectedAsset() { + const a = getSelectedAsset(); + if (!a) return; + // remove collider + if (a._colliderHandle != null) { + try { + rapierWorld?.removeCollider?.(a._colliderHandle, true); + } catch {} + } + // remove visual + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (obj?.parent) obj.parent.remove(obj); + _assetBumpVelocities.delete(a.id); + assets = assets.filter((x) => x.id !== a.id); + selectedAssetId = null; + saveTagsForWorld(); + renderAssetsList(); + setStatus("Asset deleted."); +} + +async function interactSelectedAssetDebug() { + const a = getSelectedAsset(); + if (!a) return; + const state = a.currentStateId || a.currentState || "A"; + const outgoing = (a.actions || []).filter((x) => x.from === state); + const act = outgoing[0] || null; + if (!act) { + setStatus("Selected asset has no valid action from its current state."); + return; + } + const ok = await applyAssetAction(a.id, act.id); + setStatus(ok ? `Asset interacted: ${act.label}` : "Asset interaction failed."); +} + +function _newId(prefix) { + return `${prefix}${Date.now().toString(16)}${Math.random().toString(16).slice(2, 6)}`; +} + +function _cloneAssetWithFreshIds(src) { + // Ensure latest schema and interactions exist + const a = normalizeAsset(structuredClone ? structuredClone(src) : JSON.parse(JSON.stringify(src))); + + const newAssetId = _newId("asset_"); + const stateIdMap = new Map(); + const newStates = []; + + for (const s of a.states || []) { + const newSid = _newId("s"); + stateIdMap.set(s.id, newSid); + newStates.push({ + id: newSid, + name: s.name || "", + glbName: s.glbName || "", + dataBase64: s.dataBase64 || "", + interactions: [], + }); + } + + // Copy interactions (and rewrite state ids) + for (const s of a.states || []) { + const fromNew = stateIdMap.get(s.id); + const dstState = newStates.find((x) => x.id === fromNew); + if (!dstState) continue; + const ints = Array.isArray(s.interactions) ? s.interactions : []; + dstState.interactions = ints + .map((it) => ({ + id: _newId("it_"), + label: it.label || "toggle", + to: stateIdMap.get(it.to) || stateIdMap.get(s.id) || fromNew, + })) + .filter((it) => it.to && it.to !== fromNew); + } + + const curOld = a.currentStateId || a.currentState || (a.states?.[0]?.id ?? null); + const curNew = stateIdMap.get(curOld) || newStates[0]?.id || null; + + // Offset placement slightly forward so it doesn't overlap + let transform = a.transform ? structuredClone(a.transform) : null; + const obj = assetsGroup.getObjectByName(`asset:${src.id}`); + if (!transform) { + transform = { position: { x: 0, y: 0, z: 0 }, rotation: { x: 0, y: 0, z: 0 }, scale: { x: 1, y: 1, z: 1 } }; + } + if (obj) { + transform.position = { x: obj.position.x, y: obj.position.y, z: obj.position.z }; + transform.rotation = { x: obj.rotation.x, y: obj.rotation.y, z: obj.rotation.z }; + transform.scale = { x: obj.scale.x, y: obj.scale.y, z: obj.scale.z }; + } else { + transform.position = transform.position || { x: 0, y: 0, z: 0 }; + transform.rotation = transform.rotation || { x: 0, y: 0, z: 0 }; + transform.scale = transform.scale || { x: 1, y: 1, z: 1 }; + } + const fwd = new THREE.Vector3(); + camera.getWorldDirection(fwd); + const offset = 0.7; + transform.position.x += fwd.x * offset; + transform.position.y += fwd.y * offset; + transform.position.z += fwd.z * offset; + + const duplicated = { + id: newAssetId, + title: (src.title || "").trim() ? `${src.title} (copy)` : "Asset (copy)", + notes: src.notes || "", + states: newStates, + currentStateId: curNew, + transform, + actions: [], + _colliderHandle: null, + }; + + // Build actions from interactions for runtime/agent/debug use + duplicated.actions = []; + for (const s of duplicated.states) { + for (const it of s.interactions || []) { + duplicated.actions.push({ id: it.id, label: it.label, from: s.id, to: it.to }); + } + } + + return duplicated; +} + +async function duplicateSelectedAsset() { + const a = getSelectedAsset(); + if (!a) return; + const dup = _cloneAssetWithFreshIds(a); + assets.push(dup); + saveTagsForWorld(); + await instantiateAsset(dup); + renderAssetsList(); + selectAsset(dup.id); + setStatus("Asset duplicated."); +} + +async function buildRapierTriMeshColliderFromObject(obj) { + await ensureRapierLoaded(); + const verts = []; + const indices = []; + let vertBase = 0; + const tmpPos = new THREE.Vector3(); + obj.updateMatrixWorld(true); + + obj.traverse((m) => { + if (!m.isMesh) return; + const geom = m.geometry; + const posAttr = geom?.attributes?.position; + if (!posAttr) return; + const indexAttr = geom.index; + const matWorld = m.matrixWorld; + + for (let i = 0; i < posAttr.count; i++) { + tmpPos.fromBufferAttribute(posAttr, i).applyMatrix4(matWorld); + verts.push(tmpPos.x, tmpPos.y, tmpPos.z); + } + + if (indexAttr) { + for (let i = 0; i < indexAttr.count; i++) indices.push(indexAttr.getX(i) + vertBase); + } else { + for (let i = 0; i < posAttr.count; i++) indices.push(vertBase + i); + } + vertBase += posAttr.count; + }); + + if (verts.length < 9 || indices.length < 3) return null; + const desc = RAPIER.ColliderDesc.trimesh(verts, indices).setFriction(0.9); + return rapierWorld.createCollider(desc); +} + +async function rebuildAssetCollider(assetId) { + const a = assets.find((x) => x.id === assetId); + if (!a) return; + await ensureRapierLoaded(); + if (!rapierWorld || !RAPIER) return; + + // Remove existing collider + if (a._colliderHandle != null) { + try { + if (typeof a._colliderHandle === 'object' && a._colliderHandle.handle !== undefined) { + rapierWorld.removeCollider(a._colliderHandle, true); + } + } catch (e) { + console.warn(`[COLLIDER] Failed to remove collider for ${assetId}:`, e); + } + a._colliderHandle = null; + } + + const obj = assetsGroup.getObjectByName(`asset:${assetId}`); + if (!obj) return; + const collider = await buildRapierTriMeshColliderFromObject(obj); + if (collider) { + a._colliderHandle = collider; + } +} + +function removeAssetColliderHandle(asset) { + if (!asset || asset._colliderHandle == null || !rapierWorld) return; + try { + if (typeof asset._colliderHandle === "object" && asset._colliderHandle.handle !== undefined) { + rapierWorld.removeCollider(asset._colliderHandle, true); + } + } catch (e) { + console.warn(`[COLLIDER] Failed to remove collider for ${asset.id}:`, e); + } + asset._colliderHandle = null; +} + +async function rebuildAssets() { + while (assetsGroup.children.length) assetsGroup.remove(assetsGroup.children[0]); + for (const a of assets) { + try { + await instantiateAsset(a); + } catch (e) { + console.warn("Failed to rebuild asset", a?.glb?.name, e); + } + } + selectAsset(selectedAssetId); +} + +// ============================================================================= +// PRIMITIVES – Parametric Shape System (Level Editor) +// ============================================================================= + +function createPrimitiveGeometry(type, dims) { + dims = dims || {}; + const num = (v, fallback) => { + const n = Number(v); + return Number.isFinite(n) ? n : fallback; + }; + const degToRad = (deg, fallback = 0) => (Number.isFinite(deg) ? deg : fallback) * Math.PI / 180; + const clampInt = (v, fallback, min = 1) => Math.max(min, Math.floor(Number(v) || fallback)); + const clamp01 = (v, fallback = 0) => { + const n = Number(v); + if (!Number.isFinite(n)) return fallback; + return Math.max(0, Math.min(1, n)); + }; + switch (type) { + case "box": { + const width = Math.max(0.01, num(dims.width, 1)); + const height = Math.max(0.01, num(dims.height, 1)); + const depth = Math.max(0.01, num(dims.depth, 1)); + const edgeRadius = Math.max(0, num(dims.edgeRadius, 0)); + if (edgeRadius > 0) { + const radius = Math.min(edgeRadius, width * 0.5, height * 0.5, depth * 0.5); + const edgeSegments = clampInt(dims.edgeSegments, 4, 1); + return new RoundedBoxGeometry(width, height, depth, edgeSegments, radius); + } + return new THREE.BoxGeometry( + width, + height, + depth, + clampInt(dims.widthSegments, 1, 1), + clampInt(dims.heightSegments, 1, 1), + clampInt(dims.depthSegments, 1, 1) + ); + } + case "sphere": + return new THREE.SphereGeometry( + Math.max(0.01, num(dims.radius, 0.5)), + clampInt(dims.widthSegments, 32, 3), + clampInt(dims.heightSegments, 16, 2), + degToRad(num(dims.phiStartDeg, 0), 0), + degToRad(num(dims.phiLengthDeg, 360), 360), + degToRad(num(dims.thetaStartDeg, 0), 0), + degToRad(num(dims.thetaLengthDeg, 180), 180) + ); + case "cylinder": + return new THREE.CylinderGeometry( + Math.max(0.01, num(dims.radiusTop, 0.5)), + Math.max(0.01, num(dims.radiusBottom, 0.5)), + Math.max(0.01, num(dims.height, 1)), + clampInt(dims.radialSegments, 32, 3), + clampInt(dims.heightSegments, 1, 1), + clamp01(dims.openEnded, 0) >= 0.5 + ); + case "cone": + return new THREE.ConeGeometry( + Math.max(0.01, num(dims.radius, 0.5)), + Math.max(0.01, num(dims.height, 1)), + clampInt(dims.radialSegments, 32, 3), + clampInt(dims.heightSegments, 1, 1), + clamp01(dims.openEnded, 0) >= 0.5 + ); + case "torus": + return new THREE.TorusGeometry( + Math.max(0.01, num(dims.radius, 0.5)), + Math.max(0.01, num(dims.tube, 0.15)), + clampInt(dims.radialSegments, 16, 3), + clampInt(dims.tubularSegments, 48, 3), + degToRad(num(dims.arcDeg, 360), 360) + ); + case "plane": + return new THREE.PlaneGeometry( + Math.max(0.01, num(dims.width, 2)), + Math.max(0.01, num(dims.height, 2)), + clampInt(dims.widthSegments, 1, 1), + clampInt(dims.heightSegments, 1, 1) + ); + default: + return new THREE.BoxGeometry(1, 1, 1); + } +} + +const _textureLoader = new THREE.TextureLoader(); +const _textureCache = new Map(); // dataUrl → THREE.Texture + +function createPrimitiveMaterial(mat) { + mat = mat || {}; + const uv = mat.uvTransform || {}; + const clamp01 = (v, fallback = 0) => { + const n = Number(v); + if (!Number.isFinite(n)) return fallback; + return Math.max(0, Math.min(1, n)); + }; + const hardness = clamp01(mat.hardness, 0); + const fluffiness = clamp01(mat.fluffiness, 0); + const params = { + color: new THREE.Color(mat.color || "#808080"), + roughness: mat.softness ?? mat.roughness ?? 0.7, + metalness: mat.metalness ?? 0.0, + specularIntensity: mat.specularIntensity ?? 1.0, + specularColor: new THREE.Color(mat.specularColor || "#ffffff"), + envMapIntensity: mat.envMapIntensity ?? 1.0, + opacity: mat.opacity ?? 1.0, + transparent: (mat.opacity ?? 1.0) < 1 || (mat.transmission ?? 0) > 0, + transmission: mat.transmission ?? 0.0, + ior: mat.ior ?? 1.45, + thickness: mat.thickness ?? 0.0, + attenuationColor: new THREE.Color(mat.attenuationColor || "#ffffff"), + attenuationDistance: Math.max(0.01, mat.attenuationDistance ?? 1.0), + iridescence: mat.iridescence ?? 0.0, + iridescenceIOR: mat.ior ?? 1.45, + emissive: new THREE.Color(mat.emissive || "#000000"), + emissiveIntensity: mat.emissiveIntensity ?? 0.0, + clearcoat: Math.max(mat.clearcoat ?? 0.0, hardness * 0.85), + clearcoatRoughness: Math.min(mat.clearcoatRoughness ?? 0.0, 1 - hardness * 0.8), + sheen: fluffiness, + sheenRoughness: 0.9, + sheenColor: new THREE.Color(mat.sheenColor || mat.color || "#808080"), + side: mat.doubleSided === false ? THREE.FrontSide : THREE.DoubleSide, + flatShading: mat.flatShading === true, + wireframe: mat.wireframe === true, + alphaTest: clamp01(mat.alphaCutoff, 0), + depthWrite: (mat.opacity ?? 1.0) >= 1 && (mat.transmission ?? 0) <= 0, + }; + if (mat.textureDataUrl) { + let baseTex = _textureCache.get(mat.textureDataUrl); + if (!baseTex) { + baseTex = _textureLoader.load(mat.textureDataUrl); + baseTex.colorSpace = THREE.SRGBColorSpace; + baseTex.wrapS = baseTex.wrapT = THREE.RepeatWrapping; + _textureCache.set(mat.textureDataUrl, baseTex); + } + const tex = baseTex.clone(); + tex.needsUpdate = true; + tex.repeat.set(uv.repeatX ?? 1, uv.repeatY ?? 1); + tex.offset.set(uv.offsetX ?? 0, uv.offsetY ?? 0); + tex.rotation = ((uv.rotationDeg ?? 0) * Math.PI) / 180; + tex.center.set(0.5, 0.5); + const textureSoftness = clamp01(mat.textureSoftness, 0.25); + const textureHardness = clamp01(mat.textureHardness, 0.5); + const maxAniso = renderer?.capabilities?.getMaxAnisotropy?.() || 1; + const targetAniso = Math.max(1, Math.round(1 + textureHardness * (maxAniso - 1))); + tex.anisotropy = Math.max(1, Math.round(targetAniso * (1 - textureSoftness * 0.85))); + tex.minFilter = textureSoftness > 0.6 ? THREE.LinearMipmapLinearFilter : THREE.LinearMipmapNearestFilter; + tex.magFilter = textureSoftness > 0.75 ? THREE.LinearFilter : (textureHardness > 0.9 ? THREE.NearestFilter : THREE.LinearFilter); + tex.generateMipmaps = true; + params.map = tex; + } + return new THREE.MeshPhysicalMaterial(params); +} + +function sanitizePrimitiveCutouts(cutouts) { + if (!Array.isArray(cutouts)) return []; + const out = []; + for (const c of cutouts) { + if (!c || typeof c !== "object") continue; + if (!Array.isArray(c.targetToSourceMatrix) || c.targetToSourceMatrix.length !== 16) continue; + const type = String(c.type || ""); + if (!["box", "sphere", "cylinder", "cone", "torus"].includes(type)) continue; + out.push({ + type, + targetToSourceMatrix: c.targetToSourceMatrix.map((n) => Number(n) || 0), + dimensions: { ...(c.dimensions || {}) }, + }); + if (out.length >= 8) break; + } + return out; +} + +function applyPrimitiveCutoutShader(mesh, primData) { + if (!mesh?.material?.isMeshPhysicalMaterial) return; + const cutouts = sanitizePrimitiveCutouts(primData?.cutouts); + if (!cutouts.length) return; + const mat = mesh.material; + const maxCuts = 8; + const cutMatrices = Array.from({ length: maxCuts }, () => new THREE.Matrix4()); + const cutA = Array.from({ length: maxCuts }, () => new THREE.Vector4(0, 0, 0, 0)); + const cutB = Array.from({ length: maxCuts }, () => new THREE.Vector4(0, 0, 0, 0)); + const typeCodeFor = (t) => (t === "sphere" ? 1 : t === "box" ? 2 : t === "cylinder" ? 3 : t === "cone" ? 4 : t === "torus" ? 5 : 0); + for (let i = 0; i < cutouts.length && i < maxCuts; i++) { + const c = cutouts[i]; + cutMatrices[i].fromArray(c.targetToSourceMatrix); + const d = c.dimensions || {}; + switch (c.type) { + case "sphere": + cutA[i].set(Number(d.radius) || 0.5, 0, 0, typeCodeFor(c.type)); + break; + case "box": + cutA[i].set((Number(d.width) || 1) * 0.5, (Number(d.height) || 1) * 0.5, (Number(d.depth) || 1) * 0.5, typeCodeFor(c.type)); + break; + case "cylinder": + case "cone": + cutA[i].set(Math.max(Number(d.radiusTop) || Number(d.radius) || 0.5, Number(d.radiusBottom) || Number(d.radius) || 0.5), Number(d.height) || 1, 0, typeCodeFor(c.type)); + break; + case "torus": + cutA[i].set(Number(d.radius) || 0.5, Number(d.tube) || 0.15, 0, typeCodeFor(c.type)); + break; + default: + break; + } + } + mat.onBeforeCompile = (shader) => { + shader.uniforms.uCutoutCount = { value: cutouts.length }; + shader.uniforms.uCutoutInv = { value: cutMatrices }; + shader.uniforms.uCutoutA = { value: cutA }; + shader.uniforms.uCutoutB = { value: cutB }; + shader.vertexShader = ` +varying vec3 vPrimLocalPos; +${shader.vertexShader}`.replace( + "#include ", + `#include +vPrimLocalPos = position;` + ); + shader.fragmentShader = ` +uniform int uCutoutCount; +uniform mat4 uCutoutInv[8]; +uniform vec4 uCutoutA[8]; +uniform vec4 uCutoutB[8]; +varying vec3 vPrimLocalPos; + +float sdfBox(vec3 p, vec3 b) { + vec3 q = abs(p) - b; + return length(max(q, 0.0)) + min(max(q.x, max(q.y, q.z)), 0.0); +} +float sdfSphere(vec3 p, float r) { return length(p) - r; } +float sdfCylinderY(vec3 p, float r, float h) { + vec2 d = abs(vec2(length(p.xz), p.y)) - vec2(r, h * 0.5); + return min(max(d.x, d.y), 0.0) + length(max(d, 0.0)); +} +float sdfTorus(vec3 p, float r, float t) { + vec2 q = vec2(length(p.xz) - r, p.y); + return length(q) - t; +} +${shader.fragmentShader}`.replace( + "#include ", + `#include +for (int i = 0; i < 8; i++) { + if (i >= uCutoutCount) break; + vec3 lp = (uCutoutInv[i] * vec4(vPrimLocalPos, 1.0)).xyz; + float typ = uCutoutA[i].w; + float d = 1e6; + if (typ < 1.5) d = sdfSphere(lp, uCutoutA[i].x); + else if (typ < 2.5) d = sdfBox(lp, vec3(uCutoutA[i].x, uCutoutA[i].y, uCutoutA[i].z)); + else if (typ < 3.5) d = sdfCylinderY(lp, uCutoutA[i].x, uCutoutA[i].y); + else if (typ < 4.5) d = sdfCylinderY(lp, uCutoutA[i].x, uCutoutA[i].y); + else d = sdfTorus(lp, uCutoutA[i].x, uCutoutA[i].y); + if (d < 0.0) discard; +}` + ); + }; + mat.customProgramCacheKey = () => `cutouts:${cutouts.length}:${cutouts.map((c) => c.type).join(",")}`; + mat.needsUpdate = true; +} + +function disposePrimitiveMaterial(material) { + if (!material) return; + const mats = Array.isArray(material) ? material : [material]; + for (const m of mats) { + if (!m) continue; + const maps = [ + "map", + "alphaMap", + "aoMap", + "normalMap", + "roughnessMap", + "metalnessMap", + "emissiveMap", + "clearcoatMap", + "clearcoatRoughnessMap", + "transmissionMap", + "thicknessMap", + ]; + for (const key of maps) { + const tex = m[key]; + if (tex?.isTexture) tex.dispose(); + } + m.dispose?.(); + } +} + +// Deferred collider queue — colliders are only created at a safe frame boundary +const _pendingColliderBuilds = []; + +function flushPendingColliderBuilds() { + if (!rapierWorld || !worldBody || _pendingColliderBuilds.length === 0) return; + while (_pendingColliderBuilds.length > 0) { + const prim = _pendingColliderBuilds.shift(); + // Verify the primitive still exists and still wants physics + if (primitives.includes(prim) && prim.physics !== false) { + rebuildPrimitiveColliderSync(prim); + } + } +} + +function instantiatePrimitive(prim) { + // Remove existing + const existing = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (existing) { + existing.geometry?.dispose(); + disposePrimitiveMaterial(existing.material); + primitivesGroup.remove(existing); + } + + const geom = createPrimitiveGeometry(prim.type, prim.dimensions); + const mat = createPrimitiveMaterial(prim.material); + const mesh = new THREE.Mesh(geom, mat); + applyPrimitiveCutoutShader(mesh, prim); + mesh.name = `prim:${prim.id}`; + mesh.userData.primitiveId = prim.id; + mesh.userData.isPrimitive = true; + // Default both to true — shapes should always participate in shadows + mesh.castShadow = prim.castShadow !== false; + mesh.receiveShadow = prim.receiveShadow !== false; + + const tr = prim.transform || {}; + if (tr.position) mesh.position.set(tr.position.x, tr.position.y, tr.position.z); + if (tr.rotation) mesh.rotation.set(tr.rotation.x, tr.rotation.y, tr.rotation.z); + if (tr.scale) mesh.scale.set(tr.scale.x ?? 1, tr.scale.y ?? 1, tr.scale.z ?? 1); + + primitivesGroup.add(mesh); + + // Build collider — if Rapier is ready, do it now; otherwise queue it + if (prim.physics !== false) { + if (rapierWorld && worldBody) { + rebuildPrimitiveColliderSync(prim); + } else { + // Queue for deferred build once Rapier is ready + _pendingColliderBuilds.push(prim); + // Kick off Rapier init (non-blocking, collider will be built by flush) + ensureRapierLoaded(); + } + } +} + +// Safely remove a primitive's existing collider from the Rapier world +function removePrimitiveCollider(prim) { + if (prim._colliderHandle == null || !rapierWorld) return; + try { + if (typeof prim._colliderHandle === "object" && prim._colliderHandle.handle !== undefined) { + rapierWorld.removeCollider(prim._colliderHandle, true); + } + } catch (e) { + console.warn(`[COLLIDER] Primitive collider remove failed for ${prim.id}:`, e); + } + prim._colliderHandle = null; +} + +// SYNCHRONOUS collider creation for native Rapier shapes. +// Only falls back to async for trimesh (torus, plane). +function rebuildPrimitiveColliderSync(prim) { + if (!prim) return; + // Rapier must already be loaded for sync creation + if (!rapierWorld || !RAPIER || !worldBody) return; + + removePrimitiveCollider(prim); + if (prim.physics === false) return; + + const mesh = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (!mesh) return; + + const dims = prim.dimensions || {}; + const s = prim.transform?.scale || { x: 1, y: 1, z: 1 }; + const pos = prim.transform?.position || { x: 0, y: 0, z: 0 }; + const rot = prim.transform?.rotation || { x: 0, y: 0, z: 0 }; + + // Clamp all half-extents / radii to a safe minimum to avoid WASM traps + const clamp = (v) => Math.max(v, 0.001); + + let desc = null; + + // Use native Rapier collision shapes – far more compute-efficient than trimesh + switch (prim.type) { + case "box": + desc = RAPIER.ColliderDesc.cuboid( + clamp(((dims.width || 1) * (s.x ?? 1)) / 2), + clamp(((dims.height || 1) * (s.y ?? 1)) / 2), + clamp(((dims.depth || 1) * (s.z ?? 1)) / 2) + ); + break; + case "sphere": + desc = RAPIER.ColliderDesc.ball( + clamp((dims.radius || 0.5) * Math.max(s.x ?? 1, s.y ?? 1, s.z ?? 1)) + ); + break; + case "cylinder": + desc = RAPIER.ColliderDesc.cylinder( + clamp(((dims.height || 1) * (s.y ?? 1)) / 2), + clamp(Math.max(dims.radiusTop ?? 0.5, dims.radiusBottom ?? 0.5) * Math.max(s.x ?? 1, s.z ?? 1)) + ); + break; + case "cone": + desc = RAPIER.ColliderDesc.cone( + clamp(((dims.height || 1) * (s.y ?? 1)) / 2), + clamp((dims.radius || 0.5) * Math.max(s.x ?? 1, s.z ?? 1)) + ); + break; + case "plane": { + // PlaneGeometry lies in the XY plane (normal along +Z), so make the + // cuboid thin in Z to match the visual exactly. No rotation offset needed. + const pw = clamp(((dims.width || 2) * (s.x ?? 1)) / 2); + const ph = clamp(((dims.height || 2) * (s.y ?? 1)) / 2); + desc = RAPIER.ColliderDesc.cuboid(pw, ph, 0.005); + break; // fall through to the standard rotation/translation below + } + case "torus": { + // Torus: use trimesh async fallback (deferred, won't block) + rebuildPrimitiveColliderAsync(prim); + return; + } + default: + return; + } + + if (desc) { + desc.setTranslation(pos.x, pos.y, pos.z); + const euler = new THREE.Euler(rot.x, rot.y, rot.z); + const quat = new THREE.Quaternion().setFromEuler(euler); + desc.setRotation({ x: quat.x, y: quat.y, z: quat.z, w: quat.w }); + desc.setFriction(0.9); + try { + const collider = rapierWorld.createCollider(desc); + prim._colliderHandle = collider; + } catch (e) { + console.warn(`[COLLIDER] Failed to create primitive collider for ${prim.type}:`, e); + } + } +} + +// Async fallback only used for torus (trimesh) +async function rebuildPrimitiveColliderAsync(prim) { + if (!prim) return; + await ensureRapierLoaded(); + if (!rapierWorld || !RAPIER || !worldBody) return; + removePrimitiveCollider(prim); + if (prim.physics === false) return; + const mesh = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (!mesh) return; + try { + const collider = await buildRapierTriMeshColliderFromObject(mesh); + if (collider) prim._colliderHandle = collider; + } catch (e) { + console.warn(`[COLLIDER] Trimesh fallback failed for ${prim.id}:`, e); + } +} + +// Keep old name as alias for callers (e.g. dimension/transform change handlers) +function rebuildPrimitiveCollider(primId) { + const prim = primitives.find((p) => p.id === primId); + if (prim) rebuildPrimitiveColliderSync(prim); +} + +function addPrimitiveAtCrosshair(type) { + const spawnPos = getPlacementAtCrosshair({ raycastDistance: 250, surfaceOffset: 0.5 }).position; + addPrimitiveAtPosition(type, spawnPos); +} + +function addPrimitiveAtPosition(type, spawnPos) { + const prim = { + id: randId(), + type, + name: type.charAt(0).toUpperCase() + type.slice(1), + notes: "", + tags: [], // string tags for filtering / grouping + state: "static", // static | dynamic | interactable | trigger | decoration + metadata: {}, // arbitrary key-value pairs + dimensions: { ...(PRIMITIVE_DEFAULTS[type] || PRIMITIVE_DEFAULTS.box) }, + transform: { + position: spawnPos, + rotation: { x: 0, y: 0, z: 0 }, + scale: { x: 1, y: 1, z: 1 }, + }, + material: { + color: "#808080", + roughness: 0.7, + softness: 0.7, + hardness: 0.0, + fluffiness: 0.0, + metalness: 0.0, + specularIntensity: 1.0, + specularColor: "#ffffff", + envMapIntensity: 1.0, + opacity: 1.0, + transmission: 0.0, + ior: 1.45, + thickness: 0.0, + attenuationColor: "#ffffff", + attenuationDistance: 1.0, + iridescence: 0.0, + emissive: "#000000", + emissiveIntensity: 0.0, + clearcoat: 0.0, + clearcoatRoughness: 0.0, + alphaCutoff: 0.0, + textureSoftness: 0.25, + textureHardness: 0.5, + doubleSided: true, + flatShading: false, + wireframe: false, + uvTransform: { repeatX: 1, repeatY: 1, offsetX: 0, offsetY: 0, rotationDeg: 0 }, + textureDataUrl: null, + }, + physics: true, + castShadow: true, + receiveShadow: true, + }; + + primitives.push(prim); + instantiatePrimitive(prim); + saveTagsForWorld(); + renderPrimitivesList(); + selectPrimitive(prim.id); + setStatus(`${prim.name} placed. Use transform tools to position.`); +} + +function selectPrimitive(id) { + selectedPrimitiveId = id; + if (id) { + selectedAssetId = null; + } + renderPrimitivesList(); +} + +function getPlacementAtCrosshair({ raycastDistance = 500, fallbackDistance = 3, surfaceOffset = 0.02 } = {}) { + const hit = rapierRaycastFromCamera(raycastDistance); + if (hit) { + const n = hit.normal + ? new THREE.Vector3(hit.normal.x, hit.normal.y, hit.normal.z).normalize() + : new THREE.Vector3(0, 1, 0); + return { + hit: true, + point: { x: hit.point.x, y: hit.point.y, z: hit.point.z }, + normal: { x: n.x, y: n.y, z: n.z }, + position: { + x: hit.point.x + n.x * surfaceOffset, + y: hit.point.y + n.y * surfaceOffset, + z: hit.point.z + n.z * surfaceOffset, + }, + }; + } + + // If no collider is hit, place directly in front of the crosshair. + const dir = camera.getWorldDirection(new THREE.Vector3()); + const p = camera.getWorldPosition(new THREE.Vector3()); + return { + hit: false, + point: { + x: p.x + dir.x * fallbackDistance, + y: p.y + dir.y * fallbackDistance, + z: p.z + dir.z * fallbackDistance, + }, + normal: { x: 0, y: 1, z: 0 }, + position: { + x: p.x + dir.x * fallbackDistance, + y: p.y + dir.y * fallbackDistance, + z: p.z + dir.z * fallbackDistance, + }, + }; +} + +function getPlacementFromAgentView(agent, { raycastDistance = 500, fallbackDistance = 2.5, surfaceOffset = 0.02 } = {}) { + const [ax, ay, az] = agent?.getPosition?.() || [0, 0, 0]; + const yaw = agent?.group?.rotation?.y ?? 0; + const pitch = typeof agent?.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch); + const sp = Math.sin(pitch); + const dx = Math.sin(yaw) * cp; + const dy = sp; + const dz = Math.cos(yaw) * cp; + const eyeY = ay + PLAYER_EYE_HEIGHT * 0.9; + + if (rapierWorld && RAPIER) { + const ray = new RAPIER.Ray({ x: ax, y: eyeY, z: az }, { x: dx, y: dy, z: dz }); + const hit = rapierWorld.queryPipeline.castRayAndGetNormal( + rapierWorld.bodies, + rapierWorld.colliders, + ray, + raycastDistance, + false, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + agent?.collider?.handle + ); + if (hit) { + const toi = hit.toi ?? hit.timeOfImpact ?? 0; + const px = ax + dx * toi; + const py = eyeY + dy * toi; + const pz = az + dz * toi; + const n = hit.normal + ? new THREE.Vector3(hit.normal.x, hit.normal.y, hit.normal.z).normalize() + : new THREE.Vector3(0, 1, 0); + return { + hit: true, + point: { x: px, y: py, z: pz }, + normal: { x: n.x, y: n.y, z: n.z }, + position: { + x: px + n.x * surfaceOffset, + y: py + n.y * surfaceOffset, + z: pz + n.z * surfaceOffset, + }, + }; + } + } + + return { + hit: false, + point: { + x: ax + dx * fallbackDistance, + y: eyeY + dy * fallbackDistance, + z: az + dz * fallbackDistance, + }, + normal: { x: 0, y: 1, z: 0 }, + position: { + x: ax + dx * fallbackDistance, + y: eyeY + dy * fallbackDistance, + z: az + dz * fallbackDistance, + }, + }; +} + + +function getSelectedPrimitive() { + return primitives.find((p) => p.id === selectedPrimitiveId) || null; +} + +function buildPrimitiveCutoutFromSource(targetId, sourceId) { + const targetPrim = primitives.find((p) => p.id === targetId); + const sourcePrim = primitives.find((p) => p.id === sourceId); + if (!targetPrim || !sourcePrim) return null; + const targetMesh = primitivesGroup.getObjectByName(`prim:${targetId}`); + const sourceMesh = primitivesGroup.getObjectByName(`prim:${sourceId}`); + if (!targetMesh || !sourceMesh) return null; + targetMesh.updateMatrixWorld(true); + sourceMesh.updateMatrixWorld(true); + const targetWorldInv = new THREE.Matrix4().copy(targetMesh.matrixWorld).invert(); + const sourceInTarget = new THREE.Matrix4().multiplyMatrices(targetWorldInv, sourceMesh.matrixWorld); + const targetToSource = new THREE.Matrix4().copy(sourceInTarget).invert(); + return { + id: randId(), + type: sourcePrim.type, + targetToSourceMatrix: targetToSource.elements.slice(), + dimensions: { ...(sourcePrim.dimensions || {}) }, + }; +} + + +function getOverlappingPrimitiveIds(targetId) { + const targetMesh = primitivesGroup.getObjectByName(`prim:${targetId}`); + if (!targetMesh) return []; + const targetBox = new THREE.Box3().setFromObject(targetMesh).expandByScalar(0.01); + const out = []; + for (const p of primitives) { + if (p.id === targetId) continue; + const mesh = primitivesGroup.getObjectByName(`prim:${p.id}`); + if (!mesh) continue; + const box = new THREE.Box3().setFromObject(mesh); + if (box.isEmpty()) continue; + if (targetBox.intersectsBox(box)) out.push(p.id); + } + return out; +} + +function persistSelectedPrimitiveTransform() { + const prim = getSelectedPrimitive(); + if (!prim) return; + const obj = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (!obj) return; + prim.transform = { + position: { x: obj.position.x, y: obj.position.y, z: obj.position.z }, + rotation: { x: obj.rotation.x, y: obj.rotation.y, z: obj.rotation.z }, + scale: { x: obj.scale.x, y: obj.scale.y, z: obj.scale.z }, + }; + saveTagsForWorld(); + rebuildPrimitiveColliderSync(prim); +} + +function deletePrimitive(id) { + const idx = primitives.findIndex((p) => p.id === id); + if (idx === -1) return; + const prim = primitives[idx]; + + // Remove collider safely + removePrimitiveCollider(prim); + + // Remove visual + const obj = primitivesGroup.getObjectByName(`prim:${id}`); + if (obj) { + obj.geometry?.dispose(); + disposePrimitiveMaterial(obj.material); + primitivesGroup.remove(obj); + } + + primitives.splice(idx, 1); + if (selectedPrimitiveId === id) { + selectedPrimitiveId = null; + } + saveTagsForWorld(); + renderPrimitivesList(); + setStatus("Primitive deleted."); +} + +function duplicatePrimitive(id) { + const src = primitives.find((p) => p.id === id); + if (!src) return; + // Strip runtime objects (collider handle has circular refs) before deep clone + const { _colliderHandle, ...serializable } = src; + const clone = JSON.parse(JSON.stringify(serializable)); + clone.id = randId(); + clone.name = src.name + " copy"; + clone._colliderHandle = null; + // Offset slightly + if (clone.transform?.position) { + clone.transform.position.x += 1; + } + primitives.push(clone); + instantiatePrimitive(clone); + saveTagsForWorld(); + renderPrimitivesList(); + selectPrimitive(clone.id); + setStatus("Primitive duplicated."); +} + +function updatePrimitiveMaterial(primId) { + const prim = primitives.find((p) => p.id === primId); + if (!prim) return; + const mesh = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (!mesh) return; + disposePrimitiveMaterial(mesh.material); + mesh.material = createPrimitiveMaterial(prim.material); + applyPrimitiveCutoutShader(mesh, prim); +} + +function updatePrimitiveDimensions(primId) { + const prim = primitives.find((p) => p.id === primId); + if (!prim) return; + const mesh = primitivesGroup.getObjectByName(`prim:${prim.id}`); + if (!mesh) return; + mesh.geometry?.dispose(); + mesh.geometry = createPrimitiveGeometry(prim.type, prim.dimensions); + rebuildPrimitiveCollider(prim.id); +} + +function renderPrimitivesList() { + // No-op in sim-only mode (editor primitives list not present) +} + +function rebuildAllPrimitives() { + // Remove all existing colliders first + for (const p of primitives) { + removePrimitiveCollider(p); + } + // Remove all visual meshes + while (primitivesGroup.children.length) { + const c = primitivesGroup.children[0]; + c.geometry?.dispose(); + disposePrimitiveMaterial(c.material); + primitivesGroup.remove(c); + } + // Rebuild all + for (const p of primitives) { + try { + instantiatePrimitive(p); + } catch (e) { + console.warn("Failed to rebuild primitive", p.id, e); + } + } +} + +// ============================================================================= +// EDITOR LIGHTS – User-placed lights with visible proxy icons +// ============================================================================= + +function instantiateEditorLight(lightData) { + // Remove existing + removeEditorLightObjects(lightData.id); + + let lightObj; + const color = new THREE.Color(lightData.color || "#ffffff"); + const intensity = lightData.intensity ?? 1.0; + + switch (lightData.type) { + case "point": + lightObj = new THREE.PointLight(color, intensity, lightData.distance || 0); + break; + case "spot": { + lightObj = new THREE.SpotLight( + color, + intensity, + lightData.distance || 0, + lightData.angle ?? Math.PI / 4, + lightData.penumbra ?? 0.1 + ); + const tgt = lightData.target || { x: 0, y: 0, z: 0 }; + lightObj.target.position.set(tgt.x, tgt.y, tgt.z); + lightsGroup.add(lightObj.target); + break; + } + case "directional": + default: { + lightObj = new THREE.DirectionalLight(color, intensity); + const tgt = lightData.target || { x: 0, y: 0, z: 0 }; + lightObj.target.position.set(tgt.x, tgt.y, tgt.z); + lightsGroup.add(lightObj.target); + break; + } + } + + const pos = lightData.position || { x: 5, y: 10, z: 5 }; + lightObj.position.set(pos.x, pos.y, pos.z); + lightObj.castShadow = lightData.castShadow ?? false; + lightObj.name = `light:${lightData.id}`; + lightObj.userData.editorLightId = lightData.id; + lightObj.userData.isEditorLight = true; + + // Configure shadow map for this light (only renders when castShadow=true). + // Use 512 for point/spot (6-face cubemap = expensive) and 1024 for directional. + if (lightObj.shadow) { + const res = lightObj.isDirectionalLight ? 1024 : 512; + lightObj.shadow.mapSize.width = res; + lightObj.shadow.mapSize.height = res; + lightObj.shadow.bias = -0.003; + if (lightObj.shadow.camera) { + if (lightObj.isDirectionalLight) { + lightObj.shadow.camera.near = 0.5; + lightObj.shadow.camera.far = 50; + lightObj.shadow.camera.left = -20; + lightObj.shadow.camera.right = 20; + lightObj.shadow.camera.top = 20; + lightObj.shadow.camera.bottom = -20; + } else { + lightObj.shadow.camera.near = 0.5; + lightObj.shadow.camera.far = Math.min(lightData.distance || 30, 30); + } + } + } + + lightsGroup.add(lightObj); + lightData._lightObj = lightObj; + lightData._proxyObj = null; + lightData._helperObj = null; +} + +function removeEditorLightObjects(id) { + const names = [`light:${id}`, `lightHelper:${id}`, `lightProxy:${id}`]; + for (const n of names) { + const obj = lightsGroup.getObjectByName(n); + if (obj) { + // Remove target if it exists (directional/spot) + if (obj.target && obj.target.parent) obj.target.parent.remove(obj.target); + // Dispose children meshes + obj.traverse?.((c) => { + if (c.geometry) c.geometry.dispose(); + if (c.material) { + if (c.material.map) c.material.map.dispose(); + c.material.dispose(); + } + }); + lightsGroup.remove(obj); + } + } +} + + + +function rebuildAllEditorLights() { + // Remove all light objects + while (lightsGroup.children.length) { + const c = lightsGroup.children[0]; + c.traverse?.((m) => { m.geometry?.dispose(); m.material?.dispose(); }); + lightsGroup.remove(c); + } + for (const ld of editorLights) { + ld._lightObj = null; + ld._helperObj = null; + ld._proxyObj = null; + try { + instantiateEditorLight(ld); + } catch (e) { + console.warn("Failed to rebuild light", ld.id, e); + } + } + // Enable/disable the renderer shadow map based on whether any light casts shadows + syncShadowMapEnabled(); +} + +// ============================================================================= +// DETAILS PANEL & TRANSFORM XYZ – UE-style unified properties +// ============================================================================= + +const RAD2DEG = 180 / Math.PI; +const DEG2RAD = Math.PI / 180; + +// Dynamically enable/disable the shadow map system. +// When no light casts shadows, the renderer skips ALL shadow work (zero overhead). +function enforceShadowSamplerBudget() { + // Prevent WebGL shader validation failures: + // "texture image units count exceeds MAX_TEXTURE_IMAGE_UNITS" + // Point-light shadows are especially expensive (cube map = ~6 samplers). + const budget = 8; + const costFor = (lightObj) => (lightObj?.isPointLight ? 6 : 1); + + const candidates = []; + for (const sl of sceneLights) { + if (!sl?.obj || sl.obj.visible === false) continue; + if (!sl.obj.castShadow) continue; + candidates.push({ obj: sl.obj, source: "scene", meta: sl }); + } + for (const ld of editorLights) { + if (!ld?._lightObj || ld._lightObj.visible === false) continue; + if (!ld._lightObj.castShadow) continue; + candidates.push({ obj: ld._lightObj, source: "editor", meta: ld }); + } + + // Prefer non-point shadow lights first (directional/spot), then points. + candidates.sort((a, b) => { + const ac = costFor(a.obj); + const bc = costFor(b.obj); + if (ac !== bc) return ac - bc; // cheaper first + return 0; + }); + + let used = 0; + for (const c of candidates) { + const cost = costFor(c.obj); + if (used + cost <= budget) { + used += cost; + continue; + } + c.obj.castShadow = false; + // keep data model consistent so UI reflects actual runtime state + if (c.source === "editor") c.meta.castShadow = false; + } +} + +function syncShadowMapEnabled() { + enforceShadowSamplerBudget(); + let anyCast = false; + // Check scene lights + for (const sl of sceneLights) { + if (sl.obj?.castShadow && sl.obj?.visible !== false) { anyCast = true; break; } + } + // Check editor lights + if (!anyCast) { + for (const ld of editorLights) { + if (ld._lightObj?.castShadow && ld._lightObj?.visible !== false) { anyCast = true; break; } + } + } + if (renderer.shadowMap.enabled !== anyCast) { + renderer.shadowMap.enabled = anyCast; + // When toggling shadow maps, Three.js needs to recompile materials + scene.traverse((obj) => { if (obj.material) obj.material.needsUpdate = true; }); + } + if (anyCast) renderer.shadowMap.needsUpdate = true; +} + +function renderSceneInMode(mode) { + const savedOverride = scene.overrideMaterial; + const savedBg = scene.background; + const savedAssets = assetsGroup.visible; + const savedPrims = primitivesGroup.visible; + const savedLights = lightsGroup.visible; + const savedTags = tagsGroup.visible; + const savedLidar = lidarVizGroup.visible; + const savedOverlay = rgbdPcOverlayGroup.visible; + + if (mode === "rgb") { + scene.overrideMaterial = null; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + scene.background = DEFAULT_SCENE_BG; + renderer.render(scene, camera); + } else if (mode === "lidar") { + scene.overrideMaterial = null; + assetsGroup.visible = false; + primitivesGroup.visible = false; + lightsGroup.visible = false; + tagsGroup.visible = false; + lidarVizGroup.visible = true; + rgbdPcOverlayGroup.visible = rgbdPcOverlayOnLidar && _rgbdPcOverlayLastCount > 0; + scene.background = RGBD_BG; + renderer.render(scene, camera); + } + + scene.overrideMaterial = savedOverride; + scene.background = savedBg; + assetsGroup.visible = savedAssets; + primitivesGroup.visible = savedPrims; + lightsGroup.visible = savedLights; + tagsGroup.visible = savedTags; + lidarVizGroup.visible = savedLidar; + rgbdPcOverlayGroup.visible = savedOverlay; +} + +function renderCompareViews() { + const sz = renderer.getSize(new THREE.Vector2()); + const W = sz.x; + const H = sz.y; + const halfW = Math.floor(W / 2); + const halfH = Math.floor(H / 2); + + renderer.setScissorTest(true); + renderer.autoClear = false; + + renderer.setViewport(0, 0, W, H); + renderer.setScissor(0, 0, W, H); + renderer.setClearColor(0x000000, 1); + renderer.clear(true, true, true); + + // Top-left: RGB + renderer.setViewport(0, halfH, halfW, halfH); + renderer.setScissor(0, halfH, halfW, halfH); + renderer.setClearColor(DEFAULT_SCENE_BG, 1); + renderer.clear(true, true, true); + renderSceneInMode("rgb"); + + // Top-right: RGB-D + renderRgbdMetricPassOffscreen(); + rgbdVizMaterial.uniforms.uGrayMode.value = rgbdVizMode === "gray" ? 1.0 : 0.0; + renderer.setRenderTarget(null); + renderer.setViewport(halfW, halfH, W - halfW, halfH); + renderer.setScissor(halfW, halfH, W - halfW, halfH); + renderer.setClearColor(RGBD_BG, 1); + renderer.clear(true, true, true); + renderer.render(rgbdVizScene, rgbdPostCamera); + + // Bottom-center: LiDAR + const lidarX = Math.floor((W - halfW) / 2); + renderer.setViewport(lidarX, 0, halfW, halfH); + renderer.setScissor(lidarX, 0, halfW, halfH); + renderer.setClearColor(RGBD_BG, 1); + renderer.clear(true, true, true); + renderSceneInMode("lidar"); + + renderer.setScissorTest(false); + renderer.autoClear = true; + renderer.setViewport(0, 0, W, H); + renderer.setScissor(0, 0, W, H); +} + +function renderActiveView() { + syncShadowMapEnabled(); + if (simCompareView) { + renderCompareViews(); + } else if (simSensorViewMode === "rgbd") { + renderRgbdView(); + } else { + renderer.render(scene, camera); + } +} + +const _tmpCamPos = new THREE.Vector3(); +const _tmpCamDir = new THREE.Vector3(); +const _raycaster = new THREE.Raycaster(); + +function rapierRaycastFromCamera(maxToi = 250) { + if (!rapierWorld || !RAPIER) return null; + // Query pipeline is kept current by rapierWorld.step() in updateRapier + + const o = camera.getWorldPosition(_tmpCamPos); + const d = camera.getWorldDirection(_tmpCamDir).normalize(); + + const ray = new RAPIER.Ray({ x: o.x, y: o.y, z: o.z }, { x: d.x, y: d.y, z: d.z }); + const hit = rapierWorld.queryPipeline.castRayAndGetNormal( + rapierWorld.bodies, + rapierWorld.colliders, + ray, + maxToi, + false, // hollow: can hit boundary even if ray starts inside + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + playerCollider?.handle + ); + if (!hit) return null; + const toi = hit.toi ?? hit.timeOfImpact ?? 0; + const p = { x: o.x + d.x * toi, y: o.y + d.y * toi, z: o.z + d.z * toi }; + const n = hit.normal ? { x: hit.normal.x, y: hit.normal.y, z: hit.normal.z } : null; + return { point: p, normal: n, colliderHandle: hit.colliderHandle ?? null, toi }; +} + +function isShapeFreeAt(shape, rot, pos, excludeColliderHandle = null) { + if (!rapierWorld || !RAPIER) return false; + const hit = rapierWorld.queryPipeline.intersectionWithShape( + rapierWorld.bodies, + rapierWorld.colliders, + pos, + rot, + shape, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + excludeColliderHandle + ); + return hit == null; +} + +function findNearbyFreeSpotForCollider(collider, startPos, maxR = 2.0, step = 0.12) { + if (!collider) return null; + const shape = collider.shape; + const rot = collider.rotation(); + const exclude = collider.handle; + + if (isShapeFreeAt(shape, rot, startPos, exclude)) return startPos; + + const dirs = [ + [1, 0, 0], + [-1, 0, 0], + [0, 0, 1], + [0, 0, -1], + [1, 0, 1], + [1, 0, -1], + [-1, 0, 1], + [-1, 0, -1], + [0, 1, 0], + [0, -1, 0], + ]; + for (let r = step; r <= maxR; r += step) { + for (const [dx, dy, dz] of dirs) { + const len = Math.hypot(dx, dy, dz) || 1; + const pos = { x: startPos.x + (dx / len) * r, y: startPos.y + (dy / len) * r, z: startPos.z + (dz / len) * r }; + if (isShapeFreeAt(shape, rot, pos, exclude)) return pos; + } + } + return null; +} + +function removeAiAgent(agent, reason = "manual") { + if (!agent) return; + const removedId = String(agent.id || ""); + try { + aiAgents = aiAgents.filter((a) => a !== agent); + agentUiPush(`${new Date().toLocaleTimeString()}\nAGENT DESPAWN\n${agent.id} (${reason})`); + agent.dispose?.(); + } catch {} + if (removedId) { + _agentTasks.delete(removedId); + agentInspectorStateById.delete(removedId); + } + if (removedId) removeAgentBadge(removedId); + if (agentCameraFollowId === removedId) { + disableAgentCameraFollow(); + } + if (selectedAgentInspectorId === removedId) { + selectedAgentInspectorId = aiAgents[0]?.id || null; + if (selectedAgentInspectorId) renderAgentInspector(selectedAgentInspectorId); + else { + clearAgentInspectorViews(); + } + } + if (aiAgents.length === 0) { + disableAgentCameraFollow(); + } + renderAgentTaskUi(); +} + +function stopAiAgent(agent, reason = "manual-stop") { + if (!agent) return; + try { + if (agent.vlm) agent.vlm.enabled = false; + agent._plan = null; + agent._pendingDecision = null; + agent._setThought?.("Stopped"); + } catch {} + agentUiPush(`${new Date().toLocaleTimeString()}\nAGENT STOP\n${agent.id} (${reason})`); + renderAgentTaskUi(); +} + +function despawnEphemeralAgents(reason = "task-end") { + const doomed = aiAgents.filter((a) => a?._ephemeral === true); + for (const a of doomed) removeAiAgent(a, reason); +} + +function createAiAgent({ ephemeral = false } = {}) { + const endpoint = localStorage.getItem("sparkWorldVlmEndpoint") || "/vlm/decision"; + const model = resolveActiveVlmModel(); + const nearbyRange = 2.5; + const id = `agent-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; + let agentRef = null; + const agent = new AiAvatar({ + id, + scene, + rapierWorld, + RAPIER, + getWorldKey: () => worldKey, + getTags: () => tags, + getPlayerPosition: () => (Array.isArray(window.__playerPosition) ? window.__playerPosition : [0, 0, 0]), + avatarUrl: ["/agent-model/unitree_go2.glb", "/agent-model/robot.glb"], + senseRadius: 3.0, + walkSpeed: 2.0, + // Headless mode in dimos: skip visual rendering, keep colliders for physics + headless: false, + vlm: { + // In dimos mode, VLM is disabled — agent pose is driven externally via /odom. + // Ephemeral workers auto-enable; manually spawned agents start idle. + enabled: dimosMode ? false : true, + showSpeechBubbleInScene: false, + holdPositionWhenIdle: true, + endpoint, + model, + getModel: resolveActiveVlmModel, + actions: ACTIVE_VLM_ACTIONS, + buildPrompt: () => buildActiveVlmPrompt(), + request: requestVlmDecision, + captureBase64: async (a) => { + // If a sensor mode is active (RGB-D, LiDAR, Compare), capture the + // on-screen view which already renders in that mode via renderActiveView. + if (simSensorViewMode !== "rgb" || simCompareView) { + const [ax, ay, az] = a.getPosition?.() || [0, 0, 0]; + const yaw = a.group?.rotation?.y ?? 0; + const pitch = typeof a.pitch === "number" ? a.pitch : 0; + const cp = Math.cos(pitch), sp = Math.sin(pitch); + const eyeY = ay + PLAYER_EYE_HEIGHT * 0.9; + const prevPos = camera.position.clone(); + const prevQuat = camera.quaternion.clone(); + camera.position.set(ax, eyeY, az); + camera.lookAt(ax + Math.sin(yaw) * cp, eyeY + sp, az + Math.cos(yaw) * cp); + camera.updateProjectionMatrix(); + camera.updateMatrixWorld(true); + renderActiveView(); + const dataUrl = renderer.domElement.toDataURL("image/jpeg", 0.8); + camera.position.copy(prevPos); + camera.quaternion.copy(prevQuat); + camera.updateProjectionMatrix(); + camera.updateMatrixWorld(true); + const idx = dataUrl.indexOf("base64,"); + return idx !== -1 ? dataUrl.slice(idx + 7) : null; + } + return captureAgentPovBase64({ + agent: a, + renderer, + scene, + mainCamera: camera, + width: 960, + height: 432, + eyeHeight: PLAYER_EYE_HEIGHT * 0.9, + fov: camera.fov, + near: camera.near, + far: camera.far, + headLamp: null, + jpegQuality: 0.8, + }); + }, + decideEverySteps: ACTIVE_VLM_DEFAULTS.decideEverySteps, + stepMeters: ACTIVE_VLM_DEFAULTS.stepMeters, + getTask: () => ({ ..._getAgentTask(id) }), + getNearbyAssets: (a) => getNearbyAssetsForAgent(a, nearbyRange), + getNearbyPrimitives: (a) => getNearbyPrimitivesForAgent(a, nearbyRange), + isEditorMode: () => false, + interactAsset: ({ agent: a, assetId, actionId }) => agentInteractAsset({ agent: a, assetId, actionId }), + pickUpAsset: ({ agent: a, assetId }) => agentPickUpAsset(a, assetId), + dropAsset: ({ agent: a }) => agentDropAsset(a), + getHeldAsset: (a) => getAgentHeldAsset(a), + onCapture: (base64) => { + const id = agentRef?.id || ""; + const s = getOrCreateAgentInspectorState(id); + s.shot = base64 || ""; + if (!selectedAgentInspectorId) selectedAgentInspectorId = id; + if (selectedAgentInspectorId === id) agentUiSetShot(base64); + }, + onRequest: ({ endpoint: ep, model: m, prompt, context, imageBase64, messages }) => { + const id = agentRef?.id || ""; + const req = { + endpoint: ep, + model: m, + prompt, + context, + imageBytes: imageBase64 ? Math.floor((imageBase64.length * 3) / 4) : null, + messages, + }; + const s = getOrCreateAgentInspectorState(id); + s.request = req; + if (!selectedAgentInspectorId) selectedAgentInspectorId = id; + if (selectedAgentInspectorId === id) renderAgentInspector(id); + agentUiPush(`${new Date().toLocaleTimeString()}\nREQUEST ${m}\n${ep}\nagent=${id}`); + }, + onResponse: ({ raw, parsed }) => { + const id = agentRef?.id || ""; + const resp = { raw, parsed }; + const s = getOrCreateAgentInspectorState(id); + s.response = resp; + if (!selectedAgentInspectorId) selectedAgentInspectorId = id; + if (selectedAgentInspectorId === id) agentUiSetResponse(resp); + const action = typeof parsed?.action === "string" ? parsed.action : ""; + agentUiPush(`${new Date().toLocaleTimeString()}\nRESPONSE\n${action}\nagent=${id}`); + }, + onActionApplied: ({ action, params, plan }) => { + const id = agentRef?.id || ""; + agentUiPush( + `${new Date().toLocaleTimeString()}\nAPPLY ${action}\nparams=${JSON.stringify(params || {})}\nplan=${JSON.stringify(plan || {})}\nagent=${id}` + ); + }, + onTaskFinished: ({ summary }) => { + const agent = agentRef; + const summaryText = String(summary || "").trim(); + agentUiPush(`${new Date().toLocaleTimeString()}\nTASK FINISH${agent ? ` [${agent.id}]` : ""}\n${summaryText}`); + + // End this specific agent's task + if (agent) { + const task = _agentTasks.get(agent.id); + if (task) { + task.active = false; + task.finishedAt = Date.now(); + task.finishedReason = "model"; + task.lastSummary = summaryText; + } + } + + // Auto-despawn if ephemeral or configured to despawn after task + const shouldDespawn = + agent && + (agent._ephemeral === true || agent._autoDespawnAfterTask === true); + if (shouldDespawn) { + _agentTasks.delete(agent.id); + removeAiAgent(agent, "task-complete"); + } + + // Check if any agents still have active tasks + const anyActive = [..._agentTasks.values()].some((t) => t.active); + if (!anyActive) { + agentTask.active = false; + agentTask.finishedAt = Date.now(); + agentTask.finishedReason = "model"; + agentTask.lastSummary = summaryText; + disableAgentCameraFollow(); + } + renderAgentTaskUi(); + }, + onError: (err) => { + const id = agentRef?.id || ""; + agentUiPush(`${new Date().toLocaleTimeString()}\nERROR\n${String(err?.message || err)}\nagent=${id}`); + }, + onDecision: (d) => { + const thought = typeof d?.thought === "string" ? d.thought : ""; + const action = typeof d?.action === "string" ? d.action : ""; + const id = agentRef?.id || ""; + agentUiPush(`${new Date().toLocaleTimeString()}\nDECISION\n${thought}\n${action}\nagent=${id}`); + }, + }, + }); + agentRef = agent; + agent._ephemeral = !!ephemeral; + // Manually spawned editor agents should clean themselves up after task completion. + agent._autoDespawnAfterTask = true; + // Only inherit the active task if this agent was spawned as part of a worker pool (ephemeral). + // Manually spawned agents start idle and wait for their own task assignment. + agent._taskStartedAt = ephemeral ? Number(agentTask.startedAt || 0) : 0; + getOrCreateAgentInspectorState(id); + if (!selectedAgentInspectorId) selectedAgentInspectorId = id; + renderSelectedAgentControls(); + return agent; +} + + +async function ensureRapierLoaded() { + if (RAPIER) return; + if (!_rapierInitPromise) { + _rapierInitPromise = _doRapierInit(); + } + return _rapierInitPromise; +} + +async function _doRapierInit() { + RAPIER = await import("@dimforge/rapier3d-compat"); + await RAPIER.init(); + rapierWorld = new RAPIER.World({ x: 0, y: -9.81, z: 0 }); + worldBody = rapierWorld.createRigidBody(RAPIER.RigidBodyDesc.fixed()); + + const radius = PLAYER_RADIUS; + const halfHeight = PLAYER_HALF_HEIGHT; + playerBody = rapierWorld.createRigidBody( + RAPIER.RigidBodyDesc.kinematicPositionBased().setTranslation(0, 3, 0) + ); + playerCollider = rapierWorld.createCollider( + RAPIER.ColliderDesc.capsule(halfHeight, radius).setFriction(0.0).setSensor(ghostMode), + playerBody + ); + + characterController = rapierWorld.createCharacterController(0.02); + characterController.setSlideEnabled(true); + characterController.enableAutostep(0.55, 0.25, true); + characterController.enableSnapToGround(0.25); + characterController.setMaxSlopeClimbAngle(Math.PI / 3); + characterController.setMinSlopeSlideAngle(Math.PI / 2); +} + +async function spawnOrMoveAiAtAim({ createNew = false, silent = false, ephemeral = false } = {}) { + await ensureRapierLoaded(); + const hit = rapierRaycastFromCamera(500); + const placement = hit + ? { point: hit.point, normal: hit.normal || { x: 0, y: 1, z: 0 } } + : getPlacementAtCrosshair({ raycastDistance: 500, fallbackDistance: 3, surfaceOffset: 0.0 }); + if (!hit && !silent) { + setStatus("No collider hit; spawned AI using crosshair fallback placement."); + } + + let agent = createNew ? null : aiAgents[0] || null; + if (!agent) { + if (aiAgents.length >= MAX_AGENT_COUNT) { + if (!silent) setStatus(`Agent cap reached (${MAX_AGENT_COUNT}).`); + return; + } + agent = createAiAgent({ ephemeral }); + aiAgents.push(agent); + } else if (ephemeral) { + agent._ephemeral = true; + } + + const n = placement.normal + ? new THREE.Vector3(placement.normal.x, placement.normal.y, placement.normal.z).normalize() + : new THREE.Vector3(0, 1, 0); + const offset = Math.max(0.12, (agent.radius ?? PLAYER_RADIUS) + 0.06); + const p0 = placement.point; + const candA = { x: p0.x + n.x * offset, y: p0.y + n.y * offset, z: p0.z + n.z * offset }; + const candB = { x: p0.x - n.x * offset, y: p0.y - n.y * offset, z: p0.z - n.z * offset }; + + let chosen = null; + chosen = findNearbyFreeSpotForCollider(agent.collider, candA, 2.0, 0.12); + if (!chosen) chosen = findNearbyFreeSpotForCollider(agent.collider, candB, 2.0, 0.12); + if (!chosen) chosen = findNearbyFreeSpotForCollider(agent.collider, { x: p0.x, y: p0.y + offset, z: p0.z }, 2.5, 0.12); + // Fallback: use placement point directly (slightly above surface) rather than failing silently. + if (!chosen) { + chosen = { x: p0.x + n.x * 0.5, y: p0.y + n.y * 0.5 + 0.5, z: p0.z + n.z * 0.5 }; + console.warn("[Spawn] No free collision spot found – using direct fallback placement at", chosen); + } + + agent.setPosition(chosen.x, chosen.y, chosen.z); + if (agentTask.active) { + agent._taskStartedAt = agentTask.startedAt; + } + renderAgentTaskUi(); + if (!silent) { + const label = createNew ? "AI worker spawned." : "AI placed."; + setStatus(`${label} (${aiAgents.length} total)`); + } +} + +function pickAgentFromRay(raycaster) { + const agentRoots = aiAgents.map((a) => a?.group).filter(Boolean); + if (agentRoots.length === 0) return null; + + // First try exact mesh hits. + const agentHits = raycaster.intersectObjects(agentRoots, true); + if (agentHits.length > 0) { + let obj = agentHits[0].object; + while (obj && !(typeof obj.name === "string" && obj.name.startsWith("AiAvatar:"))) obj = obj.parent; + const agentId = obj?.name?.slice("AiAvatar:".length) || ""; + const agent = aiAgents.find((a) => a.id === agentId) || null; + if (agent) return agent; + } + + // Fallback: broad proximity test against agent centers. + let best = null; + let bestT = Infinity; + const origin = raycaster.ray.origin; + const dir = raycaster.ray.direction; + const tmp = new THREE.Vector3(); + const to = new THREE.Vector3(); + const pickRadius = 0.45; // generous click radius for tiny capsules + + for (const a of aiAgents) { + if (!a?.group || a.group.visible === false) continue; + const [x, y, z] = a.getPosition?.() || [0, 0, 0]; + // Aim around torso center for better clickability. + tmp.set(x, y + (a.halfHeight || 0.25), z); + const d2 = raycaster.ray.distanceSqToPoint(tmp); + if (d2 > pickRadius * pickRadius) continue; + + // Prefer nearest along-ray candidate in front of camera. + to.copy(tmp).sub(origin); + const t = to.dot(dir); + if (t <= 0) continue; + if (t < bestT) { + bestT = t; + best = a; + } + } + return best; +} + +function pickAgentFromScreenPoint(clientX, clientY, canvasRect) { + if (!Number.isFinite(clientX) || !Number.isFinite(clientY) || !canvasRect || aiAgents.length === 0) return null; + const thresholdPx = 46; + let best = null; + let bestD2 = thresholdPx * thresholdPx; + const v = new THREE.Vector3(); + for (const a of aiAgents) { + if (!a?.group || a.group.visible === false) continue; + const [x, y, z] = a.getPosition?.() || [0, 0, 0]; + v.set(x, y + (a.halfHeight || 0.25), z).project(camera); + if (v.z < -1 || v.z > 1) continue; // behind camera / clipped + const sx = canvasRect.left + (v.x * 0.5 + 0.5) * canvasRect.width; + const sy = canvasRect.top + (-v.y * 0.5 + 0.5) * canvasRect.height; + const dx = sx - clientX; + const dy = sy - clientY; + const d2 = dx * dx + dy * dy; + if (d2 < bestD2) { + bestD2 = d2; + best = a; + } + } + return best; +} + +function ensureAgentBadgeLayer() { + if (agentBadgeLayerEl) return; + installAgentBadgeEventDelegation(); + const el = document.createElement("div"); + el.id = "agent-badge-layer"; + el.className = "agent-badge-layer"; + document.body.appendChild(el); + agentBadgeLayerEl = el; +} + +function getOrCreateAgentBadge(agentId) { + const id = String(agentId || ""); + if (!id) return null; + ensureAgentBadgeLayer(); + if (agentBadgeElsById.has(id)) return agentBadgeElsById.get(id); + const btn = document.createElement("button"); + btn.type = "button"; + btn.className = "agent-badge"; + btn.textContent = id; + btn.dataset.agentId = id; + btn.addEventListener("pointerdown", (e) => { + e.preventDefault(); + e.stopPropagation(); + selectAgentInspector(id); + setStatus(`Inspecting ${id}. Use right panel controls.`); + }); + document.body.appendChild(btn); + agentBadgeElsById.set(id, btn); + return btn; +} + +function installAgentBadgeEventDelegation() { + if (typeof document === "undefined") return; + if (document.body?.dataset?.agentBadgeDelegationInstalled === "1") return; + if (document.body) document.body.dataset.agentBadgeDelegationInstalled = "1"; + // Capture-phase delegation to beat canvas/overlay handlers. + document.addEventListener( + "pointerdown", + (e) => { + const target = e.target; + if (!(target instanceof HTMLElement)) return; + const badge = target.closest(".agent-badge"); + if (!badge) return; + const id = String(badge.dataset.agentId || "").trim(); + if (!id) return; + e.preventDefault(); + e.stopPropagation(); + selectAgentInspector(id); + setStatus(`Inspecting ${id}. Use right panel controls.`); + }, + true + ); +} + +function removeAgentBadge(agentId) { + const id = String(agentId || ""); + const el = agentBadgeElsById.get(id); + if (el?.parentElement) el.parentElement.removeChild(el); + agentBadgeElsById.delete(id); +} + + +function pickTagMarkerFromCamera() { + _raycaster.setFromCamera({ x: 0, y: 0 }, camera); + const hits = _raycaster.intersectObjects(tagsGroup.children, false); + for (const h of hits) { + const obj = h.object; + if (obj?.userData?.isRadius) continue; + const id = obj?.userData?.tagId; + if (id) return id; + } + return null; +} + + + +// Lock pointer and interact on click for FPS navigation. +canvas.addEventListener("click", async (e) => { + if (!controls.isLocked) { + controls.enabled = true; + try { controls.lock(); } catch {} + } else if (e.button === 0) { + await handlePlayerInteraction(); + } +}); + +// Right-click to lock pointer (for FPS navigation) +canvas.addEventListener("contextmenu", (e) => { + e.preventDefault(); + if (!controls.isLocked) { + controls.enabled = true; + try { controls.lock(); } catch {} + } +}); + +controls.addEventListener("lock", () => { + controls.enabled = true; +}); +controls.addEventListener("unlock", () => { + setStatus("Click to look around."); +}); + +resetBtn?.addEventListener("click", () => { + controls.object.position.set(0, 1.7, 4); +}); + + +function setGhostMode(enabled) { + ghostMode = !!enabled; + // Ghost mode indicator shown in status + if (enabled) setStatus("Ghost mode ON"); + + // Disable collisions by turning the player collider into a sensor. + // (Sensors don't generate contact forces, so you can pass through walls.) + try { + if (playerCollider && typeof playerCollider.setSensor === "function") { + playerCollider.setSensor(ghostMode); + } + } catch { + // ignore + } +} + +// Ghost mode toggled via 'G' key only + +// Tagging UI +document.documentElement.dataset.mode = "sim"; +simPanelCollapseBtn?.addEventListener("click", () => { + simPanelCollapsed = true; + applySimPanelCollapsedState(); +}); +simPanelOpenBtn?.addEventListener("click", () => { + simPanelCollapsed = false; + applySimPanelCollapsedState(); +}); +simCameraModeToggleBtn?.addEventListener("click", () => { + simUserCameraMode = simUserCameraMode === "user" ? "agent" : "user"; + localStorage.setItem("sparkWorldSimCameraMode", simUserCameraMode); + updateSimCameraModeToggleUi(); + if (simUserCameraMode === "user") { + if (agentCameraFollow) disableAgentCameraFollow(); + } else if (agentTask.active) { + enableAgentCameraFollow(); + } +}); +simViewRgbdBtn?.addEventListener("click", () => { + simCompareView = false; + setSimSensorViewMode("rgbd"); +}); +simRgbdGrayBtn?.addEventListener("click", () => { + rgbdVizMode = "gray"; + updateSimSensorButtons(); + if (simSensorViewMode === "rgbd") setStatus("RGB-D: metric grayscale"); +}); +simRgbdColormapBtn?.addEventListener("click", () => { + rgbdVizMode = "colormap"; + updateSimSensorButtons(); + if (simSensorViewMode === "rgbd") setStatus("RGB-D: metric colormap"); +}); +simRgbdAutoRangeBtn?.addEventListener("click", () => { + rgbdAutoRange = !rgbdAutoRange; + updateSimSensorButtons(); + if (simSensorViewMode === "rgbd") setStatus(rgbdAutoRange ? "RGB-D auto-range ON (p5/p95)" : "RGB-D auto-range OFF"); +}); +simRgbdNoiseBtn?.addEventListener("click", () => { + rgbdNoiseEnabled = !rgbdNoiseEnabled; + updateSimSensorButtons(); + setStatus(rgbdNoiseEnabled ? "RGB-D noise ON" : "RGB-D noise OFF"); +}); +simRgbdSpeckleBtn?.addEventListener("click", () => { + rgbdSpeckleEnabled = !rgbdSpeckleEnabled; + updateSimSensorButtons(); + setStatus(rgbdSpeckleEnabled ? "RGB-D speckle ON" : "RGB-D speckle OFF"); +}); +simRgbdMinEl?.addEventListener("input", () => { + if (rgbdAutoRange) return; + const minV = Number(simRgbdMinEl.value); + const maxV = Number(simRgbdMaxEl?.value ?? rgbdRangeMaxM); + setRgbdRange(minV, maxV); +}); +simRgbdMaxEl?.addEventListener("input", () => { + if (rgbdAutoRange) return; + const minV = Number(simRgbdMinEl?.value ?? rgbdRangeMinM); + const maxV = Number(simRgbdMaxEl.value); + setRgbdRange(minV, maxV); +}); +simRgbdPcOverlayBtn?.addEventListener("click", () => { + rgbdPcOverlayOnLidar = !rgbdPcOverlayOnLidar; + _rgbdPcOverlayLastUpdateMs = 0; + _rgbdPcOverlayLastPose = null; + _rgbdPcOverlayDirty = rgbdPcOverlayOnLidar; + if (!rgbdPcOverlayOnLidar) { + _rgbdPcGeom.setDrawRange(0, 0); + _rgbdPcOverlayLastCount = 0; + _rgbdPcOverlayDirty = false; + } + if (rgbdPcOverlayOnLidar) { + // Overlay button should directly enter combined LiDAR+RGBD-PC debug mode. + simCompareView = false; + lidarOrderedDebugView = false; + if (simSensorViewMode !== "lidar") simSensorViewMode = "lidar"; + applySimSensorViewMode(); + } + // Actual visibility is finalized by updateRgbdPcOverlayCloud once points are generated. + rgbdPcOverlayGroup.visible = false; + updateSimSensorButtons(); + setStatus(rgbdPcOverlayOnLidar ? `RGB-D->PointCloud overlay ON (${_rgbdPcOverlayLastCount} pts)` : "RGB-D->PointCloud overlay OFF"); +}); +simViewLidarBtn?.addEventListener("click", () => { + // Main LiDAR button always maps to accumulated unordered 3D point cloud. + simCompareView = false; + lidarOrderedDebugView = false; + if (simSensorViewMode !== "lidar") { + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + } + setSimSensorViewMode("lidar"); + if (rgbdPcOverlayOnLidar) _rgbdPcOverlayDirty = true; +}); +simViewCompareBtn?.addEventListener("click", () => { + simCompareView = !simCompareView; + if (simCompareView) { + // Auto-collapse panel so tiles get full canvas width. + simPanelCollapsed = true; + applySimPanelCollapsedState(); + simSensorViewMode = "lidar"; + lidarOrderedDebugView = false; + if (rgbdPcOverlayOnLidar) _rgbdPcOverlayDirty = true; + setStatus("Compare view: RGB | RGB-D | LiDAR"); + } else { + simPanelCollapsed = false; + applySimPanelCollapsedState(); + setStatus("Compare view OFF"); + } + applySimSensorViewMode(); +}); +simLidarColorRangeBtn?.addEventListener("click", () => { + lidarColorByRange = !lidarColorByRange; + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") { + updateLidarPointCloud(); + setStatus(lidarColorByRange ? "LiDAR: range-color mode" : "LiDAR: intensity mode"); + } +}); +simLidarOrderedDebugBtn?.addEventListener("click", () => { + // Single Sweep is the explicit ring/scan debug view. + lidarOrderedDebugView = true; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + if (simSensorViewMode !== "lidar") simSensorViewMode = "lidar"; + updateSimSensorButtons(); + applySimSensorViewMode(); + setStatus("LiDAR: single sweep view"); +}); +simLidarNoiseBtn?.addEventListener("click", () => { + lidarNoiseEnabled = !lidarNoiseEnabled; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + setStatus(lidarNoiseEnabled ? "LiDAR noise ON" : "LiDAR noise OFF"); +}); +simLidarMultiReturnBtn?.addEventListener("click", () => { + lidarMultiReturnMode = lidarMultiReturnMode === "strongest" ? "last" : "strongest"; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + setStatus(`LiDAR return mode: ${lidarMultiReturnMode}`); +}); +spawnAiBtn?.addEventListener("click", async () => { + try { + await spawnOrMoveAiAtAim({ createNew: false, ephemeral: false }); + } catch (err) { + console.error("[Spawn] Error spawning agent:", err); + setStatus("Spawn failed: " + (err?.message || String(err))); + } +}); + +// --- Blob shadow live-adjustment helpers --- +// Updates the blob shadow mesh in-place without rebuilding the entire asset. +function updateBlobShadowLive(assetId) { + const a = assets.find((x) => x.id === assetId); + if (!a?.castShadow) return; + const root = assetsGroup.getObjectByName(`asset:${assetId}`); + if (!root) return; + const blob = root.getObjectByName(`blobShadow:${assetId}`); + if (!blob) return; + const bs = a.blobShadow || {}; + // Opacity + if (blob.material) blob.material.opacity = bs.opacity ?? 0.5; + // Scale + stretch + const baseDiam = blob.userData._baseDiameter || 1; + const userScale = bs.scale ?? 1.0; + const stretch = bs.stretch ?? 1.0; + const d = baseDiam * userScale; + blob.scale.set(d * stretch, 1, d / stretch); + // Rotation (Y axis, degrees → radians) + blob.rotation.y = ((bs.rotationDeg ?? 0) * Math.PI) / 180; + // Offset + blob.position.x = bs.offsetX ?? 0; + const baseY = blob.userData._baseLocalY ?? blob.position.y; + blob.position.y = baseY + (bs.offsetY ?? 0); + blob.position.z = bs.offsetZ ?? 0; +} + + +// Initialize agent UI visibility/content. +applySimPanelCollapsedState(); +renderAgentTaskUi(); +agentTaskStartBtn?.addEventListener("click", () => { + if (agentTask.active) return; + void startAgentTask(agentTaskInputEl?.value); +}); +agentTaskEndBtn?.addEventListener("click", () => endAgentTask("manual")); +// Enter key in command input starts task; stop propagation so WASD doesn't trigger +agentTaskInputEl?.addEventListener("keydown", (e) => { + e.stopPropagation(); + if (e.key === "Enter" && !agentTask.active && aiAgents.length > 0) { + void startAgentTask(agentTaskInputEl.value); + } +}); + +// Shared import logic — used by both editor Import and sim Load Level +async function importLevelFromJSON(json, options = {}) { + const importedTags = Array.isArray(json?.tags) ? json.tags : Array.isArray(json) ? json : null; + const preserveAssetsWhenMissing = options.preserveAssetsWhenMissing === true; + const importedAssets = Array.isArray(json?.assets) + ? json.assets + : (preserveAssetsWhenMissing ? assets : []); + const importedPrimitives = Array.isArray(json?.primitives) ? json.primitives : []; + const importedLights = Array.isArray(json?.lights) ? json.lights : []; + const importedSceneSettings = json && typeof json === "object" && json.sceneSettings + ? normalizeSceneSettings(json.sceneSettings) + : null; + if (!importedTags) throw new Error("Invalid level file."); + // Clean up old primitive colliders + for (const p of primitives) removePrimitiveCollider(p); + tags = importedTags; + assets = importedAssets; + primitives = importedPrimitives; + editorLights = importedLights; + if (importedSceneSettings) sceneSettings = importedSceneSettings; + if (!options.skipWorldSave) saveTagsForWorld(); + rebuildTagMarkers(); + await rebuildAssets(); + rebuildAllPrimitives(); + rebuildAllEditorLights(); + renderTagsList(); + renderAssetsList(); + renderPrimitivesList(); + renderTagPanel(); + applySceneSkySettings(); + applySceneRgbBackground(); + syncShadowMapEnabled(); +} + + +// Sim-mode "Load Level JSON" input (only exists in sim.html) +const simLevelImportEl = document.getElementById("sim-level-import"); +simLevelImportEl?.addEventListener("change", async (e) => { + const file = e.target.files?.[0]; + if (!file) return; + try { + setStatus("Loading level..."); + const text = await file.text(); + await importLevelFromJSON(JSON.parse(text)); + await ensureRapierLoaded(); + spawnPlayerInsideScene(); + setStatus("Level loaded. Click to enter, then spawn an agent."); + } catch (err) { + console.error(err); + setStatus(err?.message || "Failed to load level."); + } finally { + e.target.value = ""; + } +}); + +canvas?.addEventListener("mousedown", () => { + const id = pickTagMarkerFromCamera(); + if (!id) return; + selectedTagId = id; + draftTag = null; + updateMarkerMaterials(); + renderTagsList(); + renderTagPanel(); +}); + + +// ============================================================================= +// PRIMITIVE & LIGHT EVENT HANDLERS +// ============================================================================= + + +// Expose tag data for "simulation mode" consumers. +globalThis.sparkWorld = globalThis.sparkWorld || {}; +globalThis.sparkWorld.getWorldKey = () => worldKey; +globalThis.sparkWorld.getTags = () => tags.slice(); +globalThis.sparkWorld.getAiAgents = () => aiAgents.map((a) => ({ id: a.id, position: a.getPosition?.() })); + +function teleportPlayerTo(x, y, z) { + if (!playerBody) return; + playerBody.setTranslation({ x, y, z }, true); + playerBody.setLinvel({ x: 0, y: 0, z: 0 }, true); +} + +// Find a reasonable interior floor Y by casting a ray straight down through the +// loaded scene meshes and picking the lowest up-facing surface. This skips the +// roof when a building has both a roof and an interior floor, so the player +// lands inside rather than on top. +function _findSceneFloorY(x = 0, z = 0) { + const bbox = new THREE.Box3(); + const tmp = new THREE.Box3(); + try { tmp.setFromObject(assetsGroup); if (tmp.isEmpty() === false) bbox.union(tmp); } catch {} + try { tmp.setFromObject(primitivesGroup); if (tmp.isEmpty() === false) bbox.union(tmp); } catch {} + const fromY = bbox.isEmpty() ? 50 : bbox.max.y + 5; + const raycaster = new THREE.Raycaster( + new THREE.Vector3(x, fromY, z), + new THREE.Vector3(0, -1, 0), + 0, + fromY + 500, + ); + const hits = [ + ...raycaster.intersectObject(assetsGroup, true), + ...raycaster.intersectObject(primitivesGroup, true), + ]; + let floorY = null; + for (const h of hits) { + const n = h.face?.normal; + if (!n) continue; + // Transform face normal to world space to test "up-facing" + const worldN = n.clone().transformDirection(h.object.matrixWorld); + if (worldN.y < 0.5) continue; // skip walls/ceilings + if (floorY == null || h.point.y < floorY) floorY = h.point.y; + } + return floorY; +} + +function spawnPlayerInsideScene() { + const floorY = _findSceneFloorY(0, 0); + if (floorY == null) return false; + const y = floorY + PLAYER_EYE_HEIGHT; + camera.position.set(0, y, 0); + teleportPlayerTo(0, y, 0); + return true; +} + + +function safeDisableGhost() { + // If we're currently inside occupied geometry, turning collisions back on will + // trap the character (penetration state). Use Rapier query pipeline to find a safe spot. + if (!playerBody) return setGhostMode(false); + const p = playerBody.translation(); + + // Use Rapier query pipeline to find a non-penetrating spot. + if (rapierWorld && playerCollider) { + try { + const shape = playerCollider.shape; + const rot = playerCollider.rotation(); + const here = { x: p.x, y: p.y, z: p.z }; + + const intersectsHere = rapierWorld.queryPipeline.intersectionWithShape( + rapierWorld.bodies, + rapierWorld.colliders, + here, + rot, + shape, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + playerCollider.handle + ); + + // If we're not intersecting anything solid, we can safely disable ghost immediately. + if (intersectsHere == null) { + setGhostMode(false); + setStatus("Ghost disabled."); + return; + } + + const tryOffsets = (maxR, step) => { + for (let r = step; r <= maxR; r += step) { + // sample a handful of directions per radius + const dirs = [ + [1, 0, 0], + [-1, 0, 0], + [0, 0, 1], + [0, 0, -1], + [1, 0, 1], + [1, 0, -1], + [-1, 0, 1], + [-1, 0, -1], + [0, 1, 0], + [0, -1, 0], + ]; + for (const [dx, dy, dz] of dirs) { + const len = Math.hypot(dx, dy, dz) || 1; + const pos = { x: p.x + (dx / len) * r, y: p.y + (dy / len) * r, z: p.z + (dz / len) * r }; + const hit = rapierWorld.queryPipeline.intersectionWithShape( + rapierWorld.bodies, + rapierWorld.colliders, + pos, + rot, + shape, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS, + undefined, + playerCollider.handle + ); + if (hit == null) return pos; + } + } + return null; + }; + + const pos = tryOffsets(2.5, 0.15); + if (pos) { + teleportPlayerTo(pos.x, pos.y, pos.z); + setGhostMode(false); + setStatus("Ghost disabled (moved to nearest free space)."); + return; + } + } catch { + // ignore + } + } + + setStatus("Couldn't find free space to disable Ghost. Staying in Ghost mode."); + setGhostMode(true); +} + +window.addEventListener("keydown", (e) => { + const tagName = e.target?.tagName?.toLowerCase?.(); + const isTyping = + tagName === "input" || tagName === "textarea" || tagName === "select" || e.target?.isContentEditable; + if (!isTyping) { + if (e.code === "KeyB") { + void spawnOrMoveAiAtAim({ createNew: false, ephemeral: false }); + e.preventDefault(); + } + } + if (e.code === "KeyW") keys.forward = true; + if (e.code === "KeyS") keys.backward = true; + if (e.code === "KeyA") keys.left = true; + if (e.code === "KeyD") keys.right = true; + if (e.code === "Space") keys.up = true; + if (e.code === "ShiftLeft" || e.code === "ShiftRight") keys.down = true; + if (e.code === "KeyF") flyMode = !flyMode; + if (e.code === "KeyG") { + if (ghostMode) safeDisableGhost(); + else setGhostMode(true); + } + + // === PLAYER INTERACTION KEYS === + // E key to interact with asset at crosshair + if (e.code === "KeyE" && controls?.isLocked && !isTyping) { + handlePlayerInteraction(); + e.preventDefault(); + } + if (e.code === "KeyR" && controls?.isLocked && !isTyping && !isInteractionPopupVisible()) { + if (cycleInteractableTarget(1)) { + updatePlayerInteractionHint(); + e.preventDefault(); + } + } + + // Escape to close interaction popup + if (e.code === "Escape" && isInteractionPopupVisible()) { + hideInteractionPopup(); + // Re-lock pointer after closing popup + controls?.lock?.(); + e.preventDefault(); + } + + // Number keys 1-9 to select action when popup is visible + if (isInteractionPopupVisible() && _currentInteractableAsset) { + const numMatch = e.code.match(/^(?:Digit|Numpad)([1-9])$/); + if (numMatch) { + const idx = parseInt(numMatch[1], 10) - 1; + const { asset, actions } = _currentInteractableAsset; + if (idx >= 0 && idx < actions.length) { + const actionId = actions[idx].id; + + // Hide popup and re-lock pointer FIRST (before async operations) + hideInteractionPopup(); + + // Execute the action + if (actionId === "__PICK_UP__") { + playerPickUpAsset(asset.id); + } else { + executePlayerInteraction(asset.id, actionId); + } + + // Re-lock pointer (use setTimeout since pointer lock may need a moment) + setTimeout(() => { + try { + controls?.lock?.(); + } catch (err) { + // Pointer lock requires user gesture, may fail silently + } + }, 10); + + e.preventDefault(); + e.stopPropagation(); + return; + } + } + } +}); + +window.addEventListener("keyup", (e) => { + if (e.code === "KeyW") keys.forward = false; + if (e.code === "KeyS") keys.backward = false; + if (e.code === "KeyA") keys.left = false; + if (e.code === "KeyD") keys.right = false; + if (e.code === "Space") keys.up = false; + if (e.code === "ShiftLeft" || e.code === "ShiftRight") keys.down = false; +}); + + +// Shadow catcher: a large transparent ground plane that only shows shadows. +// ShadowMaterial is fully transparent where there's no shadow, so the splat +// floor shows through, but shadows appear as dark patches on top. +const shadowCatcherMat = new THREE.ShadowMaterial({ opacity: 0.35 }); +const shadowCatcher = new THREE.Mesh( + new THREE.PlaneGeometry(200, 200), + shadowCatcherMat +); +shadowCatcher.rotation.x = -Math.PI / 2; // lie flat +shadowCatcher.position.y = 0.001; // just above grid to avoid z-fighting +shadowCatcher.receiveShadow = true; +shadowCatcher.name = "__shadowCatcher"; +scene.add(shadowCatcher); +// Add to scene lights registry so it's controllable from the editor +sceneLights.push({ id: "_shadow_ground", label: "Shadow Ground", obj: shadowCatcher, type: "shadow_ground" }); + +function _hasBumpableAssets() { + for (const a of assets) { if (a?.bumpable) return true; } + return false; +} + +function updateBumpableAssets(dt, playerPos, agentPushers = []) { + if (!playerPos || !_hasBumpableAssets()) { + _playerPosPrevForBumpValid = false; + return; + } + if (!_playerPosPrevForBumpValid) { + _playerPosPrevForBump.copy(playerPos); + _playerPosPrevForBumpValid = true; + return; + } + const playerVel = new THREE.Vector3().subVectors(playerPos, _playerPosPrevForBump).divideScalar(Math.max(dt, 1e-3)); + _playerPosPrevForBump.copy(playerPos); + const speedXZ = Math.hypot(playerVel.x, playerVel.z); + const playerCanPush = !ghostMode; + const intent = new THREE.Vector3(); + const camForward = new THREE.Vector3(); + camera.getWorldDirection(camForward); + camForward.y = 0; + if (camForward.lengthSq() > 1e-6) camForward.normalize(); + const camRight = new THREE.Vector3().crossVectors(camForward, camera.up).normalize(); + if (keys.forward) intent.add(camForward); + if (keys.backward) intent.sub(camForward); + if (keys.right) intent.add(camRight); + if (keys.left) intent.sub(camRight); + if (intent.lengthSq() > 1e-6) intent.normalize(); + const intentPush = playerCanPush && intent.lengthSq() > 0; + const pushDir = intentPush ? intent.clone() : new THREE.Vector3(playerVel.x, 0, playerVel.z); + if (pushDir.lengthSq() > 1e-6) pushDir.normalize(); + const playerRadius = 0.35; + const pushThreshold = 0.05; + let anyMoved = false; + let anyColliderNeedsSync = false; + for (const a of assets) { + if (!a?.bumpable) continue; + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (!obj) continue; + const vel = _assetBumpVelocities.get(a.id) || new THREE.Vector3(); + const localCenter = obj.userData?._localSphereCenter || new THREE.Vector3(); + const worldCenter = localCenter.clone(); + obj.localToWorld(worldCenter); + const worldRadius = (obj.userData?._localSphereRadius || 0.6) * Math.max(obj.scale.x, obj.scale.y, obj.scale.z); + const dx = worldCenter.x - playerPos.x; + const dz = worldCenter.z - playerPos.z; + const dist = Math.hypot(dx, dz); + const minDist = worldRadius + playerRadius; + const ahead = pushDir.lengthSq() > 0 ? (dx * pushDir.x + dz * pushDir.z) : 0; + const lateral = pushDir.lengthSq() > 0 ? Math.abs(dx * -pushDir.z + dz * pushDir.x) : dist; + const inPushCone = intentPush && ahead > -0.05 && ahead < (minDist + 0.9) && lateral < (worldRadius + 0.55); + if (playerCanPush && (dist < (minDist + 0.35) || inPushCone) && (speedXZ > pushThreshold || intentPush)) { + const dirX = dist > 1e-3 ? dx / dist : (intentPush ? pushDir.x : (Math.sign(playerVel.x) || 1)); + const dirZ = dist > 1e-3 ? dz / dist : (intentPush ? pushDir.z : (Math.sign(playerVel.z) || 0)); + const penetration = minDist - dist; + const response = Number(a.bumpResponse) || 0.9; + const driveSpeed = Math.max(speedXZ, intentPush ? 1.4 : 0); + const intentBonus = inPushCone ? 0.35 : 0; + const impulse = Math.min(2.4, (Math.max(0, penetration) * 3 + driveSpeed * 0.35 + intentBonus) * response); + vel.x += dirX * impulse; + vel.z += dirZ * impulse; + } + // AI agents can push bumpable assets as well. + for (const ap of agentPushers) { + const apPos = ap?.pos; + const apVel = ap?.vel; + if (!apPos || !apVel) continue; + const av = Math.hypot(apVel.x || 0, apVel.z || 0); + if (av <= 0.04) continue; + const adx = worldCenter.x - apPos.x; + const adz = worldCenter.z - apPos.z; + const adist = Math.hypot(adx, adz); + const aminDist = worldRadius + Math.max(0.22, Number(ap.radius) || 0.22); + if (adist > aminDist + 0.3) continue; + const dirX = adist > 1e-3 ? adx / adist : (Math.sign(apVel.x) || 1); + const dirZ = adist > 1e-3 ? adz / adist : (Math.sign(apVel.z) || 0); + const penetration = aminDist - adist; + const response = Number(a.bumpResponse) || 0.9; + const impulse = Math.min(2.2, (Math.max(0, penetration) * 2.4 + av * 0.28) * response); + vel.x += dirX * impulse; + vel.z += dirZ * impulse; + } + const damping = Math.min(0.995, Math.max(0.65, Number(a.bumpDamping) || 0.9)); + const dampPow = Math.pow(damping, dt * 60); + vel.multiplyScalar(dampPow); + const maxSpeed = 2.5; + const speed = Math.hypot(vel.x, vel.z); + if (speed > maxSpeed) { + const s = maxSpeed / speed; + vel.x *= s; + vel.z *= s; + } + if (vel.lengthSq() < 1e-4) { + vel.set(0, 0, 0); + _assetBumpVelocities.set(a.id, vel); + continue; + } + let moveX = THREE.MathUtils.clamp(vel.x * dt, -0.2, 0.2); + let moveZ = THREE.MathUtils.clamp(vel.z * dt, -0.2, 0.2); + const myBox = new THREE.Box3().setFromObject(obj); + const testBoxX = myBox.clone().translate(new THREE.Vector3(moveX, 0, 0)); + const testBoxZ = myBox.clone().translate(new THREE.Vector3(0, 0, moveZ)); + let blockedX = false, blockedZ = false; + const checkCollision = (testBox, excludeObj) => { + for (const child of primitivesGroup.children) { + if (child === excludeObj) continue; + const cb = new THREE.Box3().setFromObject(child); + if (!cb.isEmpty() && testBox.intersectsBox(cb)) return true; + } + for (const child of assetsGroup.children) { + if (child === excludeObj) continue; + if (child.userData?.isBlobShadow) continue; + const cb = new THREE.Box3().setFromObject(child); + if (!cb.isEmpty() && testBox.intersectsBox(cb)) return true; + } + return false; + }; + if (Math.abs(moveX) > 1e-5 && checkCollision(testBoxX, obj)) { + blockedX = true; + vel.x *= -0.15; + } + if (Math.abs(moveZ) > 1e-5 && checkCollision(testBoxZ, obj)) { + blockedZ = true; + vel.z *= -0.15; + } + if (!blockedX) obj.position.x += moveX; + if (!blockedZ) obj.position.z += moveZ; + if (blockedX && blockedZ) { + _assetBumpVelocities.set(a.id, vel); + continue; + } + anyMoved = true; + anyColliderNeedsSync = true; + if (!a.transform) a.transform = {}; + if (!a.transform.position) a.transform.position = { x: 0, y: 0, z: 0 }; + a.transform.position.x = obj.position.x; + a.transform.position.z = obj.position.z; + _assetBumpVelocities.set(a.id, vel); + } + if (anyMoved) { + const now = performance.now(); + if (now - _lastBumpSaveAt > 500) { + _lastBumpSaveAt = now; + saveTagsForWorld(); + } + if (anyColliderNeedsSync && now - _lastBumpColliderSyncAt > 50) { + _lastBumpColliderSyncAt = now; + for (const a of assets) { + if (!a?.bumpable) continue; + if (!_assetBumpVelocities.has(a.id)) continue; + const v = _assetBumpVelocities.get(a.id); + if (!v || v.lengthSq() < 1e-4) continue; + rebuildAssetCollider(a.id); + } + } + } +} + +function collectAgentBumpPushers(dt) { + const pushers = []; + const alive = new Set(); + const invDt = 1 / Math.max(dt, 1e-3); + for (const agent of aiAgents) { + const id = String(agent?.id || ""); + const posRaw = agent?.body?.translation?.(); + if (!id || !posRaw) continue; + alive.add(id); + const pos = new THREE.Vector3(posRaw.x, posRaw.y, posRaw.z); + const prev = _agentPosPrevForBump.get(id); + const vel = prev ? pos.clone().sub(prev).multiplyScalar(invDt) : new THREE.Vector3(); + _agentPosPrevForBump.set(id, pos.clone()); + pushers.push({ + id, + pos, + vel, + radius: Math.max(0.2, Number(agent?.radius) || 0.2), + }); + } + for (const id of _agentPosPrevForBump.keys()) { + if (!alive.has(id)) _agentPosPrevForBump.delete(id); + } + return pushers; +} + +function updateRapier(dt) { + // No physics world loaded → free-fly camera movement so user can still navigate + if (!rapierWorld || !playerBody) { + const flySpeed = 8.0; + const fwd = new THREE.Vector3(); + camera.getWorldDirection(fwd); + const right = new THREE.Vector3().crossVectors(fwd, camera.up).normalize(); + const move = new THREE.Vector3(); + if (keys.forward) move.add(fwd); + if (keys.backward) move.sub(fwd); + if (keys.right) move.add(right); + if (keys.left) move.sub(right); + if (keys.up) move.y += 1; + if (keys.down) move.y -= 1; + if (move.lengthSq() > 0) { + move.normalize().multiplyScalar(flySpeed * dt); + controls.object.position.add(move); + avatar.position.copy(controls.object.position).y -= PLAYER_EYE_HEIGHT; + } + return; + } + + // Flush any deferred collider builds BEFORE stepping + flushPendingColliderBuilds(); + + // Step physics FIRST — this integrates last frame's kinematic moves and + // updates the query pipeline internally, avoiding the RefCell double-borrow + // that happens with manual `queryPipeline.update(colliders)`. + rapierWorld.timestep = dt; + try { + rapierWorld.step(); + _rapierStepFaultCount = 0; + } catch (e) { + _rapierStepFaultCount += 1; + console.warn(`[RAPIER] step() failed (${_rapierStepFaultCount})`, e); + // Prevent hard crash loop; skip this frame and try again next tick. + return; + } + + // Sync camera and avatar to the body position that step() just resolved + const p = playerBody.translation(); + + // Skip player movement when camera is following agent + if (agentCameraFollow) { + avatar.position.set(p.x, p.y, p.z); + return; + } + + const baseSpeed = 6.0; + const runSpeed = 10.0; + const flySpeed = 8.0; + const speed = flyMode ? flySpeed : keys.down ? runSpeed : baseSpeed; + const gravity = 20.0; + const jumpVel = 8.0; + + const forward = new THREE.Vector3(); + camera.getWorldDirection(forward); + forward.y = 0; + forward.normalize(); + const right = new THREE.Vector3().crossVectors(forward, camera.up).normalize(); + + const wish = new THREE.Vector3(); + if (keys.forward) wish.add(forward); + if (keys.backward) wish.sub(forward); + if (keys.right) wish.add(right); + if (keys.left) wish.sub(right); + if (wish.lengthSq() > 0) wish.normalize(); + + const upDown = flyMode ? (keys.up ? 1 : 0) + (keys.down ? -1 : 0) : 0; + + const t = p; // body position after step + let desired = { x: 0, y: 0, z: 0 }; + + if (ghostMode) { + desired = { + x: wish.x * flySpeed * dt, + y: ((keys.up ? 1 : 0) + (keys.down ? -1 : 0)) * flySpeed * dt, + z: wish.z * flySpeed * dt, + }; + playerBody.setNextKinematicTranslation({ + x: t.x + desired.x, + y: t.y + desired.y, + z: t.z + desired.z, + }); + } else if (flyMode) { + desired = { + x: wish.x * flySpeed * dt, + y: upDown * flySpeed * dt, + z: wish.z * flySpeed * dt, + }; + if (characterController && playerCollider) { + characterController.computeColliderMovement( + playerCollider, + desired, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS + ); + const m = characterController.computedMovement(); + const mx = m.x, my = m.y, mz = m.z; + playerBody.setNextKinematicTranslation({ x: t.x + mx, y: t.y + my, z: t.z + mz }); + } else { + playerBody.setNextKinematicTranslation({ x: t.x + desired.x, y: t.y + desired.y, z: t.z + desired.z }); + } + } else { + walkVerticalVel -= gravity * dt; + + if (keys.up && characterController?.computedGrounded?.()) { + walkVerticalVel = jumpVel; + } + + desired = { x: wish.x * speed * dt, y: walkVerticalVel * dt, z: wish.z * speed * dt }; + + if (characterController && playerCollider) { + characterController.computeColliderMovement( + playerCollider, + desired, + RAPIER.QueryFilterFlags.EXCLUDE_SENSORS + ); + const m = characterController.computedMovement(); + const mx = m.x, my = m.y, mz = m.z; + const grounded = characterController.computedGrounded(); + if (grounded && walkVerticalVel < 0) walkVerticalVel = 0; + playerBody.setNextKinematicTranslation({ x: t.x + mx, y: t.y + my, z: t.z + mz }); + } else { + playerBody.setNextKinematicTranslation({ x: t.x + desired.x, y: t.y + desired.y, z: t.z + desired.z }); + } + } + + // Safety: if Ghost is OFF, ensure the collider is not a sensor + try { + if (!ghostMode && playerCollider && typeof playerCollider.isSensor === "function" && playerCollider.isSensor()) { + playerCollider.setSensor(false); + } + } catch {} + avatar.position.set(p.x, p.y, p.z); + + // If agent camera follow is active, DON'T sync player camera to player body + // The tick() function will handle camera positioning via updateAgentCameraFollow + if (!agentCameraFollow) { + controls.object.position.set(p.x, p.y + PLAYER_EYE_HEIGHT, p.z); + } + + // Expose player position for other modules (AI, etc). + if (typeof window !== "undefined") { + window.__playerPosition = [p.x, p.y, p.z]; + } +} + +function tick() { + const rawDt = clock.getDelta(); + const physicsDt = Math.min(rawDt, 0.05); + const motionDt = Math.min(rawDt, 0.02); + + updateRapier(physicsDt); + + // Bumpable assets: only compute if any exist + if (_hasBumpableAssets()) { + const agentPushers = aiAgents.length ? collectAgentBumpPushers(physicsDt) : []; + let bumpPlayerPos = null; + if (playerBody) { + const p = playerBody.translation(); + bumpPlayerPos = new THREE.Vector3(p.x, p.y, p.z); + } else { + bumpPlayerPos = controls.object.position.clone(); + bumpPlayerPos.y -= PLAYER_EYE_HEIGHT; + } + updateBumpableAssets(physicsDt, bumpPlayerPos, agentPushers); + } + + // Update AI agents (if Rapier is initialized). + if (aiAgents.length && rapierWorld) { + const now = Date.now(); + for (const a of aiAgents) { + try { + // Keep cmd_vel integration tied to wall-clock delta even when physics dt is clamped. + a.update(motionDt, now); + } catch (e) { + console.warn("AI update failed:", e); + } + } + } + + // Update agent camera follow (after agent update, before render) + if (agentCameraFollow) { + updateAgentCameraFollow(physicsDt); + avatar.visible = false; + } + + // Update interaction hint at reduced rate + const now = performance.now(); + if (now - _lastHintUpdate > 300) { + _lastHintUpdate = now; + updateInteractionHint(); + } + + // LiDAR / sensor overlays — run when explicitly enabled OR in dimos mode + // In dimos mode, always skip browser raycasting — server handles lidar via Rapier snapshots. + const _skipBrowserLidar = dimosMode; + if (!_skipBrowserLidar && (simSensorViewMode === "lidar" || simCompareView || dimosMode)) { + lidarVizGroup.visible = true; + updateLidarPointCloud(); + if (_lidarGeom.drawRange.count <= 0 && _lidarLastNonZeroDrawCount > 0) { + _lidarGeom.setDrawRange(0, _lidarLastNonZeroDrawCount); + } + // In dimos mode, hide LiDAR viz from the main scene render — it's only + // needed for data capture + the sidebar LiDAR panel renders it separately. + if (dimosMode && simSensorViewMode !== "lidar" && !simCompareView) { + lidarVizGroup.visible = false; + } + } else if (!_skipBrowserLidar && rgbdPcOverlayOnLidar && (simSensorViewMode === "lidar" || simCompareView)) { + updateRgbdPcOverlayCloud(false); + } + + if (!_skipBrowserLidar) pushLidarPoseSample(); + + // Dimos sensor capture — GPU readback needs rAF, odom runs independently via setInterval + if (dimosMode && window.__dimosBridge) { + const bridge = window.__dimosBridge; + if (bridge._connected) { + // Lidar: skip browser→WS publish when server-side lidar is active + if (bridge._dirty.lidar && !bridge._serverLidar) { + bridge._dirty.lidar = false; + bridge._publishLidar(); + } + // Camera stream disabled for now. + // if (bridge._dirty.images) { + // bridge._dirty.images = false; + // bridge._publishImages(); + // } + } + } + + // Agent vision captures + if (hasPendingCapture()) { + processPendingCaptures().then(() => { + renderActiveView(); + requestAnimationFrame(tick); + }); + return; + } + + renderActiveView(); + requestAnimationFrame(tick); +} + +// Interaction hint elements (cached) +let _lastHintUpdate = 0; +let _crosshairEl = null; +let _interactionHintEl = null; + +function updateInteractionHint() { + // Cache DOM elements + if (!_crosshairEl) _crosshairEl = document.getElementById("crosshair"); + if (!_interactionHintEl) _interactionHintEl = document.getElementById("interaction-hint"); + + // Only show when pointer is locked and no popup is visible + if (!controls?.isLocked || isInteractionPopupVisible()) { + _crosshairEl?.classList.remove("interactable"); + if (_interactionHintEl) { + _interactionHintEl.classList.remove("visible"); + } + return; + } + + // If holding something, show drop hint + if (playerHeldAsset) { + const heldAsset = getPlayerHeldAsset(); + const heldName = heldAsset?.title || "item"; + _crosshairEl?.classList.remove("interactable"); + _crosshairEl?.classList.add("holding"); + if (_interactionHintEl) { + _interactionHintEl.innerHTML = `Holding: ${escapeHtml(heldName)} · DropE`; + _interactionHintEl.classList.add("visible"); + } + return; + } + if (playerHeldGroupId) { + const heldGroup = groups.find((g) => g.id === playerHeldGroupId); + const heldName = heldGroup?.name || "group"; + _crosshairEl?.classList.remove("interactable"); + _crosshairEl?.classList.add("holding"); + if (_interactionHintEl) { + _interactionHintEl.innerHTML = `Holding: ${escapeHtml(heldName)} · DropE`; + _interactionHintEl.classList.add("visible"); + } + return; + } + + // Not holding anything - remove holding class + _crosshairEl?.classList.remove("holding"); + + const target = getInteractableAssetAtCrosshair(); + + if (target) { + const { kind, asset, group, actions, dist, canPickUp } = target; + const title = kind === "group" ? (group?.name || "(group)") : (asset.title || "(asset)"); + + // Build action description + let actionText; + if (kind === "group") { + actionText = "Pick up"; + } else if (actions.length === 0 && canPickUp) { + actionText = "Pick up"; + } else if (actions.length === 1 && !canPickUp) { + actionText = actions[0].label || "interact"; + } else { + const count = actions.length + (canPickUp ? 1 : 0); + actionText = `${count} actions`; + } + + _crosshairEl?.classList.add("interactable"); + if (_interactionHintEl) { + const cycleHint = kind === "asset" && target.candidateCount > 1 + ? ` · Cycle ${target.candidateIndex + 1}/${target.candidateCount}R` + : ""; + _interactionHintEl.innerHTML = `${escapeHtml(title)} · ${escapeHtml(actionText)}E${cycleHint}`; + _interactionHintEl.classList.add("visible"); + } + } else { + _crosshairEl?.classList.remove("interactable"); + if (_interactionHintEl) { + _interactionHintEl.classList.remove("visible"); + } + } +} + +setStatus("Select a .ply/.spz to start."); +tick(); + +// Expose debug utilities +window.clearWorldStorage = clearWorldStorage; +window.__robovalLidar = { + // Returns the latest standardized frames (raw + deskewed + optional range image) + getLatestFrames() { + return { + raw: _lidarLatestRawFrame, + deskewed: _lidarLatestDeskewedFrame, + rangeImage: _lidarLatestRangeImage, + }; + }, + // ROS2 PointCloud2-compatible dict converter + toPointCloud2(frame) { + return to_pointcloud2(frame); + }, + // Manual export of the latest frame set to NPZ files. + async exportLatest() { + if (!_lidarLatestRawFrame || !_lidarLatestDeskewedFrame) return false; + await writeLidarFrameFiles(_lidarLatestRawFrame, _lidarLatestDeskewedFrame, _lidarLatestRangeImage); + return true; + }, + // Auto-export each LiDAR frame (warning: downloads many files in browser). + setAutoExport(enabled) { + _lidarAutoExport = !!enabled; + return _lidarAutoExport; + }, + getAutoExport() { + return _lidarAutoExport; + }, + // Force a known-good synthetic cloud to isolate renderer issues from sensor math. + setKnownGoodDebugCloud(enabled) { + _lidarUseKnownGoodDebugCloud = !!enabled; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + return _lidarUseKnownGoodDebugCloud; + }, + getKnownGoodDebugCloud() { + return _lidarUseKnownGoodDebugCloud; + }, + // Toggle ordered scan debug render (single-frame, lidar-frame) vs accumulated world cloud. + setOrderedDebugView(enabled) { + lidarOrderedDebugView = !!enabled; + if (!lidarOrderedDebugView) { + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + } + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + return lidarOrderedDebugView; + }, + getOrderedDebugView() { + return lidarOrderedDebugView; + }, + setNoiseModel(enabled) { + lidarNoiseEnabled = !!enabled; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + return lidarNoiseEnabled; + }, + getNoiseModel() { + return lidarNoiseEnabled; + }, + setMultiReturnMode(mode) { + lidarMultiReturnMode = mode === "last" ? "last" : "strongest"; + _lidarAccumFrames.length = 0; + _lidarLastAccumPose = null; + resetLidarScanState(); + updateSimSensorButtons(); + if (simSensorViewMode === "lidar") updateLidarPointCloud(); + return lidarMultiReturnMode; + }, + getMultiReturnMode() { + return lidarMultiReturnMode; + }, +}; + +window.__robovalRgbd = { + // Returns metric camera-space Z depth map in meters (Float32Array length W*H). + // Uses the same render path as on-screen RGB-D mode. + getMetricDepthFrame() { + renderRgbdView(); + const depth = readRgbdMetricDepthFrameMeters(); + if (!depth) return null; + return { + width: rgbdMetricTarget.width, + height: rgbdMetricTarget.height, + depth_m: depth, + semantics: "camera_space_z", + units: "meters", + min_depth_m: RGBD_MIN_DEPTH_M, + max_depth_m: RGBD_MAX_DEPTH_M, + }; + }, + getOverlayStats() { + return { + enabled: rgbdPcOverlayOnLidar, + visible: rgbdPcOverlayGroup.visible, + points: _rgbdPcOverlayLastCount, + rt_w: RGBD_PC_OVERLAY_RT_W, + rt_h: RGBD_PC_OVERLAY_RT_H, + dirty: _rgbdPcOverlayDirty, + }; + }, +}; + +// Debug: List all colliders in the physics world +window.debugColliders = function() { + if (!rapierWorld) { + console.log("[DEBUG] No physics world loaded"); + return; + } + + console.log("[DEBUG] === ALL COLLIDERS IN PHYSICS WORLD ==="); + let count = 0; + rapierWorld.colliders.forEach((collider) => { + const pos = collider.translation(); + const shape = collider.shape; + const isSensor = collider.isSensor(); + const handle = collider.handle; + console.log(`Collider #${count} (handle=${handle}): pos=(${pos.x.toFixed(2)}, ${pos.y.toFixed(2)}, ${pos.z.toFixed(2)}), sensor=${isSensor}, shapeType=${shape.type}`); + count++; + }); + console.log(`[DEBUG] Total colliders: ${count}`); + + // Also show asset collider handles + console.log("[DEBUG] === ASSET COLLIDERS (on asset objects) ==="); + let assetColCount = 0; + for (const a of assets) { + if (a._colliderHandle) { + const handleInfo = typeof a._colliderHandle === 'object' ? `obj.handle=${a._colliderHandle.handle}` : `num=${a._colliderHandle}`; + console.log(` ${a.id}: "${a.title}", _colliderHandle=${handleInfo}`); + assetColCount++; + } + } + console.log(`[DEBUG] Assets with colliders: ${assetColCount}`); + + // Show tracked map + console.log("[DEBUG] === _assetColliderHandles Map ==="); + console.log(`Map size: ${_assetColliderHandles.size}`); +}; + +// Debug: Remove all colliders except world/player +window.debugClearAssetColliders = function() { + if (!rapierWorld) return; + + // Helper to remove a collider (handles both object and number) + const removeCol = (handle) => { + try { + if (typeof handle === 'object' && handle.handle !== undefined) { + rapierWorld.removeCollider(handle, true); + return true; + } else if (typeof handle === 'number') { + const collider = rapierWorld.getCollider(handle); + if (collider) { + rapierWorld.removeCollider(collider, true); + return true; + } + } + } catch (e) {} + return false; + }; + + let removed = 0; + + // Remove all tracked asset colliders + _assetColliderHandles.forEach((handle, assetId) => { + if (removeCol(handle)) removed++; + }); + _assetColliderHandles.clear(); + + // Also clear colliders stored on asset objects + for (const asset of assets) { + if (asset._colliderHandle != null) { + if (removeCol(asset._colliderHandle)) removed++; + asset._colliderHandle = null; + } + } + + console.log(`[DEBUG] Cleared ${removed} asset colliders`); +}; + +// ── dimos integration mode boot ────────────────────────────────────────────── +// When dimosMode is active, auto-load a scene and spawn an agent, then connect +// the LCM bridge so sensor data flows and external /odom drives the agent. +if (dimosMode) { + (async () => { + try { + // 1. Auto-load scene + const sceneName = dimosScene || "apt"; + console.log(`[dimos] Loading scene: ${sceneName}`); + const resp = await fetch(`/sims/${sceneName}.json`); + if (!resp.ok) throw new Error(`Scene fetch failed: HTTP ${resp.status}`); + const sceneJson = await resp.json(); + await importLevelFromJSON(sceneJson); + console.log(`[dimos] Scene loaded: ${sceneName}`); + + // 2. Auto-spawn agent (wait for physics to settle) + await new Promise((r) => setTimeout(r, 1500)); + await ensureRapierLoaded(); + // Re-apply grid floor now that rapier is loaded (creates collider) + spawnPlayerInsideScene(); + const agent = createAiAgent({ ephemeral: false }); + aiAgents.push(agent); + // Place agent at a default spawn point + const spawnPos = sceneJson.dimosSpawnPoint || { x: 2, y: 0.5, z: 3 }; + agent.setPosition(spawnPos.x, spawnPos.y, spawnPos.z); + renderAgentTaskUi(); // update UI: hide spawn button, enable task controls + // Server-side physics: agent pose is driven by ServerPhysics (Deno). + // Browser just receives position updates and moves the visual avatar. + let _dimosYaw = 0; + // Bridge updates _dimosYaw via this setter when server sends pose + window.__dimosSetYaw = (yaw) => { _dimosYaw = yaw; }; + agent.update = function(_dt) { + this._syncVisual(); + }; + console.log(`[dimos] Agent spawned: ${agent.id}`); + + // 3. Set up fixed-size offscreen capture for dimos. + // Keep sensor cost independent of the headed browser window size. + const _dimosCapW = 640, _dimosCapH = 288; + const _dimosCapTarget = new THREE.WebGLRenderTarget(_dimosCapW, _dimosCapH, { + minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, + format: THREE.RGBAFormat, depthBuffer: true, stencilBuffer: false, + }); + // Go2 depth camera: 87° horizontal. At 640x288 (2.22:1 aspect), that's 46° vertical. + const _dimosFov = window.__dimosCameraFov || 46; + const _dimosCapCam = new THREE.PerspectiveCamera(_dimosFov, _dimosCapW / _dimosCapH, camera.near, camera.far); + const _dimosCapBuf = new Uint8Array(_dimosCapW * _dimosCapH * 4); + const _dimosCapCvs = document.createElement("canvas"); + _dimosCapCvs.width = _dimosCapW; + _dimosCapCvs.height = _dimosCapH; + const _dimosCapCtx = _dimosCapCvs.getContext("2d"); + const _dimosDepthTarget = new THREE.WebGLRenderTarget(_dimosCapW, _dimosCapH, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: THREE.RGBAFormat, + type: THREE.UnsignedByteType, + depthBuffer: true, + stencilBuffer: false, + }); + _dimosDepthTarget.texture.generateMipmaps = false; + _dimosDepthTarget.depthTexture = new THREE.DepthTexture(_dimosCapW, _dimosCapH, THREE.UnsignedIntType); + _dimosDepthTarget.depthTexture.minFilter = THREE.NearestFilter; + _dimosDepthTarget.depthTexture.magFilter = THREE.NearestFilter; + _dimosDepthTarget.depthTexture.generateMipmaps = false; + const _dimosMetricTarget = new THREE.WebGLRenderTarget(_dimosCapW, _dimosCapH, { + minFilter: THREE.NearestFilter, + magFilter: THREE.NearestFilter, + format: rgbdMetricUsesR32F ? THREE.RedFormat : THREE.RGBAFormat, + type: rgbdMetricTargetType, + depthBuffer: false, + stencilBuffer: false, + }); + if (rgbdMetricUsesR32F) _dimosMetricTarget.texture.internalFormat = "R32F"; + _dimosMetricTarget.texture.generateMipmaps = false; + + function _dimosReadMetricDepthFrameMeters() { + const w = _dimosMetricTarget.width; + const h = _dimosMetricTarget.height; + if (!w || !h) return null; + + if (rgbdMetricUsesR32F) { + const depth = new Float32Array(w * h); + renderer.readRenderTargetPixels(_dimosMetricTarget, 0, 0, w, h, depth); + return depth; + } + + if (_dimosMetricTarget.texture.type === THREE.FloatType) { + const raw = new Float32Array(w * h * 4); + renderer.readRenderTargetPixels(_dimosMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = raw[i * 4 + 0]; + return depth; + } + + const raw = new Uint16Array(w * h * 4); + renderer.readRenderTargetPixels(_dimosMetricTarget, 0, 0, w, h, raw); + const depth = new Float32Array(w * h); + for (let i = 0; i < w * h; i++) depth[i] = halfToFloat(raw[i * 4 + 0]); + return depth; + } + + function _dimosCaptureRgb() { + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch), sp = Math.sin(pitch); + const feetY = ay - ((agent.halfHeight || 0.25) + (agent.radius || 0.12)); + const eyeY = feetY + GO2_CAMERA_HEIGHT; + const eyeX = ax + Math.sin(yaw) * GO2_CAMERA_FORWARD; + const eyeZ = az + Math.cos(yaw) * GO2_CAMERA_FORWARD; + _dimosCapCam.position.set(eyeX, eyeY, eyeZ); + _dimosCapCam.lookAt(eyeX + Math.sin(yaw)*cp, eyeY + sp, eyeZ + Math.cos(yaw)*cp); + _dimosCapCam.updateProjectionMatrix(); + _dimosCapCam.updateMatrixWorld(true); + + const prev = renderer.getRenderTarget(); + const prevAgentVisible = agent.group?.visible; + if (agent.group) agent.group.visible = false; + renderer.setRenderTarget(_dimosCapTarget); + renderer.render(scene, _dimosCapCam); + renderer.setRenderTarget(prev); + if (agent.group) agent.group.visible = prevAgentVisible; + + renderer.readRenderTargetPixels(_dimosCapTarget, 0, 0, _dimosCapW, _dimosCapH, _dimosCapBuf); + // Flip Y — return raw RGBA pixels (no JPEG encode) + const flipped = new Uint8Array(_dimosCapW * _dimosCapH * 4); + const rowB = _dimosCapW * 4; + for (let y = 0; y < _dimosCapH; y++) { + flipped.set(_dimosCapBuf.subarray((_dimosCapH-1-y)*rowB, (_dimosCapH-y)*rowB), y*rowB); + } + return { data: flipped, width: _dimosCapW, height: _dimosCapH }; + } + + // Offscreen depth capture from agent POV using a dedicated low-res target. + function _dimosCaptureDepth() { + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + const yaw = agent.group?.rotation?.y ?? 0; + const pitch = typeof agent.pitch === "number" ? agent.pitch : 0; + const cp = Math.cos(pitch), sp = Math.sin(pitch); + const feetY = ay - ((agent.halfHeight || 0.25) + (agent.radius || 0.12)); + const eyeY = feetY + GO2_CAMERA_HEIGHT; + const eyeX = ax + Math.sin(yaw) * GO2_CAMERA_FORWARD; + const eyeZ = az + Math.cos(yaw) * GO2_CAMERA_FORWARD; + _dimosCapCam.position.set(eyeX, eyeY, eyeZ); + _dimosCapCam.lookAt(eyeX + Math.sin(yaw)*cp, eyeY + sp, eyeZ + Math.cos(yaw)*cp); + _dimosCapCam.updateProjectionMatrix(); + _dimosCapCam.updateMatrixWorld(true); + + const prevDepthTex = rgbdMetricMaterial.uniforms.uDepthTex.value; + const prevNear = rgbdMetricMaterial.uniforms.uNear.value; + const prevFar = rgbdMetricMaterial.uniforms.uFar.value; + const prevNoise = rgbdMetricMaterial.uniforms.uNoiseEnabled.value; + const prevSpeckle = rgbdMetricMaterial.uniforms.uSpeckleEnabled.value; + rgbdMetricMaterial.uniforms.uDepthTex.value = _dimosDepthTarget.depthTexture; + rgbdMetricMaterial.uniforms.uNear.value = _dimosCapCam.near; + rgbdMetricMaterial.uniforms.uFar.value = _dimosCapCam.far; + rgbdMetricMaterial.uniforms.uNoiseEnabled.value = rgbdNoiseEnabled ? 1.0 : 0.0; + rgbdMetricMaterial.uniforms.uSpeckleEnabled.value = rgbdSpeckleEnabled ? 1.0 : 0.0; + + const savedOverride = scene.overrideMaterial; + const savedAssets = assetsGroup.visible; + const savedPrims = primitivesGroup.visible; + const savedLights = lightsGroup.visible; + const savedTags = tagsGroup.visible; + const savedLidarViz = lidarVizGroup.visible; + const savedRgbdPc = rgbdPcOverlayGroup.visible; + + scene.overrideMaterial = null; + assetsGroup.visible = true; + primitivesGroup.visible = true; + lightsGroup.visible = true; + tagsGroup.visible = false; + lidarVizGroup.visible = false; + rgbdPcOverlayGroup.visible = false; + const savedAgentVisible = agent.group?.visible; + if (agent.group) agent.group.visible = false; + + renderer.setRenderTarget(_dimosDepthTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(scene, _dimosCapCam); + + renderer.setRenderTarget(_dimosMetricTarget); + renderer.setClearColor(0x000000, RGBD_CLEAR_ALPHA); + renderer.clear(true, true, true); + renderer.render(rgbdMetricScene, rgbdPostCamera); + + scene.overrideMaterial = savedOverride; + assetsGroup.visible = savedAssets; + primitivesGroup.visible = savedPrims; + lightsGroup.visible = savedLights; + tagsGroup.visible = savedTags; + lidarVizGroup.visible = savedLidarViz; + rgbdPcOverlayGroup.visible = savedRgbdPc; + if (agent.group) agent.group.visible = savedAgentVisible; + rgbdMetricMaterial.uniforms.uDepthTex.value = prevDepthTex; + rgbdMetricMaterial.uniforms.uNear.value = prevNear; + rgbdMetricMaterial.uniforms.uFar.value = prevFar; + rgbdMetricMaterial.uniforms.uNoiseEnabled.value = prevNoise; + rgbdMetricMaterial.uniforms.uSpeckleEnabled.value = prevSpeckle; + renderer.setRenderTarget(null); + + const depthData = _dimosReadMetricDepthFrameMeters(); + if (!depthData) return null; + + const dw = _dimosMetricTarget.width, dh = _dimosMetricTarget.height; + + // Flip rows: WebGL reads bottom-to-top, image convention is top-to-bottom + const flipped = new Float32Array(dw * dh); + for (let y = 0; y < dh; y++) { + flipped.set(depthData.subarray((dh - 1 - y) * dw, (dh - y) * dw), y * dw); + } + return { data: flipped, width: dw, height: dh }; + } + + // 4. Sidebar sensor panel setup (depth + LiDAR canvases) + const _dimosSidebarW = 320, _dimosSidebarH = 145; + const _dimosDepthCanvas = document.getElementById("agent-depth-canvas"); + const _dimosLidarCanvas = document.getElementById("agent-lidar-canvas"); + if (_dimosDepthCanvas) { _dimosDepthCanvas.width = _dimosSidebarW; _dimosDepthCanvas.height = _dimosSidebarH; } + if (_dimosLidarCanvas) { _dimosLidarCanvas.width = _dimosSidebarW; _dimosLidarCanvas.height = _dimosSidebarH; } + const _dimosDepthCtx = _dimosDepthCanvas?.getContext("2d"); + const _dimosLidarCtx = _dimosLidarCanvas?.getContext("2d"); + + // Small offscreen render targets for sidebar panels + const _dimosSidebarDepthTarget = new THREE.WebGLRenderTarget(_dimosSidebarW, _dimosSidebarH, { + minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, + format: THREE.RGBAFormat, depthBuffer: true, stencilBuffer: false, + }); + const _dimosSidebarLidarTarget = new THREE.WebGLRenderTarget(_dimosSidebarW, _dimosSidebarH, { + minFilter: THREE.LinearFilter, magFilter: THREE.LinearFilter, + format: THREE.RGBAFormat, depthBuffer: true, stencilBuffer: false, + }); + const _dimosSidebarReadBuf = new Uint8Array(_dimosSidebarW * _dimosSidebarH * 4); + + // Helper: render a target, readback, flip Y, draw to 2D canvas + function _dimosBlitToCanvas(rt, ctx, w, h) { + renderer.readRenderTargetPixels(rt, 0, 0, w, h, _dimosSidebarReadBuf); + const flipped = new Uint8ClampedArray(w * h * 4); + const rowB = w * 4; + for (let y = 0; y < h; y++) { + flipped.set(_dimosSidebarReadBuf.subarray((h-1-y)*rowB, (h-y)*rowB), y*rowB); + } + ctx.putImageData(new ImageData(flipped, w, h), 0, 0); + } + + /** Update the sidebar sensor panels (called after capture) */ + function _dimosUpdateSidebarPanels(rgbBase64) { + if (window.__dimosHeadless) return; + + // RGB — set img src + if (rgbBase64 && agentShotImgEl) { + agentShotImgEl.src = `data:image/jpeg;base64,${rgbBase64}`; + } + + // Depth — render colormap to small target, blit to canvas + if (_dimosDepthCtx) { + const prev = renderer.getRenderTarget(); + rgbdVizMaterial.uniforms.uGrayMode.value = rgbdVizMode === "gray" ? 1.0 : 0.0; + renderer.setRenderTarget(_dimosSidebarDepthTarget); + renderer.setClearColor(0x000000, 1); + renderer.clear(true, true, true); + renderer.render(rgbdVizScene, rgbdPostCamera); + _dimosBlitToCanvas(_dimosSidebarDepthTarget, _dimosDepthCtx, _dimosSidebarW, _dimosSidebarH); + renderer.setRenderTarget(prev); + } + + // LiDAR — render lidar scene from agent POV to small target, blit to canvas + if (_dimosLidarCtx) { + const prev = renderer.getRenderTarget(); + // Save/restore scene visibility for lidar-only render + const savedAssets = assetsGroup.visible; + const savedPrims = primitivesGroup.visible; + const savedLights = lightsGroup.visible; + const savedTags = tagsGroup.visible; + const savedLidar = lidarVizGroup.visible; + const savedOverlay = rgbdPcOverlayGroup.visible; + const savedBg = scene.background; + + assetsGroup.visible = false; + primitivesGroup.visible = false; + lightsGroup.visible = false; + tagsGroup.visible = false; + lidarVizGroup.visible = true; + rgbdPcOverlayGroup.visible = false; + scene.background = RGBD_BG; + + renderer.setRenderTarget(_dimosSidebarLidarTarget); + renderer.setClearColor(0x000000, 1); + renderer.clear(true, true, true); + renderer.render(scene, _dimosCapCam); + + // Restore + assetsGroup.visible = savedAssets; + primitivesGroup.visible = savedPrims; + lightsGroup.visible = savedLights; + tagsGroup.visible = savedTags; + lidarVizGroup.visible = savedLidar; + rgbdPcOverlayGroup.visible = savedOverlay; + scene.background = savedBg; + + _dimosBlitToCanvas(_dimosSidebarLidarTarget, _dimosLidarCtx, _dimosSidebarW, _dimosSidebarH); + renderer.setRenderTarget(prev); + } + } + + // 5. Connect dimos bridge + let _lastRgbBase64 = null; + const { DimosBridge } = await import("./dimos/dimosBridge.ts"); + const bridge = new DimosBridge({ + agent, + rates: window.__dimosSensorRates || undefined, + sensorEnable: window.__dimosSensorEnable || undefined, + sensorSources: { + captureRgb: () => { + const frame = _dimosCaptureRgb(); + if (!frame) return null; + // Render to canvas → JPEG (used for both LCM publish and eval/sidebar) + _dimosCapCtx.putImageData(new ImageData(new Uint8ClampedArray(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength), frame.width, frame.height), 0, 0); + const dataUrl = _dimosCapCvs.toDataURL("image/jpeg", 0.75); + _lastRgbBase64 = dataUrl.split("base64,")[1] || null; + if (!_lastRgbBase64) return null; + // Decode base64 → Uint8Array for JPEG LCM transport + const bin = atob(_lastRgbBase64); + const jpegBytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) jpegBytes[i] = bin.charCodeAt(i); + return { data: jpegBytes, width: frame.width, height: frame.height }; + }, + captureDepth: () => _dimosCaptureDepth(), + captureLidar: () => { + // Return world-frame points (Three.js Y-up). + // Bridge converts Y-up → ROS Z-up and labels frame_id="world". + const lLen = _lidarLatestWorldPts ? _lidarLatestWorldPts.length : -1; + if (lLen > 0) { + return { + points: _lidarLatestWorldPts, + intensity: _lidarLatestWorldIntensity, + numPoints: lLen / 3, + }; + } + const frames = window.__robovalLidar?.getLatestFrames?.(); + const src = frames?.raw; + if (!src) return null; + return { points: src.points, intensity: src.intensity, numPoints: src.points?.length / 3 || 0 }; + }, + getOdomPose: () => { + const pos = agent.getPosition?.(); + if (!pos) return null; // skip this frame instead of fallback to origin + const [ax, ay, az] = pos; + const qw = Math.cos(_dimosYaw / 2); + const qy = Math.sin(_dimosYaw / 2); + return { x: ax, y: ay, z: az, qx: 0, qy, qz: 0, qw }; + }, + }, + }); + + // Hook: after _publishSensors — RGB capture disabled for now (GPU stall) + // _dimosCaptureRgb() does a full render + readRenderTargetPixels which + // stalls the GPU pipeline. Skip it for lidar-only nav testing. + // const origPublishSensors = bridge._publishSensors.bind(bridge); + // bridge._publishSensors = function() { + // origPublishSensors(); + // _lastRgbBase64 = _dimosCaptureRgb(); + // _dimosUpdateSidebarPanels(_lastRgbBase64); + // }; + + bridge.connect(); + bridge.sceneReady = true; + window.__dimosBridge = bridge; + window.__dimosCapCam = _dimosCapCam; + window.__dimosAgent = agent; + + // Send Rapier world snapshot to bridge server for server-side physics + lidar. + // Flush any deferred collider builds first — primitives (floor, walls) may be + // queued in _pendingColliderBuilds if they were created before the render loop ran. + flushPendingColliderBuilds(); + let _snapshotColliders = 0; + rapierWorld.colliders.forEach(() => { _snapshotColliders++; }); + console.log(`[dimos] Flushed collider queue — ${_snapshotColliders} colliders in world before snapshot`); + + // Format: [DSSN 4B][spawnX f32][spawnY f32][spawnZ f32][snapshot...] + const _waitSensorWs = () => { + if (bridge.wsSensors && bridge.wsSensors.readyState === WebSocket.OPEN) { + try { + const snapshot = rapierWorld.takeSnapshot(); + const [sx, sy, sz] = agent.getPosition?.() || [2, 0.5, 3]; + const prefixed = new Uint8Array(16 + snapshot.byteLength); + const dv = new DataView(prefixed.buffer); + dv.setUint32(0, 0x44535332, false); // "DSS2" — includes spawn position + dv.setFloat32(4, sx, true); + dv.setFloat32(8, sy, true); + dv.setFloat32(12, sz, true); + prefixed.set(snapshot, 16); + bridge.wsSensors.send(prefixed.buffer); + bridge._serverLidar = true; + console.log(`[DimosBridge] sent Rapier snapshot (${(snapshot.byteLength / 1024).toFixed(0)}KB) spawn=(${sx.toFixed(1)},${sy.toFixed(1)},${sz.toFixed(1)}) — server physics + lidar active`); + } catch (e) { + console.warn("[DimosBridge] snapshot send failed:", e); + } + } else { + setTimeout(_waitSensorWs, 200); + } + }; + _waitSensorWs(); + // Expose yaw for lidar pose sampling (avoids reading Three.js Euler) + Object.defineProperty(window, '__dimosYaw', { get: () => _dimosYaw }); + + // Odom: server-side physics publishes odom directly to LCM. + // Browser no longer needs to publish odom — server is authoritative. + + // Eval harness — scores objectDistance rubric when triggered by dimsim eval runner + const { EvalHarness } = await import("./dimos/evalHarness.ts"); + const channel = new URLSearchParams(location.search).get("channel") || undefined; + const evalHarness = new EvalHarness({ + bridge, + channel, + getSceneState: () => { + const enriched = assets.map(a => { + const obj = assetsGroup.getObjectByName(`asset:${a.id}`); + if (obj) { + const bbox = new THREE.Box3().setFromObject(obj); + if (!bbox.isEmpty()) { + const center = new THREE.Vector3(); + const size = new THREE.Vector3(); + bbox.getCenter(center); + bbox.getSize(size); + return { + ...a, + transform: { x: center.x, y: center.y, z: center.z }, + _bbox: { w: size.x, h: size.y, d: size.z }, + }; + } + } + return a; + }); + return { assets: enriched }; + }, + getAgentPose: () => { + const pos = agent.getPosition?.(); + if (!pos) return null; + const camOffset = 0.3; + const cx = pos[0] + Math.sin(_dimosYaw) * camOffset; + const cz = pos[2] + Math.cos(_dimosYaw) * camOffset; + return { x: cx, y: pos[1], z: cz, yaw: _dimosYaw, pitch: 0 }; + }, + }); + window.__evalHarness = evalHarness; + + // Scene editor — script execution engine for sim editing (exec_js API) + const { SceneEditor } = await import("./dimos/sceneEditor.ts"); + const sceneEditor = new SceneEditor({ + bridge, + channel, + globals: { scene, THREE, RAPIER, rapierWorld, worldBody, renderer, camera, agent, assets, assetsGroup, gltfLoader }, + }); + window.__sceneEditor = sceneEditor; + + // Agent POV only in headless (sensor capture needs it). Headed = free orbit. + if (window.__dimosHeadless) { + enableAgentCameraFollow(agent.id); + } + + // 7a. dimos mode UI cleanup handled in CSS via body.dimos-mode class + // (panel hiding) and .shortcuts-floating in index.html (WASD strip). + + // 7b. Debug panel (integration diagnostics) — hidden for now + if (false && !window.__dimosHeadless) { + const dbg = document.createElement("div"); + dbg.id = "dimos-debug"; + dbg.style.cssText = "position:fixed;bottom:8px;left:8px;z-index:99999;background:rgba(0,0,0,0.88);color:#0f0;font:11px/1.4 monospace;padding:10px 14px;border-radius:8px;max-width:460px;max-height:400px;overflow-y:auto;pointer-events:auto;user-select:text;"; + document.body.appendChild(dbg); + + const _dbgState = { + bridgeConn: false, + sensorFps: 0, + agentPos: { x: 0, y: 0, z: 0 }, + agentYaw: 0, + cmdVel: { angY: 0, linZ: 0 }, + _sensorCount: 0, + _sensorLastTs: Date.now(), + }; + + // Hook lidar publish for FPS counter + const _origPubLidar2 = bridge._publishLidar; + bridge._publishLidar = function() { + _dbgState._sensorCount++; + _origPubLidar2.call(bridge); + }; + + // Update loop + setInterval(() => { + const now = Date.now(); + const dt = (now - _dbgState._sensorLastTs) / 1000; + if (dt >= 1) { + _dbgState.sensorFps = Math.round(_dbgState._sensorCount / dt); + _dbgState._sensorCount = 0; + _dbgState._sensorLastTs = now; + } + + const [ax, ay, az] = agent.getPosition?.() || [0, 0, 0]; + _dbgState.agentPos = { x: ax.toFixed(2), y: ay.toFixed(2), z: az.toFixed(2) }; + _dbgState.agentYaw = (agent.group?.rotation?.y ?? 0).toFixed(3); + _dbgState.bridgeConn = bridge.ws?.readyState === WebSocket.OPEN; + const vel = bridge.getCmdVel(); + _dbgState.cmdVel = { angY: vel.angY.toFixed(3), linZ: vel.linZ.toFixed(3) }; + + dbg.innerHTML = ` +
dimos integration
+
Bridge: ${_dbgState.bridgeConn ? 'connected' : 'disconnected'} | Sensors: ${_dbgState.sensorFps} fps
+
Agent: (${_dbgState.agentPos.x}, ${_dbgState.agentPos.y}, ${_dbgState.agentPos.z}) yaw=${_dbgState.agentYaw}
+
cmd_vel: angY=${_dbgState.cmdVel.angY} linZ=${_dbgState.cmdVel.linZ}
+ `; + }, 500); + } + + console.log("[dimos] Bridge connected. Sensor publishing active."); + } catch (err) { + console.error("[dimos] Initialization failed:", err); + } + })(); +} diff --git a/misc/DimSim/src/main.js b/misc/DimSim/src/main.js new file mode 100644 index 0000000000..f237e9a49d --- /dev/null +++ b/misc/DimSim/src/main.js @@ -0,0 +1,4 @@ +// DimSim engine entry point. +// Imports the full engine from ./engine.js (copied from SimStudio/src/main.js). +// To update: bash copy-sources.sh +import "./engine.js"; diff --git a/misc/DimSim/src/style.css b/misc/DimSim/src/style.css new file mode 100644 index 0000000000..5ae5a2e7c4 --- /dev/null +++ b/misc/DimSim/src/style.css @@ -0,0 +1,3188 @@ +/* ============================================================================ + SimStudio - Professional Training Environment + ============================================================================ */ + +:root { + color-scheme: dark; + + /* Core Colors — flat terminal/cyberdeck */ + --bg-primary: #06080b; + --bg-secondary: #0b0e13; + --bg-tertiary: #10141b; + --bg-elevated: rgba(9, 12, 17, 0.96); + + /* Surface Colors */ + --surface-1: rgba(255, 255, 255, 0.02); + --surface-2: rgba(255, 255, 255, 0.04); + --surface-3: rgba(255, 255, 255, 0.07); + + /* Border Colors — hairline */ + --border-subtle: rgba(255, 255, 255, 0.06); + --border-default: rgba(255, 255, 255, 0.10); + --border-strong: rgba(255, 255, 255, 0.16); + + /* Text Colors */ + --text-primary: rgba(255, 255, 255, 0.95); + --text-secondary: rgba(255, 255, 255, 0.68); + --text-tertiary: rgba(255, 255, 255, 0.44); + + /* Accent Colors — dimos-style cool cyan-white */ + --accent-primary: #d6dde6; + --accent-primary-hover: #ffffff; + --accent-primary-subtle: rgba(214, 221, 230, 0.14); + --accent-primary-glow: rgba(214, 221, 230, 0.28); + + --accent-success: #7fe3c4; + --accent-success-subtle: rgba(127, 227, 196, 0.14); + + --accent-warning: #f5c06b; + --accent-warning-subtle: rgba(245, 192, 107, 0.12); + + --accent-info: #7fd4f5; + --accent-info-subtle: rgba(127, 212, 245, 0.12); + + /* Shadows — flat, no heavy drop shadows */ + --shadow-sm: 0 0 0 1px rgba(0, 0, 0, 0.2); + --shadow-md: 0 0 0 1px rgba(0, 0, 0, 0.3); + --shadow-lg: 0 0 0 1px rgba(0, 0, 0, 0.4); + --shadow-xl: 0 0 0 1px rgba(0, 0, 0, 0.5); + + /* Transitions */ + --transition-fast: 0.1s linear; + --transition-normal: 0.15s linear; + --transition-slow: 0.2s linear; + + /* Border Radius — sharp corners */ + --radius-sm: 0px; + --radius-md: 0px; + --radius-lg: 0px; + --radius-xl: 2px; + + /* Font Stacks */ + --font-mono: 'JetBrains Mono', 'IBM Plex Mono', 'SF Mono', 'Fira Code', 'Menlo', 'Monaco', 'Consolas', ui-monospace, monospace; + --font-sans: 'Inter', ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; +} + +/* ============================================================================ + Base Styles + ============================================================================ */ + +html, body { + height: 100%; + margin: 0; +} + +body { + overflow: hidden; + font-family: var(--font-sans); + background: var(--bg-primary); + color: var(--text-primary); + font-size: 13px; + line-height: 1.5; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +/* Google Font Import */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800;900&display=swap'); + +/* ============================================================================ + Canvas + ============================================================================ */ + +#c { + position: fixed; + inset: 0; + width: 100%; + height: 100%; + display: block; +} + +/* ============================================================================ + Overlay Layout + ============================================================================ */ + +#overlay { + position: fixed; + inset: 0; + pointer-events: none; + z-index: 100; + padding: 0; + box-sizing: border-box; + display: grid; + grid-template-columns: 330px 1fr 380px; + grid-template-rows: auto 1fr; + grid-template-areas: + "top top top" + "left . right"; + gap: 0; +} + +/* Collapsed states for edit mode panels */ +#overlay.left-collapsed { + grid-template-columns: 0 1fr 380px; +} +#overlay.right-collapsed { + grid-template-columns: 330px 1fr 0; +} +#overlay.left-collapsed.right-collapsed { + grid-template-columns: 0 1fr 0; +} + +html[data-mode="sim"] #overlay { + grid-template-columns: 1fr 340px; + grid-template-rows: 1fr; + grid-template-areas: ". panel"; + padding: 16px; + gap: 16px; +} + +html[data-mode="sim"] #overlay.sim-panel-collapsed { + grid-template-columns: 1fr 0; +} + +/* ============================================================================ + Slim Toolbar (UE-inspired) + ============================================================================ */ + +#overlay-top { + pointer-events: auto; + grid-area: top; + display: flex; + gap: 4px; + align-items: center; + padding: 4px 8px; + background: #1a1d24; + border-bottom: 1px solid rgba(255,255,255,0.08); + box-shadow: 0 2px 8px rgba(0,0,0,0.4); + flex-wrap: wrap; + overflow: visible; + min-height: 36px; + position: relative; + z-index: 10; +} + +/* Toolbar generic button */ +.tb-btn { + display: inline-flex; + align-items: center; + gap: 5px; + padding: 5px 10px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--text-secondary); + background: transparent; + border: 1px solid transparent; + border-radius: 0; + cursor: pointer; + white-space: nowrap; + transition: all 0.1s linear; +} +.tb-btn:hover { background: rgba(255,255,255,0.04); color: var(--text-primary); border-color: var(--border-default); } +.tb-btn.active { color: var(--accent-primary); background: var(--accent-primary-subtle); border-color: var(--accent-primary); } +.tb-btn.tb-primary { background: transparent; color: var(--accent-primary); border: 1px solid var(--accent-primary); } +.tb-btn.tb-primary:hover { background: var(--accent-primary-subtle); color: var(--accent-primary-hover); border-color: var(--accent-primary-hover); } +.tb-btn.tb-danger { color: #ef4444; } +.tb-btn.tb-danger:hover { background: rgba(239,68,68,0.12); color: #f87171; } +.tb-btn.tb-muted { color: var(--text-tertiary); font-weight: 500; } +.tb-btn.tb-muted:hover { color: var(--text-secondary); } +.tb-btn svg { flex-shrink: 0; } + +/* File upload styled as toolbar button */ +.tb-file-label { cursor: pointer; } +.tb-file-label input[type="file"] { display: none; } + +/* Toolbar select */ +.tb-select { + padding: 4px 8px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + background: rgba(255,255,255,0.03); + border: 1px solid var(--border-default); + border-radius: 0; + color: var(--text-primary); + cursor: pointer; + outline: none; + max-width: 130px; +} +.tb-select:focus { border-color: var(--accent-primary); } + +/* Toolbar separator */ +.tb-sep { + width: 1px; + height: 20px; + background: rgba(255,255,255,0.1); + margin: 0 2px; + flex-shrink: 0; +} +.tb-spacer { flex: 1; } + +/* Toolbar transform group */ +.tb-transform { + display: flex; + gap: 2px; + padding: 2px; + background: rgba(255,255,255,0.02); + border: 1px solid var(--border-subtle); + border-radius: 0; +} + +/* Toolbar status */ +.tb-status { + font-size: 11px; + color: var(--text-tertiary); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + max-width: 200px; +} +.tb-status:empty { display: none; } + +/* Toolbar group */ +.tb-group { display: flex; gap: 3px; align-items: center; } + +.workspace-tab-strip { + display: inline-flex; + gap: 0; + padding: 0; + background: transparent; + border: 1px solid var(--border-default); + border-radius: 0; + margin: 0 4px; +} +.workspace-tab-strip .tb-btn { + padding: 4px 12px; + font-size: 11px; + font-weight: 500; + border-radius: 0; + border: none; + color: var(--text-tertiary); + transition: all 0.1s linear; +} +.workspace-tab-strip .tb-btn:hover { + color: var(--text-secondary); + background: rgba(255,255,255,0.04); +} +.workspace-tab-strip .tb-btn.active { + color: var(--accent-primary-hover); + background: var(--accent-primary-subtle); + box-shadow: inset 0 -2px 0 var(--accent-primary); +} + +/* Builder-mode inline shape palette */ +.builder-shape-bar { + display: inline-flex; + gap: 2px; + padding: 2px; + background: rgba(255,255,255,0.02); + border: 1px solid var(--border-subtle); + border-radius: 0; +} +.builder-shape-bar .tb-btn { + padding: 4px 8px; + font-size: 12px; + gap: 3px; +} +.builder-shape-bar .shape-icon { + font-size: 11px; + opacity: 0.7; +} + +/* Advanced dropdown */ +.tb-advanced { position: relative; } +.tb-advanced > summary { list-style: none; } +.tb-advanced > summary::-webkit-details-marker { display: none; } +.tb-advanced-body { + position: absolute; + top: calc(100% + 4px); + right: 0; + z-index: 50; + background: var(--bg-secondary); + border: 1px solid var(--border-default); + border-radius: 0; + padding: 4px; + display: flex; + gap: 4px; + box-shadow: none; +} + +/* Legacy world-selector compat (now .tb-group) */ +.world-selector { display: contents; } + +/* ============================================================================ + Buttons + ============================================================================ */ + +button, +.file span { + cursor: pointer; + user-select: none; + padding: 7px 14px; + border-radius: 0; + border: 1px solid var(--border-default); + background: transparent; + color: var(--text-primary); + font-weight: 500; + font-size: 12px; + font-family: var(--font-mono); + letter-spacing: 0.05em; + text-transform: uppercase; + transition: all var(--transition-fast); + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; +} + +/* Toolbar buttons override the heavy defaults */ +.tb-btn, #overlay-top button, #overlay-top .file span { + padding: 5px 10px; + border-radius: 0; + border: 1px solid transparent; + background: transparent; + font-size: 11px; + font-family: var(--font-mono); + transform: none !important; +} + +button:hover, +.file span:hover { + border-color: var(--accent-primary); + background: var(--accent-primary-subtle); + color: var(--accent-primary-hover); +} + +button:active, +.file span:active { + transform: none; +} + +button:disabled { + opacity: 0.5; + cursor: not-allowed; + transform: none; +} + +/* Primary Action Button */ +button.primary { + background: transparent; + border: 1px solid var(--accent-primary); + color: var(--accent-primary); + box-shadow: none; +} + +button.primary:hover { + background: var(--accent-primary-subtle); + border-color: var(--accent-primary-hover); + color: var(--accent-primary-hover); + box-shadow: none; +} + +/* Portal Button */ +#portal-create-btn { + background: var(--surface-2); + border: 1px solid var(--border-default); + color: var(--text-primary); + box-shadow: none; +} + +#portal-create-btn:hover { + background: var(--surface-3); + border-color: var(--border-strong); + box-shadow: var(--shadow-sm); +} + +/* Portal Exit Modal */ +#portal-exit-modal .modal-title { + background: var(--surface-1); +} + +#portal-exit-modal .modal-hint { + border-left-color: var(--accent-primary); + background: var(--accent-primary-subtle); +} + +#portal-exit-world-name { + color: var(--text-primary); + font-weight: 700; +} + +/* Watermark logo */ +.watermark-logo { + position: fixed; + bottom: 16px; + right: 16px; + height: 32px; + width: auto; + opacity: 0.35; + pointer-events: none; + z-index: 9999; + user-select: none; +} + +/* ============================================ + PORTAL LOADING SCREEN + ============================================ */ +.portal-loading { + position: fixed; + inset: 0; + z-index: 10000; + background: radial-gradient(ellipse at center, #1a1035 0%, #0a0a0f 70%, #000 100%); + display: flex; + align-items: center; + justify-content: center; + opacity: 1; + transition: opacity 0.5s ease-out; +} + +.portal-loading.hidden { + opacity: 0; + pointer-events: none; +} + +.portal-loading.fade-out { + opacity: 0; +} + +.portal-loading-content { + display: flex; + flex-direction: column; + align-items: center; + gap: 40px; +} + +.portal-loading-effect { + position: relative; + width: 150px; + height: 150px; +} + +.portal-ring { + position: absolute; + inset: 0; + border-radius: 50%; + border: 3px solid transparent; + border-top-color: #8b5cf6; + border-bottom-color: #a78bfa; + animation: portal-spin 1.5s linear infinite; +} + +.portal-ring:nth-child(2) { + inset: 15px; + border-top-color: #7c3aed; + border-bottom-color: #8b5cf6; + animation-duration: 2s; + animation-direction: reverse; +} + +.portal-ring:nth-child(3) { + inset: 30px; + border-top-color: #6d28d9; + border-bottom-color: #7c3aed; + animation-duration: 2.5s; +} + +@keyframes portal-spin { + 0% { + transform: rotate(0deg); + } + 100% { + transform: rotate(360deg); + } +} + +/* Inner glow effect */ +.portal-loading-effect::before { + content: ''; + position: absolute; + inset: 40px; + border-radius: 50%; + background: radial-gradient(circle, rgba(139, 92, 246, 0.4) 0%, rgba(109, 40, 217, 0.2) 50%, transparent 70%); + animation: portal-pulse 2s ease-in-out infinite; +} + +@keyframes portal-pulse { + 0%, 100% { + transform: scale(0.9); + opacity: 0.6; + } + 50% { + transform: scale(1.1); + opacity: 1; + } +} + +/* Particle effect */ +.portal-loading-effect::after { + content: ''; + position: absolute; + inset: 20px; + border-radius: 50%; + background: transparent; + box-shadow: + 0 0 30px rgba(139, 92, 246, 0.5), + 0 0 60px rgba(139, 92, 246, 0.3), + 0 0 90px rgba(139, 92, 246, 0.1); + animation: portal-glow 2s ease-in-out infinite alternate; +} + +@keyframes portal-glow { + 0% { + box-shadow: + 0 0 30px rgba(139, 92, 246, 0.5), + 0 0 60px rgba(139, 92, 246, 0.3), + 0 0 90px rgba(139, 92, 246, 0.1); + } + 100% { + box-shadow: + 0 0 40px rgba(167, 139, 250, 0.6), + 0 0 80px rgba(139, 92, 246, 0.4), + 0 0 120px rgba(139, 92, 246, 0.2); + } +} + +.portal-loading-text { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + text-align: center; +} + +#portal-loading-title { + font-size: 18px; + font-weight: 600; + color: #e2e8f0; + letter-spacing: 1px; +} + +#portal-loading-dest { + font-size: 24px; + font-weight: 700; + color: #a78bfa; + text-shadow: 0 0 20px rgba(139, 92, 246, 0.5); +} + +/* Success Button */ +button.success { + background: linear-gradient(135deg, var(--accent-success) 0%, #16a34a 100%); + border: none; + color: white; +} + +/* Danger Button */ +button.danger { + background: linear-gradient(135deg, #ef4444 0%, #dc2626 100%); + border: none; + color: white; +} + +/* Ghost Button */ +button.ghost { + background: transparent; + border-color: transparent; + color: var(--text-secondary); +} + +button.ghost:hover { + background: var(--surface-2); + color: var(--text-primary); +} + +.file input { + display: none; +} + +/* ============================================================================ + Form Elements + ============================================================================ */ + +input[type="text"], +input[type="email"], +input[type="password"], +textarea, +.select { + width: 100%; + box-sizing: border-box; + padding: 10px 14px; + border-radius: var(--radius-md); + border: 1px solid var(--border-default); + background: var(--surface-2); + color: var(--text-primary); + font-weight: 500; + font-size: 14px; + outline: none; + transition: all var(--transition-fast); +} + +input[type="text"]:hover, +textarea:hover, +.select:hover { + border-color: var(--border-strong); +} + +input[type="text"]:focus, +textarea:focus, +.select:focus { + border-color: var(--accent-primary); + box-shadow: 0 0 0 3px var(--accent-primary-subtle); +} + +input[type="text"]::placeholder, +textarea::placeholder { + color: var(--text-tertiary); +} + +textarea { + resize: vertical; + min-height: 100px; + font-family: inherit; + line-height: 1.5; +} + +/* Checkbox Styling */ +input[type="checkbox"] { + width: 18px; + height: 18px; + border-radius: var(--radius-sm); + border: 2px solid var(--border-strong); + background: var(--surface-2); + cursor: pointer; + appearance: none; + -webkit-appearance: none; + transition: all var(--transition-fast); + position: relative; +} + +input[type="checkbox"]:checked { + background: var(--accent-primary); + border-color: var(--accent-primary); +} + +input[type="checkbox"]:checked::after { + content: "✓"; + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + color: white; + font-size: 12px; + font-weight: 700; +} + +input[type="checkbox"]:hover { + border-color: var(--accent-primary); +} + +/* ============================================================================ + Status Bar + ============================================================================ */ + +.status { + margin-left: auto; + padding: 6px 12px; + font-size: 12px; + font-weight: 500; + color: var(--text-secondary); + background: var(--surface-1); + border-radius: var(--radius-sm); + max-width: 300px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +/* ============================================================================ + Details / Accordion + ============================================================================ */ + +.details { + border: 1px solid var(--border-default); + border-radius: var(--radius-md); + background: var(--surface-1); + padding: 0; + overflow: hidden; +} + +.details > summary { + cursor: pointer; + user-select: none; + font-weight: 600; + color: var(--text-secondary); + font-size: 12px; + padding: 10px 14px; + list-style: none; + transition: all var(--transition-fast); + display: flex; + align-items: center; + gap: 8px; +} + +.details > summary::-webkit-details-marker { + display: none; +} + +.details > summary::before { + content: "▸"; + font-size: 10px; + transition: transform var(--transition-fast); +} + +.details[open] > summary::before { + transform: rotate(90deg); +} + +.details > summary:hover { + background: var(--surface-2); + color: var(--text-primary); +} + +.details-body { + padding: 12px 14px; + border-top: 1px solid var(--border-subtle); + display: flex; + flex-direction: column; + gap: 12px; +} + +.details-row { + display: flex; + align-items: center; + gap: 10px; + flex-wrap: wrap; +} + +/* ============================================================================ + Slider + ============================================================================ */ + +.slider { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 0; +} + +.slider-label { + font-size: 10px; + color: var(--text-tertiary); + font-weight: 600; + min-width: 40px; + text-transform: uppercase; + letter-spacing: 0.02em; +} + +.slider input[type="range"] { + flex: 1; + height: 3px; + border-radius: 2px; + background: rgba(255,255,255,0.1); + appearance: none; + -webkit-appearance: none; +} + +.slider input[type="range"]::-webkit-slider-thumb { + appearance: none; + -webkit-appearance: none; + width: 12px; + height: 12px; + border-radius: 50%; + background: var(--accent-primary); + cursor: pointer; + border: 1.5px solid rgba(255,255,255,0.3); +} + +.slider-value { + font-variant-numeric: tabular-nums; + font-weight: 600; + font-size: 10px; + color: var(--text-secondary); + min-width: 30px; + text-align: right; +} + + +/* ============================================================================ + Side Panel (UE-inspired) + ============================================================================ */ + +.side-panel { + pointer-events: auto; + width: 100%; + height: calc(100vh - 36px); /* below toolbar */ + display: flex; + flex-direction: column; + background: #1a1d24; + overflow: hidden; +} + +.side-panel-left { + grid-area: left; + border-right: 1px solid rgba(255,255,255,0.06); +} + +.side-panel-right { + grid-area: right; + border-left: 1px solid rgba(255,255,255,0.06); +} + +/* Hide panels when overlay is collapsed */ +#overlay.left-collapsed .side-panel-left { + display: none; +} +#overlay.right-collapsed .side-panel-right { + display: none; +} + +html[data-mode="sim"] .side-panel { + grid-area: panel; + height: calc(100vh - 32px); + border-radius: 0; + border: 1px solid var(--border-default); + backdrop-filter: blur(20px); + background: var(--bg-elevated); +} + +.panel-header { + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px 10px; + border-bottom: 1px solid rgba(255,255,255,0.06); + background: rgba(255,255,255,0.02); +} + +.panel-title { + font-family: var(--font-mono); + font-weight: 600; + font-size: 11px; + letter-spacing: 0.16em; + text-transform: uppercase; + color: var(--text-secondary); + display: flex; + align-items: center; + gap: 8px; +} + +.panel-title::before { + content: "["; + color: var(--accent-primary); + font-family: var(--font-mono); + font-weight: 500; + font-size: 12px; + background: transparent; + width: auto; + height: auto; + border-radius: 0; + letter-spacing: 0; +} +.panel-title::after { + content: "]"; + color: var(--accent-primary); + font-family: var(--font-mono); + font-weight: 500; + font-size: 12px; + margin-left: 2px; + letter-spacing: 0; +} + +.mode-toggle { + padding: 3px 8px !important; + font-size: 11px !important; + font-weight: 500 !important; + font-family: var(--font-mono) !important; + background: transparent !important; + border: 1px solid var(--border-default) !important; + color: var(--text-tertiary) !important; + display: flex !important; + align-items: center !important; + gap: 5px !important; + border-radius: 0 !important; + cursor: pointer; + transition: all 0.1s !important; + text-transform: none !important; + letter-spacing: 0 !important; +} +.mode-toggle:hover { background: rgba(255,255,255,0.06) !important; color: var(--text-primary) !important; } +.mode-icon { font-size: 9px; } + +.panel-content { + flex: 1; + overflow-y: auto; + overflow-x: hidden; +} + +.panel-footer { + flex-shrink: 0; + padding: 5px 10px; + border-top: 1px solid rgba(255,255,255,0.06); + background: rgba(255,255,255,0.02); +} + +/* Panel collapse/expand buttons */ +.panel-collapse-btn { + padding: 4px 8px !important; + font-size: 10px !important; + font-weight: 500 !important; + font-family: var(--font-mono) !important; + background: transparent !important; + border: 1px solid var(--border-default) !important; + color: var(--text-tertiary) !important; + border-radius: 0 !important; + cursor: pointer; + transition: all 0.1s !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; + line-height: 1 !important; +} +.panel-collapse-btn:hover { background: rgba(255,255,255,0.06) !important; color: var(--text-primary) !important; } + +/* Left panel open button (shown when collapsed) */ +.left-panel-open { + pointer-events: auto; + position: fixed; + top: 44px; + left: 8px; + width: 30px; + height: 30px; + border-radius: 0; + border: 1px solid var(--border-default); + background: var(--bg-elevated); + color: var(--text-secondary); + cursor: pointer; + z-index: 150; + font-size: 14px; + display: flex; + align-items: center; + justify-content: center; + backdrop-filter: blur(12px); +} +.left-panel-open:hover { color: var(--text-primary); background: rgba(255,255,255,0.08); } +.left-panel-open.hidden { display: none; } + +/* Right AI panel open button (shown when collapsed) */ +.ai-panel-open { + pointer-events: auto; + position: fixed; + top: 44px; + right: 8px; + width: 30px; + height: 30px; + border-radius: 0; + border: 1px solid var(--border-default); + background: var(--bg-elevated); + color: var(--text-secondary); + cursor: pointer; + z-index: 150; + font-size: 14px; + display: flex; + align-items: center; + justify-content: center; + backdrop-filter: blur(12px); +} +.ai-panel-open:hover { color: var(--text-primary); background: rgba(255,255,255,0.08); } +.ai-panel-open.hidden { display: none; } + +.sim-panel-open { + pointer-events: auto; + position: fixed; + top: 16px; + right: 16px; + width: 30px; + height: 30px; + border-radius: 0; + border: 1px solid var(--border-default); + background: var(--bg-elevated); + color: var(--text-secondary); + cursor: pointer; + z-index: 150; + font-family: var(--font-mono); +} +.sim-panel-open:hover { color: var(--text-primary); background: rgba(255,255,255,0.08); } +.sim-panel-open.hidden { display: none; } + +.shortcuts { + display: flex; + flex-wrap: wrap; + gap: 6px 10px; + font-size: 10px; + font-family: var(--font-mono); + letter-spacing: 0.04em; + color: var(--text-tertiary); +} + +.shortcuts b { + color: var(--accent-primary); + font-weight: 500; + background: transparent; + border: 1px solid var(--border-default); + padding: 0 4px; + border-radius: 0; + font-size: 9px; + font-family: var(--font-mono); +} + +/* Floating bottom-left shortcuts strip (always visible in sim mode) */ +.shortcuts-floating { + position: fixed; + bottom: 16px; + left: 16px; + padding: 8px 12px; + background: rgba(8, 10, 14, 0.82); + border: 1px solid rgba(255, 255, 255, 0.12); + border-radius: 0; + font-size: 11px; + letter-spacing: 0.06em; + text-transform: uppercase; + z-index: 9998; + backdrop-filter: blur(4px); +} +.shortcuts-floating b { + font-size: 10px; + padding: 1px 5px; +} + +/* Dimos-only: hide the sim side panel and command bar — leave only + top-left status + bottom-left WASD shortcuts visible. */ +body.dimos-mode #agent-panel, +body.dimos-mode #sim-panel-open, +body.dimos-mode #agent-command-bar { + display: none !important; +} + +/* Tag form / tag-selected (legacy compat) */ +.tag-selected { display: none; } +.tag-form { /* managed by details-panel */ } +.tag-form.hidden { display: none; } + +/* Floating status for sim mode */ +.status-floating { + position: fixed; + top: 16px; + left: 16px; + padding: 6px 12px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.04em; + color: var(--text-secondary); + background: var(--bg-elevated); + border: 1px solid var(--border-default); + border-radius: 0; + backdrop-filter: blur(12px); + pointer-events: auto; + max-width: 300px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} +.status-floating:empty { display: none; } + +/* ============================================================================ + Outliner (Scene tree, collapsible sections) + ============================================================================ */ + +.outliner { + border-bottom: 1px solid rgba(255,255,255,0.06); +} + +.ol-section { + border-bottom: 1px solid rgba(255,255,255,0.04); +} +.ol-section:last-child { border-bottom: none; } +.ol-section > summary { list-style: none; } +.ol-section > summary::-webkit-details-marker { display: none; } + +.ol-header { + display: flex; + align-items: center; + gap: 6px; + padding: 5px 10px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text-secondary); + cursor: pointer; + user-select: none; + transition: background 0.1s; +} +.ol-header:hover { background: rgba(255,255,255,0.04); } + +.ol-header::before { + content: "▸"; + font-size: 9px; + color: var(--text-tertiary); + transition: transform 0.15s; + display: inline-block; + width: 10px; +} +.ol-section[open] > .ol-header::before { transform: rotate(90deg); } + +.ol-icon { font-size: 12px; } +.ol-count { + margin-left: auto; + font-size: 10px; + font-weight: 500; + color: var(--text-tertiary); + background: rgba(255,255,255,0.04); + padding: 1px 6px; + border-radius: 0; + min-width: 16px; + text-align: center; + font-family: var(--font-mono); +} + +.ol-list { + display: flex; + flex-direction: column; + gap: 1px; + padding: 0 0 2px 0; + max-height: 160px; + overflow-y: auto; +} + +/* Compact outliner items (replaces old .tag-item) */ +.tag-item, .ol-item { + display: flex; + align-items: center; + gap: 6px; + padding: 4px 10px 4px 26px; + cursor: pointer; + font-size: 12px; + font-weight: 500; + color: var(--text-secondary); + transition: all 0.08s; + border: none; + background: transparent; + border-radius: 0; +} + +.tag-item:hover { + background: rgba(255,255,255,0.04); + color: var(--text-primary); +} + +.tag-item small { + margin-left: auto; + color: var(--text-tertiary); + font-size: 10px; + font-weight: 400; + margin-top: 0; + display: inline; + flex-shrink: 0; +} + +.tag-item.active { + background: var(--accent-primary-subtle); + color: var(--accent-primary-hover); + border: none; + box-shadow: inset 3px 0 0 var(--accent-primary); +} + +/* ============================================================================ + Groups in Outliner + ============================================================================ */ + +.ol-group { + border-left: 2px solid transparent; + margin-bottom: 2px; +} +.ol-group.active { + border-left-color: var(--accent-primary); + background: rgba(99,102,241,0.04); +} + +.ol-group-header { + display: flex; + align-items: center; + gap: 5px; + padding: 4px 8px 4px 14px; + cursor: pointer; + font-size: 12px; + font-weight: 600; + color: var(--text-secondary); + transition: background 0.08s; + user-select: none; +} +.ol-group-collapse-btn { + width: 18px; + height: 18px; + padding: 0; + border: 1px solid transparent; + border-radius: 4px; + background: transparent; + color: var(--text-tertiary); + font-size: 11px; + line-height: 1; + cursor: pointer; +} +.ol-group-collapse-btn:hover { + background: rgba(255,255,255,0.06); + color: var(--text-primary); +} +.ol-group-header:hover { + background: rgba(255,255,255,0.05); + color: var(--text-primary); +} + +.ol-group-icon { + font-size: 11px; + flex-shrink: 0; +} +.ol-group-name { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.ol-group-count { + font-size: 10px; + font-weight: 400; + color: var(--text-tertiary); + flex-shrink: 0; +} +.ol-group-pickable { + font-size: 11px; + flex-shrink: 0; + opacity: 0.85; +} + +.ol-group-actions { + display: flex; + gap: 2px; + margin-left: auto; + flex-shrink: 0; + opacity: 0; + transition: opacity 0.1s; +} +.ol-group-header:hover .ol-group-actions { + opacity: 1; +} + +.ol-group-btn { + padding: 1px 6px; + font-size: 10px; + font-weight: 600; + font-family: inherit; + border: 1px solid var(--border-default); + border-radius: var(--radius-sm); + background: var(--surface-2); + color: var(--text-tertiary); + cursor: pointer; + transition: all 0.08s; +} +.ol-group-btn:hover { + background: var(--surface-3); + color: var(--text-primary); + border-color: var(--border-strong); +} + +.ol-group-children { + padding-left: 12px; + border-left: 1px solid var(--border-subtle); + margin-left: 18px; +} +.ol-group-child { + padding-left: 14px !important; + font-size: 11px !important; +} + +/* Multi-select (shift+click) */ +.tag-item.multi-selected { + background: rgba(139, 92, 246, 0.12); + color: var(--text-primary); + box-shadow: inset 3px 0 0 #8B5CF6; +} +.tag-item.multi-selected:hover { + background: rgba(139, 92, 246, 0.18); +} + +/* Group action bar (shown when 2+ shapes are shift-selected) */ +.ol-group-action-bar { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 10px; + background: rgba(139, 92, 246, 0.08); + border: 1px solid rgba(139, 92, 246, 0.25); + border-radius: var(--radius-md); + margin: 4px 6px; +} +.ol-group-action-label { + font-size: 11px; + font-weight: 600; + color: #a78bfa; + flex: 1; + white-space: nowrap; +} +.ol-group-action-btn { + padding: 3px 10px; + font-size: 11px; + font-weight: 600; + font-family: inherit; + border: none; + border-radius: var(--radius-sm); + cursor: pointer; + white-space: nowrap; + transition: all 0.1s; + background: #8B5CF6; + color: #fff; +} +.ol-group-action-btn:hover { + background: #7C3AED; +} +.ol-group-cancel-btn { + background: var(--surface-2); + color: var(--text-secondary); + border: 1px solid var(--border-default); +} +.ol-group-cancel-btn:hover { + background: var(--surface-3); +} + +/* ============================================================================ + Details Panel (Properties of selected object) + ============================================================================ */ + +.details-panel { + border-top: 2px solid rgba(99,102,241,0.3); +} +.details-panel.hidden { display: none; } + +.details-title { + padding: 6px 10px; + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--accent-primary-hover); + background: rgba(99,102,241,0.06); + border-bottom: 1px solid rgba(255,255,255,0.04); +} + +/* Details collapsible sections */ +.dt-section { + border-bottom: 1px solid rgba(255,255,255,0.04); +} +.dt-section > summary { list-style: none; } +.dt-section > summary::-webkit-details-marker { display: none; } + +.dt-header { + display: flex; + align-items: center; + gap: 6px; + padding: 5px 10px; + font-size: 11px; + font-weight: 600; + color: var(--text-tertiary); + cursor: pointer; + user-select: none; + text-transform: uppercase; + letter-spacing: 0.03em; +} +.dt-header:hover { color: var(--text-secondary); } +.dt-header::before { + content: "▸"; + font-size: 9px; + transition: transform 0.15s; + display: inline-block; + width: 10px; +} +.dt-section[open] > .dt-header::before { transform: rotate(90deg); } + +.dt-body { + padding: 6px 10px 8px; +} + +/* Inputs inside details */ +.dt-input { + width: 100%; + padding: 5px 8px; + background: rgba(0,0,0,0.3); + border: 1px solid var(--border-default); + border-radius: 0; + color: var(--text-primary); + font-family: var(--font-mono); + font-size: 12px; + margin-bottom: 6px; + transition: border-color 0.1s; + resize: vertical; +} +.dt-input:focus { outline: none; border-color: var(--accent-primary); } + +.dt-select { + flex: 1; + padding: 4px 8px; + background: rgba(0,0,0,0.3); + border: 1px solid var(--border-default); + border-radius: 0; + color: var(--text-primary); + font-family: var(--font-mono); + font-size: 12px; +} + +.dt-row { + display: flex; + gap: 6px; + align-items: center; + margin-bottom: 6px; +} + +/* Asset-builder transition rows: keep remove button visible in narrow panels */ +.builder-interaction-row { + flex-wrap: wrap; +} +.builder-interaction-row .builder-transition-label { + flex: 1 1 180px; + min-width: 150px; + margin-bottom: 0; +} +.builder-interaction-row .builder-transition-target { + flex: 0 1 140px; + min-width: 120px; +} +.builder-interaction-row .builder-transition-remove { + flex: 0 0 auto; +} + +.dt-actions { + display: flex; + gap: 4px; +} +.dt-actions .tb-btn { flex: 1; justify-content: center; } + +/* ============================================================================ + Transform XYZ Grid (UE-style) + ============================================================================ */ + +.xform-row { + display: grid; + grid-template-columns: 56px 1fr 1fr 1fr; + gap: 3px; + margin-bottom: 3px; + align-items: center; +} + +.xform-label { + font-size: 10px; + font-weight: 600; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.03em; + padding-left: 2px; +} + +.xform-in { + width: 100%; + padding: 4px 6px; + font-size: 11px; + font-family: var(--font-mono); + font-weight: 500; + background: rgba(0,0,0,0.3); + border: 1px solid var(--border-subtle); + border-radius: 0; + color: var(--text-primary); + text-align: right; + -moz-appearance: textfield; +} +.xform-in::-webkit-outer-spin-button, +.xform-in::-webkit-inner-spin-button { -webkit-appearance: none; margin: 0; } +.xform-in:focus { outline: none; border-color: rgba(255,255,255,0.2); } + +/* Color-coded X/Y/Z borders (UE style) */ +.xform-x { border-left: 2px solid #ef4444; } +.xform-y { border-left: 2px solid #22c55e; } +.xform-z { border-left: 2px solid #3b82f6; } + +/* Legacy compat: hide old elements that are now in toolbar */ +.panel-section { padding: 0; border: none; } +.section-label { display: none; } +.asset-tools { display: none; /* now in toolbar */ } + +/* ============================================================================ + Modal + ============================================================================ */ + +.modal { + pointer-events: auto; + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.7); + backdrop-filter: blur(4px); + display: grid; + place-items: center; + z-index: 200; + animation: modalFadeIn 0.2s ease; +} + +@keyframes modalFadeIn { + from { opacity: 0; } + to { opacity: 1; } +} + +.modal-card { + width: min(560px, calc(100vw - 48px)); + border-radius: 0; + border: 1px solid var(--border-default); + background: var(--bg-secondary); + box-shadow: none; + overflow: hidden; + max-height: calc(100vh - 96px); + display: flex; + flex-direction: column; + animation: modalSlideIn 0.3s ease; +} + +@keyframes modalSlideIn { + from { + opacity: 0; + transform: translateY(-20px) scale(0.98); + } + to { + opacity: 1; + transform: translateY(0) scale(1); + } +} + +.modal-title { + padding: 16px 20px; + font-weight: 500; + font-size: 13px; + font-family: var(--font-mono); + letter-spacing: 0.14em; + text-transform: uppercase; + color: var(--accent-primary); + border-bottom: 1px solid var(--border-subtle); + background: var(--surface-1); +} + +.modal-body { + padding: 20px; + overflow-y: auto; +} + +.modal-body input, +.modal-body textarea { + margin-bottom: 12px; +} + +.modal-subtitle { + margin: 16px 0 10px; + font-size: 12px; + color: var(--text-secondary); + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.modal-hint { + margin-top: 12px; + padding: 12px 14px; + font-size: 12px; + color: var(--text-tertiary); + line-height: 1.5; + background: var(--surface-1); + border-radius: var(--radius-md); + border-left: 3px solid var(--accent-info); +} + +.modal-hint b { + color: var(--text-secondary); +} + +/* Asset States */ +.asset-states { + display: flex; + flex-direction: column; + gap: 10px; +} + +.asset-state-row { + border: 1px solid var(--border-default); + border-radius: var(--radius-md); + padding: 14px; + background: var(--surface-1); + display: flex; + flex-direction: column; + gap: 10px; +} + +.asset-state-row-top { + display: flex; + gap: 10px; + align-items: center; + flex-wrap: wrap; +} + +.asset-state-row-top label { + font-size: 12px; + color: var(--text-secondary); + font-weight: 600; + display: flex; + align-items: center; + gap: 6px; +} + +.asset-interactions-title { + font-size: 11px; + color: var(--text-tertiary); + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.05em; + margin-top: 4px; +} + +.asset-interactions { + display: flex; + flex-direction: column; + gap: 8px; +} + +.asset-interaction-row { + display: flex; + gap: 8px; + align-items: center; + flex-wrap: wrap; +} + +.asset-interaction-row input[type="text"] { + flex: 1; + min-width: 140px; + margin: 0; + padding: 8px 12px; + font-size: 13px; +} + +/* ============================================================================ + Agent Panel Components + ============================================================================ */ + +/* ---------- Floating Command Bar ---------- */ +.agent-cmd-bar { + pointer-events: auto; + position: fixed; + bottom: 24px; + left: 50%; + transform: translateX(-50%); + display: flex; + align-items: center; + gap: 6px; + padding: 6px 10px; + background: var(--bg-elevated); + border: 1px solid var(--border-default); + border-radius: 0; + backdrop-filter: blur(16px); + box-shadow: none; + z-index: 160; + max-width: min(600px, calc(100vw - 32px)); + transition: opacity var(--transition-fast); + opacity: 0.75; +} +.agent-cmd-bar:hover, +.agent-cmd-bar:focus-within, +.agent-cmd-bar.active { + opacity: 1; +} + +.agent-cmd-spawn { + flex-shrink: 0; + padding: 6px 12px; + font-size: 11px; + font-weight: 500; + letter-spacing: 0.06em; + text-transform: uppercase; + background: transparent; + color: var(--text-primary); + border: 1px solid var(--border-default); + border-radius: 0; + cursor: pointer; + white-space: nowrap; + transition: all var(--transition-fast); + font-family: var(--font-mono); +} +.agent-cmd-spawn:hover { + background: var(--accent-primary-subtle); + border-color: var(--accent-primary); + color: var(--accent-primary-hover); +} + +.agent-cmd-status { + flex-shrink: 0; + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + white-space: nowrap; + max-width: 80px; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; +} +.agent-cmd-status:empty { + display: none; +} + +.agent-cmd-input { + flex: 1; + min-width: 140px; + padding: 6px 10px; + font-size: 12px; + font-weight: 500; + font-family: var(--font-mono); + background: rgba(0,0,0,0.35); + color: var(--text-primary); + border: 1px solid var(--border-default); + border-radius: 0; + outline: none; + transition: border-color var(--transition-fast); +} +.agent-cmd-input:focus { + border-color: var(--accent-primary); +} +.agent-cmd-input::placeholder { + color: var(--text-tertiary); +} + +.agent-cmd-btn { + flex-shrink: 0; + padding: 6px 14px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.06em; + text-transform: uppercase; + background: transparent; + color: var(--accent-primary); + border: 1px solid var(--accent-primary); + border-radius: 0; + cursor: pointer; + white-space: nowrap; + transition: all var(--transition-fast); +} +.agent-cmd-btn:hover { + background: var(--accent-primary-subtle); + color: var(--accent-primary-hover); + border-color: var(--accent-primary-hover); +} +.agent-cmd-btn:disabled { + opacity: 0.35; + cursor: not-allowed; +} +.agent-cmd-stop { + background: transparent; + color: var(--text-secondary); + border: 1px solid var(--border-default); +} +.agent-cmd-stop:hover { + background: rgba(239,68,68,0.08); + border-color: #ef4444; + color: #f87171; +} + +.agent-control-strip { + padding: 8px 10px; + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + background: rgba(255, 255, 255, 0.02); +} +.agent-control-selected { + font-size: 11px; + font-weight: 600; + color: var(--text-tertiary); + margin-bottom: 6px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.agent-control-actions { + display: flex; + gap: 6px; +} +.agent-control-actions .tb-btn { + flex: 1; + justify-content: center; + padding: 5px 8px; + font-size: 11px; +} +.agent-control-task-row { + margin-top: 6px; + display: flex; + gap: 6px; +} +.agent-control-task-input { + flex: 1; + min-width: 0; + padding: 6px 8px; + font-size: 11px; + color: var(--text-primary); + background: var(--surface-1); + border: 1px solid var(--border-default); + border-radius: var(--radius-sm); + outline: none; +} +.agent-control-task-input:focus { + border-color: var(--accent-primary); +} + +.agent-badge-layer { + position: fixed; + inset: 0; + pointer-events: none; + z-index: 170; +} +.agent-badge { + position: fixed; + transform: translate(-50%, -50%); + pointer-events: auto; + z-index: 220; + touch-action: none; + padding: 3px 8px; + font-size: 10px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--text-primary); + background: rgba(16, 22, 32, 0.9); + border: 1px solid var(--border-strong); + border-radius: 0; + cursor: pointer; + white-space: nowrap; + box-shadow: none; +} +.agent-badge.active { + border-color: var(--accent-primary); + box-shadow: 0 0 0 2px var(--accent-primary-subtle), var(--shadow-md); +} + +/* ===== Vibe Creator – Right Panel Layout ===== */ +.vibe-panel-header { + flex-shrink: 0; + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px 10px; + border-bottom: 1px solid rgba(255,255,255,0.06); + background: rgba(255,255,255,0.02); +} + +.vibe-panel-title { + font-weight: 600; + font-size: 11px; + font-family: var(--font-mono); + letter-spacing: 0.16em; + text-transform: uppercase; + color: var(--text-secondary); + display: flex; + align-items: center; + gap: 6px; +} +.vibe-panel-title::before { + content: ">"; + font-family: var(--font-mono); + font-size: 12px; + color: var(--accent-primary); +} + +.vibe-tabs { + display: flex; + gap: 4px; + padding: 6px 10px; + border-bottom: 1px solid var(--border-subtle); + background: rgba(255,255,255,0.01); +} +.vibe-tab-btn { + padding: 5px 10px; + border-radius: 0; + border: 1px solid transparent; + background: transparent; + color: var(--text-tertiary); + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.08em; + text-transform: uppercase; + cursor: pointer; +} +.vibe-tab-btn:hover { + color: var(--text-secondary); + background: rgba(255,255,255,0.04); +} +.vibe-tab-btn.active { + color: var(--text-primary); + border-color: rgba(139, 92, 246, 0.35); + background: rgba(139, 92, 246, 0.15); +} + +.vibe-content { + flex: 1; + min-height: 0; + display: flex; +} +.vibe-tab-pane { + flex: 1; + min-height: 0; + display: flex; + flex-direction: column; +} + +.vibe-assets-help { + font-size: 11px; + color: var(--text-tertiary); + line-height: 1.6; + padding: 10px; + border-bottom: 1px solid var(--border-subtle); +} +.vibe-asset-controls { + display: flex; + gap: 6px; + padding: 10px; + border-bottom: 1px solid var(--border-subtle); +} +.vibe-asset-aux-controls { + display: flex; + gap: 6px; + padding: 8px 10px; + border-bottom: 1px solid var(--border-subtle); +} +.vibe-asset-aux-controls .vibe-btn { + flex: 1; + padding: 6px 8px; + font-size: 11px; + border: 1px solid var(--border-default); + background: var(--surface-2); + color: var(--text-secondary); +} +.vibe-asset-aux-controls .vibe-btn:hover { + background: var(--surface-3); + color: var(--text-primary); +} +.vibe-asset-list { + flex: 1; + min-height: 0; + overflow-y: auto; + padding: 8px; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(132px, 1fr)); + grid-auto-rows: max-content; + align-content: start; + align-items: start; + gap: 10px; +} +.vibe-asset-empty { + font-size: 12px; + color: var(--text-tertiary); + padding: 16px 10px; +} +.vibe-asset-item { + border: 1px solid var(--border-subtle); + border-radius: 0; + background: rgba(255,255,255,0.02); + padding: 6px; + height: auto; + align-self: start; +} +.vibe-asset-thumb { + width: 100%; + aspect-ratio: 1 / 1; + object-fit: cover; + border-radius: 0; + border: 1px solid rgba(255,255,255,0.08); + background: #0b0d11; + display: block; +} +.vibe-asset-item-head { + display: flex; + justify-content: space-between; + align-items: center; + gap: 8px; + margin: 6px 2px 0; +} +.vibe-asset-name { + font-size: 11px; + font-weight: 600; + color: var(--text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.vibe-asset-status { + font-size: 9px; + font-family: var(--font-mono); + text-transform: uppercase; + letter-spacing: 0.08em; + padding: 2px 6px; + border-radius: 0; + border: 1px solid var(--border-default); + color: var(--text-tertiary); +} +.vibe-asset-status.approved { + border-color: rgba(74, 222, 128, 0.4); + color: #86efac; + background: rgba(74, 222, 128, 0.12); +} +.vibe-asset-prompt { + font-size: 10px; + color: var(--text-tertiary); + line-height: 1.4; + margin: 4px 2px 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.vibe-asset-actions { + display: flex; + gap: 4px; + margin-top: 6px; +} +.vibe-asset-actions .vibe-btn { + flex: 1; + padding: 5px 6px; + font-size: 10px; +} +.vibe-asset-item[draggable="true"] { + cursor: grab; +} +.vibe-asset-item[draggable="true"]:active { + cursor: grabbing; +} + +/* Staging editor visual mode */ +body.staging-mode { + background: #000; +} +body.staging-mode .watermark-logo { + opacity: 0.08; +} +body.staging-mode #overlay-top { + background: rgba(8, 10, 14, 0.92); + border-bottom-color: rgba(255,255,255,0.08); +} +/* In builder mode, change left panel header to reflect context */ +body.staging-mode .panel-title::after { + content: " · Asset Builder"; + color: var(--accent-primary); + font-weight: 600; +} +/* Hide the shape dropdown in builder mode — inline bar replaces it */ +body.staging-mode .shape-dropdown-wrapper { + display: none; +} + +/* Bottom input bar pinned to panel bottom */ +.vibe-bar { + flex-shrink: 0; + display: flex; + align-items: center; + gap: 6px; + padding: 8px 10px; + background: rgba(255,255,255,0.02); + border-top: 1px solid rgba(255,255,255,0.06); +} +.vibe-bar.active { + background: rgba(139, 92, 246, 0.04); + border-top-color: rgba(139, 92, 246, 0.15); +} + +.vibe-icon { + display: none; +} + +.vibe-input { + flex: 1; + min-width: 0; + padding: 6px 10px; + font-size: 12px; + font-weight: 500; + font-family: var(--font-mono); + background: rgba(0,0,0,0.35); + color: var(--text-primary); + border: 1px solid var(--border-default); + border-radius: 0; + outline: none; + transition: border-color var(--transition-fast); +} +.vibe-input:focus { + border-color: var(--accent-primary); +} +.vibe-input::placeholder { + color: var(--text-tertiary); +} +.vibe-input:disabled { + opacity: 0.5; +} + +.vibe-mode-toggle { + flex-shrink: 0; + display: flex; + align-items: center; + gap: 4px; + font-size: 11px; + font-weight: 600; + color: var(--text-tertiary); + cursor: pointer; + user-select: none; + padding: 4px 6px; + border-radius: var(--radius-sm); + transition: all var(--transition-fast); +} +.vibe-mode-toggle:hover { + color: var(--text-secondary); + background: var(--surface-1); +} +.vibe-mode-toggle input[type="checkbox"] { + width: 14px; + height: 14px; + margin: 0; + cursor: pointer; +} + +.vibe-btn { + flex-shrink: 0; + padding: 6px 12px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.06em; + text-transform: uppercase; + border: 1px solid var(--border-default); + border-radius: 0; + cursor: pointer; + white-space: nowrap; + transition: all var(--transition-fast); + background: transparent; + color: var(--text-secondary); +} +.vibe-btn:disabled { + opacity: 0.35; + cursor: not-allowed; +} +.vibe-btn-primary { + background: transparent; + color: var(--accent-primary); + border: 1px solid var(--accent-primary); +} +.vibe-btn-primary:hover { + background: var(--accent-primary-subtle); + color: var(--accent-primary-hover); + border-color: var(--accent-primary-hover); + box-shadow: none; +} +.vibe-btn-stop { + background: transparent; + color: var(--text-secondary); + border: 1px solid var(--border-default); +} +.vibe-btn-stop:hover { + background: rgba(239,68,68,0.08); + border-color: #ef4444; + color: #f87171; +} + +.vibe-status { + flex-shrink: 0; + font-size: 11px; + font-weight: 500; + color: var(--text-tertiary); + white-space: nowrap; + max-width: 180px; + overflow: hidden; + text-overflow: ellipsis; +} +.vibe-status:empty { + display: none; +} + +/* ===== Vibe Creator – Task Tracker (in-panel) ===== */ +.vibe-tracker { + flex: 1; + overflow-y: auto; + padding: 6px 0; + font-size: 12px; + scrollbar-width: thin; +} +.vibe-stream-details { + margin: 8px 10px 2px; + border: 1px solid var(--border-subtle); + border-radius: 0; + background: rgba(255,255,255,0.02); + overflow: hidden; +} +.vibe-stream-summary { + cursor: pointer; + padding: 8px 10px; + font-size: 11px; + color: var(--text-secondary); + border-bottom: 1px solid rgba(255,255,255,0.05); +} +.vibe-stream-body { + margin: 0; + padding: 10px; + max-height: 160px; + overflow: auto; + white-space: pre-wrap; + word-break: break-word; + font-size: 10px; + line-height: 1.45; + color: #cdd6f4; + background: rgba(8, 10, 14, 0.7); +} +.vibe-tracker::-webkit-scrollbar { + width: 4px; +} +.vibe-tracker::-webkit-scrollbar-thumb { + background: var(--border-default); + border-radius: 2px; +} + +/* Tracker header */ +.vt-header { + padding: 6px 14px 8px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.12em; + text-transform: uppercase; + color: var(--accent-primary); + border-bottom: 1px solid var(--border-subtle); + display: flex; + align-items: center; + gap: 8px; +} +.vt-count { + font-size: 10px; + font-weight: 500; + font-family: var(--font-mono); + color: var(--text-tertiary); + background: transparent; + border: 1px solid var(--border-default); + padding: 0 6px; + border-radius: 0; +} + +/* Task items */ +.vt-task { + border-top: 1px solid transparent; +} +.vt-task + .vt-task { + border-top-color: var(--border-subtle); +} + +.vt-task-row { + display: flex; + align-items: center; + gap: 8px; + padding: 5px 14px; + min-height: 28px; +} + +.vt-icon { + flex-shrink: 0; + width: 16px; + text-align: center; + font-size: 11px; + line-height: 1; +} + +/* Status-specific icon colors */ +.vt-pending .vt-icon { + color: var(--text-tertiary); +} +.vt-active .vt-icon { + color: #8B5CF6; + animation: vt-pulse 1.2s ease-in-out infinite; +} +.vt-done .vt-icon { + color: #4ade80; +} +.vt-failed .vt-icon { + color: #ef4444; +} + +@keyframes vt-pulse { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.4; } +} + +.vt-title { + flex: 1; + font-weight: 500; + color: var(--text-secondary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.vt-active .vt-title { + color: var(--text-primary); + font-weight: 600; +} +.vt-done .vt-title { + color: var(--text-secondary); +} +.vt-pending .vt-title { + color: var(--text-tertiary); +} + +.vt-meta { + flex-shrink: 0; + font-size: 10px; + font-weight: 500; + color: var(--text-tertiary); + white-space: nowrap; +} +.vt-active .vt-meta { + color: #a78bfa; +} + +/* Task detail sub-entries */ +.vt-details { + padding: 0 14px 4px 38px; +} +.vt-detail { + font-size: 11px; + color: var(--text-tertiary); + line-height: 1.6; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.vt-detail::before { + content: "› "; + color: var(--border-strong); +} + +/* Final polish note */ +.vt-final { + padding: 6px 14px; + font-size: 11px; + color: var(--text-secondary); + border-top: 1px solid var(--border-subtle); + display: flex; + align-items: baseline; + gap: 6px; +} +.vt-final-icon { + color: #fbbf24; + font-size: 12px; +} + +/* Vibe Creator – Empty state (shown when no generation is active) */ +.vibe-empty-state { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 40px 20px; + gap: 12px; + text-align: center; + height: 100%; + min-height: 200px; +} +.vibe-empty-icon { + font-size: 32px; + opacity: 0.25; + user-select: none; +} +.vibe-empty-text { + font-size: 12px; + color: var(--text-tertiary); + max-width: 240px; + line-height: 1.6; +} + +/* Vibe Creator – Activity bar (what the agent is doing right now) */ +.vt-activity { + padding: 8px 14px; + font-size: 11px; + font-weight: 500; + color: #a78bfa; + background: rgba(139, 92, 246, 0.06); + border-top: 1px solid rgba(139, 92, 246, 0.15); + line-height: 1.5; + word-wrap: break-word; + white-space: normal; + animation: vt-pulse 1.2s ease-in-out infinite; +} +.vt-activity.hidden { + display: none; +} + +/* Agent Vision */ +.agent-vision { + border-bottom: 1px solid var(--border-subtle); +} + +.agent-shot-img { + width: 100%; + height: auto; + display: block; + background: var(--bg-primary); + aspect-ratio: 2.2 / 1; + object-fit: cover; +} + +.agent-decision-content { + margin: 0; + padding: 10px 12px; + font-size: 11px; + line-height: 1.5; + color: var(--text-primary); + white-space: pre-wrap; + word-break: break-word; + font-family: var(--font-mono); + background: var(--surface-1); + border-radius: 0; + border: 1px solid var(--border-subtle); + max-height: 160px; + overflow-y: auto; +} + +/* Collapsible Sections */ +.agent-collapse { + border-bottom: 1px solid var(--border-subtle); +} + +.agent-collapse:last-child { + border-bottom: none; +} + +.agent-collapse > summary { + cursor: pointer; + padding: 9px 12px; + font-size: 11px; + color: var(--text-secondary); + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.1em; + text-transform: uppercase; + user-select: none; + transition: all var(--transition-fast); + display: flex; + align-items: center; + gap: 8px; + list-style: none; +} + +.agent-collapse > summary::-webkit-details-marker { + display: none; +} + +.agent-collapse > summary::before { + content: "▸"; + font-size: 10px; + transition: transform var(--transition-fast); +} + +.agent-collapse[open] > summary::before { + transform: rotate(90deg); +} + +.agent-collapse > summary:hover { + background: var(--surface-1); + color: var(--text-primary); +} + +.agent-collapse-content { + padding: 0 12px 12px; +} + +.agent-observation-panel { + margin: 16px 12px 16px; + padding: 10px; + border: 1px solid var(--border-subtle); + border-radius: var(--radius-md); + background: var(--surface-1); +} + +.agent-observation-panel .section-label { + margin-bottom: 8px; +} + +.agent-meta { + font-size: 11px; + color: var(--text-tertiary); + margin-bottom: 8px; + line-height: 1.5; +} + +/* Sub-collapse (nested details) */ +.agent-sub-collapse { + border: 1px solid var(--border-subtle); + border-radius: 0; + margin-bottom: 6px; + background: var(--surface-1); +} + +.agent-sub-collapse:last-child { + margin-bottom: 0; +} + +.agent-sub-collapse > summary { + cursor: pointer; + padding: 7px 10px; + font-size: 10px; + color: var(--text-tertiary); + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.08em; + text-transform: uppercase; + user-select: none; + transition: all var(--transition-fast); + display: flex; + align-items: center; + gap: 6px; + list-style: none; +} + +.agent-sub-collapse > summary::-webkit-details-marker { + display: none; +} + +.agent-sub-collapse > summary::before { + content: "▸"; + font-size: 9px; + transition: transform var(--transition-fast); +} + +.agent-sub-collapse[open] > summary::before { + transform: rotate(90deg); +} + +.agent-sub-collapse > summary:hover { + color: var(--text-secondary); +} + +.agent-pre { + margin: 0; + padding: 10px; + font-size: 10px; + line-height: 1.5; + color: var(--text-secondary); + white-space: pre-wrap; + word-break: break-word; + font-family: var(--font-mono); + background: var(--bg-primary); + max-height: 300px; + overflow-y: auto; + border-top: 1px solid var(--border-subtle); +} + +/* Activity Log */ +.agent-log { + padding: 8px 12px 12px; + display: flex; + flex-direction: column; + gap: 4px; + max-height: 200px; + overflow-y: auto; +} + +.agent-log-item { + padding: 8px 10px; + border-radius: 0; + background: var(--surface-1); + border-left: 2px solid var(--border-default); + font-size: 10px; + font-family: var(--font-mono); + color: var(--text-secondary); + white-space: pre-wrap; + line-height: 1.4; +} + +/* ============================================================================ + Crosshair + ============================================================================ */ + +#crosshair { + position: fixed; + left: 50%; + top: 50%; + width: 20px; + height: 20px; + transform: translate(-50%, -50%); + pointer-events: none; + opacity: 0.9; +} + +#crosshair::before, +#crosshair::after { + content: ""; + position: absolute; + left: 50%; + top: 50%; + background: rgba(255, 255, 255, 0.9); + transform: translate(-50%, -50%); + border-radius: 1px; + transition: all var(--transition-fast); +} + +#crosshair::before { + width: 12px; + height: 2px; +} + +#crosshair::after { + width: 2px; + height: 12px; +} + +#crosshair.interactable::before, +#crosshair.interactable::after { + background: var(--accent-primary); + box-shadow: 0 0 10px var(--accent-primary-glow); +} + +#crosshair.interactable { + opacity: 1; +} + +#crosshair.holding::before, +#crosshair.holding::after { + background: var(--accent-warning); + box-shadow: 0 0 10px rgba(245, 158, 11, 0.5); +} + +#crosshair.holding { + opacity: 1; +} + +/* ============================================================================ + Interaction Hint + ============================================================================ */ + +#interaction-hint { + position: fixed; + left: 50%; + top: 50%; + transform: translate(-50%, -50%) translateY(40px); + pointer-events: none; + opacity: 0; + transition: all var(--transition-normal); + z-index: 101; + background: var(--bg-elevated); + border: 1px solid var(--accent-primary); + border-radius: 0; + padding: 6px 12px; + font-size: 11px; + font-weight: 500; + font-family: var(--font-mono); + letter-spacing: 0.06em; + text-transform: uppercase; + color: var(--text-primary); + white-space: nowrap; + backdrop-filter: blur(12px); + box-shadow: none; +} + +#interaction-hint.visible { + opacity: 1; + transform: translate(-50%, -50%) translateY(35px); +} + +#interaction-hint .hint-key { + display: inline-flex; + align-items: center; + justify-content: center; + min-width: 20px; + height: 20px; + background: transparent; + border: 1px solid var(--accent-primary); + border-radius: 0; + padding: 0 6px; + margin-left: 8px; + font-size: 10px; + font-weight: 500; + font-family: var(--font-mono); + color: var(--accent-primary); + box-shadow: none; +} + +/* ============================================================================ + Interaction Popup + ============================================================================ */ + +#interaction-popup { + position: fixed; + left: 50%; + top: 50%; + transform: translate(-50%, -50%) translateY(70px); + background: var(--bg-elevated); + border: 1px solid var(--border-default); + border-radius: 0; + padding: 6px; + z-index: 150; + min-width: 220px; + backdrop-filter: blur(20px); + box-shadow: none; + animation: popupSlideIn 0.2s ease; + flex-direction: column; + gap: 4px; + pointer-events: auto; +} + +@keyframes popupSlideIn { + from { + opacity: 0; + transform: translate(-50%, -50%) translateY(60px) scale(0.95); + } + to { + opacity: 1; + transform: translate(-50%, -50%) translateY(70px) scale(1); + } +} + +.interact-action-btn { + width: 100%; + padding: 10px 14px !important; + text-align: left !important; + justify-content: flex-start !important; + margin-bottom: 4px; + background: var(--surface-1) !important; + border: 1px solid var(--border-subtle) !important; +} + +.interact-action-btn:last-child { + margin-bottom: 0; +} + +.interact-action-btn:hover { + background: var(--accent-primary-subtle) !important; + border-color: rgba(99, 102, 241, 0.3) !important; +} + +/* ============================================================================ + Visibility Utilities + ============================================================================ */ + +.hidden { + display: none !important; +} + +.edit-only, +.sim-only { + /* default shown */ +} + +html[data-mode="sim"] .edit-only { + display: none !important; +} + +html[data-mode="edit"] .sim-only { + display: none !important; +} + +/* Keep the agent command bar available in edit mode for iterative scene edits. */ +html[data-mode="edit"] #agent-command-bar.sim-only { + display: flex !important; +} + +/* In edit mode, keep Agent Vision/Response visible as a floating inspector. */ +html[data-mode="edit"] #agent-panel.sim-only { + display: flex !important; + position: fixed; + right: 16px; + bottom: 72px; + width: 360px; + max-height: 56vh; + z-index: 145; + border: 1px solid var(--border-default); + border-radius: var(--radius-lg); + backdrop-filter: blur(20px); + background: var(--bg-elevated); +} + +/* ============================================================================ + Responsive Design + ============================================================================ */ + +@media (max-width: 1100px) { + #overlay { + grid-template-columns: 280px 1fr 320px; + } + .tb-btn span { display: none; } /* hide text labels, keep icons */ + .tb-btn svg + span { display: none; } +} + +@media (max-width: 900px) { + #overlay { + grid-template-columns: 0 1fr 320px; + } + .side-panel-left { display: none; } + .left-panel-open { display: none !important; } +} + +@media (max-width: 640px) { + #overlay { + grid-template-columns: 1fr; + grid-template-rows: auto 1fr; + grid-template-areas: "top" "."; + } + .side-panel { + height: auto; + max-height: 50vh; + } + .side-panel-left, .side-panel-right { display: none; } +} + +/* ============================================================================ + Scrollbar Styling + ============================================================================ */ + +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: transparent; +} + +::-webkit-scrollbar-thumb { + background: var(--surface-3); + border-radius: 0; +} + +::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.15); +} + +/* ============================================================================ + Selection + ============================================================================ */ + +::selection { + background: var(--accent-primary-subtle); + color: var(--text-primary); +} + +/* ============================================================================ + Level Editor – Shape Dropdown, Primitives, Lights + ============================================================================ */ + +/* Shape Dropdown */ +.shape-dropdown-wrapper { + position: relative; + display: inline-block; +} + +.shape-dropdown-menu { + position: absolute; + top: calc(100% + 4px); + left: 0; + z-index: 200; + background: var(--bg-secondary); + border: 1px solid var(--border-default); + border-radius: 0; + box-shadow: none; + padding: 4px; + min-width: 140px; + display: flex; + flex-direction: column; + gap: 2px; +} + +.shape-dropdown-menu.hidden { + display: none; +} + +.shape-dropdown-menu button { + display: flex; + align-items: center; + gap: 8px; + width: 100%; + padding: 8px 12px; + background: transparent; + border: none; + color: var(--text-primary); + font-size: 13px; + font-family: inherit; + font-weight: 500; + border-radius: var(--radius-sm); + cursor: pointer; + transition: background var(--transition-fast); + text-align: left; +} + +.shape-dropdown-menu button:hover { + background: var(--surface-2); +} + +.shape-icon { + display: inline-flex; + width: 18px; + justify-content: center; + font-size: 14px; + opacity: 0.7; +} + +/* Primitive & Light Properties Panel */ +.prim-props { + padding: 0; +} + +.prim-props.hidden { + display: none; +} + +.prop-group { + margin-bottom: 6px; +} + +.prop-group-title { + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--text-tertiary); + margin-bottom: 4px; +} + +.prop-row, .dt-row { + display: flex; + align-items: center; + gap: 6px; + margin-bottom: 6px; +} + +.prop-label { + font-size: 11px; + font-weight: 600; + color: var(--text-tertiary); + min-width: 48px; + flex-shrink: 0; +} + +input[type="color"] { + -webkit-appearance: none; + appearance: none; + width: 32px; + height: 24px; + border: 1px solid var(--border-default); + border-radius: 0; + background: transparent; + cursor: pointer; + padding: 2px; +} + +input[type="color"]::-webkit-color-swatch-wrapper { padding: 0; } +input[type="color"]::-webkit-color-swatch { border: none; border-radius: 0; } + +.prop-check { + display: flex; + align-items: center; + gap: 6px; + font-size: 11px; + color: var(--text-secondary); + cursor: pointer; + padding: 4px 0; +} + +.prop-check input[type="checkbox"] { + accent-color: var(--accent-primary); + width: 13px; + height: 13px; +} + +/* Blob shadow sub-controls */ +.blob-shadow-controls { + padding: 4px 0 4px 19px; /* indent under the checkbox */ + display: flex; + flex-direction: column; + gap: 3px; +} +.blob-shadow-controls.hidden { display: none; } +.blob-shadow-controls .prop-row { + display: flex; + align-items: center; + gap: 6px; + font-size: 11px; + color: var(--text-secondary); +} +.blob-shadow-controls .prop-row label { + min-width: 52px; + flex-shrink: 0; +} +.blob-shadow-controls .prop-row input[type="range"] { + flex: 1; + height: 3px; + accent-color: var(--accent-primary); +} +.blob-shadow-controls .prop-row span { + min-width: 28px; + text-align: right; + font-variant-numeric: tabular-nums; +} +.blob-shadow-controls .prop-row input[type="number"] { + width: 60px; +} + +.prop-row-3 { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: 6px; + margin-bottom: 8px; +} + +.prop-input-group { + display: flex; + flex-direction: column; + gap: 2px; +} + +.prop-input-group label { + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + color: var(--text-tertiary); +} + +.prop-input-group input[type="number"] { + width: 100%; + padding: 6px 8px; + background: var(--surface-1); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + color: var(--text-primary); + font-family: inherit; + font-size: 12px; + -moz-appearance: textfield; +} + +.prop-input-group input[type="number"]::-webkit-outer-spin-button, +.prop-input-group input[type="number"]::-webkit-inner-spin-button { + -webkit-appearance: none; + margin: 0; +} + +.prop-input-group input[type="number"]:focus { + outline: none; + border-color: var(--accent-primary); +} + +.file-sm { + font-size: 12px !important; + padding: 4px 10px !important; +} + +.btn-sm { + font-size: 12px; + padding: 4px 10px; + background: var(--surface-2); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + color: var(--text-secondary); + cursor: pointer; + font-family: inherit; + transition: all var(--transition-fast); +} + +.btn-sm:hover { + background: var(--surface-3); + color: var(--text-primary); +} + +.btn-sm.danger { + color: #ef4444; +} + +.btn-sm.danger:hover { + background: rgba(239, 68, 68, 0.15); + color: #f87171; +} + +/* Primitive textarea */ +.prim-props textarea { + width: 100%; + padding: 8px 10px; + background: var(--surface-1); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + color: var(--text-primary); + font-family: inherit; + font-size: 13px; + margin-bottom: 8px; + resize: vertical; + min-height: 40px; + transition: border-color var(--transition-fast); +} + +.prim-props textarea:focus { + outline: none; + border-color: var(--accent-primary); +} + +/* Metadata Key-Value Editor */ +.meta-kv-list { + display: flex; + flex-direction: column; + gap: 4px; +} + +.meta-kv-empty { + font-size: 11px; + color: var(--text-tertiary); + font-style: italic; + padding: 4px 0; +} + +.meta-kv-row { + display: flex; + gap: 4px; + align-items: center; +} + +.meta-kv-key, +.meta-kv-val { + flex: 1; + min-width: 0; + padding: 5px 8px; + background: var(--surface-1); + border: 1px solid var(--border-subtle); + border-radius: var(--radius-sm); + color: var(--text-primary); + font-family: inherit; + font-size: 12px; + transition: border-color var(--transition-fast); +} + +.meta-kv-key { + flex: 0.45; + font-weight: 600; + color: var(--accent-info); +} + +.meta-kv-val { + flex: 0.55; +} + +.meta-kv-key:focus, +.meta-kv-val:focus { + outline: none; + border-color: var(--accent-primary); +} + +/* ============================================================================ + Agent Sensor Panels (sidebar) + ============================================================================ */ +.agent-sensor-label { + font-size: 10px; + font-weight: 600; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.5px; + padding: 4px 8px 2px; +} + +.agent-sensor-canvas { + width: 100%; + height: auto; + display: block; + background: #000; + aspect-ratio: 2.2 / 1; +} diff --git a/misc/DimSim/update-sims.sh b/misc/DimSim/update-sims.sh new file mode 100644 index 0000000000..a672030781 --- /dev/null +++ b/misc/DimSim/update-sims.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Regenerate public/sims/manifest.json from the .json files in public/sims/. +# Run after adding or removing scene files. +set -e +DIR="$(cd "$(dirname "$0")" && pwd)/public/sims" +cd "$DIR" +echo -n "[" > manifest.json +first=true +for f in *.json; do + [ "$f" = "manifest.json" ] && continue + name="${f%.json}" + if [ "$first" = true ]; then first=false; else echo -n "," >> manifest.json; fi + echo -n "\"$name\"" >> manifest.json +done +echo "]" >> manifest.json +echo "Updated manifest: $(cat manifest.json)" diff --git a/misc/DimSim/vite.config.js b/misc/DimSim/vite.config.js new file mode 100644 index 0000000000..4d21fd3a4b --- /dev/null +++ b/misc/DimSim/vite.config.js @@ -0,0 +1,22 @@ +import { defineConfig } from "vite"; + +export default defineConfig({ + optimizeDeps: { + exclude: ["@sparkjsdev/spark", "@dimforge/rapier3d-compat"], + }, + assetsInclude: ["**/*.wasm"], + build: { + assetsInlineLimit: 0, + rollupOptions: { + external: [/^https:\/\/esm\.sh\//], + }, + }, + server: { + proxy: { + "/vlm": { + target: "http://127.0.0.1:8000", + changeOrigin: true, + }, + }, + }, +}); diff --git a/misc/DimSim/vlm-server/asset-library.json b/misc/DimSim/vlm-server/asset-library.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/misc/DimSim/vlm-server/asset-library.json @@ -0,0 +1 @@ +[] diff --git a/pyproject.toml b/pyproject.toml index 47b8a25f62..8f7fcccf6b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -568,4 +568,7 @@ ignore = [ "dimos/manipulation/manipulation_module.py", "dimos/navigation/nav_stack/modules/*/main.cpp", "dimos/navigation/nav_stack/common/*.hpp", + "misc/DimSim/src/AiAvatar.js", + "misc/DimSim/src/style.css", + "misc/DimSim/src/engine.js", ]