From d273aec0960136b66b475c2653ab2a46ccf31891 Mon Sep 17 00:00:00 2001 From: Jason717717 <1354471825@qq.com> Date: Sat, 10 May 2025 06:06:46 -0500 Subject: [PATCH 1/5] feat: add support for viewing images and point cloud in rosbag --- .../sr_viz/threeD/package-lock.json | 71 +++++ .../visualization/sr_viz/threeD/package.json | 2 + .../sr_viz/threeD/src/app/page.tsx | 24 ++ .../threeD/src/app/rosbagViewer/page.tsx | 36 +++ .../threeD/src/components/PanelManager.tsx | 277 ++++++++++++++++++ .../threeD/src/components/PointCloudPanel.tsx | 174 +++++++++++ .../threeD/src/components/RosbagViewer.tsx | 155 ++++++++++ .../threeD/src/components/Scrubber2.tsx | 156 ++++++++++ .../threeD/src/components/ScrubberContext.tsx | 32 ++ .../threeD/src/components/VideoPanel.tsx | 255 ++++++++++++++++ .../threeD/src/hooks/usePlaybackTime.ts | 2 +- 11 files changed, 1183 insertions(+), 1 deletion(-) create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/app/rosbagViewer/page.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx create mode 100644 GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json b/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json index e9ef8df23..5414b9c5c 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json +++ b/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", + "@foxglove/rosbag": "^0.4.1", "@mui/icons-material": "^7.0.2", "@mui/material": "^7.0.2", "@react-three/drei": "^10.0.6", @@ -19,6 +20,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-icons": "^5.5.0", + "react-resizable-panels": "^3.0.1", "three": "^0.175.0", "urdf-loader": "^0.12.4" }, @@ -503,6 +505,56 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@foxglove/message-definition": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.2.0.tgz", + "integrity": "sha512-IQHIGCvBZR8GIua9nEpS+hsMF3gm1bfbrrnjG0rgtcFBWiNuKbzx4vIP8OIwDC+8wtwcFdfJhf4Vp5TPFiUUcQ==" + }, + "node_modules/@foxglove/rosbag": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@foxglove/rosbag/-/rosbag-0.4.1.tgz", + "integrity": "sha512-Tbt1SUz+xnKOX+bvhbHiVjNMpTM28hl3zAku8fJ7+8qTZjJPrW95p3W4Aiz6YVWO+NMhCns5wQqOv8rvBpCotg==", + "dependencies": { + "@foxglove/rosmsg": "^4.0.0", + "@foxglove/rosmsg-serialization": "^2.0.0", + "@foxglove/rostime": "^1.1.2", + "heap": "^0.2.7" + } + }, + "node_modules/@foxglove/rosmsg": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg/-/rosmsg-4.2.2.tgz", + "integrity": "sha512-bAY7+b/3AJnR6pIkUWbDdFCRGwCc/WaI9V/51nkj0iNP16ZMJ6s5xPDwKYMpaNIlFSyJG1QE69AmRPRYzP6oEQ==", + "dependencies": { + "@foxglove/message-definition": "^0.2.0", + "md5-typescript": "^1.0.5" + }, + "bin": { + "gendeps2": "bin/gendeps2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rosmsg-serialization": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg-serialization/-/rosmsg-serialization-2.0.3.tgz", + "integrity": "sha512-IN/VREnP+3eGNBQJUXQxKfX1ZG/rc0/kkRdZMhqkuFhCywGHwOGNo8KNVEuEO6EfYygREArPoRB1V9L0MUjZ2Q==", + "dependencies": { + "@foxglove/message-definition": "^0.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rostime": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@foxglove/rostime/-/rostime-1.1.2.tgz", + "integrity": "sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -4108,6 +4160,11 @@ "node": ">= 0.4" } }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "node_modules/hls.js": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.2.tgz", @@ -5026,6 +5083,11 @@ "node": ">= 0.4" } }, + "node_modules/md5-typescript": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/md5-typescript/-/md5-typescript-1.0.5.tgz", + "integrity": "sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -5595,6 +5657,15 @@ "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==" }, + "node_modules/react-resizable-panels": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-3.0.1.tgz", + "integrity": "sha512-6ruCEyw0iqXRcXEktPQn1HL553DNhrdLisCyEdSpzhkmo9bPqZxskJZ+aGeFqJ1qPvIWxuAiag82kvLSb2JZTQ==", + "peerDependencies": { + "react": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc", + "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + } + }, "node_modules/react-transition-group": { "version": "4.4.5", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/package.json b/GEMstack/onboard/visualization/sr_viz/threeD/package.json index f86ed25a2..ccf138646 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/package.json +++ b/GEMstack/onboard/visualization/sr_viz/threeD/package.json @@ -11,6 +11,7 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", + "@foxglove/rosbag": "^0.4.1", "@mui/icons-material": "^7.0.2", "@mui/material": "^7.0.2", "@react-three/drei": "^10.0.6", @@ -20,6 +21,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-icons": "^5.5.0", + "react-resizable-panels": "^3.0.1", "three": "^0.175.0", "urdf-loader": "^0.12.4" }, diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx index 1e2d0ad24..ef623d394 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx @@ -5,13 +5,18 @@ import ControlPanel from "@/components/ControlPanel"; import CanvasWrapper from "@/components/CanvasWrapper"; import Scrubber from "@/components/Scrubber"; import { usePlaybackTime } from "@/hooks/usePlaybackTime"; +import { useRouter } from "next/navigation"; +import Button from "@mui/material/Button"; +import PageviewIcon from '@mui/icons-material/Pageview'; export default function HomePage() { + const router = useRouter(); const { time, reset, restart, play, + setPlay, togglePlay, speed, setPlaybackSpeed, @@ -31,8 +36,27 @@ export default function HomePage() { } }, []); + const handleRedirect = () => { + setPlay(false); + router.replace("/rosbagViewer"); + }; + return (
+
+
+ +
+
{ + router.replace("/"); + }; + return ( +
+
+
+ +
+
+ +
+ ); +} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx new file mode 100644 index 000000000..d77688006 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx @@ -0,0 +1,277 @@ +"use client"; + +import React, { useState } from "react"; +import { VideoPanel } from "./VideoPanel"; +import { PointCloudPanel } from "./PointCloudPanel"; +import { Button, Menu, MenuItem, ListItemIcon, ListItemText } from "@mui/material"; +import MenuIcon from "@mui/icons-material/Menu"; +import { PanelGroup, Panel, PanelResizeHandle } from "react-resizable-panels"; +import HorizontalSplitIcon from '@mui/icons-material/HorizontalSplit'; +import VerticalSplitIcon from '@mui/icons-material/VerticalSplit'; +import ImageIcon from '@mui/icons-material/Image'; +import HubIcon from '@mui/icons-material/Hub'; +import CloseIcon from '@mui/icons-material/Close'; + +type PanelNode = + | { + id: number; + type: "video" | "pointcloud" | "text"; + } + | { + id: number; + split: "horizontal" | "vertical"; + children: PanelNode[]; + }; + +let panelIdCounter = 2; + +function createPanel(type: "video" | "pointcloud" | "text"): PanelNode { + return { id: panelIdCounter++, type }; +} + +export const PanelManager = ({ + messageMap, +}: { + messageMap: Record; +}) => { + const [rootPanel, setRootPanel] = useState(createPanel("video")); + + const renderPanelNode = (node: PanelNode): JSX.Element => { + if ("split" in node) { + return ( + + + {node.children.map((child, idx) => ( + + {renderPanelNode(child)} + {idx < node.children.length - 1 && ( + + )} + + ))} + + + ); + } + + return ( + + { + const newSplit: PanelNode = { + id: panelIdCounter++, + split: direction, + children: [node, createPanel("video")], + }; + replacePanel( + rootPanel, + node.id, + newSplit, + setRootPanel + ); + }} + onClose={() => closePanel(rootPanel, node.id, setRootPanel)} + onChangeType={(newType) => + changePanelType( + rootPanel, + node.id, + newType, + setRootPanel + ) + } + rootPanel={rootPanel} + /> + {node.type === "video" && ( + + )} + {node.type === "pointcloud" && ( + + )} + + ); + }; + + return ( +
+ {/*
+ +
*/} + + {renderPanelNode(rootPanel)} + +
+ ); +}; + +function replacePanel( + root: PanelNode, + targetId: number, + replacement: PanelNode, + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode { + if (node.id === targetId) return replacement; + if ("split" in node) { + return { + ...node, + children: node.children.map(helper), + }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function closePanel( + root: PanelNode, + targetId: number, + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode | null { + if (node.id === targetId) return null; + if ("split" in node) { + const newChildren = node.children + .map(helper) + .filter((child) => child !== null) as PanelNode[]; + return { ...node, children: newChildren }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function changePanelType( + root: PanelNode, + targetId: number, + newType: "video" | "pointcloud" | "text", + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode { + if (node.id === targetId) { + return { ...node, type: newType }; + } + if ("split" in node) { + return { + ...node, + children: node.children.map(helper), + }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function getNumLeaves(node: PanelNode): number { + if ("split" in node) { + return node.children.reduce((sum, child) => sum + getNumLeaves(child), 0); + } + return 1; +} + +const PanelMenu = ({ + onSplit, + onClose, + onChangeType, + rootPanel, +}: { + onSplit: (dir: "horizontal" | "vertical") => void; + onClose: () => void; + onChangeType: (newType: "video" | "pointcloud") => void; + rootPanel: PanelNode; +}) => { + const [anchorEl, setAnchorEl] = useState(null); + const open = Boolean(anchorEl); + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => setAnchorEl(null); + + return ( +
+ + + { + onSplit("horizontal"); + handleClose(); + }} + > + + + + + Split Horizontally + + + { + onSplit("vertical"); + handleClose(); + }} + > + + + + + Split Vertically + + + { + onChangeType("video"); + handleClose(); + }} + > + + + + + Switch to Video Panel + + + { + onChangeType("pointcloud"); + handleClose(); + }} + > + + + + + Switch to PointCloud Panel + + + {getNumLeaves(rootPanel) > 1 && ( + { + onClose(); + handleClose(); + }} + > + + + + + Close Panel + + + )} + +
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx new file mode 100644 index 000000000..494ff8888 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx @@ -0,0 +1,174 @@ +"use client"; + +import React, { useEffect, useRef } from "react"; +import { useScrubber } from "./ScrubberContext"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; + +function parsePointCloud2(msg: any): THREE.Points { + const { data, point_step, fields } = msg; + const positions: number[] = []; + const colors: number[] = []; + + const fieldMap = Object.fromEntries(fields.map((f: any) => [f.name, f.offset])); + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + + for (let i = 0; i < data.length; i += point_step) { + const x = dv.getFloat32(i + fieldMap["x"], true); + const y = dv.getFloat32(i + fieldMap["y"], true); + const z = dv.getFloat32(i + fieldMap["z"], true); + + if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) continue; + + positions.push(x, y, z); + + if ("rgb" in fieldMap) { + const rgb = dv.getUint32(i + fieldMap["rgb"], true); + const r = (rgb >> 16) & 0xff; + const g = (rgb >> 8) & 0xff; + const b = rgb & 0xff; + colors.push(r / 255, g / 255, b / 255); + } else { + colors.push(1, 1, 1); + } + } + + const geometry = new THREE.BufferGeometry(); + geometry.setAttribute("position", new THREE.Float32BufferAttribute(positions, 3)); + geometry.setAttribute("color", new THREE.Float32BufferAttribute(colors, 3)); + const material = new THREE.PointsMaterial({ size: 0.05, vertexColors: true }); + + return new THREE.Points(geometry, material); + } + + function buildTransformLookup(tfMessages: any[], time: number) { + const latestTransforms = new Map(); + + for (const msg of tfMessages) { + const transforms = msg.data.transforms || []; + for (const t of transforms) { + if (t.header.stamp.sec + t.header.stamp.nsec * 1e-9 < time) continue; + const key = `${t.header.frame_id}->${t.child_frame_id}`; + if ( + !latestTransforms.has(key) + ) { + latestTransforms.set(key, t); + } + } + } + + return latestTransforms; + } + + function getTransformMatrix( + lookup: Map, + from: string, + to: string + ): THREE.Matrix4 | null { + const key = `${from}->${to}`; + const tf = lookup.get(key); + if (!tf) return null; + + const { translation, rotation } = tf.transform; + const position = new THREE.Vector3(translation.x, translation.y, translation.z); + const quaternion = new THREE.Quaternion(rotation.x, rotation.y, rotation.z, rotation.w); + + const matrix = new THREE.Matrix4(); + matrix.makeRotationFromQuaternion(quaternion); + matrix.setPosition(position); + + return matrix; + } + +export const PointCloudPanel = ({ messages, tfMessages }: { messages: any[], tfMessages: any[] }) => { + const mountRef = useRef(null); + const { startTime, currentTime } = useScrubber(); + const rendererRef = useRef(null); + const cameraRef = useRef(); + const sceneRef = useRef(); + const controlsRef = useRef(); + const pointCloudRef = useRef(); + + useEffect(() => { + if (!mountRef.current) return; + + const scene = new THREE.Scene(); + const camera = new THREE.PerspectiveCamera(75, mountRef.current.clientWidth / mountRef.current.clientHeight, 0.1, 1000); + camera.position.z = 5; + + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setSize(mountRef.current.clientWidth, mountRef.current.clientHeight); + mountRef.current.appendChild(renderer.domElement); + + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + + sceneRef.current = scene; + cameraRef.current = camera; + rendererRef.current = renderer; + controlsRef.current = controls; + + const animate = () => { + requestAnimationFrame(animate); + controls.update(); + renderer.render(scene, camera); + }; + animate(); + + return () => { + if (rendererRef.current) { + rendererRef.current.dispose(); + const canvas = rendererRef.current.domElement; + canvas.remove(); + } + }; + }, []); + + useEffect(() => { + const scene = sceneRef.current; + if (!scene) return; + + const msg = messages.find((m) => m.timestamp >= startTime + currentTime && m.data.header.frame_id === "velodyne"); + if (!msg) return; + + if (pointCloudRef.current) { + scene.remove(pointCloudRef.current); + pointCloudRef.current.geometry.dispose(); + (pointCloudRef.current.material as THREE.Material).dispose(); + } + + const newCloud = parsePointCloud2(msg.data); + pointCloudRef.current = newCloud; + + const transforms = buildTransformLookup(tfMessages, currentTime + startTime); + const cloudFrame = msg.data.header.frame_id; + const targetFrame = "base_link"; + + const matrix = getTransformMatrix(transforms, cloudFrame, targetFrame); + + if (matrix) { + newCloud.applyMatrix4(matrix); + } + scene.add(newCloud); + }, [startTime, currentTime, messages, tfMessages]); + + useEffect(() => { + if (!mountRef.current || !rendererRef.current || !cameraRef.current) return; + + const observer = new ResizeObserver(() => { + const width = mountRef.current!.clientWidth; + const height = mountRef.current!.clientHeight; + + rendererRef.current!.setSize(width, height); + cameraRef.current!.aspect = width / height; + cameraRef.current!.updateProjectionMatrix(); + }); + + observer.observe(mountRef.current); + + return () => observer.disconnect(); +}, []); + + + return
; +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx new file mode 100644 index 000000000..d103d8c2f --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx @@ -0,0 +1,155 @@ +"use client"; + +import React, { useState } from "react"; +import { Bag } from "@foxglove/rosbag"; +import Button from "@mui/material/Button"; +import CloudUploadIcon from "@mui/icons-material/CloudUpload"; +import { styled } from "@mui/material/styles"; +import { ScrubberProvider } from "./ScrubberContext"; +import { Scrubber2 } from "@/components/Scrubber2"; +import { PanelManager } from "./PanelManager"; + +const VisuallyHiddenInput = styled("input")({ + clip: "rect(0 0 0 0)", + clipPath: "inset(50%)", + height: 1, + overflow: "hidden", + position: "absolute", + bottom: 0, + left: 0, + whiteSpace: "nowrap", + width: 1, +}); + +function fileToReader(file: File) { + return { + size: () => file.size, + read: async (offset: number, length: number) => { + const blob = file.slice(offset, offset + length); + const arrayBuffer = await blob.arrayBuffer(); + return new Uint8Array(arrayBuffer); + }, + }; +} + +export default function RosbagViewer() { + const [topics, setTopics] = useState([]); + const [types, setTypes] = useState([]); + const [loading, setLoading] = useState(false); + const [messageMap, setMessageMap] = useState>({ + video: [], + pointcloud: [], + }); + + const [duration, setDuration] = useState(0); + const [startTime, setStartTime] = useState(0); + + const handleFileUpload = async ( + event: React.ChangeEvent + ) => { + const file = event.target.files?.[0]; + if (!file) return; + + setMessageMap({ + video: [], + pointcloud: [], + tf: [], + }); + setLoading(true); + const reader = fileToReader(file); + const bag = new Bag(reader); + await bag.open(); + console.log("Bag opened:", bag); + const topicNames = Array.from(bag.connections.values()).map( + (conn) => conn.topic + ); + const topicTypes = Array.from(bag.connections.values()).map( + (conn) => conn.type + ); + setTopics(topicNames); + setTypes(topicTypes); + console.log("Topics:", topicNames, topicTypes); + + const videoMessages: any[] = []; + const pointcloudMessages: any[] = []; + const tfMessages: any[] = []; + const velodyneScanMessages: any[] = []; + for await (const msg of bag.messageIterator()) { + const entry = { + timestamp: msg.timestamp.sec + msg.timestamp.nsec * 1e-9, + topic: msg.topic, + data: msg.message, + }; + const type = topicTypes[topicNames.indexOf(msg.topic)]; + if (type.includes("Image")) videoMessages.push(entry); + if (type.includes("PointCloud2")) pointcloudMessages.push(entry); + if (type.includes("TFMessage")) tfMessages.push(entry); + if (type.includes("VelodyneScan")) velodyneScanMessages.push(entry); + } + setLoading(false); + console.log("Messages parsed:", videoMessages, pointcloudMessages, tfMessages, velodyneScanMessages); + setMessageMap({ + video: videoMessages, + pointcloud: pointcloudMessages, + tf: tfMessages, + }); + if (videoMessages.length > 0) { + setDuration( + Math.max( + videoMessages[videoMessages.length - 1].timestamp - + videoMessages[0].timestamp, + 0 + ) + ); + } + if (pointcloudMessages.length > 0) { + setDuration((prev) => + Math.max( + prev, + pointcloudMessages[pointcloudMessages.length - 1] + .timestamp - pointcloudMessages[0].timestamp, + 0 + ) + ); + } + const videoStart = videoMessages[0]?.timestamp || 0; + const pointcloudStart = pointcloudMessages[0]?.timestamp || 0; + const start = (videoStart > 0 && pointcloudStart > 0) ? Math.min(videoStart, pointcloudStart) : Math.max(videoStart, pointcloudStart); + setStartTime(start); + }; + + return ( +
+
+
+ +
+
+ + + + + +
+ ); +} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx new file mode 100644 index 000000000..d52b3e287 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx @@ -0,0 +1,156 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { useScrubber } from "./ScrubberContext"; +import { IconButton, Slider, Menu, MenuItem } from "@mui/material"; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import PauseIcon from "@mui/icons-material/Pause"; +import SpeedIcon from "@mui/icons-material/Speed"; +import RefreshIcon from "@mui/icons-material/Refresh"; + +export const Scrubber2 = ({ duration, startTime, loading }: { duration: number, startTime: number, loading: boolean }) => { + const { + setStartTime, + currentTime, + setCurrentTime, + isPlaying, + setIsPlaying, + } = useScrubber(); + const frameRate = 30; + const [anchorEl, setAnchorEl] = useState(null); + const [selectedSpeed, setSelectedSpeed] = useState(1); + const speedOptions = [0.5, 1, 1.5, 2, 3]; + const open = Boolean(anchorEl); + const [isDragging, setIsDragging] = useState(false); + + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => { + setAnchorEl(null); + }; + const handleMenuItemClick = (speed: number) => { + setSelectedSpeed(speed); + handleClose(); + }; + const handleSliderChange = (_: Event, newValue: number) => { + setCurrentTime(newValue); + }; + const handleSliderChangeCommitted = () => { + setIsDragging(false); + } + const togglePlay = () => { + setIsPlaying((prev: boolean) => !prev); + } + const restart = () => { + setCurrentTime(0); + }; + const formatDuration = (time: number) => { + const minutes = Math.floor(time / 60); + const seconds = Math.floor(time % 60); + return `${minutes < 10 ? "0" : ""}${minutes}:${ + seconds < 10 ? "0" : "" + }${seconds}`; + }; + const handleContextMenu = (event: React.MouseEvent) => { + event.preventDefault(); + }; + useEffect(() => { + if (!isPlaying) return; + const interval = setInterval(() => { + setCurrentTime((prev: number) => + Math.min(prev + selectedSpeed * frameRate / 1000, duration) + ); + }, 1000 / frameRate); + return () => clearInterval(interval); + }, [isPlaying, duration, selectedSpeed]); + + useEffect(() => { + setStartTime(startTime); + setCurrentTime(0); + }, [startTime]); + + useEffect(() => { + if (currentTime >= duration) { + setIsPlaying(false); + } + }, [currentTime, duration]); + + return ( +
+ + {isPlaying ? ( + + ) : ( + + )} + +
+
+ {currentTime < duration + ? formatDuration(currentTime) + : formatDuration(duration)} +
+ setIsDragging(true)} + onChangeCommitted={handleSliderChangeCommitted} + value={currentTime} + /> +
{formatDuration(duration)}
+
+
+ + + + + {speedOptions.map((speed) => ( + handleMenuItemClick(speed)} + > + {speed}x + + ))} + + + + +
+
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx new file mode 100644 index 000000000..180c79356 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx @@ -0,0 +1,32 @@ +"use client"; + +import React, { createContext, useContext, useState } from "react"; + +const ScrubberContext = createContext(null); + +export const ScrubberProvider = ({ + children, +}: { + children: React.ReactNode; +}) => { + const [startTime, setStartTime] = useState(0); + const [currentTime, setCurrentTime] = useState(0); + const [isPlaying, setIsPlaying] = useState(false); + + return ( + + {children} + + ); +}; + +export const useScrubber = () => useContext(ScrubberContext); diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx new file mode 100644 index 000000000..82b12f402 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx @@ -0,0 +1,255 @@ +"use client"; + +import React, { useEffect, useRef, useState } from "react"; +import { useScrubber } from "./ScrubberContext"; + +export function decodeImage(msg: { + encoding: string; + data: Uint8Array; + height: number; + width: number; + step: number; +}): ImageData { + const { encoding, data, height, width, step } = msg; + const rgba = new Uint8ClampedArray(width * height * 4); + + if (encoding === "bgr8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const srcIdx = i * step + j * 3; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = data[srcIdx + 2]; + rgba[dstIdx + 1] = data[srcIdx + 1]; + rgba[dstIdx + 2] = data[srcIdx]; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "rgb8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const srcIdx = i * step + j * 3; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = data[srcIdx]; + rgba[dstIdx + 1] = data[srcIdx + 1]; + rgba[dstIdx + 2] = data[srcIdx + 2]; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "mono8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const gray = data[i * step + j]; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = gray; + rgba[dstIdx + 1] = gray; + rgba[dstIdx + 2] = gray; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "bayer_grbg8") { + const get = (x: number, y: number): number => + x >= 0 && y >= 0 && x < width && y < height + ? data[y * width + x] + : 0; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + let r = 0, + g = 0, + b = 0; + + const val = get(x, y); + + if (y % 2 === 0) { + if (x % 2 === 0) { + g = val; + r = (get(x - 1, y) + get(x + 1, y)) / 2; + b = (get(x, y - 1) + get(x, y + 1)) / 2; + } else { + r = val; + g = + (get(x - 1, y) + + get(x + 1, y) + + get(x, y - 1) + + get(x, y + 1)) / + 4; + b = + (get(x - 1, y - 1) + + get(x + 1, y - 1) + + get(x - 1, y + 1) + + get(x + 1, y + 1)) / + 4; + } + } else { + if (x % 2 === 0) { + b = val; + g = + (get(x - 1, y) + + get(x + 1, y) + + get(x, y - 1) + + get(x, y + 1)) / + 4; + r = + (get(x - 1, y - 1) + + get(x + 1, y - 1) + + get(x - 1, y + 1) + + get(x + 1, y + 1)) / + 4; + } else { + g = val; + r = (get(x, y - 1) + get(x, y + 1)) / 2; + b = (get(x - 1, y) + get(x + 1, y)) / 2; + } + } + + const i = (y * width + x) * 4; + rgba[i] = r; + rgba[i + 1] = g; + rgba[i + 2] = b; + rgba[i + 3] = 255; + } + } + } else { + throw new Error(`Unsupported encoding: ${encoding}`); + } + + return new ImageData(rgba, width, height); +} + +type VideoPanelProps = { + messages: any[]; +}; + +export const VideoPanel: React.FC = ({ messages }) => { + const { startTime, currentTime } = useScrubber(); + const canvasRef = useRef(null); + const containerRef = useRef(null); + const imageRef = useRef(null); + + const [zoom, setZoom] = useState(1); + const [offset, setOffset] = useState({ x: 0, y: 0 }); + const [isDragging, setIsDragging] = useState(false); + const lastMouse = useRef({ x: 0, y: 0 }); + + useEffect(() => { + const msg = messages.find( + (m) => m.timestamp >= startTime + currentTime + ); + if (!msg || !msg.data) return; + + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d", { willReadFrequently: true }); + if (!canvas || !ctx) return; + + try { + const imageData = decodeImage(msg.data); + imageRef.current = imageData; + drawImage(); + } catch (e) { + console.error("Failed to decode image:", e); + } + }, [messages, currentTime]); + + const drawImage = () => { + requestAnimationFrame(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d", { willReadFrequently: true }); + const imageData = imageRef.current as ImageData; + if (!canvas || !ctx || !imageData) return; + + const { width: cw, height: ch } = canvas; + ctx.clearRect(0, 0, cw, ch); + + const imgAspect = imageData.width / imageData.height; + const canvasAspect = cw / ch; + + let drawW = imageData.width * zoom; + let drawH = imageData.height * zoom; + + if (imgAspect > canvasAspect) { + drawW = cw * zoom; + drawH = drawW / imgAspect; + } else { + drawH = ch * zoom; + drawW = drawH * imgAspect; + } + + const drawX = (cw - drawW) / 2 + offset.x; + const drawY = (ch - drawH) / 2 + offset.y; + + const offscreen = document.createElement("canvas"); + offscreen.width = imageData.width; + offscreen.height = imageData.height; + offscreen.getContext("2d")?.putImageData(imageData, 0, 0); + + ctx.drawImage(offscreen, drawX, drawY, drawW, drawH); + }); + }; + + useEffect(() => { + const container = containerRef.current; + if (!container) return; + + const handleWheel = (e: WheelEvent) => { + e.preventDefault(); + const zoomDelta = -e.deltaY * 0.001; + setZoom((z) => Math.min(Math.max(z + zoomDelta, 0.1), 10)); + }; + + container.addEventListener("wheel", handleWheel, { passive: false }); + return () => container.removeEventListener("wheel", handleWheel); + }, []); + + useEffect(() => { + drawImage(); + }, [zoom, offset]); + + useEffect(() => { + const handleResize = () => { + const container = containerRef.current; + const canvas = canvasRef.current; + if (container && canvas) { + canvas.width = container.clientWidth; + canvas.height = container.clientHeight; + drawImage(); + } + }; + + window.addEventListener("resize", handleResize); + handleResize(); + return () => window.removeEventListener("resize", handleResize); + }, []); + + const handleMouseDown = (e: React.MouseEvent) => { + setIsDragging(true); + lastMouse.current = { x: e.clientX, y: e.clientY }; + }; + + const handleMouseUp = () => { + setIsDragging(false); + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (!isDragging) return; + const dx = e.clientX - lastMouse.current.x; + const dy = e.clientY - lastMouse.current.y; + lastMouse.current = { x: e.clientX, y: e.clientY }; + setOffset((prev) => ({ x: prev.x + dx, y: prev.y + dy })); + }; + + return ( +
+ 0 ? (isDragging ? "cursor-grabbing" : "cursor-grab") : ''}`} + /> +
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts b/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts index fda840914..95843bb0f 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts @@ -57,5 +57,5 @@ export function usePlaybackTime() { return () => cancelAnimationFrame(rafId); }, [play, speed]); - return { time, reset, restart, play, togglePlay, speed, setPlaybackSpeed, moveToTime, duration, setDuration }; + return { time, reset, restart, play, setPlay, togglePlay, speed, setPlaybackSpeed, moveToTime, duration, setDuration }; } From 937066ddff0951a8bf66320bb0fa87b5636c16f9 Mon Sep 17 00:00:00 2001 From: Jason717717 <1354471825@qq.com> Date: Sat, 10 May 2025 06:36:37 -0500 Subject: [PATCH 2/5] feat: add a switch to turn on and off drag and zoom --- .../threeD/src/components/VideoPanel.tsx | 153 ++++++++++++------ 1 file changed, 104 insertions(+), 49 deletions(-) diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx index 82b12f402..22a81ac90 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx @@ -2,6 +2,7 @@ import React, { useEffect, useRef, useState } from "react"; import { useScrubber } from "./ScrubberContext"; +import { Switch, FormControlLabel } from "@mui/material"; export function decodeImage(msg: { encoding: string; @@ -124,8 +125,9 @@ export const VideoPanel: React.FC = ({ messages }) => { const { startTime, currentTime } = useScrubber(); const canvasRef = useRef(null); const containerRef = useRef(null); - const imageRef = useRef(null); + const imageRef = useRef(null); + const [enableInteraction, setEnableInteraction] = useState(false); const [zoom, setZoom] = useState(1); const [offset, setOffset] = useState({ x: 0, y: 0 }); const [isDragging, setIsDragging] = useState(false); @@ -138,55 +140,65 @@ export const VideoPanel: React.FC = ({ messages }) => { if (!msg || !msg.data) return; const canvas = canvasRef.current; - const ctx = canvas?.getContext("2d", { willReadFrequently: true }); - if (!canvas || !ctx) return; + const ctx = canvas?.getContext("2d"); try { const imageData = decodeImage(msg.data); imageRef.current = imageData; - drawImage(); + + if (!enableInteraction) { + if (canvas && ctx) { + canvas.width = imageData.width; + canvas.height = imageData.height; + ctx.putImageData(imageData, 0, 0); + } + } else { + drawImage(); + } } catch (e) { console.error("Failed to decode image:", e); } - }, [messages, currentTime]); + }, [messages, currentTime, enableInteraction]); const drawImage = () => { - requestAnimationFrame(() => { - const canvas = canvasRef.current; - const ctx = canvas?.getContext("2d", { willReadFrequently: true }); - const imageData = imageRef.current as ImageData; - if (!canvas || !ctx || !imageData) return; - - const { width: cw, height: ch } = canvas; - ctx.clearRect(0, 0, cw, ch); - - const imgAspect = imageData.width / imageData.height; - const canvasAspect = cw / ch; - - let drawW = imageData.width * zoom; - let drawH = imageData.height * zoom; - - if (imgAspect > canvasAspect) { - drawW = cw * zoom; - drawH = drawW / imgAspect; - } else { - drawH = ch * zoom; - drawW = drawH * imgAspect; - } - - const drawX = (cw - drawW) / 2 + offset.x; - const drawY = (ch - drawH) / 2 + offset.y; - - const offscreen = document.createElement("canvas"); - offscreen.width = imageData.width; - offscreen.height = imageData.height; - offscreen.getContext("2d")?.putImageData(imageData, 0, 0); - - ctx.drawImage(offscreen, drawX, drawY, drawW, drawH); - }); + requestAnimationFrame(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + const imageData = imageRef.current; + if (!canvas || !ctx || !imageData) return; + + const { width: cw, height: ch } = canvas; + ctx.clearRect(0, 0, cw, ch); + + const imgAspect = imageData.width / imageData.height; + const canvasAspect = cw / ch; + + let drawW = imageData.width * zoom; + let drawH = imageData.height * zoom; + + if (imgAspect > canvasAspect) { + drawW = cw * zoom; + drawH = drawW / imgAspect; + } else { + drawH = ch * zoom; + drawW = drawH * imgAspect; + } + + const drawX = (cw - drawW) / 2 + offset.x; + const drawY = (ch - drawH) / 2 + offset.y; + + const offscreen = document.createElement("canvas"); + offscreen.width = imageData.width; + offscreen.height = imageData.height; + offscreen.getContext("2d")?.putImageData(imageData, 0, 0); + + ctx.drawImage(offscreen, drawX, drawY, drawW, drawH); + }); }; useEffect(() => { + if (!enableInteraction) return; + const container = containerRef.current; if (!container) return; @@ -198,11 +210,12 @@ export const VideoPanel: React.FC = ({ messages }) => { container.addEventListener("wheel", handleWheel, { passive: false }); return () => container.removeEventListener("wheel", handleWheel); - }, []); + }, [enableInteraction]); useEffect(() => { + if (!enableInteraction) return; drawImage(); - }, [zoom, offset]); + }, [zoom, offset, enableInteraction]); useEffect(() => { const handleResize = () => { @@ -211,44 +224,86 @@ export const VideoPanel: React.FC = ({ messages }) => { if (container && canvas) { canvas.width = container.clientWidth; canvas.height = container.clientHeight; - drawImage(); + if (enableInteraction) drawImage(); } }; window.addEventListener("resize", handleResize); handleResize(); return () => window.removeEventListener("resize", handleResize); - }, []); + }, [enableInteraction]); + + useEffect(() => { + if (!enableInteraction) { + setZoom(1); + setOffset({ x: 0, y: 0 }); + } + }, [enableInteraction]); const handleMouseDown = (e: React.MouseEvent) => { + if (!enableInteraction) return; setIsDragging(true); lastMouse.current = { x: e.clientX, y: e.clientY }; }; - const handleMouseUp = () => { - setIsDragging(false); - }; - const handleMouseMove = (e: React.MouseEvent) => { - if (!isDragging) return; + if (!enableInteraction || !isDragging) return; const dx = e.clientX - lastMouse.current.x; const dy = e.clientY - lastMouse.current.y; lastMouse.current = { x: e.clientX, y: e.clientY }; setOffset((prev) => ({ x: prev.x + dx, y: prev.y + dy })); }; + const handleMouseUp = () => { + if (!enableInteraction) return; + setIsDragging(false); + }; + return (
+
+ + setEnableInteraction(e.target.checked) + } + /> + } + sx={{ + backgroundColor: "white", + opacity: 0.25, + borderRadius: "9999px", + paddingRight: "10px", + "&:hover": { + opacity: 0.5, + }, + }} + /> +
+ 0 ? (isDragging ? "cursor-grabbing" : "cursor-grab") : ''}`} + className={`w-full h-full ${ + enableInteraction && isDragging + ? "cursor-grabbing" + : enableInteraction + ? "cursor-grab" + : "" + }`} + style={{ + width: "100%", + height: enableInteraction ? "100%" : "auto", + }} />
); From ff9bd5ab6ee562a43973ec00f5075ac7a3f5c093 Mon Sep 17 00:00:00 2001 From: Jason717717 <1354471825@qq.com> Date: Sat, 10 May 2025 06:44:24 -0500 Subject: [PATCH 3/5] format files --- .../sr_viz/threeD/src/app/page.tsx | 124 ++++---- .../threeD/src/components/PanelManager.tsx | 43 ++- .../threeD/src/components/PointCloudPanel.tsx | 294 ++++++++++-------- .../threeD/src/components/RosbagViewer.tsx | 57 ++-- .../threeD/src/components/Scrubber2.tsx | 16 +- 5 files changed, 301 insertions(+), 233 deletions(-) diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx index ef623d394..6800c8fed 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx @@ -7,67 +7,77 @@ import Scrubber from "@/components/Scrubber"; import { usePlaybackTime } from "@/hooks/usePlaybackTime"; import { useRouter } from "next/navigation"; import Button from "@mui/material/Button"; -import PageviewIcon from '@mui/icons-material/Pageview'; +import PageviewIcon from "@mui/icons-material/Pageview"; export default function HomePage() { - const router = useRouter(); - const { - time, - reset, - restart, - play, - setPlay, - togglePlay, - speed, - setPlaybackSpeed, - moveToTime, - duration, - setDuration, - } = usePlaybackTime(); + const router = useRouter(); + const { + time, + reset, + restart, + play, + setPlay, + togglePlay, + speed, + setPlaybackSpeed, + moveToTime, + duration, + setDuration, + } = usePlaybackTime(); - const [searchParams, setSearchParams] = useState<{ folder?: string; file?: string }>({}); + const [searchParams, setSearchParams] = useState<{ + folder?: string; + file?: string; + }>({}); - useEffect(() => { - if (typeof window !== "undefined") { - const params = new URLSearchParams(window.location.search); - const folder = params.get("folder") || undefined; - const file = params.get("file") || undefined; - setSearchParams({ folder, file }); - } - }, []); + useEffect(() => { + if (typeof window !== "undefined") { + const params = new URLSearchParams(window.location.search); + const folder = params.get("folder") || undefined; + const file = params.get("file") || undefined; + setSearchParams({ folder, file }); + } + }, []); - const handleRedirect = () => { - setPlay(false); - router.replace("/rosbagViewer"); - }; + const handleRedirect = () => { + setPlay(false); + router.replace("/rosbagViewer"); + }; - return ( -
-
-
- -
-
- - - -
- ); + return ( +
+
+
+ +
+
+ + + +
+ ); } diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx index d77688006..0c01c0704 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx @@ -3,14 +3,20 @@ import React, { useState } from "react"; import { VideoPanel } from "./VideoPanel"; import { PointCloudPanel } from "./PointCloudPanel"; -import { Button, Menu, MenuItem, ListItemIcon, ListItemText } from "@mui/material"; +import { + Button, + Menu, + MenuItem, + ListItemIcon, + ListItemText, +} from "@mui/material"; import MenuIcon from "@mui/icons-material/Menu"; import { PanelGroup, Panel, PanelResizeHandle } from "react-resizable-panels"; -import HorizontalSplitIcon from '@mui/icons-material/HorizontalSplit'; -import VerticalSplitIcon from '@mui/icons-material/VerticalSplit'; -import ImageIcon from '@mui/icons-material/Image'; -import HubIcon from '@mui/icons-material/Hub'; -import CloseIcon from '@mui/icons-material/Close'; +import HorizontalSplitIcon from "@mui/icons-material/HorizontalSplit"; +import VerticalSplitIcon from "@mui/icons-material/VerticalSplit"; +import ImageIcon from "@mui/icons-material/Image"; +import HubIcon from "@mui/icons-material/Hub"; +import CloseIcon from "@mui/icons-material/Close"; type PanelNode = | { @@ -175,7 +181,10 @@ function changePanelType( function getNumLeaves(node: PanelNode): number { if ("split" in node) { - return node.children.reduce((sum, child) => sum + getNumLeaves(child), 0); + return node.children.reduce( + (sum, child) => sum + getNumLeaves(child), + 0 + ); } return 1; } @@ -213,9 +222,7 @@ const PanelMenu = ({ - - Split Horizontally - + Split Horizontally { @@ -226,9 +233,7 @@ const PanelMenu = ({ - - Split Vertically - + Split Vertically { @@ -239,9 +244,7 @@ const PanelMenu = ({ - - Switch to Video Panel - + Switch to Video Panel { @@ -252,9 +255,7 @@ const PanelMenu = ({ - - Switch to PointCloud Panel - + Switch to PointCloud Panel {getNumLeaves(rootPanel) > 1 && ( - - Close Panel - + Close Panel )} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx index 494ff8888..c669e9e60 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx @@ -9,166 +9,204 @@ function parsePointCloud2(msg: any): THREE.Points { const { data, point_step, fields } = msg; const positions: number[] = []; const colors: number[] = []; - - const fieldMap = Object.fromEntries(fields.map((f: any) => [f.name, f.offset])); + + const fieldMap = Object.fromEntries( + fields.map((f: any) => [f.name, f.offset]) + ); const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); - + for (let i = 0; i < data.length; i += point_step) { - const x = dv.getFloat32(i + fieldMap["x"], true); - const y = dv.getFloat32(i + fieldMap["y"], true); - const z = dv.getFloat32(i + fieldMap["z"], true); - - if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) continue; - - positions.push(x, y, z); - - if ("rgb" in fieldMap) { - const rgb = dv.getUint32(i + fieldMap["rgb"], true); - const r = (rgb >> 16) & 0xff; - const g = (rgb >> 8) & 0xff; - const b = rgb & 0xff; - colors.push(r / 255, g / 255, b / 255); - } else { - colors.push(1, 1, 1); - } + const x = dv.getFloat32(i + fieldMap["x"], true); + const y = dv.getFloat32(i + fieldMap["y"], true); + const z = dv.getFloat32(i + fieldMap["z"], true); + + if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) + continue; + + positions.push(x, y, z); + + if ("rgb" in fieldMap) { + const rgb = dv.getUint32(i + fieldMap["rgb"], true); + const r = (rgb >> 16) & 0xff; + const g = (rgb >> 8) & 0xff; + const b = rgb & 0xff; + colors.push(r / 255, g / 255, b / 255); + } else { + colors.push(1, 1, 1); + } } - + const geometry = new THREE.BufferGeometry(); - geometry.setAttribute("position", new THREE.Float32BufferAttribute(positions, 3)); + geometry.setAttribute( + "position", + new THREE.Float32BufferAttribute(positions, 3) + ); geometry.setAttribute("color", new THREE.Float32BufferAttribute(colors, 3)); - const material = new THREE.PointsMaterial({ size: 0.05, vertexColors: true }); + const material = new THREE.PointsMaterial({ + size: 0.05, + vertexColors: true, + }); return new THREE.Points(geometry, material); - } +} - function buildTransformLookup(tfMessages: any[], time: number) { +function buildTransformLookup(tfMessages: any[], time: number) { const latestTransforms = new Map(); - + for (const msg of tfMessages) { - const transforms = msg.data.transforms || []; - for (const t of transforms) { - if (t.header.stamp.sec + t.header.stamp.nsec * 1e-9 < time) continue; - const key = `${t.header.frame_id}->${t.child_frame_id}`; - if ( - !latestTransforms.has(key) - ) { - latestTransforms.set(key, t); + const transforms = msg.data.transforms || []; + for (const t of transforms) { + if (t.header.stamp.sec + t.header.stamp.nsec * 1e-9 < time) + continue; + const key = `${t.header.frame_id}->${t.child_frame_id}`; + if (!latestTransforms.has(key)) { + latestTransforms.set(key, t); + } } - } } - + return latestTransforms; - } - - function getTransformMatrix( +} + +function getTransformMatrix( lookup: Map, from: string, to: string - ): THREE.Matrix4 | null { +): THREE.Matrix4 | null { const key = `${from}->${to}`; const tf = lookup.get(key); if (!tf) return null; - + const { translation, rotation } = tf.transform; - const position = new THREE.Vector3(translation.x, translation.y, translation.z); - const quaternion = new THREE.Quaternion(rotation.x, rotation.y, rotation.z, rotation.w); - + const position = new THREE.Vector3( + translation.x, + translation.y, + translation.z + ); + const quaternion = new THREE.Quaternion( + rotation.x, + rotation.y, + rotation.z, + rotation.w + ); + const matrix = new THREE.Matrix4(); matrix.makeRotationFromQuaternion(quaternion); matrix.setPosition(position); - - return matrix; - } - -export const PointCloudPanel = ({ messages, tfMessages }: { messages: any[], tfMessages: any[] }) => { - const mountRef = useRef(null); - const { startTime, currentTime } = useScrubber(); - const rendererRef = useRef(null); - const cameraRef = useRef(); - const sceneRef = useRef(); - const controlsRef = useRef(); - const pointCloudRef = useRef(); - - useEffect(() => { - if (!mountRef.current) return; - - const scene = new THREE.Scene(); - const camera = new THREE.PerspectiveCamera(75, mountRef.current.clientWidth / mountRef.current.clientHeight, 0.1, 1000); - camera.position.z = 5; - - const renderer = new THREE.WebGLRenderer({ antialias: true }); - renderer.setSize(mountRef.current.clientWidth, mountRef.current.clientHeight); - mountRef.current.appendChild(renderer.domElement); - - const controls = new OrbitControls(camera, renderer.domElement); - controls.enableDamping = true; - - sceneRef.current = scene; - cameraRef.current = camera; - rendererRef.current = renderer; - controlsRef.current = controls; - - const animate = () => { - requestAnimationFrame(animate); - controls.update(); - renderer.render(scene, camera); - }; - animate(); - - return () => { - if (rendererRef.current) { - rendererRef.current.dispose(); - const canvas = rendererRef.current.domElement; - canvas.remove(); - } - }; - }, []); - - useEffect(() => { - const scene = sceneRef.current; - if (!scene) return; - - const msg = messages.find((m) => m.timestamp >= startTime + currentTime && m.data.header.frame_id === "velodyne"); - if (!msg) return; - - if (pointCloudRef.current) { - scene.remove(pointCloudRef.current); - pointCloudRef.current.geometry.dispose(); - (pointCloudRef.current.material as THREE.Material).dispose(); - } - const newCloud = parsePointCloud2(msg.data); - pointCloudRef.current = newCloud; + return matrix; +} + +export const PointCloudPanel = ({ + messages, + tfMessages, +}: { + messages: any[]; + tfMessages: any[]; +}) => { + const mountRef = useRef(null); + const { startTime, currentTime } = useScrubber(); + const rendererRef = useRef(null); + const cameraRef = useRef(); + const sceneRef = useRef(); + const controlsRef = useRef(); + const pointCloudRef = useRef(); + + useEffect(() => { + if (!mountRef.current) return; + + const scene = new THREE.Scene(); + const camera = new THREE.PerspectiveCamera( + 75, + mountRef.current.clientWidth / mountRef.current.clientHeight, + 0.1, + 1000 + ); + camera.position.z = 5; + + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setSize( + mountRef.current.clientWidth, + mountRef.current.clientHeight + ); + mountRef.current.appendChild(renderer.domElement); + + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + + sceneRef.current = scene; + cameraRef.current = camera; + rendererRef.current = renderer; + controlsRef.current = controls; + + const animate = () => { + requestAnimationFrame(animate); + controls.update(); + renderer.render(scene, camera); + }; + animate(); + + return () => { + if (rendererRef.current) { + rendererRef.current.dispose(); + const canvas = rendererRef.current.domElement; + canvas.remove(); + } + }; + }, []); + + useEffect(() => { + const scene = sceneRef.current; + if (!scene) return; + + const msg = messages.find( + (m) => + m.timestamp >= startTime + currentTime && + m.data.header.frame_id === "velodyne" + ); + if (!msg) return; + + if (pointCloudRef.current) { + scene.remove(pointCloudRef.current); + pointCloudRef.current.geometry.dispose(); + (pointCloudRef.current.material as THREE.Material).dispose(); + } - const transforms = buildTransformLookup(tfMessages, currentTime + startTime); - const cloudFrame = msg.data.header.frame_id; - const targetFrame = "base_link"; + const newCloud = parsePointCloud2(msg.data); + pointCloudRef.current = newCloud; - const matrix = getTransformMatrix(transforms, cloudFrame, targetFrame); + const transforms = buildTransformLookup( + tfMessages, + currentTime + startTime + ); + const cloudFrame = msg.data.header.frame_id; + const targetFrame = "base_link"; - if (matrix) { - newCloud.applyMatrix4(matrix); - } - scene.add(newCloud); - }, [startTime, currentTime, messages, tfMessages]); + const matrix = getTransformMatrix(transforms, cloudFrame, targetFrame); - useEffect(() => { - if (!mountRef.current || !rendererRef.current || !cameraRef.current) return; + if (matrix) { + newCloud.applyMatrix4(matrix); + } + scene.add(newCloud); + }, [startTime, currentTime, messages, tfMessages]); - const observer = new ResizeObserver(() => { - const width = mountRef.current!.clientWidth; - const height = mountRef.current!.clientHeight; + useEffect(() => { + if (!mountRef.current || !rendererRef.current || !cameraRef.current) + return; - rendererRef.current!.setSize(width, height); - cameraRef.current!.aspect = width / height; - cameraRef.current!.updateProjectionMatrix(); - }); + const observer = new ResizeObserver(() => { + const width = mountRef.current!.clientWidth; + const height = mountRef.current!.clientHeight; - observer.observe(mountRef.current); + rendererRef.current!.setSize(width, height); + cameraRef.current!.aspect = width / height; + cameraRef.current!.updateProjectionMatrix(); + }); - return () => observer.disconnect(); -}, []); + observer.observe(mountRef.current); + return () => observer.disconnect(); + }, []); - return
; + return
; }; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx index d103d8c2f..4d3cda06b 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx @@ -49,7 +49,7 @@ export default function RosbagViewer() { ) => { const file = event.target.files?.[0]; if (!file) return; - + setMessageMap({ video: [], pointcloud: [], @@ -87,7 +87,13 @@ export default function RosbagViewer() { if (type.includes("VelodyneScan")) velodyneScanMessages.push(entry); } setLoading(false); - console.log("Messages parsed:", videoMessages, pointcloudMessages, tfMessages, velodyneScanMessages); + console.log( + "Messages parsed:", + videoMessages, + pointcloudMessages, + tfMessages, + velodyneScanMessages + ); setMessageMap({ video: videoMessages, pointcloud: pointcloudMessages, @@ -114,7 +120,10 @@ export default function RosbagViewer() { } const videoStart = videoMessages[0]?.timestamp || 0; const pointcloudStart = pointcloudMessages[0]?.timestamp || 0; - const start = (videoStart > 0 && pointcloudStart > 0) ? Math.min(videoStart, pointcloudStart) : Math.max(videoStart, pointcloudStart); + const start = + videoStart > 0 && pointcloudStart > 0 + ? Math.min(videoStart, pointcloudStart) + : Math.max(videoStart, pointcloudStart); setStartTime(start); }; @@ -123,32 +132,36 @@ export default function RosbagViewer() {
- +
); diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx index d52b3e287..24dd41340 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx @@ -8,7 +8,15 @@ import PauseIcon from "@mui/icons-material/Pause"; import SpeedIcon from "@mui/icons-material/Speed"; import RefreshIcon from "@mui/icons-material/Refresh"; -export const Scrubber2 = ({ duration, startTime, loading }: { duration: number, startTime: number, loading: boolean }) => { +export const Scrubber2 = ({ + duration, + startTime, + loading, +}: { + duration: number; + startTime: number; + loading: boolean; +}) => { const { setStartTime, currentTime, @@ -38,10 +46,10 @@ export const Scrubber2 = ({ duration, startTime, loading }: { duration: number, }; const handleSliderChangeCommitted = () => { setIsDragging(false); - } + }; const togglePlay = () => { setIsPlaying((prev: boolean) => !prev); - } + }; const restart = () => { setCurrentTime(0); }; @@ -59,7 +67,7 @@ export const Scrubber2 = ({ duration, startTime, loading }: { duration: number, if (!isPlaying) return; const interval = setInterval(() => { setCurrentTime((prev: number) => - Math.min(prev + selectedSpeed * frameRate / 1000, duration) + Math.min(prev + (selectedSpeed * frameRate) / 1000, duration) ); }, 1000 / frameRate); return () => clearInterval(interval); From d91f562c6003d4e5982edf116c46ff27a48a9b4a Mon Sep 17 00:00:00 2001 From: Jason717717 <1354471825@qq.com> Date: Sat, 10 May 2025 06:50:11 -0500 Subject: [PATCH 4/5] delete duplicate --- log_dashboard/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/log_dashboard/requirements.txt b/log_dashboard/requirements.txt index dd4bd7d24..e6dac0d83 100644 --- a/log_dashboard/requirements.txt +++ b/log_dashboard/requirements.txt @@ -1,5 +1,4 @@ flask -flask-cors pyyaml cachelib numpy From d5626d9b1a8ff184ef54254b86af0716e744b6bc Mon Sep 17 00:00:00 2001 From: Jason717717 <1354471825@qq.com> Date: Sun, 11 May 2025 08:19:02 -0500 Subject: [PATCH 5/5] feat: add support for selecting topics --- .../threeD/src/components/PanelManager.tsx | 8 +- .../threeD/src/components/PointCloudPanel.tsx | 60 +++++++++-- .../threeD/src/components/RosbagViewer.tsx | 100 ++++++++++-------- .../threeD/src/components/Scrubber2.tsx | 6 ++ .../threeD/src/components/VideoPanel.tsx | 52 ++++++++- 5 files changed, 167 insertions(+), 59 deletions(-) diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx index 0c01c0704..98aad4737 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx @@ -38,7 +38,7 @@ function createPanel(type: "video" | "pointcloud" | "text"): PanelNode { export const PanelManager = ({ messageMap, }: { - messageMap: Record; + messageMap: Record | any[]>; }) => { const [rootPanel, setRootPanel] = useState(createPanel("video")); @@ -94,12 +94,13 @@ export const PanelManager = ({ rootPanel={rootPanel} /> {node.type === "video" && ( - + )} {node.type === "pointcloud" && ( )} @@ -108,9 +109,6 @@ export const PanelManager = ({ return (
- {/*
- -
*/} {renderPanelNode(rootPanel)} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx index c669e9e60..339720961 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx @@ -1,9 +1,10 @@ "use client"; -import React, { useEffect, useRef } from "react"; +import React, { useEffect, useRef, useState } from "react"; import { useScrubber } from "./ScrubberContext"; import * as THREE from "three"; import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; +import { Select, SelectChangeEvent, MenuItem } from "@mui/material"; function parsePointCloud2(msg: any): THREE.Points { const { data, point_step, fields } = msg; @@ -25,7 +26,13 @@ function parsePointCloud2(msg: any): THREE.Points { positions.push(x, y, z); - if ("rgb" in fieldMap) { + if ("rgba" in fieldMap) { + const rgba = dv.getUint32(i + fieldMap["rgba"], true); + const r = (rgba >> 24) & 0xff; + const g = (rgba >> 16) & 0xff; + const b = (rgba >> 8) & 0xff; + colors.push(r / 255, g / 255, b / 255); + } else if ("rgb" in fieldMap) { const rgb = dv.getUint32(i + fieldMap["rgb"], true); const r = (rgb >> 16) & 0xff; const g = (rgb >> 8) & 0xff; @@ -100,9 +107,11 @@ function getTransformMatrix( export const PointCloudPanel = ({ messages, tfMessages, + initialTopic, }: { - messages: any[]; + messages: Record; tfMessages: any[]; + initialTopic?: string; }) => { const mountRef = useRef(null); const { startTime, currentTime } = useScrubber(); @@ -111,8 +120,18 @@ export const PointCloudPanel = ({ const sceneRef = useRef(); const controlsRef = useRef(); const pointCloudRef = useRef(); + const [selectedTopic, setSelectedTopic] = useState( + initialTopic || Object.keys(messages)[0] || "" + ); + + useEffect(() => { + if (initialTopic && messages[initialTopic]) { + setSelectedTopic(initialTopic); + } + }, [initialTopic]); useEffect(() => { + if (!selectedTopic || !messages[selectedTopic]) return; if (!mountRef.current) return; const scene = new THREE.Scene(); @@ -159,10 +178,8 @@ export const PointCloudPanel = ({ const scene = sceneRef.current; if (!scene) return; - const msg = messages.find( - (m) => - m.timestamp >= startTime + currentTime && - m.data.header.frame_id === "velodyne" + const msg = messages[selectedTopic].find( + (m) => m.timestamp >= startTime + currentTime ); if (!msg) return; @@ -208,5 +225,32 @@ export const PointCloudPanel = ({ return () => observer.disconnect(); }, []); - return
; + return ( +
+
+ +
+
+ ); }; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx index 4d3cda06b..d2d9d2867 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx @@ -36,9 +36,14 @@ export default function RosbagViewer() { const [topics, setTopics] = useState([]); const [types, setTypes] = useState([]); const [loading, setLoading] = useState(false); - const [messageMap, setMessageMap] = useState>({ - video: [], - pointcloud: [], + const [messageMap, setMessageMap] = useState<{ + video: Record; + pointcloud: Record; + tf: any[]; + }>({ + video: {}, + pointcloud: {}, + tf: [], }); const [duration, setDuration] = useState(0); @@ -59,7 +64,7 @@ export default function RosbagViewer() { const reader = fileToReader(file); const bag = new Bag(reader); await bag.open(); - console.log("Bag opened:", bag); + // console.log("Bag opened:", bag); const topicNames = Array.from(bag.connections.values()).map( (conn) => conn.topic ); @@ -68,12 +73,11 @@ export default function RosbagViewer() { ); setTopics(topicNames); setTypes(topicTypes); - console.log("Topics:", topicNames, topicTypes); + // console.log("Topics:", topicNames, topicTypes); - const videoMessages: any[] = []; - const pointcloudMessages: any[] = []; + const videoMessages: Record = {}; + const pointcloudMessages: Record = {}; const tfMessages: any[] = []; - const velodyneScanMessages: any[] = []; for await (const msg of bag.messageIterator()) { const entry = { timestamp: msg.timestamp.sec + msg.timestamp.nsec * 1e-9, @@ -81,50 +85,62 @@ export default function RosbagViewer() { data: msg.message, }; const type = topicTypes[topicNames.indexOf(msg.topic)]; - if (type.includes("Image")) videoMessages.push(entry); - if (type.includes("PointCloud2")) pointcloudMessages.push(entry); + if (type.includes("Image")) { + if (!videoMessages[msg.topic]) videoMessages[msg.topic] = []; + videoMessages[msg.topic].push(entry); + } + if (type.includes("PointCloud2")) { + if (!pointcloudMessages[msg.topic]) + pointcloudMessages[msg.topic] = []; + pointcloudMessages[msg.topic].push(entry); + } if (type.includes("TFMessage")) tfMessages.push(entry); - if (type.includes("VelodyneScan")) velodyneScanMessages.push(entry); } setLoading(false); - console.log( - "Messages parsed:", - videoMessages, - pointcloudMessages, - tfMessages, - velodyneScanMessages - ); + // console.log( + // "Messages parsed:", + // videoMessages, + // pointcloudMessages, + // tfMessages + // ); setMessageMap({ video: videoMessages, pointcloud: pointcloudMessages, tf: tfMessages, }); - if (videoMessages.length > 0) { - setDuration( - Math.max( - videoMessages[videoMessages.length - 1].timestamp - - videoMessages[0].timestamp, - 0 - ) - ); - } - if (pointcloudMessages.length > 0) { - setDuration((prev) => - Math.max( - prev, - pointcloudMessages[pointcloudMessages.length - 1] - .timestamp - pointcloudMessages[0].timestamp, - 0 - ) - ); - } - const videoStart = videoMessages[0]?.timestamp || 0; - const pointcloudStart = pointcloudMessages[0]?.timestamp || 0; - const start = + const getDuration = ( + messagesByTopic: Record + ): number => { + const durations = Object.values(messagesByTopic) + .filter((msgs) => msgs.length > 1) + .map( + (msgs) => + msgs[msgs.length - 1].timestamp - msgs[0].timestamp + ); + return durations.length > 0 ? Math.max(...durations) : 0; + }; + + const getStart = (messagesByTopic: Record): number => { + const starts = Object.values(messagesByTopic) + .filter((msgs) => msgs.length > 0) + .map((msgs) => msgs[0].timestamp); + return starts.length > 0 ? Math.min(...starts) : 0; + }; + + setDuration( + Math.max( + getDuration(videoMessages), + getDuration(pointcloudMessages), + 0 + ) + ); + const videoStart = getStart(videoMessages); + const pointcloudStart = getStart(pointcloudMessages); + setStartTime( videoStart > 0 && pointcloudStart > 0 ? Math.min(videoStart, pointcloudStart) - : Math.max(videoStart, pointcloudStart); - setStartTime(start); + : Math.max(videoStart, pointcloudStart) + ); }; return ( diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx index 24dd41340..385af9ec1 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx @@ -84,6 +84,12 @@ export const Scrubber2 = ({ } }, [currentTime, duration]); + useEffect(() => { + if (loading) { + setIsPlaying(false); + } + }, [loading]); + return (
; + initialTopic?: string; }; -export const VideoPanel: React.FC = ({ messages }) => { +export const VideoPanel: React.FC = ({ + messages, + initialTopic, +}) => { const { startTime, currentTime } = useScrubber(); const canvasRef = useRef(null); const containerRef = useRef(null); @@ -132,9 +142,19 @@ export const VideoPanel: React.FC = ({ messages }) => { const [offset, setOffset] = useState({ x: 0, y: 0 }); const [isDragging, setIsDragging] = useState(false); const lastMouse = useRef({ x: 0, y: 0 }); + const [selectedTopic, setSelectedTopic] = useState( + initialTopic || Object.keys(messages)[0] || "" + ); + + useEffect(() => { + if (initialTopic && messages[initialTopic]) { + setSelectedTopic(initialTopic); + } + }, [initialTopic]); useEffect(() => { - const msg = messages.find( + if (!selectedTopic || !messages[selectedTopic]) return; + const msg = messages[selectedTopic].find( (m) => m.timestamp >= startTime + currentTime ); if (!msg || !msg.data) return; @@ -268,6 +288,30 @@ export const VideoPanel: React.FC = ({ messages }) => { onMouseUp={handleMouseUp} onMouseLeave={handleMouseUp} > +
+ +