diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json b/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json index e9ef8df23..5414b9c5c 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json +++ b/GEMstack/onboard/visualization/sr_viz/threeD/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", + "@foxglove/rosbag": "^0.4.1", "@mui/icons-material": "^7.0.2", "@mui/material": "^7.0.2", "@react-three/drei": "^10.0.6", @@ -19,6 +20,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-icons": "^5.5.0", + "react-resizable-panels": "^3.0.1", "three": "^0.175.0", "urdf-loader": "^0.12.4" }, @@ -503,6 +505,56 @@ "node": "^18.18.0 || ^20.9.0 || >=21.1.0" } }, + "node_modules/@foxglove/message-definition": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@foxglove/message-definition/-/message-definition-0.2.0.tgz", + "integrity": "sha512-IQHIGCvBZR8GIua9nEpS+hsMF3gm1bfbrrnjG0rgtcFBWiNuKbzx4vIP8OIwDC+8wtwcFdfJhf4Vp5TPFiUUcQ==" + }, + "node_modules/@foxglove/rosbag": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/@foxglove/rosbag/-/rosbag-0.4.1.tgz", + "integrity": "sha512-Tbt1SUz+xnKOX+bvhbHiVjNMpTM28hl3zAku8fJ7+8qTZjJPrW95p3W4Aiz6YVWO+NMhCns5wQqOv8rvBpCotg==", + "dependencies": { + "@foxglove/rosmsg": "^4.0.0", + "@foxglove/rosmsg-serialization": "^2.0.0", + "@foxglove/rostime": "^1.1.2", + "heap": "^0.2.7" + } + }, + "node_modules/@foxglove/rosmsg": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg/-/rosmsg-4.2.2.tgz", + "integrity": "sha512-bAY7+b/3AJnR6pIkUWbDdFCRGwCc/WaI9V/51nkj0iNP16ZMJ6s5xPDwKYMpaNIlFSyJG1QE69AmRPRYzP6oEQ==", + "dependencies": { + "@foxglove/message-definition": "^0.2.0", + "md5-typescript": "^1.0.5" + }, + "bin": { + "gendeps2": "bin/gendeps2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rosmsg-serialization": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@foxglove/rosmsg-serialization/-/rosmsg-serialization-2.0.3.tgz", + "integrity": "sha512-IN/VREnP+3eGNBQJUXQxKfX1ZG/rc0/kkRdZMhqkuFhCywGHwOGNo8KNVEuEO6EfYygREArPoRB1V9L0MUjZ2Q==", + "dependencies": { + "@foxglove/message-definition": "^0.2.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@foxglove/rostime": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@foxglove/rostime/-/rostime-1.1.2.tgz", + "integrity": "sha512-vWuTJCuGv0xvgwOlrZ1y2MevmNMVxWcUU/HwlmYXi/jUq/kRaACStU18uyuZ3LzdNKaffkti0rTcXWESaQjwQw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -4108,6 +4160,11 @@ "node": ">= 0.4" } }, + "node_modules/heap": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/heap/-/heap-0.2.7.tgz", + "integrity": "sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==" + }, "node_modules/hls.js": { "version": "1.6.2", "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.2.tgz", @@ -5026,6 +5083,11 @@ "node": ">= 0.4" } }, + "node_modules/md5-typescript": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/md5-typescript/-/md5-typescript-1.0.5.tgz", + "integrity": "sha512-ovAc4EtiNt2dY8JPhPr/wkC9h4U5k/nuClNVcG0Ga3V1rMlYpAY24ZaaymFXJlz+ccJ6UMPo3FSaVKe7czBsXw==" + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -5595,6 +5657,15 @@ "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==" }, + "node_modules/react-resizable-panels": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/react-resizable-panels/-/react-resizable-panels-3.0.1.tgz", + "integrity": "sha512-6ruCEyw0iqXRcXEktPQn1HL553DNhrdLisCyEdSpzhkmo9bPqZxskJZ+aGeFqJ1qPvIWxuAiag82kvLSb2JZTQ==", + "peerDependencies": { + "react": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc", + "react-dom": "^16.14.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + } + }, "node_modules/react-transition-group": { "version": "4.4.5", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/package.json b/GEMstack/onboard/visualization/sr_viz/threeD/package.json index f86ed25a2..ccf138646 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/package.json +++ b/GEMstack/onboard/visualization/sr_viz/threeD/package.json @@ -11,6 +11,7 @@ "dependencies": { "@emotion/react": "^11.14.0", "@emotion/styled": "^11.14.0", + "@foxglove/rosbag": "^0.4.1", "@mui/icons-material": "^7.0.2", "@mui/material": "^7.0.2", "@react-three/drei": "^10.0.6", @@ -20,6 +21,7 @@ "react": "^19.0.0", "react-dom": "^19.0.0", "react-icons": "^5.5.0", + "react-resizable-panels": "^3.0.1", "three": "^0.175.0", "urdf-loader": "^0.12.4" }, diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx index 180333927..3b0989248 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/page.tsx @@ -6,53 +6,80 @@ import CanvasWrapper from "@/components/CanvasWrapper"; import Scrubber from "@/components/Scrubber"; import { VehicleInfoPanel } from "@/components/VehicleInfoPanel"; import { usePlaybackTime } from "@/hooks/usePlaybackTime"; +import { useRouter } from "next/navigation"; +import Button from "@mui/material/Button"; +import PageviewIcon from "@mui/icons-material/Pageview"; export default function HomePage() { - const { - time, - reset, - restart, - play, - togglePlay, - speed, - setPlaybackSpeed, - moveToTime, - duration, - setDuration, - } = usePlaybackTime(); + const router = useRouter(); + const { + time, + reset, + restart, + play, + setPlay, + togglePlay, + speed, + setPlaybackSpeed, + moveToTime, + duration, + setDuration, + } = usePlaybackTime(); - const [searchParams, setSearchParams] = useState<{ - folder?: string; - file?: string; - }>({}); + const [searchParams, setSearchParams] = useState<{ + folder?: string; + file?: string; + }>({}); - useEffect(() => { - if (typeof window !== "undefined") { - const params = new URLSearchParams(window.location.search); - const folder = params.get("folder") || undefined; - const file = params.get("file") || undefined; - setSearchParams({ folder, file }); - } - }, []); + useEffect(() => { + if (typeof window !== "undefined") { + const params = new URLSearchParams(window.location.search); + const folder = params.get("folder") || undefined; + const file = params.get("file") || undefined; + setSearchParams({ folder, file }); + } + }, []); - return ( -
- - - - -
- ); + const handleRedirect = () => { + setPlay(false); + router.replace("/rosbagViewer"); + }; + + return ( +
+
+
+ +
+
+ + + + +
+ ); } diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/app/rosbagViewer/page.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/rosbagViewer/page.tsx new file mode 100644 index 000000000..71e151eee --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/app/rosbagViewer/page.tsx @@ -0,0 +1,36 @@ +"use client"; + +import RosbagViewer from "@/components/RosbagViewer"; +import { useRouter } from "next/navigation"; +import Button from "@mui/material/Button"; +import HomeIcon from "@mui/icons-material/Home"; + +export default function RosbagViewerPage() { + const router = useRouter(); + const handleRedirect = () => { + router.replace("/"); + }; + return ( +
+
+
+ +
+
+ +
+ ); +} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx new file mode 100644 index 000000000..98aad4737 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PanelManager.tsx @@ -0,0 +1,274 @@ +"use client"; + +import React, { useState } from "react"; +import { VideoPanel } from "./VideoPanel"; +import { PointCloudPanel } from "./PointCloudPanel"; +import { + Button, + Menu, + MenuItem, + ListItemIcon, + ListItemText, +} from "@mui/material"; +import MenuIcon from "@mui/icons-material/Menu"; +import { PanelGroup, Panel, PanelResizeHandle } from "react-resizable-panels"; +import HorizontalSplitIcon from "@mui/icons-material/HorizontalSplit"; +import VerticalSplitIcon from "@mui/icons-material/VerticalSplit"; +import ImageIcon from "@mui/icons-material/Image"; +import HubIcon from "@mui/icons-material/Hub"; +import CloseIcon from "@mui/icons-material/Close"; + +type PanelNode = + | { + id: number; + type: "video" | "pointcloud" | "text"; + } + | { + id: number; + split: "horizontal" | "vertical"; + children: PanelNode[]; + }; + +let panelIdCounter = 2; + +function createPanel(type: "video" | "pointcloud" | "text"): PanelNode { + return { id: panelIdCounter++, type }; +} + +export const PanelManager = ({ + messageMap, +}: { + messageMap: Record | any[]>; +}) => { + const [rootPanel, setRootPanel] = useState(createPanel("video")); + + const renderPanelNode = (node: PanelNode): JSX.Element => { + if ("split" in node) { + return ( + + + {node.children.map((child, idx) => ( + + {renderPanelNode(child)} + {idx < node.children.length - 1 && ( + + )} + + ))} + + + ); + } + + return ( + + { + const newSplit: PanelNode = { + id: panelIdCounter++, + split: direction, + children: [node, createPanel("video")], + }; + replacePanel( + rootPanel, + node.id, + newSplit, + setRootPanel + ); + }} + onClose={() => closePanel(rootPanel, node.id, setRootPanel)} + onChangeType={(newType) => + changePanelType( + rootPanel, + node.id, + newType, + setRootPanel + ) + } + rootPanel={rootPanel} + /> + {node.type === "video" && ( + + )} + {node.type === "pointcloud" && ( + + )} + + ); + }; + + return ( +
+ + {renderPanelNode(rootPanel)} + +
+ ); +}; + +function replacePanel( + root: PanelNode, + targetId: number, + replacement: PanelNode, + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode { + if (node.id === targetId) return replacement; + if ("split" in node) { + return { + ...node, + children: node.children.map(helper), + }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function closePanel( + root: PanelNode, + targetId: number, + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode | null { + if (node.id === targetId) return null; + if ("split" in node) { + const newChildren = node.children + .map(helper) + .filter((child) => child !== null) as PanelNode[]; + return { ...node, children: newChildren }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function changePanelType( + root: PanelNode, + targetId: number, + newType: "video" | "pointcloud" | "text", + setRoot: (r: PanelNode) => void +) { + function helper(node: PanelNode): PanelNode { + if (node.id === targetId) { + return { ...node, type: newType }; + } + if ("split" in node) { + return { + ...node, + children: node.children.map(helper), + }; + } + return node; + } + const newRoot = helper(root); + setRoot(newRoot); +} + +function getNumLeaves(node: PanelNode): number { + if ("split" in node) { + return node.children.reduce( + (sum, child) => sum + getNumLeaves(child), + 0 + ); + } + return 1; +} + +const PanelMenu = ({ + onSplit, + onClose, + onChangeType, + rootPanel, +}: { + onSplit: (dir: "horizontal" | "vertical") => void; + onClose: () => void; + onChangeType: (newType: "video" | "pointcloud") => void; + rootPanel: PanelNode; +}) => { + const [anchorEl, setAnchorEl] = useState(null); + const open = Boolean(anchorEl); + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => setAnchorEl(null); + + return ( +
+ + + { + onSplit("horizontal"); + handleClose(); + }} + > + + + + Split Horizontally + + { + onSplit("vertical"); + handleClose(); + }} + > + + + + Split Vertically + + { + onChangeType("video"); + handleClose(); + }} + > + + + + Switch to Video Panel + + { + onChangeType("pointcloud"); + handleClose(); + }} + > + + + + Switch to PointCloud Panel + + {getNumLeaves(rootPanel) > 1 && ( + { + onClose(); + handleClose(); + }} + > + + + + Close Panel + + )} + +
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx new file mode 100644 index 000000000..339720961 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/PointCloudPanel.tsx @@ -0,0 +1,256 @@ +"use client"; + +import React, { useEffect, useRef, useState } from "react"; +import { useScrubber } from "./ScrubberContext"; +import * as THREE from "three"; +import { OrbitControls } from "three/examples/jsm/controls/OrbitControls"; +import { Select, SelectChangeEvent, MenuItem } from "@mui/material"; + +function parsePointCloud2(msg: any): THREE.Points { + const { data, point_step, fields } = msg; + const positions: number[] = []; + const colors: number[] = []; + + const fieldMap = Object.fromEntries( + fields.map((f: any) => [f.name, f.offset]) + ); + const dv = new DataView(data.buffer, data.byteOffset, data.byteLength); + + for (let i = 0; i < data.length; i += point_step) { + const x = dv.getFloat32(i + fieldMap["x"], true); + const y = dv.getFloat32(i + fieldMap["y"], true); + const z = dv.getFloat32(i + fieldMap["z"], true); + + if (!Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(z)) + continue; + + positions.push(x, y, z); + + if ("rgba" in fieldMap) { + const rgba = dv.getUint32(i + fieldMap["rgba"], true); + const r = (rgba >> 24) & 0xff; + const g = (rgba >> 16) & 0xff; + const b = (rgba >> 8) & 0xff; + colors.push(r / 255, g / 255, b / 255); + } else if ("rgb" in fieldMap) { + const rgb = dv.getUint32(i + fieldMap["rgb"], true); + const r = (rgb >> 16) & 0xff; + const g = (rgb >> 8) & 0xff; + const b = rgb & 0xff; + colors.push(r / 255, g / 255, b / 255); + } else { + colors.push(1, 1, 1); + } + } + + const geometry = new THREE.BufferGeometry(); + geometry.setAttribute( + "position", + new THREE.Float32BufferAttribute(positions, 3) + ); + geometry.setAttribute("color", new THREE.Float32BufferAttribute(colors, 3)); + const material = new THREE.PointsMaterial({ + size: 0.05, + vertexColors: true, + }); + + return new THREE.Points(geometry, material); +} + +function buildTransformLookup(tfMessages: any[], time: number) { + const latestTransforms = new Map(); + + for (const msg of tfMessages) { + const transforms = msg.data.transforms || []; + for (const t of transforms) { + if (t.header.stamp.sec + t.header.stamp.nsec * 1e-9 < time) + continue; + const key = `${t.header.frame_id}->${t.child_frame_id}`; + if (!latestTransforms.has(key)) { + latestTransforms.set(key, t); + } + } + } + + return latestTransforms; +} + +function getTransformMatrix( + lookup: Map, + from: string, + to: string +): THREE.Matrix4 | null { + const key = `${from}->${to}`; + const tf = lookup.get(key); + if (!tf) return null; + + const { translation, rotation } = tf.transform; + const position = new THREE.Vector3( + translation.x, + translation.y, + translation.z + ); + const quaternion = new THREE.Quaternion( + rotation.x, + rotation.y, + rotation.z, + rotation.w + ); + + const matrix = new THREE.Matrix4(); + matrix.makeRotationFromQuaternion(quaternion); + matrix.setPosition(position); + + return matrix; +} + +export const PointCloudPanel = ({ + messages, + tfMessages, + initialTopic, +}: { + messages: Record; + tfMessages: any[]; + initialTopic?: string; +}) => { + const mountRef = useRef(null); + const { startTime, currentTime } = useScrubber(); + const rendererRef = useRef(null); + const cameraRef = useRef(); + const sceneRef = useRef(); + const controlsRef = useRef(); + const pointCloudRef = useRef(); + const [selectedTopic, setSelectedTopic] = useState( + initialTopic || Object.keys(messages)[0] || "" + ); + + useEffect(() => { + if (initialTopic && messages[initialTopic]) { + setSelectedTopic(initialTopic); + } + }, [initialTopic]); + + useEffect(() => { + if (!selectedTopic || !messages[selectedTopic]) return; + if (!mountRef.current) return; + + const scene = new THREE.Scene(); + const camera = new THREE.PerspectiveCamera( + 75, + mountRef.current.clientWidth / mountRef.current.clientHeight, + 0.1, + 1000 + ); + camera.position.z = 5; + + const renderer = new THREE.WebGLRenderer({ antialias: true }); + renderer.setSize( + mountRef.current.clientWidth, + mountRef.current.clientHeight + ); + mountRef.current.appendChild(renderer.domElement); + + const controls = new OrbitControls(camera, renderer.domElement); + controls.enableDamping = true; + + sceneRef.current = scene; + cameraRef.current = camera; + rendererRef.current = renderer; + controlsRef.current = controls; + + const animate = () => { + requestAnimationFrame(animate); + controls.update(); + renderer.render(scene, camera); + }; + animate(); + + return () => { + if (rendererRef.current) { + rendererRef.current.dispose(); + const canvas = rendererRef.current.domElement; + canvas.remove(); + } + }; + }, []); + + useEffect(() => { + const scene = sceneRef.current; + if (!scene) return; + + const msg = messages[selectedTopic].find( + (m) => m.timestamp >= startTime + currentTime + ); + if (!msg) return; + + if (pointCloudRef.current) { + scene.remove(pointCloudRef.current); + pointCloudRef.current.geometry.dispose(); + (pointCloudRef.current.material as THREE.Material).dispose(); + } + + const newCloud = parsePointCloud2(msg.data); + pointCloudRef.current = newCloud; + + const transforms = buildTransformLookup( + tfMessages, + currentTime + startTime + ); + const cloudFrame = msg.data.header.frame_id; + const targetFrame = "base_link"; + + const matrix = getTransformMatrix(transforms, cloudFrame, targetFrame); + + if (matrix) { + newCloud.applyMatrix4(matrix); + } + scene.add(newCloud); + }, [startTime, currentTime, messages, tfMessages]); + + useEffect(() => { + if (!mountRef.current || !rendererRef.current || !cameraRef.current) + return; + + const observer = new ResizeObserver(() => { + const width = mountRef.current!.clientWidth; + const height = mountRef.current!.clientHeight; + + rendererRef.current!.setSize(width, height); + cameraRef.current!.aspect = width / height; + cameraRef.current!.updateProjectionMatrix(); + }); + + observer.observe(mountRef.current); + + return () => observer.disconnect(); + }, []); + + return ( +
+
+ +
+
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx new file mode 100644 index 000000000..d2d9d2867 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/RosbagViewer.tsx @@ -0,0 +1,184 @@ +"use client"; + +import React, { useState } from "react"; +import { Bag } from "@foxglove/rosbag"; +import Button from "@mui/material/Button"; +import CloudUploadIcon from "@mui/icons-material/CloudUpload"; +import { styled } from "@mui/material/styles"; +import { ScrubberProvider } from "./ScrubberContext"; +import { Scrubber2 } from "@/components/Scrubber2"; +import { PanelManager } from "./PanelManager"; + +const VisuallyHiddenInput = styled("input")({ + clip: "rect(0 0 0 0)", + clipPath: "inset(50%)", + height: 1, + overflow: "hidden", + position: "absolute", + bottom: 0, + left: 0, + whiteSpace: "nowrap", + width: 1, +}); + +function fileToReader(file: File) { + return { + size: () => file.size, + read: async (offset: number, length: number) => { + const blob = file.slice(offset, offset + length); + const arrayBuffer = await blob.arrayBuffer(); + return new Uint8Array(arrayBuffer); + }, + }; +} + +export default function RosbagViewer() { + const [topics, setTopics] = useState([]); + const [types, setTypes] = useState([]); + const [loading, setLoading] = useState(false); + const [messageMap, setMessageMap] = useState<{ + video: Record; + pointcloud: Record; + tf: any[]; + }>({ + video: {}, + pointcloud: {}, + tf: [], + }); + + const [duration, setDuration] = useState(0); + const [startTime, setStartTime] = useState(0); + + const handleFileUpload = async ( + event: React.ChangeEvent + ) => { + const file = event.target.files?.[0]; + if (!file) return; + + setMessageMap({ + video: [], + pointcloud: [], + tf: [], + }); + setLoading(true); + const reader = fileToReader(file); + const bag = new Bag(reader); + await bag.open(); + // console.log("Bag opened:", bag); + const topicNames = Array.from(bag.connections.values()).map( + (conn) => conn.topic + ); + const topicTypes = Array.from(bag.connections.values()).map( + (conn) => conn.type + ); + setTopics(topicNames); + setTypes(topicTypes); + // console.log("Topics:", topicNames, topicTypes); + + const videoMessages: Record = {}; + const pointcloudMessages: Record = {}; + const tfMessages: any[] = []; + for await (const msg of bag.messageIterator()) { + const entry = { + timestamp: msg.timestamp.sec + msg.timestamp.nsec * 1e-9, + topic: msg.topic, + data: msg.message, + }; + const type = topicTypes[topicNames.indexOf(msg.topic)]; + if (type.includes("Image")) { + if (!videoMessages[msg.topic]) videoMessages[msg.topic] = []; + videoMessages[msg.topic].push(entry); + } + if (type.includes("PointCloud2")) { + if (!pointcloudMessages[msg.topic]) + pointcloudMessages[msg.topic] = []; + pointcloudMessages[msg.topic].push(entry); + } + if (type.includes("TFMessage")) tfMessages.push(entry); + } + setLoading(false); + // console.log( + // "Messages parsed:", + // videoMessages, + // pointcloudMessages, + // tfMessages + // ); + setMessageMap({ + video: videoMessages, + pointcloud: pointcloudMessages, + tf: tfMessages, + }); + const getDuration = ( + messagesByTopic: Record + ): number => { + const durations = Object.values(messagesByTopic) + .filter((msgs) => msgs.length > 1) + .map( + (msgs) => + msgs[msgs.length - 1].timestamp - msgs[0].timestamp + ); + return durations.length > 0 ? Math.max(...durations) : 0; + }; + + const getStart = (messagesByTopic: Record): number => { + const starts = Object.values(messagesByTopic) + .filter((msgs) => msgs.length > 0) + .map((msgs) => msgs[0].timestamp); + return starts.length > 0 ? Math.min(...starts) : 0; + }; + + setDuration( + Math.max( + getDuration(videoMessages), + getDuration(pointcloudMessages), + 0 + ) + ); + const videoStart = getStart(videoMessages); + const pointcloudStart = getStart(pointcloudMessages); + setStartTime( + videoStart > 0 && pointcloudStart > 0 + ? Math.min(videoStart, pointcloudStart) + : Math.max(videoStart, pointcloudStart) + ); + }; + + return ( +
+
+
+ +
+
+ + + + + +
+ ); +} diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx new file mode 100644 index 000000000..385af9ec1 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/Scrubber2.tsx @@ -0,0 +1,170 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { useScrubber } from "./ScrubberContext"; +import { IconButton, Slider, Menu, MenuItem } from "@mui/material"; +import PlayArrowIcon from "@mui/icons-material/PlayArrow"; +import PauseIcon from "@mui/icons-material/Pause"; +import SpeedIcon from "@mui/icons-material/Speed"; +import RefreshIcon from "@mui/icons-material/Refresh"; + +export const Scrubber2 = ({ + duration, + startTime, + loading, +}: { + duration: number; + startTime: number; + loading: boolean; +}) => { + const { + setStartTime, + currentTime, + setCurrentTime, + isPlaying, + setIsPlaying, + } = useScrubber(); + const frameRate = 30; + const [anchorEl, setAnchorEl] = useState(null); + const [selectedSpeed, setSelectedSpeed] = useState(1); + const speedOptions = [0.5, 1, 1.5, 2, 3]; + const open = Boolean(anchorEl); + const [isDragging, setIsDragging] = useState(false); + + const handleClick = (event: React.MouseEvent) => { + setAnchorEl(event.currentTarget); + }; + const handleClose = () => { + setAnchorEl(null); + }; + const handleMenuItemClick = (speed: number) => { + setSelectedSpeed(speed); + handleClose(); + }; + const handleSliderChange = (_: Event, newValue: number) => { + setCurrentTime(newValue); + }; + const handleSliderChangeCommitted = () => { + setIsDragging(false); + }; + const togglePlay = () => { + setIsPlaying((prev: boolean) => !prev); + }; + const restart = () => { + setCurrentTime(0); + }; + const formatDuration = (time: number) => { + const minutes = Math.floor(time / 60); + const seconds = Math.floor(time % 60); + return `${minutes < 10 ? "0" : ""}${minutes}:${ + seconds < 10 ? "0" : "" + }${seconds}`; + }; + const handleContextMenu = (event: React.MouseEvent) => { + event.preventDefault(); + }; + useEffect(() => { + if (!isPlaying) return; + const interval = setInterval(() => { + setCurrentTime((prev: number) => + Math.min(prev + (selectedSpeed * frameRate) / 1000, duration) + ); + }, 1000 / frameRate); + return () => clearInterval(interval); + }, [isPlaying, duration, selectedSpeed]); + + useEffect(() => { + setStartTime(startTime); + setCurrentTime(0); + }, [startTime]); + + useEffect(() => { + if (currentTime >= duration) { + setIsPlaying(false); + } + }, [currentTime, duration]); + + useEffect(() => { + if (loading) { + setIsPlaying(false); + } + }, [loading]); + + return ( +
+ + {isPlaying ? ( + + ) : ( + + )} + +
+
+ {currentTime < duration + ? formatDuration(currentTime) + : formatDuration(duration)} +
+ setIsDragging(true)} + onChangeCommitted={handleSliderChangeCommitted} + value={currentTime} + /> +
{formatDuration(duration)}
+
+
+ + + + + {speedOptions.map((speed) => ( + handleMenuItemClick(speed)} + > + {speed}x + + ))} + + + + +
+
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx new file mode 100644 index 000000000..180c79356 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/ScrubberContext.tsx @@ -0,0 +1,32 @@ +"use client"; + +import React, { createContext, useContext, useState } from "react"; + +const ScrubberContext = createContext(null); + +export const ScrubberProvider = ({ + children, +}: { + children: React.ReactNode; +}) => { + const [startTime, setStartTime] = useState(0); + const [currentTime, setCurrentTime] = useState(0); + const [isPlaying, setIsPlaying] = useState(false); + + return ( + + {children} + + ); +}; + +export const useScrubber = () => useContext(ScrubberContext); diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx new file mode 100644 index 000000000..69f39f3e4 --- /dev/null +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/components/VideoPanel.tsx @@ -0,0 +1,354 @@ +"use client"; + +import React, { useEffect, useRef, useState } from "react"; +import { useScrubber } from "./ScrubberContext"; +import { + Switch, + FormControlLabel, + Select, + SelectChangeEvent, + MenuItem, +} from "@mui/material"; + +export function decodeImage(msg: { + encoding: string; + data: Uint8Array; + height: number; + width: number; + step: number; +}): ImageData { + const { encoding, data, height, width, step } = msg; + const rgba = new Uint8ClampedArray(width * height * 4); + + if (encoding === "bgr8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const srcIdx = i * step + j * 3; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = data[srcIdx + 2]; + rgba[dstIdx + 1] = data[srcIdx + 1]; + rgba[dstIdx + 2] = data[srcIdx]; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "rgb8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const srcIdx = i * step + j * 3; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = data[srcIdx]; + rgba[dstIdx + 1] = data[srcIdx + 1]; + rgba[dstIdx + 2] = data[srcIdx + 2]; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "mono8") { + for (let i = 0; i < height; i++) { + for (let j = 0; j < width; j++) { + const gray = data[i * step + j]; + const dstIdx = (i * width + j) * 4; + rgba[dstIdx] = gray; + rgba[dstIdx + 1] = gray; + rgba[dstIdx + 2] = gray; + rgba[dstIdx + 3] = 255; + } + } + } else if (encoding === "bayer_grbg8") { + const get = (x: number, y: number): number => + x >= 0 && y >= 0 && x < width && y < height + ? data[y * width + x] + : 0; + + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + let r = 0, + g = 0, + b = 0; + + const val = get(x, y); + + if (y % 2 === 0) { + if (x % 2 === 0) { + g = val; + r = (get(x - 1, y) + get(x + 1, y)) / 2; + b = (get(x, y - 1) + get(x, y + 1)) / 2; + } else { + r = val; + g = + (get(x - 1, y) + + get(x + 1, y) + + get(x, y - 1) + + get(x, y + 1)) / + 4; + b = + (get(x - 1, y - 1) + + get(x + 1, y - 1) + + get(x - 1, y + 1) + + get(x + 1, y + 1)) / + 4; + } + } else { + if (x % 2 === 0) { + b = val; + g = + (get(x - 1, y) + + get(x + 1, y) + + get(x, y - 1) + + get(x, y + 1)) / + 4; + r = + (get(x - 1, y - 1) + + get(x + 1, y - 1) + + get(x - 1, y + 1) + + get(x + 1, y + 1)) / + 4; + } else { + g = val; + r = (get(x, y - 1) + get(x, y + 1)) / 2; + b = (get(x - 1, y) + get(x + 1, y)) / 2; + } + } + + const i = (y * width + x) * 4; + rgba[i] = r; + rgba[i + 1] = g; + rgba[i + 2] = b; + rgba[i + 3] = 255; + } + } + } else { + throw new Error(`Unsupported encoding: ${encoding}`); + } + + return new ImageData(rgba, width, height); +} + +type VideoPanelProps = { + messages: Record; + initialTopic?: string; +}; + +export const VideoPanel: React.FC = ({ + messages, + initialTopic, +}) => { + const { startTime, currentTime } = useScrubber(); + const canvasRef = useRef(null); + const containerRef = useRef(null); + const imageRef = useRef(null); + + const [enableInteraction, setEnableInteraction] = useState(false); + const [zoom, setZoom] = useState(1); + const [offset, setOffset] = useState({ x: 0, y: 0 }); + const [isDragging, setIsDragging] = useState(false); + const lastMouse = useRef({ x: 0, y: 0 }); + const [selectedTopic, setSelectedTopic] = useState( + initialTopic || Object.keys(messages)[0] || "" + ); + + useEffect(() => { + if (initialTopic && messages[initialTopic]) { + setSelectedTopic(initialTopic); + } + }, [initialTopic]); + + useEffect(() => { + if (!selectedTopic || !messages[selectedTopic]) return; + const msg = messages[selectedTopic].find( + (m) => m.timestamp >= startTime + currentTime + ); + if (!msg || !msg.data) return; + + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + + try { + const imageData = decodeImage(msg.data); + imageRef.current = imageData; + + if (!enableInteraction) { + if (canvas && ctx) { + canvas.width = imageData.width; + canvas.height = imageData.height; + ctx.putImageData(imageData, 0, 0); + } + } else { + drawImage(); + } + } catch (e) { + console.error("Failed to decode image:", e); + } + }, [messages, currentTime, enableInteraction]); + + const drawImage = () => { + requestAnimationFrame(() => { + const canvas = canvasRef.current; + const ctx = canvas?.getContext("2d"); + const imageData = imageRef.current; + if (!canvas || !ctx || !imageData) return; + + const { width: cw, height: ch } = canvas; + ctx.clearRect(0, 0, cw, ch); + + const imgAspect = imageData.width / imageData.height; + const canvasAspect = cw / ch; + + let drawW = imageData.width * zoom; + let drawH = imageData.height * zoom; + + if (imgAspect > canvasAspect) { + drawW = cw * zoom; + drawH = drawW / imgAspect; + } else { + drawH = ch * zoom; + drawW = drawH * imgAspect; + } + + const drawX = (cw - drawW) / 2 + offset.x; + const drawY = (ch - drawH) / 2 + offset.y; + + const offscreen = document.createElement("canvas"); + offscreen.width = imageData.width; + offscreen.height = imageData.height; + offscreen.getContext("2d")?.putImageData(imageData, 0, 0); + + ctx.drawImage(offscreen, drawX, drawY, drawW, drawH); + }); + }; + + useEffect(() => { + if (!enableInteraction) return; + + const container = containerRef.current; + if (!container) return; + + const handleWheel = (e: WheelEvent) => { + e.preventDefault(); + const zoomDelta = -e.deltaY * 0.001; + setZoom((z) => Math.min(Math.max(z + zoomDelta, 0.1), 10)); + }; + + container.addEventListener("wheel", handleWheel, { passive: false }); + return () => container.removeEventListener("wheel", handleWheel); + }, [enableInteraction]); + + useEffect(() => { + if (!enableInteraction) return; + drawImage(); + }, [zoom, offset, enableInteraction]); + + useEffect(() => { + const handleResize = () => { + const container = containerRef.current; + const canvas = canvasRef.current; + if (container && canvas) { + canvas.width = container.clientWidth; + canvas.height = container.clientHeight; + if (enableInteraction) drawImage(); + } + }; + + window.addEventListener("resize", handleResize); + handleResize(); + return () => window.removeEventListener("resize", handleResize); + }, [enableInteraction]); + + useEffect(() => { + if (!enableInteraction) { + setZoom(1); + setOffset({ x: 0, y: 0 }); + } + }, [enableInteraction]); + + const handleMouseDown = (e: React.MouseEvent) => { + if (!enableInteraction) return; + setIsDragging(true); + lastMouse.current = { x: e.clientX, y: e.clientY }; + }; + + const handleMouseMove = (e: React.MouseEvent) => { + if (!enableInteraction || !isDragging) return; + const dx = e.clientX - lastMouse.current.x; + const dy = e.clientY - lastMouse.current.y; + lastMouse.current = { x: e.clientX, y: e.clientY }; + setOffset((prev) => ({ x: prev.x + dx, y: prev.y + dy })); + }; + + const handleMouseUp = () => { + if (!enableInteraction) return; + setIsDragging(false); + }; + + return ( +
+
+ +
+
+ + setEnableInteraction(e.target.checked) + } + /> + } + sx={{ + backgroundColor: "white", + opacity: 0.25, + borderRadius: "9999px", + paddingRight: "10px", + "&:hover": { + opacity: 0.5, + }, + }} + /> +
+ + +
+ ); +}; diff --git a/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts b/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts index fda840914..95843bb0f 100644 --- a/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts +++ b/GEMstack/onboard/visualization/sr_viz/threeD/src/hooks/usePlaybackTime.ts @@ -57,5 +57,5 @@ export function usePlaybackTime() { return () => cancelAnimationFrame(rafId); }, [play, speed]); - return { time, reset, restart, play, togglePlay, speed, setPlaybackSpeed, moveToTime, duration, setDuration }; + return { time, reset, restart, play, setPlay, togglePlay, speed, setPlaybackSpeed, moveToTime, duration, setDuration }; } diff --git a/log_dashboard/requirements.txt b/log_dashboard/requirements.txt index dd4bd7d24..e6dac0d83 100644 --- a/log_dashboard/requirements.txt +++ b/log_dashboard/requirements.txt @@ -1,5 +1,4 @@ flask -flask-cors pyyaml cachelib numpy