Skip to content

Fixes, added isaac5 configurations #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
saihv wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Fixes, added isaac5 configurations #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6630fb9
Fixes, added isaac5 configurations
saihv Jan 4, 2026
2dfff57
Fix configs, update models in notebooks
saihv Jan 5, 2026
3894ab6
Add drone forest fire example, arm planning example
saihv Jan 6, 2026
4bb41be
convert arm planning example to notebook
saihv Jan 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions configs/airgen/drone_forestfire.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
{
"sim": {
"sim_type": "airgen",
"scene_name": "forest_fire",
"kwargs": {
"geo": false
},
"settings": {
"SimMode": "Multirotor",
"Vehicles": {
"Drone": {
"VehicleType": "SimpleFlight",
"VehicleModel": "Astro"
}
},
"OriginGeopoint": {
"Latitude": 47.62094998919241,
"Longitude": -122.35554810901883,
"Altitude": 100
}
}
},
"grid": {
"entities": {
"robot": [
{
"name": "airgen-drone",
"kwargs": {}
}
],
"model": []
}
}
}
35 changes: 35 additions & 0 deletions configs/airgen/legged_basic.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{
"sim": {
"sim_type": "airgen",
"scene_name": "blocks",
"kwargs": {
"geo": false
},
"settings": {
"SettingsVersion": 2.0,
"SimMode": "CVMover",
"Vehicles": {
"Drone": {
"VehicleType": "CVMover",
"VehicleModel": "UnitreeGo2"
}
},
"OriginGeopoint": {
"Latitude": 47.62094998919241,
"Longitude": -122.35554810901883,
"Altitude": 100
}
}
},
"grid": {
"entities": {
"robot": [
{
"name": "airgen-quad",
"kwargs": {}
}
],
"model": []
}
}
}
2 changes: 1 addition & 1 deletion configs/airgen/multi_drone.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"sim": {
"sim_type": "airgen",
"scene_name": "electric_central",
"scene_name": "nordic_forest",
"kwargs": {
"geo": false
},
Expand Down
23 changes: 23 additions & 0 deletions configs/isaac5/franka_japi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_tabletop",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_franka_japi"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-arm",
"kwargs": {}
}
],
"model": []
}
}
}
23 changes: 23 additions & 0 deletions configs/isaac5/franka_keyboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_tabletop",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_franka_kb"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-arm",
"kwargs": {}
}
],
"model": []
}
}
}
23 changes: 23 additions & 0 deletions configs/isaac5/g1_warehouse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_fullwarehouse",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_g1_rlagent"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-humanoid",
"kwargs": {}
}
],
"model": []
}
}
}
23 changes: 23 additions & 0 deletions configs/isaac5/go2_warehouse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_warehouse",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_go2_rlagent"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-locomotion",
"kwargs": {}
}
],
"model": []
}
}
}
23 changes: 23 additions & 0 deletions configs/isaac5/rizon_kb.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_tabletop",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_rizon4_kb"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-arm",
"kwargs": {}
}
],
"model": []
}
}
}
23 changes: 23 additions & 0 deletions configs/isaac5/ur5_japi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
{
"sim": {
"sim_type": "isaac5",
"scene_name": "isaac_tabletop",
"kwargs": {
"geo": false
},
"settings": {
"robot_name": "isaac_ur5e_japi"
}
},
"grid": {
"entities": {
"robot": [
{
"name": "isaac-arm",
"kwargs": {}
}
],
"model": []
}
}
}
6 changes: 3 additions & 3 deletions notebooks/ai_models_car.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@
"outputs": [],
"source": [
"# set the weather, wind, and timeofday parameters and import AI models\n",
"from grid.model.perception.detection.gdino import GroundingDINO\n",
"from grid.model.perception.detection.owlv2 import OWLv2\n",
"from grid.model.perception.segmentation.gsam2 import GSAM2\n",
"\n",
"detection = GroundingDINO()\n",
"detection = OWLv2()\n",
"segmentation = GSAM2()\n",
"\n",
"client = airgen_car_0.client\n",
Expand Down Expand Up @@ -59,7 +59,7 @@
"metadata": {},
"outputs": [],
"source": [
"def runAIModels(client: airgen.VehicleClient) -> dict:\n",
"def runAIModels(client: airgen.CVMoverClient) -> dict:\n",
" img, _ = client.getImages(\"front_center\",[airgen.ImageType.Scene])[0]\n",
" boxes, scores, labels = detection.run(img,\"car.\")\n",
" mask = segmentation.run(img,\"road\")\n",
Expand Down
6 changes: 3 additions & 3 deletions notebooks/ai_models_drone.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,16 +31,16 @@
"source": [
"import airgen\n",
"import numpy as np\n",
"from grid.model.perception.detection.gdino import GroundingDINO\n",
"from grid.model.perception.detection.owlv2 import OWLv2\n",
"\n",
"groundingdino = GroundingDINO()\n",
"owlv2 = OWLv2()\n",
"\n",
"# Define a detect function that captures an RGB image from the front camera and uses an object detector\n",
"# Please seperate the object name with a dot (.)\n",
"\n",
"def detect(drone, object_name=\"fire.\"):\n",
" rgb_image, pose = drone.getImages(\"front_center\", [airgen.ImageType.Scene])[0]\n",
" boxes, scores, labels = groundingdino.run(rgb_image, object_name)\n",
" boxes, scores, labels = owlv2.run(rgb_image, object_name)\n",
" return boxes, scores, labels\n",
"\n",
"# Let's search for the fire by rotating 360 degrees and trying to detect fire at each step\n",
Expand Down
4 changes: 2 additions & 2 deletions notebooks/car_data_capture.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@
"source": [
"import h5py, os\n",
"from typing import List\n",
"from airgen import ImageType, VehicleClient\n",
"from airgen import ImageType, CarClient\n",
"from airgen.utils.visualize import rr_log_airgen_image\n",
"from grid import GRID_USER_SESSION_BLOB_DIR\n",
"\n",
"\n",
"def capture_data(car: VehicleClient, sensor_types: List[str], capture_types: List[str], hdf5_file_path: str, idx=0):\n",
"def capture_data(car: CarClient, sensor_types: List[str], capture_types: List[str], hdf5_file_path: str, idx=0):\n",
" with h5py.File(hdf5_file_path, 'a') as hdf5_file:\n",
" frame_group = hdf5_file.create_group(f\"frame_{idx}\")\n",
" sensor_group = frame_group.create_group(\"sensor_data\")\n",
Expand Down
4 changes: 2 additions & 2 deletions notebooks/data_capture.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@
"source": [
"# define sensor and image collection function\n",
"from typing import List, Literal\n",
"from airgen import ImageType, VehicleClient\n",
"from airgen import ImageType, MultirotorClient\n",
"from airgen.utils.visualize import rr_log_airgen_image\n",
"\n",
"def data_capture(drone: VehicleClient, sensor_types: List[str], capture_types: Literal['rgb', 'depth', 'segmentation']):\n",
"def data_capture(drone: MultirotorClient, sensor_types: List[str], capture_types: Literal['rgb', 'depth', 'segmentation']):\n",
" frame_data = {\"sensor_data\": {}, \"image_data\": {}}\n",
" capture_name_map = {\"rgb\": ImageType.Scene, \"depth\": ImageType.DepthPerspective, \"segmentation\": ImageType.Segmentation}\n",
"\n",
Expand Down
4 changes: 2 additions & 2 deletions notebooks/data_generation.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@
"source": [
"# define sensor and image collection function\n",
"from typing import List, Literal\n",
"from airgen import ImageType, VehicleClient\n",
"from airgen import ImageType, CVMoverClient\n",
"from airgen.utils.visualize import rr_log_airgen_image\n",
"import cv2\n",
"import numpy as np\n",
"from grid import GRID_USER_SESSION_BLOB_DIR\n",
"\n",
"def data_capture(car: VehicleClient, sensor_types: List[str], capture_types: Literal['rgb', 'depth', 'segmentation', 'visdepth'], idx: int = 0):\n",
"def data_capture(car: CVMoverClient, sensor_types: List[str], capture_types: Literal['rgb', 'depth', 'segmentation', 'visdepth'], idx: int = 0):\n",
" frame_data = {\"sensor_data\": {}, \"image_data\": {}}\n",
" capture_name_map = {\"rgb\": ImageType.Scene, \"depth\": ImageType.DepthPerspective, \"segmentation\": ImageType.Segmentation, \"visdepth\": ImageType.DepthVis}\n",
"\n",
Expand Down
2 changes: 1 addition & 1 deletion notebooks/depth_point_cloud.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
"import rerun as rr\n",
"from airgen.utils.vision import depth2pointcloud\n",
"\n",
"client = airgen.VehicleClient()\n",
"client = airgen.CVMoverClient()\n",
"\n",
"images = client.getImages(\"front_center\", [airgen.ImageType.Scene, airgen.ImageType.DepthPerspective])\n",
"capture = {\n",
Expand Down
4 changes: 2 additions & 2 deletions notebooks/drone_data_capture.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,12 +31,12 @@
"source": [
"import h5py, os\n",
"from typing import List\n",
"from airgen import ImageType, VehicleClient\n",
"from airgen import ImageType, MultirotorClient\n",
"from airgen.utils.visualize import rr_log_airgen_image\n",
"from grid import GRID_USER_SESSION_BLOB_DIR\n",
"\n",
"\n",
"def capture_data(drone: VehicleClient, sensor_types: List[str], capture_types: List[str], hdf5_file_path: str, idx=0):\n",
"def capture_data(drone: MultirotorClient, sensor_types: List[str], capture_types: List[str], hdf5_file_path: str, idx=0):\n",
" with h5py.File(hdf5_file_path, 'a') as hdf5_file:\n",
" frame_group = hdf5_file.create_group(f\"frame_{idx}\")\n",
" sensor_group = frame_group.create_group(\"sensor_data\")\n",
Expand Down
11 changes: 4 additions & 7 deletions notebooks/grid-isaac/humanoid_ai_models.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,6 @@
"metadata": {},
"outputs": [],
"source": [
"# Start the humanoid simulation\n",
"humanoidisaacsim_0.run()\n",
"\n",
"# Import necessary utilities for velocity control, logging, and timing\n",
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist bot Jan 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

The call to humanoidisaacsim_0.run() was removed, but it appears to be necessary for the simulation to start. Subsequent cells that interact with the simulation (e.g., humanoidisaacsim_0.locomotion.moveByVelocity(...)) will fail without it. Other Isaac Sim notebooks in the repository still include an explicit .run() call, suggesting it's still necessary. Please restore this call to ensure the notebook functions correctly. The cell's code should be:

# Start the humanoid simulation
humanoidisaacsim_0.run()

# Import necessary utilities for velocity control, logging, and timing
from grid.utils.types import Velocity
from grid.utils import log
import time

"from grid.utils.types import Velocity\n",
"from grid.utils import log\n",
Expand Down Expand Up @@ -74,13 +71,13 @@
"# Import and initialize perception models for segmentation, depth estimation, and vision-language processing.\n",
"\n",
"from grid.model.perception.segmentation.oneformer import OneFormer \n",
"from grid.model.perception.depth.depth_anything_v2 import DepthAnything_V2 \n",
"from grid.model.perception.vlm.llava import LLaVA\n",
"from grid.model.perception.depth.zoedepth import ZoeDepth \n",
"from grid.model.perception.vlm.moondream import MoonDream\n",
"\n",
"# Create model instances\n",
"seg_model = OneFormer()\n",
"depth_model = DepthAnything_V2()\n",
"vlm_model = LLaVA()\n",
"depth_model = ZoeDepth()\n",
"vlm_model = MoonDream()\n",
"\n",
"# Retrieve the latest RGB image from the simulation for processing\n",
"rgb = humanoidisaacsim_0.getImage()\n",
Expand Down
4 changes: 1 addition & 3 deletions notebooks/grid-isaac/humanoid_detect_navigate.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@
"source": [
"from grid.model.perception.detection.owlv2 import OWLv2\n",
"det_model = OWLv2()\n",
"obj = \"forklift\"\n",
"\n",
"humanoidisaacsim_0.run()"
"obj = \"forklift\""
Copy link
Copy Markdown

@gemini-code-assist gemini-code-assist bot Jan 4, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

The call to humanoidisaacsim_0.run() has been removed from this cell. The next cell in the notebook calls humanoidisaacsim_0.getImage(...), which will likely fail if the simulation has not been started. Please restore the humanoidisaacsim_0.run() call to ensure the notebook executes correctly.

    "obj = \"forklift\"\n",
    "\n",
    "humanoidisaacsim_0.run()"

]
},
{
Expand Down
6 changes: 2 additions & 4 deletions notebooks/grid-isaac/locomotion_safexploration.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,7 @@
"\n",
"import numpy as np\n",
"import time\n",
"import cv2\n",
"\n",
"locomotionisaacsim_0.run()"
"import cv2"
]
},
{
Expand Down Expand Up @@ -163,7 +161,7 @@
" \"\"\"\n",
" Continuously captures images from the agent, processes them for depth, and navigates safely.\n",
" \"\"\"\n",
" depth = MIDAS()\n",
" depth = MIDAS(run_local=True)\n",
"\n",
" while True:\n",
" rgb_image = agent.getImage()\n",
Expand Down
Loading