diff --git a/README.md b/README.md index 9f6d805a..3e6a3df6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ - vis4d + vis4d @@ -12,7 +12,7 @@ ## Quickstart -You can checkout our [documentation](https://docs.vis.xyz/4d/index.html). +You can checkout our [documentation](https://vis4d.readthedocs.io). You can use the [template](https://github.com/SysCV/vis4d-template) here to start your own project with Vis4D. @@ -24,7 +24,7 @@ Installation is as easy as python3 -m pip install vis4d ``` -[For more detailed information, check out our installation guide](docs/source/user_guide/install.rst) +[For more detailed information, check out our installation guide](https://vis4d.readthedocs.io/en/latest/user_guide/install.html) ## Basic CLI usage @@ -33,9 +33,6 @@ python3 -m pip install vis4d ```bash # vis4d.engine vis4d fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 - -# vis4d.pl -vis4d-pl fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 ``` - To test a model @@ -43,9 +40,6 @@ vis4d-pl fit --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 ```bash # vis4d.engine vis4d test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 - -# vis4d.pl -vis4d-pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 ``` ## DDP @@ -55,21 +49,13 @@ vis4d-pl test --config vis4d/zoo/faster_rcnn/faster_rcnn_coco.py --gpus 1 - Local machine / SLURM interactivate job (`job-name=bash`) ```bash -# vis4d.engine -./scripts/dist_train.sh - -# vis4d.pl -vis4d-pl fit --config --gpus +vis4d fit --config --gpus ``` - SLURM ```bash -# vis4d.engine -srun vis4d fit --config --gpus --slurm True - -# vis4d.pl -srun vis4d-pl fit --config --gpus +srun vis4d fit --config --gpus ``` ### Testing @@ -77,21 +63,13 @@ srun vis4d-pl fit --config --gpus - Local machine / SLURM interactivate job (`job-name=bash`) ```bash -# vis4d.engine -./scripts/dist_test.sh - -# vis4d.pl -vis4d-pl test --config --gpus +vis4d test --config --gpus ``` - SLURM ```bash -# vis4d.engine -srun vis4d test --config --gpus --slurm True - -# vis4d.pl -srun vis4d-pl test --config --gpus +srun vis4d test --config --gpus ``` ## Acknowledgement diff --git a/docs/source/user_guide/visualization.ipynb b/docs/source/user_guide/visualization.ipynb index 6c1c285c..0bfe6576 100644 --- a/docs/source/user_guide/visualization.ipynb +++ b/docs/source/user_guide/visualization.ipynb @@ -270,7 +270,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -286,7 +286,7 @@ ], "source": [ "img = draw_masks(images[1], masks[1], class_ids = class_ids[1])\n", - "img = draw_bboxes(img, boxes[0], class_ids = classes[0], scores = scores[0], track_ids = tracks[0], class_id_mapping = {1 : \"Human\", 51: \"Pan\"})\n", + "img = draw_bboxes(img, boxes[0], class_ids = classes[0], scores = scores[0], track_ids = tracks[0], class_id_mapping = {1 : \"Human\", 51: \"Pan\"}).as_numpy_image()\n", "imshow(img)" ] }, diff --git a/pyproject.toml b/pyproject.toml index 4aa3652c..6cc6bd5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -136,11 +136,10 @@ dependencies = {file = ["requirements/install.txt"]} include = ["vis4d*"] [project.urls] -"Documentation" = "https://docs.vis.xyz/4d" +"Documentation" = "https://vis4d.readthedocs.io" "Source" = "https://github.com/syscv/vis4d" -"Tracker" = "https://github.com/syscv/" +"Tracker" = "https://github.com/syscv/vis4d/issues" [project.scripts] vis4d = "vis4d.engine.run:entrypoint" -vis4d-pl = "vis4d.pl.run:entrypoint" vis4d-zoo = "vis4d.zoo.run:entrypoint" diff --git a/scripts/eval_nusc/README.md b/scripts/eval_nusc/README.md new file mode 100644 index 00000000..f4969522 --- /dev/null +++ b/scripts/eval_nusc/README.md @@ -0,0 +1,27 @@ +# nuScenes 3D Detection and Tracking Evaluation + +This folder contains the code and python environment to run nuScenes 3D detection and tracking evaluation locally. + +### Installation +- Python: 3.6 + +```bash +pip install -r nusc.txt +``` + +### Run +```bash +# Detection +python run.py \ +--input $FOLDER_OF_PREDICTION \ +--version $VERSION \ +--dataroot $NUSC_DATA_ROOT \ +--mode detection + +# Tracking +python run.py \ +--input $FOLDER_OF_PREDICTION \ +--version $VERSION \ +--dataroot $NUSC_DATA_ROOT \ +--mode tracking +``` \ No newline at end of file diff --git a/scripts/eval_nusc/nusc.txt b/scripts/eval_nusc/nusc.txt new file mode 100644 index 00000000..28c59498 --- /dev/null +++ b/scripts/eval_nusc/nusc.txt @@ -0,0 +1,84 @@ +argon2-cffi==21.3.0 +argon2-cffi-bindings==21.2.0 +async-generator==1.10 +attrs==22.2.0 +backcall==0.2.0 +bleach==4.1.0 +cachetools==4.2.4 +certifi==2021.5.30 +cffi==1.15.1 +comm==0.1.4 +cycler==0.11.0 +dataclasses==0.8 +decorator==5.1.1 +defusedxml==0.7.1 +descartes==1.1.0 +entrypoints==0.4 +fire==0.5.0 +importlib-metadata==4.8.3 +importlib-resources==5.4.0 +ipykernel==5.5.6 +ipython==7.16.3 +ipython-genutils==0.2.0 +ipywidgets==7.8.1 +jedi==0.17.2 +Jinja2==3.0.3 +joblib==1.1.1 +jsonschema==3.2.0 +jupyter==1.0.0 +jupyter-client==7.1.2 +jupyter-console==6.4.3 +jupyter-core==4.9.2 +jupyterlab-pygments==0.1.2 +jupyterlab-widgets==1.1.7 +kiwisolver==1.3.1 +MarkupSafe==2.0.1 +matplotlib==3.3.4 +mistune==0.8.4 +motmetrics==1.1.3 +nbclient==0.5.9 +nbconvert==6.0.7 +nbformat==5.1.3 +nest-asyncio==1.5.8 +notebook==6.4.10 +numpy==1.19.5 +nuscenes-devkit==1.1.10 +opencv-python==4.5.4.58 +packaging==21.3 +pandas==1.1.5 +pandocfilters==1.5.0 +parso==0.7.1 +pexpect==4.9.0 +pickleshare==0.7.5 +Pillow==8.4.0 +prometheus-client==0.17.1 +prompt-toolkit==3.0.36 +ptyprocess==0.7.0 +pycocotools==2.0.7 +pycparser==2.21 +Pygments==2.14.0 +pyparsing==3.1.1 +pyquaternion==0.9.9 +pyrsistent==0.18.0 +python-dateutil==2.8.2 +pytz==2023.3.post1 +pyzmq==25.1.2 +qtconsole==5.2.2 +QtPy==2.0.1 +scikit-learn==0.24.2 +scipy==1.5.4 +Send2Trash==1.8.2 +Shapely==1.8.5 +six==1.16.0 +termcolor==1.1.0 +terminado==0.12.1 +testpath==0.6.0 +threadpoolctl==3.1.0 +tornado==6.1 +tqdm==4.64.1 +traitlets==4.3.3 +typing_extensions==4.1.1 +wcwidth==0.2.13 +webencodings==0.5.1 +widgetsnbextension==3.6.6 +zipp==3.6.0 diff --git a/scripts/eval_nusc/run.py b/scripts/eval_nusc/run.py new file mode 100755 index 00000000..e80d9f35 --- /dev/null +++ b/scripts/eval_nusc/run.py @@ -0,0 +1,130 @@ +"""nuScenes evaluation pipeline for Vis4D.""" + +import argparse +import os +import json + +from nuscenes import NuScenes +from nuscenes.eval.detection.evaluate import NuScenesEval +from nuscenes.eval.detection.config import config_factory + +from nuscenes.eval.tracking.evaluate import TrackingEval as track_eval +from nuscenes.eval.tracking.utils import print_final_metrics +from nuscenes.eval.tracking.data_classes import TrackingConfig, TrackingMetrics +from nuscenes.eval.common.config import config_factory as track_configs + + +def eval_detection( + version: str, + dataroot: str, + output_dir: str, + result_path: str, + eval_set: str, +) -> None: + """Evaluate detection results.""" + nusc = NuScenes(version=version, dataroot=dataroot, verbose=True) + + nusc_eval = NuScenesEval( + nusc, + config=config_factory("detection_cvpr_2019"), + result_path=result_path, + eval_set=eval_set, + output_dir=output_dir, + verbose=True, + ) + _ = nusc_eval.main(render_curves=False) + + +def eval_tracking( + version: str, output_dir: str, result_path: str, root: str, eval_set: str +) -> None: + """Evaluate tracking results.""" + nusc_eval = track_eval( + config=track_configs("tracking_nips_2019"), + result_path=result_path, + eval_set=eval_set, + output_dir=output_dir, + verbose=True, + nusc_version=version, + nusc_dataroot=root, + ) + _ = nusc_eval.main() + + +def evaluate( + version: str, + dataroot: str, + mode: str, + output_dir: str, + result_path: str, + root: str, +) -> None: + """nuScenes evaluation.""" + if "mini" in version: + eval_set = "mini_val" + else: + eval_set = "val" + + if mode == "tracking": + eval_tracking(version, output_dir, result_path, root, eval_set) + else: + eval_detection(version, dataroot, output_dir, result_path, eval_set) + + +def print_metric_summary(metric_summary_path: str) -> None: + """Print metric summary.""" + with open(metric_summary_path, "r") as f: + metrics = json.load(f) + + cfg = TrackingConfig.deserialize(metrics["cfg"]) + tm = TrackingMetrics(cfg=cfg) + tm.add_runtime(metrics["eval_time"]) + tm.label_metrics = metrics["label_metrics"] + + print_final_metrics(metrics) + + +if __name__ == "__main__": + """Main.""" + parser = argparse.ArgumentParser(description="NuScenes eval for Vis4D.") + parser.add_argument( + "--input", + "-i", + help=( + "Folder path to the nuScenes format detection / tracking results." + ), + ) + parser.add_argument( + "--version", + "-v", + choices=["trainval", "mini"], + help="NuScenes dataset version to convert.", + ) + parser.add_argument( + "--dataroot", + "-d", + help="NuScenes dataset root.", + default="./data/nuscenes", + ) + parser.add_argument( + "-m", + "--mode", + default="tracking", + choices=["tracking", "detection"], + help="Conversion mode: detection or tracking.", + ) + args = parser.parse_args() + + if args.mode == "detection": + metric = "detect_3d" + else: + metric = "track_3d" + + evaluate( + f"v1.0-{args.version}", + args.dataroot, + args.mode, + args.input, + os.path.join(args.input, f"{metric}_predictions.json"), + args.dataroot, + ) diff --git a/tests/vis4d-test-data b/tests/vis4d-test-data index f21d3e31..8e92136c 160000 --- a/tests/vis4d-test-data +++ b/tests/vis4d-test-data @@ -1 +1 @@ -Subproject commit f21d3e3163f7f2d4010ceab8f0202023f9ce5881 +Subproject commit 8e92136c584c169438351da12193f65088fbc0c9 diff --git a/vis4d/engine/flag.py b/vis4d/engine/flag.py index fa4fe376..184b9fa7 100644 --- a/vis4d/engine/flag.py +++ b/vis4d/engine/flag.py @@ -4,12 +4,6 @@ from .parser import DEFINE_config_file -# TODO: Currently this does not allow to load multpile config files. -# Would be nice to extend functionality to chain multiple config files using -# e.g. --config=model_1.py --config=loader_args.py -# or --config=my_config.py --config.train_dl=different_dl.py - -# TODO: Support resume from folder and load config directly from it. _CONFIG = DEFINE_config_file("config", method_name="get_config") _GPUS = flags.DEFINE_integer("gpus", default=0, help="Number of GPUs") _CKPT = flags.DEFINE_string("ckpt", default=None, help="Checkpoint path") @@ -17,10 +11,6 @@ _SHOW_CONFIG = flags.DEFINE_bool( "print-config", default=False, help="If set, prints the configuration." ) -_SWEEP = DEFINE_config_file("sweep", method_name="get_sweep") -_SLURM = flags.DEFINE_bool( - "slurm", default=False, help="If set, setup slurm running jobs." -) _VIS = flags.DEFINE_bool( "vis", default=False, @@ -34,7 +24,5 @@ "_CKPT", "_RESUME", "_SHOW_CONFIG", - "_SWEEP", - "_SLURM", "_VIS", ] diff --git a/vis4d/engine/run.py b/vis4d/engine/run.py index 7c8205ba..de8e731f 100644 --- a/vis4d/engine/run.py +++ b/vis4d/engine/run.py @@ -104,7 +104,7 @@ def main(argv: ArgsType) -> None: if not vis and isinstance(callback, VisualizerCallback): rank_zero_info( - f"{callback.visualizer} is not used." + f"{callback.visualizer} is not used. " "Please set --vis=True to use it." ) continue diff --git a/vis4d/op/track3d/cc_3dt.py b/vis4d/op/track3d/cc_3dt.py index b7caa87c..837b2b76 100644 --- a/vis4d/op/track3d/cc_3dt.py +++ b/vis4d/op/track3d/cc_3dt.py @@ -68,6 +68,7 @@ def __init__( nms_class_iou_thr: float = 0.7, nms_conf_thr: float = 0.5, with_cats: bool = True, + with_velocities: bool = False, bbox_affinity_weight: float = 0.5, ) -> None: """Creates an instance of the class. @@ -83,10 +84,12 @@ def __init__( another detection. nms_class_iou_thr (float): Maximum IoU of a high score detection with another of a different class. + nms_conf_thr (float): Confidence threshold for NMS. with_cats (bool): If to consider category information for tracking (i.e. all detections within a track must have consistent category labels). - nms_conf_thr (float): Confidence threshold for NMS. + with_velocities (bool): If to use predicted velocities for + matching. bbox_affinity_weight (float): Weight of bbox affinity in the overall affinity score. """ @@ -98,6 +101,7 @@ def __init__( self.nms_class_iou_thr = nms_class_iou_thr self.nms_conf_thr = nms_conf_thr self.with_cats = with_cats + self.with_velocities = with_velocities self.bbox_affinity_weight = bbox_affinity_weight self.feat_affinity_weight = 1 - bbox_affinity_weight @@ -110,7 +114,10 @@ def _filter_detections( scores_3d: Tensor, class_ids: Tensor, embeddings: Tensor, - ) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: + velocities: Tensor | None = None, + ) -> tuple[ + Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor | None, Tensor + ]: """Remove overlapping objects across classes via nms. Args: @@ -121,6 +128,7 @@ def _filter_detections( scores_3d (Tensor): [N,] Tensor of 3D confidence scores. class_ids (Tensor): [N,] Tensor of class ids. embeddings (Tensor): [N, C] tensor of appearance embeddings. + velocities (Tensor | None): [N, 3] Tensor of velocities. Returns: tuple[Tensor]: filtered detections, scores, class_ids, @@ -142,6 +150,10 @@ def _filter_detections( detections_3d[inds], scores_3d[inds], ) + + if velocities is not None: + velocities = velocities[inds] + valids = embeddings.new_ones((len(detections),), dtype=torch.bool) ious = bbox_iou(detections, detections) @@ -158,12 +170,17 @@ def _filter_detections( if (ious[i, :i] > thr).any(): valids[i] = False + detections = detections[valids] scores = scores[valids] detections_3d = detections_3d[valids] scores_3d = scores_3d[valids] class_ids = class_ids[valids] embeddings = embeddings[valids] + + if velocities is not None: + velocities = velocities[valids] + return ( detections, scores, @@ -171,12 +188,14 @@ def _filter_detections( scores_3d, class_ids, embeddings, + velocities, inds[valids], ) - @staticmethod def depth_ordering( + self, obsv_boxes_3d: Tensor, + obsv_velocities: Tensor | None, memory_boxes_3d_predict: Tensor, memory_boxes_3d: Tensor, memory_velocities: Tensor, @@ -197,11 +216,11 @@ def depth_ordering( # Moving distance should be aligned motion_weight_list = [] - obsv_velocities = ( + moving_dist = ( obsv_boxes_3d[:, :3, None] - memory_boxes_3d[:, :3, None].transpose(2, 0) ).transpose(1, 2) - for v in obsv_velocities: + for v in moving_dist: motion_weight_list.append( F.pairwise_distance( # pylint: disable=not-callable v, memory_velocities[:, :3] @@ -210,22 +229,41 @@ def depth_ordering( motion_weight = torch.cat(motion_weight_list, dim=0) motion_weight = torch.exp(-torch.div(motion_weight, 5.0)) - # Moving direction should be aligned - # Set to 0.5 when two vector not within +-90 degree - cos_sim_list = [] - obsv_direct = ( - obsv_boxes_3d[:, :2, None] - - memory_boxes_3d[:, :2, None].transpose(2, 0) - ).transpose(1, 2) - for d in obsv_direct: - cos_sim_list.append( - F.cosine_similarity( # pylint: disable=not-callable - d, memory_velocities[:, :2] - ).unsqueeze(0) + # Velocity scores + if self.with_velocities: + assert ( + obsv_velocities is not None + ), "Please provide velocities if with_velocities=True!" + + velsim_weight_list = [] + obsvvv_velocities = obsv_velocities.unsqueeze(1).expand_as( + moving_dist ) - cos_sim = torch.cat(cos_sim_list, dim=0) - cos_sim = torch.add(cos_sim, 1.0) - cos_sim = torch.div(cos_sim, 2.0) + for v in obsvvv_velocities: + velsim_weight_list.append( + F.pairwise_distance( # pylint: disable=not-callable + v, memory_velocities[:, -3:] + ).unsqueeze(0) + ) + velsim_weight = torch.cat(velsim_weight_list, dim=0) + cos_sim = torch.exp(-velsim_weight / 5.0) + else: + # Moving direction should be aligned + # Set to 0.5 when two vector not within +-90 degree + cos_sim_list = [] + obsv_direct = ( + obsv_boxes_3d[:, :2, None] + - memory_boxes_3d[:, :2, None].transpose(2, 0) + ).transpose(1, 2) + for d in obsv_direct: + cos_sim_list.append( + F.cosine_similarity( # pylint: disable=not-callable + d, memory_velocities[:, :2] + ).unsqueeze(0) + ) + cos_sim = torch.cat(cos_sim_list, dim=0) + cos_sim = torch.add(cos_sim, 1.0) + cos_sim = torch.div(cos_sim, 2.0) scores_depth = ( cos_sim * centroid_weight + (1.0 - cos_sim) * motion_weight @@ -242,6 +280,7 @@ def __call__( detection_scores_3d: Tensor, detection_class_ids: Tensor, detection_embeddings: Tensor, + obs_velocities: Tensor | None = None, memory_boxes_3d: Tensor | None = None, memory_track_ids: Tensor | None = None, memory_class_ids: Tensor | None = None, @@ -260,6 +299,7 @@ def __call__( detection_scores_3d (Tensor): [N,] confidence scores in 3D. detection_class_ids (Tensor): [N,] class indices. detection_embeddings (Tensor): [N, C] appearance embeddings. + obs_velocities (Tensor | None): [N, 3] velocities of detections. memory_boxes_3d (Tensor): [M, 7] boxes in memory. memory_track_ids (Tensor): [M,] track ids in memory. memory_class_ids (Tensor): [M,] class indices in memory. @@ -280,6 +320,7 @@ def __call__( detection_scores_3d, detection_class_ids, detection_embeddings, + obs_velocities, permute_inds, ) = self._filter_detections( detections, @@ -289,6 +330,7 @@ def __call__( detection_scores_3d, detection_class_ids, detection_embeddings, + obs_velocities, ) if with_depth_confidence: @@ -324,6 +366,7 @@ def __call__( # Depth Ordering scores_depth = self.depth_ordering( detections_3d, + obs_velocities, memory_boxes_3d_predict, memory_boxes_3d, memory_velocities, diff --git a/vis4d/state/track3d/cc_3dt.py b/vis4d/state/track3d/cc_3dt.py index e2509ddd..3d992869 100644 --- a/vis4d/state/track3d/cc_3dt.py +++ b/vis4d/state/track3d/cc_3dt.py @@ -60,6 +60,7 @@ def __init__( num_frames: int = 5, fps: int = 2, update_3d_score: bool = True, + use_velocities: bool = False, add_backdrops: bool = True, ) -> None: """Creates an instance of the class.""" @@ -88,6 +89,7 @@ def __init__( self.fps = fps self.update_3d_score = update_3d_score self.add_backdrops = add_backdrops + self.use_velocities = use_velocities def reset(self) -> None: """Empty the memory.""" @@ -289,6 +291,7 @@ def __call__( memory_boxes_3d_predict = None memory_velocities = None + obs_velocities = boxes_3d[:, 9:] obs_boxes_3d = torch.cat( [boxes_3d[:, :6], boxes_3d[:, 8].unsqueeze(1)], dim=1 ) @@ -301,6 +304,7 @@ def __call__( scores_3d, class_ids, embeddings, + obs_velocities, memory_boxes_3d, memory_track_ids, memory_class_ids, @@ -512,6 +516,12 @@ def update_track( + velocity ) / (self.tracklets[track_id]["acc_frame"] + 1) + # Use predicted velocity if available + if self.use_velocities: + self.tracklets[track_id]["velocity"][4:] = self.tracklets[ + track_id + ]["box_3d"][9:12] + self.tracklets[track_id]["last_frame"] = frame_id self.tracklets[track_id]["acc_frame"] += 1 diff --git a/vis4d/vis/image/functional.py b/vis4d/vis/image/functional.py index b9355c6f..a9de324a 100644 --- a/vis4d/vis/image/functional.py +++ b/vis4d/vis/image/functional.py @@ -30,7 +30,6 @@ def imshow( image: ArrayLike, image_mode: str = "RGB", image_viewer: ImageViewerBackend = MatplotlibImageViewer(), - file_path: str | None = None, ) -> None: """Shows a single image. @@ -39,14 +38,10 @@ def imshow( image_mode (str, optional): Image Mode. Defaults to "RGB". image_viewer (ImageViewerBackend, optional): The Image viewer backend to use. Defaults to MatplotlibImageViewer(). - file_path (str): The path to save the image to. Defaults to None. """ image = preprocess_image(image, image_mode) image_viewer.show_images([image]) - if file_path is not None: - image_viewer.save_images([image], [file_path]) - def draw_masks( image: ArrayLike, @@ -93,7 +88,7 @@ def draw_bboxes( image_mode: str = "RGB", box_width: int = 1, canvas: CanvasBackend = PillowCanvasBackend(), -) -> NDArrayUI8: +) -> CanvasBackend: """Draws the predicted bounding boxes into the given image. Args: @@ -131,7 +126,7 @@ def draw_bboxes( for corners, label, color in zip(*box_data): canvas.draw_box(corners, color, box_width) canvas.draw_text((corners[0], corners[1]), label) - return canvas.as_numpy_image() + return canvas def imshow_bboxes( @@ -167,7 +162,7 @@ class id to class name file_path (str): The path to save the image to. Defaults to None. """ image = preprocess_image(image, mode=image_mode) - img = draw_bboxes( + canvas = draw_bboxes( image, boxes, scores, @@ -178,7 +173,10 @@ class id to class name image_mode, box_width, ) - imshow(img, image_mode, image_viewer, file_path) + imshow(canvas.as_numpy_image(), image_mode, image_viewer) + + if file_path is not None: + canvas.save_to_disk(file_path) def draw_bbox3d( @@ -195,11 +193,11 @@ def draw_bbox3d( canvas: CanvasBackend = PillowCanvasBackend(), width: int = 4, camera_near_clip: float = 0.15, -) -> NDArrayUI8: +) -> CanvasBackend: """Draw 3D box onto image.""" image = preprocess_image(image, image_mode) image_hw = (image.shape[0], image.shape[1]) - boxes3d_data = preprocess_boxes3d( + _, corners, labels, colors, _ = preprocess_boxes3d( image_hw, boxes3d, intrinsics, @@ -212,13 +210,15 @@ def draw_bbox3d( ) canvas.create_canvas(image) - for _, corners, label, color, _ in zip(*boxes3d_data): - canvas.draw_box_3d(corners, color, intrinsics, width, camera_near_clip) + for corner, label, color in zip(corners, labels, colors): + canvas.draw_box_3d(corner, color, intrinsics, width, camera_near_clip) - selected_corner = project_point(corners[0], intrinsics) - canvas.draw_text((selected_corner[0], selected_corner[1]), label) + selected_corner = project_point(corner[0], intrinsics) + canvas.draw_text( + (selected_corner[0], selected_corner[1]), label, color=color + ) - return canvas.as_numpy_image() + return canvas def imshow_bboxes3d( @@ -237,7 +237,7 @@ def imshow_bboxes3d( ) -> None: """Show image with bounding boxes.""" image = preprocess_image(image, mode=image_mode) - img = draw_bbox3d( + canvas = draw_bbox3d( image, boxes3d, intrinsics, @@ -249,7 +249,10 @@ def imshow_bboxes3d( n_colors=n_colors, image_mode=image_mode, ) - imshow(img, image_mode, image_viewer, file_path) + imshow(canvas.as_numpy_image(), image_mode, image_viewer) + + if file_path is not None: + canvas.save_to_disk(file_path) def imshow_masks( @@ -259,8 +262,6 @@ def imshow_masks( n_colors: int = 50, image_mode: str = "RGB", canvas: CanvasBackend = PillowCanvasBackend(), - image_viewer: ImageViewerBackend = MatplotlibImageViewer(), - file_path: str | None = None, ) -> None: """Shows semantic masks overlayed over the given image. @@ -275,15 +276,10 @@ def imshow_masks( image_mode (str, optional): Image Mode.. Defaults to "RGB". canvas (CanvasBackend, optional): Canvas backend to use. Defaults to PillowCanvasBackend(). - image_viewer (ImageViewerBackend, optional): The Image viewer backend - to use. Defaults to MatplotlibImageViewer(). - file_path (str): The path to save the image to. Defaults to None. """ imshow( draw_masks(image, masks, class_ids, n_colors, image_mode, canvas), image_mode, - image_viewer, - file_path, ) @@ -352,7 +348,6 @@ def imshow_track_matches( ref_track_ids: list[ArrayLikeInt], image_mode: str = "RGB", image_viewer: ImageViewerBackend = MatplotlibImageViewer(), - file_path: str | None = None, ) -> None: """Visualize paired bounding boxes successively for batched frame pairs. @@ -369,7 +364,6 @@ def imshow_track_matches( image_mode (str, optional): Color mode if the image. Defaults to "RGB". image_viewer (ImageViewerBackend, optional): The Image viewer backend to use. Defaults to MatplotlibImageViewer(). - file_path (str): The path to save the image to. Defaults to None. """ key_imgs_np = tuple( array_to_numpy(img, n_dims=3, dtype=np.float32) for img in key_imgs @@ -410,22 +404,23 @@ def imshow_track_matches( key_box[key_i], image_mode=image_mode, image_viewer=image_viewer, - file_path=file_path, ) imshow_bboxes( ref_image, ref_box[ref_i], image_mode=image_mode, image_viewer=image_viewer, - file_path=file_path, ) else: # stack imgs horizontal - k_img = draw_bboxes( + k_canvas = draw_bboxes( key_image, key_box[batch_i], image_mode=image_mode ) - r_img = draw_bboxes( + r_canvas = draw_bboxes( ref_image, ref_box[batch_i], image_mode=image_mode ) - stacked_img = np.vstack([k_img, r_img]) - imshow(stacked_img, image_mode, image_viewer, file_path) + k_np_img = k_canvas.as_numpy_image() + r_np_img = r_canvas.as_numpy_image() + stacked_img = np.vstack([k_np_img, r_np_img]) + + imshow(stacked_img, image_mode, image_viewer) diff --git a/vis4d/vis/image/util.py b/vis4d/vis/image/util.py index d32f223a..a66dc65b 100644 --- a/vis4d/vis/image/util.py +++ b/vis4d/vis/image/util.py @@ -186,7 +186,7 @@ def preprocess_boxes3d( list[list[tuple[float, float, float]]], list[str], list[tuple[int, int, int]], - list[int], + list[int | None], ]: """Preprocesses bounding boxes. @@ -233,7 +233,7 @@ def preprocess_boxes3d( corners_proc: list[list[tuple[float, float, float]]] = [] colors_proc: list[tuple[int, int, int]] = [] labels_proc: list[str] = [] - track_ids_proc: list[int] = [] + track_ids_proc: list[int | None] = [] if len(mask) == 1: if not mask[0]: @@ -281,8 +281,7 @@ def preprocess_boxes3d( category = None labels_proc.append(_get_box_label(category, score, track_id)) - if track_id is not None: - track_ids_proc.append(track_id) + track_ids_proc.append(track_id) return centers_proc, corners_proc, labels_proc, colors_proc, track_ids_proc diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py index cfb13fcd..b73805f9 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_1x_bdd100k.py @@ -127,7 +127,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py index b9e12b39..07f13eea 100644 --- a/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/faster_rcnn/faster_rcnn_r50_3x_bdd100k.py @@ -128,7 +128,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py index a9e50b76..fc0c70bb 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_1x_bdd100k.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py index 25e93e1c..c0752de0 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_3x_bdd100k.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py index c4fcec93..d3d0d33b 100644 --- a/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py +++ b/vis4d/zoo/bdd100k/mask_rcnn/mask_rcnn_r50_5x_bdd100k.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py index c4c3701f..5bd49455 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r101_80k_bdd100k.py @@ -171,7 +171,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py index 6301745b..06b57c46 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_40k_bdd100k.py @@ -161,7 +161,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py index a3682d24..94623b06 100644 --- a/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py +++ b/vis4d/zoo/bdd100k/semantic_fpn/semantic_fpn_r50_80k_bdd100k.py @@ -161,7 +161,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/bevformer/bevformer_base.py b/vis4d/zoo/bevformer/bevformer_base.py index 5c386e5b..9a77c808 100644 --- a/vis4d/zoo/bevformer/bevformer_base.py +++ b/vis4d/zoo/bevformer/bevformer_base.py @@ -137,7 +137,7 @@ def get_config() -> ExperimentConfig: velocity_thres=0.2, ), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL ), diff --git a/vis4d/zoo/bevformer/bevformer_tiny.py b/vis4d/zoo/bevformer/bevformer_tiny.py index db82f2a5..55353090 100644 --- a/vis4d/zoo/bevformer/bevformer_tiny.py +++ b/vis4d/zoo/bevformer/bevformer_tiny.py @@ -175,7 +175,7 @@ def get_config() -> ExperimentConfig: velocity_thres=0.2, ), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL ), diff --git a/vis4d/zoo/bevformer/bevformer_vis.py b/vis4d/zoo/bevformer/bevformer_vis.py index 97b411da..1c60e877 100644 --- a/vis4d/zoo/bevformer/bevformer_vis.py +++ b/vis4d/zoo/bevformer/bevformer_vis.py @@ -50,7 +50,7 @@ def get_config() -> ExperimentConfig: vis_freq=1, plot_trajectory=False, ), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( MultiSensorCallbackConnector, key_mapping=CONN_NUSC_BBOX_3D_VIS, diff --git a/vis4d/zoo/cc_3dt/README.md b/vis4d/zoo/cc_3dt/README.md index 29d13638..c762ca22 100644 --- a/vis4d/zoo/cc_3dt/README.md +++ b/vis4d/zoo/cc_3dt/README.md @@ -34,43 +34,46 @@ We provide the converted [BEVFormer detection results](https://dl.cv.ethz.ch/vis ### Train the Tracking Model ```bash # R50 -python -m vis4d.pl fit --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py --gpus 8 +vis4d fit --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py --gpus 8 # R101 -python -m vis4d.pl fit --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py --gpus 8 +vis4d fit --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py --gpus 8 ``` ### Train VeloLSTM motion model Generate the pure detection results on training set first. ```bash -python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py --ckpt ${checkpoint_path} --gpus ${num_gpus} +vis4d test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py --ckpt ${checkpoint_path} --gpus ${num_gpus} ``` Then train the VeloLSTM motion model by updating the pure detection results path in the [config](./velo_lstm_frcnn_r101_fpn_100e_nusc.py#L74). ```bash -python -m vis4d.pl fit --config vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py --gpus 4 +vis4d fit --config vis4d/zoo/cc_3dt/velo_lstm_frcnn_r101_fpn_100e_nusc.py --gpus 4 ``` ### Inference Run with KF3D motion model. ```bash # R50 -python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py --ckpt ${tracking_model_checkpoint_path} --gpus ${num_gpus} +vis4d test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r50_fpn_kf3d_12e_nusc.py --ckpt ${tracking_model_checkpoint_path} --gpus ${num_gpus} # R101 -python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py --ckpt ${tracking_model_checkpoint_path} --gpus ${num_gpus} +vis4d test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py --ckpt ${tracking_model_checkpoint_path} --gpus ${num_gpus} ``` Run with VeloLSTM motion model. ```bash -python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py --ckpt ${tracking_model_checkpoint_path} --config.velo_lstm_ckpt ${velo_lstm_cehckpoint_path} --gpus ${num_gpus} +vis4d test --config vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_velo_lstm_24e_nusc.py --ckpt ${tracking_model_checkpoint_path} --config.velo_lstm_ckpt ${velo_lstm_cehckpoint_path} --gpus ${num_gpus} ``` Run with VeloLSTM motion model with BEVFormer detection results. ```bash -python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py --ckpt ${tracking_model_checkpoint_path} --config.velo_lstm_ckpt ${velo_lstm_cehckpoint_path} --config.pure_detection ${bevformer_pure_detection_path} --gpus ${num_gpus} +vis4d test --config vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lstm_nusc.py --ckpt ${tracking_model_checkpoint_path} --config.velo_lstm_ckpt ${velo_lstm_cehckpoint_path} --config.pure_detection ${bevformer_pure_detection_path} --gpus ${num_gpus} ``` +## CR-3DT +We also update the [config](./cc_3dt_pp_kf3d.py) for running CC-3DT++ used in [CR-3DT](https://github.com/ETH-PBL/cc-3dt-pp/tree/main). Feel free to try! + ## Citation ``` @inproceedings{cc3dt, @@ -79,4 +82,11 @@ python -m vis4d.pl test --config vis4d/zoo/cc_3dt/cc_3dt_bevformer_base_velo_lst booktitle={6th Annual Conference on Robot Learning}, year={2022} } + +@article{baumann2024cr3dt, + title={CR3DT: Camera-RADAR Fusion for 3D Detection and Tracking}, + author={Baumann, Nicolas and Baumgartner, Michael and Ghignone, Edoardo and K{\"u}hne, Jonas and Fischer, Tobias and Yang, Yung-Hsu and Pollefeys, Marc and Magno, Michele}, + journal={arXiv preprint arXiv:2403.15313}, + year={2024} +} ``` diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py index 5d2c928b..1fba9627 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc.py @@ -168,7 +168,7 @@ def get_config() -> ExperimentConfig: split=test_split, ), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL ), @@ -180,7 +180,7 @@ def get_config() -> ExperimentConfig: EvaluatorCallback, evaluator=class_config(NuScenesTrack3DEvaluator), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_TRACK3D_EVAL ), diff --git a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py index 3a9afd1b..9eacb7da 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_frcnn_r101_fpn_pure_det_nusc.py @@ -74,7 +74,7 @@ def get_config() -> ExperimentConfig: save_only=True, ), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( MultiSensorCallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL, diff --git a/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py index 192b09b2..d650410e 100644 --- a/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py +++ b/vis4d/zoo/cc_3dt/cc_3dt_nusc_test.py @@ -82,7 +82,7 @@ def get_config() -> ExperimentConfig: save_only=True, ), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL ), @@ -94,7 +94,7 @@ def get_config() -> ExperimentConfig: EvaluatorCallback, evaluator=class_config(NuScenesTrack3DEvaluator), save_predictions=True, - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_NUSC_TRACK3D_EVAL ), diff --git a/vis4d/zoo/cc_3dt/cc_3dt_pp_kf3d.py b/vis4d/zoo/cc_3dt/cc_3dt_pp_kf3d.py new file mode 100644 index 00000000..e5a99663 --- /dev/null +++ b/vis4d/zoo/cc_3dt/cc_3dt_pp_kf3d.py @@ -0,0 +1,175 @@ +# pylint: disable=duplicate-code +"""CC-3DT++ on nuScenes.""" +from __future__ import annotations + +from vis4d.config import class_config +from vis4d.config.typing import DataConfig, ExperimentConfig +from vis4d.data.const import CommonKeys as K +from vis4d.data.datasets.nuscenes import NuScenes +from vis4d.data.datasets.nuscenes_detection import NuScenesDetection +from vis4d.data.io.hdf5 import HDF5Backend +from vis4d.engine.callbacks import EvaluatorCallback +from vis4d.engine.connectors import ( + CallbackConnector, + MultiSensorDataConnector, + data_key, +) +from vis4d.eval.nuscenes import ( + NuScenesDet3DEvaluator, + NuScenesTrack3DEvaluator, +) +from vis4d.model.track3d.cc_3dt import CC3DT +from vis4d.op.base import ResNet +from vis4d.op.track3d.cc_3dt import CC3DTrackAssociation +from vis4d.state.track3d.cc_3dt import CC3DTrackGraph +from vis4d.zoo.base import get_default_callbacks_cfg +from vis4d.zoo.cc_3dt.cc_3dt_frcnn_r101_fpn_kf3d_24e_nusc import ( + get_config as get_kf3d_cfg, +) +from vis4d.zoo.cc_3dt.data import ( + CONN_NUSC_DET3D_EVAL, + CONN_NUSC_TRACK3D_EVAL, + get_test_dataloader, +) + +CONN_NUSC_BBOX_3D_TEST = { + "images_list": data_key(K.images, sensors=NuScenes.CAMERAS), + "images_hw": data_key(K.original_hw, sensors=NuScenes.CAMERAS), + "intrinsics_list": data_key(K.intrinsics, sensors=NuScenes.CAMERAS), + "extrinsics_list": data_key(K.extrinsics, sensors=NuScenes.CAMERAS), + "frame_ids": K.frame_ids, + "pred_boxes3d": data_key("pred_boxes3d", sensors=["LIDAR_TOP"]), + "pred_boxes3d_classes": data_key( + "pred_boxes3d_classes", sensors=["LIDAR_TOP"] + ), + "pred_boxes3d_scores": data_key( + "pred_boxes3d_scores", sensors=["LIDAR_TOP"] + ), + "pred_boxes3d_velocities": data_key( + "pred_boxes3d_velocities", sensors=["LIDAR_TOP"] + ), +} + + +def get_config() -> ExperimentConfig: + """Returns the config dict for CC-3DT on nuScenes. + + Returns: + ExperimentConfig: The configuration + """ + ###################################################### + ## General Config ## + ###################################################### + config = get_kf3d_cfg().ref_mode() + + config.experiment_name = "cc_3dt_pp_kf3d_nusc" + + ###################################################### + ## Datasets with augmentations ## + ###################################################### + config.pure_detection = "" + + data_root = "data/nuscenes" + version = "v1.0-trainval" + test_split = "val" + + data = DataConfig() + + data.train_dataloader = None + + test_dataset = class_config( + NuScenesDetection, + data_root=data_root, + version=version, + split=test_split, + keys_to_load=[K.images, K.original_images], + data_backend=class_config(HDF5Backend), + pure_detection=config.pure_detection, + cache_as_binary=True, + cached_file_path=f"{data_root}/val.pkl", + ) + + data.test_dataloader = get_test_dataloader( + test_dataset=test_dataset, samples_per_gpu=1, workers_per_gpu=1 + ) + + config.data = data + + ###################################################### + ## MODEL & LOSS ## + ###################################################### + basemodel = class_config( + ResNet, resnet_name="resnet101", pretrained=True, trainable_layers=3 + ) + + track_graph = class_config( + CC3DTrackGraph, + track=class_config( + CC3DTrackAssociation, + init_score_thr=0.2, + obj_score_thr=0.1, + match_score_thr=0.3, + nms_class_iou_thr=0.3, + bbox_affinity_weight=0.75, + with_velocities=True, + ), + update_3d_score=False, + use_velocities=True, + add_backdrops=False, + ) + + config.model = class_config( + CC3DT, + basemodel=basemodel, + track_graph=track_graph, + detection_range=[40, 40, 40, 50, 50, 50, 50, 50, 30, 30], + ) + + ###################################################### + ## DATA CONNECTOR ## + ###################################################### + config.test_data_connector = class_config( + MultiSensorDataConnector, key_mapping=CONN_NUSC_BBOX_3D_TEST + ) + + ###################################################### + ## CALLBACKS ## + ###################################################### + # Logger and Checkpoint + callbacks = get_default_callbacks_cfg(config.output_dir) + + # Evaluator + callbacks.append( + class_config( + EvaluatorCallback, + evaluator=class_config( + NuScenesDet3DEvaluator, + data_root=data_root, + version=version, + split=test_split, + ), + save_predictions=True, + output_dir=config.output_dir, + test_connector=class_config( + CallbackConnector, key_mapping=CONN_NUSC_DET3D_EVAL + ), + ) + ) + + callbacks.append( + class_config( + EvaluatorCallback, + evaluator=class_config( + NuScenesTrack3DEvaluator, metadata=("use_camera", "use_radar") + ), + save_predictions=True, + output_dir=config.output_dir, + test_connector=class_config( + CallbackConnector, key_mapping=CONN_NUSC_TRACK3D_EVAL + ), + ) + ) + + config.callbacks = callbacks + + return config.value_mode() diff --git a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py index 260bd8c0..58b3b46d 100644 --- a/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py +++ b/vis4d/zoo/mask_rcnn/mask_rcnn_coco.py @@ -155,7 +155,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=remap_pred_keys(CONN_BBOX_2D_VIS, "boxes"), diff --git a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py index 497186f7..8bef10fb 100644 --- a/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_frcnn_r50_fpn_augs_1x_bdd100k.py @@ -128,7 +128,7 @@ def get_config() -> ExperimentConfig: visualizer=class_config( BoundingBoxVisualizer, vis_freq=500, image_mode="BGR" ), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_TRACK_VIS ), diff --git a/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py index 160a62bb..6c0a527d 100644 --- a/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py +++ b/vis4d/zoo/qdtrack/qdtrack_yolox_x_25e_bdd100k.py @@ -116,7 +116,7 @@ def get_config() -> ExperimentConfig: visualizer=class_config( BoundingBoxVisualizer, vis_freq=500, image_mode="BGR" ), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_TRACK_VIS ), diff --git a/vis4d/zoo/retinanet/retinanet_coco.py b/vis4d/zoo/retinanet/retinanet_coco.py index e0b8df8b..c7891fbc 100644 --- a/vis4d/zoo/retinanet/retinanet_coco.py +++ b/vis4d/zoo/retinanet/retinanet_coco.py @@ -169,7 +169,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS, diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py index 90df0163..f7ede48a 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_12e_shift.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py index afe37738..43597d95 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_36e_shift.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py index 76bcc820..b22c36ef 100644 --- a/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py +++ b/vis4d/zoo/shift/faster_rcnn/faster_rcnn_r50_6e_shift_all_domains.py @@ -133,7 +133,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(BoundingBoxVisualizer, vis_freq=100), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py index 53ce37b7..a48a6116 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_12e_shift.py @@ -134,7 +134,7 @@ def get_config() -> FieldConfigDict: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py index 2a3b3eb4..e32d43e4 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_36e_shift.py @@ -134,7 +134,7 @@ def get_config() -> FieldConfigDict: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py index d03e1c13..86b1f2b2 100644 --- a/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py +++ b/vis4d/zoo/shift/mask_rcnn/mask_rcnn_r50_6e_shift_all_domains.py @@ -134,7 +134,7 @@ def get_config() -> FieldConfigDict: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=25), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_INS_MASK_2D_VIS ), diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py index b41cede5..5bc10baf 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift.py @@ -163,7 +163,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py index 94ed983f..c1725965 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_160k_shift_all_domains.py @@ -165,7 +165,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py index 70bc8918..cb8bd7da 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift.py @@ -163,7 +163,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py index 789ab175..cac4fa90 100644 --- a/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py +++ b/vis4d/zoo/shift/semantic_fpn/semantic_fpn_r50_40k_shift_all_domains.py @@ -163,7 +163,7 @@ def get_config() -> ExperimentConfig: class_config( VisualizerCallback, visualizer=class_config(SegMaskVisualizer, vis_freq=20), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_SEG_VIS ), diff --git a/vis4d/zoo/yolox/yolox_s_300e_coco.py b/vis4d/zoo/yolox/yolox_s_300e_coco.py index 8b654980..d0d9b137 100644 --- a/vis4d/zoo/yolox/yolox_s_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_s_300e_coco.py @@ -113,7 +113,7 @@ def get_config() -> ExperimentConfig: visualizer=class_config( BoundingBoxVisualizer, vis_freq=100, image_mode="BGR" ), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ), diff --git a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py index 7d63b07c..df3d36d8 100644 --- a/vis4d/zoo/yolox/yolox_tiny_300e_coco.py +++ b/vis4d/zoo/yolox/yolox_tiny_300e_coco.py @@ -116,7 +116,7 @@ def get_config() -> ExperimentConfig: visualizer=class_config( BoundingBoxVisualizer, vis_freq=100, image_mode="BGR" ), - save_prefix=config.output_dir, + output_dir=config.output_dir, test_connector=class_config( CallbackConnector, key_mapping=CONN_BBOX_2D_VIS ),