From 2b0b2d3e5404f21867ad92c42e555e8923079568 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 15:09:48 +0100
Subject: [PATCH 1/8] add iou matchmaker algoirthm as a configurable option

---
 python-sdk/nuscenes/eval/common/utils.py      | 32 ++++++++++++++++++-
 .../nuscenes/eval/detection/data_classes.py   |  4 ++-
 .../eval/detection/tests/test_utils.py        | 17 ++++++++--
 python-sdk/nuscenes/eval/tracking/algo.py     | 11 +++++--
 4 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/python-sdk/nuscenes/eval/common/utils.py b/python-sdk/nuscenes/eval/common/utils.py
index 248ec7795..866d25a55 100644
--- a/python-sdk/nuscenes/eval/common/utils.py
+++ b/python-sdk/nuscenes/eval/common/utils.py
@@ -5,12 +5,42 @@
 
 import numpy as np
 from pyquaternion import Quaternion
-
+from shapely import affinity
+from shapely.geometry import Polygon
 from nuscenes.eval.common.data_classes import EvalBox
 from nuscenes.utils.data_classes import Box
 
 DetectionBox = Any  # Workaround as direct imports lead to cyclic dependencies.
 
+def create_polygon_from_box(bbox: EvalBox):
+    l = bbox.size[0]
+    w = bbox.size[1]
+    poly_veh = Polygon(((0.5*l,0.5*w),(-0.5*l,0.5*w),(-0.5*l,-0.5*w),(0.5*l,-0.5*w),(0.5*l,0.5*w)))
+    poly_rot = affinity.rotate(poly_veh,quaternion_yaw(Quaternion(bbox.rotation)),use_radians=True)
+    poly_glob = affinity.translate(poly_rot,bbox.translation[0],bbox.translation[1])
+    return poly_glob
+
+def intersection_over_union(gt_poly: Polygon, pred_poly: Polygon):
+    intersection = gt_poly.intersection(pred_poly).area
+    iou = intersection/(gt_poly.area + pred_poly.area - intersection)
+    return iou
+
+def iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
+    """
+    1 - IOU percentage between the boxes (xy only).
+    :param gt_box: GT annotation sample.
+    :param pred_box: Predicted sample.
+    :return: 1 - IOU.
+    """
+    # Do a cheaper first pass before calculating IOU i.e. check if the circles that enclose the two
+    # boxes overlap
+    gt_radius = np.linalg.norm(0.5*np.array([gt_box.size[0],gt_box.size[1]]))
+    pred_radius = np.linalg.norm(0.5*np.array([pred_box.size[0],pred_box.size[1]]))
+    if (center_distance(gt_box,pred_box) < pred_radius + gt_radius):
+        iou = intersection_over_union(create_polygon_from_box(gt_box),create_polygon_from_box(pred_box))
+    else:
+        iou = 0.0
+    return 1.0 - iou
 
 def center_distance(gt_box: EvalBox, pred_box: EvalBox) -> float:
     """
diff --git a/python-sdk/nuscenes/eval/detection/data_classes.py b/python-sdk/nuscenes/eval/detection/data_classes.py
index 8e0a7fcc8..b49e4756a 100644
--- a/python-sdk/nuscenes/eval/detection/data_classes.py
+++ b/python-sdk/nuscenes/eval/detection/data_classes.py
@@ -7,7 +7,7 @@
 import numpy as np
 
 from nuscenes.eval.common.data_classes import MetricData, EvalBox
-from nuscenes.eval.common.utils import center_distance
+from nuscenes.eval.common.utils import center_distance, iou_complement
 from nuscenes.eval.detection.constants import DETECTION_NAMES, ATTRIBUTE_NAMES, TP_METRICS
 
 
@@ -74,6 +74,8 @@ def dist_fcn_callable(self):
         """ Return the distance function corresponding to the dist_fcn string. """
         if self.dist_fcn == 'center_distance':
             return center_distance
+        elif self.dist_fcn == "iou_complement":
+            return iou_complement
         else:
             raise Exception('Error: Unknown distance function %s!' % self.dist_fcn)
 
diff --git a/python-sdk/nuscenes/eval/detection/tests/test_utils.py b/python-sdk/nuscenes/eval/detection/tests/test_utils.py
index 2f2d9a21f..69eefcc5f 100644
--- a/python-sdk/nuscenes/eval/detection/tests/test_utils.py
+++ b/python-sdk/nuscenes/eval/detection/tests/test_utils.py
@@ -7,8 +7,8 @@
 from numpy.testing import assert_array_almost_equal
 from pyquaternion import Quaternion
 
-from nuscenes.eval.common.utils import attr_acc, scale_iou, yaw_diff, angle_diff, center_distance, velocity_l2, \
-    cummean
+from nuscenes.eval.common.utils import attr_acc, scale_iou, yaw_diff, angle_diff, iou_complement, center_distance, \
+    velocity_l2, cummean
 from nuscenes.eval.detection.data_classes import DetectionBox
 
 
@@ -128,6 +128,19 @@ def rad(x):
         period = 360
         self.assertAlmostEqual(rad(180), abs(angle_diff(rad(a), rad(b), rad(period))))
 
+    def test_iou_complement_no_overlap(self):
+        # Two boxes specified, no overlap
+        sa = DetectionBox(translation=(1.0, 0.0, 1.0), size=(2,1,1))
+        sr = DetectionBox(translation=(3.5, 0.0, 1.0), size=(3,1,2))
+        self.assertAlmostEqual(iou_complement(sa, sr), 1.0)
+
+    def test_iou_complement_overlap(self):
+        # Two boxes specified, one rotated by 90 degrees in z axis, should attain 25% overlap
+        sa = DetectionBox(rotation=(0,0,0,0), translation=(1.0, 0.5, 2.0), size=(2,1,1))
+        sr = DetectionBox(rotation=(0.70710678118,0,0,0.70710678118),
+                          translation=(0.5, 1.5, 1), size=(3,1,2))
+        self.assertAlmostEqual(iou_complement(sa, sr), 0.75)
+
     def test_center_distance(self):
         """Test for center_distance()."""
 
diff --git a/python-sdk/nuscenes/eval/tracking/algo.py b/python-sdk/nuscenes/eval/tracking/algo.py
index 670872163..db9116515 100644
--- a/python-sdk/nuscenes/eval/tracking/algo.py
+++ b/python-sdk/nuscenes/eval/tracking/algo.py
@@ -23,6 +23,7 @@
 except ModuleNotFoundError:
     raise unittest.SkipTest('Skipping test as pandas was not found!')
 
+from nuscenes.eval.common.utils import iou_complement
 from nuscenes.eval.tracking.constants import MOT_METRIC_MAP, TRACKING_METRICS
 from nuscenes.eval.tracking.data_classes import TrackingBox, TrackingMetricData
 from nuscenes.eval.tracking.mot import MOTAccumulatorCustom
@@ -257,13 +258,19 @@ def accumulate_threshold(self, threshold: float = None) -> Tuple[pandas.DataFram
 
                 # Calculate distances.
                 # Note that the distance function is hard-coded to achieve significant speedups via vectorization.
-                assert self.dist_fcn.__name__ == 'center_distance'
                 if len(frame_gt) == 0 or len(frame_pred) == 0:
                     distances = np.ones((0, 0))
-                else:
+                elif self.dist_fcn.__name__ == 'center_distance':
                     gt_boxes = np.array([b.translation[:2] for b in frame_gt])
                     pred_boxes = np.array([b.translation[:2] for b in frame_pred])
                     distances = sklearn.metrics.pairwise.euclidean_distances(gt_boxes, pred_boxes)
+                elif self.dist_fcn.__name__ == 'iou_complement':
+                    distances = np.zeros((len(frame_gt),len(frame_pred)))
+                    for i in range(len(frame_gt)):
+                        for j in range(len(frame_pred)):
+                             distances[i,j] = iou_complement(frame_gt[i],frame_pred[j])
+                else:
+                    raise Exception('Error: Unknown distance function %s!' % self.dist_fcn.__name__)
 
                 # Distances that are larger than the threshold won't be associated.
                 assert len(distances) == 0 or not np.all(np.isnan(distances))

From c4396e51b8bdb6346bcfa9f6dc3274060bcb4964 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 15:52:57 +0100
Subject: [PATCH 2/8] uncomment skip test command, update data classes and unit
 tests

---
 .../nuscenes/eval/tracking/data_classes.py    |  4 +-
 .../nuscenes/eval/tracking/tests/test_algo.py |  1 +
 .../eval/tracking/tests/test_evaluate.py      | 94 ++++++++++++-------
 3 files changed, 66 insertions(+), 33 deletions(-)

diff --git a/python-sdk/nuscenes/eval/tracking/data_classes.py b/python-sdk/nuscenes/eval/tracking/data_classes.py
index 31d219fad..742e999f0 100644
--- a/python-sdk/nuscenes/eval/tracking/data_classes.py
+++ b/python-sdk/nuscenes/eval/tracking/data_classes.py
@@ -6,7 +6,7 @@
 import numpy as np
 
 from nuscenes.eval.common.data_classes import MetricData, EvalBox
-from nuscenes.eval.common.utils import center_distance
+from nuscenes.eval.common.utils import center_distance, iou_complement
 from nuscenes.eval.tracking.constants import TRACKING_METRICS, AMOT_METRICS
 
 
@@ -86,6 +86,8 @@ def dist_fcn_callable(self):
         """ Return the distance function corresponding to the dist_fcn string. """
         if self.dist_fcn == 'center_distance':
             return center_distance
+        elif self.dist_fcn == "iou_complement":
+            return iou_complement
         else:
             raise Exception('Error: Unknown distance function %s!' % self.dist_fcn)
 
diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_algo.py b/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
index 44d6b67a3..1c52a54f1 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
@@ -19,6 +19,7 @@ def single_scene() -> Tuple[str, Dict[str, Dict[int, List[TrackingBox]]]]:
         class_name = 'car'
         box = TrackingBox(translation=(0, 0, 0), tracking_id='ta', tracking_name=class_name,
                           tracking_score=0.5)
+        box.size = [3,1,1]
         timestamp_boxes_gt = {
             0: [copy.deepcopy(box)],
             1: [copy.deepcopy(box)],
diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
index 5142a50a6..931a33376 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
@@ -7,38 +7,27 @@
 import shutil
 import sys
 import unittest
-from typing import Any, Dict, List, Optional
-from unittest.mock import patch
+from typing import Dict, Optional, Any
 
 import numpy as np
+from tqdm import tqdm
+
 from nuscenes import NuScenes
 from nuscenes.eval.common.config import config_factory
 from nuscenes.eval.tracking.evaluate import TrackingEval
 from nuscenes.eval.tracking.utils import category_to_tracking_name
-from nuscenes.utils.splits import get_scenes_of_split
-from parameterized import parameterized
-from tqdm import tqdm
+from nuscenes.utils.splits import create_splits_scenes
 
 
 class TestMain(unittest.TestCase):
     res_mockup = 'nusc_eval.json'
     res_eval_folder = 'tmp'
-    splits_file_mockup = 'mocked_splits.json'
-
-    def setUp(self):
-        with open(self.splits_file_mockup, 'w') as f:
-            json.dump({
-                "mini_custom_train": ["scene-0061", "scene-0553"],
-                "mini_custom_val": ["scene-0103", "scene-0916"]
-            }, f, indent=2)
 
     def tearDown(self):
         if os.path.exists(self.res_mockup):
             os.remove(self.res_mockup)
         if os.path.exists(self.res_eval_folder):
             shutil.rmtree(self.res_eval_folder)
-        if os.path.exists(self.splits_file_mockup):
-            os.remove(self.splits_file_mockup)
 
     @staticmethod
     def _mock_submission(nusc: NuScenes,
@@ -86,10 +75,10 @@ def random_id(instance_token: str, _add_errors: bool = False) -> str:
         mock_results = {}
 
         # Get all samples in the current evaluation split.
-        scenes_of_eval_split : List[str] = get_scenes_of_split(split_name=split, nusc=nusc)
+        splits = create_splits_scenes()
         val_samples = []
         for sample in nusc.sample:
-            if nusc.get('scene', sample['scene_token'])['name'] in scenes_of_eval_split:
+            if nusc.get('scene', sample['scene_token'])['name'] in splits[split]:
                 val_samples.append(sample)
 
         # Prepare results.
@@ -145,10 +134,13 @@ def random_id(instance_token: str, _add_errors: bool = False) -> str:
         }
         return mock_submission
 
+    @unittest.skip
     def basic_test(self,
                    eval_set: str = 'mini_val',
                    add_errors: bool = False,
-                   render_curves: bool = False) -> Dict[str, Any]:
+                   render_curves: bool = False,
+                   dist_fcn: str = '',
+                   dist_th_tp: float = 0.0) -> Dict[str, Any]:
         """
         Run the evaluation with fixed randomness on the specified subset, with or without introducing errors in the
         submission.
@@ -174,13 +166,17 @@ def basic_test(self,
             json.dump(mock, f, indent=2)
 
         cfg = config_factory('tracking_nips_2019')
+
+        # Override dist fcn and threshold
+        cfg.dist_fcn = dist_fcn
+        cfg.dist_th_tp = dist_th_tp
+
         nusc_eval = TrackingEval(cfg, self.res_mockup, eval_set=eval_set, output_dir=self.res_eval_folder,
                                  nusc_version=version, nusc_dataroot=os.environ['NUSCENES'], verbose=False)
         metrics = nusc_eval.main(render_curves=render_curves)
 
         return metrics
-
-    @unittest.skip
+    #@unittest.skip
     def test_delta_mock(self,
                         eval_set: str = 'mini_val',
                         render_curves: bool = False):
@@ -192,7 +188,8 @@ def test_delta_mock(self,
         :param render_curves: Whether to render stats curves to disk.
         """
         # Run the evaluation with errors.
-        metrics = self.basic_test(eval_set, add_errors=True, render_curves=render_curves)
+        metrics = self.basic_test(eval_set, add_errors=True, render_curves=render_curves,
+                                dist_fcn='center_distance', dist_th_tp=2.0)
 
         # Compare metrics to known solution.
         if eval_set == 'mini_val':
@@ -204,14 +201,23 @@ def test_delta_mock(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
-    @parameterized.expand([
-        ('mini_val',),
-        ('mini_custom_train',)
-    ])
-    @patch('nuscenes.utils.splits._get_custom_splits_file_path')
+        # Run again with the alternative iou_complement dist_fcn
+        metrics = self.basic_test(eval_set, add_errors=True, render_curves=render_curves,
+                                  dist_fcn='iou_complement', dist_th_tp=0.999999)
+
+        # Compare metrics to known solution.
+        if eval_set == 'mini_val':
+            self.assertAlmostEqual(metrics['amota'], 0.231839679131956)
+            self.assertAlmostEqual(metrics['amotp'], 1.3629342647309446)
+            self.assertAlmostEqual(metrics['motar'], 0.27918315466340504)
+            self.assertAlmostEqual(metrics['mota'], 0.22922560056448252)
+            self.assertAlmostEqual(metrics['motp'], 0.7541595548820258)
+        else:
+            print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
+
+    @unittest.skip
     def test_delta_gt(self,
-                      eval_set: str,
-                      mock__get_custom_splits_file_path: str,
+                      eval_set: str = 'mini_val',
                       render_curves: bool = False):
         """
         This tests runs the evaluation with the ground truth used as predictions.
@@ -221,15 +227,14 @@ def test_delta_gt(self,
         :param eval_set: Which set to evaluate on.
         :param render_curves: Whether to render stats curves to disk.
         """
-        mock__get_custom_splits_file_path.return_value = self.splits_file_mockup
-
         # Run the evaluation without errors.
-        metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves)
+        metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves,
+                                  dist_fcn='center_distance', dist_th_tp=2.0)
 
         # Compare metrics to known solution. Do not check:
         # - MT/TP (hard to figure out here).
         # - AMOTA/AMOTP (unachieved recall values lead to hard unintuitive results).
-        if eval_set in ['mini_val', 'mini_custom_train']:
+        if eval_set == 'mini_val':
             self.assertAlmostEqual(metrics['amota'], 1.0)
             self.assertAlmostEqual(metrics['amotp'], 0.0, delta=1e-5)
             self.assertAlmostEqual(metrics['motar'], 1.0)
@@ -247,6 +252,31 @@ def test_delta_gt(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
+        # Run again with the alternative iou_complement dist_fcn
+        # Note that very precise threshold specified given results are identical
+        metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves,
+                                  dist_fcn='iou_complement', dist_th_tp=1e-6)
+
+        # Compare metrics to known solution. Do not check:
+        # - MT/TP (hard to figure out here).
+        # - AMOTA/AMOTP (unachieved recall values lead to hard unintuitive results).
+        if eval_set == 'mini_val':
+            self.assertAlmostEqual(metrics['amota'], 1.0)
+            self.assertAlmostEqual(metrics['amotp'], 0.0, delta=1e-5)
+            self.assertAlmostEqual(metrics['motar'], 1.0)
+            self.assertAlmostEqual(metrics['recall'], 1.0)
+            self.assertAlmostEqual(metrics['mota'], 1.0)
+            self.assertAlmostEqual(metrics['motp'], 0.0, delta=1e-5)
+            self.assertAlmostEqual(metrics['faf'], 0.0)
+            self.assertAlmostEqual(metrics['ml'], 0.0)
+            self.assertAlmostEqual(metrics['fp'], 0.0)
+            self.assertAlmostEqual(metrics['fn'], 0.0)
+            self.assertAlmostEqual(metrics['ids'], 0.0)
+            self.assertAlmostEqual(metrics['frag'], 0.0)
+            self.assertAlmostEqual(metrics['tid'], 0.0)
+            self.assertAlmostEqual(metrics['lgd'], 0.0)
+        else:
+            print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
 if __name__ == '__main__':
     unittest.main()

From 3105d86cfadf9d1988b84deb1ba0a20c024025d2 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 16:36:23 +0100
Subject: [PATCH 3/8] add unit test for iou_complement used in detection
 evaluation

---
 .../nuscenes/eval/detection/tests/test_evaluate.py    | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py b/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
index 9a9057b9e..9d57912ed 100644
--- a/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
@@ -149,5 +149,16 @@ def test_delta(self, eval_split, mock__get_custom_splits_file_path):
         # 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split, and the equal mini_custom_val split.
         self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
 
+        # Evaluate again but use the iou_complement distance function
+        # 1. Score = 0.16651633528966858. Measured on forked repo sbarkby/nuscenes-devkit April 22nd 2024.
+        cfg.dist_fcn = "iou_complement"
+        cfg.dist_ths = [0,0.999999]
+        cfg.dist_th_tp = 0.999999
+
+        nusc_eval = DetectionEval(nusc, cfg, self.res_mockup, eval_set=eval_split, output_dir=self.res_eval_folder,
+                                  verbose=False)
+        metrics, md_list = nusc_eval.evaluate()
+        self.assertAlmostEqual(metrics.nd_score, 0.16651633528966858)
+
 if __name__ == '__main__':
     unittest.main()

From 127e868030ea0733cc14077c0385f7d4770723d7 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 16:52:00 +0100
Subject: [PATCH 4/8] remove size definition in unit test (not required)

---
 python-sdk/nuscenes/eval/tracking/tests/test_algo.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_algo.py b/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
index 1c52a54f1..44d6b67a3 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_algo.py
@@ -19,7 +19,6 @@ def single_scene() -> Tuple[str, Dict[str, Dict[int, List[TrackingBox]]]]:
         class_name = 'car'
         box = TrackingBox(translation=(0, 0, 0), tracking_id='ta', tracking_name=class_name,
                           tracking_score=0.5)
-        box.size = [3,1,1]
         timestamp_boxes_gt = {
             0: [copy.deepcopy(box)],
             1: [copy.deepcopy(box)],

From e8ae65d6a0cf5aecc388e0e9b47bb91aca4430ba Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 17:06:02 +0100
Subject: [PATCH 5/8] rebase on recent changes

---
 .../eval/tracking/tests/test_evaluate.py      | 44 ++++++++++++-------
 1 file changed, 29 insertions(+), 15 deletions(-)

diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
index 931a33376..a6b308678 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
@@ -7,27 +7,37 @@
 import shutil
 import sys
 import unittest
-from typing import Dict, Optional, Any
+from typing import Any, Dict, List, Optional
+from unittest.mock import patch
 
 import numpy as np
-from tqdm import tqdm
-
 from nuscenes import NuScenes
 from nuscenes.eval.common.config import config_factory
 from nuscenes.eval.tracking.evaluate import TrackingEval
 from nuscenes.eval.tracking.utils import category_to_tracking_name
-from nuscenes.utils.splits import create_splits_scenes
+from nuscenes.utils.splits import get_scenes_of_split
+from parameterized import parameterized
+from tqdm import tqdm
 
 
 class TestMain(unittest.TestCase):
     res_mockup = 'nusc_eval.json'
     res_eval_folder = 'tmp'
-
+    splits_file_mockup = 'mocked_splits.json'
+
+    def setUp(self):
+        with open(self.splits_file_mockup, 'w') as f:
+            json.dump({
+                "mini_custom_train": ["scene-0061", "scene-0553"],
+                "mini_custom_val": ["scene-0103", "scene-0916"]
+            }, f, indent=2)
     def tearDown(self):
         if os.path.exists(self.res_mockup):
             os.remove(self.res_mockup)
         if os.path.exists(self.res_eval_folder):
             shutil.rmtree(self.res_eval_folder)
+        if os.path.exists(self.splits_file_mockup):
+            os.remove(self.splits_file_mockup)
 
     @staticmethod
     def _mock_submission(nusc: NuScenes,
@@ -75,7 +85,7 @@ def random_id(instance_token: str, _add_errors: bool = False) -> str:
         mock_results = {}
 
         # Get all samples in the current evaluation split.
-        splits = create_splits_scenes()
+        scenes_of_eval_split : List[str] = get_scenes_of_split(split_name=split, nusc=nusc)
         val_samples = []
         for sample in nusc.sample:
             if nusc.get('scene', sample['scene_token'])['name'] in splits[split]:
@@ -134,7 +144,6 @@ def random_id(instance_token: str, _add_errors: bool = False) -> str:
         }
         return mock_submission
 
-    @unittest.skip
     def basic_test(self,
                    eval_set: str = 'mini_val',
                    add_errors: bool = False,
@@ -167,7 +176,7 @@ def basic_test(self,
 
         cfg = config_factory('tracking_nips_2019')
 
-        # Override dist fcn and threshold
+        # Update dist fcn and threshold with those specified
         cfg.dist_fcn = dist_fcn
         cfg.dist_th_tp = dist_th_tp
 
@@ -176,7 +185,8 @@ def basic_test(self,
         metrics = nusc_eval.main(render_curves=render_curves)
 
         return metrics
-    #@unittest.skip
+
+    @unittest.skip
     def test_delta_mock(self,
                         eval_set: str = 'mini_val',
                         render_curves: bool = False):
@@ -215,9 +225,14 @@ def test_delta_mock(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
-    @unittest.skip
+    @parameterized.expand([
+        ('mini_val',),
+        ('mini_custom_train',)
+    ])
+    @patch('nuscenes.utils.splits._get_custom_splits_file_path')
     def test_delta_gt(self,
-                      eval_set: str = 'mini_val',
+                      eval_set: str,
+                      mock__get_custom_splits_file_path: str,
                       render_curves: bool = False):
         """
         This tests runs the evaluation with the ground truth used as predictions.
@@ -234,7 +249,7 @@ def test_delta_gt(self,
         # Compare metrics to known solution. Do not check:
         # - MT/TP (hard to figure out here).
         # - AMOTA/AMOTP (unachieved recall values lead to hard unintuitive results).
-        if eval_set == 'mini_val':
+        if eval_set in ['mini_val', 'mini_custom_train']:
             self.assertAlmostEqual(metrics['amota'], 1.0)
             self.assertAlmostEqual(metrics['amotp'], 0.0, delta=1e-5)
             self.assertAlmostEqual(metrics['motar'], 1.0)
@@ -252,15 +267,14 @@ def test_delta_gt(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
-        # Run again with the alternative iou_complement dist_fcn
-        # Note that very precise threshold specified given results are identical
+        # Run again with the alternative iou_complement dist_fcn (and a very precise threshold)
         metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves,
                                   dist_fcn='iou_complement', dist_th_tp=1e-6)
 
         # Compare metrics to known solution. Do not check:
         # - MT/TP (hard to figure out here).
         # - AMOTA/AMOTP (unachieved recall values lead to hard unintuitive results).
-        if eval_set == 'mini_val':
+        if eval_set in ['mini_val', 'mini_custom_train']:
             self.assertAlmostEqual(metrics['amota'], 1.0)
             self.assertAlmostEqual(metrics['amotp'], 0.0, delta=1e-5)
             self.assertAlmostEqual(metrics['motar'], 1.0)

From 58968091954d4bd73dfff1bb54e9390c0a31df20 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Mon, 22 Apr 2024 17:36:37 +0100
Subject: [PATCH 6/8] rebase again

---
 python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
index a6b308678..046139716 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
@@ -31,6 +31,7 @@ def setUp(self):
                 "mini_custom_train": ["scene-0061", "scene-0553"],
                 "mini_custom_val": ["scene-0103", "scene-0916"]
             }, f, indent=2)
+
     def tearDown(self):
         if os.path.exists(self.res_mockup):
             os.remove(self.res_mockup)
@@ -88,7 +89,7 @@ def random_id(instance_token: str, _add_errors: bool = False) -> str:
         scenes_of_eval_split : List[str] = get_scenes_of_split(split_name=split, nusc=nusc)
         val_samples = []
         for sample in nusc.sample:
-            if nusc.get('scene', sample['scene_token'])['name'] in splits[split]:
+            if nusc.get('scene', sample['scene_token'])['name'] in scenes_of_eval_split:
                 val_samples.append(sample)
 
         # Prepare results.
@@ -242,6 +243,8 @@ def test_delta_gt(self,
         :param eval_set: Which set to evaluate on.
         :param render_curves: Whether to render stats curves to disk.
         """
+        mock__get_custom_splits_file_path.return_value = self.splits_file_mockup
+
         # Run the evaluation without errors.
         metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves,
                                   dist_fcn='center_distance', dist_th_tp=2.0)

From a6b3a34515375fa47f8442ef4160fcdf64b4e9fa Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Tue, 23 Apr 2024 09:55:12 +0100
Subject: [PATCH 7/8] add function descriptions, guard against machine
 precision when calculating iou

---
 python-sdk/nuscenes/eval/common/utils.py | 25 +++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/python-sdk/nuscenes/eval/common/utils.py b/python-sdk/nuscenes/eval/common/utils.py
index 866d25a55..a554b5011 100644
--- a/python-sdk/nuscenes/eval/common/utils.py
+++ b/python-sdk/nuscenes/eval/common/utils.py
@@ -13,6 +13,11 @@
 DetectionBox = Any  # Workaround as direct imports lead to cyclic dependencies.
 
 def create_polygon_from_box(bbox: EvalBox):
+    """
+    Convert an EvalBox into a Polygon
+    :param bbox: A EvalBox describing center, rotation and size.
+    :return: A Polygon describing the xy vertices.
+    """
     l = bbox.size[0]
     w = bbox.size[1]
     poly_veh = Polygon(((0.5*l,0.5*w),(-0.5*l,0.5*w),(-0.5*l,-0.5*w),(0.5*l,-0.5*w),(0.5*l,0.5*w)))
@@ -21,13 +26,22 @@ def create_polygon_from_box(bbox: EvalBox):
     return poly_glob
 
 def intersection_over_union(gt_poly: Polygon, pred_poly: Polygon):
+    """
+    IOU percentage between two input polygons (xy only).
+    :param gt_poly: GT annotation sample.
+    :param pred_poly: Predicted sample.
+    :return: IOU.
+    """
     intersection = gt_poly.intersection(pred_poly).area
     iou = intersection/(gt_poly.area + pred_poly.area - intersection)
+
+     # Guard against machine precision (i.e. when dealing with perfect overlap)
+    iou = min(iou,1.0)
     return iou
 
 def iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
     """
-    1 - IOU percentage between the boxes (xy only).
+    1 - IOU percentage between two input boxes (xy only).
     :param gt_box: GT annotation sample.
     :param pred_box: Predicted sample.
     :return: 1 - IOU.
@@ -36,11 +50,12 @@ def iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
     # boxes overlap
     gt_radius = np.linalg.norm(0.5*np.array([gt_box.size[0],gt_box.size[1]]))
     pred_radius = np.linalg.norm(0.5*np.array([pred_box.size[0],pred_box.size[1]]))
-    if (center_distance(gt_box,pred_box) < pred_radius + gt_radius):
-        iou = intersection_over_union(create_polygon_from_box(gt_box),create_polygon_from_box(pred_box))
+    if (center_distance(gt_box,pred_box) >= pred_radius + gt_radius):
+        iou_complement = 1.0
     else:
-        iou = 0.0
-    return 1.0 - iou
+        iou_complement = 1.0 - intersection_over_union(create_polygon_from_box(gt_box),
+                                      create_polygon_from_box(pred_box))
+    return iou_complement
 
 def center_distance(gt_box: EvalBox, pred_box: EvalBox) -> float:
     """

From 49028a194e9db5c4af147f0dcb25f728db7188f9 Mon Sep 17 00:00:00 2001
From: Stephen Barkby <stephen.barkby@oxbotica.com>
Date: Wed, 24 Apr 2024 11:56:30 +0100
Subject: [PATCH 8/8] rename to bev_iou_complement distance function, add unit
 tests

---
 python-sdk/nuscenes/eval/common/utils.py      | 30 +++++++-------
 .../nuscenes/eval/detection/data_classes.py   |  6 +--
 .../eval/detection/tests/test_evaluate.py     |  4 +-
 .../eval/detection/tests/test_utils.py        | 41 +++++++++++++++----
 python-sdk/nuscenes/eval/tracking/algo.py     |  6 +--
 .../nuscenes/eval/tracking/data_classes.py    |  6 +--
 .../eval/tracking/tests/test_evaluate.py      |  8 ++--
 7 files changed, 64 insertions(+), 37 deletions(-)

diff --git a/python-sdk/nuscenes/eval/common/utils.py b/python-sdk/nuscenes/eval/common/utils.py
index a554b5011..b97f22e4f 100644
--- a/python-sdk/nuscenes/eval/common/utils.py
+++ b/python-sdk/nuscenes/eval/common/utils.py
@@ -12,11 +12,11 @@
 
 DetectionBox = Any  # Workaround as direct imports lead to cyclic dependencies.
 
-def create_polygon_from_box(bbox: EvalBox):
+def create_2d_polygon_from_box(bbox: EvalBox) -> Polygon:
     """
-    Convert an EvalBox into a Polygon
-    :param bbox: A EvalBox describing center, rotation and size.
-    :return: A Polygon describing the xy vertices.
+    Convert an EvalBox into a 2D Polygon
+    :param bbox: An EvalBox describing center, rotation and size.
+    :return: A 2D Polygon describing the xy vertices.
     """
     l = bbox.size[0]
     w = bbox.size[1]
@@ -25,23 +25,23 @@ def create_polygon_from_box(bbox: EvalBox):
     poly_glob = affinity.translate(poly_rot,bbox.translation[0],bbox.translation[1])
     return poly_glob
 
-def intersection_over_union(gt_poly: Polygon, pred_poly: Polygon):
+def bev_iou(gt_poly: Polygon, pred_poly: Polygon) -> float:
     """
-    IOU percentage between two input polygons (xy only).
+    Birds Eye View IOU percentage between two input polygons (xy only).
     :param gt_poly: GT annotation sample.
     :param pred_poly: Predicted sample.
     :return: IOU.
     """
     intersection = gt_poly.intersection(pred_poly).area
-    iou = intersection/(gt_poly.area + pred_poly.area - intersection)
+    bev_iou = intersection/(gt_poly.area + pred_poly.area - intersection)
 
      # Guard against machine precision (i.e. when dealing with perfect overlap)
-    iou = min(iou,1.0)
-    return iou
+    bev_iou = min(bev_iou,1.0)
+    return bev_iou
 
-def iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
+def bev_iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
     """
-    1 - IOU percentage between two input boxes (xy only).
+    1 - BEV_IOU percentage between two input boxes (xy only).
     :param gt_box: GT annotation sample.
     :param pred_box: Predicted sample.
     :return: 1 - IOU.
@@ -51,11 +51,11 @@ def iou_complement(gt_box: EvalBox, pred_box: EvalBox) -> float:
     gt_radius = np.linalg.norm(0.5*np.array([gt_box.size[0],gt_box.size[1]]))
     pred_radius = np.linalg.norm(0.5*np.array([pred_box.size[0],pred_box.size[1]]))
     if (center_distance(gt_box,pred_box) >= pred_radius + gt_radius):
-        iou_complement = 1.0
+        bev_iou_complement = 1.0
     else:
-        iou_complement = 1.0 - intersection_over_union(create_polygon_from_box(gt_box),
-                                      create_polygon_from_box(pred_box))
-    return iou_complement
+        bev_iou_complement = 1.0 - bev_iou(create_2d_polygon_from_box(gt_box),
+                                      create_2d_polygon_from_box(pred_box))
+    return bev_iou_complement
 
 def center_distance(gt_box: EvalBox, pred_box: EvalBox) -> float:
     """
diff --git a/python-sdk/nuscenes/eval/detection/data_classes.py b/python-sdk/nuscenes/eval/detection/data_classes.py
index b49e4756a..e9682a85b 100644
--- a/python-sdk/nuscenes/eval/detection/data_classes.py
+++ b/python-sdk/nuscenes/eval/detection/data_classes.py
@@ -7,7 +7,7 @@
 import numpy as np
 
 from nuscenes.eval.common.data_classes import MetricData, EvalBox
-from nuscenes.eval.common.utils import center_distance, iou_complement
+from nuscenes.eval.common.utils import center_distance, bev_iou_complement
 from nuscenes.eval.detection.constants import DETECTION_NAMES, ATTRIBUTE_NAMES, TP_METRICS
 
 
@@ -74,8 +74,8 @@ def dist_fcn_callable(self):
         """ Return the distance function corresponding to the dist_fcn string. """
         if self.dist_fcn == 'center_distance':
             return center_distance
-        elif self.dist_fcn == "iou_complement":
-            return iou_complement
+        elif self.dist_fcn == "bev_iou_complement":
+            return bev_iou_complement
         else:
             raise Exception('Error: Unknown distance function %s!' % self.dist_fcn)
 
diff --git a/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py b/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
index 9d57912ed..6f073209d 100644
--- a/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/detection/tests/test_evaluate.py
@@ -149,9 +149,9 @@ def test_delta(self, eval_split, mock__get_custom_splits_file_path):
         # 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split, and the equal mini_custom_val split.
         self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
 
-        # Evaluate again but use the iou_complement distance function
+        # Evaluate again but use the bev_iou_complement distance function
         # 1. Score = 0.16651633528966858. Measured on forked repo sbarkby/nuscenes-devkit April 22nd 2024.
-        cfg.dist_fcn = "iou_complement"
+        cfg.dist_fcn = "bev_iou_complement"
         cfg.dist_ths = [0,0.999999]
         cfg.dist_th_tp = 0.999999
 
diff --git a/python-sdk/nuscenes/eval/detection/tests/test_utils.py b/python-sdk/nuscenes/eval/detection/tests/test_utils.py
index 69eefcc5f..0c1172c1f 100644
--- a/python-sdk/nuscenes/eval/detection/tests/test_utils.py
+++ b/python-sdk/nuscenes/eval/detection/tests/test_utils.py
@@ -7,8 +7,8 @@
 from numpy.testing import assert_array_almost_equal
 from pyquaternion import Quaternion
 
-from nuscenes.eval.common.utils import attr_acc, scale_iou, yaw_diff, angle_diff, iou_complement, center_distance, \
-    velocity_l2, cummean
+from nuscenes.eval.common.utils import attr_acc, scale_iou, yaw_diff, angle_diff, create_2d_polygon_from_box, bev_iou, \
+     bev_iou_complement, center_distance, velocity_l2, cummean
 from nuscenes.eval.detection.data_classes import DetectionBox
 
 
@@ -128,18 +128,45 @@ def rad(x):
         period = 360
         self.assertAlmostEqual(rad(180), abs(angle_diff(rad(a), rad(b), rad(period))))
 
-    def test_iou_complement_no_overlap(self):
+    def test_create_2d_polygon_from_box(self):
+        # Create a box rotated 30 degrees and offset of (2,4), check against hand calculated math
+        poly = create_2d_polygon_from_box(DetectionBox(rotation=(0.96592582628,0,0,0.2588190451),
+                          translation=(2, 4, 1), size=(3,1,2)))
+        self.assertAlmostEqual(poly.exterior.coords[0][0],3.04903810568)
+        self.assertAlmostEqual(poly.exterior.coords[0][1],5.18301270189)
+        self.assertAlmostEqual(poly.exterior.coords[1][0],0.45096189432)
+        self.assertAlmostEqual(poly.exterior.coords[1][1],3.6830127019)
+        self.assertAlmostEqual(poly.exterior.coords[2][0],0.95096189432)
+        self.assertAlmostEqual(poly.exterior.coords[2][1],2.81698729811)
+        self.assertAlmostEqual(poly.exterior.coords[3][0],3.54903810568)
+        self.assertAlmostEqual(poly.exterior.coords[3][1],4.3169872981)
+
+    def test_bev_iou(self):
+        # Two boxes specified, no overlap
+        sa = create_2d_polygon_from_box(DetectionBox(translation=(1.0, 0.0, 1.0),
+                                                     size=(2,1,1)))
+        sr = create_2d_polygon_from_box(DetectionBox(translation=(3.5, 0.0, 1.0),
+                                                     size=(3,1,2)))
+        self.assertAlmostEqual(bev_iou(sa, sr), 0.0)
+
+        # Two boxes specified, one rotated by 90 degrees in z axis, should attain 1m^2 overlap
+        sa = create_2d_polygon_from_box(DetectionBox(rotation=(0,0,0,0), translation=(1.0, 0.5, 2.0),
+                                                     size=(2,1,1)))
+        sr = create_2d_polygon_from_box(DetectionBox(rotation=(0.70710678118,0,0,0.70710678118),
+                          translation=(0.5, 1.5, 1), size=(3,1,2)))
+        self.assertAlmostEqual(bev_iou(sa, sr), 0.25)
+
+    def test_bev_iou_complement(self):
         # Two boxes specified, no overlap
         sa = DetectionBox(translation=(1.0, 0.0, 1.0), size=(2,1,1))
         sr = DetectionBox(translation=(3.5, 0.0, 1.0), size=(3,1,2))
-        self.assertAlmostEqual(iou_complement(sa, sr), 1.0)
+        self.assertAlmostEqual(bev_iou_complement(sa, sr), 1.0)
 
-    def test_iou_complement_overlap(self):
-        # Two boxes specified, one rotated by 90 degrees in z axis, should attain 25% overlap
+        # Two boxes specified, one rotated by 90 degrees in z axis, should attain 1m^2 overlap
         sa = DetectionBox(rotation=(0,0,0,0), translation=(1.0, 0.5, 2.0), size=(2,1,1))
         sr = DetectionBox(rotation=(0.70710678118,0,0,0.70710678118),
                           translation=(0.5, 1.5, 1), size=(3,1,2))
-        self.assertAlmostEqual(iou_complement(sa, sr), 0.75)
+        self.assertAlmostEqual(bev_iou_complement(sa, sr), 0.75)
 
     def test_center_distance(self):
         """Test for center_distance()."""
diff --git a/python-sdk/nuscenes/eval/tracking/algo.py b/python-sdk/nuscenes/eval/tracking/algo.py
index db9116515..3b89bec94 100644
--- a/python-sdk/nuscenes/eval/tracking/algo.py
+++ b/python-sdk/nuscenes/eval/tracking/algo.py
@@ -23,7 +23,7 @@
 except ModuleNotFoundError:
     raise unittest.SkipTest('Skipping test as pandas was not found!')
 
-from nuscenes.eval.common.utils import iou_complement
+from nuscenes.eval.common.utils import bev_iou_complement
 from nuscenes.eval.tracking.constants import MOT_METRIC_MAP, TRACKING_METRICS
 from nuscenes.eval.tracking.data_classes import TrackingBox, TrackingMetricData
 from nuscenes.eval.tracking.mot import MOTAccumulatorCustom
@@ -264,11 +264,11 @@ def accumulate_threshold(self, threshold: float = None) -> Tuple[pandas.DataFram
                     gt_boxes = np.array([b.translation[:2] for b in frame_gt])
                     pred_boxes = np.array([b.translation[:2] for b in frame_pred])
                     distances = sklearn.metrics.pairwise.euclidean_distances(gt_boxes, pred_boxes)
-                elif self.dist_fcn.__name__ == 'iou_complement':
+                elif self.dist_fcn.__name__ == 'bev_iou_complement':
                     distances = np.zeros((len(frame_gt),len(frame_pred)))
                     for i in range(len(frame_gt)):
                         for j in range(len(frame_pred)):
-                             distances[i,j] = iou_complement(frame_gt[i],frame_pred[j])
+                             distances[i,j] = bev_iou_complement(frame_gt[i],frame_pred[j])
                 else:
                     raise Exception('Error: Unknown distance function %s!' % self.dist_fcn.__name__)
 
diff --git a/python-sdk/nuscenes/eval/tracking/data_classes.py b/python-sdk/nuscenes/eval/tracking/data_classes.py
index 742e999f0..629938849 100644
--- a/python-sdk/nuscenes/eval/tracking/data_classes.py
+++ b/python-sdk/nuscenes/eval/tracking/data_classes.py
@@ -6,7 +6,7 @@
 import numpy as np
 
 from nuscenes.eval.common.data_classes import MetricData, EvalBox
-from nuscenes.eval.common.utils import center_distance, iou_complement
+from nuscenes.eval.common.utils import center_distance, bev_iou_complement
 from nuscenes.eval.tracking.constants import TRACKING_METRICS, AMOT_METRICS
 
 
@@ -86,8 +86,8 @@ def dist_fcn_callable(self):
         """ Return the distance function corresponding to the dist_fcn string. """
         if self.dist_fcn == 'center_distance':
             return center_distance
-        elif self.dist_fcn == "iou_complement":
-            return iou_complement
+        elif self.dist_fcn == "bev_iou_complement":
+            return bev_iou_complement
         else:
             raise Exception('Error: Unknown distance function %s!' % self.dist_fcn)
 
diff --git a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
index 046139716..bad4b8800 100644
--- a/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
+++ b/python-sdk/nuscenes/eval/tracking/tests/test_evaluate.py
@@ -212,9 +212,9 @@ def test_delta_mock(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
-        # Run again with the alternative iou_complement dist_fcn
+        # Run again with the alternative bev_iou_complement dist_fcn
         metrics = self.basic_test(eval_set, add_errors=True, render_curves=render_curves,
-                                  dist_fcn='iou_complement', dist_th_tp=0.999999)
+                                  dist_fcn='bev_iou_complement', dist_th_tp=0.999999)
 
         # Compare metrics to known solution.
         if eval_set == 'mini_val':
@@ -270,9 +270,9 @@ def test_delta_gt(self,
         else:
             print('Skipping checks due to choice of custom eval_set: %s' % eval_set)
 
-        # Run again with the alternative iou_complement dist_fcn (and a very precise threshold)
+        # Run again with the alternative bev_iou_complement dist_fcn (and a very precise threshold)
         metrics = self.basic_test(eval_set, add_errors=False, render_curves=render_curves,
-                                  dist_fcn='iou_complement', dist_th_tp=1e-6)
+                                  dist_fcn='bev_iou_complement', dist_th_tp=1e-6)
 
         # Compare metrics to known solution. Do not check:
         # - MT/TP (hard to figure out here).