Build and test with Python 3.12

.github/actions/run-python-tests/action.yml

@@ -0,0 +1,62 @@
+name: 'Run Python Tests'
+description: 'Run Python Tests'
+inputs:
+  python-version:
+    description: 'Python version'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Display Python version
+      shell: bash
+      env:
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: echo "Running Python tests with version ${PYTHON_VERSION}"
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ inputs.python-version }}
+    - name: Install datasets
+      shell: bash
+      run: |
+        mkdir -p ${NUSCENES} && mkdir -p ${NUIMAGES}
+
+        echo "Installing: v1.0-mini.tgz"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/v1.0/v1.0-mini.tgz | tar -xzf - -C ${NUSCENES} --exclude sweeps
+
+        echo "Installing: nuimages-v1.0-mini.tgz"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/nuimages-v1.0/nuimages-v1.0-mini.tgz | tar -xzf - -C ${NUIMAGES}
+
+        echo "Installing: nuScenes-lidarseg-mini-v1.0.tar.bz2"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/nuscenes-lidarseg-v1.0/nuScenes-lidarseg-mini-v1.0.tar.bz2 | tar -xjf - -C ${NUSCENES}
+
+        echo "Installing: nuScenes-panoptic-v1.0-mini.tar.gz"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/nuscenes-panoptic-v1.0/nuScenes-panoptic-v1.0-mini.tar.gz | tar -xzf - --strip-components=1 -C ${NUSCENES}
+
+        echo "Installing: nuScenes-map-expansion-v1.3.zip"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/v1.0/nuScenes-map-expansion-v1.3.zip -o nuScenes-map-expansion-v1.3.zip
+        unzip -q nuScenes-map-expansion-v1.3.zip -d ${NUSCENES}/maps/
+
+        echo "Installing: can_bus.zip"
+        curl -fsSL https://motional-nuscenes.s3-ap-northeast-1.amazonaws.com/public/v1.0/can_bus.zip -o can_bus.zip
+        unzip -q can_bus.zip -d ${NUSCENES} can_bus/scene-0001_*
+
+        echo "Removing zip files . . ."
+        rm nuScenes-map-expansion-v1.3.zip can_bus.zip
+    - name: Install dependencies
+      shell: bash
+      env:
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: |
+        PYTHON_VERSION_UNDERSCORE=${PYTHON_VERSION//./_}
+        pip install -r setup/requirements_${PYTHON_VERSION_UNDERSCORE}_lock.txt
+    - name: Run Python unit tests
+      shell: bash
+      run: |
+        python -m unittest discover python-sdk
+    - name: Run Jupyter notebook tests
+      shell: bash
+      run: |
+        pip install jupyter -q
+        export PYTHONPATH="${PYTHONPATH}:$(pwd)/python-sdk"
+        ./setup/test_tutorial.sh --ci

.github/workflows/pipeline.yml

@@ -0,0 +1,24 @@
+name: nuscenes-devkit CI pipeline
+on: [pull_request]
+env:
+  NUSCENES: data/sets/nuscenes
+  NUIMAGES: data/sets/nuimages
+jobs:
+  test-in-3-9:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - id: run-test-in-3-9
+        uses: ./.github/actions/run-python-tests
+        with:
+          python-version: 3.9
+  test-in-3-12:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+      - id: run-test-in-3-12
+        uses: ./.github/actions/run-python-tests
+        with:
+          python-version: 3.12

python-sdk/nuscenes/eval/tracking/constants.py

@@ -17,6 +17,7 @@
 MOT_METRIC_MAP = {  # Mapping from motmetrics names to metric names used here.
     'num_frames': '',  # Used in FAF.
     'num_objects': 'gt',  # Used in MOTAR computation.
+    'pred_frequencies': '', # Only needed in background.
     'num_predictions': '',  # Only printed out.
     'num_matches': 'tp',  # Used in MOTAR computation and printed out.
     'motar': 'motar',  # Only used in AMOTA.

python-sdk/nuscenes/eval/tracking/evaluate.py

@@ -5,10 +5,9 @@
 import json
 import os
 import time
-from typing import Tuple, List, Dict, Any
+from typing import Any, Dict, List, Tuple
 
 import numpy as np
-
 from nuscenes import NuScenes
 from nuscenes.eval.common.config import config_factory
 from nuscenes.eval.common.loaders import (
@@ -21,9 +20,14 @@
     load_prediction_of_sample_tokens,
 )
 from nuscenes.eval.tracking.algo import TrackingEvaluation
-from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, MOT_METRIC_MAP, LEGACY_METRICS
-from nuscenes.eval.tracking.data_classes import TrackingMetrics, TrackingMetricDataList, TrackingConfig, TrackingBox, \
-    TrackingMetricData
+from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, LEGACY_METRICS, MOT_METRIC_MAP
+from nuscenes.eval.tracking.data_classes import (
+    TrackingBox,
+    TrackingConfig,
+    TrackingMetricData,
+    TrackingMetricDataList,
+    TrackingMetrics,
+)
 from nuscenes.eval.tracking.loaders import create_tracks
 from nuscenes.eval.tracking.render import recall_metric_curve, summary_plot
 from nuscenes.eval.tracking.utils import print_final_metrics

python-sdk/nuscenes/eval/tracking/metrics.py

@@ -7,7 +7,7 @@
 py-motmetrics at:
 https://github.com/cheind/py-motmetrics
 """
-from typing import Any
+from typing import Any, Optional
 
 import numpy as np
 
@@ -109,7 +109,7 @@ def longest_gap_duration(df: DataFrame, obj_frequencies: DataFrame) -> float:
 
 
 def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, num_false_positives: int,
-          num_objects: int, alpha: float = 1.0) -> float:
+          num_objects: int, alpha: float = 1.0, ana: Optional[dict] = None) -> float:
     """
     Initializes a MOTAR class which refers to the modified MOTA metric at https://www.nuscenes.org/tracking.
     Note that we use the measured recall, which is not identical to the hypothetical recall of the
@@ -121,6 +121,7 @@ def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, n
     :param num_false_positives: The number of false positives.
     :param num_objects: The total number of objects of this class in the GT.
     :param alpha: MOTAR weighting factor (previously 0.2).
+    :param ana: something for caching, introduced by motmetrics 1.4.0
     :return: The MOTAR or nan if there are no GT objects.
     """
     recall = num_matches / num_objects

python-sdk/nuscenes/eval/tracking/mot.py

@@ -6,50 +6,74 @@
 
 py-motmetrics at:
 https://github.com/cheind/py-motmetrics
+
+Notes by Michael Hoss:
+For Python 3.10, we need to update the version of py-motmetrics to 1.4.0.
+Then, to keep this code working, we need to change back the types of OId HId to object because they are
+strings in nuscenes-devkit, whereas motmetrics changed these types to float from 1.1.3 to 1.4.0.
 """
 from collections import OrderedDict
 from itertools import count
 
-import motmetrics
 import numpy as np
 import pandas as pd
+from motmetrics import MOTAccumulator
 
+_INDEX_FIELDS = ['FrameId', 'Event']
 
-class MOTAccumulatorCustom(motmetrics.mot.MOTAccumulator):
+class MOTAccumulatorCustom(MOTAccumulator):
+    """This custom class was created by nuscenes-devkit to use a faster implementation of
+    `new_event_dataframe_with_data` under compatibility with motmetrics<=1.1.3.
+    Now that we use motmetrics==1.4.0, we need to use this custom implementation to use
+    objects instead of strings for OId and HId.
+    """
     def __init__(self):
         super().__init__()
 
     @staticmethod
     def new_event_dataframe_with_data(indices, events):
-        """
-        Create a new DataFrame filled with data.
-        This version overwrites the original in MOTAccumulator achieves about 2x speedups.
+        """Create a new DataFrame filled with data.
 
         Params
         ------
-        indices: list
-            list of tuples (frameid, eventid)
-        events: list
-            list of events where each event is a list containing
-            'Type', 'OId', HId', 'D'
+        indices: dict
+            dict of lists with fields 'FrameId' and 'Event'
+        events: dict
+            dict of lists with fields 'Type', 'OId', 'HId', 'D'
         """
-        idx = pd.MultiIndex.from_tuples(indices, names=['FrameId', 'Event'])
-        df = pd.DataFrame(events, index=idx, columns=['Type', 'OId', 'HId', 'D'])
+
+        if len(events) == 0:
+            return MOTAccumulatorCustom.new_event_dataframe()
+
+        raw_type = pd.Categorical(
+            events['Type'],
+            categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'],
+            ordered=False)
+        series = [
+            pd.Series(raw_type, name='Type'),
+            pd.Series(events['OId'], dtype=object, name='OId'),  # OId is string in nuscenes-devkit
+            pd.Series(events['HId'], dtype=object, name='HId'),  # HId is string in nuscenes-devkit
+            pd.Series(events['D'], dtype=float, name='D')
+        ]
+
+        idx = pd.MultiIndex.from_arrays(
+            [indices[field] for field in _INDEX_FIELDS],
+            names=_INDEX_FIELDS)
+        df = pd.concat(series, axis=1)
+        df.index = idx
         return df
 
     @staticmethod
     def new_event_dataframe():
-        """ Create a new DataFrame for event tracking. """
+        """Create a new DataFrame for event tracking."""
         idx = pd.MultiIndex(levels=[[], []], codes=[[], []], names=['FrameId', 'Event'])
-        cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH'])
+        cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'])
         df = pd.DataFrame(
             OrderedDict([
-                ('Type', pd.Series(cats)),  # Type of event. One of FP (false positive), MISS, SWITCH, MATCH
-                ('OId', pd.Series(dtype=object)),
-                # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways.
-                ('HId', pd.Series(dtype=object)),
-                # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways.
-                ('D', pd.Series(dtype=float)),  # Distance or NaN when FP or MISS
+                ('Type', pd.Series(cats)),          # Type of event. One of FP (false positive), MISS, SWITCH, MATCH
+                ('OId', pd.Series(dtype=object)),      # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways.
+                ('HId', pd.Series(dtype=object)),      # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways.
+                ('D', pd.Series(dtype=float)),      # Distance or NaN when FP or MISS
             ]),
             index=idx
         )
@@ -63,8 +87,7 @@ def events(self):
         return self.cached_events_df
 
     @staticmethod
-    def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True,
-                               return_mappings=False):
+    def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, return_mappings=False):
         """Merge dataframes.
 
         Params
@@ -104,24 +127,29 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd
 
             # Update index
             if update_frame_indices:
-                next_frame_id = max(r.index.get_level_values(0).max() + 1,
-                                    r.index.get_level_values(0).unique().shape[0])
+                # pylint: disable=cell-var-from-loop
+                next_frame_id = max(r.index.get_level_values(0).max() + 1, r.index.get_level_values(0).unique().shape[0])
                 if np.isnan(next_frame_id):
                     next_frame_id = 0
-                copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1]))
+                if not copy.index.empty:
+                    copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1]))
                 infos['frame_offset'] = next_frame_id
 
             # Update object / hypothesis ids
             if update_oids:
+                # pylint: disable=cell-var-from-loop
                 oid_map = dict([oid, str(next(new_oid))] for oid in copy['OId'].dropna().unique())
                 copy['OId'] = copy['OId'].map(lambda x: oid_map[x], na_action='ignore')
                 infos['oid_map'] = oid_map
 
             if update_hids:
+                # pylint: disable=cell-var-from-loop
                 hid_map = dict([hid, str(next(new_hid))] for hid in copy['HId'].dropna().unique())
                 copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore')
                 infos['hid_map'] = hid_map
 
+            # Avoid pandas warning. But is this legit/do we need such a column later on again?
+            # copy = copy.dropna(axis=1, how='all')
             r = pd.concat((r, copy))
             mapping_infos.append(infos)
 

python-sdk/nuscenes/eval/tracking/utils.py

@@ -3,7 +3,7 @@
 
 import unittest
 import warnings
-from typing import Optional, Dict
+from typing import Dict, Optional
 
 import numpy as np
 
@@ -14,8 +14,15 @@
     raise unittest.SkipTest('Skipping test as motmetrics was not found!')
 
 from nuscenes.eval.tracking.data_classes import TrackingMetrics
-from nuscenes.eval.tracking.metrics import motar, mota_custom, motp_custom, faf, track_initialization_duration, \
-    longest_gap_duration, num_fragmentations_custom
+from nuscenes.eval.tracking.metrics import (
+    faf,
+    longest_gap_duration,
+    mota_custom,
+    motar,
+    motp_custom,
+    num_fragmentations_custom,
+    track_initialization_duration,
+)
 
 
 def category_to_tracking_name(category_name: str) -> Optional[str]:
@@ -148,8 +155,8 @@ def create_motmetrics() -> MetricsHost:
     # Register standard metrics.
     fields = [
         'num_frames', 'obj_frequencies', 'num_matches', 'num_switches', 'num_false_positives', 'num_misses',
-        'num_detections', 'num_objects', 'num_predictions', 'mostly_tracked', 'mostly_lost', 'num_fragmentations',
-        'motp', 'mota', 'precision', 'recall', 'track_ratios'
+        'num_detections', 'num_objects', 'pred_frequencies', 'num_predictions', 'mostly_tracked', 'mostly_lost',
+        'num_fragmentations', 'motp', 'mota', 'precision', 'recall', 'track_ratios'
     ]
     for field in fields:
         mh.register(getattr(motmetrics.metrics, field), formatter='{:d}'.format)

python-sdk/nuscenes/map_expansion/map_api.py

@@ -29,7 +29,7 @@
 from nuscenes.utils.geometry_utils import view_points
 
 # Recommended style to use as the plots will show grids.
-plt.style.use('seaborn-whitegrid')
+plt.style.use('seaborn-v0_8-whitegrid')
 
 # Define a map geometry type for polygons and lines.
 Geometry = Union[Polygon, LineString]
@@ -1820,8 +1820,8 @@ def mask_for_polygons(polygons: MultiPolygon, mask: np.ndarray) -> np.ndarray:
         def int_coords(x):
             # function to round and convert to int
             return np.array(x).round().astype(np.int32)
-        exteriors = [int_coords(poly.exterior.coords) for poly in polygons]
-        interiors = [int_coords(pi.coords) for poly in polygons for pi in poly.interiors]
+        exteriors = [int_coords(poly.exterior.coords) for poly in polygons.geoms]
+        interiors = [int_coords(pi.coords) for poly in polygons.geoms for pi in poly.interiors]
         cv2.fillPoly(mask, exteriors, 1)
         cv2.fillPoly(mask, interiors, 0)
         return mask
@@ -1885,7 +1885,7 @@ def _polygon_geom_to_mask(self,
                                                         [1.0, 0.0, 0.0, 1.0, trans_x, trans_y])
                 new_polygon = affinity.scale(new_polygon, xfact=scale_width, yfact=scale_height, origin=(0, 0))
 
-                if new_polygon.geom_type is 'Polygon':
+                if new_polygon.geom_type == 'Polygon':
                     new_polygon = MultiPolygon([new_polygon])
                 map_mask = self.mask_for_polygons(new_polygon, map_mask)
 
@@ -1922,7 +1922,7 @@ def _line_geom_to_mask(self,
 
         map_mask = np.zeros(canvas_size, np.uint8)
 
-        if layer_name is 'traffic_light':
+        if layer_name == 'traffic_light':
             return None
 
         for line in layer_geom:
@@ -1968,7 +1968,7 @@ def _get_layer_polygon(self,
                                                       origin=(patch_x, patch_y), use_radians=False)
                         new_polygon = affinity.affine_transform(new_polygon,
                                                                 [1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
-                        if new_polygon.geom_type is 'Polygon':
+                        if new_polygon.geom_type == 'Polygon':
                             new_polygon = MultiPolygon([new_polygon])
                         polygon_list.append(new_polygon)
 
@@ -1983,7 +1983,7 @@ def _get_layer_polygon(self,
                                                       origin=(patch_x, patch_y), use_radians=False)
                         new_polygon = affinity.affine_transform(new_polygon,
                                                                 [1.0, 0.0, 0.0, 1.0, -patch_x, -patch_y])
-                        if new_polygon.geom_type is 'Polygon':
+                        if new_polygon.geom_type == 'Polygon':
                             new_polygon = MultiPolygon([new_polygon])
                         polygon_list.append(new_polygon)
 
@@ -2003,7 +2003,7 @@ def _get_layer_line(self,
         if layer_name not in self.map_api.non_geometric_line_layers:
             raise ValueError("{} is not a line layer".format(layer_name))
 
-        if layer_name is 'traffic_light':
+        if layer_name == 'traffic_light':
             return None
 
         patch_x = patch_box[0]