Feature/python312

python-sdk/nuscenes/eval/tracking/constants.py

@@ -17,6 +17,7 @@
 MOT_METRIC_MAP = {  # Mapping from motmetrics names to metric names used here.
     'num_frames': '',  # Used in FAF.
     'num_objects': 'gt',  # Used in MOTAR computation.
+    'pred_frequencies': '', # Only needed in background.
     'num_predictions': '',  # Only printed out.
     'num_matches': 'tp',  # Used in MOTAR computation and printed out.
     'motar': 'motar',  # Only used in AMOTA.

python-sdk/nuscenes/eval/tracking/evaluate.py

@@ -5,17 +5,21 @@
 import json
 import os
 import time
-from typing import Tuple, List, Dict, Any
+from typing import Any, Dict, List, Tuple
 
 import numpy as np
-
 from nuscenes import NuScenes
 from nuscenes.eval.common.config import config_factory
-from nuscenes.eval.common.loaders import load_prediction, load_gt, add_center_dist, filter_eval_boxes
+from nuscenes.eval.common.loaders import add_center_dist, filter_eval_boxes, load_gt, load_prediction
 from nuscenes.eval.tracking.algo import TrackingEvaluation
-from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, MOT_METRIC_MAP, LEGACY_METRICS
-from nuscenes.eval.tracking.data_classes import TrackingMetrics, TrackingMetricDataList, TrackingConfig, TrackingBox, \
-    TrackingMetricData
+from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, LEGACY_METRICS, MOT_METRIC_MAP
+from nuscenes.eval.tracking.data_classes import (
+    TrackingBox,
+    TrackingConfig,
+    TrackingMetricData,
+    TrackingMetricDataList,
+    TrackingMetrics,
+)
 from nuscenes.eval.tracking.loaders import create_tracks
 from nuscenes.eval.tracking.render import recall_metric_curve, summary_plot
 from nuscenes.eval.tracking.utils import print_final_metrics

python-sdk/nuscenes/eval/tracking/metrics.py

@@ -7,7 +7,7 @@
 py-motmetrics at:
 https://github.com/cheind/py-motmetrics
 """
-from typing import Any
+from typing import Any, Optional
 
 import numpy as np
 
@@ -109,7 +109,7 @@ def longest_gap_duration(df: DataFrame, obj_frequencies: DataFrame) -> float:
 
 
 def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, num_false_positives: int,
-          num_objects: int, alpha: float = 1.0) -> float:
+          num_objects: int, alpha: float = 1.0, ana: Optional[dict] = None) -> float:
     """
     Initializes a MOTAR class which refers to the modified MOTA metric at https://www.nuscenes.org/tracking.
     Note that we use the measured recall, which is not identical to the hypothetical recall of the
@@ -121,6 +121,7 @@ def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, n
     :param num_false_positives: The number of false positives.
     :param num_objects: The total number of objects of this class in the GT.
     :param alpha: MOTAR weighting factor (previously 0.2).
+    :param ana: something for caching, introduced by motmetrics 1.4.0
     :return: The MOTAR or nan if there are no GT objects.
     """
     recall = num_matches / num_objects

python-sdk/nuscenes/eval/tracking/mot.py

@@ -6,50 +6,74 @@
 
 py-motmetrics at:
 https://github.com/cheind/py-motmetrics
+
+Notes by Michael Hoss:
+For Python 3.10, we need to update the version of py-motmetrics to 1.4.0.
+Then, to keep this code working, we need to change back the types of OId HId to object because they are
+strings in nuscenes-devkit, whereas motmetrics changed these types to float from 1.1.3 to 1.4.0.
 """
 from collections import OrderedDict
 from itertools import count
 
-import motmetrics
 import numpy as np
 import pandas as pd
+from motmetrics import MOTAccumulator
 
+_INDEX_FIELDS = ['FrameId', 'Event']
 
-class MOTAccumulatorCustom(motmetrics.mot.MOTAccumulator):
+class MOTAccumulatorCustom(MOTAccumulator):
+    """This custom class was created by nuscenes-devkit to use a faster implementation of
+    `new_event_dataframe_with_data` under compatibility with motmetrics<=1.1.3.
+    Now that we use motmetrics==1.4.0, we need to use this custom implementation to use
+    objects instead of strings for OId and HId.
+    """
     def __init__(self):
         super().__init__()
 
     @staticmethod
     def new_event_dataframe_with_data(indices, events):
-        """
-        Create a new DataFrame filled with data.
-        This version overwrites the original in MOTAccumulator achieves about 2x speedups.
+        """Create a new DataFrame filled with data.
 
         Params
         ------
-        indices: list
-            list of tuples (frameid, eventid)
-        events: list
-            list of events where each event is a list containing
-            'Type', 'OId', HId', 'D'
+        indices: dict
+            dict of lists with fields 'FrameId' and 'Event'
+        events: dict
+            dict of lists with fields 'Type', 'OId', 'HId', 'D'
         """
-        idx = pd.MultiIndex.from_tuples(indices, names=['FrameId', 'Event'])
-        df = pd.DataFrame(events, index=idx, columns=['Type', 'OId', 'HId', 'D'])
+
+        if len(events) == 0:
+            return MOTAccumulatorCustom.new_event_dataframe()
+
+        raw_type = pd.Categorical(
+            events['Type'],
+            categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'],
+            ordered=False)
+        series = [
+            pd.Series(raw_type, name='Type'),
+            pd.Series(events['OId'], dtype=object, name='OId'),  # OId is string in nuscenes-devkit
+            pd.Series(events['HId'], dtype=object, name='HId'),  # HId is string in nuscenes-devkit
+            pd.Series(events['D'], dtype=float, name='D')
+        ]
+
+        idx = pd.MultiIndex.from_arrays(
+            [indices[field] for field in _INDEX_FIELDS],
+            names=_INDEX_FIELDS)
+        df = pd.concat(series, axis=1)
+        df.index = idx
         return df
 
     @staticmethod
     def new_event_dataframe():
-        """ Create a new DataFrame for event tracking. """
+        """Create a new DataFrame for event tracking."""
         idx = pd.MultiIndex(levels=[[], []], codes=[[], []], names=['FrameId', 'Event'])
-        cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH'])
+        cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'])
         df = pd.DataFrame(
             OrderedDict([
-                ('Type', pd.Series(cats)),  # Type of event. One of FP (false positive), MISS, SWITCH, MATCH
-                ('OId', pd.Series(dtype=object)),
-                # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways.
-                ('HId', pd.Series(dtype=object)),
-                # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways.
-                ('D', pd.Series(dtype=float)),  # Distance or NaN when FP or MISS
+                ('Type', pd.Series(cats)),          # Type of event. One of FP (false positive), MISS, SWITCH, MATCH
+                ('OId', pd.Series(dtype=object)),      # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways.
+                ('HId', pd.Series(dtype=object)),      # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways.
+                ('D', pd.Series(dtype=float)),      # Distance or NaN when FP or MISS
             ]),
             index=idx
         )
@@ -63,8 +87,7 @@ def events(self):
         return self.cached_events_df
 
     @staticmethod
-    def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True,
-                               return_mappings=False):
+    def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, return_mappings=False):
         """Merge dataframes.
 
         Params
@@ -104,24 +127,29 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd
 
             # Update index
             if update_frame_indices:
-                next_frame_id = max(r.index.get_level_values(0).max() + 1,
-                                    r.index.get_level_values(0).unique().shape[0])
+                # pylint: disable=cell-var-from-loop
+                next_frame_id = max(r.index.get_level_values(0).max() + 1, r.index.get_level_values(0).unique().shape[0])
                 if np.isnan(next_frame_id):
                     next_frame_id = 0
-                copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1]))
+                if not copy.index.empty:
+                    copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1]))
                 infos['frame_offset'] = next_frame_id
 
             # Update object / hypothesis ids
             if update_oids:
+                # pylint: disable=cell-var-from-loop
                 oid_map = dict([oid, str(next(new_oid))] for oid in copy['OId'].dropna().unique())
                 copy['OId'] = copy['OId'].map(lambda x: oid_map[x], na_action='ignore')
                 infos['oid_map'] = oid_map
 
             if update_hids:
+                # pylint: disable=cell-var-from-loop
                 hid_map = dict([hid, str(next(new_hid))] for hid in copy['HId'].dropna().unique())
                 copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore')
                 infos['hid_map'] = hid_map
 
+            # Avoid pandas warning. But is this legit/do we need such a column later on again?
+            # copy = copy.dropna(axis=1, how='all')
             r = pd.concat((r, copy))
             mapping_infos.append(infos)
 

python-sdk/nuscenes/eval/tracking/utils.py

@@ -3,7 +3,7 @@
 
 import unittest
 import warnings
-from typing import Optional, Dict
+from typing import Dict, Optional
 
 import numpy as np
 
@@ -14,8 +14,15 @@
     raise unittest.SkipTest('Skipping test as motmetrics was not found!')
 
 from nuscenes.eval.tracking.data_classes import TrackingMetrics
-from nuscenes.eval.tracking.metrics import motar, mota_custom, motp_custom, faf, track_initialization_duration, \
-    longest_gap_duration, num_fragmentations_custom
+from nuscenes.eval.tracking.metrics import (
+    faf,
+    longest_gap_duration,
+    mota_custom,
+    motar,
+    motp_custom,
+    num_fragmentations_custom,
+    track_initialization_duration,
+)
 
 
 def category_to_tracking_name(category_name: str) -> Optional[str]:
@@ -148,8 +155,8 @@ def create_motmetrics() -> MetricsHost:
     # Register standard metrics.
     fields = [
         'num_frames', 'obj_frequencies', 'num_matches', 'num_switches', 'num_false_positives', 'num_misses',
-        'num_detections', 'num_objects', 'num_predictions', 'mostly_tracked', 'mostly_lost', 'num_fragmentations',
-        'motp', 'mota', 'precision', 'recall', 'track_ratios'
+        'num_detections', 'num_objects', 'pred_frequencies', 'num_predictions', 'mostly_tracked', 'mostly_lost',
+        'num_fragmentations', 'motp', 'mota', 'precision', 'recall', 'track_ratios'
     ]
     for field in fields:
         mh.register(getattr(motmetrics.metrics, field), formatter='{:d}'.format)

python-sdk/nuscenes/nuscenes.py

@@ -1023,9 +1023,9 @@ def render_pointcloud_in_image(self,
         if ax is None:
             fig, ax = plt.subplots(1, 1, figsize=(9, 16))
             if lidarseg_preds_bin_path:
-                fig.canvas.set_window_title(sample_token + '(predictions)')
+                fig.canvas.manager.set_window_title(sample_token + '(predictions)')
             else:
-                fig.canvas.set_window_title(sample_token)
+                fig.canvas.manager.set_window_title(sample_token)
         else:  # Set title on if rendering as part of render_sample.
             ax.set_title(camera_channel)
         ax.imshow(im)

setup/requirements/requirements_base.txt

@@ -1,12 +1,12 @@
 cachetools
 descartes
 fire
-matplotlib<3.6.0
-numpy>=1.22.0
+matplotlib
+numpy>=1.22.0,<2.0
 opencv-python>=4.5.4.58
 Pillow>6.2.1
 pyquaternion>=0.9.5
 scikit-learn
 scipy
-Shapely<2.0.0
+Shapely~=2.0.3
 tqdm

setup/requirements/requirements_tracking.txt

@@ -1,2 +1,2 @@
-motmetrics<=1.1.3
+motmetrics==1.4.0
 pandas>=0.24