diff --git a/python-sdk/nuscenes/eval/tracking/constants.py b/python-sdk/nuscenes/eval/tracking/constants.py index c90fc63d..5fde8db4 100644 --- a/python-sdk/nuscenes/eval/tracking/constants.py +++ b/python-sdk/nuscenes/eval/tracking/constants.py @@ -17,6 +17,7 @@ MOT_METRIC_MAP = { # Mapping from motmetrics names to metric names used here. 'num_frames': '', # Used in FAF. 'num_objects': 'gt', # Used in MOTAR computation. + 'pred_frequencies': '', # Only needed in background. 'num_predictions': '', # Only printed out. 'num_matches': 'tp', # Used in MOTAR computation and printed out. 'motar': 'motar', # Only used in AMOTA. diff --git a/python-sdk/nuscenes/eval/tracking/evaluate.py b/python-sdk/nuscenes/eval/tracking/evaluate.py index 8cc1d456..941847b9 100644 --- a/python-sdk/nuscenes/eval/tracking/evaluate.py +++ b/python-sdk/nuscenes/eval/tracking/evaluate.py @@ -5,17 +5,21 @@ import json import os import time -from typing import Tuple, List, Dict, Any +from typing import Any, Dict, List, Tuple import numpy as np - from nuscenes import NuScenes from nuscenes.eval.common.config import config_factory -from nuscenes.eval.common.loaders import load_prediction, load_gt, add_center_dist, filter_eval_boxes +from nuscenes.eval.common.loaders import add_center_dist, filter_eval_boxes, load_gt, load_prediction from nuscenes.eval.tracking.algo import TrackingEvaluation -from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, MOT_METRIC_MAP, LEGACY_METRICS -from nuscenes.eval.tracking.data_classes import TrackingMetrics, TrackingMetricDataList, TrackingConfig, TrackingBox, \ - TrackingMetricData +from nuscenes.eval.tracking.constants import AVG_METRIC_MAP, LEGACY_METRICS, MOT_METRIC_MAP +from nuscenes.eval.tracking.data_classes import ( + TrackingBox, + TrackingConfig, + TrackingMetricData, + TrackingMetricDataList, + TrackingMetrics, +) from nuscenes.eval.tracking.loaders import create_tracks from nuscenes.eval.tracking.render import recall_metric_curve, summary_plot from nuscenes.eval.tracking.utils import print_final_metrics diff --git a/python-sdk/nuscenes/eval/tracking/metrics.py b/python-sdk/nuscenes/eval/tracking/metrics.py index fad93305..b4215b18 100644 --- a/python-sdk/nuscenes/eval/tracking/metrics.py +++ b/python-sdk/nuscenes/eval/tracking/metrics.py @@ -7,7 +7,7 @@ py-motmetrics at: https://github.com/cheind/py-motmetrics """ -from typing import Any +from typing import Any, Optional import numpy as np @@ -109,7 +109,7 @@ def longest_gap_duration(df: DataFrame, obj_frequencies: DataFrame) -> float: def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, num_false_positives: int, - num_objects: int, alpha: float = 1.0) -> float: + num_objects: int, alpha: float = 1.0, ana: Optional[dict] = None) -> float: """ Initializes a MOTAR class which refers to the modified MOTA metric at https://www.nuscenes.org/tracking. Note that we use the measured recall, which is not identical to the hypothetical recall of the @@ -121,6 +121,7 @@ def motar(df: DataFrame, num_matches: int, num_misses: int, num_switches: int, n :param num_false_positives: The number of false positives. :param num_objects: The total number of objects of this class in the GT. :param alpha: MOTAR weighting factor (previously 0.2). + :param ana: something for caching, introduced by motmetrics 1.4.0 :return: The MOTAR or nan if there are no GT objects. """ recall = num_matches / num_objects diff --git a/python-sdk/nuscenes/eval/tracking/mot.py b/python-sdk/nuscenes/eval/tracking/mot.py index aa18421b..53b813f3 100644 --- a/python-sdk/nuscenes/eval/tracking/mot.py +++ b/python-sdk/nuscenes/eval/tracking/mot.py @@ -6,50 +6,74 @@ py-motmetrics at: https://github.com/cheind/py-motmetrics + +Notes by Michael Hoss: +For Python 3.10, we need to update the version of py-motmetrics to 1.4.0. +Then, to keep this code working, we need to change back the types of OId HId to object because they are +strings in nuscenes-devkit, whereas motmetrics changed these types to float from 1.1.3 to 1.4.0. """ from collections import OrderedDict from itertools import count -import motmetrics import numpy as np import pandas as pd +from motmetrics import MOTAccumulator +_INDEX_FIELDS = ['FrameId', 'Event'] -class MOTAccumulatorCustom(motmetrics.mot.MOTAccumulator): +class MOTAccumulatorCustom(MOTAccumulator): + """This custom class was created by nuscenes-devkit to use a faster implementation of + `new_event_dataframe_with_data` under compatibility with motmetrics<=1.1.3. + Now that we use motmetrics==1.4.0, we need to use this custom implementation to use + objects instead of strings for OId and HId. + """ def __init__(self): super().__init__() @staticmethod def new_event_dataframe_with_data(indices, events): - """ - Create a new DataFrame filled with data. - This version overwrites the original in MOTAccumulator achieves about 2x speedups. + """Create a new DataFrame filled with data. Params ------ - indices: list - list of tuples (frameid, eventid) - events: list - list of events where each event is a list containing - 'Type', 'OId', HId', 'D' + indices: dict + dict of lists with fields 'FrameId' and 'Event' + events: dict + dict of lists with fields 'Type', 'OId', 'HId', 'D' """ - idx = pd.MultiIndex.from_tuples(indices, names=['FrameId', 'Event']) - df = pd.DataFrame(events, index=idx, columns=['Type', 'OId', 'HId', 'D']) + + if len(events) == 0: + return MOTAccumulatorCustom.new_event_dataframe() + + raw_type = pd.Categorical( + events['Type'], + categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE'], + ordered=False) + series = [ + pd.Series(raw_type, name='Type'), + pd.Series(events['OId'], dtype=object, name='OId'), # OId is string in nuscenes-devkit + pd.Series(events['HId'], dtype=object, name='HId'), # HId is string in nuscenes-devkit + pd.Series(events['D'], dtype=float, name='D') + ] + + idx = pd.MultiIndex.from_arrays( + [indices[field] for field in _INDEX_FIELDS], + names=_INDEX_FIELDS) + df = pd.concat(series, axis=1) + df.index = idx return df @staticmethod def new_event_dataframe(): - """ Create a new DataFrame for event tracking. """ + """Create a new DataFrame for event tracking.""" idx = pd.MultiIndex(levels=[[], []], codes=[[], []], names=['FrameId', 'Event']) - cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH']) + cats = pd.Categorical([], categories=['RAW', 'FP', 'MISS', 'SWITCH', 'MATCH', 'TRANSFER', 'ASCEND', 'MIGRATE']) df = pd.DataFrame( OrderedDict([ - ('Type', pd.Series(cats)), # Type of event. One of FP (false positive), MISS, SWITCH, MATCH - ('OId', pd.Series(dtype=object)), - # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways. - ('HId', pd.Series(dtype=object)), - # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways. - ('D', pd.Series(dtype=float)), # Distance or NaN when FP or MISS + ('Type', pd.Series(cats)), # Type of event. One of FP (false positive), MISS, SWITCH, MATCH + ('OId', pd.Series(dtype=object)), # Object ID or -1 if FP. Using float as missing values will be converted to NaN anyways. + ('HId', pd.Series(dtype=object)), # Hypothesis ID or NaN if MISS. Using float as missing values will be converted to NaN anyways. + ('D', pd.Series(dtype=float)), # Distance or NaN when FP or MISS ]), index=idx ) @@ -63,8 +87,7 @@ def events(self): return self.cached_events_df @staticmethod - def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, - return_mappings=False): + def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, update_hids=True, return_mappings=False): """Merge dataframes. Params @@ -104,24 +127,29 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd # Update index if update_frame_indices: - next_frame_id = max(r.index.get_level_values(0).max() + 1, - r.index.get_level_values(0).unique().shape[0]) + # pylint: disable=cell-var-from-loop + next_frame_id = max(r.index.get_level_values(0).max() + 1, r.index.get_level_values(0).unique().shape[0]) if np.isnan(next_frame_id): next_frame_id = 0 - copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1])) + if not copy.index.empty: + copy.index = copy.index.map(lambda x: (x[0] + next_frame_id, x[1])) infos['frame_offset'] = next_frame_id # Update object / hypothesis ids if update_oids: + # pylint: disable=cell-var-from-loop oid_map = dict([oid, str(next(new_oid))] for oid in copy['OId'].dropna().unique()) copy['OId'] = copy['OId'].map(lambda x: oid_map[x], na_action='ignore') infos['oid_map'] = oid_map if update_hids: + # pylint: disable=cell-var-from-loop hid_map = dict([hid, str(next(new_hid))] for hid in copy['HId'].dropna().unique()) copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore') infos['hid_map'] = hid_map + # Avoid pandas warning. But is this legit/do we need such a column later on again? + # copy = copy.dropna(axis=1, how='all') r = pd.concat((r, copy)) mapping_infos.append(infos) diff --git a/python-sdk/nuscenes/eval/tracking/utils.py b/python-sdk/nuscenes/eval/tracking/utils.py index da078f29..3bbd59e9 100644 --- a/python-sdk/nuscenes/eval/tracking/utils.py +++ b/python-sdk/nuscenes/eval/tracking/utils.py @@ -3,7 +3,7 @@ import unittest import warnings -from typing import Optional, Dict +from typing import Dict, Optional import numpy as np @@ -14,8 +14,15 @@ raise unittest.SkipTest('Skipping test as motmetrics was not found!') from nuscenes.eval.tracking.data_classes import TrackingMetrics -from nuscenes.eval.tracking.metrics import motar, mota_custom, motp_custom, faf, track_initialization_duration, \ - longest_gap_duration, num_fragmentations_custom +from nuscenes.eval.tracking.metrics import ( + faf, + longest_gap_duration, + mota_custom, + motar, + motp_custom, + num_fragmentations_custom, + track_initialization_duration, +) def category_to_tracking_name(category_name: str) -> Optional[str]: @@ -148,8 +155,8 @@ def create_motmetrics() -> MetricsHost: # Register standard metrics. fields = [ 'num_frames', 'obj_frequencies', 'num_matches', 'num_switches', 'num_false_positives', 'num_misses', - 'num_detections', 'num_objects', 'num_predictions', 'mostly_tracked', 'mostly_lost', 'num_fragmentations', - 'motp', 'mota', 'precision', 'recall', 'track_ratios' + 'num_detections', 'num_objects', 'pred_frequencies', 'num_predictions', 'mostly_tracked', 'mostly_lost', + 'num_fragmentations', 'motp', 'mota', 'precision', 'recall', 'track_ratios' ] for field in fields: mh.register(getattr(motmetrics.metrics, field), formatter='{:d}'.format) diff --git a/python-sdk/nuscenes/nuscenes.py b/python-sdk/nuscenes/nuscenes.py index c36c023c..9bd61d1b 100644 --- a/python-sdk/nuscenes/nuscenes.py +++ b/python-sdk/nuscenes/nuscenes.py @@ -1023,9 +1023,9 @@ def render_pointcloud_in_image(self, if ax is None: fig, ax = plt.subplots(1, 1, figsize=(9, 16)) if lidarseg_preds_bin_path: - fig.canvas.set_window_title(sample_token + '(predictions)') + fig.canvas.manager.set_window_title(sample_token + '(predictions)') else: - fig.canvas.set_window_title(sample_token) + fig.canvas.manager.set_window_title(sample_token) else: # Set title on if rendering as part of render_sample. ax.set_title(camera_channel) ax.imshow(im) diff --git a/setup/requirements/requirements_base.txt b/setup/requirements/requirements_base.txt index aa8eff2f..85b6c6a6 100644 --- a/setup/requirements/requirements_base.txt +++ b/setup/requirements/requirements_base.txt @@ -1,12 +1,12 @@ cachetools descartes fire -matplotlib<3.6.0 -numpy>=1.22.0 +matplotlib +numpy>=1.22.0,<2.0 opencv-python>=4.5.4.58 Pillow>6.2.1 pyquaternion>=0.9.5 scikit-learn scipy -Shapely<2.0.0 +Shapely~=2.0.3 tqdm diff --git a/setup/requirements/requirements_tracking.txt b/setup/requirements/requirements_tracking.txt index abcc4d75..50ff8142 100644 --- a/setup/requirements/requirements_tracking.txt +++ b/setup/requirements/requirements_tracking.txt @@ -1,2 +1,2 @@ -motmetrics<=1.1.3 +motmetrics==1.4.0 pandas>=0.24