Update files for realsense deployment

Author: roshambo919

avstack/geometry/datastructs.py

@@ -7,7 +7,13 @@
 
 from . import transformations as tforms
 from .base import q_mult_vec
-from .refchoc import ReferenceDecoder, ReferenceFrame, PassiveReferenceFrame, Rotation, Vector
+from .refchoc import (
+    PassiveReferenceFrame,
+    ReferenceDecoder,
+    ReferenceFrame,
+    Rotation,
+    Vector,
+)
 
 
 class VectorDecoder(json.JSONDecoder):
@@ -71,7 +77,9 @@ def object_hook(json_object):
 
 
 class Position(Vector):
-    def _pull_from_reference(self, reference: Union[ReferenceFrame, PassiveReferenceFrame]):
+    def _pull_from_reference(
+        self, reference: Union[ReferenceFrame, PassiveReferenceFrame]
+    ):
         return reference.x
 
     @property
@@ -84,7 +92,9 @@ def factory():
 
 
 class Velocity(Vector):
-    def _pull_from_reference(self, reference: Union[ReferenceFrame, PassiveReferenceFrame]):
+    def _pull_from_reference(
+        self, reference: Union[ReferenceFrame, PassiveReferenceFrame]
+    ):
         return reference.v
 
     @staticmethod
@@ -93,7 +103,9 @@ def factory():
 
 
 class Acceleration(Vector):
-    def _pull_from_reference(self, reference: Union[ReferenceFrame, PassiveReferenceFrame]):
+    def _pull_from_reference(
+        self, reference: Union[ReferenceFrame, PassiveReferenceFrame]
+    ):
         return reference.acc
 
     @staticmethod
@@ -102,7 +114,9 @@ def factory():
 
 
 class Attitude(Rotation):
-    def _pull_from_reference(self, reference: Union[ReferenceFrame, PassiveReferenceFrame]):
+    def _pull_from_reference(
+        self, reference: Union[ReferenceFrame, PassiveReferenceFrame]
+    ):
         return reference.q
 
     @staticmethod
@@ -111,7 +125,9 @@ def factory():
 
 
 class AngularVelocity(Rotation):
-    def _pull_from_reference(self, reference: Union[ReferenceFrame, PassiveReferenceFrame]):
+    def _pull_from_reference(
+        self, reference: Union[ReferenceFrame, PassiveReferenceFrame]
+    ):
         return reference.ang
 
     @staticmethod

avstack/modules/perception/base.py

@@ -65,7 +65,7 @@ def map_checkpoint_to_latest(mm_root, checkpoint_file):
                 chk_path = f.readlines()[0].rstrip()
             return chk_path
         else:
-            return None
+            return ""
 
     def parse_mm_model(self):
         raise NotImplementedError("Implement this in the subclass.")

avstack/modules/perception/object2dfv.py

@@ -176,9 +176,9 @@ def parse_mm_model_from_checkpoint(model, dataset, epoch):
                 checkpoint_file = "checkpoints/coco/rtmdet_m_8xb32-300e_coco_20220719_112220-229f527c.pth"
             elif dataset == "rccars-oneclass":
                 threshold = 0.5
-                config_file = "configs/rccars/rtmdet_m_8xb32-300e_rccars-oneclass.py"
+                config_file = "work_dirs/rccars/rtmdet_m_8xb32-300e_rccars-oneclass.py"
                 checkpoint_file = (
-                    f"work_dirs/rtmdet_m_8xb32-300e_rccars-oneclass/{epoch_str}.pth"
+                    f"work_dirs/rccars/rtmdet_m_8xb32-300e_rccars-oneclass.pth"
                 )
             else:
                 raise NotImplementedError(f"{model}, {dataset} not compatible yet")

deployment/mmdeploy/test_deployment_fps.py

@@ -1,15 +1,14 @@
-from mmdeploy_runtime import Detector
+import argparse
 import os
-import cv2
 import time
-import numpy as np
-import argparse
 
-
-from avstack.geometry import GlobalOrigin3D
+import cv2
+import numpy as np
 from avstack.calibration import CameraCalibration
-from avstack.sensors import ImageData
+from avstack.geometry import GlobalOrigin3D
 from avstack.modules.perception.object2dfv import MMDetObjectDetector2D
+from avstack.sensors import ImageData
+from mmdeploy_runtime import Detector
 
 
 def run_with_timing(detector, img, n_inferences):
@@ -19,17 +18,13 @@ def run_with_timing(detector, img, n_inferences):
         _ = detector(img)
         t2 = time.time()
         if i > 1:  # just in case first two times are slow...
-            timing.append((t2 - t1)*1000)
-    print(f'Timing Results:\n   {np.mean(timing):4.2f} +/- {np.std(timing):4.2f} ms')
+            timing.append((t2 - t1) * 1000)
+    print(f"Timing Results:\n   {np.mean(timing):4.2f} +/- {np.std(timing):4.2f} ms")
 
 
 def main_mmdeploy(model_path, img, n_inferences=50):
-    detector = Detector(
-        model_path=model_path,
-        device_name='cuda',
-        device_id=0
-    )
-    print('Running mmdeploy:')
+    detector = Detector(model_path=model_path, device_name="cuda", device_id=0)
+    print("Running mmdeploy:")
     run_with_timing(detector, img, n_inferences)
 
 
@@ -40,28 +35,34 @@ def main_mmdet(dataset, model, img, n_inferences=50):
         gpu=0,
     )
     cam_calib = CameraCalibration(
-        reference=GlobalOrigin3D,
-        P=np.random.rand(3,4),
-        img_shape=img.shape
+        reference=GlobalOrigin3D, P=np.random.rand(3, 4), img_shape=img.shape
     )
     img = ImageData(timestamp=0, frame=0, data=img, calibration=cam_calib, source_ID=0)
-    print('Running mmdetection')
+    print("Running mmdetection")
     run_with_timing(detector, img, n_inferences)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('--dataset', default="coco", type=str, help="Name of the dataset to use")
-    parser.add_argument('--model', default="cascade_rcnn", type=str, help="Name of the model to use")
-    parser.add_argument('--runtime', default='tensorrt', type=str, help="NN runtime to use for deploy")
+    parser.add_argument(
+        "--dataset", default="coco", type=str, help="Name of the dataset to use"
+    )
+    parser.add_argument(
+        "--model", default="cascade_rcnn", type=str, help="Name of the model to use"
+    )
+    parser.add_argument(
+        "--runtime", default="tensorrt", type=str, help="NN runtime to use for deploy"
+    )
     args = parser.parse_args()
-    
-    mmdep_model_path = os.path.join('mmdeploy_models', f'{args.model}_{args.dataset}_{args.runtime}')
+
+    mmdep_model_path = os.path.join(
+        "mmdeploy_models", f"{args.model}_{args.dataset}_{args.runtime}"
+    )
     if not os.path.exists(mmdep_model_path):
         raise FileNotFoundError("Cannot find model for mmdeploy")
 
     MMDET_BASE = "../../third_party/mmdetection"
     img = cv2.imread(os.path.join(MMDET_BASE, "demo", "demo.jpg"))
     n_inferences = 20
     main_mmdeploy(mmdep_model_path, img, n_inferences=n_inferences)
-    main_mmdet(args.dataset, args.model, img, n_inferences=n_inferences)
+    main_mmdet(args.dataset, args.model, img, n_inferences=n_inferences)

deployment/mmdeploy/test_model_deployment.py

@@ -1,24 +1,21 @@
-from mmdeploy_runtime import Detector
+import argparse
 import os
+
 import cv2
-import argparse
+from mmdeploy_runtime import Detector
 
 
 def main(model_path, img):
-    detector = Detector(
-        model_path=model_path,
-        device_name='cuda',
-        device_id=0
-    )
+    detector = Detector(model_path=model_path, device_name="cuda", device_id=0)
     bboxes, labels, _ = detector(img)
     print(bboxes, labels)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('model_path', type=str, help='Path to converted model')
+    parser.add_argument("model_path", type=str, help="Path to converted model")
     args = parser.parse_args()
-    
+
     MMDET_BASE = "../../third_party/mmdetection"
     img = cv2.imread(os.path.join(MMDET_BASE, "demo", "demo.jpg"))
-    main(args.model_path, img)
+    main(args.model_path, img)

models/config/mmseg/unet_fov.py

@@ -1,19 +1,19 @@
 _base_ = [
-    "../../../third_party/mmsegmentation/configs/" + '_base_/datasets/fov_bev.py',
-    "../../../third_party/mmsegmentation/configs/" + '_base_/default_runtime.py',
-    "../../../third_party/mmsegmentation/configs/" + '_base_/schedules/schedule_5k.py'
+    "../../../third_party/mmsegmentation/configs/" + "_base_/datasets/fov_bev.py",
+    "../../../third_party/mmsegmentation/configs/" + "_base_/default_runtime.py",
+    "../../../third_party/mmsegmentation/configs/" + "_base_/schedules/schedule_5k.py",
 ]
 
 # model settings
 crop_size = (256, 256)
-norm_cfg = dict(type='SyncBN', requires_grad=True)
-data_preprocessor = dict(type='SegDataPreProcessor')
+norm_cfg = dict(type="SyncBN", requires_grad=True)
+data_preprocessor = dict(type="SegDataPreProcessor")
 model = dict(
-    type='EncoderDecoder',
+    type="EncoderDecoder",
     data_preprocessor=data_preprocessor,
     pretrained=None,
     backbone=dict(
-        type='UNet',
+        type="UNet",
         in_channels=1,
         base_channels=64,
         num_stages=5,
@@ -26,11 +26,12 @@
         with_cp=False,
         conv_cfg=None,
         norm_cfg=norm_cfg,
-        act_cfg=dict(type='ReLU'),
-        upsample_cfg=dict(type='InterpConv'),
-        norm_eval=False),
+        act_cfg=dict(type="ReLU"),
+        upsample_cfg=dict(type="InterpConv"),
+        norm_eval=False,
+    ),
     decode_head=dict(
-        type='FCNHead',
+        type="FCNHead",
         in_channels=64,
         in_index=4,
         channels=64,
@@ -40,10 +41,10 @@
         num_classes=2,
         norm_cfg=norm_cfg,
         align_corners=False,
-        loss_decode=dict(
-            type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)),
+        loss_decode=dict(type="CrossEntropyLoss", use_sigmoid=False, loss_weight=1.0),
+    ),
     auxiliary_head=dict(
-        type='FCNHead',
+        type="FCNHead",
         in_channels=128,
         in_index=3,
         channels=64,
@@ -53,14 +54,15 @@
         num_classes=2,
         norm_cfg=norm_cfg,
         align_corners=False,
-        loss_decode=dict(
-            type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)),
+        loss_decode=dict(type="CrossEntropyLoss", use_sigmoid=False, loss_weight=0.4),
+    ),
     # model training and testing settings
     train_cfg=dict(),
-    test_cfg=dict(mode='slide', crop_size=crop_size))
+    test_cfg=dict(mode="slide", crop_size=crop_size),
+)
 
 
 # scheduling configuration
 # train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=36, val_interval=36)
 # val_cfg = dict(type='ValLoop')
-# test_cfg = dict(type='TestLoop')
+# test_cfg = dict(type='TestLoop')

models/download_mmdet_models.sh

@@ -96,6 +96,9 @@ download_models "coco" "v2.0" "cascade_rcnn" "$COCO_CMRCNN"
 # CARLA_CRCNN_INF="cascade-rcnn_r50_fpn_1x_carla_infrastructure"
 # download_custom_models "carla" "carla" "$CARLA_CRCNN_INF"
 
+RCCARS_RTMDET="rtmdet_m_8xb32-300e_rccars-oneclass"
+download_custom_models "rccars" "rccars" "$RCCARS_RTMDET"
+
 
 echo "Adding symbolic link to mmdet directory"
 ln -sfnT $(realpath "$MMDET_CKPT") "$THISDIR/../third_party/mmdetection/checkpoints"

setup.py

@@ -2,5 +2,6 @@
 
 import setuptools
 
+
 if __name__ == "__main__":
-    setuptools.setup()
+    setuptools.setup()