validation_dataset: Creation

.devcontainer/build_and_push_images.sh

@@ -1,2 +1,2 @@
-docker buildx build -t omavteam/radarmeetsvision:blearn-latest -f blearn/Dockerfile --push .
 docker buildx build -t omavteam/radarmeetsvision:latest -f desktop/Dockerfile --push .
+docker buildx build -t omavteam/radarmeetsvision:blearn-latest -f blearn/Dockerfile --push .

.devcontainer/desktop/Dockerfile

@@ -33,24 +33,9 @@ RUN groupadd -r asl && \
 COPY install_ros_noetic_and_deps.sh /tmp/install_ros_noetic_and_deps.sh
 RUN bash /tmp/install_ros_noetic_and_deps.sh
 
-# Spinnaker: Install spinnaker SDK for flir cameras
-COPY spinnaker/spinnaker-3.2.0.62-amd64 /tmp/spinnaker-3.2.0.62-amd64
-WORKDIR /tmp/spinnaker-3.2.0.62-amd64
-# Figured out by running: RUN echo $(debconf-show libgentl)
-RUN echo "libspinnaker libspinnaker/accepted-flir-eula boolean true" | debconf-set-selections
-RUN echo "libspinnaker libspinnaker/present-flir-eula boolean true" | debconf-set-selections
-RUN /tmp/spinnaker-3.2.0.62-amd64/install_spinnaker.sh
-RUN /tmp/spinnaker-3.2.0.62-amd64/configure_usbfs.sh
-
 # Git-prompt: Source
 RUN echo 'if [ -f "$HOME/.bash-git-prompt/gitprompt.sh" ]; then GIT_PROMPT_ONLY_IN_REPO=1; source "$HOME/.bash-git-prompt/gitprompt.sh"; fi' >> /home/asl/.bash_aliases
 
-# TI: Download and install UniFlash
-WORKDIR /tmp
-RUN wget https://dr-download.ti.com/software-development/software-programming-tool/MD-QeJBJLj8gq/8.7.0/uniflash_sl.8.7.0.4818.run && \
-    chmod +x uniflash_sl.8.7.0.4818.run && \
-    ./uniflash_sl.8.7.0.4818.run --mode unattended --prefix /home/asl/ti/uniflash --debuglevel 0
-
 # Clean up
 RUN rm -rf /tmp/* /var/lib/apt/lists/* /var/tmp/* /var/cache/apt/archives/*
 

.devcontainer/requirements.txt

@@ -1 +1,2 @@
 colorlog
+laspy

radarmeetsvision/interface.py

@@ -22,6 +22,7 @@ def __init__(self):
         self.batch_size = None
         self.criterion = None
         self.depth_min_max = None
+        self.depth_prior_dir = None
         self.device = get_device()
         self.encoder = None
         self.lr = None
@@ -39,7 +40,11 @@ def reset_previous_best(self):
         return {'d1': 0, 'd2': 0, 'd3': 0, 'abs_rel': 100, 'sq_rel': 100, 'rmse': 100, 'rmse_log': 100, 'log10': 100, 'silog': 100,
         'average_depth': 0.0}
 
-    def set_use_depth_prior(self, use):
+    def set_use_depth_prior(self, use, depth_prior_dir=None):
+        if depth_prior_dir is not None and Path(depth_prior_dir).is_dir():
+            logger.info(f"Overwriting depth prior dir: {depth_prior_dir}")
+            self.depth_prior_dir = Path(depth_prior_dir)
+
         self.use_depth_prior = use
 
     def set_encoder(self, encoder):
@@ -87,6 +92,8 @@ def load_model(self, pretrained_from=None):
             self.model = get_model(pretrained_from, self.use_depth_prior, self.encoder, self.max_depth, output_channels=self.output_channels)
             self.model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(self.model)
             self.model.to(self.device)
+        else:
+            logger.error(f"One or multiple undefined: {self.encoder}, {self.max_depth}, {self.output_channels}, {self.use_depth_prior}")
 
         return self.model
 
@@ -123,7 +130,7 @@ def get_dataset_loader(self, task, datasets_dir, dataset_list, index_list=None):
             if index_list != None:
                 index_min, index_max = index_list[i][0], index_list[i][1]
 
-            dataset = BlearnDataset(dataset_dir, task, self.size, index_min, index_max)
+            dataset = BlearnDataset(dataset_dir, task, self.size, index_min, index_max, self.depth_prior_dir)
 
             if len(dataset) > 0:
                 datasets.append(dataset)
@@ -138,6 +145,9 @@ def get_dataset_loader(self, task, datasets_dir, dataset_list, index_list=None):
 
         return loader, dataset
 
+    def get_single_dataset_loader(self, dataset_dir, min_index=0, max_index=-1):
+        dataset = BlearnDataset(dataset_dir, 'all', self.size, min_index, max_index, self.depth_prior_dir)
+        return DataLoader(dataset, batch_size=self.batch_size, pin_memory=True, drop_last=True)
 
     def update_best_result(self, results, nsamples):
         if nsamples:
@@ -155,18 +165,23 @@ def update_best_result(self, results, nsamples):
 
     def prepare_sample(self, sample, random_flip=False):
         image = sample['image'].to(self.device)
-        depth_target = sample['depth'].to(self.device)
-        depth_prior = sample['depth_prior'].to(self.device)
-        valid_mask = sample['valid_mask'].to(self.device)
-
-        mask = (valid_mask == 1) & (depth_target >= self.min_depth) & (depth_target <= self.max_depth)
 
+        depth_prior = sample['depth_prior'].to(self.device)
         if self.use_depth_prior:
             depth_prior = depth_prior.unsqueeze(1)
             image = torch.cat((image, depth_prior), axis=1)
 
-        if random_flip:
-            image, depth_target, valid_mask = randomly_flip(image, depth_target, mask)
+        if 'depth' in sample.keys() and 'valid_mask' in sample.keys():
+            depth_target = sample['depth'].to(self.device)
+            valid_mask = sample['valid_mask'].to(self.device)
+            mask = (valid_mask == 1) & (depth_target >= self.min_depth) & (depth_target <= self.max_depth)
+
+            if random_flip:
+                image, depth_target, mask = randomly_flip(image, depth_target, mask)
+
+        else:
+            depth_target, mask = None, None
+
 
         return image, depth_prior, depth_target, mask
 
@@ -200,26 +215,31 @@ def train_epoch(self, epoch, train_loader):
                 logger.info('Iter: {}/{}, LR: {:.7f}, Loss: {:.3f}'.format(i, len(train_loader), self.optimizer.param_groups[0]['lr'], total_loss/(i + 1.0)))
 
 
-    def validate_epoch(self, epoch, val_loader):
+    def validate_epoch(self, epoch, val_loader, iteration_callback=None):
         self.model.eval()
 
         self.results, self.results_per_sample, nsamples = get_empty_results(self.device)
         for i, sample in enumerate(val_loader):
             image, _, depth_target, mask = self.prepare_sample(sample, random_flip=False)
 
             # TODO: Maybe not hardcode 10 here?
-            if mask.sum() > 10:
+            if mask is None or mask.sum() > 10:
                 with torch.no_grad():
                     prediction = self.model(image)
                     prediction = interpolate_shape(prediction, depth_target)
                     depth_prediction = get_depth_from_prediction(prediction, image)
 
-                    current_results = eval_depth(depth_prediction[mask], depth_target[mask])
-                    if current_results is not None:
-                        for k in self.results.keys():
-                            self.results[k] += current_results[k]
-                            self.results_per_sample[k].append(current_results[k])
-                        nsamples += 1
+                    # TODO: Expand on this interface
+                    if iteration_callback is not None:
+                        iteration_callback(i, depth_prediction)
+
+                    if mask is not None:
+                        current_results = eval_depth(depth_prediction[mask], depth_target[mask])
+                        if current_results is not None:
+                            for k in self.results.keys():
+                                self.results[k] += current_results[k]
+                                self.results_per_sample[k].append(current_results[k])
+                            nsamples += 1
 
                 if i % 10 == 0:
                     abs_rel = (self.results["abs_rel"]/nsamples).item()

radarmeetsvision/metric_depth_network/__init__.py

@@ -4,7 +4,7 @@
 #
 ######################################################################
 
-from .common import interpolate_shape, get_depth_from_prediction
+from .common import interpolate_shape, get_depth_from_prediction, get_confidence_from_prediction
 from .dataset import BlearnDataset
 from .depth_anything_v2 import get_model, DepthAnythingV2
 from .util import *

radarmeetsvision/metric_depth_network/common.py

@@ -28,15 +28,16 @@ def get_confidence_from_prediction(prediction):
     return prediction[:, 1:, :, :]
 
 def interpolate_shape(prediction, target, mode='bilinear'):
-    if len(target.shape) > 2:
-        target = target.squeeze()
+    if target is not None:
+        if len(target.shape) > 2:
+            target = target.squeeze()
 
-    interp_shape = (target.shape[0], target.shape[1])
+        interp_shape = (target.shape[0], target.shape[1])
 
-    if mode == 'nearest':
-        if len(prediction.shape) < 4:
-            prediction = prediction.unsqueeze(0)
-        prediction = F.interpolate(prediction, interp_shape, mode=mode)
-    else:
-        prediction = F.interpolate(prediction, interp_shape, mode=mode, align_corners=True)
+        if mode == 'nearest':
+            if len(prediction.shape) < 4:
+                prediction = prediction.unsqueeze(0)
+            prediction = F.interpolate(prediction, interp_shape, mode=mode)
+        else:
+            prediction = F.interpolate(prediction, interp_shape, mode=mode, align_corners=True)
     return prediction

radarmeetsvision/metric_depth_network/dataset/blearndataset.py

@@ -18,12 +18,12 @@
 from torchvision.transforms import Compose
 from scipy.spatial import cKDTree
 
-from .transform import Resize, PrepareForNet, Crop
+from .transform import Resize, PrepareForNet, Crop, NormalizeImage
 
 logger = logging.getLogger(__name__)
 
 class BlearnDataset(Dataset):
-    def __init__(self, dataset_dir, mode, size, index_min=0, index_max=-1):
+    def __init__(self, dataset_dir, mode, size, index_min=0, index_max=-1, depth_prior_dir=None):
         self.mode = mode
         self.size = size
 
@@ -37,7 +37,7 @@ def __init__(self, dataset_dir, mode, size, index_min=0, index_max=-1):
                 resize_method='lower_bound',
                 image_interpolation_method=cv2.INTER_CUBIC,
             ),
-            # TODO: Think if we want to add this here, again, evaluate, where do these numbers come from?
+            # TODO: Check these numbers again
             # NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
             PrepareForNet(),
         ] + ([Crop(size[0])] if 'train' in self.mode else []))
@@ -59,6 +59,9 @@ def __init__(self, dataset_dir, mode, size, index_min=0, index_max=-1):
         self.depth_min, self.depth_max, self.depth_range = self.get_depth_range()
 
         self.depth_prior_dir = self.dir / "depth_prior"
+        if depth_prior_dir is not None:
+            self.depth_prior_dir = depth_prior_dir
+
         self.depth_prior_template = "{:05d}_ra.npy"
 
         self.height = size[0]
@@ -97,6 +100,7 @@ def get_filelist(self, index_min=0, index_max=-1):
             filelist = all_indexes
 
         else:
+            logger.error(f'Mode not supported: {self.mode}')
             filelist = None
 
         # Log the number of files selected
@@ -120,15 +124,19 @@ def __getitem__(self, item):
         depth = self.get_depth(index)
         depth_prior = self.get_depth_prior(index, image_cv.copy(), depth)
 
-        sample = self.transform({'image': image, 'depth': depth, 'depth_prior': depth_prior})
+        if depth is not None:
+            sample = self.transform({'image': image, 'depth': depth, 'depth_prior': depth_prior})
+            sample['depth'] = torch.from_numpy(sample['depth'])
+            sample['depth'] = torch.nan_to_num(sample['depth'], nan=0.0)
+            sample['valid_mask'] = ((sample['depth'] > self.depth_min) & (sample['depth'] <= self.depth_max))
+        else:
+            sample = self.transform({'image': image, 'depth_prior': depth_prior})
+
         sample['image'] = torch.from_numpy(sample['image'])
-        sample['depth'] = torch.from_numpy(sample['depth'])
-        sample['depth'] = torch.nan_to_num(sample['depth'], nan=0.0)
 
         sample['depth_prior'] = torch.from_numpy(sample['depth_prior'])
         sample['depth_prior'] = torch.nan_to_num(sample['depth_prior'], nan=0.0)
 
-        sample['valid_mask'] = ((sample['depth'] > self.depth_min) & (sample['depth'] <= self.depth_max))
         sample['image_path'] = str(img_path)
 
         return sample
@@ -223,14 +231,15 @@ def get_depth_range(self):
                 logger.info(f'Found norm range {norm_range:.3f} m')
 
         else:
-            logger.error(f"Could not find: {depth_norm_file}")
+            logger.warning(f"Could not find: {depth_norm_file}")
 
         return norm_min, norm_max, norm_range
 
     def get_depth_prior(self, index, img_copy, depth):
         if self.depth_prior_dir.is_dir():
             depth_prior = np.load(str(self.depth_prior_dir / self.depth_prior_template.format(index))).astype(np.float32)
-            if depth_prior.max() <= 1.0:
+            # TODO: Better detecting if dataset is normalized or not
+            if not (depth_prior > 0.0).any() and self.depth_min is not None and self.depth_range is not None:
                 depth_prior_valid_mask = (depth_prior > 0.0) & (depth_prior <= 1.0)
                 depth_prior[depth_prior_valid_mask] *= self.depth_range + self.depth_min
 

radarmeetsvision/metric_depth_network/dataset/transform.py

@@ -238,7 +238,7 @@ def __call__(self, sample):
             sample["mask"] = sample["mask"].astype(np.float32)
             sample["mask"] = np.ascontiguousarray(sample["mask"])
 
-        if "depth" in sample:
+        if "depth" in sample and sample["depth"] is not None:
             depth = sample["depth"].astype(np.float32)
             sample["depth"] = np.ascontiguousarray(depth)