Merge branch '84-add-subpix' into 'release'

Resolve "Ajout du sous-pixellique"

See merge request 3d/PandoraBox/pandora2d!95

docs/source/userguide/step_by_step/matching_cost.rst

@@ -54,11 +54,20 @@ Configuration and parameters
      - [1, 1]
      - list[int >0, int >0]
      - No
+   * - subpix
+     - Subpix parameter for computing subpixel disparities
+     - int
+     - 1
+     - [1,2,4]
+     - No
 
 
 .. note::
     The order of steps should be [row, col].
 
+.. warning::
+    The subpix parameter can only take values 1, 2 and 4.
+
 
 **Example**
 
@@ -77,7 +86,8 @@ Configuration and parameters
             {
                 "matching_cost_method": "ssd",
                 "window_size": 7,
-                "step" : [5, 5]
+                "step" : [5, 5],
+                "subpix": 4,
             },
             //...
         }

pandora2d/img_tools.py

@@ -35,7 +35,7 @@
 
 import copy
 from collections.abc import Sequence
-from typing import List, Dict, NamedTuple, Any
+from typing import List, Dict, Union, NamedTuple, Any
 
 from math import floor
 import xarray as xr
@@ -207,7 +207,7 @@ def add_disparity_grid(dataset: xr.Dataset, col_disparity: List[int], row_dispar
     return dataset
 
 
-def shift_img_pandora2d(img_right: xr.Dataset, dec_row: int) -> xr.Dataset:
+def shift_disp_row_img(img_right: xr.Dataset, dec_row: int) -> xr.Dataset:
     """
     Return a Dataset that contains the shifted right images
 
@@ -219,14 +219,14 @@ def shift_img_pandora2d(img_right: xr.Dataset, dec_row: int) -> xr.Dataset:
     :return: img_right_shift: Dataset containing the shifted image
     :rtype: xr.Dataset
     """
-    # dimensions of images
-    nrow_, ncol_ = img_right["im"].shape
+    # coordinates of images
+    row = img_right.get("row")
+    col = img_right.get("col")
+
     # shifted image by scipy
     data = shift(img_right["im"].data, (-dec_row, 0), cval=img_right.attrs["no_data_img"])
     # create shifted image dataset
-    img_right_shift = xr.Dataset(
-        {"im": (["row", "col"], data)}, coords={"row": np.arange(nrow_), "col": np.arange(ncol_)}
-    )
+    img_right_shift = xr.Dataset({"im": (["row", "col"], data)}, coords={"row": row, "col": col})
     # add attributes to dataset
     img_right_shift.attrs = {
         "no_data_img": img_right.attrs["no_data_img"],
@@ -338,7 +338,9 @@ def remove_roi_margins(dataset: xr.Dataset, cfg: Dict):
     return new_dataset
 
 
-def row_zoom_img(img: np.ndarray, ny: int, subpix: int, coords: Coordinates, ind: int) -> xr.Dataset:
+def row_zoom_img(
+    img: np.ndarray, ny: int, subpix: int, coords: Coordinates, ind: int, no_data: Union[int, str]
+) -> xr.Dataset:
     """
     Return a list that contains the shifted right images in row
 
@@ -354,21 +356,32 @@ def row_zoom_img(img: np.ndarray, ny: int, subpix: int, coords: Coordinates, ind
     :type coords: Coordinates
     :param ind: index of range(subpix)
     :type ind: int
+    :param no_data: no_data value in img
+    :type no_data: Union[int, str]
     :return: an array that contains the shifted right images in row
     :rtype: array of xarray.Dataset
     """
 
     shift = 1 / subpix
     # For each index, shift the right image for subpixel precision 1/subpix*index
     data = zoom(img, ((ny * subpix - (subpix - 1)) / float(ny), 1), order=1)[ind::subpix, :]
-    row = np.arange(coords.get("row")[0] + shift * ind, coords.get("row")[-1], step=1)  # type: np.ndarray
+
+    # Add a row full of no data at the end of data have the same shape as img
+    # It enables to use Pandora's compute_cost_volume() methods,
+    # which only accept left and right images of the same shape.
+    data = np.pad(data, ((0, 1), (0, 0)), "constant", constant_values=no_data)
+
+    row = np.arange(coords.get("row")[0] + shift * ind, coords.get("row")[-1] + 1, step=1)  # type: np.ndarray
+
     return xr.Dataset(
         {"im": (["row", "col"], data)},
         coords={"row": row, "col": coords.get("col")},
     )
 
 
-def col_zoom_img(img: np.ndarray, nx: int, subpix: int, coords: Coordinates, ind: int) -> xr.Dataset:
+def col_zoom_img(
+    img: np.ndarray, nx: int, subpix: int, coords: Coordinates, ind: int, no_data: Union[int, str]
+) -> xr.Dataset:
     """
     Return a list that contains the shifted right images in col
 
@@ -384,21 +397,29 @@ def col_zoom_img(img: np.ndarray, nx: int, subpix: int, coords: Coordinates, ind
     :type coords: Coordinates
     :param ind: index of range(subpix)
     :type ind: int
+    :param no_data: no_data value in img
+    :type no_data: Union[int, str]
     :return: an array that contains the shifted right images in col
     :rtype: array of xarray.Dataset
     """
 
     shift = 1 / subpix
     # For each index, shift the right image for subpixel precision 1/subpix*index
     data = zoom(img, (1, (nx * subpix - (subpix - 1)) / float(nx)), order=1)[:, ind::subpix]
-    col = np.arange(coords.get("col")[0] + shift * ind, coords.get("col")[-1], step=1)  # type: np.ndarray
+
+    # Add a col full of no data at the end of data to have the same shape as img
+    # It enables to use Pandora's compute_cost_volume() methods,
+    # which only accept left and right images of the same shape.
+    data = np.pad(data, ((0, 0), (0, 1)), "constant", constant_values=no_data)
+
+    col = np.arange(coords.get("col")[0] + shift * ind, coords.get("col")[-1] + 1, step=1)  # type: np.ndarray
     return xr.Dataset(
         {"im": (["row", "col"], data)},
         coords={"row": coords.get("row"), "col": col},
     )
 
 
-def shift_subpix_img(img_right: xr.Dataset, subpix: int, column: bool = True) -> List[xr.Dataset]:
+def shift_subpix_img(img_right: xr.Dataset, subpix: int, row: bool = True) -> List[xr.Dataset]:
     """
     Return an array that contains the shifted right images
 
@@ -415,13 +436,27 @@ def shift_subpix_img(img_right: xr.Dataset, subpix: int, column: bool = True) ->
 
     if subpix > 1:
         for ind in np.arange(1, subpix):
-            if column:
+            if row:
                 img_right_shift.append(
-                    col_zoom_img(img_right["im"].data, img_right.sizes["col"], subpix, img_right.coords, ind)
+                    row_zoom_img(
+                        img_right["im"].data,
+                        img_right.sizes["row"],
+                        subpix,
+                        img_right.coords,
+                        ind,
+                        img_right.attrs["no_data_img"],
+                    ).assign_attrs(img_right.attrs)
                 )
             else:
                 img_right_shift.append(
-                    row_zoom_img(img_right["im"].data, img_right.sizes["row"], subpix, img_right.coords, ind)
+                    col_zoom_img(
+                        img_right["im"].data,
+                        img_right.sizes["col"],
+                        subpix,
+                        img_right.coords,
+                        ind,
+                        img_right.attrs["no_data_img"],
+                    ).assign_attrs(img_right.attrs)
                 )
 
     return img_right_shift

pandora2d/matching_cost/matching_cost.py

@@ -24,7 +24,7 @@
 This module contains functions associated to the matching cost computation step.
 """
 import copy
-from typing import Dict, List, cast, Union
+from typing import Dict, cast, Union
 from json_checker import And, Checker
 
 import xarray as xr
@@ -45,6 +45,7 @@ class MatchingCost:
 
     _WINDOW_SIZE = 5
     _STEP = [1, 1]
+    _SUBPIX = 1
     margins = HalfWindowMargins()
 
     def __init__(self, cfg: Dict) -> None:
@@ -61,6 +62,7 @@ def __init__(self, cfg: Dict) -> None:
         # Cast to int in order to help mypy because self.cfg is a Dict, and it can not know the type of step.
         self._step_row = cast(int, self.cfg["step"][0])
         self._step_col = cast(int, self.cfg["step"][1])
+        self._subpix = cast(int, self.cfg["subpix"])
 
         # Init pandora items
         self.pandora_matching_cost_: Union[matching_cost.AbstractMatchingCost, None] = None
@@ -79,11 +81,14 @@ def check_conf(self, cfg: Dict) -> Dict[str, str]:
             cfg["window_size"] = self._WINDOW_SIZE
         if "step" not in cfg:
             cfg["step"] = self._STEP
+        if "subpix" not in cfg:
+            cfg["subpix"] = self._SUBPIX
 
         schema = {
             "matching_cost_method": And(str, lambda mc: mc in ["ssd", "sad", "zncc", "mc_cnn"]),
             "window_size": And(int, lambda ws: ws > 0, lambda ws: ws % 2 != 0),
             "step": And(list, lambda x: len(x) == 2, lambda y: all(val >= 1 for val in y)),
+            "subpix": And(int, lambda sp: sp in [1, 2, 4]),
         }
 
         checker = Checker(schema)
@@ -96,8 +101,8 @@ def allocate_cost_volumes(
         cost_volume_attr: dict,
         row: np.ndarray,
         col: np.ndarray,
-        col_disparity: List[int],
-        row_disparity: List[int],
+        disp_range_col: np.ndarray,
+        disp_range_row: np.ndarray,
         np_data: np.ndarray = None,
     ) -> xr.Dataset:
         """
@@ -109,31 +114,23 @@ def allocate_cost_volumes(
         :type row: np.ndarray
         :param col: dimension of the image (columns)
         :type col: np.ndarray
-        :param col_disparity: min and max disparities for columns.
-        :type col_disparity: List[int]
-        :param row_disparity: min and max disparities for rows.
-        :type row_disparity: List[int]
+        :param disp_range_col: columns disparity range.
+        :type disp_range_col: np.ndarray
+        :param disp_range_row: rows disparity range.
+        :type disp_range_row: np.ndarray
         :param np_data: 4D numpy.ndarray og cost_volumes. Defaults to None.
         :type np_data: np.ndarray
         :return: cost_volumes: 4D Dataset containing the cost_volumes
         :rtype: cost_volumes: xr.Dataset
         """
 
-        disp_min_col, disp_max_col = col_disparity
-        disp_min_row, disp_max_row = row_disparity
-
-        disparity_range_col = np.arange(disp_min_col, disp_max_col + 1)
-        disparity_range_row = np.arange(disp_min_row, disp_max_row + 1)
-
         # Create the cost volume
         if np_data is None:
-            np_data = np.zeros(
-                (len(row), len(col), len(disparity_range_col), len(disparity_range_row)), dtype=np.float32
-            )
+            np_data = np.zeros((len(row), len(col), len(disp_range_col), len(disp_range_row)), dtype=np.float32)
 
         cost_volumes = xr.Dataset(
             {"cost_volumes": (["row", "col", "disp_col", "disp_row"], np_data)},
-            coords={"row": row, "col": col, "disp_col": disparity_range_col, "disp_row": disparity_range_row},
+            coords={"row": row, "col": col, "disp_col": disp_range_col, "disp_row": disp_range_row},
         )
 
         cost_volumes.attrs = cost_volume_attr
@@ -226,12 +223,37 @@ def compute_cost_volumes(
         min_row, max_row = self.pandora_matching_cost_.get_min_max_from_grid(grid_min_row, grid_max_row)
         min_col, max_col = self.pandora_matching_cost_.get_min_max_from_grid(grid_min_col, grid_max_col)
 
+        # Array with all x disparities
+        disps_col = self.pandora_matching_cost_.get_disparity_range(min_col, max_col, self._subpix)
         # Array with all y disparities
-        disps_row = range(min_row, max_row + 1)
+        disps_row = self.pandora_matching_cost_.get_disparity_range(min_row, max_row, self._subpix)
+
         row_index = None
+
+        # Contains the shifted right images (with subpixel)
+        imgs_right_shift_subpixel = img_tools.shift_subpix_img(img_right, self._subpix)
+
         for idx, disp_row in enumerate(disps_row):
+
+            i_right = int((disp_row % 1) * self._subpix)
+
+            # Images contained in imgs_right_shift_subpixel are already shifted by 1/subpix.
+            # In order for img_right_shift to contain the right image shifted from disp_row,
+            # we call img_tools.shift_disp_row_img with np.floor(disp_row).
+
+            # For example if subpix=2 and disp_row=1.5
+            # i_right=1
+            # imgs_right_shift_subpixel[i_right] is shifted by 0.5
+            # In img_tools.shift_disp_row_img we shift it by np.floor(1.5)=1 --> In addition it is shifted by 1.5
+
+            # Another example if subpix=4 and disp_row=-1.25
+            # i_right=3
+            # imgs_right_shift_subpixel[i_right] is shifted by 0.75
+            # In img_tools.shift_disp_row_img we shift it by np.floor(-1.25)=-2 --> In addition it is shifted by -1.25
+
             # Shift image in the y axis
-            img_right_shift = img_tools.shift_img_pandora2d(img_right, disp_row)
+            img_right_shift = img_tools.shift_disp_row_img(imgs_right_shift_subpixel[i_right], np.floor(disp_row))
+
             # Compute cost volume
             cost_volume = self.pandora_matching_cost_.compute_cost_volume(img_left, img_right_shift, self.grid_)
             # Mask cost volume
@@ -262,7 +284,7 @@ def compute_cost_volumes(
                 col_coords = cost_volume["cost_volume"].coords["col"]
 
                 cost_volumes = self.allocate_cost_volumes(
-                    cost_volume.attrs, row_coords, col_coords, [min_col, max_col], [min_row, max_row], None
+                    cost_volume.attrs, row_coords, col_coords, disps_col, disps_row, None
                 )
 
             # Add current cost volume to the cost_volumes dataset