Mhs/das 2216/quick fixes

CHANGELOG.md

@@ -1,6 +1,14 @@
 # Changelog
 
+## [v1.2.0] - 2024-10-10
+
+### Changed
+
+- [[DAS-2216](https://bugs.earthdata.nasa.gov/browse/DAS-2216)]
+  The Swath Projector has been updated with quick fixes to add support for TEMPO level 2 data.
+
 ## [v1.1.1] - 2024-09-16
+
 ### Changed
 
 - [[TRT-558](https://bugs.earthdata.nasa.gov/browse/TRT-558)]
@@ -12,6 +20,7 @@
   also been renamed to `earthdata_varinfo_config.json`.
 
 ## [v1.1.0] - 2024-08-29
+
 ### Changed
 
 - [[DAS-1934](https://bugs.earthdata.nasa.gov/browse/DAS-1934)]
@@ -37,14 +46,15 @@ include updated documentation and files outlined by the
 
 Repository structure changes include:
 
-* Migrating `pymods` directory to `swath_projector`.
-* Migrating `swotrepr.py` to `swath_projector/adapter.py`.
-* Addition of `swath_projector/main.py`.
+- Migrating `pymods` directory to `swath_projector`.
+- Migrating `swotrepr.py` to `swath_projector/adapter.py`.
+- Addition of `swath_projector/main.py`.
 
 For more information on internal releases prior to NASA open-source approval,
 see legacy-CHANGELOG.md.
 
-[v1.1.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0)
-[v1.1.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
-[v1.0.1]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
-[v1.0.0]:(https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0)
+[v1.2.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.2.0)
+[v1.1.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.1)
+[v1.1.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.1.0)
+[v1.0.1]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.1)
+[v1.0.0]: (https://github.com/nasa/harmony-swath-projector/releases/tag/1.0.0)

bin/project_local_granule.py

@@ -125,8 +125,8 @@ def project_granule(
                         {
                             'url': local_file_path,
                             'temporal': {
-                                'start': '2021-01-03T23:45:00.000Z',
-                                'end': '2020-01-04T00:00:00.000Z',
+                                'start': '2020-01-03T23:45:00.000Z',
+                                'end': '2025-01-04T00:00:00.000Z',
                             },
                             'bbox': [-180, -90, 180, 90],
                         }
@@ -141,5 +141,5 @@ def project_granule(
 
     reprojector = SwathProjectorAdapter(message, config=config(False))
 
-    with patch('swotrepr.shutil.rmtree', side_effect=rmtree_side_effect):
+    with patch('swath_projector.adapter.shutil.rmtree', side_effect=rmtree_side_effect):
         reprojector.invoke()

docker/service_version.txt

@@ -1 +1 @@
-1.1.1
+1.2.0

swath_projector/earthdata_varinfo_config.json

@@ -1,12 +1,35 @@
 {
   "Identification": "Swath Projector VarInfo configuration",
-  "Version": 3,
+  "Version": 4,
   "CollectionShortNamePath": [
-    "ShortName"
+    "ShortName",
+    "collection_shortname"
   ],
   "Mission": {
-    "VNP10": "VIIRS"
+    "VNP10": "VIIRS",
+    "TEMPO_O3TOT_L2": "TEMPO"
   },
+  "ExcludedScienceVariables": [
+    {
+      "Applicability": {
+        "Mission": "TEMPO",
+        "ShortNamePath": "TEMPO_O3TOT_L2"
+      },
+      "VariablePattern": [
+        "/support_data/a_priori_layer_o3",
+        "/support_data/cal_adjustment",
+        "/support_data/dNdR",
+        "/support_data/layer_efficiency",
+        "/support_data/lut_wavelength",
+        "/support_data/N_value",
+        "/support_data/N_value_residual",
+        "/support_data/ozone_sensitivity_ratio",
+        "/support_data/step_1_N_value_residual",
+        "/support_data/step_2_N_value_residual",
+        "/support_data/temp_sensitivity_ratio"
+      ]
+    }
+  ],
   "MetadataOverrides": [
     {
       "Applicability": {
@@ -21,6 +44,20 @@
         }
       ],
       "_Description": "VNP10 SnowData variables have incorrect relative paths for coordinates."
+    },
+    {
+      "Applicability": {
+        "Mission": "TEMPO",
+        "ShortNamePath": "TEMPO_O3TOT_L2",
+        "VariablePattern": "^/product/.*|^/support_data/.*"
+      },
+      "Attributes": [
+        {
+          "Name": "coordinates",
+          "Value": "/geolocation/latitude, /geolocation/longitude"
+        }
+      ],
+      "_Description": "TEMPO_O3TOT_L2 variables only contain basenames for coordinates, which are found in sibling hierarchical groups. This rule fully qualifies the paths to these coordinates."
     }
   ]
 }

swath_projector/interpolation.py

@@ -268,6 +268,7 @@ def get_ewa_results(
         ewa_information['target_area'],
         variable['values'],
         maximum_weight_mode=maximum_weight_mode,
+        rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
     )
 
     if variable['fill_value'] is not None:

swath_projector/utilities.py

@@ -31,7 +31,8 @@ def get_variable_values(
     As the variable data are returned as a `numpy.ma.MaskedArray`, the will
     return no data in the filled pixels. To ensure that the data are
     correctly handled, the fill value is applied to masked pixels using the
-    `filled` method.
+    `filled` method. The variable values are transposed if the `along-track`
+    dimension size is less than the `across-track` dimension size.
 
     """
     # TODO: Remove in favour of apply2D or process_subdimension.
@@ -42,11 +43,15 @@ def get_variable_values(
     if len(variable[:].shape) == 1:
         return make_array_two_dimensional(variable[:])
     elif 'time' in input_file.variables and 'time' in variable.dimensions:
-        # Assumption: Array = (1, y, x)
-        return variable[0][:].filled(fill_value=fill_value)
+        # Assumption: Array = (time, along-track, across-track)
+        return transpose_if_xdim_less_than_ydim(
+            variable[0][:].filled(fill_value=fill_value)
+        )
     else:
-        # Assumption: Array = (y, x)
-        return variable[:].filled(fill_value=fill_value)
+        # Assumption: Array = (along-track, across-track)
+        return transpose_if_xdim_less_than_ydim(
+            variable[:].filled(fill_value=fill_value)
+        )
 
 
 def get_coordinate_variable(
@@ -62,7 +67,12 @@ def get_coordinate_variable(
         if coordinate_substring in coordinate.split('/')[-1] and variable_in_dataset(
             coordinate, dataset
         ):
-            return dataset[coordinate]
+            # QuickFix (DAS-2216) for short and wide swaths
+            if dataset[coordinate].ndim == 1:
+                return dataset[coordinate]
+
+            return transpose_if_xdim_less_than_ydim(dataset[coordinate])
+
     raise MissingCoordinatesError(coordinates_tuple)
 
 
@@ -216,3 +226,23 @@ def make_array_two_dimensional(one_dimensional_array: np.ndarray) -> np.ndarray:
 
     """
     return np.expand_dims(one_dimensional_array, 1)
+
+
+def transpose_if_xdim_less_than_ydim(variable: np.ndarray) -> np.ndarray:
+    """Return transposed variable when variable is wider than tall.
+
+    QuickFix (DAS-2216): We presume that a swath has more rows than columns and
+    if that's not the case we transpose it so that it does.
+    """
+    if variable.ndim != 2:
+        raise ValueError(
+            f'Input variable must be 2 dimensional, but got {variable.ndim} dimensions.'
+        )
+    if variable.shape[0] < variable.shape[1]:
+        transposed_variable = np.ma.transpose(variable).copy()
+        if hasattr(variable, 'mask'):
+            transposed_variable.mask = np.ma.transpose(variable[:].mask)
+
+        return transposed_variable
+
+    return variable

tests/unit/test_interpolation.py

@@ -384,6 +384,7 @@ def test_resample_ewa(
                 self.mock_target_area,
                 mock_values,
                 maximum_weight_mode=False,
+                rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
             )
             mock_write_output.assert_called_once_with(
                 self.mock_target_area,
@@ -423,6 +424,7 @@ def test_resample_ewa(
                 self.mock_target_area,
                 mock_values,
                 maximum_weight_mode=False,
+                rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
             )
             mock_write_output.assert_called_once_with(
                 self.mock_target_area,
@@ -491,6 +493,7 @@ def test_resample_ewa_nn(
                 self.mock_target_area,
                 mock_values,
                 maximum_weight_mode=True,
+                rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
             )
             mock_write_output.assert_called_once_with(
                 self.mock_target_area,
@@ -530,6 +533,7 @@ def test_resample_ewa_nn(
                 self.mock_target_area,
                 mock_values,
                 maximum_weight_mode=True,
+                rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
             )
             mock_write_output.assert_called_once_with(
                 self.mock_target_area,
@@ -581,6 +585,7 @@ def test_resample_ewa_nn(
                 harmony_target_area,
                 mock_values,
                 maximum_weight_mode=True,
+                rows_per_scan=2,  # Added in QuickFix DAS-2216 to be fixed in DAS-2220
             )
 
             # The Harmony target area should be given to the output function

tests/unit/test_utilities.py

@@ -16,6 +16,7 @@
     get_variable_values,
     make_array_two_dimensional,
     qualify_reference,
+    transpose_if_xdim_less_than_ydim,
     variable_in_dataset,
 )
 
@@ -79,7 +80,9 @@ def test_get_variable_values(self):
                 wind_speed_values = get_variable_values(dataset, wind_speed, None)
                 self.assertIsInstance(wind_speed_values, np.ndarray)
                 self.assertEqual(len(wind_speed_values.shape), 2)
-                self.assertEqual(wind_speed_values.shape, wind_speed.shape)
+                self.assertEqual(
+                    wind_speed_values.shape, np.ma.transpose(wind_speed).shape
+                )
 
         with self.subTest('Masked values are set to fill value.'):
             fill_value = 210
@@ -358,3 +361,53 @@ def test_make_array_two_dimensional(self):
 
         self.assertEqual(len(output_array.shape), 2)
         np.testing.assert_array_equal(output_array, expected_output)
+
+
+class TestTransposeIfXdimLessThanYdim(TestCase):
+
+    def test_wider_than_tall(self):
+        """Test case where x dim <  y dim and should transpose."""
+        input_array = np.ma.array([[1, 2, 3], [4, 5, 6]])
+        expected_output = np.ma.array([[1, 4], [2, 5], [3, 6]])
+        result = transpose_if_xdim_less_than_ydim(input_array)
+        np.testing.assert_array_equal(result, expected_output)
+        self.assertEqual(result.shape, (3, 2))
+
+    def test_taller_than_wide(self):
+        """Test case where x < y and should not transpose."""
+        input_array = np.ma.array([[1, 2], [3, 4], [5, 6]])
+        result = transpose_if_xdim_less_than_ydim(input_array)
+        np.testing.assert_array_equal(result, input_array)
+        self.assertEqual(result.shape, (3, 2))
+
+    def test_square_array(self):
+        """Test case where y dim == x dim and should not transpose."""
+        input_array = np.ma.array([[1, 2], [3, 4]])
+        result = transpose_if_xdim_less_than_ydim(input_array)
+        np.testing.assert_array_equal(result, input_array)
+        self.assertEqual(result.shape, (2, 2))
+
+    def test_1d_array(self):
+        """Test case with a 1D array"""
+        input_array = np.ma.array([1, 2, 3])
+        with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'):
+            transpose_if_xdim_less_than_ydim(input_array)
+
+    def test_3d_array(self):
+        """Test case with a 3D array"""
+        input_array = np.ma.array([[[1, 2], [3, 4]], [[5, 6], [7, 8]]])
+        with self.assertRaisesRegex(ValueError, 'variable must be 2 dimensional'):
+            transpose_if_xdim_less_than_ydim(input_array)
+
+    def test_masked_array(self):
+        """Test case with a masked array"""
+        input_array = np.ma.array(
+            [[1, 2, 3], [4, 5, 6]], mask=[[True, False, False], [False, True, False]]
+        )
+        expected_output = np.ma.array(
+            [[1, 4], [2, 5], [3, 6]],
+            mask=[[True, False], [False, True], [False, False]],
+        )
+        result = transpose_if_xdim_less_than_ydim(input_array)
+        np.testing.assert_array_equal(result, expected_output)
+        np.testing.assert_array_equal(result.mask, expected_output.mask)