fix nan handling for numeric range fields

datacube/drivers/postgis/_fields.py

@@ -330,11 +330,9 @@ def value_to_alchemy(self, value: DateFieldLike) -> DateFieldLike:
         """
         Wrap a value as needed for this field type.
         """
-        if isinstance(value, datetime):
+        if isinstance(value, (datetime, str)):
             return self.normalise_value(value)
-        elif isinstance(value, str):
-            return tz_as_utc(datetime.fromisoformat(value))
-        elif isinstance(value, (ColumnElement, str)):
+        elif isinstance(value, ColumnElement):
             # SQLAlchemy expression or string are parsed in pg as dates.
             # NB: Do not cast here - casting here breaks expected behaviour in other timezones
             return value
@@ -373,10 +371,6 @@ def day(self):
             alchemy_expression=cast(func.date_trunc('day', self.alchemy_expression), TIMESTAMP(timezone=True))
         )
 
-    @property
-    def alchemy_expression(self):
-        return self._alchemy_offset_value(self.offset, self.aggregation.pg_calc).label(self.name)
-
 
 class RangeDocField(PgDocField):
     """
@@ -442,6 +436,14 @@ def value_to_alchemy(self, value):
             type_=NUMRANGE,
         )
 
+    def search_value_to_alchemy(self, value):
+        low, high = value
+        if isinstance(low, float) and math.isnan(low):
+            raise UnindexableValue("Cannot index NaNs")
+        if isinstance(high, float) and math.isnan(high):
+            raise UnindexableValue("Cannot index NaNs")
+        return self.value_to_alchemy(value)
+
     def between(self, low, high):
         """
         :rtype: Expression

integration_tests/conftest.py

@@ -93,6 +93,7 @@ def eo3_dataset_paths():
         str(EO3_TESTDIR / "ls8_dataset4.yaml"),
         str(EO3_TESTDIR / "wo_dataset.yaml"),
         str(EO3_TESTDIR / "s2_africa_dataset.yaml"),
+        str(EO3_TESTDIR / "s2_africa_dataset2.yaml"),
     ]
 
 
@@ -164,6 +165,15 @@ def eo3_africa_dataset_doc():
     )
 
 
+@pytest.fixture
+def eo3_africa_dataset2_doc():
+    return (
+        get_eo3_test_data_doc("s2_africa_dataset2.yaml"),
+        's3://deafrica-sentinel-2/sentinel-s2-l2a-cogs/39/M/XR/'
+        '2020/9/S2A_39MXR_20200909_0_L2A/S2A_39MXR_20200909_0_L2A.json'
+    )
+
+
 @pytest.fixture
 def datasets_with_unembedded_lineage_doc():
     return [
@@ -354,6 +364,13 @@ def africa_eo3_dataset(index, africa_s2_eo3_product, eo3_africa_dataset_doc):
                      *eo3_africa_dataset_doc)
 
 
+@pytest.fixture
+def africa_eo3_dataset2(index, africa_s2_eo3_product, eo3_africa_dataset2_doc):
+    return doc_to_ds(index,
+                     africa_s2_eo3_product.name,
+                     *eo3_africa_dataset2_doc)
+
+
 @pytest.fixture
 def nrt_dataset(index, extended_eo3_metadata_type, ls8_eo3_product, nrt_dataset_doc):
     return doc_to_ds_no_add(
@@ -569,7 +586,7 @@ def index_pair_populated_empty(cfg_env_pair, uninitialised_postgres_db_pair,
                                base_eo3_product_doc, extended_eo3_product_doc, africa_s2_product_doc,
                                eo3_ls8_dataset_doc, eo3_ls8_dataset2_doc,
                                eo3_ls8_dataset3_doc, eo3_ls8_dataset4_doc,
-                               eo3_wo_dataset_doc, eo3_africa_dataset_doc):
+                               eo3_wo_dataset_doc, eo3_africa_dataset_doc, eo3_africa_dataset2_doc):
     populated_cfg, empty_cfg = cfg_env_pair
     populated_idx = index_connect(populated_cfg, validate_connection=False)
     empty_idx = index_connect(empty_cfg, validate_connection=False)
@@ -585,7 +602,7 @@ def index_pair_populated_empty(cfg_env_pair, uninitialised_postgres_db_pair,
         populated_idx.products.add_document(prod_doc)
     for ds_doc, ds_path in (eo3_ls8_dataset_doc, eo3_ls8_dataset2_doc,
                             eo3_ls8_dataset3_doc, eo3_ls8_dataset4_doc,
-                            eo3_wo_dataset_doc, eo3_africa_dataset_doc):
+                            eo3_wo_dataset_doc, eo3_africa_dataset_doc, eo3_africa_dataset2_doc):
         doc_to_ds(populated_idx, ds_doc['product']['name'], ds_doc, ds_path)
     assert list(populated_idx.products.get_all()) != list(empty_idx.products.get_all())
     assert list(empty_idx.products.get_all()) == []

integration_tests/data/eo3/s2_africa_dataset2.yaml

@@ -0,0 +1,115 @@
+---
+# Dataset
+# url: https://explorer.digitalearth.africa/dataset/d50adeb2-15be-562b-86dd-24e30357bec7.odc-metadata.yaml
+$schema: https://schemas.opendatacube.org/dataset
+id: d50adeb2-15be-562b-86dd-24e30357bec7
+
+label: S2A_MSIL2A_20200909T065031_N0214_R077_T39MXR_20200909T083937
+product:
+  name: s2_l2a
+
+location: 
+  s3://deafrica-sentinel-2/sentinel-s2-l2a-cogs/39/M/XR/2020/9/S2A_39MXR_20200909_0_L2A/S2A_39MXR_20200909_0_L2A.json
+
+crs: epsg:32739
+geometry:
+  type: Polygon
+  coordinates: [[[7.09791e+05, 9.55145e+06], [7.09791e+05, 9.551499e+06], [7.09791e+05,
+        9.55145e+06], [7.09791e+05, 9.55145e+06]]]
+grids:
+  g20m:
+    shape: [5490, 5490]
+    transform: [2.e+01, 0.e+00, 6.e+05, 0.e+00, -2.e+01, 9.60004e+06, 0.e+00, 0.e+00,
+      1.e+00]
+  g60m:
+    shape: [1830, 1830]
+    transform: [6.e+01, 0.e+00, 6.e+05, 0.e+00, -6.e+01, 9.60004e+06, 0.e+00, 0.e+00,
+      1.e+00]
+  g320m:
+    shape: [343, 343]
+    transform: [3.2e+02, 0.e+00, 6.e+05, 0.e+00, -3.2e+02, 9.60004e+06, 0.e+00, 0.e+00,
+      1.e+00]
+  default:
+    shape: [10980, 10980]
+    transform: [1.e+01, 0.e+00, 6.e+05, 0.e+00, -1.e+01, 9.60004e+06, 0.e+00, 0.e+00,
+      1.e+00]
+
+properties:
+  datetime: '2020-09-09T06:50:53Z'
+  eo:cloud_cover: 7.346e+01
+  eo:constellation: sentinel-2
+  eo:gsd: 10  # Ground sample distance (m)
+  eo:instrument: MSI
+  eo:off_nadir: 0
+  eo:platform: sentinel-2a
+  odc:file_format: GeoTIFF
+  odc:processing_datetime: '2020-09-09T06:50:53Z'
+  odc:region_code: 39MXR
+  proj:epsg: 32739
+  sentinel:grid_square: XR
+  sentinel:latitude_band: M
+  sentinel:product_id: S2A_MSIL2A_20200909T065031_N0214_R077_T39MXR_20200909T083937
+  sentinel:sequence: '0'
+  sentinel:utm_zone: 39
+  sentinel:valid_cloud_cover: true
+
+measurements:
+  AOT:
+    grid: g60m
+    path: AOT.tif
+  B01:
+    grid: g60m
+    path: B01.tif
+  B02:
+    path: B02.tif
+  B03:
+    path: B03.tif
+  B04:
+    path: B04.tif
+  B05:
+    grid: g20m
+    path: B05.tif
+  B06:
+    grid: g20m
+    path: B06.tif
+  B07:
+    grid: g20m
+    path: B07.tif
+  B08:
+    path: B08.tif
+  B09:
+    grid: g60m
+    path: B09.tif
+  B11:
+    grid: g20m
+    path: B11.tif
+  B12:
+    grid: g20m
+    path: B12.tif
+  B8A:
+    grid: g20m
+    path: B8A.tif
+  SCL:
+    grid: g20m
+    path: SCL.tif
+  WVP:
+    path: WVP.tif
+  visual:
+    path: TCI.tif
+  overview:
+    grid: g320m
+    path: L2A_PVI.tif
+
+accessories:
+  info:
+    path: 
+      https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/39/M/XR/2020/9/9/0/tileInfo.json
+  metadata:
+    path: 
+      https://roda.sentinel-hub.com/sentinel-s2-l2a/tiles/39/M/XR/2020/9/9/0/metadata.xml
+  thumbnail:
+    path: 
+      https://roda.sentinel-hub.com/sentinel-s2-l1c/tiles/39/M/XR/2020/9/9/0/preview.jpg
+
+lineage: {}
+...

integration_tests/index/test_postgis_index.py

@@ -76,6 +76,7 @@ def test_spatial_index_populate(index: Index,
 def test_spatial_index_crs_validity(index: Index,
                                     ls8_eo3_product, ls8_eo3_dataset,
                                     africa_s2_eo3_product, africa_eo3_dataset):
+    # TODO: potentially include africa_eo3_dataset2 in this test
     epsg4326 = CRS("EPSG:4326")
     epsg3577 = CRS("EPSG:3577")
     index.create_spatial_index(epsg3577)
@@ -189,6 +190,7 @@ def test_spatial_extent(index,
                         ls8_eo3_dataset, ls8_eo3_dataset2,
                         ls8_eo3_dataset3, ls8_eo3_dataset4,
                         africa_s2_eo3_product, africa_eo3_dataset):
+    # TODO: include africa_eo3_dataset2 in this test
     epsg4326 = CRS("EPSG:4326")
     epsg3577 = CRS("EPSG:3577")
     index.create_spatial_index(epsg3577)