From a52aedee2baa238640942a6b1a8333af9ce75f33 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Tue, 11 Jun 2024 15:36:14 +1000 Subject: [PATCH 1/4] WCS fixes. (#1027) * Check for error after retrieving ranges. * Use correct dtype when writing out empty data in WCS1 (not required in WCS2). * Update issue template. * Ensure correct default behaviour when output and subsetting CRSs match. * Pin datacube<1.9 * Add (very simple) test. (cherry picked from commit 79b796fd6b76b76c4274094644a14b50c72b0413) --- .github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md | 6 +-- datacube_ows/templates/wms_capabilities.xml | 2 +- datacube_ows/wcs1_utils.py | 6 ++- datacube_ows/wcs_scaler.py | 41 ++++++++++----------- integration_tests/test_wcs_server.py | 7 ++++ 5 files changed, 35 insertions(+), 27 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md index b1fbc3ac..62b019d2 100644 --- a/.github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE/ISSUE_TEMPLATE.md @@ -18,11 +18,11 @@ assignees: '' 4. ## Context (Environment) -### `datacube-ows` version (datacube-ows --version): - +### `datacube-ows` version (datacube-ows-update --version): + ### `ows_config.py` file (link, sample code) - + ### datacube product metadata (datacube product show product_name) diff --git a/datacube_ows/templates/wms_capabilities.xml b/datacube_ows/templates/wms_capabilities.xml index 275e24b7..8b5fa2f0 100644 --- a/datacube_ows/templates/wms_capabilities.xml +++ b/datacube_ows/templates/wms_capabilities.xml @@ -35,8 +35,8 @@ {% endif %} {%- endmacro %} {% macro render_named_layer(lyr, depth=0) -%} - {% if not lyr.hide %} {% set lyr_ranges = lyr.ranges %} + {% if not lyr.hide %} {{ lyr.name }} {{ lyr.title }} diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index 2af75468..2e0dc3d1 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -357,7 +357,8 @@ def get_coverage_data(req, qprof): nparrays = { band: (("time", yname, xname), numpy.full((len(req.times), len(yvals), len(xvals)), - req.layer.band_idx.nodata_val(band)) + req.layer.band_idx.nodata_val(band), + dtype=req.layer.band_idx.dtype_val(band)) ) for band in req.bands } @@ -365,7 +366,8 @@ def get_coverage_data(req, qprof): nparrays = { band: (("time", xname, yname), numpy.full((len(req.times), len(xvals), len(yvals)), - req.layer.band_idx.nodata_val(band)) + req.layer.band_idx.nodata_val(band), + dtype=req.layer.band_idx.dtype_val(band)) ) for band in req.bands } diff --git a/datacube_ows/wcs_scaler.py b/datacube_ows/wcs_scaler.py index 503ab75b..187640c9 100644 --- a/datacube_ows/wcs_scaler.py +++ b/datacube_ows/wcs_scaler.py @@ -130,27 +130,26 @@ def to_crs(self, new_crs): grid = self.layer.grids[new_crs] skip_x_xform = False skip_y_xform = False - if self.crs != new_crs: - if not self.subsetted.x and not self.subsetted.y: - # Neither axis subsetted - self.min.x = self.layer.ranges["bboxes"][new_crs]["left"] - self.max.x = self.layer.ranges["bboxes"][new_crs]["right"] - self.min.y = self.layer.ranges["bboxes"][new_crs]["bottom"] - self.max.y = self.layer.ranges["bboxes"][new_crs]["top"] - self.crs = new_crs - elif not self.subsetted.x or not self.subsetted.y: - # One axis subsetted - if self.subsetted.x: - self.min.y = self.layer.ranges["bboxes"][self.crs]["bottom"] - self.max.y = self.layer.ranges["bboxes"][self.crs]["top"] - skip_y_xform = True - if self.subsetted.y: - self.min.x = self.layer.ranges["bboxes"][self.crs]["left"] - self.max.x = self.layer.ranges["bboxes"][self.crs]["right"] - skip_x_xform = True - else: - # Both axes subsetted - pass + if not self.subsetted.x and not self.subsetted.y: + # Neither axis subsetted + self.min.x = self.layer.ranges["bboxes"][new_crs]["left"] + self.max.x = self.layer.ranges["bboxes"][new_crs]["right"] + self.min.y = self.layer.ranges["bboxes"][new_crs]["bottom"] + self.max.y = self.layer.ranges["bboxes"][new_crs]["top"] + self.crs = new_crs + elif not self.subsetted.x or not self.subsetted.y: + # One axis subsetted + if self.subsetted.x: + self.min.y = self.layer.ranges["bboxes"][self.crs]["bottom"] + self.max.y = self.layer.ranges["bboxes"][self.crs]["top"] + skip_y_xform = True + if self.subsetted.y: + self.min.x = self.layer.ranges["bboxes"][self.crs]["left"] + self.max.x = self.layer.ranges["bboxes"][self.crs]["right"] + skip_x_xform = True + else: + # Both axes subsetted + pass if self.crs != new_crs: is_point = False diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index 22d35e74..ad7a03a6 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -1099,6 +1099,13 @@ def test_wcs20_getcoverage_geotiff_bigimage(ows_server): scalesize="x(3000),y(3000)", ) assert "too much data for a single request" in str(e.value) + # Test default request + with pytest.raises(ServiceException) as e: + output = wcs.getCoverage( + identifier=[layer.name], + format="application/x-netcdf", + ) + assert "too much data for a single request" in str(e.value) def test_wcs20_getcoverage_netcdf(ows_server): From 5134b404b9e4bd3c89e248b4c7cff410ea9e66c2 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 12 Jun 2024 10:31:03 +1000 Subject: [PATCH 2/4] Plug up some permissions management gaps. --- .../grants/read_only/002_grant_range_read_requires_role.sql | 3 +++ .../grants/read_only/003_grant_agdc_user_requires_role.sql | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 datacube_ows/sql/postgres/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql create mode 100644 datacube_ows/sql/postgres/ows_schema/grants/read_only/003_grant_agdc_user_requires_role.sql diff --git a/datacube_ows/sql/postgres/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql b/datacube_ows/sql/postgres/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql new file mode 100644 index 00000000..dc1bc3ee --- /dev/null +++ b/datacube_ows/sql/postgres/ows_schema/grants/read_only/002_grant_range_read_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting select on layer ranges table to {role} + +GRANT SELECT ON ows.layer_ranges TO {role}; diff --git a/datacube_ows/sql/postgres/ows_schema/grants/read_only/003_grant_agdc_user_requires_role.sql b/datacube_ows/sql/postgres/ows_schema/grants/read_only/003_grant_agdc_user_requires_role.sql new file mode 100644 index 00000000..5783f1b9 --- /dev/null +++ b/datacube_ows/sql/postgres/ows_schema/grants/read_only/003_grant_agdc_user_requires_role.sql @@ -0,0 +1,3 @@ +-- Granting agdc_user role to {role} + +GRANT agdc_user to {role}; From 378f6f598cc6e1e490876fc69e2c7042171dab72 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 12 Jun 2024 10:31:37 +1000 Subject: [PATCH 3/4] Fix old style ranges accessors in the WCS2 Scaler. --- datacube_ows/wcs_scaler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/datacube_ows/wcs_scaler.py b/datacube_ows/wcs_scaler.py index 187640c9..2810e1bb 100644 --- a/datacube_ows/wcs_scaler.py +++ b/datacube_ows/wcs_scaler.py @@ -132,20 +132,20 @@ def to_crs(self, new_crs): skip_y_xform = False if not self.subsetted.x and not self.subsetted.y: # Neither axis subsetted - self.min.x = self.layer.ranges["bboxes"][new_crs]["left"] - self.max.x = self.layer.ranges["bboxes"][new_crs]["right"] - self.min.y = self.layer.ranges["bboxes"][new_crs]["bottom"] - self.max.y = self.layer.ranges["bboxes"][new_crs]["top"] + self.min.x = self.layer.ranges.bboxes[new_crs]["left"] + self.max.x = self.layer.ranges.bboxes[new_crs]["right"] + self.min.y = self.layer.ranges.bboxes[new_crs]["bottom"] + self.max.y = self.layer.ranges.bboxes[new_crs]["top"] self.crs = new_crs elif not self.subsetted.x or not self.subsetted.y: # One axis subsetted if self.subsetted.x: - self.min.y = self.layer.ranges["bboxes"][self.crs]["bottom"] - self.max.y = self.layer.ranges["bboxes"][self.crs]["top"] + self.min.y = self.layer.ranges.bboxes[self.crs]["bottom"] + self.max.y = self.layer.ranges.bboxes[self.crs]["top"] skip_y_xform = True if self.subsetted.y: - self.min.x = self.layer.ranges["bboxes"][self.crs]["left"] - self.max.x = self.layer.ranges["bboxes"][self.crs]["right"] + self.min.x = self.layer.ranges.bboxes[self.crs]["left"] + self.max.x = self.layer.ranges.bboxes[self.crs]["right"] skip_x_xform = True else: # Both axes subsetted @@ -188,14 +188,14 @@ def to_crs(self, new_crs): proj_geom = geom.to_crs(new_crs_obj) bbox = proj_geom.boundingbox if skip_x_xform: - self.min.x = self.layer.ranges["bboxes"][new_crs]["left"] - self.max.x = self.layer.ranges["bboxes"][new_crs]["right"] + self.min.x = self.layer.ranges.bboxes[new_crs]["left"] + self.max.x = self.layer.ranges.bboxes[new_crs]["right"] else: self.min.x = bbox.left self.max.x = bbox.right if skip_y_xform: - self.min.y = self.layer.ranges["bboxes"][new_crs]["bottom"] - self.max.y = self.layer.ranges["bboxes"][new_crs]["top"] + self.min.y = self.layer.ranges.bboxes[new_crs]["bottom"] + self.max.y = self.layer.ranges.bboxes[new_crs]["top"] else: self.min.y = bbox.bottom self.max.y = bbox.top From 8091fc531690100fbda354ab15e8785fa84346d8 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Wed, 12 Jun 2024 10:48:33 +1000 Subject: [PATCH 4/4] Update WCS Scaler tests to use new format ranges. --- tests/test_wcs_scaler.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/tests/test_wcs_scaler.py b/tests/test_wcs_scaler.py index b4732ad4..dce56dc7 100644 --- a/tests/test_wcs_scaler.py +++ b/tests/test_wcs_scaler.py @@ -10,6 +10,7 @@ from affine import Affine from datacube_ows.ows_configuration import OWSConfig, OWSProductLayer +from datacube_ows.index.api import CoordRange, LayerExtent from datacube_ows.wcs_scaler import (SpatialParameter, WCSScaler, WCSScalerUnknownDimension) @@ -124,20 +125,11 @@ def layer_crs_geom(): times = [datetime.date(2013, 1, 1), datetime.date(2014, 1, 1), datetime.date(2015, 1, 1), datetime.date(2016, 1, 1), datetime.date(2017, 1, 1), datetime.date(2018, 1, 1)] product_layer.dynamic = False - product_layer._ranges = { - 'lat': { - 'min': -34.5250413940276, - 'max': -33.772472435988 - }, - 'lon': { - 'min': 150.330509919584, - 'max': 151.258021405841 - }, - 'times': times, - 'start_time': datetime.date(2013, 1, 1), - 'end_time': datetime.date(2018, 1, 1), - 'time_set': set(times), - 'bboxes': { + product_layer._ranges = LayerExtent( + lat=CoordRange(min=-34.5250413940276, max=-33.772472435988), + lon=CoordRange(min=150.330509919584, max=151.258021405841), + times=times, + bboxes={ 'EPSG:3111': { 'top': 5725844.213533809, 'left': -1623290.9363678931, 'right': 3983581.449863785, 'bottom': 1042109.9920098772 @@ -155,7 +147,7 @@ def layer_crs_geom(): 'right': 157.105656164263, 'bottom': -45.761684927317 } } - } + ) return product_layer