Skip to content

Commit

Permalink
Merge pull request #181 from opendatacube/dynamic_products
Browse files Browse the repository at this point in the history 
Support for "dynamic" products, where ranges are not cached in memory.
  • Loading branch information
@pindge
pindge authored Jan 9, 2020
2 parents 3b690b7 + 789d5a1 commit 9a7ed06
Show file tree
Hide file tree
Showing 6 changed files with 65 additions and 51 deletions.
5 changes: 5 additions & 0 deletions datacube_ows/ows_cfg_example.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1172,6 +1172,9 @@
# Resource limits.
# See reusable resource limit declarations above for documentation.
"resource_limits": standard_resource_limits,
# If "dynamic" is False (the default) the the ranges for the product are cached in memory.
# Dynamic products slow down the generation of the GetCapabilities document - use sparingly.
"dynamic": False,
"flags": {
# Data may include flags that mark which pixels have missing or poor-quality data,
# or contain cloud, or cloud-shadow, etc. This section describes how
Expand Down Expand Up @@ -1467,6 +1470,8 @@
"product_names": ["s2a_nrt_granule", "s2b_nrt_granule"],
"bands": sentinel2_bands,
"resource_limits": standard_resource_limits,
# Near Real Time datasets are being regularly updated - do not cache ranges in memory.
"dynamic": True,
"flags": {
"band": "quality",
"ignore_time": False,
Expand Down
44 changes: 31 additions & 13 deletions datacube_ows/ows_configuration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from collections.abc import Mapping, Sequence
from slugify import slugify

from datacube_ows.cube_pool import cube
from datacube_ows.cube_pool import cube, get_cube, release_cube
from datacube_ows.band_mapper import StyleDef
from datacube_ows.ogc_utils import get_function, ConfigException, ProductLayerException, FunctionWrapper

Expand Down Expand Up @@ -315,16 +315,9 @@ def __init__(self, cfg, global_cfg, dc, parent_layer=None):
raise ConfigException("Invalid time resolution value %s in named layer %s" % (self.time_resolution, self.name))
except KeyError:
raise ConfigException("Required product names entry missing in named layer %s" % self.name)
from datacube_ows.product_ranges import get_ranges
try:
self.ranges = get_ranges(dc, self)
if self.ranges is None:
raise Exception("Null product range")
except Exception as a:
range_failure = "get_ranges failed for layer %s: %s" % (self.name, str(a))
raise ConfigException(range_failure)
self.bboxes = self.extract_bboxes()
# sub-ranges???
self.dynamic = cfg.get("dynamic", False)
self.force_range_update(dc)
# TODO: sub-ranges
self.band_idx = BandIndex(self.product, cfg.get("bands"), dc)
try:
self.parse_resource_limits(cfg.get("resource_limits", {"wms": {}, "wcs": {}}))
Expand Down Expand Up @@ -502,8 +495,33 @@ def parse_product_names(self, cfg):
def parse_pq_names(self, cfg):
raise NotImplementedError()

def force_range_update(self, ext_dc=None):
if ext_dc:
dc = ext_dc
else:
dc = get_cube()
from datacube_ows.product_ranges import get_ranges
self._ranges = None
try:
self._ranges = get_ranges(dc, self)
if self._ranges is None:
raise Exception("Null product range")
except Exception as a:
range_failure = "get_ranges failed for layer %s: %s" % (self.name, str(a))
raise ConfigException(range_failure)
finally:
if not ext_dc:
release_cube(dc)
self.bboxes = self.extract_bboxes()

@property
def ranges(self):
if self.dynamic:
self.force_range_update()
return self._ranges

def extract_bboxes(self):
if self.ranges is None:
if self._ranges is None:
return {}
return {
crs_id: {"right": bbox["bottom"],
Expand All @@ -518,7 +536,7 @@ def extract_bboxes(self):
"top": bbox["top"],
"bottom": bbox["bottom"]
}
for crs_id, bbox in self.ranges["bboxes"].items()
for crs_id, bbox in self._ranges["bboxes"].items()
}
def layer_count(self):
return 1
Expand Down
11 changes: 5 additions & 6 deletions datacube_ows/templates/wcs_capabilities.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,17 +113,16 @@ xsi:schemaLocation="http://www.opengis.net/wcs http://schemas.opengis.net/wcs/1.
<ContentMetadata>
{% for product in cfg.product_index.values() %}
{% if product.wcs %}
{% set product_ranges = product.ranges %}
<CoverageOfferingBrief>
<description>{{ product.definition.description }}</description>
<name>{{ product.name }}</name>
<label>{{ product.title }}</label>
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84">
<gml:pos dimension="2">{{ product.ranges.lon.min }} {{ product.ranges.lat.min }}</gml:pos>
<gml:pos dimension="2">{{ product.ranges.lon.max }} {{ product.ranges.lat.max }}</gml:pos>
{% if not product.wcs_sole_time %}
<gml:timePosition>{{ product.ranges.start_time.isoformat() }}</gml:timePosition>
<gml:timePosition>{{ product.ranges.end_time.isoformat() }}</gml:timePosition>
{% endif %}
<gml:pos dimension="2">{{ product_ranges.lon.min }} {{ product_ranges.lat.min }}</gml:pos>
<gml:pos dimension="2">{{ product_ranges.lon.max }} {{ product_ranges.lat.max }}</gml:pos>
<gml:timePosition>{{ product_ranges.start_time.isoformat() }}</gml:timePosition>
<gml:timePosition>{{ product_ranges.end_time.isoformat() }}</gml:timePosition>
</lonLatEnvelope>
</CoverageOfferingBrief>
{% endif %}
Expand Down
36 changes: 13 additions & 23 deletions datacube_ows/templates/wcs_desc_coverage.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,26 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.opengis.net/wcs http://schemas.opengis.net/wcs/1.0.0/describeCoverage.xsd">
{% for product in products %}
{% set product_ranges = product.ranges %}
<CoverageOffering>
<description>{{ product.definition.description }}</description>
<name>{{ product.name }}</name>
<label>{{ product.title }}</label>
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84">
<gml:pos>{{ product.ranges.lon.min }} {{ product.ranges.lat.min }}</gml:pos>
<gml:pos>{{ product.ranges.lon.max }} {{ product.ranges.lat.max }}</gml:pos>
{% if not product.wcs_sole_time %}
<gml:timePosition>{{ product.ranges.start_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
<gml:timePosition>{{ product.ranges.end_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
{% endif %}
<gml:pos>{{ product_ranges.lon.min }} {{ product_ranges.lat.min }}</gml:pos>
<gml:pos>{{ product_ranges.lon.max }} {{ product_ranges.lat.max }}</gml:pos>
<gml:timePosition>{{ product_ranges.start_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
<gml:timePosition>{{ product_ranges.end_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
</lonLatEnvelope>

<domainSet>
<spatialDomain>
{% if not product.wcs_sole_time %}
<gml:EnvelopeWithTimePeriod srsName="{{ cfg.default_geographic_CRS }}">
<gml:pos>{{ product.ranges.lon.min }} {{ product.ranges.lat.min }}</gml:pos>
<gml:pos>{{ product.ranges.lon.max }} {{ product.ranges.lat.max }}</gml:pos>
<gml:timePosition>{{ product.ranges.start_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
<gml:timePosition>{{ product.ranges.end_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
</gml:EnvelopeWithTimePeriod>
{% else %}
<gml:Envelope srsName="{{ cfg.default_geographic_CRS }}">
<gml:pos dimension="2">{{ product.ranges.lon.min }} {{ product.ranges.lat.min }}</gml:pos>
<gml:pos dimension="2">{{ product.ranges.lon.max }} {{ product.ranges.lat.max }}</gml:pos>
</gml:Envelope>
{% endif %}
<gml:EnvelopeWithTimePeriod srsName="{{ cfg.default_geographic_CRS }}">
<gml:pos>{{ product_ranges.lon.min }} {{ product_ranges.lat.min }}</gml:pos>
<gml:pos>{{ product_ranges.lon.max }} {{ product_ranges.lat.max }}</gml:pos>
<gml:timePosition>{{ product_ranges.start_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
<gml:timePosition>{{ product_ranges.end_time.isoformat() }}T00:00:00.000Z</gml:timePosition>
</gml:EnvelopeWithTimePeriod>
{% if product.grid_high_x %}
<!-- Real RectifiedGrid section -->
<gml:RectifiedGrid srsName="{{ product.native_CRS }}" dimension="2">
Expand All @@ -49,7 +41,7 @@
<gml:axisName>{{ product.native_CRS_def["vertical_coord"] }}</gml:axisName>
<gml:origin srsName="{{ product.native_CRS }}">
<gml:pos>
{{ [product.ranges["bboxes"][product.native_CRS]["left"], product.ranges["bboxes"][product.native_CRS]["right"]]|min }} {{ [product.ranges["bboxes"][product.native_CRS]["top"], product.ranges["bboxes"][product.native_CRS]["bottom"]]|min }}
{{ [product_ranges["bboxes"][product.native_CRS]["left"], product_ranges["bboxes"][product.native_CRS]["right"]]|min }} {{ [product_ranges["bboxes"][product.native_CRS]["top"], product_ranges["bboxes"][product.native_CRS]["bottom"]]|min }}
</gml:pos>
</gml:origin>
<gml:offsetVector>{{ product.resolution_x }} 0.0</gml:offsetVector>
Expand All @@ -76,13 +68,11 @@
</gml:RectifiedGrid>
{% endif %}
</spatialDomain>
{% if not product.wcs_sole_time %}
<temporalDomain>
{% for t in product.ranges.times %}
{% for t in product_ranges.times %}
<gml:timePosition>{{ t.isoformat() }}T00:00:00.000Z</gml:timePosition>
{% endfor %}
</temporalDomain>
{% endif %}
</domainSet>
<rangeSet>
<RangeSet>
Expand Down
11 changes: 6 additions & 5 deletions datacube_ows/templates/wms_capabilities.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
{% endif %}
{%- endmacro %}
{% macro render_named_layer(lyr) -%}
{% set lyr_ranges = lyr.ranges %}
<Layer queryable="1">
<Name>{{ lyr.name }}</Name>
<Title>{{ lyr.title }}</Title>
Expand All @@ -45,10 +46,10 @@
{% endfor %}
</KeywordList>
<EX_GeographicBoundingBox>
<westBoundLongitude>{{ lyr.ranges.lon.min }}</westBoundLongitude>
<eastBoundLongitude>{{ lyr.ranges.lon.max }}</eastBoundLongitude>
<southBoundLatitude>{{ lyr.ranges.lat.min }}</southBoundLatitude>
<northBoundLatitude>{{ lyr.ranges.lat.max }}</northBoundLatitude>
<westBoundLongitude>{{ lyr_ranges.lon.min }}</westBoundLongitude>
<eastBoundLongitude>{{ lyr_ranges.lon.max }}</eastBoundLongitude>
<southBoundLatitude>{{ lyr_ranges.lat.min }}</southBoundLatitude>
<northBoundLatitude>{{ lyr_ranges.lat.max }}</northBoundLatitude>
</EX_GeographicBoundingBox>
{% for crsid, bbox in lyr.bboxes.items() %}
<BoundingBox CRS="{{ crsid }}"
Expand All @@ -65,7 +66,7 @@
/>
{% endfor %}
<Dimension name="time" units="ISO8601">
{% for t in lyr.ranges.times %}{{ t }}{% if not loop.last %},{% endif %}{% endfor %}
{% for t in lyr_ranges.times %}{{ t }}{% if not loop.last %},{% endif %}{% endfor %}
</Dimension>
{% for style in lyr.styles %}
<Style>
Expand Down
9 changes: 5 additions & 4 deletions datacube_ows/templates/wmts_capabilities.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,13 +125,14 @@ xsi:schemaLocation="http://www.opengis.net/wcs http://schemas.opengis.net/wmts/1
{% if show_contents %}
<Contents>
{% for product in cfg.product_index.values() %}
{% if product.ranges %}
{% set product_ranges = product.ranges %}
{% if product_ranges %}
<Layer>
<ows:Title>{{ product.title }}</ows:Title>
<ows:Abstract>{{ product.abstract }}</ows:Abstract>
<ows:WGS84BoundingBox>
<ows:LowerCorner>{{ product.ranges.lon.min }} {{ product.ranges.lat.min }}</ows:LowerCorner>
<ows:UpperCorner>{{ product.ranges.lon.max }} {{ product.ranges.lat.max }}</ows:UpperCorner>
<ows:LowerCorner>{{ product_ranges.lon.min }} {{ product_ranges.lat.min }}</ows:LowerCorner>
<ows:UpperCorner>{{ product_ranges.lon.max }} {{ product_ranges.lat.max }}</ows:UpperCorner>
</ows:WGS84BoundingBox>
<ows:Identifier>{{ product.name }}</ows:Identifier>
{% for style in product.styles %}
Expand All @@ -148,7 +149,7 @@ xsi:schemaLocation="http://www.opengis.net/wcs http://schemas.opengis.net/wmts/1
<Dimension>
<ows:Identifier>Time</ows:Identifier>
<ows:Title>Date</ows:Title>
{% for t in product.ranges.times %}
{% for t in product_ranges.times %}
<Value>{{ t }}</Value>
{% endfor %}
</Dimension>
Expand Down

0 comments on commit 9a7ed06

Please sign in to comment.