From 3609c788c35da96ab5d6fb7ad022cd944232c0fa Mon Sep 17 00:00:00 2001 From: Johan Schreurs Date: Wed, 9 Aug 2023 11:00:26 +0200 Subject: [PATCH 01/12] Issue #259 Make pyproj an optional dependency when parsing CRS string / EPSG code --- openeo/util.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/openeo/util.py b/openeo/util.py index 1a4794536..a595c6f9b 100644 --- a/openeo/util.py +++ b/openeo/util.py @@ -15,7 +15,6 @@ from urllib.parse import urljoin import requests -import pyproj.crs import shapely.geometry.base from deprecated import deprecated @@ -675,11 +674,10 @@ def crs_to_epsg_code(crs: Union[str, int, dict, None]) -> Optional[int]: if crs in (None, "", {}): return None - # TODO: decide: are more fine-grained checks more helpful than always raising EPSGCodeNotFound? if not isinstance(crs, (int, str, dict)): raise TypeError("The allowed type for the parameter 'crs' are: str, int, dict and None") - # if We want to stop processing as soon as we have an int value, then we + # If we want to stop processing as soon as we have an int value, then we # should not accept values that are complete non-sense, as best as we can. crs_intermediate = crs if isinstance(crs, int): @@ -693,10 +691,26 @@ def crs_to_epsg_code(crs: Union[str, int, dict, None]) -> Optional[int]: # So we need to process it with pyproj, below. logger.debug("crs_to_epsg_code received crs input that was not an int: crs={crs}, exception caught: {exc}") + if isinstance(crs_intermediate, int): + if crs_intermediate <= 0: + raise ValueError(f"When crs is an integer value it has to be > 0.") + else: + return crs_intermediate + try: - converted_crs = pyproj.crs.CRS.from_user_input(crs_intermediate) - except pyproj.exceptions.CRSError as exc: - logger.error(f"Could not convert CRS string to EPSG code: crs={crs}, exception: {exc}", exc_info=True) - raise ValueError(crs) from exc + import pyproj.crs + except ImportError as exc: + message = ( + f"Cannot convert CRS string: {crs}. " + + "Need pyproj to convert this CRS string but the pyproj library is not installed." + ) + logger.error(message) + raise ValueError(message) from ImportError else: - return converted_crs.to_epsg() + try: + converted_crs = pyproj.crs.CRS.from_user_input(crs) + except pyproj.exceptions.CRSError as exc: + logger.error(f"Could not convert CRS string to EPSG code: crs={crs}, exception: {exc}", exc_info=True) + raise ValueError(crs) from exc + else: + return converted_crs.to_epsg() From a79ec77b2f61600bb84b027f158a36f3d90bd20a Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 13:38:41 +0200 Subject: [PATCH 02/12] fixup! Issue #425 `load_stac`: support lambda based property filtering --- openeo/rest/connection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index dd44f3928..dd46605fb 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1185,7 +1185,7 @@ def load_stac( spatial_extent: Optional[Dict[str, float]] = None, temporal_extent: Optional[List[Union[str, datetime.datetime, datetime.date]]] = None, bands: Optional[List[str]] = None, - properties: Optional[dict] = None, + properties: Optional[Dict[str, Union[str, PGNode, Callable]]] = None, ) -> DataCube: """ Loads data from a static STAC catalog or a STAC API Collection and returns the data as a processable :py:class:`DataCube`. From 9ee1641e62fb086db9ab12694b6e45662929de0b Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 15:29:31 +0200 Subject: [PATCH 03/12] Issue #459 VectorCube: basic support for `filter_bands`, `filter_bbox`, `filter_labels` and `filter_vector` --- CHANGELOG.md | 2 +- openeo/rest/_testing.py | 35 ++- openeo/rest/vectorcube.py | 73 +++++- openeo/util.py | 36 +-- tests/rest/datacube/test_vectorcube.py | 307 ++++++++++++++++++++----- tests/test_util.py | 72 +++++- 6 files changed, 430 insertions(+), 95 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9dc9894e..a6eefb00a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Initial `load_geojson` support with `Connection.load_geojson()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) - Initial `load_url` (for vector cubes) support with `Connection.load_url()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) - Support lambda based property filtering in `Connection.load_stac()` ([#425](https://github.com/Open-EO/openeo-python-client/issues/425)) - +- `VectorCube`: initial support for `filter_bands`, `filter_bbox`, `filter_labels` and `filter_vector` ([#459](https://github.com/Open-EO/openeo-python-client/issues/459)) ### Changed diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index 73458f82a..fb44674e7 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -1,6 +1,8 @@ import re +from typing import Union, Optional -from openeo import Connection +from openeo import Connection, DataCube +from openeo.rest.vectorcube import VectorCube class DummyBackend: @@ -91,8 +93,33 @@ def get_batch_pg(self) -> dict: assert len(self.batch_jobs) == 1 return self.batch_jobs[max(self.batch_jobs.keys())]["pg"] - def get_pg(self) -> dict: - """Get one and only batch process graph (sync or batch)""" + def get_pg(self, process_id: Optional[str] = None) -> dict: + """ + Get one and only batch process graph (sync or batch) + + :param process_id: just return single process graph node with this process_id + :return: process graph (flat graph representation) or process graph node + """ pgs = self.sync_requests + [b["pg"] for b in self.batch_jobs.values()] assert len(pgs) == 1 - return pgs[0] + pg = pgs[0] + if process_id: + # Just return single node (by process_id) + found = [node for node in pg.values() if node.get("process_id") == process_id] + if len(found) != 1: + raise RuntimeError( + f"Expected single process graph node with {process_id=}, but found {len(found)}: {found}" + ) + return found[0] + return pg + + def execute(self, cube: Union[DataCube, VectorCube], process_id: Optional[str] = None) -> dict: + """ + Execute given cube (synchronously) and return observed process graph (or subset thereof). + + :param cube: cube to execute on dummy back-end + :param process_id: just return single process graph node with this process_id + :return: process graph (flat graph representation) or process graph node + """ + cube.execute() + return self.get_pg(process_id=process_id) diff --git a/openeo/rest/vectorcube.py b/openeo/rest/vectorcube.py index c90f8a662..cd7a6b4ec 100644 --- a/openeo/rest/vectorcube.py +++ b/openeo/rest/vectorcube.py @@ -1,7 +1,7 @@ import json import pathlib import typing -from typing import List, Optional, Union +from typing import List, Optional, Union, Tuple, Callable import shapely.geometry.base @@ -14,7 +14,7 @@ from openeo.rest._datacube import THIS, UDF, _ProcessGraphAbstraction, build_child_callback from openeo.rest.job import BatchJob from openeo.rest.mlmodel import MlModel -from openeo.util import dict_no_none, guess_format +from openeo.util import dict_no_none, guess_format, crs_to_epsg_code, to_bbox_dict, InvalidBBoxException if typing.TYPE_CHECKING: # Imports for type checking only (circular import issue at runtime). @@ -327,6 +327,75 @@ def create_job( send_job = legacy_alias(create_job, name="send_job", since="0.10.0") + @openeo_process + def filter_bands(self, bands: List[str]) -> "VectorCube": + """ + .. versionadded:: 0.22.0 + """ + # TODO #459 docs + return self.process( + process_id="filter_bands", + arguments={"data": THIS, "bands": bands}, + ) + + @openeo_process + def filter_bbox( + self, + *, + west: Optional[float] = None, + south: Optional[float] = None, + east: Optional[float] = None, + north: Optional[float] = None, + extent: Optional[Union[dict, List[float], Tuple[float, float, float, float], Parameter]] = None, + crs: Optional[int] = None, + ) -> "VectorCube": + """ + .. versionadded:: 0.22.0 + """ + # TODO #459 docs + if any(c is not None for c in [west, south, east, north]): + if extent is not None: + raise InvalidBBoxException("Don't specify both west/south/east/north and extent") + extent = dict_no_none(west=west, south=south, east=east, north=north) + + if isinstance(extent, Parameter): + pass + else: + extent = to_bbox_dict(extent, crs=crs) + return self.process( + process_id="filter_bbox", + arguments={"data": THIS, "extent": extent}, + ) + + @openeo_process + def filter_labels( + self, condition: Union[PGNode, Callable], dimension: str, context: Optional[dict] = None + ) -> "VectorCube": + """ + .. versionadded:: 0.22.0 + """ + # TODO #459 docs + condition = build_child_callback(condition, parent_parameters=["value"]) + return self.process( + process_id="filter_labels", + arguments=dict_no_none(data=THIS, condition=condition, dimension=dimension, context=context), + ) + + @openeo_process + def filter_vector( + self, geometries: Union["VectorCube", shapely.geometry.base.BaseGeometry, dict], relation: str = "intersects" + ) -> "VectorCube": + """ + .. versionadded:: 0.22.0 + """ + # TODO #459 docs + if not isinstance(geometries, (VectorCube, Parameter)): + geometries = self.load_geojson(connection=self.connection, data=geometries) + return self.process( + process_id="filter_vector", + arguments={"data": THIS, "geometries": geometries, "relation": relation}, + ) + @openeo_process def fit_class_random_forest( self, diff --git a/openeo/util.py b/openeo/util.py index a595c6f9b..129a33115 100644 --- a/openeo/util.py +++ b/openeo/util.py @@ -520,6 +520,10 @@ def in_interactive_mode() -> bool: return hasattr(sys, "ps1") +class InvalidBBoxException(ValueError): + pass + + class BBoxDict(dict): """ Dictionary based helper to easily create/work with bounding box dictionaries @@ -528,18 +532,18 @@ class BBoxDict(dict): .. versionadded:: 0.10.1 """ - def __init__(self, *, west: float, south: float, east: float, north: float, crs: Optional[str] = None): + def __init__(self, *, west: float, south: float, east: float, north: float, crs: Optional[Union[str, int]] = None): super().__init__(west=west, south=south, east=east, north=north) if crs is not None: - # TODO: #259, should we covert EPSG strings to int here with crs_to_epsg_code? - # self.update(crs=crs_to_epsg_code(crs)) - self.update(crs=crs) + self.update(crs=crs_to_epsg_code(crs)) # TODO: provide west, south, east, north, crs as @properties? Read-only or read-write? @classmethod def from_any(cls, x: Any, *, crs: Optional[str] = None) -> 'BBoxDict': if isinstance(x, dict): + if crs and "crs" in x and crs != x["crs"]: + raise InvalidBBoxException(f"Two CRS values specified: {crs} and {x['crs']}") return cls.from_dict({"crs": crs, **x}) elif isinstance(x, (list, tuple)): return cls.from_sequence(x, crs=crs) @@ -547,31 +551,31 @@ def from_any(cls, x: Any, *, crs: Optional[str] = None) -> 'BBoxDict': return cls.from_sequence(x.bounds, crs=crs) # TODO: support other input? E.g.: WKT string, GeoJson-style dictionary (Polygon, FeatureCollection, ...) else: - raise ValueError(f"Can not construct BBoxDict from {x!r}") + raise InvalidBBoxException(f"Can not construct BBoxDict from {x!r}") @classmethod def from_dict(cls, data: dict) -> 'BBoxDict': """Build from dictionary with at least keys "west", "south", "east", and "north".""" expected_fields = {"west", "south", "east", "north"} - # TODO: also support converting support case fields? - if not all(k in data for k in expected_fields): - raise ValueError( - f"Expecting fields {expected_fields}, but only found {expected_fields.intersection(data.keys())}." - ) - return cls( - west=data["west"], south=data["south"], east=data["east"], north=data["north"], - crs=data.get("crs") - ) + # TODO: also support upper case fields? + # TODO: optional support for parameterized bbox fields? + missing = expected_fields.difference(data.keys()) + if missing: + raise InvalidBBoxException(f"Missing bbox fields {sorted(missing)}") + invalid = {k: data[k] for k in expected_fields if not isinstance(data[k], (int, float))} + if invalid: + raise InvalidBBoxException(f"Non-numerical bbox fields {invalid}.") + return cls(west=data["west"], south=data["south"], east=data["east"], north=data["north"], crs=data.get("crs")) @classmethod def from_sequence(cls, seq: Union[list, tuple], crs: Optional[str] = None) -> 'BBoxDict': """Build from sequence of 4 bounds (west, south, east and north).""" if len(seq) != 4: - raise ValueError(f"Expected sequence with 4 items, but got {len(seq)}.") + raise InvalidBBoxException(f"Expected sequence with 4 items, but got {len(seq)}.") return cls(west=seq[0], south=seq[1], east=seq[2], north=seq[3], crs=crs) -def to_bbox_dict(x: Any, *, crs: Optional[str] = None) -> BBoxDict: +def to_bbox_dict(x: Any, *, crs: Optional[Union[str, int]] = None) -> BBoxDict: """ Convert given data or object to a bounding box dictionary (having keys "west", "south", "east", "north", and optionally "crs"). diff --git a/tests/rest/datacube/test_vectorcube.py b/tests/rest/datacube/test_vectorcube.py index ec2a552c5..beb447da7 100644 --- a/tests/rest/datacube/test_vectorcube.py +++ b/tests/rest/datacube/test_vectorcube.py @@ -1,3 +1,4 @@ +import re from pathlib import Path import pytest @@ -7,12 +8,32 @@ from openeo.internal.graph_building import PGNode from openeo.rest._testing import DummyBackend from openeo.rest.vectorcube import VectorCube +from openeo.util import InvalidBBoxException +import openeo.processes @pytest.fixture def vector_cube(con100) -> VectorCube: - pgnode = PGNode(process_id="create_vector_cube") - return VectorCube(graph=pgnode, connection=con100) + """Dummy vector cube""" + return con100.load_geojson({"type": "Point", "coordinates": [1, 2]}) + + +def test_vector_cube_fixture(vector_cube, dummy_backend): + assert dummy_backend.execute(vector_cube) == { + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + "result": True, + } + } + + +def test_vector_cube_fixture_process_id(vector_cube, dummy_backend): + assert dummy_backend.execute(vector_cube, process_id="load_geojson") == { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + "result": True, + } def test_raster_to_vector(con100): @@ -21,28 +42,16 @@ def test_raster_to_vector(con100): vector_cube_tranformed = vector_cube.run_udf(udf="python source code", runtime="Python") assert vector_cube_tranformed.flat_graph() == { - 'loadcollection1': { - 'arguments': { - 'id': 'S2', - 'spatial_extent': None, - 'temporal_extent': None - }, - 'process_id': 'load_collection' + "loadcollection1": { + "arguments": {"id": "S2", "spatial_extent": None, "temporal_extent": None}, + "process_id": "load_collection", }, - 'rastertovector1': { - 'arguments': { - 'data': {'from_node': 'loadcollection1'} - }, - 'process_id': 'raster_to_vector' + "rastertovector1": {"arguments": {"data": {"from_node": "loadcollection1"}}, "process_id": "raster_to_vector"}, + "runudf1": { + "arguments": {"data": {"from_node": "rastertovector1"}, "runtime": "Python", "udf": "python source code"}, + "process_id": "run_udf", + "result": True, }, - 'runudf1': { - 'arguments': { - 'data': {'from_node': 'rastertovector1'}, - 'runtime': 'Python', - 'udf': 'python source code' - }, - 'process_id': 'run_udf', - 'result': True} } @@ -68,11 +77,14 @@ def test_download_auto_save_result_only_file( raise ValueError(exec_mode) assert dummy_backend.get_pg() == { - "createvectorcube1": {"process_id": "create_vector_cube", "arguments": {}}, + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, "saveresult1": { "process_id": "save_result", "arguments": { - "data": {"from_node": "createvectorcube1"}, + "data": {"from_node": "loadgeojson1"}, "format": expected_format, "options": {}, }, @@ -80,7 +92,6 @@ def test_download_auto_save_result_only_file( }, } assert output_path.read_bytes() == DummyBackend.DEFAULT_RESULT - assert output_path.read_bytes() == DummyBackend.DEFAULT_RESULT @pytest.mark.parametrize( @@ -110,11 +121,14 @@ def test_download_auto_save_result_with_format( raise ValueError(exec_mode) assert dummy_backend.get_pg() == { - "createvectorcube1": {"process_id": "create_vector_cube", "arguments": {}}, + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, "saveresult1": { "process_id": "save_result", "arguments": { - "data": {"from_node": "createvectorcube1"}, + "data": {"from_node": "loadgeojson1"}, "format": expected_format, "options": {}, }, @@ -138,11 +152,14 @@ def test_download_auto_save_result_with_options(vector_cube, dummy_backend, tmp_ raise ValueError(exec_mode) assert dummy_backend.get_pg() == { - "createvectorcube1": {"process_id": "create_vector_cube", "arguments": {}}, + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, "saveresult1": { "process_id": "save_result", "arguments": { - "data": {"from_node": "createvectorcube1"}, + "data": {"from_node": "loadgeojson1"}, "format": "GeoJSON", "options": {"precision": 7}, }, @@ -176,10 +193,13 @@ def test_save_result_and_download( raise ValueError(exec_mode) assert dummy_backend.get_pg() == { - "createvectorcube1": {"process_id": "create_vector_cube", "arguments": {}}, + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, "saveresult1": { "process_id": "save_result", - "arguments": {"data": {"from_node": "createvectorcube1"}, "format": expected_format, "options": {}}, + "arguments": {"data": {"from_node": "loadgeojson1"}, "format": expected_format, "options": {}}, "result": True, }, } @@ -197,8 +217,7 @@ def test_save_result_and_download( def test_load_geojson_basic(con100, data, dummy_backend): vc = VectorCube.load_geojson(connection=con100, data=data) assert isinstance(vc, VectorCube) - vc.execute() - assert dummy_backend.get_pg() == { + assert dummy_backend.execute(vc) == { "loadgeojson1": { "process_id": "load_geojson", "arguments": { @@ -216,8 +235,7 @@ def test_load_geojson_path(con100, dummy_backend, tmp_path, path_type): path.write_text("""{"type": "Polygon", "coordinates": [[[1, 2], [3, 2], [3, 4], [1, 4], [1, 2]]]}""") vc = VectorCube.load_geojson(connection=con100, data=path_type(path)) assert isinstance(vc, VectorCube) - vc.execute() - assert dummy_backend.get_pg() == { + assert dummy_backend.execute(vc) == { "loadgeojson1": { "process_id": "load_geojson", "arguments": { @@ -232,8 +250,7 @@ def test_load_geojson_path(con100, dummy_backend, tmp_path, path_type): def test_load_geojson_parameter(con100, dummy_backend): vc = VectorCube.load_geojson(connection=con100, data=Parameter.datacube()) assert isinstance(vc, VectorCube) - vc.execute() - assert dummy_backend.get_pg() == { + assert dummy_backend.execute(vc) == { "loadgeojson1": { "process_id": "load_geojson", "arguments": {"data": {"from_parameter": "data"}, "properties": []}, @@ -245,8 +262,7 @@ def test_load_geojson_parameter(con100, dummy_backend): def test_load_url(con100, dummy_backend): vc = VectorCube.load_url(connection=con100, url="https://example.com/geometry.json", format="GeoJSON") assert isinstance(vc, VectorCube) - vc.execute() - assert dummy_backend.get_pg() == { + assert dummy_backend.execute(vc) == { "loadurl1": { "process_id": "load_url", "arguments": {"url": "https://example.com/geometry.json", "format": "GeoJSON"}, @@ -263,34 +279,205 @@ def test_load_url(con100, dummy_backend): ("wibbles", True), ], ) -def test_apply_dimension(con100, dummy_backend, dimension, expect_warning, caplog): - vc = con100.load_geojson({"type": "Point", "coordinates": [1, 2]}) - result = vc.apply_dimension("sort", dimension=dimension) - result.execute() - assert dummy_backend.get_pg() == { +def test_apply_dimension(vector_cube, dummy_backend, dimension, expect_warning, caplog): + vc = vector_cube.apply_dimension("sort", dimension=dimension) + assert dummy_backend.execute(vc, process_id="apply_dimension") == { + "process_id": "apply_dimension", + "arguments": { + "data": {"from_node": "loadgeojson1"}, + "dimension": dimension, + "process": { + "process_graph": { + "sort1": { + "process_id": "sort", + "arguments": {"data": {"from_parameter": "data"}}, + "result": True, + } + } + }, + }, + "result": True, + } + + assert ( + f"Invalid dimension {dimension!r}. Should be one of ['geometry', 'properties']" in caplog.messages + ) == expect_warning + + +def test_filter_bands(vector_cube, dummy_backend): + vc = vector_cube.filter_bands(["B01", "B02"]) + assert dummy_backend.execute(vc, process_id="filter_bands") == { + "process_id": "filter_bands", + "arguments": {"data": {"from_node": "loadgeojson1"}, "bands": ["B01", "B02"]}, + "result": True, + } + + +def test_filter_bbox_wsen(vector_cube, dummy_backend): + vc = vector_cube.filter_bbox(west=1, south=2, east=3, north=4) + assert dummy_backend.execute(vc, process_id="filter_bbox") == { + "process_id": "filter_bbox", + "arguments": {"data": {"from_node": "loadgeojson1"}, "extent": {"west": 1, "south": 2, "east": 3, "north": 4}}, + "result": True, + } + + +@pytest.mark.parametrize( + "extent", + [ + [1, 2, 3, 4], + (1, 2, 3, 4), + {"west": 1, "south": 2, "east": 3, "north": 4}, + ], +) +def test_filter_bbox_extent(vector_cube, dummy_backend, extent): + vc = vector_cube.filter_bbox(extent=extent) + assert dummy_backend.execute(vc, process_id="filter_bbox") == { + "process_id": "filter_bbox", + "arguments": {"data": {"from_node": "loadgeojson1"}, "extent": {"west": 1, "south": 2, "east": 3, "north": 4}}, + "result": True, + } + + +def test_filter_bbox_extent_parameter(vector_cube, dummy_backend): + vc = vector_cube.filter_bbox(extent=Parameter(name="the_extent")) + assert dummy_backend.execute(vc, process_id="filter_bbox") == { + "process_id": "filter_bbox", + "arguments": {"data": {"from_node": "loadgeojson1"}, "extent": {"from_parameter": "the_extent"}}, + "result": True, + } + + +@pytest.mark.parametrize( + ["kwargs", "expected"], + [ + ({}, "Can not construct BBoxDict from None"), + ({"west": 123, "south": 456, "east": 789}, "Missing bbox fields ['north']"), + ({"west": 123, "extent": [1, 2, 3, 4]}, "Don't specify both west/south/east/north and extent"), + ({"extent": [1, 2, 3]}, "Expected sequence with 4 items, but got 3."), + ({"extent": {"west": 1, "south": 2, "east": 3, "norht": 4}}, "Missing bbox fields ['north']"), + ], +) +def test_filter_bbox_invalid(vector_cube, dummy_backend, kwargs, expected): + with pytest.raises(InvalidBBoxException, match=re.escape(expected)): + _ = vector_cube.filter_bbox(**kwargs) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"extent": [1, 2, 3, 4], "crs": 4326}, + {"extent": {"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}}, + {"extent": {"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}, "crs": 4326}, + {"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}, + {"extent": [1, 2, 3, 4], "crs": "EPSG:4326"}, + ], +) +def test_filter_bbox_crs(vector_cube, dummy_backend, kwargs): + vc = vector_cube.filter_bbox(**kwargs) + assert dummy_backend.execute(vc, process_id="filter_bbox") == { + "process_id": "filter_bbox", + "arguments": { + "data": {"from_node": "loadgeojson1"}, + "extent": {"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}, + }, + "result": True, + } + + +def test_filter_labels_eq(vector_cube, dummy_backend): + vc = vector_cube.filter_labels(condition=lambda v: v == "B02", dimension="properties") + assert dummy_backend.execute(vc, process_id="filter_labels") == { + "process_id": "filter_labels", + "arguments": { + "data": {"from_node": "loadgeojson1"}, + "condition": { + "process_graph": { + "eq1": { + "arguments": {"x": {"from_parameter": "value"}, "y": "B02"}, + "process_id": "eq", + "result": True, + } + } + }, + "dimension": "properties", + }, + "result": True, + } + + +def test_filter_labels_contains(vector_cube, dummy_backend): + vc = vector_cube.filter_labels( + condition=lambda v: openeo.processes.array_contains(["B02", "B03"], v), dimension="properties" + ) + assert dummy_backend.execute(vc, process_id="filter_labels") == { + "process_id": "filter_labels", + "arguments": { + "data": {"from_node": "loadgeojson1"}, + "condition": { + "process_graph": { + "arraycontains1": { + "arguments": {"data": ["B02", "B03"], "value": {"from_parameter": "value"}}, + "process_id": "array_contains", + "result": True, + } + } + }, + "dimension": "properties", + }, + "result": True, + } + + +def test_filter_vector_vector_cube(vector_cube, con100, dummy_backend): + geometries = con100.load_geojson({"type": "Point", "coordinates": [3, 4]}) + vc = vector_cube.filter_vector(geometries=geometries) + assert dummy_backend.execute(vc) == { "loadgeojson1": { "process_id": "load_geojson", - "arguments": {"data": {"coordinates": [1, 2], "type": "Point"}, "properties": []}, + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, + "loadgeojson2": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [3, 4]}, "properties": []}, }, - "applydimension1": { - "process_id": "apply_dimension", + "filtervector1": { "arguments": { "data": {"from_node": "loadgeojson1"}, - "dimension": dimension, - "process": { - "process_graph": { - "sort1": { - "process_id": "sort", - "arguments": {"data": {"from_parameter": "data"}}, - "result": True, - } - } - }, + "geometries": {"from_node": "loadgeojson2"}, + "relation": "intersects", }, + "process_id": "filter_vector", "result": True, }, } - assert ( - f"Invalid dimension {dimension!r}. Should be one of ['geometry', 'properties']" in caplog.messages - ) == expect_warning + +@pytest.mark.parametrize( + "geometries", + [ + shapely.geometry.Point(3, 4), + {"type": "Point", "coordinates": [3, 4]}, + ], +) +def test_filter_vector_shapely(vector_cube, dummy_backend, geometries): + vc = vector_cube.filter_vector(geometries=geometries) + assert dummy_backend.execute(vc) == { + "loadgeojson1": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [1, 2]}, "properties": []}, + }, + "loadgeojson2": { + "process_id": "load_geojson", + "arguments": {"data": {"type": "Point", "coordinates": [3, 4]}, "properties": []}, + }, + "filtervector1": { + "arguments": { + "data": {"from_node": "loadgeojson1"}, + "geometries": {"from_node": "loadgeojson2"}, + "relation": "intersects", + }, + "process_id": "filter_vector", + "result": True, + }, + } diff --git a/tests/test_util.py b/tests/test_util.py index 311c3b4c5..5bb0b8dc4 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -34,6 +34,7 @@ str_truncate, to_bbox_dict, url_join, + InvalidBBoxException, ) @@ -685,7 +686,18 @@ class TestBBoxDict: def test_init(self): assert BBoxDict(west=1, south=2, east=3, north=4) == {"west": 1, "south": 2, "east": 3, "north": 4} assert BBoxDict(west=1, south=2, east=3, north=4, crs="EPSG:4326") == { - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326", + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, + } + assert BBoxDict(west=1, south=2, east=3, north=4, crs=4326) == { + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, } def test_repr(self): @@ -699,36 +711,72 @@ def test_to_json(self): def test_to_bbox_dict_from_sequence(self): assert to_bbox_dict([1, 2, 3, 4]) == {"west": 1, "south": 2, "east": 3, "north": 4} assert to_bbox_dict((1, 2, 3, 4)) == {"west": 1, "south": 2, "east": 3, "north": 4} + assert to_bbox_dict([1, 2, 3, 4], crs=4326) == { + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, + } assert to_bbox_dict([1, 2, 3, 4], crs="EPSG:4326") == { - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326", + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, } def test_to_bbox_dict_from_sequence_mismatch(self): - with pytest.raises(ValueError, match="Expected sequence with 4 items, but got 3."): + with pytest.raises(InvalidBBoxException, match="Expected sequence with 4 items, but got 3."): to_bbox_dict([1, 2, 3]) - with pytest.raises(ValueError, match="Expected sequence with 4 items, but got 5."): + with pytest.raises(InvalidBBoxException, match="Expected sequence with 4 items, but got 5."): to_bbox_dict([1, 2, 3, 4, 5]) def test_to_bbox_dict_from_dict(self): assert to_bbox_dict({"west": 1, "south": 2, "east": 3, "north": 4}) == { "west": 1, "south": 2, "east": 3, "north": 4 } + assert to_bbox_dict({"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}) == { + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, + } assert to_bbox_dict({"west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326"}) == { - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326" + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, } assert to_bbox_dict({"west": 1, "south": 2, "east": 3, "north": 4}, crs="EPSG:4326") == { - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326", + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": 4326, } - assert to_bbox_dict({ - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326", "color": "red", "other": "garbage", - }) == { - "west": 1, "south": 2, "east": 3, "north": 4, "crs": "EPSG:4326" - } + assert to_bbox_dict( + { + "west": 1, + "south": 2, + "east": 3, + "north": 4, + "crs": "EPSG:4326", + "color": "red", + "other": "garbage", + } + ) == {"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326} def test_to_bbox_dict_from_dict_missing_field(self): - with pytest.raises(ValueError, match="but only found {'east'}"): + with pytest.raises(InvalidBBoxException, match=re.escape("Missing bbox fields ['north', 'south', 'west']")): to_bbox_dict({"east": 3}) + def test_to_bbox_dict_multiple_crs(self): + with pytest.raises(InvalidBBoxException, match="Two CRS values specified: EPSG:32631 and 4326"): + _ = to_bbox_dict({"west": 1, "south": 2, "east": 3, "north": 4, "crs": 4326}, crs="EPSG:32631") + def test_to_bbox_dict_from_geometry(self): geometry = shapely.geometry.Polygon([(4, 2), (7, 4), (5, 8), (3, 3), (4, 2)]) assert to_bbox_dict(geometry) == {"west": 3, "south": 2, "east": 7, "north": 8} From ba32cca30301efcb2a242fc33e69849ae567407b Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 16:48:08 +0200 Subject: [PATCH 04/12] f-string fixup for python<3.8 --- openeo/rest/_testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index fb44674e7..267df89d7 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -108,7 +108,7 @@ def get_pg(self, process_id: Optional[str] = None) -> dict: found = [node for node in pg.values() if node.get("process_id") == process_id] if len(found) != 1: raise RuntimeError( - f"Expected single process graph node with {process_id=}, but found {len(found)}: {found}" + f"Expected single process graph node with process_id {process_id!r}, but found {len(found)}: {found}" ) return found[0] return pg From b2a0e00ef9f61c75a91167328ded9a09f6a43522 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 18:47:19 +0200 Subject: [PATCH 05/12] Issue #259/#453#458 finetune crs normalization some more - rename to `normalize_crs` because it is not only about EPSG output, WKT2 output is also allowed - reworked and simplified pyproj availability handling: when available: use it fully, when not: do best effort normalization - Move tests into class for better grouping and overview - add test coverage for "no pyproj available" code path --- CHANGELOG.md | 6 +- openeo/rest/datacube.py | 6 +- openeo/rest/vectorcube.py | 2 +- openeo/util.py | 119 ++++++-------- tests/test_util.py | 329 ++++++++++++++++++++------------------ 5 files changed, 230 insertions(+), 232 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6eefb00a..c24da85cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - -- Processes that take a CRS as argument now try harder to convert your input into a proper EPSG code, to avoid unexpected results when an invalid argument gets sent to the backend. +- Processes that take a CRS as argument now try harder to normalize your input to + a CRS representation that aligns with the openEO API (using `pyproj` library when available) + ([#259](https://github.com/Open-EO/openeo-python-client/issues/259)) - Initial `load_geojson` support with `Connection.load_geojson()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) - Initial `load_url` (for vector cubes) support with `Connection.load_url()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) - Support lambda based property filtering in `Connection.load_stac()` ([#425](https://github.com/Open-EO/openeo-python-client/issues/425)) @@ -27,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix: MultibackendJobManager should stop when finished, also when job finishes with error ([#452](https://github.com/Open-EO/openeo-python-client/issues/432)) + ## [0.21.1] - 2023-07-19 ### Fixed diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index e234e0b4a..1cff5d08e 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -37,7 +37,7 @@ from openeo.rest.service import Service from openeo.rest.udp import RESTUserDefinedProcess from openeo.rest.vectorcube import VectorCube -from openeo.util import get_temporal_extent, dict_no_none, rfc3339, guess_format, crs_to_epsg_code +from openeo.util import get_temporal_extent, dict_no_none, rfc3339, guess_format, normalize_crs if typing.TYPE_CHECKING: # Imports for type checking only (circular import issue at runtime). @@ -332,7 +332,7 @@ def filter_bbox( " Use keyword arguments or tuple/list argument instead.") west, east, north, south = args[:4] if len(args) > 4: - crs = crs_to_epsg_code(args[4]) + crs = normalize_crs(args[4]) elif len(args) == 1 and (isinstance(args[0], (list, tuple)) and len(args[0]) == 4 or isinstance(args[0], (dict, shapely.geometry.base.BaseGeometry, Parameter))): bbox = args[0] @@ -834,7 +834,7 @@ def _get_geometry_argument( # TODO: don't warn when the crs is Lon-Lat like EPSG:4326? warnings.warn(f"Geometry with non-Lon-Lat CRS {crs!r} is only supported by specific back-ends.") # TODO #204 alternative for non-standard CRS in GeoJSON object? - epsg_code = crs_to_epsg_code(crs) + epsg_code = normalize_crs(crs) if epsg_code is not None: # proj did recognize the CRS crs_name = f"EPSG:{epsg_code}" diff --git a/openeo/rest/vectorcube.py b/openeo/rest/vectorcube.py index cd7a6b4ec..a266c67bb 100644 --- a/openeo/rest/vectorcube.py +++ b/openeo/rest/vectorcube.py @@ -14,7 +14,7 @@ from openeo.rest._datacube import THIS, UDF, _ProcessGraphAbstraction, build_child_callback from openeo.rest.job import BatchJob from openeo.rest.mlmodel import MlModel -from openeo.util import dict_no_none, guess_format, crs_to_epsg_code, to_bbox_dict, InvalidBBoxException +from openeo.util import dict_no_none, guess_format, to_bbox_dict, InvalidBBoxException if typing.TYPE_CHECKING: # Imports for type checking only (circular import issue at runtime). diff --git a/openeo/util.py b/openeo/util.py index 129a33115..ff2c86ed7 100644 --- a/openeo/util.py +++ b/openeo/util.py @@ -18,6 +18,13 @@ import shapely.geometry.base from deprecated import deprecated +try: + # pyproj is an optional dependency + import pyproj +except ImportError: + pyproj = None + + logger = logging.getLogger(__name__) @@ -535,7 +542,7 @@ class BBoxDict(dict): def __init__(self, *, west: float, south: float, east: float, north: float, crs: Optional[Union[str, int]] = None): super().__init__(west=west, south=south, east=east, north=north) if crs is not None: - self.update(crs=crs_to_epsg_code(crs)) + self.update(crs=normalize_crs(crs)) # TODO: provide west, south, east, north, crs as @properties? Read-only or read-write? @@ -635,86 +642,58 @@ def get(self, fraction: float) -> str: return f"{self.left}{bar:{self.fill}<{width}s}{self.right}" -def crs_to_epsg_code(crs: Union[str, int, dict, None]) -> Optional[int]: - """Convert a CRS string or int to an integer EPGS code, where CRS usually comes from user input. - - Three cases: - - - If it is already an integer we just keep it. - - If it is None it stays None, and empty strings become None as well. - - If it is a string we try to parse it with the pyproj library. - - Strings of the form "EPSG:" will be converted to teh value - - For any other strings formats, it will work if pyproj supports is, - otherwise it won't. - - The result is **always** an EPSG code, so the CRS should be one that is - defined in EPSG. For any other definitions pyproj will only give you the - closest EPSG match and that result is possibly inaccurate. - - Note that we also need to support WKT string (WKT2), - see also: https://github.com/Open-EO/openeo-processes/issues/58 +def normalize_crs(crs: Any, *, use_pyproj: bool = True) -> Union[None, int, str]: + """ + Normalize given data structure (typically just an int or string) + that encodes a CRS (Coordinate Reference System) to an EPSG (int) code or WKT2 CRS string. - For very the oldest supported version of Python: v3.6 there is a problem - because the pyproj version that is compatible with Python 3.6 is too old - and does not properly support WKT2. + Behavior and data structure support depends on the availability of the ``pyproj`` library: + - If the ``pyproj`` library is available: use that to do parsing and conversion. + This means that anything that is supported by ``pyproj.CRS.from_user_input`` is allowed. + See the ``pyproj`` docs for more details. + - Otherwise, some best effort validation is done: + EPSG looking int/str values will be parsed as such, other strings will be assumed to be WKT2 already. + Other data structures will not be accepted. - For a list of CRS input formats that proj supports - see: https://pyproj4.github.io/pyproj/stable/api/crs/crs.html#pyproj.crs.CRS.from_user_input + :param crs: data structure that encodes a CRS, typically just an int or string value. + If the ``pyproj`` library is available, everything supported by it is allowed + :param use_pyproj: whether ``pyproj`` should be leveraged at all + (mainly useful for testing the "no pyproj available" code path) - :param crs: - Input from user for the Coordinate Reference System to convert to an - EPSG code. + :return: EPSG code as int, or WKT2 string. Or None if input was empty . :raises ValueError: - When the crs is a not a supported CRS string. - :raises TypeError: - When crs is none of the supported types: str, int, None + When the given CRS data can not be parsed/converted/normalized. - :return: An EPGS code if it could be found, otherwise None """ - - # Only convert to the default if it is an explicitly allowed type. if crs in (None, "", {}): return None - if not isinstance(crs, (int, str, dict)): - raise TypeError("The allowed type for the parameter 'crs' are: str, int, dict and None") - - # If we want to stop processing as soon as we have an int value, then we - # should not accept values that are complete non-sense, as best as we can. - crs_intermediate = crs - if isinstance(crs, int): - crs_intermediate = crs - elif isinstance(crs, str): - # This conversion is needed to support strings that only contain an integer, - # e.g. "4326" though it is a string, is a otherwise a correct EPSG code. + if pyproj and use_pyproj: try: - crs_intermediate = int(crs) - except ValueError as exc: - # So we need to process it with pyproj, below. - logger.debug("crs_to_epsg_code received crs input that was not an int: crs={crs}, exception caught: {exc}") - - if isinstance(crs_intermediate, int): - if crs_intermediate <= 0: - raise ValueError(f"When crs is an integer value it has to be > 0.") - else: - return crs_intermediate - - try: - import pyproj.crs - except ImportError as exc: - message = ( - f"Cannot convert CRS string: {crs}. " - + "Need pyproj to convert this CRS string but the pyproj library is not installed." - ) - logger.error(message) - raise ValueError(message) from ImportError + # (if available:) let pyproj do the validation/parsing + crs_obj = pyproj.CRS.from_user_input(crs) + # Convert back to EPSG int or WKT2 string + crs = crs_obj.to_epsg() or crs_obj.to_wkt() + except pyproj.ProjError as e: + raise ValueError(f"Failed to normalize CRS data with pyproj: {crs}") from e else: - try: - converted_crs = pyproj.crs.CRS.from_user_input(crs) - except pyproj.exceptions.CRSError as exc: - logger.error(f"Could not convert CRS string to EPSG code: crs={crs}, exception: {exc}", exc_info=True) - raise ValueError(crs) from exc + # Best effort simple validation/normalization + if isinstance(crs, int) and crs > 0: + # Assume int is already valid EPSG code + pass + elif isinstance(crs, str): + # Parse as EPSG int code if it looks like that, + # otherwise: leave it as-is, assuming it is a valid WKT2 CRS string + if re.match(r"^(epsg:)?\d+$", crs.strip(), flags=re.IGNORECASE): + crs = int(crs.split(":")[-1]) + elif "GEOGCRS[" in crs: + # Very simple WKT2 CRS detection heuristic + logger.warning(f"Assuming this is a valid WK2 CRS string: {repr_truncate(crs)}") + else: + raise ValueError(f"Can not normalize CRS string {repr_truncate(crs)}") else: - return converted_crs.to_epsg() + raise ValueError(f"Can not normalize CRS data {type(crs)}") + + return crs diff --git a/tests/test_util.py b/tests/test_util.py index 5bb0b8dc4..6876ae6f5 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -20,7 +20,7 @@ SimpleProgressBar, TimingLogger, clip, - crs_to_epsg_code, + normalize_crs, deep_get, deep_set, dict_no_none, @@ -818,7 +818,9 @@ def test_clip_and_overflow(self): assert pgb.get(1.5) == "[=####################################=]" -WKT2_FOR_EPSG32631 = """ +class TestNormalizeCrs: + WKT2_FOR_EPSG4326 = 'GEOGCRS["WGS 84",ENSEMBLE["World Geodetic System 1984 ensemble",MEMBER["World Geodetic System 1984 (Transit)"],MEMBER["World Geodetic System 1984 (G730)"],MEMBER["World Geodetic System 1984 (G873)"],MEMBER["World Geodetic System 1984 (G1150)"],MEMBER["World Geodetic System 1984 (G1674)"],MEMBER["World Geodetic System 1984 (G1762)"],MEMBER["World Geodetic System 1984 (G2139)"],ELLIPSOID["WGS 84",6378137,298.257223563,LENGTHUNIT["metre",1]],ENSEMBLEACCURACY[2.0]],PRIMEM["Greenwich",0,ANGLEUNIT["degree",0.0174532925199433]],CS[ellipsoidal,2],AXIS["geodetic latitude (Lat)",north,ORDER[1],ANGLEUNIT["degree",0.0174532925199433]],AXIS["geodetic longitude (Lon)",east,ORDER[2],ANGLEUNIT["degree",0.0174532925199433]],USAGE[SCOPE["Horizontal component of 3D system."],AREA["World."],BBOX[-90,-180,90,180]],ID["EPSG",4326]]' + WKT2_FOR_EPSG32631 = """ PROJCRS["WGS 84 / UTM zone 31N", BASEGEOGCRS["WGS 84", ENSEMBLE["World Geodetic System 1984 ensemble", @@ -867,170 +869,185 @@ def test_clip_and_overflow(self): ID["EPSG",32631]] """ + @pytest.mark.parametrize( + ["epsg_input", "expected"], + [ + ("epsg:4326", 4326), + ("EPSG:4326", 4326), + ("Epsg:4326", 4326), + ("epsg:32165", 32165), + ("EPSG:32165", 32165), + ("Epsg:32165", 32165), + (4326, 4326), + (32165, 32165), + ("4326", 4326), + ("32165", 32165), + (None, None), + ("", None), + ({}, None), # Should treat empty dict for PROJJSON the same way as "" or None + # also likely to occur + ("WGS84", 4326), + ], + ) + def test_normalize_crs_succeeds_with_correct_crses(self, epsg_input, expected): + """Happy path, values that are allowed""" + assert normalize_crs(epsg_input) == expected -@pytest.mark.parametrize( - ["epsg_input", "expected"], - [ - ("epsg:4326", 4326), - ("EPSG:4326", 4326), - ("Epsg:4326", 4326), - ("epsg:32165", 32165), - ("EPSG:32165", 32165), - ("Epsg:32165", 32165), - (4326, 4326), - (32165, 32165), - ("4326", 4326), - ("32165", 32165), - (None, None), - ("", None), - ({}, None), # Should treat empty dict for PROJJSON the same way as "" or None - # also likely to occur - ("WGS84", 4326), - ], -) -def test_crs_to_epsg_code_succeeds_with_correct_crses(epsg_input, expected): - """Happy path, values that are allowed""" - assert crs_to_epsg_code(epsg_input) == expected - - -@pytest.mark.skipif(sys.version_info < (3, 7), reason="WKT2 format not supported by pyproj 3.0 / python 3.6") -def test_crs_to_epsg_code_succeeds_with_wkt2_input(): - """Test can handle WKT2 strings. - - We need to support WKT2: - See also https://github.com/Open-EO/openeo-processes/issues/58 - - - WARNING: - ======= - - Older versions of pyproj do not support this format. - In particular, pyproj 3.0 which is the version we get on python 3.6, would - fail on this test, and is marked with a skipif for that reason. - """ - assert crs_to_epsg_code(WKT2_FOR_EPSG32631) == 32631 - - -PROJJSON_FOR_EPSG32631 = { - "$schema": "https://proj.org/schemas/v0.4/projjson.schema.json", - "type": "ProjectedCRS", - "name": "WGS 84 / UTM zone 31N", - "base_crs": { - "name": "WGS 84", - "datum_ensemble": { - "name": "World Geodetic System 1984 ensemble", - "members": [ - {"name": "World Geodetic System 1984 (Transit)", "id": {"authority": "EPSG", "code": 1166}}, - {"name": "World Geodetic System 1984 (G730)", "id": {"authority": "EPSG", "code": 1152}}, - {"name": "World Geodetic System 1984 (G873)", "id": {"authority": "EPSG", "code": 1153}}, - {"name": "World Geodetic System 1984 (G1150)", "id": {"authority": "EPSG", "code": 1154}}, - {"name": "World Geodetic System 1984 (G1674)", "id": {"authority": "EPSG", "code": 1155}}, - {"name": "World Geodetic System 1984 (G1762)", "id": {"authority": "EPSG", "code": 1156}}, - {"name": "World Geodetic System 1984 (G2139)", "id": {"authority": "EPSG", "code": 1309}}, + @pytest.mark.parametrize( + ["epsg_input", "expected"], + [ + ("epsg:4326", 4326), + ("EPSG:4326", 4326), + ("Epsg:4326", 4326), + ("epsg:32165", 32165), + ("EPSG:32165", 32165), + ("Epsg:32165", 32165), + (4326, 4326), + (32165, 32165), + ("4326", 4326), + ("32165", 32165), + (None, None), + ("", None), + ({}, None), # Should treat empty dict for PROJJSON the same way as "" or None + ('GEOGCRS["looks like WKT2"]', 'GEOGCRS["looks like WKT2"]'), + ], + ) + def test_normalize_crs_without_pyproj_succeeds_with_correct_crses(self, epsg_input, expected): + """Happy path, values that are allowed""" + assert normalize_crs(epsg_input, use_pyproj=False) == expected + + def test_normalize_crs_without_pyproj_accept_non_epsg_string(self, caplog): + """Happy path, values that are allowed""" + caplog.set_level(logging.WARNING) + crs = self.WKT2_FOR_EPSG4326 + assert normalize_crs(crs, use_pyproj=False) == crs + assert ( + """Assuming this is a valid WK2 CRS string: 'GEOGCRS["WGS 84",ENSEMBLE["World Geodetic System 1984 ensem...'""" + in caplog.text + ) + + @pytest.mark.skipif(sys.version_info < (3, 7), reason="WKT2 format not supported by pyproj 3.0 / python 3.6") + def test_normalize_crs_succeeds_with_wkt2_input(self): + """Test can handle WKT2 strings. + + We need to support WKT2: + See also https://github.com/Open-EO/openeo-processes/issues/58 + + + WARNING: + ======= + + Older versions of pyproj do not support this format. + In particular, pyproj 3.0 which is the version we get on python 3.6, would + fail on this test, and is marked with a skipif for that reason. + """ + assert normalize_crs(self.WKT2_FOR_EPSG32631) == 32631 + + PROJJSON_FOR_EPSG32631 = { + "$schema": "https://proj.org/schemas/v0.4/projjson.schema.json", + "type": "ProjectedCRS", + "name": "WGS 84 / UTM zone 31N", + "base_crs": { + "name": "WGS 84", + "datum_ensemble": { + "name": "World Geodetic System 1984 ensemble", + "members": [ + {"name": "World Geodetic System 1984 (Transit)", "id": {"authority": "EPSG", "code": 1166}}, + {"name": "World Geodetic System 1984 (G730)", "id": {"authority": "EPSG", "code": 1152}}, + {"name": "World Geodetic System 1984 (G873)", "id": {"authority": "EPSG", "code": 1153}}, + {"name": "World Geodetic System 1984 (G1150)", "id": {"authority": "EPSG", "code": 1154}}, + {"name": "World Geodetic System 1984 (G1674)", "id": {"authority": "EPSG", "code": 1155}}, + {"name": "World Geodetic System 1984 (G1762)", "id": {"authority": "EPSG", "code": 1156}}, + {"name": "World Geodetic System 1984 (G2139)", "id": {"authority": "EPSG", "code": 1309}}, + ], + "ellipsoid": {"name": "WGS 84", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563}, + "accuracy": "2.0", + "id": {"authority": "EPSG", "code": 6326}, + }, + "coordinate_system": { + "subtype": "ellipsoidal", + "axis": [ + {"name": "Geodetic latitude", "abbreviation": "Lat", "direction": "north", "unit": "degree"}, + {"name": "Geodetic longitude", "abbreviation": "Lon", "direction": "east", "unit": "degree"}, + ], + }, + "id": {"authority": "EPSG", "code": 4326}, + }, + "conversion": { + "name": "UTM zone 31N", + "method": {"name": "Transverse Mercator", "id": {"authority": "EPSG", "code": 9807}}, + "parameters": [ + { + "name": "Latitude of natural origin", + "value": 0, + "unit": "degree", + "id": {"authority": "EPSG", "code": 8801}, + }, + { + "name": "Longitude of natural origin", + "value": 3, + "unit": "degree", + "id": {"authority": "EPSG", "code": 8802}, + }, + { + "name": "Scale factor at natural origin", + "value": 0.9996, + "unit": "unity", + "id": {"authority": "EPSG", "code": 8805}, + }, + {"name": "False easting", "value": 500000, "unit": "metre", "id": {"authority": "EPSG", "code": 8806}}, + {"name": "False northing", "value": 0, "unit": "metre", "id": {"authority": "EPSG", "code": 8807}}, ], - "ellipsoid": {"name": "WGS 84", "semi_major_axis": 6378137, "inverse_flattening": 298.257223563}, - "accuracy": "2.0", - "id": {"authority": "EPSG", "code": 6326}, }, "coordinate_system": { - "subtype": "ellipsoidal", + "subtype": "Cartesian", "axis": [ - {"name": "Geodetic latitude", "abbreviation": "Lat", "direction": "north", "unit": "degree"}, - {"name": "Geodetic longitude", "abbreviation": "Lon", "direction": "east", "unit": "degree"}, + {"name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre"}, + {"name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre"}, ], }, - "id": {"authority": "EPSG", "code": 4326}, - }, - "conversion": { - "name": "UTM zone 31N", - "method": {"name": "Transverse Mercator", "id": {"authority": "EPSG", "code": 9807}}, - "parameters": [ - { - "name": "Latitude of natural origin", - "value": 0, - "unit": "degree", - "id": {"authority": "EPSG", "code": 8801}, - }, - { - "name": "Longitude of natural origin", - "value": 3, - "unit": "degree", - "id": {"authority": "EPSG", "code": 8802}, - }, - { - "name": "Scale factor at natural origin", - "value": 0.9996, - "unit": "unity", - "id": {"authority": "EPSG", "code": 8805}, - }, - {"name": "False easting", "value": 500000, "unit": "metre", "id": {"authority": "EPSG", "code": 8806}}, - {"name": "False northing", "value": 0, "unit": "metre", "id": {"authority": "EPSG", "code": 8807}}, - ], - }, - "coordinate_system": { - "subtype": "Cartesian", - "axis": [ - {"name": "Easting", "abbreviation": "E", "direction": "east", "unit": "metre"}, - {"name": "Northing", "abbreviation": "N", "direction": "north", "unit": "metre"}, - ], - }, - "scope": "Engineering survey, topographic mapping.", - "area": "Between 0°E and 6°E, northern hemisphere between equator and 84°N, onshore and offshore. Algeria. Andorra. Belgium. Benin. Burkina Faso. Denmark - North Sea. France. Germany - North Sea. Ghana. Luxembourg. Mali. Netherlands. Niger. Nigeria. Norway. Spain. Togo. United Kingdom (UK) - North Sea.", - "bbox": {"south_latitude": 0, "west_longitude": 0, "north_latitude": 84, "east_longitude": 6}, - "id": {"authority": "EPSG", "code": 32631}, -} - - -@pytest.mark.skipif(sys.version_info < (3, 8), reason="PROJJSON format not supported by pyproj v3.2 / python < v3.8") -def test_crs_to_epsg_code_succeeds_with_correct_projjson(): - json_str = json.dumps(PROJJSON_FOR_EPSG32631) - - # It should work with both a JSON string as well as the dict that - # represents that same JSON. - assert crs_to_epsg_code(json_str) == 32631 - assert crs_to_epsg_code(PROJJSON_FOR_EPSG32631) == 32631 - - -@pytest.mark.parametrize( - ["epsg_input", "expected"], - [ - ("+proj=latlon", 4326), - ("+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs", 32631), - ], -) -def test_crs_to_epsg_code_succeeds_with_correct_projstring(epsg_input, expected): - """These are more advanced inputs that pyproj should support, though - the proj format is now discouraged, in favor of WKT2 and PROJJSON. - - See also https://github.com/Open-EO/openeo-processes/issues/58 - - Contrary to WKT, it seems less likely that users would ask for these - proj options. Hence a separate test. - """ - assert crs_to_epsg_code(epsg_input) == expected + "scope": "Engineering survey, topographic mapping.", + "area": "Between 0°E and 6°E, northern hemisphere between equator and 84°N, onshore and offshore. Algeria. Andorra. Belgium. Benin. Burkina Faso. Denmark - North Sea. France. Germany - North Sea. Ghana. Luxembourg. Mali. Netherlands. Niger. Nigeria. Norway. Spain. Togo. United Kingdom (UK) - North Sea.", + "bbox": {"south_latitude": 0, "west_longitude": 0, "north_latitude": 84, "east_longitude": 6}, + "id": {"authority": "EPSG", "code": 32631}, + } + @pytest.mark.skipif( + sys.version_info < (3, 8), reason="PROJJSON format not supported by pyproj v3.2 / python < v3.8" + ) + def test_normalize_crs_succeeds_with_correct_projjson( + self, + ): + json_str = json.dumps(self.PROJJSON_FOR_EPSG32631) -@pytest.mark.parametrize( - "epsg_input", - ["doesnotexist", "unknownauthority:123", "4326.0"], -) -def test_crs_to_epsg_code_handles_incorrect_crs(epsg_input): - with pytest.raises(ValueError): - crs_to_epsg_code(epsg_input) + # It should work with both a JSON string as well as the dict that + # represents that same JSON. + assert normalize_crs(json_str) == 32631 + assert normalize_crs(self.PROJJSON_FOR_EPSG32631) == 32631 + @pytest.mark.parametrize( + ["epsg_input", "expected"], + [ + ("+proj=latlon", 4326), + ("+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs", 32631), + ], + ) + def test_normalize_crs_succeeds_with_correct_projstring(self, epsg_input, expected): + """These are more advanced inputs that pyproj should support, though + the proj format is now discouraged, in favor of WKT2 and PROJJSON. -@pytest.mark.parametrize("epsg_input", [0.0, 1.0, 10.0, 4326.0, []]) -def test_crs_to_epsg_code_raises_typeerror(epsg_input): - """Verify we restrict the allowed input types to int, str and None.""" - with pytest.raises(TypeError): - crs_to_epsg_code(epsg_input) + See also https://github.com/Open-EO/openeo-processes/issues/58 + Contrary to WKT, it seems less likely that users would ask for these + proj options. Hence a separate test. + """ + assert normalize_crs(epsg_input) == expected -@pytest.mark.parametrize( - "epsg_input", - [0, "0", -1, "-1", -321654643], -) -def test_crs_to_epsg_code_raises_valueerror(epsg_input): - """EPSG codes can not be 0 or negative.""" - with pytest.raises(ValueError): - crs_to_epsg_code(epsg_input) + @pytest.mark.parametrize( + "epsg_input", + ["doesnotexist", "unknownauthority:123", "4326.0", 0.0, 123.456, 4326.0, [], -4326, "-4326", {"foo": "bar"}], + ) + @pytest.mark.parametrize("use_pyproj", [False, True]) + def test_normalize_crs_handles_incorrect_crs(self, epsg_input, use_pyproj): + with pytest.raises(ValueError): + normalize_crs(epsg_input, use_pyproj=use_pyproj) From cf57056069c56f80ea65370e550dd17e3204cca0 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 20:55:17 +0200 Subject: [PATCH 06/12] Issue #259/#453/#458 Fixup pyproj related test skips Skip based on pyproj version iso python version (allows testing the skips on higher python versions too) --- openeo/util.py | 2 +- setup.py | 1 + tests/test_util.py | 27 +++++++++++++++++---------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/openeo/util.py b/openeo/util.py index ff2c86ed7..bd9d8814a 100644 --- a/openeo/util.py +++ b/openeo/util.py @@ -677,7 +677,7 @@ def normalize_crs(crs: Any, *, use_pyproj: bool = True) -> Union[None, int, str] # Convert back to EPSG int or WKT2 string crs = crs_obj.to_epsg() or crs_obj.to_wkt() except pyproj.ProjError as e: - raise ValueError(f"Failed to normalize CRS data with pyproj: {crs}") from e + raise ValueError(f"Failed to normalize CRS data with pyproj: {crs!r}") from e else: # Best effort simple validation/normalization if isinstance(crs, int) and crs > 0: diff --git a/setup.py b/setup.py index b6decb29f..4045e7887 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ "geopandas", "flake8>=5.0.0", "time_machine", + "pyproj", # Pyproj is an optional, best-effort runtime dependency ] docs_require = [ diff --git a/tests/test_util.py b/tests/test_util.py index 6876ae6f5..2052af2a3 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -8,9 +8,11 @@ import unittest.mock as mock from typing import List, Union +import pyproj import pytest import shapely.geometry +from openeo.capabilities import ComparableVersion from openeo.util import ( BBoxDict, ContextTimer, @@ -891,6 +893,10 @@ class TestNormalizeCrs: ) def test_normalize_crs_succeeds_with_correct_crses(self, epsg_input, expected): """Happy path, values that are allowed""" + if isinstance(epsg_input, str) and epsg_input.isnumeric() and pyproj.__version__ < ComparableVersion("3.3.1"): + # TODO #460 this skip is only necessary for python 3.6 and lower + pytest.skip("pyproj below 3.3.1 does not support int-like strings") + assert normalize_crs(epsg_input) == expected @pytest.mark.parametrize( @@ -926,23 +932,22 @@ def test_normalize_crs_without_pyproj_accept_non_epsg_string(self, caplog): in caplog.text ) - @pytest.mark.skipif(sys.version_info < (3, 7), reason="WKT2 format not supported by pyproj 3.0 / python 3.6") + @pytest.mark.skipif( + # TODO #460 this skip is only necessary for python 3.6 and lower + pyproj.__version__ < ComparableVersion("3.1.0"), + reason="WKT2 format support requires pypro 3.1.0 or higher", + ) def test_normalize_crs_succeeds_with_wkt2_input(self): """Test can handle WKT2 strings. We need to support WKT2: See also https://github.com/Open-EO/openeo-processes/issues/58 - - - WARNING: - ======= - - Older versions of pyproj do not support this format. - In particular, pyproj 3.0 which is the version we get on python 3.6, would - fail on this test, and is marked with a skipif for that reason. """ assert normalize_crs(self.WKT2_FOR_EPSG32631) == 32631 + def test_normalize_crs_without_pyproj_succeeds_with_wkt2_input(self): + assert normalize_crs(self.WKT2_FOR_EPSG32631, use_pyproj=False) == self.WKT2_FOR_EPSG32631 + PROJJSON_FOR_EPSG32631 = { "$schema": "https://proj.org/schemas/v0.4/projjson.schema.json", "type": "ProjectedCRS", @@ -1013,7 +1018,9 @@ def test_normalize_crs_succeeds_with_wkt2_input(self): } @pytest.mark.skipif( - sys.version_info < (3, 8), reason="PROJJSON format not supported by pyproj v3.2 / python < v3.8" + # TODO #460 this skip is only necessary for python 3.6 and lower + pyproj.__version__ < ComparableVersion("3.3.0"), + reason="PROJJSON format requires pyproj 3.3.0 or higher", ) def test_normalize_crs_succeeds_with_correct_projjson( self, From 49f6c263aaa414798716213fe1a87a3b3fa14279 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 21:27:30 +0200 Subject: [PATCH 07/12] fixup! Issue #259/#453/#458 Fixup pyproj related test skips --- setup.py | 2 +- tests/rest/datacube/test_datacube100.py | 90 ++++++++----------------- tests/test_util.py | 4 +- 3 files changed, 30 insertions(+), 66 deletions(-) diff --git a/setup.py b/setup.py index 4045e7887..99fe8cae3 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ "geopandas", "flake8>=5.0.0", "time_machine", - "pyproj", # Pyproj is an optional, best-effort runtime dependency + "pyproj", # Pyproj is an optional, best-effort runtime dependency # TODO #460 set a high enough minimum version when py3.6 support can be dropped ] docs_require = [ diff --git a/tests/rest/datacube/test_datacube100.py b/tests/rest/datacube/test_datacube100.py index d50e292d6..d3183d394 100644 --- a/tests/rest/datacube/test_datacube100.py +++ b/tests/rest/datacube/test_datacube100.py @@ -13,6 +13,7 @@ import textwrap from typing import Optional +import pyproj import pytest import requests import shapely.geometry @@ -20,6 +21,7 @@ import openeo.metadata import openeo.processes from openeo.api.process import Parameter +from openeo.capabilities import ComparableVersion from openeo.internal.graph_building import PGNode from openeo.internal.process_graph_visitor import ProcessGraphVisitException from openeo.internal.warnings import UserDeprecationWarning @@ -182,12 +184,21 @@ } -CRS_VALUES_SUPPORTED_FOR_ALL_PYTHON_VERSIONS = [ - "EPSG:32631", - "32631", - 32631, - "+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs", # is also EPSG:32631, in proj format -] +def _get_normalizable_crs_inputs(): + """ + Dynamic (proj version based) generation of supported CRS inputs (to normalize). + :return: + """ + yield "EPSG:32631" + yield 32631 + if pyproj.__version__ >= ComparableVersion("3.3.1"): + # pyproj below 3.3.1 does not support int-like strings + # TODO #460 this skip is only necessary for python 3.6 and lower + yield "32631" + yield "+proj=utm +zone=31 +datum=WGS84 +units=m +no_defs" # is also EPSG:32631, in proj format + if pyproj.__version__ >= ComparableVersion("3.1.0"): + # WKT2 format support requires pyproj 3.1.0 or higher + yield WKT2_FOR_EPSG23631 def _get_leaf_node(cube: DataCube) -> dict: @@ -405,7 +416,7 @@ def test_aggregate_spatial_types(con100: Connection, polygon, expected_geometrie } -@pytest.mark.parametrize("crs", CRS_VALUES_SUPPORTED_FOR_ALL_PYTHON_VERSIONS) +@pytest.mark.parametrize("crs", _get_normalizable_crs_inputs()) def test_aggregate_spatial_with_crs(con100: Connection, recwarn, crs: str): img = con100.load_collection("S2") polygon = shapely.geometry.box(0, 0, 1, 1) @@ -430,36 +441,10 @@ def test_aggregate_spatial_with_crs(con100: Connection, recwarn, crs: str): } -@pytest.mark.skipif(sys.version_info < (3, 7), reason="WKT2 format not supported by pyproj 3.0 / python 3.6") -def test_aggregate_spatial_with_crs_as_wkt(con100: Connection, recwarn): - """Separate test coverage for WKT, so we can skip it for Python3.6""" - crs = WKT2_FOR_EPSG23631 - img = con100.load_collection("S2") - polygon = shapely.geometry.box(0, 0, 1, 1) - masked = img.aggregate_spatial(geometries=polygon, reducer="mean", crs=crs) - warnings = [str(w.message) for w in recwarn] - assert f"Geometry with non-Lon-Lat CRS {crs!r} is only supported by specific back-ends." in warnings - assert sorted(masked.flat_graph().keys()) == ["aggregatespatial1", "loadcollection1"] - assert masked.flat_graph()["aggregatespatial1"] == { - "process_id": "aggregate_spatial", - "arguments": { - "data": {"from_node": "loadcollection1"}, - "geometries": { - "type": "Polygon", - "coordinates": (((1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0), (1.0, 0.0)),), - "crs": {"properties": {"name": "EPSG:32631"}, "type": "name"}, - }, - "reducer": { - "process_graph": { - "mean1": {"process_id": "mean", "arguments": {"data": {"from_parameter": "data"}}, "result": True} - } - }, - }, - "result": True, - } - - -@pytest.mark.skipif(sys.version_info < (3, 8), reason="PROJJSON format not supported by pyproj 3.2 / python < v3.8") +@pytest.mark.skipif( + pyproj.__version__ < ComparableVersion("3.3.0"), + reason="PROJJSON format support requires pyproj 3.3.0 or higher", +) @pytest.mark.parametrize("crs", [PROJJSON_FOR_EPSG23631, json.dumps(PROJJSON_FOR_EPSG23631)]) def test_aggregate_spatial_with_crs_as_projjson(con100: Connection, recwarn, crs): """Separate test coverage for PROJJSON, so we can skip it for Python versions below 3.8""" @@ -686,7 +671,7 @@ def test_mask_polygon_types(con100: Connection, polygon, expected_mask): } -@pytest.mark.parametrize("crs", CRS_VALUES_SUPPORTED_FOR_ALL_PYTHON_VERSIONS) +@pytest.mark.parametrize("crs", _get_normalizable_crs_inputs()) def test_mask_polygon_with_crs(con100: Connection, recwarn, crs: str): img = con100.load_collection("S2") polygon = shapely.geometry.box(0, 0, 1, 1) @@ -709,31 +694,10 @@ def test_mask_polygon_with_crs(con100: Connection, recwarn, crs: str): } -@pytest.mark.skipif(sys.version_info < (3, 7), reason="WKT2 format not supported by pyproj 3.0 / python 3.6") -def test_mask_polygon_with_crs_as_wkt(con100: Connection, recwarn): - """Separate test coverage for WKT, so we can skip it for Python3.6""" - crs = WKT2_FOR_EPSG23631 - img = con100.load_collection("S2") - polygon = shapely.geometry.box(0, 0, 1, 1) - masked = img.mask_polygon(mask=polygon, srs=crs) - warnings = [str(w.message) for w in recwarn] - assert f"Geometry with non-Lon-Lat CRS {crs!r} is only supported by specific back-ends." in warnings - assert sorted(masked.flat_graph().keys()) == ["loadcollection1", "maskpolygon1"] - assert masked.flat_graph()["maskpolygon1"] == { - "process_id": "mask_polygon", - "arguments": { - "data": {"from_node": "loadcollection1"}, - "mask": { - "type": "Polygon", "coordinates": (((1.0, 0.0), (1.0, 1.0), (0.0, 1.0), (0.0, 0.0), (1.0, 0.0)),), - # All listed test inputs for crs should be converted to "EPSG:32631" - "crs": {"type": "name", "properties": {"name": "EPSG:32631"}}, - }, - }, - "result": True, - } - - -@pytest.mark.skipif(sys.version_info < (3, 8), reason="PROJJSON format not supported by pyproj 3.2 / python < v3.8") +@pytest.mark.skipif( + pyproj.__version__ < ComparableVersion("3.3.0"), + reason="PROJJSON format support requires pyproj 3.3.0 or higher", +) @pytest.mark.parametrize("crs", [PROJJSON_FOR_EPSG23631, json.dumps(PROJJSON_FOR_EPSG23631)]) def test_mask_polygon_with_crs_as_projjson(con100: Connection, recwarn, crs): """Separate test coverage for PROJJSON, so we can skip it for Python versions below 3.8""" diff --git a/tests/test_util.py b/tests/test_util.py index 2052af2a3..a823c6ecc 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -935,7 +935,7 @@ def test_normalize_crs_without_pyproj_accept_non_epsg_string(self, caplog): @pytest.mark.skipif( # TODO #460 this skip is only necessary for python 3.6 and lower pyproj.__version__ < ComparableVersion("3.1.0"), - reason="WKT2 format support requires pypro 3.1.0 or higher", + reason="WKT2 format support requires pyproj 3.1.0 or higher", ) def test_normalize_crs_succeeds_with_wkt2_input(self): """Test can handle WKT2 strings. @@ -1020,7 +1020,7 @@ def test_normalize_crs_without_pyproj_succeeds_with_wkt2_input(self): @pytest.mark.skipif( # TODO #460 this skip is only necessary for python 3.6 and lower pyproj.__version__ < ComparableVersion("3.3.0"), - reason="PROJJSON format requires pyproj 3.3.0 or higher", + reason="PROJJSON format support requires pyproj 3.3.0 or higher", ) def test_normalize_crs_succeeds_with_correct_projjson( self, From 14c5f398a7f03270a022c98a45a381478cfa253f Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 21:47:45 +0200 Subject: [PATCH 08/12] fixup! Add `VectorCube.apply_dimension` --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c24da85cb..8cf8bde1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#259](https://github.com/Open-EO/openeo-python-client/issues/259)) - Initial `load_geojson` support with `Connection.load_geojson()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) - Initial `load_url` (for vector cubes) support with `Connection.load_url()` ([#424](https://github.com/Open-EO/openeo-python-client/issues/424)) +- Add `VectorCube.apply_dimension()` ([Open-EO/openeo-python-driver#197](https://github.com/Open-EO/openeo-python-driver/issues/197)) - Support lambda based property filtering in `Connection.load_stac()` ([#425](https://github.com/Open-EO/openeo-python-client/issues/425)) - `VectorCube`: initial support for `filter_bands`, `filter_bbox`, `filter_labels` and `filter_vector` ([#459](https://github.com/Open-EO/openeo-python-client/issues/459)) From 6670708efda418e6a137598dcc8aa86603328e1c Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 21:57:06 +0200 Subject: [PATCH 09/12] Release 0.22.0 --- CHANGELOG.md | 15 +++++++++++++-- openeo/_version.py | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf8bde1a..c0d5cd9e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,23 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + ## [Unreleased] ### Added +### Changed + +### Removed + +### Fixed + + + +## [0.22.0] - 2023-08-09 + +### Added + - Processes that take a CRS as argument now try harder to normalize your input to a CRS representation that aligns with the openEO API (using `pyproj` library when available) ([#259](https://github.com/Open-EO/openeo-python-client/issues/259)) @@ -23,8 +36,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `Connection` based requests: always use finite timeouts by default (20 minutes in general, 30 minutes for synchronous execute requests) ([#454](https://github.com/Open-EO/openeo-python-client/issues/454)) -### Removed - ### Fixed - Fix: MultibackendJobManager should stop when finished, also when job finishes with error ([#452](https://github.com/Open-EO/openeo-python-client/issues/432)) diff --git a/openeo/_version.py b/openeo/_version.py index 2ce0f8458..5963297e7 100644 --- a/openeo/_version.py +++ b/openeo/_version.py @@ -1 +1 @@ -__version__ = "0.22.0a1" +__version__ = "0.22.0" From 6f1c10ce6e6cebf3d2332553825a0ce68417101d Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 22:07:36 +0200 Subject: [PATCH 10/12] _version.py: bump to 0.23.0a1 --- openeo/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo/_version.py b/openeo/_version.py index 5963297e7..0c8bc47d0 100644 --- a/openeo/_version.py +++ b/openeo/_version.py @@ -1 +1 @@ -__version__ = "0.22.0" +__version__ = "0.23.0a1" From b666ed9814e7add4071ca1aef9f6b837f2ebfd76 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 22:10:42 +0200 Subject: [PATCH 11/12] Jenkinsfile: attempt to speed up jenkins build don't install sphinx and related doc building tools to just run tests --- Jenkinsfile | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index eba3afe54..cbb52bda4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,7 @@ pythonPipeline { package_name = 'openeo' wipeout_workspace = true python_version = ["3.8"] - extras_require = 'dev' + extras_require = 'tests' upload_dev_wheels = false wheel_repo = 'python-openeo' wheel_repo_dev = 'python-openeo' diff --git a/setup.py b/setup.py index 99fe8cae3..28fcd0eea 100644 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ 'oschmod>=0.3.12; sys_platform == "win32"', ], extras_require={ + "tests": tests_require, "dev": tests_require + docs_require, "docs": docs_require, "oschmod": [ # install oschmod even when platform is not Windows, e.g. for testing in CI. From 5bb0da8800f50bdb445f64b6f1527c40aca5869f Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Wed, 9 Aug 2023 22:27:00 +0200 Subject: [PATCH 12/12] Add tests for `build_child_callback` --- tests/rest/datacube/test_base_datacube.py | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/rest/datacube/test_base_datacube.py diff --git a/tests/rest/datacube/test_base_datacube.py b/tests/rest/datacube/test_base_datacube.py new file mode 100644 index 000000000..7392bb425 --- /dev/null +++ b/tests/rest/datacube/test_base_datacube.py @@ -0,0 +1,56 @@ +from openeo.internal.graph_building import PGNode +from openeo.rest._datacube import build_child_callback, UDF + + +def test_build_child_callback_str(): + pg = build_child_callback("add", parent_parameters=["x", "y"]) + assert isinstance(pg["process_graph"], PGNode) + assert pg["process_graph"].flat_graph() == { + "add1": { + "process_id": "add", + "arguments": {"x": {"from_parameter": "x"}, "y": {"from_parameter": "y"}}, + "result": True, + } + } + + +def test_build_child_callback_pgnode(): + pg = build_child_callback( + PGNode(process_id="add", arguments={"x": {"from_parameter": "x"}, "y": 1}), parent_parameters=["x"] + ) + assert isinstance(pg["process_graph"], PGNode) + assert pg["process_graph"].flat_graph() == { + "add1": { + "process_id": "add", + "arguments": {"x": {"from_parameter": "x"}, "y": 1}, + "result": True, + } + } + + +def test_build_child_callback_lambda(): + pg = build_child_callback(lambda x: x + 1, parent_parameters=["x"]) + assert isinstance(pg["process_graph"], PGNode) + assert pg["process_graph"].flat_graph() == { + "add1": { + "process_id": "add", + "arguments": {"x": {"from_parameter": "x"}, "y": 1}, + "result": True, + } + } + + +def test_build_child_callback_udf(): + pg = build_child_callback(UDF(code="def fun(x):\n return x + 1", runtime="Python"), parent_parameters=["data"]) + assert isinstance(pg["process_graph"], PGNode) + assert pg["process_graph"].flat_graph() == { + "runudf1": { + "process_id": "run_udf", + "arguments": { + "data": {"from_parameter": "data"}, + "runtime": "Python", + "udf": "def fun(x):\n return x + 1", + }, + "result": True, + } + }