From 1c962ae6731dc7e93669fed00e12539e45c0410c Mon Sep 17 00:00:00 2001 From: Alex Leith Date: Thu, 22 Jun 2023 11:26:02 +1000 Subject: [PATCH 1/4] Add a test and fix for invalid geometry --- apps/dc_tools/odc/apps/dc_tools/_stac.py | 7 ++++++- apps/dc_tools/tests/conftest.py | 22 +++++++++++++++++++++ apps/dc_tools/tests/test_stac_transform.py | 23 +++++++++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/apps/dc_tools/odc/apps/dc_tools/_stac.py b/apps/dc_tools/odc/apps/dc_tools/_stac.py index 21ce868d9..26b7c060f 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_stac.py +++ b/apps/dc_tools/odc/apps/dc_tools/_stac.py @@ -270,7 +270,12 @@ def round_coords(c1, c2): if geom.is_valid: return geom.transform(round_coords) else: - return None + # Attempt to fix it if it's not valid + print( + "Caution, invalid geometry, attempting to fix. This may result in invalid data." + ) + geom = geom.convex_hull + return geom.transform(round_coords) def stac_transform_absolute(input_stac): diff --git a/apps/dc_tools/tests/conftest.py b/apps/dc_tools/tests/conftest.py index 52b55f784..4eb3362f0 100644 --- a/apps/dc_tools/tests/conftest.py +++ b/apps/dc_tools/tests/conftest.py @@ -18,6 +18,7 @@ from datacube.model import MetadataType from datacube.utils import documents from odc.apps.dc_tools.add_update_products import add_update_products +from datacube.utils.geometry import Geometry, CRS TEST_DATA_FOLDER: Path = Path(__file__).parent.joinpath("data") LANDSAT_STAC: str = "ga_ls8c_ard_3-1-0_088080_2020-05-25_final.stac-item.json" @@ -338,3 +339,24 @@ def odc_db_for_archive(odc_test_db_with_products: Datacube): assert result.exit_code == 0 return odc_test_db_with_products + + +@pytest.fixture +def geometry_with_a_twist(): + # From https://planetarycomputer.microsoft.com/api/stac/v1/collections/ + # landsat-c2-l2/items/LC08_L2SR_073071_20130427_02_T1 + return Geometry( + { + "type": "Polygon", + "coordinates": [ + [ + [-179.9145508, -14.923636], + [-180.0228509, -15.3692234], + [-178.5768881, -16.8780443], + [-179.983072, -16.5695723], + [-179.9145508, -14.923636], + ] + ], + }, + crs=CRS("epsg:4326"), + ) diff --git a/apps/dc_tools/tests/test_stac_transform.py b/apps/dc_tools/tests/test_stac_transform.py index 509220080..f15c63967 100644 --- a/apps/dc_tools/tests/test_stac_transform.py +++ b/apps/dc_tools/tests/test_stac_transform.py @@ -3,7 +3,28 @@ """ from datacube.utils.changes import get_doc_changes -from odc.apps.dc_tools._stac import stac_transform +from odc.apps.dc_tools._stac import stac_transform, _geographic_to_projected + + +def test_geographic_to_projected(geometry_with_a_twist): + transformed = _geographic_to_projected(geometry_with_a_twist, "EPSG:900913") + + assert transformed is not None + + expected = { + "type": "Polygon", + "coordinates": ( + ( + (-20027996.181356553, -1680401.0306884265), + (20034964.592237074, -1731788.8882795917), + (-19879088.250739392, -1906633.287019358), + (-20035623.926449094, -1870777.708859445), + (-20027996.181356553, -1680401.0306884265), + ), + ), + } + + assert transformed.json == expected def test_esri_lulc_stac_transform(esri_lulc_stac): From b2726480c5d492f4395faf06e1a3a392a8744b13 Mon Sep 17 00:00:00 2001 From: Alex Leith Date: Thu, 22 Jun 2023 11:29:42 +1000 Subject: [PATCH 2/4] Change target CRS to one that works in the region --- apps/dc_tools/tests/test_stac_transform.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/dc_tools/tests/test_stac_transform.py b/apps/dc_tools/tests/test_stac_transform.py index f15c63967..e85c5be33 100644 --- a/apps/dc_tools/tests/test_stac_transform.py +++ b/apps/dc_tools/tests/test_stac_transform.py @@ -7,7 +7,7 @@ def test_geographic_to_projected(geometry_with_a_twist): - transformed = _geographic_to_projected(geometry_with_a_twist, "EPSG:900913") + transformed = _geographic_to_projected(geometry_with_a_twist, "EPSG:3832") assert transformed is not None @@ -15,11 +15,11 @@ def test_geographic_to_projected(geometry_with_a_twist): "type": "Polygon", "coordinates": ( ( - (-20027996.181356553, -1680401.0306884265), - (20034964.592237074, -1731788.8882795917), - (-19879088.250739392, -1906633.287019358), - (-20035623.926449094, -1870777.708859445), - (-20027996.181356553, -1680401.0306884265), + (3498004.815848052, -1894234.3033218135), + (3341469.1401383593, -1858599.0018098454), + (3337040.973246038, -1720470.5990706773), + (3349096.8852309, -1669403.410330984), + (3498004.815848052, -1894234.3033218135), ), ), } From 9b94467c1cad648b0c0e5cc89feae5b1a4e89c12 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 27 Aug 2024 15:39:36 +1000 Subject: [PATCH 3/4] Fix default grid handling --- apps/dc_tools/odc/apps/dc_tools/_stac.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/dc_tools/odc/apps/dc_tools/_stac.py b/apps/dc_tools/odc/apps/dc_tools/_stac.py index 26b7c060f..ebc9efcd1 100644 --- a/apps/dc_tools/odc/apps/dc_tools/_stac.py +++ b/apps/dc_tools/odc/apps/dc_tools/_stac.py @@ -1,6 +1,7 @@ """ Tools for STAC to EO3 translation """ + import math import numpy from eodatasets3.serialise import from_doc @@ -227,7 +228,10 @@ def _get_path(asset, force_relative=False): # If transform specified here in the asset it should override # the properties-specified transform. transform = asset.get("proj:transform") or proj_transform - grid = f"g{transform[0]:g}m" + if transform is not None: + grid = f"g{transform[0]:g}m" + else: + grid = default_grid # As per transform, shape here overrides properties shape = asset.get("proj:shape") or proj_shape From 02fa14247727fde951b66974f14b80726817e276 Mon Sep 17 00:00:00 2001 From: Alex Date: Mon, 2 Sep 2024 09:09:16 +1000 Subject: [PATCH 4/4] Remove print statement --- apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py index 34ec66b8d..0b4c3739c 100644 --- a/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py +++ b/apps/dc_tools/odc/apps/dc_tools/stac_api_to_dc.py @@ -311,9 +311,6 @@ def cli( publish_action=publish_action, ) - print( - f"Added {added} Datasets, failed {failed} Datasets, skipped {skipped} Datasets" - ) if statsd_setting: statsd_gauge_reporting( added, ["app:stac_api_to_dc", "action:added"], statsd_setting