From 3fbbf5f7ece4195a624a541739332a4da642cedb Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 14:46:37 -0700 Subject: [PATCH 01/23] Add tile and _get_reader functions to CustomSTACReader --- titiler/stacapi/backend.py | 143 +++++++++++++++++++++++++++++++++++-- titiler/stacapi/models.py | 10 ++- 2 files changed, 146 insertions(+), 7 deletions(-) diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index 35993c8..4b8b55c 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -1,7 +1,7 @@ """titiler-stacapi custom Mosaic Backend and Custom STACReader.""" import json -from typing import Any, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List, Optional, Tuple, Type, Union, Sequence, Set import attr import planetary_computer as pc @@ -19,13 +19,17 @@ from rasterio.crs import CRS from rasterio.warp import transform, transform_bounds from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS -from rio_tiler.errors import InvalidAssetName -from rio_tiler.io import Reader +from rio_tiler.errors import InvalidAssetName, TileOutsideBounds, MissingAssets, AssetAsBandError, ExpressionMixingWarning +from rio_tiler.io import Reader, XarrayReader from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData from rio_tiler.mosaic import mosaic_reader -from rio_tiler.types import AssetInfo, BBox +from rio_tiler.tasks import multi_arrays +from rio_tiler.types import BBox, Indexes from urllib3 import Retry +import warnings + +from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.utils import Timer @@ -34,6 +38,19 @@ retry_config = RetrySettings() stac_config = STACSettings() +valid_types = { + "image/tiff; application=geotiff", + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; profile=cloud-optimized; application=geotiff", + "image/vnd.stac.geotiff; cloud-optimized=true", + "image/tiff", + "image/x.geotiff", + "image/jp2", + "application/x-hdf5", + "application/x-hdf", + "application/vnd+zarr", + "application/x-netcdf", +} @attr.s class CustomSTACReader(MultiBaseReader): @@ -63,6 +80,8 @@ class CustomSTACReader(MultiBaseReader): ctx: Any = attr.ib(default=rasterio.Env) + include_asset_types: Set[str] = attr.ib(default=valid_types) + def __attrs_post_init__(self) -> None: """Set reader spatial infos and list of valid assets.""" self.bounds = self.input["bbox"] @@ -77,11 +96,28 @@ def _minzoom(self): def _maxzoom(self): return self.tms.maxzoom + def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: + """Get Asset Reader.""" + asset_type = asset_info.get("type", None) + + if asset_type and asset_type in [ + "application/x-hdf5", + "application/x-hdf", + "application/vnd.zarr", + "application/x-netcdf", + "application/netcdf" + + ]: + return XarrayReader + + return Reader + def _get_asset_info(self, asset: str) -> AssetInfo: - """Validate asset names and return asset's url. + """ + Validate asset names and return asset's info. Args: - asset (str): STAC asset name. + AssetInfo (str): STAC asset info. Returns: str: STAC asset href. @@ -100,6 +136,9 @@ def _get_asset_info(self, asset: str) -> AssetInfo: info = AssetInfo(url=url, env={}) + if asset_info.get("type"): + info.media_type = asset_info["type"] + if header_size := asset_info.get("file:header_size"): info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size @@ -114,6 +153,98 @@ def _get_asset_info(self, asset: str) -> AssetInfo: return info + def tile( + self, + tile_x: int, + tile_y: int, + tile_z: int, + assets: Union[Sequence[str], str] = None, + expression: Optional[str] = None, + asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset + asset_as_band: bool = False, + **kwargs: Any, + ) -> ImageData: + """Read and merge Wep Map tiles from multiple assets. + + Args: + tile_x (int): Tile's horizontal index. + tile_y (int): Tile's vertical index. + tile_z (int): Tile's zoom level index. + assets (sequence of str or str, optional): assets to fetch info from. + expression (str, optional): rio-tiler expression for the asset list (e.g. asset1/asset2+asset3). + asset_indexes (dict, optional): Band indexes for each asset (e.g {"asset1": 1, "asset2": (1, 2,)}). + kwargs (optional): Options to forward to the `self.reader.tile` method. + + Returns: + rio_tiler.models.ImageData: ImageData instance with data, mask and tile spatial info. + + """ + if not self.tile_exists(tile_x, tile_y, tile_z): + raise TileOutsideBounds( + f"Tile {tile_z}/{tile_x}/{tile_y} is outside image bounds" + ) + + if isinstance(assets, str): + assets = (assets,) + + if assets and expression: + warnings.warn( + "Both expression and assets passed; expression will overwrite assets parameter.", + ExpressionMixingWarning, + ) + + if expression: + assets = self.parse_expression(expression, asset_as_band=asset_as_band) + + if not assets: + raise MissingAssets( + "assets must be passed either via `expression` or `assets` options." + ) + + asset_indexes = asset_indexes or {} + + # We fall back to `indexes` if provided + indexes = kwargs.pop("indexes", None) + + def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: + idx = asset_indexes.get(asset) or indexes # type: ignore + + asset_info = self._get_asset_info(asset) + reader = self._get_reader(asset_info) + + with self.ctx(**asset_info.get("env", {})): + with reader( + asset_info["url"], tms=self.tms, **self.reader_options + ) as src: + data = src.tile(*args, indexes=idx, **kwargs) + + self._update_statistics( + data, + indexes=idx, + statistics=asset_info.get("dataset_statistics"), + ) + + metadata = data.metadata or {} + if m := asset_info.get("metadata"): + metadata.update(m) + data.metadata = {asset: metadata} + + if asset_as_band: + if len(data.band_names) > 1: + raise AssetAsBandError( + "Can't use `asset_as_band` for multibands asset" + ) + data.band_names = [asset] + else: + data.band_names = [f"{asset}_{n}" for n in data.band_names] + + return data + + img = multi_arrays(assets, _reader, tile_x, tile_y, tile_z, **kwargs) + if expression: + return img.apply_expression(expression) + + return img @attr.s class STACAPIBackend(BaseBackend): diff --git a/titiler/stacapi/models.py b/titiler/stacapi/models.py index dfcd3a4..4473205 100644 --- a/titiler/stacapi/models.py +++ b/titiler/stacapi/models.py @@ -6,7 +6,7 @@ """ -from typing import List, Optional +from typing import List, Optional, Dict, Sequence, Tuple from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -82,3 +82,11 @@ class Landing(BaseModel): title: Optional[str] = None description: Optional[str] = None links: List[Link] + +class AssetInfo(TypedDict, total=False): + """Asset Reader Options.""" + + url: str + env: Optional[Dict] + metadata: Optional[Dict] + dataset_statistics: Optional[Sequence[Tuple[float, float]]] \ No newline at end of file From acbd215bb638deac0be2ca014245d09215415e2d Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 14:57:46 -0700 Subject: [PATCH 02/23] Linting --- titiler/stacapi/backend.py | 27 +++++++++++++++++---------- titiler/stacapi/models.py | 6 ++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index 4b8b55c..f9ba98f 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -1,7 +1,8 @@ """titiler-stacapi custom Mosaic Backend and Custom STACReader.""" import json -from typing import Any, Dict, List, Optional, Tuple, Type, Union, Sequence, Set +import warnings +from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Type, Union import attr import planetary_computer as pc @@ -19,7 +20,13 @@ from rasterio.crs import CRS from rasterio.warp import transform, transform_bounds from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS -from rio_tiler.errors import InvalidAssetName, TileOutsideBounds, MissingAssets, AssetAsBandError, ExpressionMixingWarning +from rio_tiler.errors import ( + AssetAsBandError, + ExpressionMixingWarning, + InvalidAssetName, + MissingAssets, + TileOutsideBounds, +) from rio_tiler.io import Reader, XarrayReader from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData @@ -27,10 +34,8 @@ from rio_tiler.tasks import multi_arrays from rio_tiler.types import BBox, Indexes from urllib3 import Retry -import warnings from titiler.stacapi.models import AssetInfo - from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.utils import Timer @@ -52,6 +57,7 @@ "application/x-netcdf", } + @attr.s class CustomSTACReader(MultiBaseReader): """Simplified STAC Reader. @@ -105,8 +111,7 @@ def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: "application/x-hdf", "application/vnd.zarr", "application/x-netcdf", - "application/netcdf" - + "application/netcdf", ]: return XarrayReader @@ -137,10 +142,10 @@ def _get_asset_info(self, asset: str) -> AssetInfo: info = AssetInfo(url=url, env={}) if asset_info.get("type"): - info.media_type = asset_info["type"] + info["type"] = asset_info["type"] if header_size := asset_info.get("file:header_size"): - info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size + info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size # type: ignore if bands := asset_info.get("raster:bands"): stats = [ @@ -153,12 +158,12 @@ def _get_asset_info(self, asset: str) -> AssetInfo: return info - def tile( + def tile( # noqa: C901 self, tile_x: int, tile_y: int, tile_z: int, - assets: Union[Sequence[str], str] = None, + assets: Union[Sequence[str], str] = (), expression: Optional[str] = None, asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset asset_as_band: bool = False, @@ -191,6 +196,7 @@ def tile( warnings.warn( "Both expression and assets passed; expression will overwrite assets parameter.", ExpressionMixingWarning, + stacklevel=2, ) if expression: @@ -246,6 +252,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: return img + @attr.s class STACAPIBackend(BaseBackend): """STACAPI Mosaic Backend.""" diff --git a/titiler/stacapi/models.py b/titiler/stacapi/models.py index 4473205..7033206 100644 --- a/titiler/stacapi/models.py +++ b/titiler/stacapi/models.py @@ -6,7 +6,7 @@ """ -from typing import List, Optional, Dict, Sequence, Tuple +from typing import Dict, List, Optional, Sequence, Tuple, TypedDict from pydantic import BaseModel, Field from typing_extensions import Annotated @@ -83,10 +83,12 @@ class Landing(BaseModel): description: Optional[str] = None links: List[Link] + class AssetInfo(TypedDict, total=False): """Asset Reader Options.""" url: str env: Optional[Dict] + type: str metadata: Optional[Dict] - dataset_statistics: Optional[Sequence[Tuple[float, float]]] \ No newline at end of file + dataset_statistics: Optional[Sequence[Tuple[float, float]]] From b6b36814676292e835b5dd6ff27cf8ffa2ea1a81 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 15:31:42 -0700 Subject: [PATCH 03/23] Add stub xarray reader class --- tests/test_items.py | 44 -------------- titiler/stacapi/backend.py | 5 +- titiler/stacapi/dependencies.py | 55 ----------------- titiler/stacapi/main.py | 22 +------ titiler/stacapi/stac_reader.py | 103 -------------------------------- titiler/stacapi/xarray.py | 15 +++++ 6 files changed, 20 insertions(+), 224 deletions(-) delete mode 100644 tests/test_items.py delete mode 100644 titiler/stacapi/stac_reader.py create mode 100644 titiler/stacapi/xarray.py diff --git a/tests/test_items.py b/tests/test_items.py deleted file mode 100644 index da96630..0000000 --- a/tests/test_items.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Test titiler.stacapi Item endpoints.""" - -import json -import os -from unittest.mock import patch - -import pystac -import pytest - -from .conftest import mock_rasterio_open - -item_json = os.path.join( - os.path.dirname(__file__), "fixtures", "20200307aC0853900w361030.json" -) - - -@patch("rio_tiler.io.rasterio.rasterio") -@patch("titiler.stacapi.dependencies.get_stac_item") -def test_stac_items(get_stac_item, rio, app): - """test STAC items endpoints.""" - rio.open = mock_rasterio_open - - with open(item_json, "r") as f: - get_stac_item.return_value = pystac.Item.from_dict(json.loads(f.read())) - - response = app.get( - "/collections/noaa-emergency-response/items/20200307aC0853900w361030/assets", - ) - assert response.status_code == 200 - assert response.json() == ["cog"] - - with pytest.warns(UserWarning): - response = app.get( - "/collections/noaa-emergency-response/items/20200307aC0853900w361030/info", - ) - assert response.status_code == 200 - assert response.json()["cog"] - - response = app.get( - "/collections/noaa-emergency-response/items/20200307aC0853900w361030/info", - params={"assets": "cog"}, - ) - assert response.status_code == 200 - assert response.json()["cog"] diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index f9ba98f..0ca996d 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -27,7 +27,7 @@ MissingAssets, TileOutsideBounds, ) -from rio_tiler.io import Reader, XarrayReader +from rio_tiler.io import Reader from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData from rio_tiler.mosaic import mosaic_reader @@ -38,6 +38,7 @@ from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.utils import Timer +from titiler.stacapi.xarray import XarrayReader cache_config = CacheSettings() retry_config = RetrySettings() @@ -222,6 +223,8 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: with reader( asset_info["url"], tms=self.tms, **self.reader_options ) as src: + if type(src) == XarrayReader: + raise NotImplementedError("XarrayReader not yet implemented") data = src.tile(*args, indexes=idx, **kwargs) self._update_statistics( diff --git a/titiler/stacapi/dependencies.py b/titiler/stacapi/dependencies.py index 9d81592..257e392 100644 --- a/titiler/stacapi/dependencies.py +++ b/titiler/stacapi/dependencies.py @@ -102,61 +102,6 @@ def STACApiParams( api_url=request.app.state.stac_url, ) - -@cached( # type: ignore - TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl), - key=lambda url, collection_id, item_id, headers, **kwargs: hashkey( - url, collection_id, item_id, json.dumps(headers) - ), -) -def get_stac_item( - url: str, - collection_id: str, - item_id: str, - headers: Optional[Dict] = None, -) -> pystac.Item: - """Get STAC Item from STAC API.""" - stac_api_io = StacApiIO( - max_retries=Retry( - total=retry_config.retry, - backoff_factor=retry_config.retry_factor, - ), - headers=headers, - ) - results = ItemSearch( - f"{url}/search", - stac_io=stac_api_io, - collections=[collection_id], - ids=[item_id], - modifier=pc.sign_inplace, - ) - items = list(results.items()) - if not items: - raise HTTPException( - 404, - f"Could not find Item {item_id} in {collection_id} collection.", - ) - - return items[0] - - -def ItemIdParams( - collection_id: Annotated[ - str, - Path(description="STAC Collection Identifier"), - ], - item_id: Annotated[str, Path(description="STAC Item Identifier")], - api_params=Depends(STACApiParams), -) -> pystac.Item: - """STAC Item dependency for the MultiBaseTilerFactory.""" - return get_stac_item( - api_params["api_url"], - collection_id, - item_id, - headers=api_params.get("headers", {}), - ) - - def STACSearchParams( request: Request, collection_id: Annotated[ diff --git a/titiler/stacapi/main.py b/titiler/stacapi/main.py index aa5fc30..8277964 100644 --- a/titiler/stacapi/main.py +++ b/titiler/stacapi/main.py @@ -18,11 +18,10 @@ from titiler.mosaic.errors import MOSAIC_STATUS_CODES from titiler.stacapi import __version__ as titiler_stacapi_version from titiler.stacapi import models -from titiler.stacapi.dependencies import ItemIdParams, OutputType, STACApiParams +from titiler.stacapi.dependencies import OutputType, STACApiParams from titiler.stacapi.enums import MediaType from titiler.stacapi.factory import MosaicTilerFactory from titiler.stacapi.settings import ApiSettings, STACAPISettings -from titiler.stacapi.stac_reader import STACReader from titiler.stacapi.utils import create_html_response settings = ApiSettings() @@ -99,25 +98,6 @@ collection.router, tags=["STAC Collection"], prefix="/collections/{collection_id}" ) -############################################################################### -# STAC Item Endpoints -# Notes: The `MultiBaseTilerFactory` from titiler.core.factory expect a `URL` as query parameter -# but in this project we use a custom `path_dependency=ItemIdParams`, which define `{collection_id}` and `{item_id}` as -# `Path` dependencies. Then the `ItemIdParams` dependency will fetch the STAC API endpoint to get the STAC Item. The Item -# will then be used in our custom `STACReader`. -stac = MultiBaseTilerFactory( - reader=STACReader, - path_dependency=ItemIdParams, - optional_headers=optional_headers, - router_prefix="/collections/{collection_id}/items/{item_id}", - add_viewer=True, -) -app.include_router( - stac.router, - tags=["STAC Item"], - prefix="/collections/{collection_id}/items/{item_id}", -) - ############################################################################### # Tiling Schemes Endpoints tms = TMSFactory() diff --git a/titiler/stacapi/stac_reader.py b/titiler/stacapi/stac_reader.py deleted file mode 100644 index 10ef9d7..0000000 --- a/titiler/stacapi/stac_reader.py +++ /dev/null @@ -1,103 +0,0 @@ -"""Custom STAC reader.""" - -from typing import Any, Dict, Optional, Set, Type - -import attr -import pystac -import rasterio -from morecantile import TileMatrixSet -from rasterio.crs import CRS -from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS -from rio_tiler.errors import InvalidAssetName -from rio_tiler.io import BaseReader, Reader, stac -from rio_tiler.types import AssetInfo - -from titiler.stacapi.settings import STACSettings - -stac_config = STACSettings() - - -@attr.s -class STACReader(stac.STACReader): - """Custom STAC Reader. - - Only accept `pystac.Item` as input (while rio_tiler.io.STACReader accepts url or pystac.Item) - - """ - - input: pystac.Item = attr.ib() - - tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) - minzoom: int = attr.ib() - maxzoom: int = attr.ib() - - geographic_crs: CRS = attr.ib(default=WGS84_CRS) - - include_assets: Optional[Set[str]] = attr.ib(default=None) - exclude_assets: Optional[Set[str]] = attr.ib(default=None) - - include_asset_types: Set[str] = attr.ib(default=stac.DEFAULT_VALID_TYPE) - exclude_asset_types: Optional[Set[str]] = attr.ib(default=None) - - reader: Type[BaseReader] = attr.ib(default=Reader) - reader_options: Dict = attr.ib(factory=dict) - - fetch_options: Dict = attr.ib(factory=dict) - - ctx: Any = attr.ib(default=rasterio.Env) - - item: pystac.Item = attr.ib(init=False) - - def __attrs_post_init__(self): - """Fetch STAC Item and get list of valid assets.""" - self.item = self.input - super().__attrs_post_init__() - - @minzoom.default - def _minzoom(self): - return self.tms.minzoom - - @maxzoom.default - def _maxzoom(self): - return self.tms.maxzoom - - def _get_asset_info(self, asset: str) -> AssetInfo: - """Validate asset names and return asset's url. - - Args: - asset (str): STAC asset name. - - Returns: - str: STAC asset href. - - """ - if asset not in self.assets: - raise InvalidAssetName( - f"'{asset}' is not valid, should be one of {self.assets}" - ) - - asset_info = self.item.assets[asset] - extras = asset_info.extra_fields - - url = asset_info.get_absolute_href() or asset_info.href - if alternate := stac_config.alternate_url: - url = asset_info.to_dict()["alternate"][alternate]["href"] - - info = AssetInfo( - url=url, - metadata=extras, - ) - - if head := extras.get("file:header_size"): - info["env"] = {"GDAL_INGESTED_BYTES_AT_OPEN": head} - - if bands := extras.get("raster:bands"): - stats = [ - (b["statistics"]["minimum"], b["statistics"]["maximum"]) - for b in bands - if {"minimum", "maximum"}.issubset(b.get("statistics", {})) - ] - if len(stats) == len(bands): - info["dataset_statistics"] = stats - - return info diff --git a/titiler/stacapi/xarray.py b/titiler/stacapi/xarray.py new file mode 100644 index 0000000..3d9b2df --- /dev/null +++ b/titiler/stacapi/xarray.py @@ -0,0 +1,15 @@ + +import attr +# from typing import Dict +import rio_tiler.io as riotio +# import xarray as xr +# import fsspec + +@attr.s +class XarrayReader(riotio.XarrayReader): + """Custom Xarray Reader.""" + + # def __attrs_post_init__(self) -> None: + # httpfs = fsspec.filesystem("https") + # da = xr.open_dataset(httpfs.open(self.input), engine="h5netcdf")[variable] + # self.input = da From 0be0f9a74496ef1c7276a6540e6cb45610afacc8 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 15:32:25 -0700 Subject: [PATCH 04/23] Linting --- titiler/stacapi/dependencies.py | 11 ++--------- titiler/stacapi/main.py | 2 +- titiler/stacapi/xarray.py | 4 ++++ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/titiler/stacapi/dependencies.py b/titiler/stacapi/dependencies.py index 257e392..6d858a1 100644 --- a/titiler/stacapi/dependencies.py +++ b/titiler/stacapi/dependencies.py @@ -1,18 +1,10 @@ """titiler-stacapi dependencies.""" -import json from typing import Dict, List, Literal, Optional, TypedDict, get_args -import planetary_computer as pc -import pystac -from cachetools import TTLCache, cached -from cachetools.keys import hashkey -from fastapi import Depends, HTTPException, Path, Query -from pystac_client import ItemSearch -from pystac_client.stac_api_io import StacApiIO +from fastapi import Path, Query from starlette.requests import Request from typing_extensions import Annotated -from urllib3 import Retry from titiler.stacapi.enums import MediaType from titiler.stacapi.settings import CacheSettings, RetrySettings @@ -102,6 +94,7 @@ def STACApiParams( api_url=request.app.state.stac_url, ) + def STACSearchParams( request: Request, collection_id: Annotated[ diff --git a/titiler/stacapi/main.py b/titiler/stacapi/main.py index 8277964..6ccf0b9 100644 --- a/titiler/stacapi/main.py +++ b/titiler/stacapi/main.py @@ -12,7 +12,7 @@ from typing_extensions import Annotated from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import AlgorithmFactory, MultiBaseTilerFactory, TMSFactory +from titiler.core.factory import AlgorithmFactory, TMSFactory from titiler.core.middleware import CacheControlMiddleware, LoggerMiddleware from titiler.core.resources.enums import OptionalHeader from titiler.mosaic.errors import MOSAIC_STATUS_CODES diff --git a/titiler/stacapi/xarray.py b/titiler/stacapi/xarray.py index 3d9b2df..5d301a7 100644 --- a/titiler/stacapi/xarray.py +++ b/titiler/stacapi/xarray.py @@ -1,10 +1,14 @@ +""" Custom Xarray Reader. """ import attr + # from typing import Dict import rio_tiler.io as riotio + # import xarray as xr # import fsspec + @attr.s class XarrayReader(riotio.XarrayReader): """Custom Xarray Reader.""" From 04f4949fdb11cdebe0e39ad6a4ad9039c27f3e03 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 16:06:08 -0700 Subject: [PATCH 05/23] Move stac reader into it's own file and write tests for _get_reader --- ...S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json | 1 + tests/test_stac_reader.py | 41 +++ titiler/stacapi/backend.py | 230 +---------------- titiler/stacapi/stac_reader.py | 240 ++++++++++++++++++ 4 files changed, 285 insertions(+), 227 deletions(-) create mode 100644 tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json create mode 100644 tests/test_stac_reader.py create mode 100644 titiler/stacapi/stac_reader.py diff --git a/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json b/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json new file mode 100644 index 0000000..0fc00a6 --- /dev/null +++ b/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json @@ -0,0 +1 @@ +{"id":"C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1","bbox":[-180.0,-90.0,180.0,90.0],"type":"Feature","links":[{"rel":"collection","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf"},{"rel":"parent","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf"},{"rel":"root","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"self","type":"application/geo+json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf/items/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1"}],"assets":{"netcdf":{"href":"https://landcoverdata.blob.core.windows.net/esa-cci-lc/netcdf/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.nc","created":"2021-05-18T10:15:40Z","cube:variables":{"crs":{"type":"auxiliary","dimensions":[]},"lat":{"axis":"Y","type":"auxiliary","unit":"degrees_north","dimensions":["lat"],"description":"latitude"},"lon":{"axis":"X","type":"auxiliary","unit":"degrees_east","dimensions":["lon"],"description":"longitude"},"time":{"axis":"T","type":"auxiliary","unit":"days since 1970-01-01 00:00:00","dimensions":["time"],"description":"time"},"lat_bounds":{"type":"auxiliary","dimensions":["lat","bounds"]},"lccs_class":{"type":"data","dimensions":["time","lat","lon"],"description":"Land cover class defined in LCCS"},"lon_bounds":{"type":"auxiliary","dimensions":["lon","bounds"]},"time_bounds":{"type":"auxiliary","dimensions":["time","bounds"]},"change_count":{"type":"data","dimensions":["time","lat","lon"],"description":"number of class changes"},"processed_flag":{"type":"data","dimensions":["time","lat","lon"],"description":"LC map processed area flag"},"observation_count":{"type":"data","dimensions":["time","lat","lon"],"description":"number of valid observations"},"current_pixel_state":{"type":"data","dimensions":["time","lat","lon"],"description":"LC pixel type mask"}},"cube:dimensions":{"lat":{"type":"lat","extent":[-90.0,90.0]},"lon":{"type":"lon","extent":[-180.0,180.0]},"time":{"type":"time","values":[18262.0]},"bounds":{"type":"bounds","values":[0,1]}},"type":"application/netcdf","roles":["data","quality"],"title":"ESA CCI Land Cover NetCDF 4 File"}},"geometry":{"type":"Polygon","coordinates":[[[-180,-90],[180,-90],[180,90],[-180,90],[-180,-90]]]},"collection":"esa-cci-lc-netcdf","properties":{"title":"ESA CCI Land Cover Map for 2020","created":"2023-01-11T18:52:07.171349Z","datetime":null,"proj:epsg":4326,"proj:shape":[129600,64800],"end_datetime":"2020-12-31T23:59:59Z","proj:transform":[-180.0,0.002777777777778,0.0,90.0,0.0,-0.002777777777778],"start_datetime":"2020-01-01T00:00:00Z","esa_cci_lc:version":"2.1.1","processing:lineage":"Produced based on the following data sources: Sentinel-3 OLCI","processing:software":{"lc-sr":"1.0","lc-user-tools(1)":"3.14","lc-user-tools(2)":"4.5","lc-classification":"1.0"}},"stac_extensions":["https://stac-extensions.github.io/projection/v1.0.0/schema.json","https://stac-extensions.github.io/processing/v1.1.0/schema.json","https://stac-extensions.github.io/datacube/v2.1.0/schema.json"],"stac_version":"1.0.0"} \ No newline at end of file diff --git a/tests/test_stac_reader.py b/tests/test_stac_reader.py new file mode 100644 index 0000000..1e8285f --- /dev/null +++ b/tests/test_stac_reader.py @@ -0,0 +1,41 @@ +"""Test titiler.stacapi.stac_reader functions.""" + +import json +import os + +import pystac +import pytest +from rio_tiler.io import Reader + +from titiler.stacapi.stac_reader import CustomSTACReader +from titiler.stacapi.xarray import XarrayReader +from titiler.stacapi.models import AssetInfo + +@pytest.mark.skip(reason="To be implemented.") +def test_asset_info(): + """Test get_asset_info function""" + pass + +empty_stac_reader = CustomSTACReader({'assets': [], 'bbox': []}) +def test_stac_reader_cog(): + """Test reader is rio_tiler.io.Reader""" + asset_info = AssetInfo( + url="https://file.tif", + type="image/tiff" + ) + assert empty_stac_reader._get_reader(asset_info) == Reader + + +def test_stac_reader_netcdf(): + """Test reader attribute is titiler.stacapi.XarrayReader""" + asset_info = AssetInfo( + url="https://file.nc", + type="application/netcdf" + ) + assert empty_stac_reader._get_reader(asset_info) == XarrayReader + +def test_tile_cog(): + pass + +def test_tile_netcdf(): + pass diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index 0ca996d..3f0c620 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -1,12 +1,10 @@ """titiler-stacapi custom Mosaic Backend and Custom STACReader.""" import json -import warnings -from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Type, Union +from typing import Any, Dict, List, Optional, Tuple, Type import attr import planetary_computer as pc -import rasterio from cachetools import TTLCache, cached from cachetools.keys import hashkey from cogeo_mosaic.backends import BaseBackend @@ -20,241 +18,19 @@ from rasterio.crs import CRS from rasterio.warp import transform, transform_bounds from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS -from rio_tiler.errors import ( - AssetAsBandError, - ExpressionMixingWarning, - InvalidAssetName, - MissingAssets, - TileOutsideBounds, -) -from rio_tiler.io import Reader -from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData from rio_tiler.mosaic import mosaic_reader -from rio_tiler.tasks import multi_arrays -from rio_tiler.types import BBox, Indexes +from rio_tiler.types import BBox from urllib3 import Retry -from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings +from titiler.stacapi.stac_reader import CustomSTACReader from titiler.stacapi.utils import Timer -from titiler.stacapi.xarray import XarrayReader cache_config = CacheSettings() retry_config = RetrySettings() stac_config = STACSettings() -valid_types = { - "image/tiff; application=geotiff", - "image/tiff; application=geotiff; profile=cloud-optimized", - "image/tiff; profile=cloud-optimized; application=geotiff", - "image/vnd.stac.geotiff; cloud-optimized=true", - "image/tiff", - "image/x.geotiff", - "image/jp2", - "application/x-hdf5", - "application/x-hdf", - "application/vnd+zarr", - "application/x-netcdf", -} - - -@attr.s -class CustomSTACReader(MultiBaseReader): - """Simplified STAC Reader. - - Inputs should be in form of: - { - "id": "IAMASTACITEM", - "collection": "mycollection", - "bbox": (0, 0, 10, 10), - "assets": { - "COG": { - "href": "https://somewhereovertherainbow.io/cog.tif" - } - } - } - - """ - - input: Dict[str, Any] = attr.ib() - tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) - minzoom: int = attr.ib() - maxzoom: int = attr.ib() - - reader: Type[BaseReader] = attr.ib(default=Reader) - reader_options: Dict = attr.ib(factory=dict) - - ctx: Any = attr.ib(default=rasterio.Env) - - include_asset_types: Set[str] = attr.ib(default=valid_types) - - def __attrs_post_init__(self) -> None: - """Set reader spatial infos and list of valid assets.""" - self.bounds = self.input["bbox"] - self.crs = WGS84_CRS # Per specification STAC items are in WGS84 - self.assets = list(self.input["assets"]) - - @minzoom.default - def _minzoom(self): - return self.tms.minzoom - - @maxzoom.default - def _maxzoom(self): - return self.tms.maxzoom - - def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: - """Get Asset Reader.""" - asset_type = asset_info.get("type", None) - - if asset_type and asset_type in [ - "application/x-hdf5", - "application/x-hdf", - "application/vnd.zarr", - "application/x-netcdf", - "application/netcdf", - ]: - return XarrayReader - - return Reader - - def _get_asset_info(self, asset: str) -> AssetInfo: - """ - Validate asset names and return asset's info. - - Args: - AssetInfo (str): STAC asset info. - - Returns: - str: STAC asset href. - - """ - if asset not in self.assets: - raise InvalidAssetName( - f"{asset} is not valid. Should be one of {self.assets}" - ) - - asset_info = self.input["assets"][asset] - - url = asset_info["href"] - if alternate := stac_config.alternate_url: - url = asset_info["alternate"][alternate]["href"] - - info = AssetInfo(url=url, env={}) - - if asset_info.get("type"): - info["type"] = asset_info["type"] - - if header_size := asset_info.get("file:header_size"): - info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size # type: ignore - - if bands := asset_info.get("raster:bands"): - stats = [ - (b["statistics"]["minimum"], b["statistics"]["maximum"]) - for b in bands - if {"minimum", "maximum"}.issubset(b.get("statistics", {})) - ] - if len(stats) == len(bands): - info["dataset_statistics"] = stats - - return info - - def tile( # noqa: C901 - self, - tile_x: int, - tile_y: int, - tile_z: int, - assets: Union[Sequence[str], str] = (), - expression: Optional[str] = None, - asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset - asset_as_band: bool = False, - **kwargs: Any, - ) -> ImageData: - """Read and merge Wep Map tiles from multiple assets. - - Args: - tile_x (int): Tile's horizontal index. - tile_y (int): Tile's vertical index. - tile_z (int): Tile's zoom level index. - assets (sequence of str or str, optional): assets to fetch info from. - expression (str, optional): rio-tiler expression for the asset list (e.g. asset1/asset2+asset3). - asset_indexes (dict, optional): Band indexes for each asset (e.g {"asset1": 1, "asset2": (1, 2,)}). - kwargs (optional): Options to forward to the `self.reader.tile` method. - - Returns: - rio_tiler.models.ImageData: ImageData instance with data, mask and tile spatial info. - - """ - if not self.tile_exists(tile_x, tile_y, tile_z): - raise TileOutsideBounds( - f"Tile {tile_z}/{tile_x}/{tile_y} is outside image bounds" - ) - - if isinstance(assets, str): - assets = (assets,) - - if assets and expression: - warnings.warn( - "Both expression and assets passed; expression will overwrite assets parameter.", - ExpressionMixingWarning, - stacklevel=2, - ) - - if expression: - assets = self.parse_expression(expression, asset_as_band=asset_as_band) - - if not assets: - raise MissingAssets( - "assets must be passed either via `expression` or `assets` options." - ) - - asset_indexes = asset_indexes or {} - - # We fall back to `indexes` if provided - indexes = kwargs.pop("indexes", None) - - def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: - idx = asset_indexes.get(asset) or indexes # type: ignore - - asset_info = self._get_asset_info(asset) - reader = self._get_reader(asset_info) - - with self.ctx(**asset_info.get("env", {})): - with reader( - asset_info["url"], tms=self.tms, **self.reader_options - ) as src: - if type(src) == XarrayReader: - raise NotImplementedError("XarrayReader not yet implemented") - data = src.tile(*args, indexes=idx, **kwargs) - - self._update_statistics( - data, - indexes=idx, - statistics=asset_info.get("dataset_statistics"), - ) - - metadata = data.metadata or {} - if m := asset_info.get("metadata"): - metadata.update(m) - data.metadata = {asset: metadata} - - if asset_as_band: - if len(data.band_names) > 1: - raise AssetAsBandError( - "Can't use `asset_as_band` for multibands asset" - ) - data.band_names = [asset] - else: - data.band_names = [f"{asset}_{n}" for n in data.band_names] - - return data - - img = multi_arrays(assets, _reader, tile_x, tile_y, tile_z, **kwargs) - if expression: - return img.apply_expression(expression) - - return img - @attr.s class STACAPIBackend(BaseBackend): diff --git a/titiler/stacapi/stac_reader.py b/titiler/stacapi/stac_reader.py new file mode 100644 index 0000000..a3e52ca --- /dev/null +++ b/titiler/stacapi/stac_reader.py @@ -0,0 +1,240 @@ +"""titiler-stacapi custom STACReader.""" + +import warnings +from typing import Any, Dict, Optional, Sequence, Set, Type, Union + +import attr +import rasterio +from morecantile import TileMatrixSet +from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS +from rio_tiler.errors import ( + AssetAsBandError, + ExpressionMixingWarning, + InvalidAssetName, + MissingAssets, + TileOutsideBounds, +) +from rio_tiler.io import Reader +from rio_tiler.io.base import BaseReader, MultiBaseReader +from rio_tiler.models import ImageData +from rio_tiler.tasks import multi_arrays +from rio_tiler.types import Indexes + +from titiler.stacapi.models import AssetInfo +from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings +from titiler.stacapi.xarray import XarrayReader + +cache_config = CacheSettings() +retry_config = RetrySettings() +stac_config = STACSettings() + +valid_types = { + "image/tiff; application=geotiff", + "image/tiff; application=geotiff; profile=cloud-optimized", + "image/tiff; profile=cloud-optimized; application=geotiff", + "image/vnd.stac.geotiff; cloud-optimized=true", + "image/tiff", + "image/x.geotiff", + "image/jp2", + "application/x-hdf5", + "application/x-hdf", + "application/vnd+zarr", + "application/x-netcdf", +} + + +@attr.s +class CustomSTACReader(MultiBaseReader): + """Simplified STAC Reader. + + Inputs should be in form of: + { + "id": "IAMASTACITEM", + "collection": "mycollection", + "bbox": (0, 0, 10, 10), + "assets": { + "COG": { + "href": "https://somewhereovertherainbow.io/cog.tif" + } + } + } + + """ + + input: Dict[str, Any] = attr.ib() + tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) + minzoom: int = attr.ib() + maxzoom: int = attr.ib() + + reader: Type[BaseReader] = attr.ib(default=Reader) + reader_options: Dict = attr.ib(factory=dict) + + ctx: Any = attr.ib(default=rasterio.Env) + + include_asset_types: Set[str] = attr.ib(default=valid_types) + + def __attrs_post_init__(self) -> None: + """Set reader spatial infos and list of valid assets.""" + self.bounds = self.input["bbox"] + self.crs = WGS84_CRS # Per specification STAC items are in WGS84 + self.assets = list(self.input["assets"]) + + @minzoom.default + def _minzoom(self): + return self.tms.minzoom + + @maxzoom.default + def _maxzoom(self): + return self.tms.maxzoom + + def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: + """Get Asset Reader.""" + asset_type = asset_info.get("type", None) + + if asset_type and asset_type in [ + "application/x-hdf5", + "application/x-hdf", + "application/vnd.zarr", + "application/x-netcdf", + "application/netcdf", + ]: + return XarrayReader + + return Reader + + def _get_asset_info(self, asset: str) -> AssetInfo: + """ + Validate asset names and return asset's info. + + Args: + AssetInfo (str): STAC asset info. + + Returns: + str: STAC asset href. + + """ + if asset not in self.assets: + raise InvalidAssetName( + f"{asset} is not valid. Should be one of {self.assets}" + ) + + asset_info = self.input["assets"][asset] + + url = asset_info["href"] + if alternate := stac_config.alternate_url: + url = asset_info["alternate"][alternate]["href"] + + info = AssetInfo(url=url, env={}) + + if asset_info.get("type"): + info["type"] = asset_info["type"] + + if header_size := asset_info.get("file:header_size"): + info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size # type: ignore + + if bands := asset_info.get("raster:bands"): + stats = [ + (b["statistics"]["minimum"], b["statistics"]["maximum"]) + for b in bands + if {"minimum", "maximum"}.issubset(b.get("statistics", {})) + ] + if len(stats) == len(bands): + info["dataset_statistics"] = stats + + return info + + def tile( # noqa: C901 + self, + tile_x: int, + tile_y: int, + tile_z: int, + assets: Union[Sequence[str], str] = (), + expression: Optional[str] = None, + asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset + asset_as_band: bool = False, + **kwargs: Any, + ) -> ImageData: + """Read and merge Wep Map tiles from multiple assets. + + Args: + tile_x (int): Tile's horizontal index. + tile_y (int): Tile's vertical index. + tile_z (int): Tile's zoom level index. + assets (sequence of str or str, optional): assets to fetch info from. + expression (str, optional): rio-tiler expression for the asset list (e.g. asset1/asset2+asset3). + asset_indexes (dict, optional): Band indexes for each asset (e.g {"asset1": 1, "asset2": (1, 2,)}). + kwargs (optional): Options to forward to the `self.reader.tile` method. + + Returns: + rio_tiler.models.ImageData: ImageData instance with data, mask and tile spatial info. + + """ + if not self.tile_exists(tile_x, tile_y, tile_z): + raise TileOutsideBounds( + f"Tile {tile_z}/{tile_x}/{tile_y} is outside image bounds" + ) + + if isinstance(assets, str): + assets = (assets,) + + if assets and expression: + warnings.warn( + "Both expression and assets passed; expression will overwrite assets parameter.", + ExpressionMixingWarning, + stacklevel=2, + ) + + if expression: + assets = self.parse_expression(expression, asset_as_band=asset_as_band) + + if not assets: + raise MissingAssets( + "assets must be passed either via `expression` or `assets` options." + ) + + asset_indexes = asset_indexes or {} + + # We fall back to `indexes` if provided + indexes = kwargs.pop("indexes", None) + + def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: + idx = asset_indexes.get(asset) or indexes # type: ignore + + asset_info = self._get_asset_info(asset) + reader = self._get_reader(asset_info) + + with self.ctx(**asset_info.get("env", {})): + with reader( + asset_info["url"], tms=self.tms, **self.reader_options + ) as src: + if type(src) == XarrayReader: + raise NotImplementedError("XarrayReader not yet implemented") + data = src.tile(*args, indexes=idx, **kwargs) + + self._update_statistics( + data, + indexes=idx, + statistics=asset_info.get("dataset_statistics"), + ) + + metadata = data.metadata or {} + if m := asset_info.get("metadata"): + metadata.update(m) + data.metadata = {asset: metadata} + + if asset_as_band: + if len(data.band_names) > 1: + raise AssetAsBandError( + "Can't use `asset_as_band` for multibands asset" + ) + data.band_names = [asset] + else: + data.band_names = [f"{asset}_{n}" for n in data.band_names] + + return data + + img = multi_arrays(assets, _reader, tile_x, tile_y, tile_z, **kwargs) + if expression: + return img.apply_expression(expression) + + return img \ No newline at end of file From a2b3376fbf590d3227966b61cf31b2c3392a0fd2 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 16:33:10 -0700 Subject: [PATCH 06/23] working on tile test --- tests/test_stac_reader.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/test_stac_reader.py b/tests/test_stac_reader.py index 1e8285f..ac8f09d 100644 --- a/tests/test_stac_reader.py +++ b/tests/test_stac_reader.py @@ -3,30 +3,37 @@ import json import os -import pystac +from .conftest import mock_rasterio_open + import pytest +from unittest.mock import patch from rio_tiler.io import Reader +from rio_tiler.models import ImageData from titiler.stacapi.stac_reader import CustomSTACReader from titiler.stacapi.xarray import XarrayReader from titiler.stacapi.models import AssetInfo +item_file = os.path.join( + os.path.dirname(__file__), "fixtures", "20200307aC0853900w361030.json" +) +item_json = json.loads(open(item_file).read()) + @pytest.mark.skip(reason="To be implemented.") def test_asset_info(): """Test get_asset_info function""" pass empty_stac_reader = CustomSTACReader({'assets': [], 'bbox': []}) -def test_stac_reader_cog(): +def test_get_reader_cog(): """Test reader is rio_tiler.io.Reader""" asset_info = AssetInfo( - url="https://file.tif", - type="image/tiff" + url="https://file.tif" ) assert empty_stac_reader._get_reader(asset_info) == Reader -def test_stac_reader_netcdf(): +def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo( url="https://file.nc", @@ -34,8 +41,13 @@ def test_stac_reader_netcdf(): ) assert empty_stac_reader._get_reader(asset_info) == XarrayReader -def test_tile_cog(): - pass +@patch("rio_tiler.io.rasterio.rasterio") +def test_tile_cog(rio): + rio.open = mock_rasterio_open + with CustomSTACReader(item_json) as reader: + img = reader.tile(0, 0, 0, assets=['cog']) + assert type(img) == ImageData +@pytest.mark.skip(reason="To be implemented.") def test_tile_netcdf(): pass From c64846ea7d4f49c9be46b7e5493acc06150efc4e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 16:33:45 -0700 Subject: [PATCH 07/23] Linting --- tests/test_stac_reader.py | 29 ++++++++++++++++------------- titiler/stacapi/stac_reader.py | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/test_stac_reader.py b/tests/test_stac_reader.py index ac8f09d..12c8dbc 100644 --- a/tests/test_stac_reader.py +++ b/tests/test_stac_reader.py @@ -2,52 +2,55 @@ import json import os - -from .conftest import mock_rasterio_open +from unittest.mock import patch import pytest -from unittest.mock import patch from rio_tiler.io import Reader from rio_tiler.models import ImageData +from titiler.stacapi.models import AssetInfo from titiler.stacapi.stac_reader import CustomSTACReader from titiler.stacapi.xarray import XarrayReader -from titiler.stacapi.models import AssetInfo + +from .conftest import mock_rasterio_open item_file = os.path.join( os.path.dirname(__file__), "fixtures", "20200307aC0853900w361030.json" ) item_json = json.loads(open(item_file).read()) + @pytest.mark.skip(reason="To be implemented.") def test_asset_info(): """Test get_asset_info function""" pass -empty_stac_reader = CustomSTACReader({'assets': [], 'bbox': []}) + +empty_stac_reader = CustomSTACReader({"assets": [], "bbox": []}) + + def test_get_reader_cog(): """Test reader is rio_tiler.io.Reader""" - asset_info = AssetInfo( - url="https://file.tif" - ) + asset_info = AssetInfo(url="https://file.tif") assert empty_stac_reader._get_reader(asset_info) == Reader def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" - asset_info = AssetInfo( - url="https://file.nc", - type="application/netcdf" - ) + asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") assert empty_stac_reader._get_reader(asset_info) == XarrayReader + @patch("rio_tiler.io.rasterio.rasterio") def test_tile_cog(rio): + """Test tile function with COG asset.""" rio.open = mock_rasterio_open with CustomSTACReader(item_json) as reader: - img = reader.tile(0, 0, 0, assets=['cog']) + img = reader.tile(0, 0, 0, assets=["cog"]) assert type(img) == ImageData + @pytest.mark.skip(reason="To be implemented.") def test_tile_netcdf(): + """Test tile function with netcdf asset.""" pass diff --git a/titiler/stacapi/stac_reader.py b/titiler/stacapi/stac_reader.py index a3e52ca..59f659a 100644 --- a/titiler/stacapi/stac_reader.py +++ b/titiler/stacapi/stac_reader.py @@ -237,4 +237,4 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: if expression: return img.apply_expression(expression) - return img \ No newline at end of file + return img From 0d35cf57cf71beb5c203386a1daf77b7fde2dd58 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 16:53:25 -0700 Subject: [PATCH 08/23] Rename stac reader to asset reader --- ...st_stac_reader.py => test_asset_reader.py} | 12 ++++---- .../{stac_reader.py => asset_reader.py} | 29 ++++--------------- titiler/stacapi/backend.py | 6 ++-- 3 files changed, 13 insertions(+), 34 deletions(-) rename tests/{test_stac_reader.py => test_asset_reader.py} (87%) rename titiler/stacapi/{stac_reader.py => asset_reader.py} (90%) diff --git a/tests/test_stac_reader.py b/tests/test_asset_reader.py similarity index 87% rename from tests/test_stac_reader.py rename to tests/test_asset_reader.py index 12c8dbc..211186b 100644 --- a/tests/test_stac_reader.py +++ b/tests/test_asset_reader.py @@ -9,7 +9,7 @@ from rio_tiler.models import ImageData from titiler.stacapi.models import AssetInfo -from titiler.stacapi.stac_reader import CustomSTACReader +from titiler.stacapi.asset_reader import AssetReader from titiler.stacapi.xarray import XarrayReader from .conftest import mock_rasterio_open @@ -26,26 +26,24 @@ def test_asset_info(): pass -empty_stac_reader = CustomSTACReader({"assets": [], "bbox": []}) - - -def test_get_reader_cog(): +def test_get_reader_any(): """Test reader is rio_tiler.io.Reader""" asset_info = AssetInfo(url="https://file.tif") + empty_stac_reader = AssetReader({}) assert empty_stac_reader._get_reader(asset_info) == Reader def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") + empty_stac_reader = AssetReader({}) assert empty_stac_reader._get_reader(asset_info) == XarrayReader - @patch("rio_tiler.io.rasterio.rasterio") def test_tile_cog(rio): """Test tile function with COG asset.""" rio.open = mock_rasterio_open - with CustomSTACReader(item_json) as reader: + with AssetReader(item_json) as reader: img = reader.tile(0, 0, 0, assets=["cog"]) assert type(img) == ImageData diff --git a/titiler/stacapi/stac_reader.py b/titiler/stacapi/asset_reader.py similarity index 90% rename from titiler/stacapi/stac_reader.py rename to titiler/stacapi/asset_reader.py index 59f659a..37252ce 100644 --- a/titiler/stacapi/stac_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -44,24 +44,11 @@ @attr.s -class CustomSTACReader(MultiBaseReader): - """Simplified STAC Reader. - - Inputs should be in form of: - { - "id": "IAMASTACITEM", - "collection": "mycollection", - "bbox": (0, 0, 10, 10), - "assets": { - "COG": { - "href": "https://somewhereovertherainbow.io/cog.tif" - } - } - } - +class AssetReader(MultiBaseReader): + """ + Asset reader for STAC items. """ - input: Dict[str, Any] = attr.ib() tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) minzoom: int = attr.ib() maxzoom: int = attr.ib() @@ -73,12 +60,6 @@ class CustomSTACReader(MultiBaseReader): include_asset_types: Set[str] = attr.ib(default=valid_types) - def __attrs_post_init__(self) -> None: - """Set reader spatial infos and list of valid assets.""" - self.bounds = self.input["bbox"] - self.crs = WGS84_CRS # Per specification STAC items are in WGS84 - self.assets = list(self.input["assets"]) - @minzoom.default def _minzoom(self): return self.tms.minzoom @@ -107,10 +88,10 @@ def _get_asset_info(self, asset: str) -> AssetInfo: Validate asset names and return asset's info. Args: - AssetInfo (str): STAC asset info. + asset (str): asset name. Returns: - str: STAC asset href. + AssetInfo: STAC asset info """ if asset not in self.assets: diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index 3f0c620..282a3e1 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -24,7 +24,7 @@ from urllib3 import Retry from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings -from titiler.stacapi.stac_reader import CustomSTACReader +from titiler.stacapi.asset_reader import AssetReader from titiler.stacapi.utils import Timer cache_config = CacheSettings() @@ -45,8 +45,8 @@ class STACAPIBackend(BaseBackend): minzoom: int = attr.ib() maxzoom: int = attr.ib() - # Use Custom STAC reader (outside init) - reader: Type[CustomSTACReader] = attr.ib(init=False, default=CustomSTACReader) + # Use custom asset reader (outside init) + reader: Type[AssetReader] = attr.ib(init=False, default=AssetReader) reader_options: Dict = attr.ib(factory=dict) # default values for bounds From 829c39c7a1e52940b809bc3b327b38ebfb11520a Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Mon, 3 Jun 2024 16:58:13 -0700 Subject: [PATCH 09/23] Fix linting --- tests/test_asset_reader.py | 3 ++- titiler/stacapi/asset_reader.py | 7 ++++--- titiler/stacapi/backend.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 211186b..6a50d57 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -8,8 +8,8 @@ from rio_tiler.io import Reader from rio_tiler.models import ImageData -from titiler.stacapi.models import AssetInfo from titiler.stacapi.asset_reader import AssetReader +from titiler.stacapi.models import AssetInfo from titiler.stacapi.xarray import XarrayReader from .conftest import mock_rasterio_open @@ -39,6 +39,7 @@ def test_get_reader_netcdf(): empty_stac_reader = AssetReader({}) assert empty_stac_reader._get_reader(asset_info) == XarrayReader + @patch("rio_tiler.io.rasterio.rasterio") def test_tile_cog(rio): """Test tile function with COG asset.""" diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index 37252ce..4c17a02 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -1,4 +1,4 @@ -"""titiler-stacapi custom STACReader.""" +"""titiler-stacapi Asset Reader.""" import warnings from typing import Any, Dict, Optional, Sequence, Set, Type, Union @@ -6,7 +6,7 @@ import attr import rasterio from morecantile import TileMatrixSet -from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS +from rio_tiler.constants import WEB_MERCATOR_TMS from rio_tiler.errors import ( AssetAsBandError, ExpressionMixingWarning, @@ -49,6 +49,7 @@ class AssetReader(MultiBaseReader): Asset reader for STAC items. """ + input: Any = attr.ib() tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) minzoom: int = attr.ib() maxzoom: int = attr.ib() @@ -91,7 +92,7 @@ def _get_asset_info(self, asset: str) -> AssetInfo: asset (str): asset name. Returns: - AssetInfo: STAC asset info + AssetInfo: Asset info """ if asset not in self.assets: diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index 282a3e1..cb492fb 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -23,8 +23,8 @@ from rio_tiler.types import BBox from urllib3 import Retry -from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.asset_reader import AssetReader +from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.utils import Timer cache_config = CacheSettings() From ff72c2ce67c71352ddd08b982f3bd7e3bae93e82 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 09:39:23 -0700 Subject: [PATCH 10/23] Raise error in _get_reader --- tests/test_asset_reader.py | 4 +-- titiler/stacapi/asset_reader.py | 37 ++++++++--------------- titiler/stacapi/factory.py | 53 ++++++++++++++++----------------- titiler/stacapi/xarray.py | 19 ------------ 4 files changed, 40 insertions(+), 73 deletions(-) delete mode 100644 titiler/stacapi/xarray.py diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 6a50d57..4289c54 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -29,14 +29,14 @@ def test_asset_info(): def test_get_reader_any(): """Test reader is rio_tiler.io.Reader""" asset_info = AssetInfo(url="https://file.tif") - empty_stac_reader = AssetReader({}) + empty_stac_reader = AssetReader({'bbox': [], 'assets': []}) assert empty_stac_reader._get_reader(asset_info) == Reader def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") - empty_stac_reader = AssetReader({}) + empty_stac_reader = AssetReader({'bbox': [], 'assets': []}) assert empty_stac_reader._get_reader(asset_info) == XarrayReader diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index 4c17a02..0bb1c16 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -6,7 +6,7 @@ import attr import rasterio from morecantile import TileMatrixSet -from rio_tiler.constants import WEB_MERCATOR_TMS +from rio_tiler.constants import WEB_MERCATOR_TMS, WGS84_CRS from rio_tiler.errors import ( AssetAsBandError, ExpressionMixingWarning, @@ -22,7 +22,6 @@ from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings -from titiler.stacapi.xarray import XarrayReader cache_config = CacheSettings() retry_config = RetrySettings() @@ -49,6 +48,7 @@ class AssetReader(MultiBaseReader): Asset reader for STAC items. """ + # bounds and assets are required input: Any = attr.ib() tms: TileMatrixSet = attr.ib(default=WEB_MERCATOR_TMS) minzoom: int = attr.ib() @@ -68,6 +68,12 @@ def _minzoom(self): @maxzoom.default def _maxzoom(self): return self.tms.maxzoom + + def __attrs_post_init__(self): + # MultibaseReader includes the spatial mixin so these attributes are required to assert that the tile exists inside the bounds of the item + self.crs = WGS84_CRS # Per specification STAC items are in WGS84 + self.bounds = self.input["bbox"] + self.assets = list(self.input["assets"]) def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: """Get Asset Reader.""" @@ -80,7 +86,7 @@ def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: "application/x-netcdf", "application/netcdf", ]: - return XarrayReader + raise NotImplementedError("XarrayReader not yet implemented") return Reader @@ -111,6 +117,8 @@ def _get_asset_info(self, asset: str) -> AssetInfo: if asset_info.get("type"): info["type"] = asset_info["type"] + # there is a file STAC extension for which `header_size` is the size of the header in the file + # if this value is present, we want to use the GDAL_INGESTED_BYTES_AT_OPEN env variable to read that many bytes at file open. if header_size := asset_info.get("file:header_size"): info["env"]["GDAL_INGESTED_BYTES_AT_OPEN"] = header_size # type: ignore @@ -132,7 +140,6 @@ def tile( # noqa: C901 tile_z: int, assets: Union[Sequence[str], str] = (), expression: Optional[str] = None, - asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset asset_as_band: bool = False, **kwargs: Any, ) -> ImageData: @@ -144,7 +151,6 @@ def tile( # noqa: C901 tile_z (int): Tile's zoom level index. assets (sequence of str or str, optional): assets to fetch info from. expression (str, optional): rio-tiler expression for the asset list (e.g. asset1/asset2+asset3). - asset_indexes (dict, optional): Band indexes for each asset (e.g {"asset1": 1, "asset2": (1, 2,)}). kwargs (optional): Options to forward to the `self.reader.tile` method. Returns: @@ -174,13 +180,7 @@ def tile( # noqa: C901 "assets must be passed either via `expression` or `assets` options." ) - asset_indexes = asset_indexes or {} - - # We fall back to `indexes` if provided - indexes = kwargs.pop("indexes", None) - def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: - idx = asset_indexes.get(asset) or indexes # type: ignore asset_info = self._get_asset_info(asset) reader = self._get_reader(asset_info) @@ -189,20 +189,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: with reader( asset_info["url"], tms=self.tms, **self.reader_options ) as src: - if type(src) == XarrayReader: - raise NotImplementedError("XarrayReader not yet implemented") - data = src.tile(*args, indexes=idx, **kwargs) - - self._update_statistics( - data, - indexes=idx, - statistics=asset_info.get("dataset_statistics"), - ) - - metadata = data.metadata or {} - if m := asset_info.get("metadata"): - metadata.update(m) - data.metadata = {asset: metadata} + data = src.tile(*args, **kwargs) if asset_as_band: if len(data.band_names) > 1: diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index ccdbe2f..7ef345a 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -195,35 +195,34 @@ def tile( scale = scale or 1 tms = self.supported_tms.get(tileMatrixSetId) - with rasterio.Env(**env): - with self.reader( - url=api_params["api_url"], - headers=api_params.get("headers", {}), - tms=tms, - reader_options={**reader_params}, - **backend_params, - ) as src_dst: - if MOSAIC_STRICT_ZOOM and ( - z < src_dst.minzoom or z > src_dst.maxzoom - ): - raise HTTPException( - 400, - f"Invalid ZOOM level {z}. Should be between {src_dst.minzoom} and {src_dst.maxzoom}", - ) - - image, assets = src_dst.tile( - x, - y, - z, - search_query=search_query, - tilesize=scale * 256, - pixel_selection=pixel_selection, - threads=MOSAIC_THREADS, - **tile_params, - **layer_params, - **dataset_params, + with self.reader( + url=api_params["api_url"], + headers=api_params.get("headers", {}), + tms=tms, + reader_options={**reader_params}, + **backend_params, + ) as src_dst: + if MOSAIC_STRICT_ZOOM and ( + z < src_dst.minzoom or z > src_dst.maxzoom + ): + raise HTTPException( + 400, + f"Invalid ZOOM level {z}. Should be between {src_dst.minzoom} and {src_dst.maxzoom}", ) + image, assets = src_dst.tile( + x, + y, + z, + search_query=search_query, + tilesize=scale * 256, + pixel_selection=pixel_selection, + threads=MOSAIC_THREADS, + **tile_params, + **layer_params, + **dataset_params, + ) + if post_process: image = post_process(image) diff --git a/titiler/stacapi/xarray.py b/titiler/stacapi/xarray.py deleted file mode 100644 index 5d301a7..0000000 --- a/titiler/stacapi/xarray.py +++ /dev/null @@ -1,19 +0,0 @@ -""" Custom Xarray Reader. """ - -import attr - -# from typing import Dict -import rio_tiler.io as riotio - -# import xarray as xr -# import fsspec - - -@attr.s -class XarrayReader(riotio.XarrayReader): - """Custom Xarray Reader.""" - - # def __attrs_post_init__(self) -> None: - # httpfs = fsspec.filesystem("https") - # da = xr.open_dataset(httpfs.open(self.input), engine="h5netcdf")[variable] - # self.input = da From 1aa53e039e7ae13c0bff30aac847170b6afee724 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:07:33 -0700 Subject: [PATCH 11/23] Working on tests --- tests/conftest.py | 2 ++ tests/test_asset_reader.py | 10 +++++----- titiler/stacapi/asset_reader.py | 1 - 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 29569ec..84531fb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ import pytest import rasterio from fastapi.testclient import TestClient +from rio_tiler.models import ImageData DATA_DIR = os.path.join(os.path.dirname(__file__), "fixtures") @@ -29,4 +30,5 @@ def mock_rasterio_open(asset): "https://noaa-eri-pds.s3.us-east-1.amazonaws.com/2020_Nashville_Tornado/20200307a_RGB", DATA_DIR, ) + import pdb; pdb.set_trace() return rasterio.open(asset) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 4289c54..8aa59f0 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -10,7 +10,6 @@ from titiler.stacapi.asset_reader import AssetReader from titiler.stacapi.models import AssetInfo -from titiler.stacapi.xarray import XarrayReader from .conftest import mock_rasterio_open @@ -33,21 +32,22 @@ def test_get_reader_any(): assert empty_stac_reader._get_reader(asset_info) == Reader +@pytest.mark.xfail(reason="To be implemented.") def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") empty_stac_reader = AssetReader({'bbox': [], 'assets': []}) - assert empty_stac_reader._get_reader(asset_info) == XarrayReader - + empty_stac_reader._get_reader(asset_info) +@pytest.mark.skip(reason="Too slow.") @patch("rio_tiler.io.rasterio.rasterio") def test_tile_cog(rio): """Test tile function with COG asset.""" rio.open = mock_rasterio_open + with AssetReader(item_json) as reader: img = reader.tile(0, 0, 0, assets=["cog"]) - assert type(img) == ImageData - + assert isinstance(img, ImageData) @pytest.mark.skip(reason="To be implemented.") def test_tile_netcdf(): diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index 0bb1c16..4802be1 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -181,7 +181,6 @@ def tile( # noqa: C901 ) def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: - asset_info = self._get_asset_info(asset) reader = self._get_reader(asset_info) From b68c0c505dc6a6459ae1190082432a07d2290f30 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:08:06 -0700 Subject: [PATCH 12/23] Linting --- tests/conftest.py | 5 +++-- tests/test_asset_reader.py | 6 ++++-- titiler/stacapi/asset_reader.py | 8 +++++--- titiler/stacapi/factory.py | 5 +---- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 84531fb..81c656a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,6 @@ import pytest import rasterio from fastapi.testclient import TestClient -from rio_tiler.models import ImageData DATA_DIR = os.path.join(os.path.dirname(__file__), "fixtures") @@ -30,5 +29,7 @@ def mock_rasterio_open(asset): "https://noaa-eri-pds.s3.us-east-1.amazonaws.com/2020_Nashville_Tornado/20200307a_RGB", DATA_DIR, ) - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() return rasterio.open(asset) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 8aa59f0..0726f5b 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -28,7 +28,7 @@ def test_asset_info(): def test_get_reader_any(): """Test reader is rio_tiler.io.Reader""" asset_info = AssetInfo(url="https://file.tif") - empty_stac_reader = AssetReader({'bbox': [], 'assets': []}) + empty_stac_reader = AssetReader({"bbox": [], "assets": []}) assert empty_stac_reader._get_reader(asset_info) == Reader @@ -36,9 +36,10 @@ def test_get_reader_any(): def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") - empty_stac_reader = AssetReader({'bbox': [], 'assets': []}) + empty_stac_reader = AssetReader({"bbox": [], "assets": []}) empty_stac_reader._get_reader(asset_info) + @pytest.mark.skip(reason="Too slow.") @patch("rio_tiler.io.rasterio.rasterio") def test_tile_cog(rio): @@ -49,6 +50,7 @@ def test_tile_cog(rio): img = reader.tile(0, 0, 0, assets=["cog"]) assert isinstance(img, ImageData) + @pytest.mark.skip(reason="To be implemented.") def test_tile_netcdf(): """Test tile function with netcdf asset.""" diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index 4802be1..723a7f2 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -18,7 +18,6 @@ from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData from rio_tiler.tasks import multi_arrays -from rio_tiler.types import Indexes from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings @@ -68,8 +67,11 @@ def _minzoom(self): @maxzoom.default def _maxzoom(self): return self.tms.maxzoom - + def __attrs_post_init__(self): + """ + Post Init. + """ # MultibaseReader includes the spatial mixin so these attributes are required to assert that the tile exists inside the bounds of the item self.crs = WGS84_CRS # Per specification STAC items are in WGS84 self.bounds = self.input["bbox"] @@ -86,7 +88,7 @@ def _get_reader(self, asset_info: AssetInfo) -> Type[BaseReader]: "application/x-netcdf", "application/netcdf", ]: - raise NotImplementedError("XarrayReader not yet implemented") + raise NotImplementedError("XarrayReader not yet implemented") return Reader diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index 7ef345a..d5d9f59 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -6,7 +6,6 @@ from urllib.parse import urlencode import jinja2 -import rasterio from cogeo_mosaic.backends import BaseBackend from fastapi import Depends, HTTPException, Path, Query from fastapi.dependencies.utils import get_dependant, request_params_to_args @@ -202,9 +201,7 @@ def tile( reader_options={**reader_params}, **backend_params, ) as src_dst: - if MOSAIC_STRICT_ZOOM and ( - z < src_dst.minzoom or z > src_dst.maxzoom - ): + if MOSAIC_STRICT_ZOOM and (z < src_dst.minzoom or z > src_dst.maxzoom): raise HTTPException( 400, f"Invalid ZOOM level {z}. Should be between {src_dst.minzoom} and {src_dst.maxzoom}", From ba2cb6f76c378c7bb5ed32b1c30aad499f5071aa Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:12:57 -0700 Subject: [PATCH 13/23] Revert change to asset indexes --- titiler/stacapi/asset_reader.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index 723a7f2..f457078 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -142,6 +142,7 @@ def tile( # noqa: C901 tile_z: int, assets: Union[Sequence[str], str] = (), expression: Optional[str] = None, + asset_indexes: Optional[Dict[str, Indexes]] = None, # Indexes for each asset asset_as_band: bool = False, **kwargs: Any, ) -> ImageData: @@ -153,6 +154,7 @@ def tile( # noqa: C901 tile_z (int): Tile's zoom level index. assets (sequence of str or str, optional): assets to fetch info from. expression (str, optional): rio-tiler expression for the asset list (e.g. asset1/asset2+asset3). + asset_indexes (dict, optional): Band indexes for each asset (e.g {"asset1": 1, "asset2": (1, 2,)}). kwargs (optional): Options to forward to the `self.reader.tile` method. Returns: @@ -181,8 +183,16 @@ def tile( # noqa: C901 raise MissingAssets( "assets must be passed either via `expression` or `assets` options." ) + + # indexes comes from the bidx query-parameter. + # but for asset based backend we usually use asset_bidx option. + asset_indexes = asset_indexes or {} + + # We fall back to `indexes` if provided + indexes = kwargs.pop("indexes", None) def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: + idx = asset_indexes.get(asset) or indexes # type: ignore asset_info = self._get_asset_info(asset) reader = self._get_reader(asset_info) @@ -190,7 +200,7 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: with reader( asset_info["url"], tms=self.tms, **self.reader_options ) as src: - data = src.tile(*args, **kwargs) + data = src.tile(*args, indexes=idx, **kwargs) if asset_as_band: if len(data.band_names) > 1: From cdff4d0f1e86e45545b9dce639f5825f57ffde4e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:13:46 -0700 Subject: [PATCH 14/23] Revert change to indexes --- titiler/stacapi/asset_reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index f457078..cdee039 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -18,6 +18,7 @@ from rio_tiler.io.base import BaseReader, MultiBaseReader from rio_tiler.models import ImageData from rio_tiler.tasks import multi_arrays +from rio_tiler.types import Indexes from titiler.stacapi.models import AssetInfo from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings @@ -183,7 +184,7 @@ def tile( # noqa: C901 raise MissingAssets( "assets must be passed either via `expression` or `assets` options." ) - + # indexes comes from the bidx query-parameter. # but for asset based backend we usually use asset_bidx option. asset_indexes = asset_indexes or {} From 5c46e0334707d659136091aa958d6cc2a04f564f Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:15:22 -0700 Subject: [PATCH 15/23] Remove unused fixture --- tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json diff --git a/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json b/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json deleted file mode 100644 index 0fc00a6..0000000 --- a/tests/fixtures/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1","bbox":[-180.0,-90.0,180.0,90.0],"type":"Feature","links":[{"rel":"collection","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf"},{"rel":"parent","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf"},{"rel":"root","type":"application/json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/"},{"rel":"self","type":"application/geo+json","href":"https://planetarycomputer.microsoft.com/api/stac/v1/collections/esa-cci-lc-netcdf/items/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1"}],"assets":{"netcdf":{"href":"https://landcoverdata.blob.core.windows.net/esa-cci-lc/netcdf/C3S-LC-L4-LCCS-Map-300m-P1Y-2020-v2.1.1.nc","created":"2021-05-18T10:15:40Z","cube:variables":{"crs":{"type":"auxiliary","dimensions":[]},"lat":{"axis":"Y","type":"auxiliary","unit":"degrees_north","dimensions":["lat"],"description":"latitude"},"lon":{"axis":"X","type":"auxiliary","unit":"degrees_east","dimensions":["lon"],"description":"longitude"},"time":{"axis":"T","type":"auxiliary","unit":"days since 1970-01-01 00:00:00","dimensions":["time"],"description":"time"},"lat_bounds":{"type":"auxiliary","dimensions":["lat","bounds"]},"lccs_class":{"type":"data","dimensions":["time","lat","lon"],"description":"Land cover class defined in LCCS"},"lon_bounds":{"type":"auxiliary","dimensions":["lon","bounds"]},"time_bounds":{"type":"auxiliary","dimensions":["time","bounds"]},"change_count":{"type":"data","dimensions":["time","lat","lon"],"description":"number of class changes"},"processed_flag":{"type":"data","dimensions":["time","lat","lon"],"description":"LC map processed area flag"},"observation_count":{"type":"data","dimensions":["time","lat","lon"],"description":"number of valid observations"},"current_pixel_state":{"type":"data","dimensions":["time","lat","lon"],"description":"LC pixel type mask"}},"cube:dimensions":{"lat":{"type":"lat","extent":[-90.0,90.0]},"lon":{"type":"lon","extent":[-180.0,180.0]},"time":{"type":"time","values":[18262.0]},"bounds":{"type":"bounds","values":[0,1]}},"type":"application/netcdf","roles":["data","quality"],"title":"ESA CCI Land Cover NetCDF 4 File"}},"geometry":{"type":"Polygon","coordinates":[[[-180,-90],[180,-90],[180,90],[-180,90],[-180,-90]]]},"collection":"esa-cci-lc-netcdf","properties":{"title":"ESA CCI Land Cover Map for 2020","created":"2023-01-11T18:52:07.171349Z","datetime":null,"proj:epsg":4326,"proj:shape":[129600,64800],"end_datetime":"2020-12-31T23:59:59Z","proj:transform":[-180.0,0.002777777777778,0.0,90.0,0.0,-0.002777777777778],"start_datetime":"2020-01-01T00:00:00Z","esa_cci_lc:version":"2.1.1","processing:lineage":"Produced based on the following data sources: Sentinel-3 OLCI","processing:software":{"lc-sr":"1.0","lc-user-tools(1)":"3.14","lc-user-tools(2)":"4.5","lc-classification":"1.0"}},"stac_extensions":["https://stac-extensions.github.io/projection/v1.0.0/schema.json","https://stac-extensions.github.io/processing/v1.1.0/schema.json","https://stac-extensions.github.io/datacube/v2.1.0/schema.json"],"stac_version":"1.0.0"} \ No newline at end of file From 31cfe3cc5c239d54782930d50a4680f6d89cff5e Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 12:16:21 -0700 Subject: [PATCH 16/23] Remove pdb statement --- tests/conftest.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 81c656a..29569ec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,7 +29,4 @@ def mock_rasterio_open(asset): "https://noaa-eri-pds.s3.us-east-1.amazonaws.com/2020_Nashville_Tornado/20200307a_RGB", DATA_DIR, ) - import pdb - - pdb.set_trace() return rasterio.open(asset) From b77432b7b21df6d2b81258128cfff3dbf2d5a208 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 13:59:23 -0700 Subject: [PATCH 17/23] Implement get_asset_info test --- tests/test_asset_reader.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 0726f5b..1d56f47 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -19,10 +19,15 @@ item_json = json.loads(open(item_file).read()) -@pytest.mark.skip(reason="To be implemented.") -def test_asset_info(): +def test_get_asset_info(): """Test get_asset_info function""" - pass + asset_reader = AssetReader(item_json) + expected_asset_info = AssetInfo( + url=item_json['assets']['cog']['href'], + type=item_json['assets']['cog']['type'], + env={} + ) + assert asset_reader._get_asset_info('cog') == expected_asset_info def test_get_reader_any(): From 26ce1213958293725a00e3316b6d2d5ca3931931 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 13:59:40 -0700 Subject: [PATCH 18/23] Linting --- tests/test_asset_reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index 1d56f47..a5dd48e 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -23,11 +23,11 @@ def test_get_asset_info(): """Test get_asset_info function""" asset_reader = AssetReader(item_json) expected_asset_info = AssetInfo( - url=item_json['assets']['cog']['href'], - type=item_json['assets']['cog']['type'], - env={} + url=item_json["assets"]["cog"]["href"], + type=item_json["assets"]["cog"]["type"], + env={}, ) - assert asset_reader._get_asset_info('cog') == expected_asset_info + assert asset_reader._get_asset_info("cog") == expected_asset_info def test_get_reader_any(): From 55e57751004f68c982784d47f6cf5be71c1592c2 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Tue, 4 Jun 2024 16:16:31 -0700 Subject: [PATCH 19/23] Remove unused settings --- titiler/stacapi/asset_reader.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/asset_reader.py index cdee039..d63f7c5 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/asset_reader.py @@ -21,10 +21,8 @@ from rio_tiler.types import Indexes from titiler.stacapi.models import AssetInfo -from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings +from titiler.stacapi.settings import STACSettings -cache_config = CacheSettings() -retry_config = RetrySettings() stac_config = STACSettings() valid_types = { From 28574cd1d2f81c31d0b85b027686861b12be540d Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 5 Jun 2024 07:56:29 -0700 Subject: [PATCH 20/23] Revert change to factory --- titiler/stacapi/factory.py | 52 ++++++++++++++++++++------------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/titiler/stacapi/factory.py b/titiler/stacapi/factory.py index d5d9f59..ccdbe2f 100644 --- a/titiler/stacapi/factory.py +++ b/titiler/stacapi/factory.py @@ -6,6 +6,7 @@ from urllib.parse import urlencode import jinja2 +import rasterio from cogeo_mosaic.backends import BaseBackend from fastapi import Depends, HTTPException, Path, Query from fastapi.dependencies.utils import get_dependant, request_params_to_args @@ -194,32 +195,35 @@ def tile( scale = scale or 1 tms = self.supported_tms.get(tileMatrixSetId) - with self.reader( - url=api_params["api_url"], - headers=api_params.get("headers", {}), - tms=tms, - reader_options={**reader_params}, - **backend_params, - ) as src_dst: - if MOSAIC_STRICT_ZOOM and (z < src_dst.minzoom or z > src_dst.maxzoom): - raise HTTPException( - 400, - f"Invalid ZOOM level {z}. Should be between {src_dst.minzoom} and {src_dst.maxzoom}", + with rasterio.Env(**env): + with self.reader( + url=api_params["api_url"], + headers=api_params.get("headers", {}), + tms=tms, + reader_options={**reader_params}, + **backend_params, + ) as src_dst: + if MOSAIC_STRICT_ZOOM and ( + z < src_dst.minzoom or z > src_dst.maxzoom + ): + raise HTTPException( + 400, + f"Invalid ZOOM level {z}. Should be between {src_dst.minzoom} and {src_dst.maxzoom}", + ) + + image, assets = src_dst.tile( + x, + y, + z, + search_query=search_query, + tilesize=scale * 256, + pixel_selection=pixel_selection, + threads=MOSAIC_THREADS, + **tile_params, + **layer_params, + **dataset_params, ) - image, assets = src_dst.tile( - x, - y, - z, - search_query=search_query, - tilesize=scale * 256, - pixel_selection=pixel_selection, - threads=MOSAIC_THREADS, - **tile_params, - **layer_params, - **dataset_params, - ) - if post_process: image = post_process(image) From 613f2f9085801d1b589d4860a00fc4188a4405c2 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 5 Jun 2024 08:16:25 -0700 Subject: [PATCH 21/23] Rename asset_reader to assets_reader --- tests/test_asset_reader.py | 12 ++++++------ .../stacapi/{asset_reader.py => assets_reader.py} | 2 +- titiler/stacapi/backend.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) rename titiler/stacapi/{asset_reader.py => assets_reader.py} (99%) diff --git a/tests/test_asset_reader.py b/tests/test_asset_reader.py index a5dd48e..06efd31 100644 --- a/tests/test_asset_reader.py +++ b/tests/test_asset_reader.py @@ -8,7 +8,7 @@ from rio_tiler.io import Reader from rio_tiler.models import ImageData -from titiler.stacapi.asset_reader import AssetReader +from titiler.stacapi.assets_reader import AssetsReader from titiler.stacapi.models import AssetInfo from .conftest import mock_rasterio_open @@ -21,19 +21,19 @@ def test_get_asset_info(): """Test get_asset_info function""" - asset_reader = AssetReader(item_json) + assets_reader = AssetsReader(item_json) expected_asset_info = AssetInfo( url=item_json["assets"]["cog"]["href"], type=item_json["assets"]["cog"]["type"], env={}, ) - assert asset_reader._get_asset_info("cog") == expected_asset_info + assert assets_reader._get_asset_info("cog") == expected_asset_info def test_get_reader_any(): """Test reader is rio_tiler.io.Reader""" asset_info = AssetInfo(url="https://file.tif") - empty_stac_reader = AssetReader({"bbox": [], "assets": []}) + empty_stac_reader = AssetsReader({"bbox": [], "assets": []}) assert empty_stac_reader._get_reader(asset_info) == Reader @@ -41,7 +41,7 @@ def test_get_reader_any(): def test_get_reader_netcdf(): """Test reader attribute is titiler.stacapi.XarrayReader""" asset_info = AssetInfo(url="https://file.nc", type="application/netcdf") - empty_stac_reader = AssetReader({"bbox": [], "assets": []}) + empty_stac_reader = AssetsReader({"bbox": [], "assets": []}) empty_stac_reader._get_reader(asset_info) @@ -51,7 +51,7 @@ def test_tile_cog(rio): """Test tile function with COG asset.""" rio.open = mock_rasterio_open - with AssetReader(item_json) as reader: + with AssetsReader(item_json) as reader: img = reader.tile(0, 0, 0, assets=["cog"]) assert isinstance(img, ImageData) diff --git a/titiler/stacapi/asset_reader.py b/titiler/stacapi/assets_reader.py similarity index 99% rename from titiler/stacapi/asset_reader.py rename to titiler/stacapi/assets_reader.py index d63f7c5..5f50f42 100644 --- a/titiler/stacapi/asset_reader.py +++ b/titiler/stacapi/assets_reader.py @@ -41,7 +41,7 @@ @attr.s -class AssetReader(MultiBaseReader): +class AssetsReader(MultiBaseReader): """ Asset reader for STAC items. """ diff --git a/titiler/stacapi/backend.py b/titiler/stacapi/backend.py index cb492fb..c7ad438 100644 --- a/titiler/stacapi/backend.py +++ b/titiler/stacapi/backend.py @@ -23,7 +23,7 @@ from rio_tiler.types import BBox from urllib3 import Retry -from titiler.stacapi.asset_reader import AssetReader +from titiler.stacapi.assets_reader import AssetsReader from titiler.stacapi.settings import CacheSettings, RetrySettings, STACSettings from titiler.stacapi.utils import Timer @@ -46,7 +46,7 @@ class STACAPIBackend(BaseBackend): maxzoom: int = attr.ib() # Use custom asset reader (outside init) - reader: Type[AssetReader] = attr.ib(init=False, default=AssetReader) + reader: Type[AssetsReader] = attr.ib(init=False, default=AssetsReader) reader_options: Dict = attr.ib(factory=dict) # default values for bounds From ae676b69eceee6546843340552c57efc7e8574ce Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 5 Jun 2024 08:36:47 -0700 Subject: [PATCH 22/23] rename tests file --- tests/{test_asset_reader.py => test_assets_reader.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_asset_reader.py => test_assets_reader.py} (100%) diff --git a/tests/test_asset_reader.py b/tests/test_assets_reader.py similarity index 100% rename from tests/test_asset_reader.py rename to tests/test_assets_reader.py From af38c92c6a1e3def353cbbaffa6247132c9e5379 Mon Sep 17 00:00:00 2001 From: Aimee Barciauskas Date: Wed, 5 Jun 2024 08:38:38 -0700 Subject: [PATCH 23/23] indexes should be optional --- titiler/stacapi/assets_reader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/titiler/stacapi/assets_reader.py b/titiler/stacapi/assets_reader.py index 5f50f42..b41037b 100644 --- a/titiler/stacapi/assets_reader.py +++ b/titiler/stacapi/assets_reader.py @@ -199,7 +199,9 @@ def _reader(asset: str, *args: Any, **kwargs: Any) -> ImageData: with reader( asset_info["url"], tms=self.tms, **self.reader_options ) as src: - data = src.tile(*args, indexes=idx, **kwargs) + if idx is not None: + kwargs.update({"indexes": idx}) + data = src.tile(*args, **kwargs) if asset_as_band: if len(data.band_names) > 1: