diff --git a/docs/conf.py b/docs/conf.py index c35f7bb..47e060b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,9 +10,10 @@ # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. # +from __future__ import annotations + import os import sys -from typing import List sys.path.insert(0, os.path.abspath("..")) @@ -47,7 +48,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. -exclude_patterns: List[str] = [] +exclude_patterns: list[str] = [] # -- Options for HTML output ------------------------------------------------- diff --git a/h5grove/content.py b/h5grove/content.py index b193855..eaa9e4d 100644 --- a/h5grove/content.py +++ b/h5grove/content.py @@ -1,6 +1,14 @@ +from __future__ import annotations +from collections.abc import Callable, Sequence +from typing import ( + Any, + Generic, + TypeVar, + cast, +) + import contextlib from pathlib import Path -from typing import Any, Callable, Dict, Generic, Optional, Sequence, TypeVar, Union import h5py import numpy as np @@ -9,7 +17,19 @@ except ImportError: pass -from .models import LinkResolution, Selection +from .models import ( + LinkResolution, + Selection, + EntityMetadata, + ExternalLinkMetadata, + SoftLinkMetadata, + AttributeMetadata, + ResolvedEntityMetadata, + GroupMetadata, + DatasetMetadata, + DatatypeMetadata, + Stats, +) from .utils import ( NotFoundError, QueryArgumentError, @@ -35,21 +55,18 @@ class EntityContent: def __init__(self, path: str): self._path = path - def metadata(self) -> Dict[str, str]: - """Entity metadata - - :returns: {"name": str, "kind": str} - """ + def metadata(self) -> EntityMetadata: + """Entity metadata""" return {"name": self.name, "kind": self.kind} @property def name(self) -> str: - """Entity name. Last member of the path.""" + """Entity name (last path segment)""" return self._path.split("/")[-1] @property def path(self) -> str: - """Path in the file.""" + """Path in the file""" return self._path @@ -61,11 +78,8 @@ def __init__(self, path: str, link: h5py.ExternalLink): self._target_file = link.filename self._target_path = link.path - def metadata(self, depth=None): - """External link metadata - - :returns: {"name": str, "target_file": str, "target_path": str, "kind": str} - """ + def metadata(self, depth=None) -> ExternalLinkMetadata: + """External link metadata""" return sorted_dict( ("target_file", self._target_file), ("target_path", self._target_path), @@ -89,12 +103,10 @@ class SoftLinkContent(EntityContent): def __init__(self, path: str, link: h5py.SoftLink) -> None: super().__init__(path) self._target_path = link.path - """ The target path of the link """ + """The target path of the link""" - def metadata(self, depth=None): - """ - :returns: {"name": str, "target_path": str, "kind": str} - """ + def metadata(self, depth=None) -> SoftLinkMetadata: + """Soft link metadata""" return sorted_dict( ("target_path", self._target_path), *super().metadata().items() ) @@ -114,19 +126,19 @@ class ResolvedEntityContent(EntityContent, Generic[T]): def __init__(self, path: str, h5py_entity: T): super().__init__(path) self._h5py_entity = h5py_entity - """ Resolved h5py entity """ + """Resolved h5py entity""" - def attributes(self, attr_keys: Optional[Sequence[str]] = None): + def attributes( + self, attr_keys: Sequence[str] | None = None + ) -> dict[str, AttributeMetadata]: """Attributes of the h5py entity. Can be filtered by keys.""" if attr_keys is None: return dict((*self._h5py_entity.attrs.items(),)) return dict((key, self._h5py_entity.attrs[key]) for key in attr_keys) - def metadata(self, depth=None): - """ - :returns: {"attributes": AttributeMetadata, "name": str, "kind": str} - """ + def metadata(self, depth=None) -> ResolvedEntityMetadata: + """Resolved entity metadata""" attribute_names = sorted(self._h5py_entity.attrs.keys()) return sorted_dict( ( @@ -143,10 +155,8 @@ def metadata(self, depth=None): class DatasetContent(ResolvedEntityContent[h5py.Dataset]): kind = "dataset" - def metadata(self, depth=None): - """ - :returns: {"attributes": AttributeMetadata, chunks": tuple, "filters": tuple, "kind": str, "name": str, "shape": tuple, "type": TypeMetadata} - """ + def metadata(self, depth=None) -> DatasetMetadata: + """Dataset metadata""" return sorted_dict( ("chunks", self._h5py_entity.chunks), ("filters", get_filters(self._h5py_entity)), @@ -157,9 +167,9 @@ def metadata(self, depth=None): def data( self, - selection: Selection = None, + selection: Selection | None = None, flatten: bool = False, - dtype: Optional[str] = "origin", + dtype: str | None = "origin", ): """Dataset data. @@ -177,13 +187,10 @@ def data( return result - def data_stats( - self, selection: Selection = None - ) -> Dict[str, Union[float, int, None]]: + def data_stats(self, selection: Selection | None = None) -> Stats: """Statistics on the data. Providing a selection will compute stats only on the selected slice. :param selection: NumPy-like indexing to define a selection as a slice - :returns: {"strict_positive_min": number | None, "positive_min": number | None, "min": number | None, "max": number | None, "mean": number | None, "std": number | None} """ data = self._get_finite_data(selection) @@ -208,7 +215,7 @@ class GroupContent(ResolvedEntityContent[h5py.Group]): def __init__(self, path: str, h5py_entity: h5py.Group, h5file: h5py.File): super().__init__(path, h5py_entity) self._h5file = h5file - """ File in which the entity was resolved. This is needed to resolve child entity. """ + """File in which the entity was resolved. This is needed to resolve child entity.""" def _get_child_metadata_content(self, depth=0): return [ @@ -218,14 +225,13 @@ def _get_child_metadata_content(self, depth=0): for child_path in self._h5py_entity.keys() ] - def metadata(self, depth: int = 1): + def metadata(self, depth: int = 1) -> GroupMetadata: """Metadata of the group. Recursively includes child metadata if depth > 0. :parameter depth: The level of child metadata resolution. - :returns: {"attributes": AttributeMetadata, "children": ChildMetadata, "name": str, "kind": str} """ if depth <= 0: - return super().metadata() + return cast(GroupMetadata, super().metadata()) return sorted_dict( ("children", self._get_child_metadata_content(depth - 1)), @@ -236,10 +242,8 @@ def metadata(self, depth: int = 1): class DatatypeContent(ResolvedEntityContent[h5py.Datatype]): kind = "datatype" - def metadata(self, depth=None): - """ - :returns: {"attributes": AttributeMetadata, "kind": str, "name": str, "type": TypeMetadata} - """ + def metadata(self, depth=None) -> DatatypeMetadata: + """Datatype metadata""" return sorted_dict( ("type", get_type_metadata(self._h5py_entity.id)), *super().metadata().items(), @@ -248,7 +252,7 @@ def metadata(self, depth=None): def create_content( h5file: h5py.File, - path: Optional[str], + path: str | None, resolve_links: LinkResolution = LinkResolution.ONLY_VALID, ): """ @@ -287,11 +291,11 @@ def create_content( @contextlib.contextmanager def get_content_from_file( - filepath: Union[str, Path], - path: Optional[str], + filepath: str | Path, + path: str | None, create_error: Callable[[int, str], Exception], - resolve_links_arg: Optional[str] = LinkResolution.ONLY_VALID, - h5py_options: Dict[str, Any] = {}, + resolve_links_arg: str | None = LinkResolution.ONLY_VALID, + h5py_options: dict[str, Any] = {}, ): f = open_file_with_error_fallback(filepath, create_error, h5py_options) @@ -316,11 +320,11 @@ def get_content_from_file( @contextlib.contextmanager def get_list_of_paths( - filepath: Union[str, Path], - base_path: Optional[str], + filepath: str | Path, + base_path: str | None, create_error: Callable[[int, str], Exception], - resolve_links_arg: Optional[str] = LinkResolution.ONLY_VALID, - h5py_options: Dict[str, Any] = {}, + resolve_links_arg: str | None = LinkResolution.ONLY_VALID, + h5py_options: dict[str, Any] = {}, ): f = open_file_with_error_fallback(filepath, create_error, h5py_options) diff --git a/h5grove/encoders.py b/h5grove/encoders.py index 12a1b44..7bb2455 100644 --- a/h5grove/encoders.py +++ b/h5grove/encoders.py @@ -1,5 +1,8 @@ +from __future__ import annotations +from collections.abc import Callable +from typing import Any + import io -from typing import Any, Callable, Dict, Optional, Union import numpy as np import orjson import h5py @@ -17,7 +20,7 @@ def bin_encode(array: np.ndarray) -> bytes: return array.tobytes() -def orjson_default(o: Any) -> Union[list, float, str, None]: +def orjson_default(o: Any) -> list | float | str | None: """Converts Python objects to JSON-serializable objects. :raises TypeError: if the object is not supported.""" @@ -37,7 +40,7 @@ def orjson_default(o: Any) -> Union[list, float, str, None]: raise TypeError -def orjson_encode(content: Any, default: Optional[Callable] = None) -> bytes: +def orjson_encode(content: Any, default: Callable | None = None) -> bytes: """Encode in JSON using orjson. :param: content: Content to encode @@ -82,15 +85,15 @@ def tiff_encode(data: np.ndarray) -> bytes: class Response: content: bytes """ Encoded `content` as bytes """ - headers: Dict[str, str] + headers: dict[str, str] """ Associated headers """ - def __init__(self, content: bytes, headers: Dict[str, str]): + def __init__(self, content: bytes, headers: dict[str, str]): self.content = content self.headers = {**headers, "Content-Length": str(len(content))} -def encode(content: Any, encoding: Optional[str] = "json") -> Response: +def encode(content: Any, encoding: str | None = "json") -> Response: """Encode content in given encoding. Warning: Not all encodings supports all types of content. diff --git a/h5grove/fastapi_utils.py b/h5grove/fastapi_utils.py index 6271caf..27d0da1 100644 --- a/h5grove/fastapi_utils.py +++ b/h5grove/fastapi_utils.py @@ -1,9 +1,11 @@ """Helpers for usage with `FastAPI `_""" +from __future__ import annotations +from collections.abc import Callable + from fastapi import APIRouter, Depends, Response, Query, Request from fastapi.routing import APIRoute from pydantic_settings import BaseSettings -from typing import List, Optional, Union, Callable from .content import ( DatasetContent, @@ -46,7 +48,7 @@ async def custom_route_handler(request: Request) -> Response: class Settings(BaseSettings): - base_dir: Union[str, None] = None + base_dir: str | None = None settings = Settings() @@ -86,7 +88,7 @@ async def get_root(): async def get_attr( file: str = Depends(add_base_path), path: str = "/", - attr_keys: Optional[List[str]] = Query(default=None), + attr_keys: list[str] | None = Query(default=None), ): """`/attr/` endpoint handler""" with get_content_from_file(file, path, create_error) as content: diff --git a/h5grove/flask_utils.py b/h5grove/flask_utils.py index 68d3ece..dbfe0f2 100644 --- a/h5grove/flask_utils.py +++ b/h5grove/flask_utils.py @@ -1,10 +1,12 @@ """Helpers for usage with `Flask `_""" +from __future__ import annotations +from collections.abc import Callable, Mapping +from typing import Any + from werkzeug.exceptions import HTTPException from flask import Blueprint, current_app, request, Response, Request import os -from typing import Any, Callable, Mapping, Optional - from .content import ( DatasetContent, @@ -29,7 +31,7 @@ def make_encoded_response( - content, format_arg: Optional[str] = "json", status: Optional[int] = None + content, format_arg: str | None = "json", status: int | None = None ) -> Response: """Prepare flask Response according to format""" h5grove_response = encode(content, format_arg) diff --git a/h5grove/models.py b/h5grove/models.py index 1584509..043a4ce 100644 --- a/h5grove/models.py +++ b/h5grove/models.py @@ -1,6 +1,7 @@ +from __future__ import annotations from enum import Enum -from typing import Dict, Tuple, Union -from typing_extensions import TypedDict +from typing import Union, Tuple, Dict, List +from typing_extensions import TypedDict, NotRequired import h5py H5pyEntity = Union[ @@ -22,6 +23,7 @@ class LinkResolution(str, Enum): StrDtype = Union[str, Dict[str, "StrDtype"]] # type: ignore # https://api.h5py.org/h5t.html +# Must use functional `TypedDict` syntax because of `class` key TypeMetadata = TypedDict( "TypeMetadata", { @@ -40,3 +42,51 @@ class LinkResolution(str, Enum): }, total=False, ) + + +class EntityMetadata(TypedDict): + name: str + kind: str + + +class ExternalLinkMetadata(EntityMetadata): + target_file: str + target_path: str + + +class SoftLinkMetadata(EntityMetadata): + target_path: str + + +class AttributeMetadata(TypedDict): + name: str + shape: tuple + type: TypeMetadata + + +class ResolvedEntityMetadata(EntityMetadata): + attributes: List[AttributeMetadata] + + +class GroupMetadata(ResolvedEntityMetadata): + children: NotRequired[List[EntityMetadata]] + + +class DatasetMetadata(ResolvedEntityMetadata): + chunks: tuple + filters: tuple + shape: tuple + type: TypeMetadata + + +class DatatypeMetadata(ResolvedEntityMetadata): + type: TypeMetadata + + +class Stats(TypedDict): + strict_positive_min: Union[int, float, None] + positive_min: Union[int, float, None] + min: Union[int, float, None] + max: Union[int, float, None] + mean: Union[int, float, None] + std: Union[int, float, None] diff --git a/h5grove/tornado_utils.py b/h5grove/tornado_utils.py index 7d026bc..4256e8c 100644 --- a/h5grove/tornado_utils.py +++ b/h5grove/tornado_utils.py @@ -1,8 +1,9 @@ """Helpers for usage with `Tornado `_""" -import os -from typing import Any, Optional +from __future__ import annotations +from typing import Any +import os from tornado.web import HTTPError, MissingArgumentError, RequestHandler from .content import ( @@ -33,7 +34,7 @@ def create_error(status_code: int, message: str): class BaseHandler(RequestHandler): """Base class for h5grove handlers""" - def initialize(self, base_dir: str, allow_origin: Optional[str] = None) -> None: + def initialize(self, base_dir: str, allow_origin: str | None = None) -> None: self.base_dir = base_dir self.allow_origin = allow_origin @@ -57,7 +58,7 @@ def get(self): self.finish() def get_response( - self, full_file_path: str, path: Optional[str], resolve_links: Optional[str] + self, full_file_path: str, path: str | None, resolve_links: str | None ) -> Response: raise NotImplementedError @@ -82,7 +83,7 @@ def head(self): class ContentHandler(BaseHandler): def get_response( - self, full_file_path: str, path: Optional[str], resolve_links: Optional[str] + self, full_file_path: str, path: str | None, resolve_links: str | None ) -> Response: with get_content_from_file( full_file_path, path, create_error, resolve_links @@ -141,7 +142,7 @@ def get_content_response(self, content: EntityContent) -> Response: class PathsHandler(BaseHandler): def get_response( - self, full_file_path: str, path: Optional[str], resolve_links: Optional[str] + self, full_file_path: str, path: str | None, resolve_links: str | None ) -> Response: with get_list_of_paths( full_file_path, path, create_error, resolve_links @@ -150,7 +151,7 @@ def get_response( # TODO: Setting the return type raises mypy errors -def get_handlers(base_dir: Optional[str], allow_origin: Optional[str] = None): +def get_handlers(base_dir: str | None, allow_origin: str | None = None): """Build h5grove handlers (`/`, `/attr/`, `/data/`, `/meta/` and `/stats/`). :param base_dir: Base directory from which the HDF5 files will be served diff --git a/h5grove/utils.py b/h5grove/utils.py index cc08e8d..e3094e9 100644 --- a/h5grove/utils.py +++ b/h5grove/utils.py @@ -1,11 +1,22 @@ +from __future__ import annotations +from collections.abc import Callable +from typing import Any, TypeVar + from pathlib import Path import h5py from h5py.version import version_tuple as h5py_version from os.path import basename import numpy as np -from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, Union -from .models import H5pyEntity, LinkResolution, Selection, StrDtype, TypeMetadata +from .models import ( + H5pyEntity, + LinkResolution, + Selection, + StrDtype, + TypeMetadata, + Stats, + AttributeMetadata, +) class NotFoundError(Exception): @@ -39,7 +50,9 @@ def _legacy_get_attr_id(entity_attrs: h5py.AttributeManager, attr_name: str): ) -def attr_metadata(entity_attrs: h5py.AttributeManager, attr_name: str) -> dict: +def attr_metadata( + entity_attrs: h5py.AttributeManager, attr_name: str +) -> AttributeMetadata: attrId = get_attr_id(entity_attrs, attr_name) return { @@ -79,7 +92,7 @@ def get_entity_from_file( return h5file[path] -def parse_slice(slice_str: str) -> Tuple[Union[slice, int], ...]: +def parse_slice(slice_str: str) -> tuple[slice | int, ...]: """ Parses a string containing a slice under NumPy format. @@ -98,7 +111,7 @@ def parse_slice(slice_str: str) -> Tuple[Union[slice, int], ...]: return tuple(parse_slice_member(s) for s in slice_members) -def parse_slice_member(slice_member: str) -> Union[slice, int]: +def parse_slice_member(slice_member: str) -> slice | int: if ":" not in slice_member: return int(slice_member) @@ -122,7 +135,7 @@ def parse_slice_member(slice_member: str) -> Union[slice, int]: raise TypeError(f"{slice_member} is not a valid slice") -def sorted_dict(*args: Tuple[str, Any]): +def sorted_dict(*args: tuple[str, Any]): return dict(sorted(args, key=lambda entry: entry[0])) @@ -219,7 +232,7 @@ def _sanitize_dtype(dtype: np.dtype) -> np.dtype: T = TypeVar("T", np.ndarray, np.number, np.bool_) -def convert(data: T, dtype: Optional[str] = "origin") -> T: +def convert(data: T, dtype: str | None = "origin") -> T: """Convert array or numpy scalar to given dtype query param :param data: nD array or scalar to convert @@ -241,14 +254,14 @@ def convert(data: T, dtype: Optional[str] = "origin") -> T: raise QueryArgumentError(f"Unsupported dtype {dtype}") -def is_numeric_data(data: Union[np.ndarray, np.number, np.bool_, bytes]) -> bool: +def is_numeric_data(data: np.ndarray | np.number | np.bool_ | bytes) -> bool: if not isinstance(data, (np.ndarray, np.number, np.bool_)): return False return np.issubdtype(data.dtype, np.number) or np.issubdtype(data.dtype, np.bool_) -def get_array_stats(data: np.ndarray) -> Dict[str, Union[float, int, None]]: +def get_array_stats(data: np.ndarray) -> Stats: if data.size == 0: return { "strict_positive_min": None, @@ -278,14 +291,14 @@ def get_array_stats(data: np.ndarray) -> Dict[str, Union[float, int, None]]: } -def hdf_path_join(prefix: Union[str, None], suffix: str): +def hdf_path_join(prefix: str | None, suffix: str): if prefix is None or prefix == "/": return f"/{suffix}" return f'{prefix.rstrip("/")}/{suffix}' -def parse_bool_arg(query_arg: Union[str, None], fallback: bool) -> bool: +def parse_bool_arg(query_arg: str | None, fallback: bool) -> bool: if query_arg is None: return fallback @@ -293,7 +306,7 @@ def parse_bool_arg(query_arg: Union[str, None], fallback: bool) -> bool: def parse_link_resolution_arg( - raw_query_arg: Union[str, None], fallback: LinkResolution + raw_query_arg: str | None, fallback: LinkResolution ) -> LinkResolution: if raw_query_arg is None: return fallback @@ -332,7 +345,7 @@ def get_dataset_slice(dataset: h5py.Dataset, selection: Selection): def get_filters( dataset: h5py.Dataset, -) -> Optional[List[Dict[str, Union[int, str]]]]: +) -> list[dict[str, int | str]] | None: property_list = dataset.id.get_create_plist() n_filters = property_list.get_nfilters() @@ -343,8 +356,8 @@ def get_filters( def get_filter_info( - filter: Tuple[int, int, Tuple[int, ...], str] -) -> Dict[str, Union[int, str]]: + filter: tuple[int, int, tuple[int, ...], str] +) -> dict[str, int | str]: # https://api.h5py.org/h5p.html#h5py.h5p.PropDCID.get_filter (filter_id, _, _, name) = filter @@ -361,9 +374,9 @@ def stringify_dtype(dtype: np.dtype) -> StrDtype: def open_file_with_error_fallback( - filepath: Union[str, Path], + filepath: str | Path, create_error: Callable[[int, str], Exception], - h5py_options: Dict[str, Any] = {}, + h5py_options: dict[str, Any] = {}, ) -> h5py.File: try: f = h5py.File(filepath, "r", **h5py_options) diff --git a/setup.cfg b/setup.cfg index 14e7687..f2926d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,6 +41,7 @@ dev = black bump2version check-manifest + eval_type_backport flake8 h5grove[fastapi] h5grove[flask] diff --git a/test/base_test.py b/test/base_test.py index df6d60d..24f2225 100644 --- a/test/base_test.py +++ b/test/base_test.py @@ -1,8 +1,10 @@ """Base class for testing with different servers""" +from __future__ import annotations +from collections.abc import Generator + import os import stat -from typing import Generator from urllib.parse import urlencode import h5py diff --git a/test/conftest.py b/test/conftest.py index ce6c6dd..e031abb 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,12 @@ +from __future__ import annotations +from collections.abc import Callable + import os import pathlib import socketserver import subprocess import sys import time -from typing import Callable, Optional from urllib.request import urlopen from urllib.error import HTTPError @@ -31,7 +33,7 @@ def _get_response(self, url: str, benchmark: Callable) -> Response: def get( self, url: str, - benchmark: Optional[Callable] = None, + benchmark: Callable | None = None, ) -> Response: """Request url and return retrieved response""" if benchmark is None: diff --git a/test/test_benchmark_data.py b/test/test_benchmark_data.py index ae75af1..213cd26 100644 --- a/test/test_benchmark_data.py +++ b/test/test_benchmark_data.py @@ -1,7 +1,9 @@ """Benchmark data requests with server apps in example/ folder""" +from __future__ import annotations +from collections.abc import Generator + import pathlib -from typing import Generator from urllib.parse import urlencode import h5py import numpy as np diff --git a/test/test_fastapi.py b/test/test_fastapi.py index 4fb7e46..71b7013 100644 --- a/test/test_fastapi.py +++ b/test/test_fastapi.py @@ -1,7 +1,9 @@ """Test fastapi_utils with fastapi testing""" +from __future__ import annotations +from collections.abc import Callable + import pathlib -from typing import Callable from fastapi import FastAPI from fastapi.testclient import TestClient import pytest diff --git a/test/test_flask.py b/test/test_flask.py index 75b4500..ddec322 100644 --- a/test/test_flask.py +++ b/test/test_flask.py @@ -1,7 +1,9 @@ """Test flask_utils blueprint with Flask testing""" +from __future__ import annotations +from collections.abc import Callable + import pathlib -from typing import Callable from flask import Flask import pytest diff --git a/test/test_tornado.py b/test/test_tornado.py index 628a2a8..7de8fc5 100644 --- a/test/test_tornado.py +++ b/test/test_tornado.py @@ -1,7 +1,9 @@ """Test tornado_utils using pytest-tornado""" +from __future__ import annotations +from collections.abc import Callable + import pathlib -from typing import Callable import pytest from tornado.httpclient import HTTPClientError import tornado.web diff --git a/test/utils.py b/test/utils.py index fcedcf2..124eb4d 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1,7 +1,9 @@ +from __future__ import annotations +from typing import NamedTuple + import io import json import numpy as np -from typing import List, NamedTuple, Tuple import tifffile @@ -12,7 +14,7 @@ class Response(NamedTuple): """Return type of :meth:`get`""" status: int - headers: List[Tuple[str, str]] + headers: list[tuple[str, str]] content: bytes def find_header_value(self, key: str): @@ -55,7 +57,7 @@ def decode_array_response( response: Response, format: str, dtype: str, - shape: Tuple[int, ...], + shape: tuple[int, ...], ) -> np.ndarray: """Decode data array response content according to given information""" content_type = response.find_header_value("content-type")