From 7ed16cb9e8fbd662e3a6689e0e2173b7aa0b62d2 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 5 Jul 2024 11:11:02 +0200 Subject: [PATCH 01/20] remove unused RelativePath --- bioimageio/spec/_internal/io.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index f46c189cf..e03952b8b 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -171,24 +171,6 @@ def _validate(cls, value: Union[PurePath, str]): return cls(PurePath(value)) -class RelativePath( - RelativePathBase[Union[AbsoluteFilePath, AbsoluteDirectory, HttpUrl]], frozen=True -): - def get_absolute( - self, root: "RootHttpUrl | Path | AnyUrl" - ) -> "AbsoluteFilePath | AbsoluteDirectory | HttpUrl": - absolute = self._get_absolute_impl(root) - if ( - isinstance(absolute, Path) - and (context := validation_context_var.get()).perform_io_checks - and str(self.root) not in context.known_files - and not absolute.exists() - ): - raise ValueError(f"{absolute} does not exist") - - return absolute - - class RelativeFilePath(RelativePathBase[Union[AbsoluteFilePath, HttpUrl]], frozen=True): def model_post_init(self, __context: Any) -> None: if not self.root.parts: # an empty path can only be a directory From ff51689d3134e2a05a25ec282a4c0a12a0e02534 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Fri, 5 Jul 2024 13:58:35 +0200 Subject: [PATCH 02/20] add docstring to RelativeFilePath --- bioimageio/spec/_internal/io.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index e03952b8b..0702c6648 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -172,6 +172,8 @@ def _validate(cls, value: Union[PurePath, str]): class RelativeFilePath(RelativePathBase[Union[AbsoluteFilePath, HttpUrl]], frozen=True): + """A path relative to the `rdf.yaml` file (also if the RDF source is a URL).""" + def model_post_init(self, __context: Any) -> None: if not self.root.parts: # an empty path can only be a directory raise ValueError(f"{self.root} is not a valid file path.") From 214285f18e8039f3a884924aecf3f1e4c4cd8ac0 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 17 Jul 2024 11:38:16 +0200 Subject: [PATCH 03/20] add docstring --- bioimageio/spec/_description.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bioimageio/spec/_description.py b/bioimageio/spec/_description.py index a4a43ad21..3c8fa2eb1 100644 --- a/bioimageio/spec/_description.py +++ b/bioimageio/spec/_description.py @@ -131,6 +131,22 @@ def build_description( context: Optional[ValidationContext] = None, format_version: Union[FormatVersionPlaceholder, str] = DISCOVER, ) -> Union[ResourceDescr, InvalidDescr]: + """build a bioimage.io resource description from an RDF's content. + + Use `load_description` if you want to build a resource description from an rdf.yaml + or bioimage.io zip-package. + + Args: + content: loaded rdf.yaml file (loaded with YAML, not bioimageio.spec) + context: validation context to use during validation + format_version: (optional) use this argument to load the resource and + convert its metadata to a higher format_version + + Returns: + An object holding all metadata of the bioimage.io resource + + """ + return build_description_impl( content, context=context, From 0b2b0a23ffad462d5b5229ed9f002fd30dc78ea1 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Wed, 17 Jul 2024 11:39:13 +0200 Subject: [PATCH 04/20] expose perform_io_checks --- bioimageio/spec/_io.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/bioimageio/spec/_io.py b/bioimageio/spec/_io.py index cb8b1d78c..811536fc6 100644 --- a/bioimageio/spec/_io.py +++ b/bioimageio/spec/_io.py @@ -10,6 +10,7 @@ build_description, dump_description, ) +from ._internal._settings import settings from ._internal.common_nodes import ResourceDescrBase from ._internal.io import BioimageioYamlContent, YamlValue from ._internal.io_utils import open_bioimageio_yaml, write_yaml @@ -23,7 +24,23 @@ def load_description( /, *, format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER, + perform_io_checks: bool = settings.perform_io_checks, ) -> Union[ResourceDescr, InvalidDescr]: + """load a bioimage.io resource description + + Args: + source: Path or URL to an rdf.yaml or a bioimage.io package + (zip-file with rdf.yaml in it) + format_version: (optional) use this argument to load the resource and + convert its metadata to a higher format_version + perform_io_checks: wether or not to perform validation that requires file io, + e.g. downloading a remote files. The existence of local + absolute file paths is still being checked. + + Returns: + An object holding all metadata of the bioimage.io resource + + """ if isinstance(source, ResourceDescrBase): name = getattr(source, "name", f"{str(source)[:10]}...") logger.warning("returning already loaded description '{}' as is", name) @@ -34,7 +51,9 @@ def load_description( return build_description( opened.content, context=ValidationContext( - root=opened.original_root, file_name=opened.original_file_name + root=opened.original_root, + file_name=opened.original_file_name, + perform_io_checks=perform_io_checks, ), format_version=format_version, ) From a999f727718504743703ececa4acbc48062ec3fd Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 10:36:38 +0200 Subject: [PATCH 05/20] rename _build_description -> _description_impl --- bioimageio/spec/_description.py | 2 +- bioimageio/spec/{_build_description.py => _description_impl.py} | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) rename bioimageio/spec/{_build_description.py => _description_impl.py} (98%) diff --git a/bioimageio/spec/_description.py b/bioimageio/spec/_description.py index 3c8fa2eb1..15fc3eaf8 100644 --- a/bioimageio/spec/_description.py +++ b/bioimageio/spec/_description.py @@ -5,7 +5,7 @@ from pydantic import Discriminator from typing_extensions import Annotated -from ._build_description import DISCOVER, build_description_impl, get_rd_class_impl +from ._description_impl import DISCOVER, build_description_impl, get_rd_class_impl from ._internal.common_nodes import InvalidDescr from ._internal.io import BioimageioYamlContent, BioimageioYamlSource from ._internal.types import FormatVersionPlaceholder diff --git a/bioimageio/spec/_build_description.py b/bioimageio/spec/_description_impl.py similarity index 98% rename from bioimageio/spec/_build_description.py rename to bioimageio/spec/_description_impl.py index 31c4dcb00..0916dae2e 100644 --- a/bioimageio/spec/_build_description.py +++ b/bioimageio/spec/_description_impl.py @@ -1,3 +1,5 @@ +"""implementation details for building a bioimage.io resource description""" + from typing import Any, Callable, List, Mapping, Optional, Type, TypeVar, Union from ._internal.common_nodes import InvalidDescr, ResourceDescrBase From d53366212efa2b24012ee6a3afebfa0f22ad9f4e Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 10:49:13 +0200 Subject: [PATCH 06/20] improve/add docstrings and add args --- bioimageio/spec/_io.py | 62 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 12 deletions(-) diff --git a/bioimageio/spec/_io.py b/bioimageio/spec/_io.py index 811536fc6..b3243ff93 100644 --- a/bioimageio/spec/_io.py +++ b/bioimageio/spec/_io.py @@ -1,8 +1,10 @@ -from typing import Literal, TextIO, Union, cast +from typing import Dict, Literal, Optional, TextIO, Union, cast from loguru import logger from pydantic import FilePath, NewPath +from bioimageio.spec._internal.io_basics import Sha256 + from ._description import ( DISCOVER, InvalidDescr, @@ -14,7 +16,7 @@ from ._internal.common_nodes import ResourceDescrBase from ._internal.io import BioimageioYamlContent, YamlValue from ._internal.io_utils import open_bioimageio_yaml, write_yaml -from ._internal.validation_context import ValidationContext +from ._internal.validation_context import validation_context_var from .common import PermissiveFileSource from .summary import ValidationSummary @@ -25,17 +27,20 @@ def load_description( *, format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER, perform_io_checks: bool = settings.perform_io_checks, + known_files: Optional[Dict[str, Sha256]] = None, ) -> Union[ResourceDescr, InvalidDescr]: """load a bioimage.io resource description Args: source: Path or URL to an rdf.yaml or a bioimage.io package - (zip-file with rdf.yaml in it) - format_version: (optional) use this argument to load the resource and - convert its metadata to a higher format_version - perform_io_checks: wether or not to perform validation that requires file io, + (zip-file with rdf.yaml in it). + format_version: (optional) Use this argument to load the resource and + convert its metadata to a higher format_version. + perform_io_checks: Wether or not to perform validation that requires file io, e.g. downloading a remote files. The existence of local absolute file paths is still being checked. + known_files: Allows to bypass download and hashing of referenced files + (even if perform_io_checks is True). Returns: An object holding all metadata of the bioimage.io resource @@ -48,13 +53,16 @@ def load_description( opened = open_bioimageio_yaml(source) + context = validation_context_var.get().replace( + root=opened.original_root, + file_name=opened.original_file_name, + perform_io_checks=perform_io_checks, + known_files=known_files, + ) + return build_description( opened.content, - context=ValidationContext( - root=opened.original_root, - file_name=opened.original_file_name, - perform_io_checks=perform_io_checks, - ), + context=context, format_version=format_version, ) @@ -64,6 +72,12 @@ def save_bioimageio_yaml_only( /, file: Union[NewPath, FilePath, TextIO], ): + """write the metadata of a resource description (`rd`) to `file` + without writing any of the referenced files in it. + + Note: To save a resource description with its associated files as a package, + use `save_bioimageio_package` or `save_bioimageio_package_as_folder`. + """ if isinstance(rd, ResourceDescrBase): content = dump_description(rd) else: @@ -77,7 +91,31 @@ def load_description_and_validate_format_only( /, *, format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER, + perform_io_checks: bool = settings.perform_io_checks, + known_files: Optional[Dict[str, Sha256]] = None, ) -> ValidationSummary: - rd = load_description(source, format_version=format_version) + """load a bioimage.io resource description + + Args: + source: Path or URL to an rdf.yaml or a bioimage.io package + (zip-file with rdf.yaml in it). + format_version: (optional) Use this argument to load the resource and + convert its metadata to a higher format_version. + perform_io_checks: Wether or not to perform validation that requires file io, + e.g. downloading a remote files. The existence of local + absolute file paths is still being checked. + known_files: Allows to bypass download and hashing of referenced files + (even if perform_io_checks is True). + + Returns: + Validation summary of the bioimage.io resource found at `source`. + + """ + rd = load_description( + source, + format_version=format_version, + perform_io_checks=perform_io_checks, + known_files=known_files, + ) assert rd.validation_summary is not None return rd.validation_summary From 93f8033636ebaa2f2cf6045c56149dad1f436e3d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 10:54:36 +0200 Subject: [PATCH 07/20] update changelog and bump post --- README.md | 4 ++++ bioimageio/spec/VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index cbda0eafa..9aaa5dac4 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,10 @@ Made with [contrib.rocks](https://contrib.rocks). ### bioimageio.spec Python package +#### bioimageio.spec 0.5.3post5 + +* expose `perform_io_checks` and `known_files` from `ValidationContext` to `load_description` and `load_description_and_validate_format_only` + #### bioimageio.spec 0.5.3post4 * fix pinning of pydantic diff --git a/bioimageio/spec/VERSION b/bioimageio/spec/VERSION index eecd35939..3e6a88c5f 100644 --- a/bioimageio/spec/VERSION +++ b/bioimageio/spec/VERSION @@ -1,3 +1,3 @@ { - "version": "0.5.3post4" + "version": "0.5.3post5" } From 1d011d54167c1a3d02cf5584e2f0604dc8a7dc59 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 11:21:26 +0200 Subject: [PATCH 08/20] add load_model_description and load_dataset_description for easier use of stricter typing --- bioimageio/spec/_description.py | 48 +++++++++++++++++++++++++++++++++ bioimageio/spec/_io.py | 44 ++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/bioimageio/spec/_description.py b/bioimageio/spec/_description.py index 15fc3eaf8..b4757c8ae 100644 --- a/bioimageio/spec/_description.py +++ b/bioimageio/spec/_description.py @@ -178,3 +178,51 @@ def update_format( ) -> BioimageioYamlContent: """update a bioimageio.yaml file without validating it""" raise NotImplementedError("Oh no! This feature is not yet implemented") + + +def ensure_description_is_model( + rd: Union[InvalidDescr, ResourceDescr], +) -> AnyModelDescr: + if isinstance(rd, InvalidDescr): + rd.validation_summary.display() + raise ValueError("resource description is invalid") + + if rd.type != "model": + rd.validation_summary.display() + raise ValueError( + f"expected a model resource, but got resource type '{rd.type}'" + ) + + assert not isinstance( + rd, + ( + GenericDescr02, + GenericDescr03, + ), + ) + + return rd + + +def ensure_description_is_dataset( + rd: Union[InvalidDescr, ResourceDescr], +) -> AnyDatasetDescr: + if isinstance(rd, InvalidDescr): + rd.validation_summary.display() + raise ValueError("resource description is invalid") + + if rd.type != "dataset": + rd.validation_summary.display() + raise ValueError( + f"expected a dataset resource, but got resource type '{rd.type}'" + ) + + assert not isinstance( + rd, + ( + GenericDescr02, + GenericDescr03, + ), + ) + + return rd diff --git a/bioimageio/spec/_io.py b/bioimageio/spec/_io.py index b3243ff93..763ee6247 100644 --- a/bioimageio/spec/_io.py +++ b/bioimageio/spec/_io.py @@ -11,6 +11,8 @@ ResourceDescr, build_description, dump_description, + ensure_description_is_dataset, + ensure_description_is_model, ) from ._internal._settings import settings from ._internal.common_nodes import ResourceDescrBase @@ -18,6 +20,8 @@ from ._internal.io_utils import open_bioimageio_yaml, write_yaml from ._internal.validation_context import validation_context_var from .common import PermissiveFileSource +from .dataset import AnyDatasetDescr +from .model import AnyModelDescr from .summary import ValidationSummary @@ -67,6 +71,46 @@ def load_description( ) +def load_model_description( + source: PermissiveFileSource, + /, + *, + format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER, + perform_io_checks: bool = settings.perform_io_checks, + known_files: Optional[Dict[str, Sha256]] = None, +) -> AnyModelDescr: + """same as `load_description`, but addtionally ensures that the loaded + description is valid and of type 'model'. + """ + rd = load_description( + source, + format_version=format_version, + perform_io_checks=perform_io_checks, + known_files=known_files, + ) + return ensure_description_is_model(rd) + + +def load_dataset_description( + source: PermissiveFileSource, + /, + *, + format_version: Union[Literal["discover"], Literal["latest"], str] = DISCOVER, + perform_io_checks: bool = settings.perform_io_checks, + known_files: Optional[Dict[str, Sha256]] = None, +) -> AnyDatasetDescr: + """same as `load_description`, but addtionally ensures that the loaded + description is valid and of type 'dataset'. + """ + rd = load_description( + source, + format_version=format_version, + perform_io_checks=perform_io_checks, + known_files=known_files, + ) + return ensure_description_is_dataset(rd) + + def save_bioimageio_yaml_only( rd: Union[ResourceDescr, BioimageioYamlContent, InvalidDescr], /, From aea79dd9529738e29020493ca890dac99b2e76b6 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 11:24:34 +0200 Subject: [PATCH 09/20] update changelog --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 9aaa5dac4..666525d71 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ Made with [contrib.rocks](https://contrib.rocks). #### bioimageio.spec 0.5.3post5 +* add `load_model_description` and `load_dataset_description` +* add `ensure_description_is_model` and `ensure_description_is_dataset` * expose `perform_io_checks` and `known_files` from `ValidationContext` to `load_description` and `load_description_and_validate_format_only` #### bioimageio.spec 0.5.3post4 From 5690bf490b3c0d15ebdfe2833cf94cd67e89d181 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 11:54:41 +0200 Subject: [PATCH 10/20] change version scheme to avoid overbearing post releases --- README.md | 11 ++++++++++- bioimageio/spec/VERSION | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 666525d71..c71b5a432 100644 --- a/README.md +++ b/README.md @@ -101,11 +101,20 @@ TODO: link to settings in dev docs Made with [contrib.rocks](https://contrib.rocks). +## 🛈 Versioining scheme + +To keep the bioimageio.spec Python package version in sync with the (model) description format version, bioimageio.spec is versioned as MAJOR.MINRO.PATCH.LIB, where MAJOR.MINRO.PATCH correspond to the latest model description format version implemented and LIB may be bumpbed for library changes that do not affect the format version. +[This change was introduced with bioimageio.spec 0.5.3.1](#bioimageiospec-0531). + ## Δ Changelog ### bioimageio.spec Python package -#### bioimageio.spec 0.5.3post5 +#### bioimageio.spec 0.5.3.1 + +note: the versioning scheme was changed as our previous `post` releases include changes beyond what a post release should entail (only changing docstrings, etc). +This was motivated by the desire to keep the library version in sync with the (model) format version to avoid confusion. +To keep this relation, but avoid overbearing post releases a library version number is now added as the 4th part MAJOR.MINOR.PATCH.LIB_VERSION. * add `load_model_description` and `load_dataset_description` * add `ensure_description_is_model` and `ensure_description_is_dataset` diff --git a/bioimageio/spec/VERSION b/bioimageio/spec/VERSION index 3e6a88c5f..019d8d9b8 100644 --- a/bioimageio/spec/VERSION +++ b/bioimageio/spec/VERSION @@ -1,3 +1,3 @@ { - "version": "0.5.3post5" + "version": "0.5.3.1" } From ae1b410a645e5c7d4394a4433ea7c04fb25e240f Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 12:33:17 +0200 Subject: [PATCH 11/20] update type annotations for pyright 1.1.373 --- bioimageio/spec/_internal/common_nodes.py | 2 +- bioimageio/spec/model/v0_5.py | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/bioimageio/spec/_internal/common_nodes.py b/bioimageio/spec/_internal/common_nodes.py index 866cbc6f4..5dda134ba 100644 --- a/bioimageio/spec/_internal/common_nodes.py +++ b/bioimageio/spec/_internal/common_nodes.py @@ -309,7 +309,7 @@ def _ignore_future_patch(cls, data: Union[Dict[Any, Any], Any], /) -> Any: return data @model_validator(mode="after") - def _set_init_validation_summary(self): + def _set_init_validation_summary(self) -> Self: context = validation_context_var.get() self._validation_summary = ValidationSummary( name="bioimageio validation", diff --git a/bioimageio/spec/model/v0_5.py b/bioimageio/spec/model/v0_5.py index c8c080225..fe45454f9 100644 --- a/bioimageio/spec/model/v0_5.py +++ b/bioimageio/spec/model/v0_5.py @@ -228,10 +228,13 @@ class AxisId(LowerCaseIdentifier): SAME_AS_TYPE = "" +ParameterizedSize_N = int + + class ParameterizedSize(Node): """Describes a range of valid tensor axis sizes as `size = min + n*step`.""" - N: ClassVar[Type[int]] = int + N: ClassVar[Type[int]] = ParameterizedSize_N """integer to parameterize this axis""" min: Annotated[int, Gt(0)] @@ -248,10 +251,10 @@ def validate_size(self, size: int) -> int: return size - def get_size(self, n: ParameterizedSize.N) -> int: + def get_size(self, n: ParameterizedSize_N) -> int: return self.min + self.step * n - def get_n(self, s: int) -> ParameterizedSize.N: + def get_n(self, s: int) -> ParameterizedSize_N: """return smallest n parameterizing a size greater or equal than `s`""" return ceil((s - self.min) / self.step) @@ -343,7 +346,7 @@ def get_size( SpaceOutputAxis, SpaceOutputAxisWithHalo, ], - n: ParameterizedSize.N, + n: ParameterizedSize_N, ): """helper method to compute concrete size for a given axis and its reference axis. If the reference axis is parameterized, `n` is used to compute the concrete size of it, see `ParameterizedSize`. @@ -2416,7 +2419,7 @@ def get_output_tensor_sizes( return tensor_sizes.outputs def get_ns(self, input_sizes: Mapping[TensorId, Mapping[AxisId, int]]): - ret: Dict[Tuple[TensorId, AxisId], ParameterizedSize.N] = {} + ret: Dict[Tuple[TensorId, AxisId], ParameterizedSize_N] = {} axes = {t.id: {a.id: a for a in t.axes} for t in self.inputs} for tid in input_sizes: for aid, s in input_sizes[tid].items(): @@ -2431,7 +2434,7 @@ def get_ns(self, input_sizes: Mapping[TensorId, Mapping[AxisId, int]]): return ret def get_tensor_sizes( - self, ns: Mapping[Tuple[TensorId, AxisId], ParameterizedSize.N], batch_size: int + self, ns: Mapping[Tuple[TensorId, AxisId], ParameterizedSize_N], batch_size: int ) -> _TensorSizes: axis_sizes = self.get_axis_sizes(ns, batch_size=batch_size) return _TensorSizes( @@ -2454,7 +2457,7 @@ def get_tensor_sizes( ) def get_axis_sizes( - self, ns: Mapping[Tuple[TensorId, AxisId], ParameterizedSize.N], batch_size: int + self, ns: Mapping[Tuple[TensorId, AxisId], ParameterizedSize_N], batch_size: int ) -> _AxisSizes: all_axes = { t.id: {a.id: a for a in t.axes} for t in chain(self.inputs, self.outputs) From c15000dc7af5d6fc1a1c27f5ddf79b7c11e131e9 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 12:47:18 +0200 Subject: [PATCH 12/20] pin pyright version (errors with latest pyright v1.1.373 can't be reproduced locally) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bf509e789..48381b082 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ "pdoc", "pre-commit", "psutil", # parallel pytest with '-n auto' - "pyright", + "pyright==1.1.366", # TODO: update pyright "pytest-xdist", # parallel pytest "pytest", "python-devtools", From 4d59a12fcb2f3936996a6496fc39aa6785943f4d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 12:56:24 +0200 Subject: [PATCH 13/20] unpin pyright --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 48381b082..bf509e789 100644 --- a/setup.py +++ b/setup.py @@ -57,7 +57,7 @@ "pdoc", "pre-commit", "psutil", # parallel pytest with '-n auto' - "pyright==1.1.366", # TODO: update pyright + "pyright", "pytest-xdist", # parallel pytest "pytest", "python-devtools", From d5474409b714307eed2e91d9abf8c7b525282c23 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 12:58:14 +0200 Subject: [PATCH 14/20] remove cast to str --- bioimageio/spec/_internal/common_nodes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bioimageio/spec/_internal/common_nodes.py b/bioimageio/spec/_internal/common_nodes.py index 5dda134ba..87e9ab6de 100644 --- a/bioimageio/spec/_internal/common_nodes.py +++ b/bioimageio/spec/_internal/common_nodes.py @@ -127,9 +127,9 @@ def _get_data(cls, valid_string_data: str) -> Dict[str, Any]: @classmethod def _validate(cls, value: str) -> Self: - contrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)] - contrained_str_adapter = TypeAdapter(cast(str, contrained_str_type)) - valid_string_data = contrained_str_adapter.validate_python(value) + constrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)] + constrained_str_adapter = TypeAdapter(constrained_str_type) + valid_string_data = constrained_str_adapter.validate_python(value) data = cls._get_data(valid_string_data) self = cls(valid_string_data) object.__setattr__(self, "_node", self._node_class.model_validate(data)) From cac7a8eb4bc1e08049284959a38a98539af3da5d Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 12:59:29 +0200 Subject: [PATCH 15/20] add assert --- bioimageio/spec/_internal/common_nodes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bioimageio/spec/_internal/common_nodes.py b/bioimageio/spec/_internal/common_nodes.py index 87e9ab6de..7dbd6e2dd 100644 --- a/bioimageio/spec/_internal/common_nodes.py +++ b/bioimageio/spec/_internal/common_nodes.py @@ -130,6 +130,7 @@ def _validate(cls, value: str) -> Self: constrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)] constrained_str_adapter = TypeAdapter(constrained_str_type) valid_string_data = constrained_str_adapter.validate_python(value) + assert isinstance(valid_string_data, str) data = cls._get_data(valid_string_data) self = cls(valid_string_data) object.__setattr__(self, "_node", self._node_class.model_validate(data)) From 91df17fd31fcc4549da007edf05446b4b5e7d9e1 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 13:10:16 +0200 Subject: [PATCH 16/20] add explicit type for constrained_str_adapter --- bioimageio/spec/_internal/common_nodes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bioimageio/spec/_internal/common_nodes.py b/bioimageio/spec/_internal/common_nodes.py index 7dbd6e2dd..98260d4d0 100644 --- a/bioimageio/spec/_internal/common_nodes.py +++ b/bioimageio/spec/_internal/common_nodes.py @@ -128,9 +128,8 @@ def _get_data(cls, valid_string_data: str) -> Dict[str, Any]: @classmethod def _validate(cls, value: str) -> Self: constrained_str_type = Annotated[str, StringConstraints(pattern=cls._pattern)] - constrained_str_adapter = TypeAdapter(constrained_str_type) + constrained_str_adapter: TypeAdapter[str] = TypeAdapter(constrained_str_type) valid_string_data = constrained_str_adapter.validate_python(value) - assert isinstance(valid_string_data, str) data = cls._get_data(valid_string_data) self = cls(valid_string_data) object.__setattr__(self, "_node", self._node_class.model_validate(data)) From d32824e9b41927acd9ca3694afc7c2be7aab5ac5 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 14:21:07 +0200 Subject: [PATCH 17/20] add explicit type when using TypeAdapter --- bioimageio/spec/_internal/field_warning.py | 2 +- bioimageio/spec/_internal/io.py | 4 +- tests/test_generic/test_v0_3.py | 85 ++++++++++------------ tests/test_internal/test_types.py | 3 +- tests/test_internal/test_validate.py | 10 ++- tests/utils.py | 2 +- 6 files changed, 52 insertions(+), 54 deletions(-) diff --git a/bioimageio/spec/_internal/field_warning.py b/bioimageio/spec/_internal/field_warning.py index 65d995e81..5c1a53c6e 100644 --- a/bioimageio/spec/_internal/field_warning.py +++ b/bioimageio/spec/_internal/field_warning.py @@ -41,7 +41,7 @@ def warn( if isinstance(typ, get_args(AnnotationMetaData)): typ = Annotated[Any, typ] - validator = TypeAdapter(typ) + validator: TypeAdapter[Any] = TypeAdapter(typ) return AfterWarner( validator.validate_python, severity=severity, msg=msg, context={"typ": typ} diff --git a/bioimageio/spec/_internal/io.py b/bioimageio/spec/_internal/io.py index 0702c6648..6627c6ed6 100644 --- a/bioimageio/spec/_internal/io.py +++ b/bioimageio/spec/_internal/io.py @@ -492,7 +492,9 @@ class HashKwargs(TypedDict): sha256: NotRequired[Optional[Sha256]] -_file_source_adapter = TypeAdapter(FileSource) +_file_source_adapter: TypeAdapter[Union[HttpUrl, RelativeFilePath, FilePath]] = ( + TypeAdapter(FileSource) +) def interprete_file_source(file_source: PermissiveFileSource) -> FileSource: diff --git a/tests/test_generic/test_v0_3.py b/tests/test_generic/test_v0_3.py index 9918d8fcd..2723f8306 100644 --- a/tests/test_generic/test_v0_3.py +++ b/tests/test_generic/test_v0_3.py @@ -1,7 +1,7 @@ from __future__ import annotations from pathlib import Path, PurePath -from typing import Any, Dict, Union +from typing import Any, Dict, Sequence, Union import pytest from pydantic import ( @@ -106,7 +106,7 @@ def test_documentation_source(): from bioimageio.spec.generic.v0_3 import DocumentationSource doc_src = "https://example.com/away.md" - adapter = TypeAdapter(DocumentationSource) + adapter: TypeAdapter[Any] = TypeAdapter(DocumentationSource) with ValidationContext(perform_io_checks=False): valid = adapter.validate_python(doc_src) @@ -118,7 +118,7 @@ def test_documentation_source_abs_path(): doc_src = UNET2D_ROOT / "README.md" assert doc_src.exists(), doc_src - adapter = TypeAdapter(DocumentationSource) + adapter: TypeAdapter[Any] = TypeAdapter(DocumentationSource) valid = adapter.validate_python(doc_src) assert str(valid) == str(doc_src) @@ -142,52 +142,43 @@ def validate_md_suffix(value: Union[AbsoluteFilePath, RelativeFilePath, HttpUrl] return validate_suffix(value, ".md", case_sensitive=True) -@pytest.mark.parametrize( - "src,adapter", - [ - (UNET2D_ROOT / "README.md", a) - for a in [ - TypeAdapter(Annotated[FilePath, WithSuffix(".md", case_sensitive=True)]), - TypeAdapter(Annotated[Path, WithSuffix(".md", case_sensitive=True)]), - TypeAdapter(Annotated[PurePath, WithSuffix(".md", case_sensitive=True)]), - TypeAdapter( - Annotated[ - Union[PurePath, HttpUrl], WithSuffix(".md", case_sensitive=True) - ] - ), - TypeAdapter( - Annotated[ - Union[AbsoluteFilePath, RelativeFilePath, HttpUrl], - WithSuffix(".md", case_sensitive=True), - ] - ), - TypeAdapter(DocumentationSource), - TypeAdapter( - Annotated[DocumentationSource, WithSuffix(".md", case_sensitive=True)] - ), +_type_adapters_for_path: Sequence[TypeAdapter[Any]] = ( + TypeAdapter(Annotated[FilePath, WithSuffix(".md", case_sensitive=True)]), + TypeAdapter(Annotated[Path, WithSuffix(".md", case_sensitive=True)]), + TypeAdapter(Annotated[PurePath, WithSuffix(".md", case_sensitive=True)]), + TypeAdapter( + Annotated[Union[PurePath, HttpUrl], WithSuffix(".md", case_sensitive=True)] + ), + TypeAdapter( + Annotated[ + Union[AbsoluteFilePath, RelativeFilePath, HttpUrl], + WithSuffix(".md", case_sensitive=True), ] - ] - + [ - (text_md_url, a) - for a in [ - TypeAdapter(Annotated[HttpUrl, WithSuffix(".md", case_sensitive=True)]), - TypeAdapter( - Annotated[ - Union[PurePath, HttpUrl], WithSuffix(".md", case_sensitive=True) - ] - ), - TypeAdapter( - Annotated[ - Union[AbsoluteFilePath, RelativeFilePath, HttpUrl], - WithSuffix(".md", case_sensitive=True), - ] - ), - TypeAdapter(DocumentationSource), - TypeAdapter( - Annotated[DocumentationSource, WithSuffix(".md", case_sensitive=True)] - ), + ), + TypeAdapter(DocumentationSource), + TypeAdapter(Annotated[DocumentationSource, WithSuffix(".md", case_sensitive=True)]), +) + +_type_adapters_for_url: Sequence[TypeAdapter[Any]] = ( + TypeAdapter(Annotated[HttpUrl, WithSuffix(".md", case_sensitive=True)]), + TypeAdapter( + Annotated[Union[PurePath, HttpUrl], WithSuffix(".md", case_sensitive=True)] + ), + TypeAdapter( + Annotated[ + Union[AbsoluteFilePath, RelativeFilePath, HttpUrl], + WithSuffix(".md", case_sensitive=True), ] - ], + ), + TypeAdapter(DocumentationSource), + TypeAdapter(Annotated[DocumentationSource, WithSuffix(".md", case_sensitive=True)]), +) + + +@pytest.mark.parametrize( + "src,adapter", + [(UNET2D_ROOT / "README.md", a) for a in _type_adapters_for_path] + + [(text_md_url, a) for a in _type_adapters_for_url], ) def test_with_suffix(src: Union[Path, HttpUrl], adapter: TypeAdapter[Any]): with ValidationContext(perform_io_checks=False): diff --git a/tests/test_internal/test_types.py b/tests/test_internal/test_types.py index 3cfff996d..d3cba4fe2 100644 --- a/tests/test_internal/test_types.py +++ b/tests/test_internal/test_types.py @@ -1,5 +1,6 @@ from datetime import datetime from pathlib import Path +from typing import Any import pytest from dateutil.parser import isoparse @@ -156,7 +157,7 @@ def test_datetime_more(value: str): ) root_adapter = TypeAdapter(Datetime) - datetime_adapter = TypeAdapter( + datetime_adapter: TypeAdapter[Any] = TypeAdapter( Annotated[ datetime, PlainSerializer(_serialize_datetime_json, when_used="json-unless-none"), diff --git a/tests/test_internal/test_validate.py b/tests/test_internal/test_validate.py index bf6ed98ad..4362e2d86 100644 --- a/tests/test_internal/test_validate.py +++ b/tests/test_internal/test_validate.py @@ -10,7 +10,9 @@ def test_single_suffix(): - adapter = TypeAdapter(Annotated[FileSource, WithSuffix(".py", case_sensitive=True)]) + adapter: TypeAdapter[FileSource] = TypeAdapter( + Annotated[FileSource, WithSuffix(".py", case_sensitive=True)] + ) with ValidationContext(root=Path(__file__).parent): _ = adapter.validate_python(Path(__file__).name) @@ -20,13 +22,15 @@ def test_single_suffix(): def test_case_sensitive_suffix(): - adapter = TypeAdapter(Annotated[FileSource, WithSuffix(".py", case_sensitive=True)]) + adapter: TypeAdapter[FileSource] = TypeAdapter( + Annotated[FileSource, WithSuffix(".py", case_sensitive=True)] + ) with ValidationContext(perform_io_checks=False), pytest.raises(ValidationError): _ = adapter.validate_python("https://example.com/lala.PY") def test_multiple_suffix(): - adapter = TypeAdapter( + adapter: TypeAdapter[FileSource] = TypeAdapter( Annotated[FileSource, WithSuffix((".py", ".md"), case_sensitive=True)] ) with ValidationContext(root=Path(__file__).parent): diff --git a/tests/utils.py b/tests/utils.py index 6e3c1715d..8161185c1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -22,7 +22,7 @@ RootModel, TypeAdapter, ValidationError, - create_model, # type: ignore + create_model, ) from ruyaml import YAML From 0c0ef9cf793738874a5e435c009e2999faa36313 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 14:29:21 +0200 Subject: [PATCH 18/20] update test_format_version_matches_library --- tests/test_model/test_format_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_model/test_format_version.py b/tests/test_model/test_format_version.py index 755e0b36a..307d8525f 100644 --- a/tests/test_model/test_format_version.py +++ b/tests/test_model/test_format_version.py @@ -2,8 +2,8 @@ def test_format_version_matches_library(): from bioimageio.spec import __version__ from bioimageio.spec.model import ModelDescr - version_wo_post = __version__.split("post")[0] - assert ModelDescr.implemented_format_version == version_wo_post, ( + lib_format_version = ".".join(__version__.split(".")[:3]) + assert ModelDescr.implemented_format_version == lib_format_version, ( ModelDescr.implemented_format_version, __version__, ) From 2dfcda0e3abde65427d8f98c4a8a54fdcb334c51 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 14:39:05 +0200 Subject: [PATCH 19/20] expose load_model_description and load_dataset_description --- bioimageio/spec/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bioimageio/spec/__init__.py b/bioimageio/spec/__init__.py index eac5514ba..910547d0c 100644 --- a/bioimageio/spec/__init__.py +++ b/bioimageio/spec/__init__.py @@ -16,10 +16,12 @@ from ._internal.common_nodes import InvalidDescr as InvalidDescr from ._internal.constants import VERSION from ._internal.validation_context import ValidationContext as ValidationContext +from ._io import load_dataset_description as load_dataset_description from ._io import load_description as load_description from ._io import ( load_description_and_validate_format_only as load_description_and_validate_format_only, ) +from ._io import load_model_description as load_model_description from ._io import save_bioimageio_yaml_only as save_bioimageio_yaml_only from ._package import save_bioimageio_package as save_bioimageio_package from ._package import ( From 6de0f55ef396771cb6f5cd808ce70630b3495530 Mon Sep 17 00:00:00 2001 From: fynnbe Date: Thu, 25 Jul 2024 14:45:05 +0200 Subject: [PATCH 20/20] expose ensure_description_is_model and ensure_description_is_dataset --- bioimageio/spec/model/v0_4.py | 2 +- bioimageio/spec/utils.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bioimageio/spec/model/v0_4.py b/bioimageio/spec/model/v0_4.py index 33845f2e3..03bf0b3a3 100644 --- a/bioimageio/spec/model/v0_4.py +++ b/bioimageio/spec/model/v0_4.py @@ -49,6 +49,7 @@ from .._internal.io import FileDescr as FileDescr from .._internal.io_basics import AbsoluteFilePath as AbsoluteFilePath from .._internal.io_basics import Sha256 as Sha256 +from .._internal.io_utils import load_array from .._internal.packaging_context import packaging_context_var from .._internal.types import Datetime as Datetime from .._internal.types import Identifier as Identifier @@ -74,7 +75,6 @@ from ..generic.v0_2 import RelativeFilePath as RelativeFilePath from ..generic.v0_2 import ResourceId as ResourceId from ..generic.v0_2 import Uploader as Uploader -from ..utils import load_array from ._v0_4_converter import convert_from_older_format diff --git a/bioimageio/spec/utils.py b/bioimageio/spec/utils.py index c037d4aa7..92ae843a2 100644 --- a/bioimageio/spec/utils.py +++ b/bioimageio/spec/utils.py @@ -1,6 +1,8 @@ import json from typing import List, TypedDict +from ._description import ensure_description_is_dataset as ensure_description_is_dataset +from ._description import ensure_description_is_model as ensure_description_is_model from ._internal.io import download as download from ._internal.io import extract_file_name as extract_file_name from ._internal.io import get_sha256 as get_sha256