From 14e2ed38bd5ce4219d15eee9a093262a947baa8e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 31 May 2024 14:00:03 -0700 Subject: [PATCH 1/4] chore: update pre-commit hooks (#1920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - https://github.com/charliermarsh/ruff-pre-commit → https://github.com/astral-sh/ruff-pre-commit - [github.com/astral-sh/ruff-pre-commit: v0.4.3 → v0.4.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.3...v0.4.5) - [github.com/codespell-project/codespell: v2.2.6 → v2.3.0](https://github.com/codespell-project/codespell/compare/v2.2.6...v2.3.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ffa3c94efa..58cbac3c45 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,14 +6,14 @@ default_stages: [commit, push] default_language_version: python: python3 repos: - - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.4.3' + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: 'v0.4.5' hooks: - id: ruff args: ["--fix", "--show-fixes"] - id: ruff-format - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell args: ["-L", "ba,ihs,kake,nd,noe,nwo,te,fo,zar", "-S", "fixture"] From 9ad01f1f8f80e9c96fb58df38435f1adad0d6def Mon Sep 17 00:00:00 2001 From: David Stansby Date: Fri, 31 May 2024 22:01:25 +0100 Subject: [PATCH 2/4] Disallow implicit re-exports (#1908) * No implict reexports * Clean up v2 mypy ignores --- pyproject.toml | 19 +++++++------------ src/zarr/array.py | 4 ++-- src/zarr/codecs/__init__.py | 31 +++++++++++++++++++++++-------- src/zarr/codecs/pipeline.py | 3 +-- src/zarr/codecs/sharding.py | 3 +-- src/zarr/metadata.py | 3 +++ src/zarr/store/__init__.py | 2 ++ 7 files changed, 39 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 947bec9369..4bcbfd0a0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -201,34 +201,30 @@ disallow_incomplete_defs = true disallow_untyped_calls = true disallow_untyped_defs = true +no_implicit_reexport = true + [[tool.mypy.overrides]] module = [ "zarr.v2.*", - "zarr.group", - "tests.*", ] -check_untyped_defs = false +ignore_errors = true [[tool.mypy.overrides]] module = [ - "zarr.v2.*", - "zarr.array_v2", + "zarr.group", + "tests.*", ] -disallow_any_generics = false +check_untyped_defs = false [[tool.mypy.overrides]] module = [ - "zarr.v2.*", - "zarr.array_v2", "zarr.group" ] disallow_incomplete_defs = false [[tool.mypy.overrides]] module = [ - "zarr.v2.*", - "zarr.array_v2", "zarr.array", "zarr.buffer" ] @@ -236,13 +232,12 @@ disallow_untyped_calls = false [[tool.mypy.overrides]] module = [ - "zarr.v2.*", - "zarr.array_v2", "zarr.array", "zarr.group", ] disallow_untyped_defs = false + [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] diff --git a/src/zarr/array.py b/src/zarr/array.py index 7da39c285e..cb780bb8d7 100644 --- a/src/zarr/array.py +++ b/src/zarr/array.py @@ -34,9 +34,9 @@ ZarrFormat, concurrent_map, ) -from zarr.config import config +from zarr.config import config, parse_indexing_order from zarr.indexing import BasicIndexer -from zarr.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata, parse_indexing_order +from zarr.metadata import ArrayMetadata, ArrayV2Metadata, ArrayV3Metadata from zarr.store import StoreLike, StorePath, make_store_path from zarr.sync import sync diff --git a/src/zarr/codecs/__init__.py b/src/zarr/codecs/__init__.py index 0f0ff55df5..3ef3a87db7 100644 --- a/src/zarr/codecs/__init__.py +++ b/src/zarr/codecs/__init__.py @@ -1,10 +1,25 @@ from __future__ import annotations -from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle # noqa: F401 -from zarr.codecs.bytes import BytesCodec, Endian # noqa: F401 -from zarr.codecs.crc32c_ import Crc32cCodec # noqa: F401 -from zarr.codecs.gzip import GzipCodec # noqa: F401 -from zarr.codecs.pipeline import BatchedCodecPipeline # noqa: F401 -from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation # noqa: F401 -from zarr.codecs.transpose import TransposeCodec # noqa: F401 -from zarr.codecs.zstd import ZstdCodec # noqa: F401 +from zarr.codecs.blosc import BloscCname, BloscCodec, BloscShuffle +from zarr.codecs.bytes import BytesCodec, Endian +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.codecs.gzip import GzipCodec +from zarr.codecs.pipeline import BatchedCodecPipeline +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.codecs.transpose import TransposeCodec +from zarr.codecs.zstd import ZstdCodec + +__all__ = [ + "BatchedCodecPipeline", + "BloscCodec", + "BloscCname", + "BloscShuffle", + "BytesCodec", + "Endian", + "Crc32cCodec", + "GzipCodec", + "ShardingCodec", + "ShardingCodecIndexLocation", + "TransposeCodec", + "ZstdCodec", +] diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/codecs/pipeline.py index 893cbc8b4b..6f493c9e81 100644 --- a/src/zarr/codecs/pipeline.py +++ b/src/zarr/codecs/pipeline.py @@ -11,12 +11,11 @@ ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, - ByteGetter, BytesBytesCodec, - ByteSetter, Codec, CodecPipeline, ) +from zarr.abc.store import ByteGetter, ByteSetter from zarr.buffer import Buffer, NDBuffer from zarr.codecs.registry import get_codec_class from zarr.common import JSON, concurrent_map, parse_named_configuration diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index a68577be68..ec5306ee80 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -14,11 +14,10 @@ ArrayBytesCodec, ArrayBytesCodecPartialDecodeMixin, ArrayBytesCodecPartialEncodeMixin, - ByteGetter, - ByteSetter, Codec, CodecPipeline, ) +from zarr.abc.store import ByteGetter, ByteSetter from zarr.buffer import Buffer, NDBuffer from zarr.chunk_grids import RegularChunkGrid from zarr.codecs.bytes import BytesCodec diff --git a/src/zarr/metadata.py b/src/zarr/metadata.py index 58cc276c29..2d8a455152 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/metadata.py @@ -41,6 +41,9 @@ _bool = bool +__all__ = ["ArrayMetadata"] + + class DataType(Enum): bool = "bool" int8 = "int8" diff --git a/src/zarr/store/__init__.py b/src/zarr/store/__init__.py index b1c3a5f720..b7cd6cc0fd 100644 --- a/src/zarr/store/__init__.py +++ b/src/zarr/store/__init__.py @@ -3,3 +3,5 @@ from zarr.store.remote import RemoteStore from zarr.store.local import LocalStore from zarr.store.memory import MemoryStore + +__all__ = ["StorePath", "StoreLike", "make_store_path", "RemoteStore", "LocalStore", "MemoryStore"] From 016964b90a0a30c10e5b6d08b4cc9d7eb15b6aa0 Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 1 Jun 2024 03:39:16 +0100 Subject: [PATCH 3/4] Make typing strict (#1879) * Disallow subclassing any * Disallow returning any * Enable strict * Fix literal import * pre-commit fixes * Remove old group imports --- .pre-commit-config.yaml | 2 ++ pyproject.toml | 23 ++++++++--------------- src/zarr/codecs/sharding.py | 2 +- src/zarr/codecs/zstd.py | 4 ++-- src/zarr/common.py | 12 ++++++++++-- src/zarr/config.py | 4 ++-- src/zarr/group.py | 4 ++-- src/zarr/v2/n5.py | 2 +- src/zarr/v2/util.py | 2 +- 9 files changed, 29 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 58cbac3c45..578a66fd07 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,3 +31,5 @@ repos: - types-setuptools - pytest - numpy + - numcodecs + - zstandard diff --git a/pyproject.toml b/pyproject.toml index 4bcbfd0a0a..616c388f99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -187,21 +187,7 @@ python_version = "3.10" ignore_missing_imports = true namespace_packages = false -warn_unused_configs = true -warn_redundant_casts = true -warn_unused_ignores = true -strict_equality = true -strict_concatenate = true - -check_untyped_defs = true -disallow_untyped_decorators = true -disallow_any_generics = true - -disallow_incomplete_defs = true -disallow_untyped_calls = true - -disallow_untyped_defs = true -no_implicit_reexport = true +strict = true [[tool.mypy.overrides]] @@ -238,6 +224,13 @@ module = [ disallow_untyped_defs = false +[[tool.mypy.overrides]] +module = [ + "zarr.metadata", + "zarr.store.remote" +] +warn_return_any = false + [tool.pytest.ini_options] minversion = "7" testpaths = ["tests"] diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index ec5306ee80..cea788840d 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -101,7 +101,7 @@ def is_all_empty(self) -> bool: return bool(np.array_equiv(self.offsets_and_lengths, MAX_UINT_64)) def get_full_chunk_map(self) -> npt.NDArray[np.bool_]: - return self.offsets_and_lengths[..., 0] != MAX_UINT_64 + return np.not_equal(self.offsets_and_lengths[..., 0], MAX_UINT_64) def get_chunk_slice(self, chunk_coords: ChunkCoords) -> tuple[int, int] | None: localized_chunk = self._localize_chunk(chunk_coords) diff --git a/src/zarr/codecs/zstd.py b/src/zarr/codecs/zstd.py index 76e625ad6a..451fae8b37 100644 --- a/src/zarr/codecs/zstd.py +++ b/src/zarr/codecs/zstd.py @@ -55,11 +55,11 @@ def to_dict(self) -> dict[str, JSON]: def _compress(self, data: npt.NDArray[Any]) -> bytes: ctx = ZstdCompressor(level=self.level, write_checksum=self.checksum) - return ctx.compress(data) + return ctx.compress(data.tobytes()) def _decompress(self, data: npt.NDArray[Any]) -> bytes: ctx = ZstdDecompressor() - return ctx.decompress(data) + return ctx.decompress(data.tobytes()) async def _decode_single( self, diff --git a/src/zarr/common.py b/src/zarr/common.py index 9527efbbce..ec5d870f92 100644 --- a/src/zarr/common.py +++ b/src/zarr/common.py @@ -7,7 +7,15 @@ from collections.abc import Iterable from dataclasses import dataclass from enum import Enum -from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, overload +from typing import ( + TYPE_CHECKING, + Any, + Literal, + ParamSpec, + TypeVar, + cast, + overload, +) if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator @@ -178,5 +186,5 @@ def parse_fill_value(data: Any) -> Any: def parse_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return data + return cast(Literal["C", "F"], data) raise ValueError(f"Expected one of ('C', 'F'), got {data} instead.") diff --git a/src/zarr/config.py b/src/zarr/config.py index 5b1640bd56..7c5b48a16c 100644 --- a/src/zarr/config.py +++ b/src/zarr/config.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Any, Literal +from typing import Any, Literal, cast from donfig import Config @@ -18,6 +18,6 @@ def parse_indexing_order(data: Any) -> Literal["C", "F"]: if data in ("C", "F"): - return data + return cast(Literal["C", "F"], data) msg = f"Expected one of ('C', 'F'), got {data} instead." raise ValueError(msg) diff --git a/src/zarr/group.py b/src/zarr/group.py index 4ff2176fd9..2401934b84 100644 --- a/src/zarr/group.py +++ b/src/zarr/group.py @@ -5,7 +5,7 @@ import logging from collections.abc import Iterator from dataclasses import asdict, dataclass, field, replace -from typing import TYPE_CHECKING, overload +from typing import TYPE_CHECKING, Literal, cast, overload import numpy.typing as npt @@ -37,7 +37,7 @@ def parse_zarr_format(data: Any) -> ZarrFormat: if data in (2, 3): - return data + return cast(Literal[2, 3], data) msg = msg = f"Invalid zarr_format. Expected one 2 or 3. Got {data}." raise ValueError(msg) diff --git a/src/zarr/v2/n5.py b/src/zarr/v2/n5.py index 4ea5e45721..a6fd39f5b8 100644 --- a/src/zarr/v2/n5.py +++ b/src/zarr/v2/n5.py @@ -780,7 +780,7 @@ def compressor_config_to_zarr(compressor_config: Dict[str, Any]) -> Optional[Dic return zarr_config -class N5ChunkWrapper(Codec): +class N5ChunkWrapper(Codec): # type: ignore[misc] codec_id = "n5_wrapper" def __init__(self, dtype, chunk_shape, compressor_config=None, compressor=None): diff --git a/src/zarr/v2/util.py b/src/zarr/v2/util.py index 48d7d30d88..6926bb2d14 100644 --- a/src/zarr/v2/util.py +++ b/src/zarr/v2/util.py @@ -444,7 +444,7 @@ def get_type(self): return type(self.obj).__name__ -class TreeTraversal(Traversal): +class TreeTraversal(Traversal): # type: ignore[misc] def get_children(self, node): return node.get_children() From b4cdf94c1ff80629ae902432567943a1cdcc443e Mon Sep 17 00:00:00 2001 From: David Stansby Date: Sat, 1 Jun 2024 03:46:08 +0100 Subject: [PATCH 4/4] Enable extra mypy error codes (#1909) * Clean up v2 mypy ignores * Enable extra mypy error codes * Specific error code --------- Co-authored-by: Joe Hamman --- pyproject.toml | 2 ++ src/zarr/buffer.py | 2 +- src/zarr/codecs/sharding.py | 2 +- src/zarr/indexing.py | 7 ++----- src/zarr/metadata.py | 7 ++++--- src/zarr/testing/store.py | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 616c388f99..9f50c33db0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -190,6 +190,8 @@ namespace_packages = false strict = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + [[tool.mypy.overrides]] module = [ "zarr.v2.*", diff --git a/src/zarr/buffer.py b/src/zarr/buffer.py index 0f055093c1..59994e70d6 100644 --- a/src/zarr/buffer.py +++ b/src/zarr/buffer.py @@ -79,7 +79,7 @@ def ravel(self, order: Literal["K", "A", "C", "F"] = "C") -> Self: ... def all(self) -> bool: ... - def __eq__(self, other: Any) -> Self: # type: ignore + def __eq__(self, other: Any) -> Self: # type: ignore[explicit-override, override] """Element-wise equal Notice diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index cea788840d..a7b6edc3b4 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -205,7 +205,7 @@ def merge_with_morton_order( ) -> _ShardBuilder: obj = cls.create_empty(chunks_per_shard) for chunk_coords in morton_order_iter(chunks_per_shard): - if tombstones is not None and chunk_coords in tombstones: + if chunk_coords in tombstones: continue for shard_dict in shard_dicts: maybe_value = shard_dict.get(chunk_coords, None) diff --git a/src/zarr/indexing.py b/src/zarr/indexing.py index 45413bc5b2..6bc83d5062 100644 --- a/src/zarr/indexing.py +++ b/src/zarr/indexing.py @@ -199,11 +199,8 @@ def is_total_slice(item: Selection, shape: ChunkCoords) -> bool: if isinstance(item, tuple): return all( ( - isinstance(dim_sel, slice) - and ( - (dim_sel == slice(None)) - or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) - ) + (dim_sel == slice(None)) + or ((dim_sel.stop - dim_sel.start == dim_len) and (dim_sel.step in [1, None])) ) for dim_sel, dim_len in zip(item, shape, strict=False) ) diff --git a/src/zarr/metadata.py b/src/zarr/metadata.py index 2d8a455152..29d0d19a06 100644 --- a/src/zarr/metadata.py +++ b/src/zarr/metadata.py @@ -432,10 +432,11 @@ def update_attributes(self, attributes: dict[str, JSON]) -> Self: def parse_dimension_names(data: None | Iterable[str]) -> tuple[str, ...] | None: if data is None: return data - if isinstance(data, Iterable) and all([isinstance(x, str) for x in data]): + elif all([isinstance(x, str) for x in data]): return tuple(data) - msg = f"Expected either None or a iterable of str, got {type(data)}" - raise TypeError(msg) + else: + msg = f"Expected either None or a iterable of str, got {type(data)}" + raise TypeError(msg) # todo: real validation diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index b317f383f6..cb4dc9f7b5 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -47,7 +47,7 @@ def test_store_mode(self, store: S, store_kwargs: dict[str, Any]) -> None: assert store.writeable with pytest.raises(AttributeError): - store.mode = "w" # type: ignore + store.mode = "w" # type: ignore[misc] # read-only kwargs = {**store_kwargs, "mode": "r"}