From c8cd1479436c5cc6707e2a475bbaa17f11cc2afa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 08:49:25 +0000 Subject: [PATCH 01/14] Bump sphinx-autoapi from 3.3.2 to 3.3.3 in the requirements group (#2441) Bumps the requirements group with 1 update: [sphinx-autoapi](https://github.com/readthedocs/sphinx-autoapi). Updates `sphinx-autoapi` from 3.3.2 to 3.3.3 - [Release notes](https://github.com/readthedocs/sphinx-autoapi/releases) - [Changelog](https://github.com/readthedocs/sphinx-autoapi/blob/main/CHANGELOG.rst) - [Commits](https://github.com/readthedocs/sphinx-autoapi/compare/v3.3.2...v3.3.3) --- updated-dependencies: - dependency-name: sphinx-autoapi dependency-type: direct:production update-type: version-update:semver-patch dependency-group: requirements ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7be199948..bb9fa4df8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ gpu = [ docs = [ 'sphinx==8.1.3', 'sphinx-autobuild>=2021.3.14', - 'sphinx-autoapi==3.3.2', + 'sphinx-autoapi==3.3.3', 'sphinx_design', 'sphinx-issues', 'sphinx-copybutton', From 4c3081c9b4678d1de438e0d2ee8e0fa368a61f17 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:33:14 -0700 Subject: [PATCH 02/14] chore: update pre-commit hooks (#2443) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.0 → v0.7.1](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.0...v0.7.1) - [github.com/pre-commit/mirrors-mypy: v1.12.1 → v1.13.0](https://github.com/pre-commit/mirrors-mypy/compare/v1.12.1...v1.13.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40cddb50c..d6df002fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.0 + rev: v0.7.1 hooks: - id: ruff args: ["--fix", "--show-fixes"] @@ -22,7 +22,7 @@ repos: hooks: - id: check-yaml - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.12.1 + rev: v1.13.0 hooks: - id: mypy files: src|tests From 0f56ac218dc81412749f8e7596bb637db20d329b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 09:27:06 -0800 Subject: [PATCH 03/14] Bump pypa/gh-action-pypi-publish in the actions group (#2457) Bumps the actions group with 1 update: [pypa/gh-action-pypi-publish](https://github.com/pypa/gh-action-pypi-publish). Updates `pypa/gh-action-pypi-publish` from 1.10.3 to 1.11.0 - [Release notes](https://github.com/pypa/gh-action-pypi-publish/releases) - [Commits](https://github.com/pypa/gh-action-pypi-publish/compare/v1.10.3...v1.11.0) --- updated-dependencies: - dependency-name: pypa/gh-action-pypi-publish dependency-type: direct:production update-type: version-update:semver-minor dependency-group: actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/releases.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml index bab53958d..bb24f0eb7 100644 --- a/.github/workflows/releases.yml +++ b/.github/workflows/releases.yml @@ -55,7 +55,7 @@ jobs: with: name: releases path: dist - - uses: pypa/gh-action-pypi-publish@v1.10.3 + - uses: pypa/gh-action-pypi-publish@v1.11.0 with: user: __token__ password: ${{ secrets.pypi_password }} From a82d0476656cd2de12145abd899c81eb8c726ed3 Mon Sep 17 00:00:00 2001 From: Tom Augspurger Date: Mon, 4 Nov 2024 11:45:57 -0600 Subject: [PATCH 04/14] Make AsyncArray.nchunks_initialized async (#2449) * Make AsyncArray.nchunks_initialized async This changes the API of AysncArray.nchunks_initialized to change it from a property to an async function. The motivation here comes from 1. general cleanliness (a property access calling async functions doing I/O feels a bit wrong) 2. Work on Array.info, where I hit a strange error, I think from jumping from a - sync Array.info_complete -> - async AsyncArray.info_complete -> - sync AsyncArray.nchunks_initialzed -> - sync collect_aiterator (async list_prefix) With this change, we'll be able to jump from sync to async just once at the boundary. ``` File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/array.py", line 3011, in info_complete return sync(self._async_array.info_complete()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/sync.py", line 141, in sync raise return_result File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/sync.py", line 100, in _runner return await coro ^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/array.py", line 1223, in info_complete "count_chunks_initialized": self.nchunks_initialized, # this should be async? ^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/array.py", line 844, in nchunks_initialized return nchunks_initialized(self) ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/array.py", line 3035, in nchunks_initialized return len(chunks_initialized(array)) ^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/array.py", line 3061, in chunks_initialized collect_aiterator(array.store_path.store.list_prefix(prefix=array.store_path.path)) File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/sync.py", line 178, in collect_aiterator return sync(_collect_aiterator(data)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/Users/tom/gh/zarr-developers/zarr-python/src/zarr/core/sync.py", line 128, in sync raise SyncError("Calling sync() from within a running loop") zarr.core.sync.SyncError: Calling sync() from within a running loop ``` * fixup --------- Co-authored-by: Davis Bennett --- src/zarr/core/array.py | 89 +++++++++++++++++++++++------------------- tests/test_array.py | 16 +++----- 2 files changed, 54 insertions(+), 51 deletions(-) diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index a4b86b85a..c3d6fca54 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -77,7 +77,7 @@ T_ArrayMetadata, ) from zarr.core.metadata.v3 import parse_node_type_array -from zarr.core.sync import collect_aiterator, sync +from zarr.core.sync import sync from zarr.errors import MetadataValidationError from zarr.registry import get_pipeline_class from zarr.storage import StoreLike, make_store_path @@ -829,17 +829,31 @@ def nchunks(self) -> int: """ return product(self.cdata_shape) - @property - def nchunks_initialized(self) -> int: + async def nchunks_initialized(self) -> int: """ - The number of chunks that have been persisted in storage. + Calculate the number of chunks that have been initialized, i.e. the number of chunks that have + been persisted to the storage backend. Returns ------- - int - The number of initialized chunks in the array. + nchunks_initialized : int + The number of chunks that have been initialized. + + Notes + ----- + On :class:`AsyncArray` this is an asynchronous method, unlike the (synchronous) + property :attr:`Array.nchunks_initialized`. + + Examples + -------- + >>> arr = await zarr.api.asynchronous.create(shape=(10,), chunks=(2,)) + >>> await arr.nchunks_initialized() + 0 + >>> await arr.setitem(slice(5), 1) + >>> await arr.nchunks_initialized() + 3 """ - return nchunks_initialized(self) + return len(await chunks_initialized(self)) def _iter_chunk_coords( self, *, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None @@ -1492,9 +1506,29 @@ def nbytes(self) -> int: @property def nchunks_initialized(self) -> int: """ - The number of chunks that have been initialized in the stored representation of this array. + Calculate the number of chunks that have been initialized, i.e. the number of chunks that have + been persisted to the storage backend. + + Returns + ------- + nchunks_initialized : int + The number of chunks that have been initialized. + + Notes + ----- + On :class:`Array` this is a (synchronous) property, unlike asynchronous function + :meth:`AsyncArray.nchunks_initialized`. + + Examples + -------- + >>> arr = await zarr.create(shape=(10,), chunks=(2,)) + >>> arr.nchunks_initialized + 0 + >>> arr[:5] = 1 + >>> arr.nchunks_initialized + 3 """ - return self._async_array.nchunks_initialized + return sync(self._async_array.nchunks_initialized()) def _iter_chunk_keys( self, origin: Sequence[int] | None = None, selection_shape: Sequence[int] | None = None @@ -2905,39 +2939,15 @@ def info(self) -> None: ) -def nchunks_initialized( - array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata] | Array, -) -> int: - """ - Calculate the number of chunks that have been initialized, i.e. the number of chunks that have - been persisted to the storage backend. - - Parameters - ---------- - array : Array - The array to inspect. - - Returns - ------- - nchunks_initialized : int - The number of chunks that have been initialized. - - See Also - -------- - chunks_initialized - """ - return len(chunks_initialized(array)) - - -def chunks_initialized( - array: Array | AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], +async def chunks_initialized( + array: AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata], ) -> tuple[str, ...]: """ Return the keys of the chunks that have been persisted to the storage backend. Parameters ---------- - array : Array + array : AsyncArray The array to inspect. Returns @@ -2950,10 +2960,9 @@ def chunks_initialized( nchunks_initialized """ - # TODO: make this compose with the underlying async iterator - store_contents = list( - collect_aiterator(array.store_path.store.list_prefix(prefix=array.store_path.path)) - ) + store_contents = [ + x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path) + ] return tuple(chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents) diff --git a/tests/test_array.py b/tests/test_array.py index 6451c7fe5..285649490 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -323,7 +323,7 @@ def test_nchunks(test_cls: type[Array] | type[AsyncArray[Any]], nchunks: int) -> @pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) -def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> None: +async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> None: """ Test that nchunks_initialized accurately returns the number of stored chunks. """ @@ -337,7 +337,7 @@ def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> N if test_cls == Array: observed = arr.nchunks_initialized else: - observed = arr._async_array.nchunks_initialized + observed = await arr._async_array.nchunks_initialized() assert observed == expected # delete chunks @@ -346,13 +346,12 @@ def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> N if test_cls == Array: observed = arr.nchunks_initialized else: - observed = arr._async_array.nchunks_initialized + observed = await arr._async_array.nchunks_initialized() expected = arr.nchunks - idx - 1 assert observed == expected -@pytest.mark.parametrize("test_cls", [Array, AsyncArray[Any]]) -def test_chunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> None: +async def test_chunks_initialized() -> None: """ Test that chunks_initialized accurately returns the keys of stored chunks. """ @@ -364,12 +363,7 @@ def test_chunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]) -> No ) for keys, region in zip(chunks_accumulated, arr._iter_chunk_regions(), strict=False): arr[region] = 1 - - if test_cls == Array: - observed = sorted(chunks_initialized(arr)) - else: - observed = sorted(chunks_initialized(arr._async_array)) - + observed = sorted(await chunks_initialized(arr._async_array)) expected = sorted(keys) assert observed == expected From 7bceb58dd69a87820b5655b9248dc0110fcbc2d9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:50:23 -0800 Subject: [PATCH 05/14] chore: update pre-commit hooks (#2462) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.7.1 → v0.7.2](https://github.com/astral-sh/ruff-pre-commit/compare/v0.7.1...v0.7.2) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d6df002fb..1c5edaec5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ default_language_version: python: python3 repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.7.2 hooks: - id: ruff args: ["--fix", "--show-fixes"] From a31046c718b2a536e665784965dd328232f00f47 Mon Sep 17 00:00:00 2001 From: Joe Hamman Date: Mon, 4 Nov 2024 20:58:56 -0800 Subject: [PATCH 06/14] Feature: store learns to delete prefixes when overwriting/creating hierarchy nodes (#2430) * fix(array): thread order parameter through to array __init__ * type fixes * move defaults * apply MemoryOrder type to ArrayV2Metadata * more more * more more more * feature(store,group,array): stores learn to delete prefixes when overwriting nodes - add Store.delete_dir and Store.delete_prefix - update array and group creation methods to call delete_dir - change list_prefix to return absolue keys * fixup * fixup * respond to review * fixup * fixup * Update src/zarr/abc/store.py * style: pre-commit fixes --------- Co-authored-by: Davis Bennett Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/zarr/abc/store.py | 18 ++++++++++++++++-- src/zarr/core/array.py | 14 ++++++++++++-- src/zarr/core/group.py | 21 ++++++++------------- src/zarr/storage/common.py | 9 +++++++++ src/zarr/storage/local.py | 5 ++--- src/zarr/storage/logging.py | 5 +++++ src/zarr/storage/memory.py | 11 ++++++++--- src/zarr/storage/remote.py | 7 ++++--- src/zarr/storage/zip.py | 2 +- src/zarr/testing/store.py | 20 +++++++++++++++++--- tests/test_array.py | 2 ++ tests/test_group.py | 23 ++++++++++++++++++++++- tests/test_store/test_logging.py | 7 ++++++- tests/test_store/test_zip.py | 6 +----- 14 files changed, 113 insertions(+), 37 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 045da7e84..eefe04d50 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -342,8 +342,8 @@ def list(self) -> AsyncGenerator[str, None]: @abstractmethod def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: """ - Retrieve all keys in the store that begin with a given prefix. Keys are returned with the - common leading prefix removed. + Retrieve all keys in the store that begin with a given prefix. Keys are returned relative + to the root of the store. Parameters ---------- @@ -371,6 +371,20 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: """ ... + async def delete_dir(self, prefix: str) -> None: + """ + Remove all keys and prefixes in the store that begin with a given prefix. + """ + if not self.supports_deletes: + raise NotImplementedError + if not self.supports_listing: + raise NotImplementedError + self._check_writable() + if not prefix.endswith("/"): + prefix += "/" + async for key in self.list_prefix(prefix): + await self.delete(key) + def close(self) -> None: """Close the store.""" self._is_open = False diff --git a/src/zarr/core/array.py b/src/zarr/core/array.py index c3d6fca54..1646959cb 100644 --- a/src/zarr/core/array.py +++ b/src/zarr/core/array.py @@ -553,7 +553,12 @@ async def _create_v3( attributes: dict[str, JSON] | None = None, exists_ok: bool = False, ) -> AsyncArray[ArrayV3Metadata]: - if not exists_ok: + if exists_ok: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=3) + else: await ensure_no_existing_node(store_path, zarr_format=3) shape = parse_shapelike(shape) @@ -605,7 +610,12 @@ async def _create_v2( attributes: dict[str, JSON] | None = None, exists_ok: bool = False, ) -> AsyncArray[ArrayV2Metadata]: - if not exists_ok: + if exists_ok: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=2) + else: await ensure_no_existing_node(store_path, zarr_format=2) if order is None: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 46f37700e..86cf191ca 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -404,7 +404,13 @@ async def from_store( zarr_format: ZarrFormat = 3, ) -> AsyncGroup: store_path = await make_store_path(store) - if not exists_ok: + + if exists_ok: + if store_path.store.supports_deletes: + await store_path.delete_dir() + else: + await ensure_no_existing_node(store_path, zarr_format=zarr_format) + else: await ensure_no_existing_node(store_path, zarr_format=zarr_format) attributes = attributes or {} group = cls( @@ -727,19 +733,8 @@ def _getitem_consolidated( async def delitem(self, key: str) -> None: store_path = self.store_path / key - if self.metadata.zarr_format == 3: - await (store_path / ZARR_JSON).delete() - - elif self.metadata.zarr_format == 2: - await asyncio.gather( - (store_path / ZGROUP_JSON).delete(), # TODO: missing_ok=False - (store_path / ZARRAY_JSON).delete(), # TODO: missing_ok=False - (store_path / ZATTRS_JSON).delete(), # TODO: missing_ok=True - ) - - else: - raise ValueError(f"unexpected zarr_format: {self.metadata.zarr_format}") + await store_path.delete_dir() if self.metadata.consolidated_metadata: self.metadata.consolidated_metadata.metadata.pop(key, None) await self._save_metadata() diff --git a/src/zarr/storage/common.py b/src/zarr/storage/common.py index 9ed8c274d..337fbc59a 100644 --- a/src/zarr/storage/common.py +++ b/src/zarr/storage/common.py @@ -101,6 +101,15 @@ async def delete(self) -> None: """ await self.store.delete(self.path) + async def delete_dir(self) -> None: + """ + Delete all keys with the given prefix from the store. + """ + path = self.path + if not path.endswith("/"): + path += "/" + await self.store.delete_dir(path) + async def set_if_not_exists(self, default: Buffer) -> None: """ Store a key to ``value`` if the key is not already present. diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index ba13c2c66..331c9857c 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -226,9 +226,8 @@ async def list(self) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - to_strip = ( - (self.root / prefix).as_posix() + "/" - ) # TODO: fixme in https://github.com/zarr-developers/zarr-python/issues/2438 + to_strip = self.root.as_posix() + "/" + prefix = prefix.rstrip("/") for p in (self.root / prefix).rglob("*"): if p.is_file(): yield p.as_posix().replace(to_strip, "") diff --git a/src/zarr/storage/logging.py b/src/zarr/storage/logging.py index 66fd1687e..be259579e 100644 --- a/src/zarr/storage/logging.py +++ b/src/zarr/storage/logging.py @@ -222,6 +222,11 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: async for key in self._store.list_dir(prefix=prefix): yield key + async def delete_dir(self, prefix: str) -> None: + # docstring inherited + with self.log(prefix): + await self._store.delete_dir(prefix=prefix) + def with_mode(self, mode: AccessModeLiteral) -> Self: # docstring inherited with self.log(mode): diff --git a/src/zarr/storage/memory.py b/src/zarr/storage/memory.py index f942d57b9..fa4ede2a8 100644 --- a/src/zarr/storage/memory.py +++ b/src/zarr/storage/memory.py @@ -1,5 +1,6 @@ from __future__ import annotations +from logging import getLogger from typing import TYPE_CHECKING, Self from zarr.abc.store import ByteRangeRequest, Store @@ -14,6 +15,9 @@ from zarr.core.common import AccessModeLiteral +logger = getLogger(__name__) + + class MemoryStore(Store): """ In-memory store for testing purposes. @@ -137,7 +141,7 @@ async def delete(self, key: str) -> None: try: del self._store_dict[key] except KeyError: - pass + logger.debug("Key %s does not exist.", key) async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, bytes]]) -> None: # docstring inherited @@ -150,9 +154,10 @@ async def list(self) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - for key in self._store_dict: + # note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix) + for key in list(self._store_dict): if key.startswith(prefix): - yield key.removeprefix(prefix) + yield key async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index 1f7d5f7a1..ca7a010bd 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -340,6 +340,7 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited - find_str = f"{self.path}/{prefix}" - for onefile in await self.fs._find(find_str, detail=False, maxdepth=None, withdirs=False): - yield onefile.removeprefix(find_str) + for onefile in await self.fs._find( + f"{self.path}/{prefix}", detail=False, maxdepth=None, withdirs=False + ): + yield onefile.removeprefix(f"{self.path}/") diff --git a/src/zarr/storage/zip.py b/src/zarr/storage/zip.py index d9e1aa130..cf9b338cf 100644 --- a/src/zarr/storage/zip.py +++ b/src/zarr/storage/zip.py @@ -244,7 +244,7 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited async for key in self.list(): if key.startswith(prefix): - yield key.removeprefix(prefix) + yield key async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: # docstring inherited diff --git a/src/zarr/testing/store.py b/src/zarr/testing/store.py index b4da75b06..3aece0f4a 100644 --- a/src/zarr/testing/store.py +++ b/src/zarr/testing/store.py @@ -213,11 +213,26 @@ async def test_exists(self, store: S) -> None: assert await store.exists("foo/zarr.json") async def test_delete(self, store: S) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) assert await store.exists("foo/zarr.json") await store.delete("foo/zarr.json") assert not await store.exists("foo/zarr.json") + async def test_delete_dir(self, store: S) -> None: + if not store.supports_deletes: + pytest.skip("store does not support deletes") + await store.set("zarr.json", self.buffer_cls.from_bytes(b"root")) + await store.set("foo-bar/zarr.json", self.buffer_cls.from_bytes(b"root")) + await store.set("foo/zarr.json", self.buffer_cls.from_bytes(b"bar")) + await store.set("foo/c/0", self.buffer_cls.from_bytes(b"chunk")) + await store.delete_dir("foo") + assert await store.exists("zarr.json") + assert await store.exists("foo-bar/zarr.json") + assert not await store.exists("foo/zarr.json") + assert not await store.exists("foo/c/0") + async def test_empty(self, store: S) -> None: assert await store.empty() await self.set( @@ -249,8 +264,7 @@ async def test_list(self, store: S) -> None: async def test_list_prefix(self, store: S) -> None: """ Test that the `list_prefix` method works as intended. Given a prefix, it should return - all the keys in storage that start with this prefix. Keys should be returned with the shared - prefix removed. + all the keys in storage that start with this prefix. """ prefixes = ("", "a/", "a/b/", "a/b/c/") data = self.buffer_cls.from_bytes(b"") @@ -264,7 +278,7 @@ async def test_list_prefix(self, store: S) -> None: expected: tuple[str, ...] = () for key in store_dict: if key.startswith(prefix): - expected += (key.removeprefix(prefix),) + expected += (key,) expected = tuple(sorted(expected)) assert observed == expected diff --git a/tests/test_array.py b/tests/test_array.py index 285649490..b8af26133 100644 --- a/tests/test_array.py +++ b/tests/test_array.py @@ -51,6 +51,8 @@ def test_array_creation_existing_node( new_dtype = "float32" if exists_ok: + if not store.supports_deletes: + pytest.skip("store does not support deletes") arr_new = Array.create( spath / "extant", shape=new_shape, diff --git a/tests/test_group.py b/tests/test_group.py index bcdc6ff0d..6bacca488 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -290,6 +290,10 @@ def test_group_open(store: Store, zarr_format: ZarrFormat, exists_ok: bool) -> N with pytest.raises(ContainsGroupError): Group.from_store(store, attributes=attrs, zarr_format=zarr_format, exists_ok=exists_ok) else: + if not store.supports_deletes: + pytest.skip( + "Store does not support deletes but `exists_ok` is True, requiring deletes to override a group" + ) group_created_again = Group.from_store( store, attributes=new_attrs, zarr_format=zarr_format, exists_ok=exists_ok ) @@ -720,6 +724,8 @@ def test_group_creation_existing_node( new_attributes = {"new": True} if exists_ok: + if not store.supports_deletes: + pytest.skip("store does not support deletes but exists_ok is True") node_new = Group.from_store( spath / "extant", attributes=new_attributes, @@ -1092,7 +1098,9 @@ async def test_require_group(store: LocalStore | MemoryStore, zarr_format: ZarrF assert foo_group.attrs == {"foo": 100} # test that we can get the group using require_group and overwrite=True - foo_group = await root.require_group("foo", overwrite=True) + if store.supports_deletes: + foo_group = await root.require_group("foo", overwrite=True) + assert foo_group.attrs == {} _ = await foo_group.create_array( "bar", shape=(10,), dtype="uint8", chunk_shape=(2,), attributes={"foo": 100} @@ -1371,3 +1379,16 @@ def test_group_deprecated_positional_args(method: str) -> None: with pytest.warns(FutureWarning, match=r"Pass name=.*, data=.* as keyword args."): arr = getattr(root, method)("foo_like", data, **kwargs) assert arr.shape == data.shape + + +@pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) +def test_delitem_removes_children(store: Store, zarr_format: ZarrFormat) -> None: + # https://github.com/zarr-developers/zarr-python/issues/2191 + g1 = zarr.group(store=store, zarr_format=zarr_format) + g1.create_group("0") + g1.create_group("0/0") + arr = g1.create_array("0/0/0", shape=(1,)) + arr[:] = 1 + del g1["0"] + with pytest.raises(KeyError): + g1["0/0"] diff --git a/tests/test_store/test_logging.py b/tests/test_store/test_logging.py index 0258244c5..50db5c1c5 100644 --- a/tests/test_store/test_logging.py +++ b/tests/test_store/test_logging.py @@ -45,10 +45,15 @@ async def test_logging_store_counter(store: Store) -> None: arr[:] = 1 assert wrapped.counter["set"] == 2 - assert wrapped.counter["get"] == 0 # 1 if overwrite=False assert wrapped.counter["list"] == 0 assert wrapped.counter["list_dir"] == 0 assert wrapped.counter["list_prefix"] == 0 + if store.supports_deletes: + assert wrapped.counter["get"] == 0 # 1 if overwrite=False + assert wrapped.counter["delete_dir"] == 1 + else: + assert wrapped.counter["get"] == 1 + assert wrapped.counter["delete_dir"] == 0 async def test_with_mode(): diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index d05422ecd..8dee47449 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -14,7 +14,6 @@ from zarr.testing.store import StoreTests if TYPE_CHECKING: - from collections.abc import Coroutine from typing import Any @@ -65,9 +64,6 @@ def test_store_supports_partial_writes(self, store: ZipStore) -> None: def test_store_supports_listing(self, store: ZipStore) -> None: assert store.supports_listing - def test_delete(self, store: ZipStore) -> Coroutine[Any, Any, None]: - pass - def test_api_integration(self, store: ZipStore) -> None: root = zarr.open_group(store=store) @@ -103,4 +99,4 @@ async def test_with_mode(self, store: ZipStore) -> None: @pytest.mark.parametrize("mode", ["a", "w"]) async def test_store_open_mode(self, store_kwargs: dict[str, Any], mode: str) -> None: - super().test_store_open_mode(store_kwargs, mode) + await super().test_store_open_mode(store_kwargs, mode) From 4a097e1715a3da6b4e82d314d59bd1f067b85606 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Tue, 5 Nov 2024 06:04:35 +0100 Subject: [PATCH 07/14] Move BatchedCodecPipeline to zarr.core (#2086) * mv BatchedCodecPipeline to zarr.core * fix for tests * import codec_pipeline and buffer in zarr.core.__init__ --- src/zarr/codecs/__init__.py | 2 -- src/zarr/codecs/registry.py | 0 src/zarr/core/__init__.py | 4 ++++ src/zarr/{codecs/pipeline.py => core/codec_pipeline.py} | 0 src/zarr/core/config.py | 2 +- tests/test_config.py | 9 +++++---- 6 files changed, 10 insertions(+), 7 deletions(-) delete mode 100644 src/zarr/codecs/registry.py rename src/zarr/{codecs/pipeline.py => core/codec_pipeline.py} (100%) diff --git a/src/zarr/codecs/__init__.py b/src/zarr/codecs/__init__.py index dc6c3f915..e407d9489 100644 --- a/src/zarr/codecs/__init__.py +++ b/src/zarr/codecs/__init__.py @@ -9,7 +9,6 @@ from zarr.codecs.bytes import BytesCodec, Endian from zarr.codecs.crc32c_ import Crc32cCodec from zarr.codecs.gzip import GzipCodec -from zarr.codecs.pipeline import BatchedCodecPipeline from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation from zarr.codecs.transpose import TransposeCodec from zarr.codecs.vlen_utf8 import VLenBytesCodec, VLenUTF8Codec @@ -17,7 +16,6 @@ from zarr.core.metadata.v3 import DataType __all__ = [ - "BatchedCodecPipeline", "BloscCname", "BloscCodec", "BloscShuffle", diff --git a/src/zarr/codecs/registry.py b/src/zarr/codecs/registry.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/zarr/core/__init__.py b/src/zarr/core/__init__.py index e69de29bb..cbacfe342 100644 --- a/src/zarr/core/__init__.py +++ b/src/zarr/core/__init__.py @@ -0,0 +1,4 @@ +from __future__ import annotations + +from zarr.core.buffer import Buffer, NDBuffer # noqa: F401 +from zarr.core.codec_pipeline import BatchedCodecPipeline # noqa: F401 diff --git a/src/zarr/codecs/pipeline.py b/src/zarr/core/codec_pipeline.py similarity index 100% rename from src/zarr/codecs/pipeline.py rename to src/zarr/core/codec_pipeline.py diff --git a/src/zarr/core/config.py b/src/zarr/core/config.py index ec2c8c47a..29f5e139f 100644 --- a/src/zarr/core/config.py +++ b/src/zarr/core/config.py @@ -47,7 +47,7 @@ def reset(self) -> None: "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { - "path": "zarr.codecs.pipeline.BatchedCodecPipeline", + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", "batch_size": 1, }, "codecs": { diff --git a/tests/test_config.py b/tests/test_config.py index c4cf794c5..ddabffb46 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,9 +11,10 @@ from zarr import Array, zeros from zarr.abc.codec import CodecInput, CodecOutput, CodecPipeline from zarr.abc.store import ByteSetter, Store -from zarr.codecs import BatchedCodecPipeline, BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec +from zarr.codecs import BloscCodec, BytesCodec, Crc32cCodec, ShardingCodec from zarr.core.array_spec import ArraySpec from zarr.core.buffer import NDBuffer +from zarr.core.codec_pipeline import BatchedCodecPipeline from zarr.core.config import BadConfigError, config from zarr.core.indexing import SelectorTuple from zarr.registry import ( @@ -45,7 +46,7 @@ def test_config_defaults_set() -> None: "threading": {"max_workers": None}, "json_indent": 2, "codec_pipeline": { - "path": "zarr.codecs.pipeline.BatchedCodecPipeline", + "path": "zarr.core.codec_pipeline.BatchedCodecPipeline", "batch_size": 1, }, "buffer": "zarr.core.buffer.cpu.Buffer", @@ -96,8 +97,8 @@ def test_config_codec_pipeline_class(store: Store) -> None: # has default value assert get_pipeline_class().__name__ != "" - config.set({"codec_pipeline.name": "zarr.codecs.pipeline.BatchedCodecPipeline"}) - assert get_pipeline_class() == zarr.codecs.pipeline.BatchedCodecPipeline + config.set({"codec_pipeline.name": "zarr.core.codec_pipeline.BatchedCodecPipeline"}) + assert get_pipeline_class() == zarr.core.codec_pipeline.BatchedCodecPipeline _mock = Mock() From f092351da288efc18b336cf02a6e6ff8622e0cfc Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 5 Nov 2024 06:09:29 +0100 Subject: [PATCH 08/14] Apply assorted ruff/Pylint rules (PLR) (#2371) * Apply ruff/Pylint rule PLR1711 PLR1711 Useless `return` statement at end of function * Apply ruff/Pylint rule PLR2044 PLR2044 Line with empty comment * Apply ruff/Pylint rule PLR6104 PLR6104 Use `+=`, `*=`, `/=` to perform an augmented assignment directly --------- Co-authored-by: Joe Hamman --- src/zarr/abc/store.py | 1 - src/zarr/codecs/sharding.py | 2 +- src/zarr/core/indexing.py | 6 +++--- tests/test_codecs/test_sharding.py | 2 +- tests/test_sync.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index eefe04d50..11cb5ef40 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -284,7 +284,6 @@ async def _set_many(self, values: Iterable[tuple[str, Buffer]]) -> None: Insert multiple (key, value) pairs into storage. """ await gather(*starmap(self.set, values)) - return @property @abstractmethod diff --git a/src/zarr/codecs/sharding.py b/src/zarr/codecs/sharding.py index e7fd14ecb..5372d5ec5 100644 --- a/src/zarr/codecs/sharding.py +++ b/src/zarr/codecs/sharding.py @@ -252,7 +252,7 @@ def create_empty( def __setitem__(self, chunk_coords: ChunkCoords, value: Buffer) -> None: chunk_start = len(self.buf) chunk_length = len(value) - self.buf = self.buf + value + self.buf += value self.index.set_chunk_slice(chunk_coords, slice(chunk_start, chunk_start + chunk_length)) def __delitem__(self, chunk_coords: ChunkCoords) -> None: diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index 1873d5c83..eda244430 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -675,7 +675,7 @@ def check(a: npt.NDArray[Any]) -> Order: def wraparound_indices(x: npt.NDArray[Any], dim_len: int) -> None: loc_neg = x < 0 if np.any(loc_neg): - x[loc_neg] = x[loc_neg] + dim_len + x[loc_neg] += dim_len def boundscheck_indices(x: npt.NDArray[Any], dim_len: int) -> None: @@ -1000,8 +1000,8 @@ def __init__( if stop < 0: stop = dim_numchunks + stop - start = start * dim_chunk_size - stop = stop * dim_chunk_size + start *= dim_chunk_size + stop *= dim_chunk_size slice_ = slice(start, stop) else: diff --git a/tests/test_codecs/test_sharding.py b/tests/test_codecs/test_sharding.py index 85315c878..78f32fef0 100644 --- a/tests/test_codecs/test_sharding.py +++ b/tests/test_codecs/test_sharding.py @@ -229,7 +229,7 @@ def test_sharding_partial_overwrite( read_data = a[0:10, 0:10, 0:10] assert np.array_equal(data, read_data) - data = data + 10 + data += 10 a[:10, :10, :10] = data read_data = a[0:10, 0:10, 0:10] assert np.array_equal(data, read_data) diff --git a/tests/test_sync.py b/tests/test_sync.py index a6216a485..bff3837e2 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -147,7 +147,7 @@ def test_open_positional_args_deprecate(): zarr.open(store, "w", shape=(1,)) -@pytest.mark.parametrize("workers", [None, 1, 2]) # +@pytest.mark.parametrize("workers", [None, 1, 2]) def test_get_executor(clean_state, workers) -> None: with zarr.config.set({"threading.max_workers": workers}): e = _get_executor() From d1075de5c1ab5f08942b3c6096b37052216deaf4 Mon Sep 17 00:00:00 2001 From: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> Date: Tue, 5 Nov 2024 06:18:36 +0100 Subject: [PATCH 09/14] Zarr.save argument check (#2446) * add argument check * test * test zarr.save with multiple arrays * fix path for zarr.save args * improve test readability * fix typing * support other array types Co-authored-by: Tom Augspurger * support NDArrayLike * fix typing * fix typing * fix typing * format --------- Co-authored-by: Tom Augspurger --- src/zarr/api/asynchronous.py | 16 ++++++++++++++-- tests/test_api.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index cd8c3543c..434e9fdc2 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -10,6 +10,7 @@ from zarr.abc.store import Store from zarr.core.array import Array, AsyncArray, get_array_metadata +from zarr.core.buffer import NDArrayLike from zarr.core.common import ( JSON, AccessModeLiteral, @@ -31,7 +32,6 @@ from collections.abc import Iterable from zarr.abc.codec import Codec - from zarr.core.buffer import NDArrayLike from zarr.core.chunk_key_encodings import ChunkKeyEncoding # TODO: this type could use some more thought @@ -393,6 +393,8 @@ async def save_array( _handle_zarr_version_or_format(zarr_version=zarr_version, zarr_format=zarr_format) or _default_zarr_version() ) + if not isinstance(arr, NDArrayLike): + raise TypeError("arr argument must be numpy or other NDArrayLike array") mode = kwargs.pop("mode", None) store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) @@ -447,16 +449,26 @@ async def save_group( or _default_zarr_version() ) + for arg in args: + if not isinstance(arg, NDArrayLike): + raise TypeError( + "All arguments must be numpy or other NDArrayLike arrays (except store, path, storage_options, and zarr_format)" + ) + for k, v in kwargs.items(): + if not isinstance(v, NDArrayLike): + raise TypeError(f"Keyword argument '{k}' must be a numpy or other NDArrayLike array") + if len(args) == 0 and len(kwargs) == 0: raise ValueError("at least one array must be provided") aws = [] for i, arr in enumerate(args): + _path = f"{path}/arr_{i}" if path is not None else f"arr_{i}" aws.append( save_array( store, arr, zarr_format=zarr_format, - path=f"{path}/arr_{i}", + path=_path, storage_options=storage_options, ) ) diff --git a/tests/test_api.py b/tests/test_api.py index 5b62e3a2f..6d9eb0e88 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -132,6 +132,33 @@ async def test_open_group_unspecified_version( assert g2.metadata.zarr_format == zarr_format +@pytest.mark.parametrize("store", ["local", "memory", "zip"], indirect=["store"]) +@pytest.mark.parametrize("n_args", [10, 1, 0]) +@pytest.mark.parametrize("n_kwargs", [10, 1, 0]) +def test_save(store: Store, n_args: int, n_kwargs: int) -> None: + data = np.arange(10) + args = [np.arange(10) for _ in range(n_args)] + kwargs = {f"arg_{i}": data for i in range(n_kwargs)} + + if n_kwargs == 0 and n_args == 0: + with pytest.raises(ValueError): + save(store) + elif n_args == 1 and n_kwargs == 0: + save(store, *args) + array = open(store) + assert isinstance(array, Array) + assert_array_equal(array[:], data) + else: + save(store, *args, **kwargs) # type: ignore[arg-type] + group = open(store) + assert isinstance(group, Group) + for array in group.array_values(): + assert_array_equal(array[:], data) + for k in kwargs.keys(): + assert k in group + assert group.nmembers() == n_args + n_kwargs + + def test_save_errors() -> None: with pytest.raises(ValueError): # no arrays provided @@ -142,6 +169,10 @@ def test_save_errors() -> None: with pytest.raises(ValueError): # no arrays provided save("data/group.zarr") + with pytest.raises(TypeError): + # mode is no valid argument and would get handled as an array + a = np.arange(10) + zarr.save("data/example.zarr", a, mode="w") def test_open_with_mode_r(tmp_path: pathlib.Path) -> None: From ca46bab505f2bf95ea4c575a70c1716a53565394 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 5 Nov 2024 07:57:43 +0100 Subject: [PATCH 10/14] Add more ruff rules (#2460) * Enforce ruff/flake8-executable rules (EXE) * Enforce ruff/flake8-future-annotations rules (FA) * Enforce ruff/flake8-logging rules (LOG) * Apply ruff/flake8-pie rule PIE790 PIE790 Unnecessary `pass` statement PIE790 Unnecessary `...` literal * Apply ruff/flake8-pie rule PIE800 PIE800 Unnecessary spread `**` * Apply ruff/flake8-pie rule PIE808 PIE808 Unnecessary `start` argument in `range` * Enforce ruff/flake8-pie rules (PIE) * Apply ruff preview rule RUF022 RUF022 `__all__` is not sorted * Sort pyproject.toml * Apply ruff/flake8-simplify rule SIM102 SIM102 Use a single `if` statement instead of nested `if` statements * Apply ruff/flake8-simplify rule SIM114 SIM114 Combine `if` branches using logical `or` operator * Apply ruff/flake8-simplify rule SIM118 SIM118 Use `key in dict` instead of `key in dict.keys()` * Enforce ruff/flake8-simplify rules (SIM) * Enforce ruff/flake8-slots rules (SLOT) --- pyproject.toml | 11 +- src/zarr/abc/codec.py | 1 - src/zarr/abc/metadata.py | 1 - src/zarr/api/asynchronous.py | 5 +- src/zarr/codecs/gzip.py | 2 +- src/zarr/core/chunk_grids.py | 2 +- src/zarr/core/group.py | 2 +- src/zarr/core/indexing.py | 11 +- src/zarr/core/metadata/__init__.py | 6 +- src/zarr/core/metadata/v3.py | 6 +- src/zarr/core/strings.py | 7 +- tests/test_api.py | 2 +- tests/test_codecs/test_codecs.py | 1 - tests/test_metadata/test_consolidated.py | 134 +++++++++-------------- 14 files changed, 84 insertions(+), 107 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bb9fa4df8..dc0e4730e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,19 +269,25 @@ extend-exclude = [ extend-select = [ "ANN", # flake8-annotations "B", # flake8-bugbear + "EXE", # flake8-executable "C4", # flake8-comprehensions + "FA", # flake8-future-annotations "FLY", # flynt "FURB", # refurb "G", # flake8-logging-format "I", # isort "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging "PERF", # Perflint + "PIE", # flake8-pie "PGH", # pygrep-hooks "PT", # flake8-pytest-style "PYI", # flake8-pyi - "RSE", # flake8-raise "RET", # flake8-return + "RSE", # flake8-raise "RUF", + "SIM", # flake8-simplify + "SLOT", # flake8-slots "TCH", # flake8-type-checking "TRY", # tryceratops "UP", # pyupgrade @@ -298,6 +304,7 @@ ignore = [ "RET505", "RET506", "RUF005", + "SIM108", "TRY003", "UP027", # deprecated "UP038", # https://github.com/astral-sh/ruff/issues/7871 @@ -319,7 +326,7 @@ ignore = [ ] [tool.ruff.lint.extend-per-file-ignores] -"tests/**" = ["ANN001", "ANN201"] +"tests/**" = ["ANN001", "ANN201", "RUF029", "SIM117", "SIM300"] [tool.mypy] python_version = "3.11" diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py index f27152e84..fabd042db 100644 --- a/src/zarr/abc/codec.py +++ b/src/zarr/abc/codec.py @@ -106,7 +106,6 @@ def validate(self, *, shape: ChunkCoords, dtype: np.dtype[Any], chunk_grid: Chun chunk_grid : ChunkGrid The array chunk grid """ - ... async def _decode_single(self, chunk_data: CodecOutput, chunk_spec: ArraySpec) -> CodecInput: raise NotImplementedError diff --git a/src/zarr/abc/metadata.py b/src/zarr/abc/metadata.py index 291ceb459..a56f98664 100644 --- a/src/zarr/abc/metadata.py +++ b/src/zarr/abc/metadata.py @@ -42,6 +42,5 @@ def from_dict(cls, data: dict[str, JSON]) -> Self: """ Create an instance of the model from a dictionary """ - ... return cls(**data) diff --git a/src/zarr/api/asynchronous.py b/src/zarr/api/asynchronous.py index 434e9fdc2..40f9b8d5f 100644 --- a/src/zarr/api/asynchronous.py +++ b/src/zarr/api/asynchronous.py @@ -878,9 +878,8 @@ async def create( warnings.warn("meta_array is not yet implemented", RuntimeWarning, stacklevel=2) mode = kwargs.pop("mode", None) - if mode is None: - if not isinstance(store, Store | StorePath): - mode = "a" + if mode is None and not isinstance(store, Store | StorePath): + mode = "a" store_path = await make_store_path(store, path=path, mode=mode, storage_options=storage_options) diff --git a/src/zarr/codecs/gzip.py b/src/zarr/codecs/gzip.py index c0ad5e138..b6e693148 100644 --- a/src/zarr/codecs/gzip.py +++ b/src/zarr/codecs/gzip.py @@ -21,7 +21,7 @@ def parse_gzip_level(data: JSON) -> int: if not isinstance(data, (int)): raise TypeError(f"Expected int, got {type(data)}") - if data not in range(0, 10): + if data not in range(10): raise ValueError( f"Expected an integer from the inclusive range (0, 9). Got {data} instead." ) diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py index ed7f8a1f4..afecc6824 100644 --- a/src/zarr/core/chunk_grids.py +++ b/src/zarr/core/chunk_grids.py @@ -182,7 +182,7 @@ def to_dict(self) -> dict[str, JSON]: def all_chunk_coords(self, array_shape: ChunkCoords) -> Iterator[ChunkCoords]: return itertools.product( - *(range(0, ceildiv(s, c)) for s, c in zip(array_shape, self.chunk_shape, strict=False)) + *(range(ceildiv(s, c)) for s, c in zip(array_shape, self.chunk_shape, strict=False)) ) def get_nchunks(self, array_shape: ChunkCoords) -> int: diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 86cf191ca..1054ba980 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -1225,7 +1225,7 @@ def _members_consolidated( # we kind of just want the top-level keys. if consolidated_metadata is not None: - for key in consolidated_metadata.metadata.keys(): + for key in consolidated_metadata.metadata: obj = self._getitem_consolidated( self.store_path, key, prefix=self.name ) # Metadata -> Group/Array diff --git a/src/zarr/core/indexing.py b/src/zarr/core/indexing.py index eda244430..723dadfb4 100644 --- a/src/zarr/core/indexing.py +++ b/src/zarr/core/indexing.py @@ -241,12 +241,13 @@ def is_pure_fancy_indexing(selection: Any, ndim: int) -> bool: # is mask selection return True - if ndim == 1: - if is_integer_list(selection) or is_integer_array(selection) or is_bool_list(selection): - return True + if ndim == 1 and ( + is_integer_list(selection) or is_integer_array(selection) or is_bool_list(selection) + ): + return True - # if not, we go through the normal path below, because a 1-tuple - # of integers is also allowed. + # if not, we go through the normal path below, because a 1-tuple + # of integers is also allowed. no_slicing = ( isinstance(selection, tuple) and len(selection) == ndim diff --git a/src/zarr/core/metadata/__init__.py b/src/zarr/core/metadata/__init__.py index f4374d9ab..43b5ec98f 100644 --- a/src/zarr/core/metadata/__init__.py +++ b/src/zarr/core/metadata/__init__.py @@ -8,10 +8,10 @@ T_ArrayMetadata = TypeVar("T_ArrayMetadata", ArrayV2Metadata, ArrayV3Metadata) __all__ = [ - "ArrayV2Metadata", - "ArrayV3Metadata", "ArrayMetadata", "ArrayMetadataDict", - "ArrayV3MetadataDict", + "ArrayV2Metadata", "ArrayV2MetadataDict", + "ArrayV3Metadata", + "ArrayV3MetadataDict", ] diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py index 7a38e9fd7..6ea9ed69f 100644 --- a/src/zarr/core/metadata/v3.py +++ b/src/zarr/core/metadata/v3.py @@ -481,9 +481,9 @@ def parse_fill_value( except (ValueError, OverflowError, TypeError) as e: raise ValueError(f"fill value {fill_value!r} is not valid for dtype {data_type}") from e # Check if the value is still representable by the dtype - if fill_value == "NaN" and np.isnan(casted_value): - pass - elif fill_value in ["Infinity", "-Infinity"] and not np.isfinite(casted_value): + if (fill_value == "NaN" and np.isnan(casted_value)) or ( + fill_value in ["Infinity", "-Infinity"] and not np.isfinite(casted_value) + ): pass elif np_dtype.kind == "f": # float comparison is not exact, especially when dtype None: assert isinstance(group, Group) for array in group.array_values(): assert_array_equal(array[:], data) - for k in kwargs.keys(): + for k in kwargs: assert k in group assert group.nmembers() == n_args + n_kwargs diff --git a/tests/test_codecs/test_codecs.py b/tests/test_codecs/test_codecs.py index 0f2f89291..dfb8e1c59 100644 --- a/tests/test_codecs/test_codecs.py +++ b/tests/test_codecs/test_codecs.py @@ -56,7 +56,6 @@ def test_sharding_pickle() -> None: """ Test that sharding codecs can be pickled """ - pass @pytest.mark.parametrize("store", ["local", "memory"], indirect=["store"]) diff --git a/tests/test_metadata/test_consolidated.py b/tests/test_metadata/test_consolidated.py index c0218602f..d9143d09d 100644 --- a/tests/test_metadata/test_consolidated.py +++ b/tests/test_metadata/test_consolidated.py @@ -87,31 +87,27 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: metadata={ "air": ArrayV3Metadata.from_dict( { - **{ - "shape": (1, 2, 3), - "chunk_grid": { - "configuration": {"chunk_shape": (1, 2, 3)}, - "name": "regular", - }, + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { - **{ - "shape": (1,), - "chunk_grid": { - "configuration": {"chunk_shape": (1,)}, - "name": "regular", - }, + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", }, **array_metadata, } ), "lon": ArrayV3Metadata.from_dict( { - **{"shape": (2,)}, + "shape": (2,), "chunk_grid": { "configuration": {"chunk_shape": (2,)}, "name": "regular", @@ -121,12 +117,10 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: ), "time": ArrayV3Metadata.from_dict( { - **{ - "shape": (3,), - "chunk_grid": { - "configuration": {"chunk_shape": (3,)}, - "name": "regular", - }, + "shape": (3,), + "chunk_grid": { + "configuration": {"chunk_shape": (3,)}, + "name": "regular", }, **array_metadata, } @@ -138,13 +132,11 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: "array": ArrayV3Metadata.from_dict( { **array_metadata, - **{ - "attributes": {"key": "child"}, - "shape": (4, 4), - "chunk_grid": { - "configuration": {"chunk_shape": (4, 4)}, - "name": "regular", - }, + "attributes": {"key": "child"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", }, } ), @@ -162,15 +154,11 @@ async def test_consolidated(self, memory_store_with_hierarchy: Store) -> None: "array": ArrayV3Metadata.from_dict( { **array_metadata, - **{ - "attributes": {"key": "grandchild"}, - "shape": (4, 4), - "chunk_grid": { - "configuration": { - "chunk_shape": (4, 4) - }, - "name": "regular", - }, + "attributes": {"key": "grandchild"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", }, } ), @@ -243,31 +231,27 @@ def test_consolidated_sync(self, memory_store): metadata={ "air": ArrayV3Metadata.from_dict( { - **{ - "shape": (1, 2, 3), - "chunk_grid": { - "configuration": {"chunk_shape": (1, 2, 3)}, - "name": "regular", - }, + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { - **{ - "shape": (1,), - "chunk_grid": { - "configuration": {"chunk_shape": (1,)}, - "name": "regular", - }, + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", }, **array_metadata, } ), "lon": ArrayV3Metadata.from_dict( { - **{"shape": (2,)}, + "shape": (2,), "chunk_grid": { "configuration": {"chunk_shape": (2,)}, "name": "regular", @@ -277,12 +261,10 @@ def test_consolidated_sync(self, memory_store): ), "time": ArrayV3Metadata.from_dict( { - **{ - "shape": (3,), - "chunk_grid": { - "configuration": {"chunk_shape": (3,)}, - "name": "regular", - }, + "shape": (3,), + "chunk_grid": { + "configuration": {"chunk_shape": (3,)}, + "name": "regular", }, **array_metadata, } @@ -357,24 +339,20 @@ def test_flatten(self): metadata={ "air": ArrayV3Metadata.from_dict( { - **{ - "shape": (1, 2, 3), - "chunk_grid": { - "configuration": {"chunk_shape": (1, 2, 3)}, - "name": "regular", - }, + "shape": (1, 2, 3), + "chunk_grid": { + "configuration": {"chunk_shape": (1, 2, 3)}, + "name": "regular", }, **array_metadata, } ), "lat": ArrayV3Metadata.from_dict( { - **{ - "shape": (1,), - "chunk_grid": { - "configuration": {"chunk_shape": (1,)}, - "name": "regular", - }, + "shape": (1,), + "chunk_grid": { + "configuration": {"chunk_shape": (1,)}, + "name": "regular", }, **array_metadata, } @@ -386,13 +364,11 @@ def test_flatten(self): "array": ArrayV3Metadata.from_dict( { **array_metadata, - **{ - "attributes": {"key": "child"}, - "shape": (4, 4), - "chunk_grid": { - "configuration": {"chunk_shape": (4, 4)}, - "name": "regular", - }, + "attributes": {"key": "child"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", }, } ), @@ -403,13 +379,11 @@ def test_flatten(self): "array": ArrayV3Metadata.from_dict( { **array_metadata, - **{ - "attributes": {"key": "grandchild"}, - "shape": (4, 4), - "chunk_grid": { - "configuration": {"chunk_shape": (4, 4)}, - "name": "regular", - }, + "attributes": {"key": "grandchild"}, + "shape": (4, 4), + "chunk_grid": { + "configuration": {"chunk_shape": (4, 4)}, + "name": "regular", }, } ) From f05413e30c0f5c35a7fb3d02e0d2f53f7e7c2bfe Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 5 Nov 2024 18:46:59 +0100 Subject: [PATCH 11/14] `AsyncGenerator` should not yield `None` (#2464) The `None` was added in 0e035fb, let's try to remove it. --- src/zarr/abc/store.py | 9 +++------ src/zarr/storage/local.py | 6 +++--- src/zarr/storage/logging.py | 6 +++--- src/zarr/storage/memory.py | 6 +++--- src/zarr/storage/remote.py | 6 +++--- src/zarr/storage/zip.py | 6 +++--- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/zarr/abc/store.py b/src/zarr/abc/store.py index 11cb5ef40..c02dc9d00 100644 --- a/src/zarr/abc/store.py +++ b/src/zarr/abc/store.py @@ -329,17 +329,16 @@ def supports_listing(self) -> bool: ... @abstractmethod - def list(self) -> AsyncGenerator[str, None]: + def list(self) -> AsyncGenerator[str]: """Retrieve all keys in the store. Returns ------- AsyncGenerator[str, None] """ - ... @abstractmethod - def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + def list_prefix(self, prefix: str) -> AsyncGenerator[str]: """ Retrieve all keys in the store that begin with a given prefix. Keys are returned relative to the root of the store. @@ -352,10 +351,9 @@ def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: ------- AsyncGenerator[str, None] """ - ... @abstractmethod - def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + def list_dir(self, prefix: str) -> AsyncGenerator[str]: """ Retrieve all keys and prefixes with a given prefix and which do not contain the character “/” after the given prefix. @@ -368,7 +366,6 @@ def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: ------- AsyncGenerator[str, None] """ - ... async def delete_dir(self, prefix: str) -> None: """ diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index 331c9857c..c1c711e26 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -217,14 +217,14 @@ async def exists(self, key: str) -> bool: path = self.root / key return await asyncio.to_thread(path.is_file) - async def list(self) -> AsyncGenerator[str, None]: + async def list(self) -> AsyncGenerator[str]: # docstring inherited to_strip = self.root.as_posix() + "/" for p in list(self.root.rglob("*")): if p.is_file(): yield p.as_posix().replace(to_strip, "") - async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited to_strip = self.root.as_posix() + "/" prefix = prefix.rstrip("/") @@ -232,7 +232,7 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: if p.is_file(): yield p.as_posix().replace(to_strip, "") - async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited base = self.root / prefix to_strip = str(base) + "/" diff --git a/src/zarr/storage/logging.py b/src/zarr/storage/logging.py index be259579e..eb65daddb 100644 --- a/src/zarr/storage/logging.py +++ b/src/zarr/storage/logging.py @@ -204,19 +204,19 @@ async def set_partial_values( with self.log(keys): return await self._store.set_partial_values(key_start_values=key_start_values) - async def list(self) -> AsyncGenerator[str, None]: + async def list(self) -> AsyncGenerator[str]: # docstring inherited with self.log(): async for key in self._store.list(): yield key - async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited with self.log(prefix): async for key in self._store.list_prefix(prefix=prefix): yield key - async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited with self.log(prefix): async for key in self._store.list_dir(prefix=prefix): diff --git a/src/zarr/storage/memory.py b/src/zarr/storage/memory.py index fa4ede2a8..df33876df 100644 --- a/src/zarr/storage/memory.py +++ b/src/zarr/storage/memory.py @@ -147,19 +147,19 @@ async def set_partial_values(self, key_start_values: Iterable[tuple[str, int, by # docstring inherited raise NotImplementedError - async def list(self) -> AsyncGenerator[str, None]: + async def list(self) -> AsyncGenerator[str]: # docstring inherited for key in self._store_dict: yield key - async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited # note: we materialize all dict keys into a list here so we can mutate the dict in-place (e.g. in delete_prefix) for key in list(self._store_dict): if key.startswith(prefix): yield key - async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited prefix = prefix.rstrip("/") diff --git a/src/zarr/storage/remote.py b/src/zarr/storage/remote.py index ca7a010bd..d471c85f5 100644 --- a/src/zarr/storage/remote.py +++ b/src/zarr/storage/remote.py @@ -322,13 +322,13 @@ async def set_partial_values( # docstring inherited raise NotImplementedError - async def list(self) -> AsyncGenerator[str, None]: + async def list(self) -> AsyncGenerator[str]: # docstring inherited allfiles = await self.fs._find(self.path, detail=False, withdirs=False) for onefile in (a.replace(self.path + "/", "") for a in allfiles): yield onefile - async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited prefix = f"{self.path}/{prefix.rstrip('/')}" try: @@ -338,7 +338,7 @@ async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: for onefile in (a.replace(prefix + "/", "") for a in allfiles): yield onefile.removeprefix(self.path).removeprefix("/") - async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited for onefile in await self.fs._find( f"{self.path}/{prefix}", detail=False, maxdepth=None, withdirs=False diff --git a/src/zarr/storage/zip.py b/src/zarr/storage/zip.py index cf9b338cf..a45cc1672 100644 --- a/src/zarr/storage/zip.py +++ b/src/zarr/storage/zip.py @@ -234,19 +234,19 @@ async def exists(self, key: str) -> bool: else: return True - async def list(self) -> AsyncGenerator[str, None]: + async def list(self) -> AsyncGenerator[str]: # docstring inherited with self._lock: for key in self._zf.namelist(): yield key - async def list_prefix(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited async for key in self.list(): if key.startswith(prefix): yield key - async def list_dir(self, prefix: str) -> AsyncGenerator[str, None]: + async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited prefix = prefix.rstrip("/") From 473308a9765eea053a2041f23b1da04da427bc1b Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Tue, 5 Nov 2024 23:07:00 +0100 Subject: [PATCH 12/14] use as_posix correctly for str and for listdir (#2467) * use as_posix for str and for listdir * release the windows workflows * unbreak gha --- .github/workflows/test.yml | 17 ++++++++--------- src/zarr/storage/local.py | 6 ++---- tests/test_store/test_local.py | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2866ed4b8..c388ba31d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,15 +33,14 @@ jobs: numpy-version: '2.1' dependency-set: 'optional' os: 'macos-latest' - # https://github.com/zarr-developers/zarr-python/issues/2438 - # - python-version: '3.11' - # numpy-version: '1.25' - # dependency-set: 'optional' - # os: 'windows-latest' - # - python-version: '3.13' - # numpy-version: '2.1' - # dependency-set: 'optional' - # os: 'windows-latest' + - python-version: '3.11' + numpy-version: '1.25' + dependency-set: 'optional' + os: 'windows-latest' + - python-version: '3.13' + numpy-version: '2.1' + dependency-set: 'optional' + os: 'windows-latest' runs-on: ${{ matrix.os }} steps: diff --git a/src/zarr/storage/local.py b/src/zarr/storage/local.py index c1c711e26..b8b67003c 100644 --- a/src/zarr/storage/local.py +++ b/src/zarr/storage/local.py @@ -132,7 +132,7 @@ def with_mode(self, mode: AccessModeLiteral) -> Self: return type(self)(root=self.root, mode=mode) def __str__(self) -> str: - return f"file://{self.root}" + return f"file://{self.root.as_posix()}" def __repr__(self) -> str: return f"LocalStore({str(self)!r})" @@ -235,11 +235,9 @@ async def list_prefix(self, prefix: str) -> AsyncGenerator[str]: async def list_dir(self, prefix: str) -> AsyncGenerator[str]: # docstring inherited base = self.root / prefix - to_strip = str(base) + "/" - try: key_iter = base.iterdir() for key in key_iter: - yield key.as_posix().replace(to_strip, "") + yield key.relative_to(base).as_posix() except (FileNotFoundError, NotADirectoryError): pass diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index 5352e3520..79236e547 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -31,7 +31,7 @@ def store_kwargs(self, tmpdir) -> dict[str, str]: return {"root": str(tmpdir), "mode": "r+"} def test_store_repr(self, store: LocalStore) -> None: - assert str(store) == f"file://{store.root!s}" + assert str(store) == f"file://{store.root.as_posix()!s}" def test_store_supports_writes(self, store: LocalStore) -> None: assert store.supports_writes From 680142f6f370d862f305a9b37f3c0ce2ce802e76 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Tue, 5 Nov 2024 23:07:15 +0100 Subject: [PATCH 13/14] Missing mandatory keyword argument (#2465) The keyword argument `shape` is mandatory. --- src/zarr/core/group.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/zarr/core/group.py b/src/zarr/core/group.py index 1054ba980..9a54b346b 100644 --- a/src/zarr/core/group.py +++ b/src/zarr/core/group.py @@ -967,7 +967,7 @@ async def create_array( @deprecated("Use AsyncGroup.create_array instead.") async def create_dataset( - self, name: str, **kwargs: Any + self, name: str, *, shape: ShapeLike, **kwargs: Any ) -> AsyncArray[ArrayV2Metadata] | AsyncArray[ArrayV3Metadata]: """Create an array. @@ -988,7 +988,7 @@ async def create_dataset( .. deprecated:: 3.0.0 The h5py compatibility methods will be removed in 3.1.0. Use `AsyncGroup.create_array` instead. """ - return await self.create_array(name, **kwargs) + return await self.create_array(name, shape=shape, **kwargs) @deprecated("Use AsyncGroup.require_array instead.") async def require_dataset( @@ -1666,7 +1666,7 @@ def create_dataset(self, name: str, **kwargs: Any) -> Array: return Array(self._sync(self._async_group.create_dataset(name, **kwargs))) @deprecated("Use Group.require_array instead.") - def require_dataset(self, name: str, **kwargs: Any) -> Array: + def require_dataset(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: """Obtain an array, creating if it doesn't exist. Arrays are known as "datasets" in HDF5 terminology. For compatibility @@ -1688,9 +1688,9 @@ def require_dataset(self, name: str, **kwargs: Any) -> Array: .. deprecated:: 3.0.0 The h5py compatibility methods will be removed in 3.1.0. Use `Group.require_array` instead. """ - return Array(self._sync(self._async_group.require_array(name, **kwargs))) + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) - def require_array(self, name: str, **kwargs: Any) -> Array: + def require_array(self, name: str, *, shape: ShapeLike, **kwargs: Any) -> Array: """Obtain an array, creating if it doesn't exist. @@ -1707,7 +1707,7 @@ def require_array(self, name: str, **kwargs: Any) -> Array: ------- a : Array """ - return Array(self._sync(self._async_group.require_array(name, **kwargs))) + return Array(self._sync(self._async_group.require_array(name, shape=shape, **kwargs))) @_deprecate_positional_args def empty(self, *, name: str, shape: ChunkCoords, **kwargs: Any) -> Array: From 498cb788b29900183dbcfd7633c48afd5cee8a95 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 7 Nov 2024 22:01:44 +0100 Subject: [PATCH 14/14] Remove useless `{...!s}` (#2471) --- src/zarr/storage/logging.py | 4 ++-- tests/test_store/test_local.py | 2 +- tests/test_store/test_zip.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zarr/storage/logging.py b/src/zarr/storage/logging.py index eb65daddb..d3e55c068 100644 --- a/src/zarr/storage/logging.py +++ b/src/zarr/storage/logging.py @@ -55,7 +55,7 @@ def _configure_logger( self, log_level: str = "DEBUG", log_handler: logging.Handler | None = None ) -> None: self.log_level = log_level - self.logger = logging.getLogger(f"LoggingStore({self._store!s})") + self.logger = logging.getLogger(f"LoggingStore({self._store})") self.logger.setLevel(log_level) if not self.logger.hasHandlers(): @@ -147,7 +147,7 @@ async def clear(self) -> None: return await self._store.clear() def __str__(self) -> str: - return f"logging-{self._store!s}" + return f"logging-{self._store}" def __repr__(self) -> str: return f"LoggingStore({repr(self._store)!r})" diff --git a/tests/test_store/test_local.py b/tests/test_store/test_local.py index 79236e547..239364a06 100644 --- a/tests/test_store/test_local.py +++ b/tests/test_store/test_local.py @@ -31,7 +31,7 @@ def store_kwargs(self, tmpdir) -> dict[str, str]: return {"root": str(tmpdir), "mode": "r+"} def test_store_repr(self, store: LocalStore) -> None: - assert str(store) == f"file://{store.root.as_posix()!s}" + assert str(store) == f"file://{store.root.as_posix()}" def test_store_supports_writes(self, store: LocalStore) -> None: assert store.supports_writes diff --git a/tests/test_store/test_zip.py b/tests/test_store/test_zip.py index 8dee47449..8f7398931 100644 --- a/tests/test_store/test_zip.py +++ b/tests/test_store/test_zip.py @@ -53,7 +53,7 @@ async def test_not_writable_store_raises(self, store_kwargs: dict[str, Any]) -> await store.set("foo", cpu.Buffer.from_bytes(b"bar")) def test_store_repr(self, store: ZipStore) -> None: - assert str(store) == f"zip://{store.path!s}" + assert str(store) == f"zip://{store.path}" def test_store_supports_writes(self, store: ZipStore) -> None: assert store.supports_writes