Skip to content

Commit

Permalink
patches for remote store
Browse files Browse the repository at this point in the history
  • Loading branch information
jhamman committed May 30, 2024
1 parent 177a48a commit 0672eed
Show file tree
Hide file tree
Showing 4 changed files with 34 additions and 30 deletions.
24 changes: 0 additions & 24 deletions src/zarr/store/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,27 +79,3 @@ def make_store_path(store_like: StoreLike | None, *, mode: OpenMode | None = Non
assert mode is not None
return StorePath(LocalStore(Path(store_like), mode=mode))
raise TypeError


def _normalize_interval_index(
data: Buffer, interval: None | tuple[int | None, int | None]
) -> tuple[int, int]:
"""
Convert an implicit interval into an explicit start and length
"""
if interval is None:
start = 0
length = len(data)
else:
maybe_start, maybe_len = interval
if maybe_start is None:
start = 0
else:
start = maybe_start

if maybe_len is None:
length = len(data) - start
else:
length = maybe_len

return (start, length)
2 changes: 1 addition & 1 deletion src/zarr/store/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from zarr.abc.store import Store
from zarr.buffer import Buffer
from zarr.common import OpenMode, concurrent_map
from zarr.store.core import _normalize_interval_index
from zarr.store.util import _normalize_interval_index


# TODO: this store could easily be extended to wrap any MutableMapping store from v2
Expand Down
13 changes: 8 additions & 5 deletions src/zarr/store/remote.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class RemoteStore(Store):
def __init__(
self,
url: UPath | str,
*,
mode: OpenMode = "r",
allowed_exceptions: tuple[type[Exception], ...] = (
FileNotFoundError,
Expand Down Expand Up @@ -81,10 +82,12 @@ async def get(
path = _dereference_path(self.path, key)

try:
return await (
self._fs._cat_file(path, start=byte_range[0], end=byte_range[1])
if byte_range
else self._fs._cat_file(path)
return Buffer.from_bytes(
await (
self._fs._cat_file(path, start=byte_range[0], end=byte_range[1])
if byte_range
else self._fs._cat_file(path)
)
)
# dear mypy: this is indeed defined as a tuple of exceptions
except self.exceptions: # type: ignore
Expand All @@ -100,7 +103,7 @@ async def set(
# write data
if byte_range:
raise NotImplementedError
await self._fs._pipe_file(path, value)
await self._fs._pipe_file(path, value.to_bytes())

async def delete(self, key: str) -> None:
path = _dereference_path(self.path, key)
Expand Down
25 changes: 25 additions & 0 deletions src/zarr/store/util.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
from zarr.buffer import Buffer


def _normalize_interval_index(
data: Buffer, interval: None | tuple[int | None, int | None]
) -> tuple[int, int]:
"""
Convert an implicit interval into an explicit start and length
"""
if interval is None:
start = 0
length = len(data)
else:
maybe_start, maybe_len = interval
if maybe_start is None:
start = 0
else:
start = maybe_start

if maybe_len is None:
length = len(data) - start
else:
length = maybe_len

return (start, length)

0 comments on commit 0672eed

Please sign in to comment.