diff --git a/.isort.cfg b/.isort.cfg index 80dec68c..cbe16a06 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -1,5 +1,5 @@ [settings] -known_third_party = cv2,dask,napari,numpy,pytest,requests,scipy,setuptools,skimage,vispy,zarr +known_third_party = cv2,dask,numpy,pytest,requests,scipy,setuptools,skimage,vispy,zarr multi_line_output=6 include_trailing_comma=False force_grid_wrap=0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 91dfde03..6ba4b2bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,11 +44,11 @@ repos: - --autofix - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 additional_dependencies: [ - flake8-blind-except, + # flake8-blind-except, FIXME flake8-builtins, flake8-rst-docstrings, flake8-logging-format, @@ -74,7 +74,7 @@ repos: --disallow-untyped-defs, --ignore-missing-imports, ] - exclude: tests/*|setup.py + exclude: tests/|setup.py - repo: https://github.com/adrienverge/yamllint.git rev: v1.24.2 diff --git a/environment.yml b/environment.yml index f6c31381..19750cad 100644 --- a/environment.yml +++ b/environment.yml @@ -1,10 +1,10 @@ -#name: z +name: z channels: - - defaults - ome - conda-forge + - defaults dependencies: - - napari + - pyqt - flake8 - ipython - mypy @@ -20,6 +20,7 @@ dependencies: - xarray - zarr >= 2.4.0 - pip: + - napari - pre-commit - pytest-qt # python.app -- only install on OSX: diff --git a/ome_zarr/data.py b/ome_zarr/data.py index b467f564..daf808e5 100644 --- a/ome_zarr/data.py +++ b/ome_zarr/data.py @@ -12,6 +12,7 @@ from skimage.segmentation import clear_border from .scale import Scaler +from .writer import write_multiscale CHANNEL_DIMENSION = 1 @@ -92,17 +93,6 @@ def rgb_to_5d(pixels: np.ndarray) -> List: return video -def write_multiscale(pyramid: List, group: zarr.Group) -> None: - """Write a pyramid with multiscale metadata to disk.""" - paths = [] - for path, dataset in enumerate(pyramid): - group.create_dataset(str(path), data=pyramid[path]) - paths.append({"path": str(path)}) - - multiscales = [{"version": "0.1", "datasets": paths}] - group.attrs["multiscales"] = multiscales - - def create_zarr( zarr_directory: str, method: Callable[..., Tuple[List, List]] = coins, diff --git a/ome_zarr/io.py b/ome_zarr/io.py index 820e4e3a..1816a7b5 100644 --- a/ome_zarr/io.py +++ b/ome_zarr/io.py @@ -13,6 +13,7 @@ import dask.array as da import requests +import zarr from .types import JSONDict @@ -106,8 +107,12 @@ class LocalZarrLocation(BaseZarrLocation): Uses the :module:`json` library for loading JSON from disk. """ - def __init__(self, path: Path) -> None: + def __init__(self, path: Path, mode: str = "r") -> None: self.__path: Path = path + self.mode = mode + if mode in ("w", "a") and not self.__path.exists(): + _ = zarr.DirectoryStore(self.basename()) + LOGGER.debug("Created DirectoryStore %s", self.basename()) super().__init__() def basename(self) -> str: @@ -132,8 +137,7 @@ def get_json(self, subpath: str) -> JSONDict: If a file does not exist, an empty response is returned rather than an exception. """ - filename = self.__path / subpath - + filename = self.subpath(subpath) if not os.path.exists(filename): LOGGER.debug(f"{filename} does not exist") return {} @@ -189,7 +193,7 @@ def get_json(self, subpath: str) -> JSONDict: return {} -def parse_url(path: str) -> Optional[BaseZarrLocation]: +def parse_url(path: str, mode: str = "r") -> Optional[BaseZarrLocation]: """Convert a path string or URL to a BaseZarrLocation subclass. >>> parse_url('does-not-exist') @@ -199,12 +203,15 @@ def parse_url(path: str) -> Optional[BaseZarrLocation]: return LocalZarrLocation(Path(path)) else: result = urlparse(path) - zarr: Optional[BaseZarrLocation] = None + zarr_loc: Optional[BaseZarrLocation] = None if result.scheme in ("", "file"): # Strips 'file://' if necessary - zarr = LocalZarrLocation(Path(result.path)) + zarr_loc = LocalZarrLocation(Path(result.path), mode=mode) + else: - zarr = RemoteZarrLocation(path) - if zarr.exists(): - return zarr + if mode != "r": + raise ValueError("Remote locations are read only") + zarr_loc = RemoteZarrLocation(path) + if zarr_loc.exists() or (mode in ("a", "w")): + return zarr_loc return None diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index b44794ff..2ac80976 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -354,6 +354,7 @@ def __init__(self, node: Node) -> None: node.metadata["visible"] = visibles node.metadata["contrast_limits"] = contrast_limits node.metadata["colormap"] = colormaps + except Exception as e: LOGGER.error(f"failed to parse metadata: {e}") diff --git a/ome_zarr/writer.py b/ome_zarr/writer.py new file mode 100644 index 00000000..48e755a5 --- /dev/null +++ b/ome_zarr/writer.py @@ -0,0 +1,86 @@ +"""Image writer utility + +""" +import logging +from typing import Any, List, Tuple, Union + +import numpy as np +import zarr + +from .scale import Scaler +from .types import JSONDict + +LOGGER = logging.getLogger("ome_zarr.writer") + + +def write_multiscale( + pyramid: List, group: zarr.Group, chunks: Union[Tuple[Any, ...], int] = None, +) -> None: + """Write a pyramid with multiscale metadata to disk.""" + paths = [] + for path, dataset in enumerate(pyramid): + # TODO: chunks here could be different per layer + group.create_dataset(str(path), data=dataset, chunks=chunks) + paths.append({"path": str(path)}) + + multiscales = [{"version": "0.1", "datasets": paths}] + group.attrs["multiscales"] = multiscales + + +def write_image( + image: np.ndarray, + group: zarr.Group, + chunks: Union[Tuple[Any, ...], int] = None, + byte_order: Union[str, List[str]] = "tczyx", + scaler: Scaler = Scaler(), + **metadata: JSONDict, +) -> None: + """Writes an image to the zarr store according to ome-zarr specification + + Parameters + ---------- + image: np.ndarray + the image data to save. A downsampling of the data will be computed + if the scaler argument is non-None. + group: zarr.Group + the group within the zarr store to store the data in + chunks: int or tuple of ints, + size of the saved chunks to store the image + byte_order: str or list of str, default "tczyx" + combination of the letters defining the order + in which the dimensions are saved + scaler: Scaler + Scaler implementation for downsampling the image argument. If None, + no downsampling will be performed. + """ + + if image.ndim > 5: + raise ValueError("Only images of 5D or less are supported") + + shape_5d: Tuple[Any, ...] = (*(1,) * (5 - image.ndim), *image.shape) + image = image.reshape(shape_5d) + + if chunks is not None: + chunks = _retuple(chunks, shape_5d) + + if scaler is not None: + image = scaler.nearest(image) + else: + LOGGER.debug("disabling pyramid") + image = [image] + + write_multiscale(image, group, chunks=chunks) + group.attrs.update(metadata) + + +def _retuple( + chunks: Union[Tuple[Any, ...], int], shape: Tuple[Any, ...] +) -> Tuple[Any, ...]: + + _chunks: Tuple[Any, ...] + if isinstance(chunks, int): + _chunks = (chunks,) + else: + _chunks = chunks + + return (*shape[: (5 - len(_chunks))], *_chunks) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..654b433b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +def pytest_addoption(parser): + """ + add `--show-viewer` as a valid command line flag + """ + parser.addoption( + "--show-viewer", + action="store_true", + default=False, + help="don't show viewer during tests", + ) diff --git a/tests/test_napari.py b/tests/test_napari.py index feb946f5..9a0feaac 100644 --- a/tests/test_napari.py +++ b/tests/test_napari.py @@ -2,19 +2,11 @@ import numpy as np import pytest -from napari.conftest import make_test_viewer # noqa from ome_zarr.data import astronaut, create_zarr from ome_zarr.napari import napari_get_reader -@pytest.fixture(autouse=True, scope="session") -def load_napari_conftest(pytestconfig): - from napari import conftest - - pytestconfig.pluginmanager.register(conftest, "napari-conftest") - - class TestNapari: @pytest.fixture(autouse=True) def initdir(self, tmpdir): @@ -67,9 +59,9 @@ def test_label(self): @pytest.mark.skipif( sys.version_info < (3, 7), reason="on_draw is missing in napari < 0.4.0", ) - def test_viewer(self, make_test_viewer): # noqa + def test_viewer(self, make_napari_viewer): # noqa """example of testing the viewer.""" - viewer = make_test_viewer() + viewer = make_napari_viewer() shapes = [(4000, 3000), (2000, 1500), (1000, 750), (500, 375)] np.random.seed(0) diff --git a/tests/test_writer.py b/tests/test_writer.py new file mode 100644 index 00000000..8bdd5baa --- /dev/null +++ b/tests/test_writer.py @@ -0,0 +1,44 @@ +import numpy as np +import pytest +import zarr + +from ome_zarr.io import parse_url +from ome_zarr.reader import Multiscales, Reader +from ome_zarr.scale import Scaler +from ome_zarr.writer import write_image + + +class TestWriter: + @pytest.fixture(autouse=True) + def initdir(self, tmpdir): + self.path = tmpdir.mkdir("data") + self.store = zarr.DirectoryStore(self.path) + self.root = zarr.group(store=self.store) + self.group = self.root.create_group("test") + + def create_data(self, shape, dtype=np.uint8, mean_val=10): + rng = np.random.default_rng(0) + return rng.poisson(mean_val, size=shape).astype(dtype) + + @pytest.fixture(params=((1, 2, 1, 256, 256),)) + def shape(self, request): + return request.param + + @pytest.fixture(params=[True, False], ids=["scale", "noop"]) + def scaler(self, request): + if request.param: + return Scaler() + else: + return None + + def test_writer(self, shape, scaler): + + data = self.create_data(shape) + write_image(image=data, group=self.group, chunks=(128, 128), scaler=scaler) + + # Verify + reader = Reader(parse_url(f"{self.path}/test")) + node = list(reader())[0] + assert Multiscales.matches(node.zarr) + assert node.data[0].shape == shape + assert node.data[0].chunks == ((1,), (2,), (1,), (128, 128), (128, 128))