Merge remote-tracking branch 'origin/master' into handle_empty_wells

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.8.1.dev0
+current_version = 0.8.2.dev0
 commit = True
 tag = True
 sign_tags = True

.readthedocs.yml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.10"
     # You can also specify other tool versions:
     # nodejs: "16"
     # rust: "1.55"
@@ -29,5 +29,4 @@ sphinx:
 # Optionally declare the Python requirements required to build your docs
 python:
   install:
-    - method: pip
-      path: .
+    - requirements: docs/requirements.txt

docs/requirements.txt

@@ -0,0 +1,8 @@
+sphinx==7.1.2
+sphinx-rtd-theme==1.3.0
+fsspec==2023.6.0
+zarr
+dask
+numpy
+scipy
+scikit-image

docs/source/conf.py

@@ -9,6 +9,7 @@
     "sphinx.ext.doctest",
     "sphinx.ext.autodoc",
     "sphinx.ext.intersphinx",
+    "sphinx_rtd_theme",
 ]
 
 # use index.rst instead of contents.rst
@@ -28,3 +29,6 @@
     "numpy": ("https://numpy.org/doc/stable/", None),
     "zarr": ("https://zarr.readthedocs.io/en/stable/", None),
 }
+
+# https://github.com/readthedocs/sphinx_rtd_theme
+html_theme = "sphinx_rtd_theme"

docs/source/python.rst

@@ -40,7 +40,7 @@ The following code creates a 3D Image in OME-Zarr with labels::
     root.attrs["omero"] = {
         "channels": [{
             "color": "00FFFF",
-            "window": {"start": 0, "end": 20},
+            "window": {"start": 0, "end": 20, "min": 0, "max": 255},
             "label": "random",
             "active": True,
         }]

ome_zarr/dask_utils.py

@@ -81,7 +81,7 @@ def downscale_nearest(image: da.Array, factors: Tuple[int, ...]) -> da.Array:
     ):
         raise ValueError(
             f"All scale factors must not be greater than the dimension length: "
-            f"({tuple(factors)}) <= ({tuple(image.shape)})"
+            f"({factors}) <= ({tuple(image.shape)})"
         )
     slices = tuple(slice(None, None, factor) for factor in factors)
     return image[slices]

ome_zarr/format.py

@@ -187,7 +187,7 @@ def init_store(self, path: str, mode: str = "r") -> FSStore:
         }
 
         mkdir = True
-        if "r" in mode or path.startswith("http") or path.startswith("s3"):
+        if "r" in mode or path.startswith(("http", "s3")):
             # Could be simplified on the fsspec side
             mkdir = False
         if mkdir:

ome_zarr/reader.py

@@ -7,7 +7,6 @@
 
 import dask.array as da
 import numpy as np
-from dask import delayed
 
 from .axes import Axes
 from .format import format_from_version
@@ -31,7 +30,7 @@ def __init__(
         self.zarr = zarr
         self.root = root
         self.seen: List[ZarrLocation] = []
-        if isinstance(root, Node) or isinstance(root, Reader):
+        if isinstance(root, (Node, Reader)):
             self.seen = root.seen
         else:
             self.seen = cast(List[ZarrLocation], root)
@@ -235,7 +234,7 @@ def __init__(self, node: Node) -> None:
                     if rgba:
                         rgba = [x / 255 for x in rgba]
 
-                    if isinstance(label_value, bool) or isinstance(label_value, int):
+                    if isinstance(label_value, (bool, int)):
                         colors[label_value] = rgba
                     else:
                         raise Exception("not bool or int")
@@ -420,38 +419,34 @@ def __init__(self, node: Node) -> None:
         self.img_metadata = image_node.metadata
         self.img_pyramid_shapes = [d.shape for d in image_node.data]
 
-        def get_field(tile_name: str, level: int) -> np.ndarray:
+        def get_field(row: int, col: int, level: int) -> da.core.Array:
             """tile_name is 'row,col'"""
-            row, col = (int(n) for n in tile_name.split(","))
             field_index = (column_count * row) + col
-            path = f"{field_index}/{level}"
-            LOGGER.debug("LOADING tile... %s", path)
+            data = None
             try:
-                data = self.zarr.load(path)
+                # handle e.g. 2x2 grid with only 3 images/fields
+                if field_index < len(image_paths):
+                    image_path = image_paths[field_index]
+                    path = f"{image_path}/{level}"
+                    data = self.zarr.load(path)
             except ValueError:
                 LOGGER.error("Failed to load %s", path)
-                data = np.zeros(self.img_pyramid_shapes[level], dtype=self.numpy_type)
+            if data is None:
+                data = da.zeros(self.img_pyramid_shapes[level], dtype=self.numpy_type)
             return data
 
-        lazy_reader = delayed(get_field)
-
         def get_lazy_well(level: int, tile_shape: tuple) -> da.Array:
             lazy_rows = []
             for row in range(row_count):
                 lazy_row: List[da.Array] = []
                 for col in range(column_count):
-                    tile_name = f"{row},{col}"
                     LOGGER.debug(
                         "creating lazy_reader. row: %s col: %s level: %s",
                         row,
                         col,
                         level,
                     )
-                    lazy_tile = da.from_delayed(
-                        lazy_reader(tile_name, level),
-                        shape=tile_shape,
-                        dtype=self.numpy_type,
-                    )
+                    lazy_tile = get_field(row, col, level)
                     lazy_row.append(lazy_tile)
                 lazy_rows.append(da.concatenate(lazy_row, axis=x_index))
             return da.concatenate(lazy_rows, axis=y_index)
@@ -486,7 +481,6 @@ def get_pyramid_lazy(self, node: Node) -> None:
         LOGGER.info("plate_data: %s", self.plate_data)
         self.rows = self.plate_data.get("rows")
         self.columns = self.plate_data.get("columns")
-        self.first_field = "0"
         self.row_names = [row["name"] for row in self.rows]
         self.col_names = [col["name"] for col in self.columns]
 
@@ -502,6 +496,7 @@ def get_pyramid_lazy(self, node: Node) -> None:
         well_spec: Optional[Well] = well_node.first(Well)
         if well_spec is None:
             raise Exception("Could not find first well")
+        self.first_field_path = well_spec.well_data["images"][0]["path"]
         self.numpy_type = well_spec.numpy_type
 
         LOGGER.debug("img_pyramid_shapes: %s", well_spec.img_pyramid_shapes)
@@ -528,15 +523,14 @@ def get_numpy_type(self, image_node: Node) -> np.dtype:
     def get_tile_path(self, level: int, row: int, col: int) -> str:
         return (
             f"{self.row_names[row]}/"
-            f"{self.col_names[col]}/{self.first_field}/{level}"
+            f"{self.col_names[col]}/{self.first_field_path}/{level}"
         )
 
     def get_stitched_grid(self, level: int, tile_shape: tuple) -> da.core.Array:
         LOGGER.debug("get_stitched_grid() level: %s, tile_shape: %s", level, tile_shape)
 
-        def get_tile(tile_name: str) -> np.ndarray:
+        def get_tile(row: int, col: int) -> da.core.Array:
             """tile_name is 'level,z,c,t,row,col'"""
-            row, col = (int(n) for n in tile_name.split(","))
 
             # check whether the Well exists at this row/column
             well_path = f"{self.row_names[row]}/{self.col_names[col]}"
@@ -545,27 +539,22 @@ def get_tile(tile_name: str) -> np.ndarray:
                 return np.zeros(tile_shape, dtype=self.numpy_type)
 
             path = self.get_tile_path(level, row, col)
-            LOGGER.debug("LOADING tile... %s with shape: %s", path, tile_shape)
+            LOGGER.debug("creating tile... %s with shape: %s", path, tile_shape)
 
             try:
+                # this is a dask array - data not loaded from source yet
                 data = self.zarr.load(path)
             except ValueError:
                 LOGGER.exception("Failed to load %s", path)
-                data = np.zeros(tile_shape, dtype=self.numpy_type)
+                data = da.zeros(tile_shape, dtype=self.numpy_type)
             return data
 
-        lazy_reader = delayed(get_tile)
-
         lazy_rows = []
         # For level 0, return whole image for each tile
         for row in range(self.row_count):
             lazy_row: List[da.Array] = []
             for col in range(self.column_count):
-                tile_name = f"{row},{col}"
-                lazy_tile = da.from_delayed(
-                    lazy_reader(tile_name), shape=tile_shape, dtype=self.numpy_type
-                )
-                lazy_row.append(lazy_tile)
+                lazy_row.append(get_tile(row, col))
             lazy_rows.append(da.concatenate(lazy_row, axis=len(self.axes) - 1))
         return da.concatenate(lazy_rows, axis=len(self.axes) - 2)
 
@@ -575,7 +564,7 @@ def get_tile_path(self, level: int, row: int, col: int) -> str:  # pragma: no co
         """251.zarr/A/1/0/labels/0/3/"""
         path = (
             f"{self.row_names[row]}/{self.col_names[col]}/"
-            f"{self.first_field}/labels/0/{level}"
+            f"{self.first_field_path}/labels/0/{level}"
         )
         return path
 
@@ -597,7 +586,7 @@ def get_pyramid_lazy(self, node: Node) -> None:  # pragma: no cover
         properties: Dict[int, Dict[str, Any]] = {}
         for row in self.row_names:
             for col in self.col_names:
-                path = f"{row}/{col}/{self.first_field}/labels/0/.zattrs"
+                path = f"{row}/{col}/{self.first_field_path}/labels/0/.zattrs"
                 labels_json = self.zarr.get_json(path).get("image-label", {})
                 # NB: assume that 'label_val' is unique across all images
                 props_list = labels_json.get("properties", [])

requirements/requirements-dev.txt

@@ -1,3 +1,4 @@
+fsspec==2023.6.0
 black
 cython >= 0.29.16
 numpy >= 1.16.0

setup.py

@@ -19,7 +19,7 @@ def read(fname):
 install_requires += (["dask"],)
 install_requires += (["distributed"],)
 install_requires += (["zarr>=2.8.1"],)
-install_requires += (["fsspec[s3]>=0.8,!=2021.07.0"],)
+install_requires += (["fsspec[s3]>=0.8,!=2021.07.0,!=2023.09.0"],)
 # See https://github.com/fsspec/filesystem_spec/issues/819
 install_requires += (["aiohttp<4"],)
 install_requires += (["requests"],)
@@ -29,7 +29,7 @@ def read(fname):
 
 setup(
     name="ome-zarr",
-    version="0.8.1.dev0",
+    version="0.8.2.dev0",
     author="The Open Microscopy Team",
     url="https://github.com/ome/ome-zarr-py",
     description="Implementation of images in Zarr files.",

tests/test_reader.py

@@ -1,10 +1,12 @@
+import dask.array as da
+import numpy as np
 import pytest
 import zarr
-from numpy import zeros
+from numpy import ones, zeros
 
 from ome_zarr.data import create_zarr
 from ome_zarr.io import parse_url
-from ome_zarr.reader import Node, Plate, Reader
+from ome_zarr.reader import Node, Plate, Reader, Well
 from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata
 
 
@@ -75,7 +77,8 @@ def test_minimal_plate(self):
         # assert len(nodes[1].specs) == 1
         # assert isinstance(nodes[1].specs[0], PlateLabels)
 
-    def test_multiwells_plate(self):
+    @pytest.mark.parametrize("field_paths", [["0", "1", "2"], ["img1", "img2", "img3"]])
+    def test_multiwells_plate(self, field_paths):
         row_names = ["A", "B", "C"]
         col_names = ["1", "2", "3", "4"]
         well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"]
@@ -84,16 +87,41 @@ def test_multiwells_plate(self):
             row, col = wp.split("/")
             row_group = self.root.require_group(row)
             well = row_group.require_group(col)
-            write_well_metadata(well, ["0", "1", "2"])
-            for field in range(3):
+            write_well_metadata(well, field_paths)
+            for field in field_paths:
                 image = well.require_group(str(field))
-                write_image(zeros((1, 1, 1, 256, 256)), image)
+                write_image(ones((1, 1, 1, 256, 256)), image)
 
         reader = Reader(parse_url(str(self.path)))
         nodes = list(reader())
         # currently reading plate labels disabled. Only 1 node
         assert len(nodes) == 1
-        assert len(nodes[0].specs) == 1
-        assert isinstance(nodes[0].specs[0], Plate)
-        # assert len(nodes[1].specs) == 1
+
+        plate_node = nodes[0]
+        assert len(plate_node.specs) == 1
+        assert isinstance(plate_node.specs[0], Plate)
+        # data should be a Dask array
+        pyramid = plate_node.data
+        assert isinstance(pyramid[0], da.Array)
+        # if we compute(), expect to get numpy array
+        result = pyramid[0].compute()
+        assert isinstance(result, np.ndarray)
+
+        # Get the plate node's array. It should be fused from the first field of all
+        # well arrays (which in this test are non-zero), with zero values for wells
+        # that failed to load (not expected) or the surplus area not filled by a well.
+        expected_num_pixels = (
+            len(well_paths) * len(field_paths[:1]) * np.prod((1, 1, 1, 256, 256))
+        )
+        pyramid_0 = pyramid[0]
+        assert np.asarray(pyramid_0).sum() == expected_num_pixels
+
         # assert isinstance(nodes[1].specs[0], PlateLabels)
+
+        reader = Reader(parse_url(f"{self.path}/{well_paths[0]}"))
+        nodes = list(reader())
+        assert isinstance(nodes[0].specs[0], Well)
+        pyramid = nodes[0].data
+        assert isinstance(pyramid[0], da.Array)
+        result = pyramid[0].compute()
+        assert isinstance(result, np.ndarray)