From 286a4f2b9364c8f07ea9e843abb8b683ece5f2c7 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 21 Aug 2023 13:00:19 +0100 Subject: [PATCH 01/26] Fix compute() on plate grid --- ome_zarr/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ccc4ea01..0a750016 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -541,7 +541,8 @@ def get_tile(tile_name: str) -> np.ndarray: LOGGER.debug("LOADING tile... %s with shape: %s", path, tile_shape) try: - data = self.zarr.load(path) + # compute() to get the data from dask - we want to return array + data = self.zarr.load(path).compute() except ValueError: LOGGER.exception("Failed to load %s", path) data = np.zeros(tile_shape, dtype=self.numpy_type) From ee9ef7dd967c121636239822f30fdc4434537eea Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 22 Aug 2023 12:10:21 +0100 Subject: [PATCH 02/26] remove wrapping of get_tile() with delayed() --- ome_zarr/reader.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 0a750016..4b6f084f 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -534,32 +534,27 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: def get_stitched_grid(self, level: int, tile_shape: tuple) -> da.core.Array: LOGGER.debug("get_stitched_grid() level: %s, tile_shape: %s", level, tile_shape) - def get_tile(tile_name: str) -> np.ndarray: + def get_tile(tile_name: str) -> da.core.Array: """tile_name is 'level,z,c,t,row,col'""" row, col = (int(n) for n in tile_name.split(",")) path = self.get_tile_path(level, row, col) - LOGGER.debug("LOADING tile... %s with shape: %s", path, tile_shape) + LOGGER.debug("creating tile... %s with shape: %s", path, tile_shape) try: - # compute() to get the data from dask - we want to return array - data = self.zarr.load(path).compute() + # this is a dask array - data not loaded from source yet + data = self.zarr.load(path) except ValueError: LOGGER.exception("Failed to load %s", path) - data = np.zeros(tile_shape, dtype=self.numpy_type) + data = da.zeros(tile_shape, dtype=self.numpy_type) return data - lazy_reader = delayed(get_tile) - lazy_rows = [] # For level 0, return whole image for each tile for row in range(self.row_count): lazy_row: List[da.Array] = [] for col in range(self.column_count): tile_name = f"{row},{col}" - lazy_tile = da.from_delayed( - lazy_reader(tile_name), shape=tile_shape, dtype=self.numpy_type - ) - lazy_row.append(lazy_tile) + lazy_row.append(get_tile(tile_name)) lazy_rows.append(da.concatenate(lazy_row, axis=len(self.axes) - 1)) return da.concatenate(lazy_rows, axis=len(self.axes) - 2) From 5d713f71c17084487da5c59300c7e355999df7f5 Mon Sep 17 00:00:00 2001 From: aeisenbarth <54448967+aeisenbarth@users.noreply.github.com> Date: Thu, 24 Aug 2023 01:04:41 +0200 Subject: [PATCH 03/26] Use field of view path of first well instead of field "0" --- ome_zarr/reader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ccc4ea01..1c55bbea 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -424,7 +424,8 @@ def get_field(tile_name: str, level: int) -> np.ndarray: """tile_name is 'row,col'""" row, col = (int(n) for n in tile_name.split(",")) field_index = (column_count * row) + col - path = f"{field_index}/{level}" + image_path = image_paths[field_index] + path = f"{image_path}/{level}" LOGGER.debug("LOADING tile... %s", path) try: data = self.zarr.load(path) @@ -486,7 +487,6 @@ def get_pyramid_lazy(self, node: Node) -> None: LOGGER.info("plate_data: %s", self.plate_data) self.rows = self.plate_data.get("rows") self.columns = self.plate_data.get("columns") - self.first_field = "0" self.row_names = [row["name"] for row in self.rows] self.col_names = [col["name"] for col in self.columns] @@ -502,6 +502,7 @@ def get_pyramid_lazy(self, node: Node) -> None: well_spec: Optional[Well] = well_node.first(Well) if well_spec is None: raise Exception("Could not find first well") + self.first_field = well_spec.well_data["images"][0]["path"] self.numpy_type = well_spec.numpy_type LOGGER.debug("img_pyramid_shapes: %s", well_spec.img_pyramid_shapes) From a64fd2e686812974e87f27b8e5a24b3eec716741 Mon Sep 17 00:00:00 2001 From: aeisenbarth <54448967+aeisenbarth@users.noreply.github.com> Date: Thu, 24 Aug 2023 01:06:41 +0200 Subject: [PATCH 04/26] Rename first_field to avoid ambiguity with field index --- ome_zarr/reader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1c55bbea..d58f879e 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -502,7 +502,7 @@ def get_pyramid_lazy(self, node: Node) -> None: well_spec: Optional[Well] = well_node.first(Well) if well_spec is None: raise Exception("Could not find first well") - self.first_field = well_spec.well_data["images"][0]["path"] + self.first_field_path = well_spec.well_data["images"][0]["path"] self.numpy_type = well_spec.numpy_type LOGGER.debug("img_pyramid_shapes: %s", well_spec.img_pyramid_shapes) @@ -529,7 +529,7 @@ def get_numpy_type(self, image_node: Node) -> np.dtype: def get_tile_path(self, level: int, row: int, col: int) -> str: return ( f"{self.row_names[row]}/" - f"{self.col_names[col]}/{self.first_field}/{level}" + f"{self.col_names[col]}/{self.first_field_path}/{level}" ) def get_stitched_grid(self, level: int, tile_shape: tuple) -> da.core.Array: @@ -569,7 +569,7 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: # pragma: no co """251.zarr/A/1/0/labels/0/3/""" path = ( f"{self.row_names[row]}/{self.col_names[col]}/" - f"{self.first_field}/labels/0/{level}" + f"{self.first_field_path}/labels/0/{level}" ) return path @@ -591,7 +591,7 @@ def get_pyramid_lazy(self, node: Node) -> None: # pragma: no cover properties: Dict[int, Dict[str, Any]] = {} for row in self.row_names: for col in self.col_names: - path = f"{row}/{col}/{self.first_field}/labels/0/.zattrs" + path = f"{row}/{col}/{self.first_field_path}/labels/0/.zattrs" labels_json = self.zarr.get_json(path).get("image-label", {}) # NB: assume that 'label_val' is unique across all images props_list = labels_json.get("properties", []) From b48da21a7eeff556a8026784d7e964bbbc4714ee Mon Sep 17 00:00:00 2001 From: Andreas Eisenbarth Date: Thu, 24 Aug 2023 16:44:40 +0200 Subject: [PATCH 05/26] Add test case for alphanumeric Plate/Well image paths --- tests/test_reader.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 556a2087..3520eabc 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,6 +1,7 @@ +import numpy as np import pytest import zarr -from numpy import zeros +from numpy import ones, zeros from ome_zarr.data import create_zarr from ome_zarr.io import parse_url @@ -75,7 +76,8 @@ def test_minimal_plate(self): # assert len(nodes[1].specs) == 1 # assert isinstance(nodes[1].specs[0], PlateLabels) - def test_multiwells_plate(self): + @pytest.mark.parametrize("field_paths", [["0", "1", "2"], ["img1", "img2", "img3"]]) + def test_multiwells_plate(self, field_paths): row_names = ["A", "B", "C"] col_names = ["1", "2", "3", "4"] well_paths = ["A/1", "A/2", "A/4", "B/2", "B/3", "C/1", "C/3", "C/4"] @@ -84,16 +86,25 @@ def test_multiwells_plate(self): row, col = wp.split("/") row_group = self.root.require_group(row) well = row_group.require_group(col) - write_well_metadata(well, ["0", "1", "2"]) - for field in range(3): + write_well_metadata(well, field_paths) + for field in field_paths: image = well.require_group(str(field)) - write_image(zeros((1, 1, 1, 256, 256)), image) + write_image(ones((1, 1, 1, 256, 256)), image) reader = Reader(parse_url(str(self.path))) nodes = list(reader()) # currently reading plate labels disabled. Only 1 node assert len(nodes) == 1 - assert len(nodes[0].specs) == 1 - assert isinstance(nodes[0].specs[0], Plate) + plate_node = nodes[0] + assert len(plate_node.specs) == 1 + assert isinstance(plate_node.specs[0], Plate) + # Get the plate node's array. It should be fused from the first field of all + # well arrays (which in this test are non-zero), with zero values for wells + # that failed to load (not expected) or the surplus area not filled by a well. + expected_num_pixels = ( + len(well_paths) * len(field_paths[:1]) * np.prod((1, 1, 1, 256, 256)) + ) + pyramid_0 = plate_node.data[0] + assert np.asarray(pyramid_0).sum() == expected_num_pixels # assert len(nodes[1].specs) == 1 # assert isinstance(nodes[1].specs[0], PlateLabels) From c9ed4310cc0d65be7403c7ece03e698970b32feb Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 24 Aug 2023 16:10:12 +0100 Subject: [PATCH 06/26] get_field() for Well grid returns dask array --- ome_zarr/reader.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 4b6f084f..14adcd33 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -7,7 +7,6 @@ import dask.array as da import numpy as np -from dask import delayed from .axes import Axes from .format import format_from_version @@ -420,7 +419,7 @@ def __init__(self, node: Node) -> None: self.img_metadata = image_node.metadata self.img_pyramid_shapes = [d.shape for d in image_node.data] - def get_field(tile_name: str, level: int) -> np.ndarray: + def get_field(tile_name: str, level: int) -> da.core.Array: """tile_name is 'row,col'""" row, col = (int(n) for n in tile_name.split(",")) field_index = (column_count * row) + col @@ -430,11 +429,9 @@ def get_field(tile_name: str, level: int) -> np.ndarray: data = self.zarr.load(path) except ValueError: LOGGER.error("Failed to load %s", path) - data = np.zeros(self.img_pyramid_shapes[level], dtype=self.numpy_type) + data = da.zeros(self.img_pyramid_shapes[level], dtype=self.numpy_type) return data - lazy_reader = delayed(get_field) - def get_lazy_well(level: int, tile_shape: tuple) -> da.Array: lazy_rows = [] for row in range(row_count): @@ -447,11 +444,7 @@ def get_lazy_well(level: int, tile_shape: tuple) -> da.Array: col, level, ) - lazy_tile = da.from_delayed( - lazy_reader(tile_name, level), - shape=tile_shape, - dtype=self.numpy_type, - ) + lazy_tile = get_field(tile_name, level) lazy_row.append(lazy_tile) lazy_rows.append(da.concatenate(lazy_row, axis=x_index)) return da.concatenate(lazy_rows, axis=y_index) From e55e41d39019549ec96cc307b8685d37d4ccb34b Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 24 Aug 2023 16:18:05 +0100 Subject: [PATCH 07/26] Use row, col instead of tile_name for get_tile() and get_field() --- ome_zarr/reader.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 14adcd33..eb47fd12 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -419,9 +419,8 @@ def __init__(self, node: Node) -> None: self.img_metadata = image_node.metadata self.img_pyramid_shapes = [d.shape for d in image_node.data] - def get_field(tile_name: str, level: int) -> da.core.Array: + def get_field(row: int, col: int, level: int) -> da.core.Array: """tile_name is 'row,col'""" - row, col = (int(n) for n in tile_name.split(",")) field_index = (column_count * row) + col path = f"{field_index}/{level}" LOGGER.debug("LOADING tile... %s", path) @@ -437,14 +436,13 @@ def get_lazy_well(level: int, tile_shape: tuple) -> da.Array: for row in range(row_count): lazy_row: List[da.Array] = [] for col in range(column_count): - tile_name = f"{row},{col}" LOGGER.debug( "creating lazy_reader. row: %s col: %s level: %s", row, col, level, ) - lazy_tile = get_field(tile_name, level) + lazy_tile = get_field(row, col, level) lazy_row.append(lazy_tile) lazy_rows.append(da.concatenate(lazy_row, axis=x_index)) return da.concatenate(lazy_rows, axis=y_index) @@ -527,9 +525,8 @@ def get_tile_path(self, level: int, row: int, col: int) -> str: def get_stitched_grid(self, level: int, tile_shape: tuple) -> da.core.Array: LOGGER.debug("get_stitched_grid() level: %s, tile_shape: %s", level, tile_shape) - def get_tile(tile_name: str) -> da.core.Array: + def get_tile(row: int, col: int) -> da.core.Array: """tile_name is 'level,z,c,t,row,col'""" - row, col = (int(n) for n in tile_name.split(",")) path = self.get_tile_path(level, row, col) LOGGER.debug("creating tile... %s with shape: %s", path, tile_shape) @@ -546,8 +543,7 @@ def get_tile(tile_name: str) -> da.core.Array: for row in range(self.row_count): lazy_row: List[da.Array] = [] for col in range(self.column_count): - tile_name = f"{row},{col}" - lazy_row.append(get_tile(tile_name)) + lazy_row.append(get_tile(row, col)) lazy_rows.append(da.concatenate(lazy_row, axis=len(self.axes) - 1)) return da.concatenate(lazy_rows, axis=len(self.axes) - 2) From bacc2988c7a2ad1d54d266401917efe931d706e7 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 24 Aug 2023 16:35:05 +0100 Subject: [PATCH 08/26] test_multiwells_plate() checks data.compute() for Plate and Well --- tests/test_reader.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/test_reader.py b/tests/test_reader.py index 556a2087..8d7776df 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,10 +1,12 @@ +import dask.array as da +import numpy as np import pytest import zarr from numpy import zeros from ome_zarr.data import create_zarr from ome_zarr.io import parse_url -from ome_zarr.reader import Node, Plate, Reader +from ome_zarr.reader import Node, Plate, Reader, Well from ome_zarr.writer import write_image, write_plate_metadata, write_well_metadata @@ -95,5 +97,19 @@ def test_multiwells_plate(self): assert len(nodes) == 1 assert len(nodes[0].specs) == 1 assert isinstance(nodes[0].specs[0], Plate) + # data should be a Dask array + pyramid = nodes[0].data + assert isinstance(pyramid[0], da.Array) + # if we compute(), expect to get numpy array + result = pyramid[0].compute() + assert isinstance(result, np.ndarray) # assert len(nodes[1].specs) == 1 # assert isinstance(nodes[1].specs[0], PlateLabels) + + reader = Reader(parse_url(f"{self.path}/{well_paths[0]}")) + nodes = list(reader()) + assert isinstance(nodes[0].specs[0], Well) + pyramid = nodes[0].data + assert isinstance(pyramid[0], da.Array) + result = pyramid[0].compute() + assert isinstance(result, np.ndarray) From 9a6b9476660223ec3fb887b5a7a7b709e5bc84ec Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 24 Aug 2023 16:49:36 +0100 Subject: [PATCH 09/26] Try using python 3.11 for readthedocs.yml --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 13ae5bed..5369a2e0 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.11" # You can also specify other tool versions: # nodejs: "16" # rust: "1.55" From 0a889eeefe4452938d8826172d8cd1ec3ff02908 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 24 Aug 2023 21:57:14 +0100 Subject: [PATCH 10/26] Add sphinx_rtd_theme to docs dependencies --- docs/source/conf.py | 1 + docs/source/requirements.txt | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 docs/source/requirements.txt diff --git a/docs/source/conf.py b/docs/source/conf.py index 854cc342..559f2df2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -9,6 +9,7 @@ "sphinx.ext.doctest", "sphinx.ext.autodoc", "sphinx.ext.intersphinx", + "sphinx_rtd_theme", ] # use index.rst instead of contents.rst diff --git a/docs/source/requirements.txt b/docs/source/requirements.txt new file mode 100644 index 00000000..691e5218 --- /dev/null +++ b/docs/source/requirements.txt @@ -0,0 +1,2 @@ +sphinx==5.3.0 +sphinx_rtd_theme==1.1.1 From da4151d95cb2b9efc0241cf4ed8b6082a220a611 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 09:17:17 +0100 Subject: [PATCH 11/26] Specify requirements.txt in .readthedocs.yml --- .readthedocs.yml | 1 + docs/{source => }/requirements.txt | 0 2 files changed, 1 insertion(+) rename docs/{source => }/requirements.txt (100%) diff --git a/.readthedocs.yml b/.readthedocs.yml index 13ae5bed..4e1ce7ea 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -31,3 +31,4 @@ python: install: - method: pip path: . + - requirements: docs/requirements.txt diff --git a/docs/source/requirements.txt b/docs/requirements.txt similarity index 100% rename from docs/source/requirements.txt rename to docs/requirements.txt From fd682c84b24fab6f1245c594e73685775aca3164 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 09:51:32 +0100 Subject: [PATCH 12/26] Remove method: pip from rtd python.install To match all examples at https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html --- .readthedocs.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 4e1ce7ea..a461605f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -29,6 +29,4 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - method: pip - path: . - requirements: docs/requirements.txt From 492b29c78321be3a617364e388e07da92e0ea44c Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 10:17:51 +0100 Subject: [PATCH 13/26] Updating to match rtd tutorial template at https://github.com/readthedocs/tutorial-template/blob/main/docs/requirements.txt --- .readthedocs.yml | 2 +- docs/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index a461605f..fc2e4840 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-20.04 tools: - python: "3.9" + python: "3.10" # You can also specify other tool versions: # nodejs: "16" # rust: "1.55" diff --git a/docs/requirements.txt b/docs/requirements.txt index 691e5218..53fc1f32 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -sphinx==5.3.0 -sphinx_rtd_theme==1.1.1 +sphinx==7.1.2 +sphinx-rtd-theme==1.3.0rc1 From 14ee7e4ede40c33a5ba6f2e9f9c69ababa2805ce Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 10:37:59 +0100 Subject: [PATCH 14/26] Add html_theme to conf.py --- docs/source/conf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 559f2df2..3825cd12 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -29,3 +29,6 @@ "numpy": ("https://numpy.org/doc/stable/", None), "zarr": ("https://zarr.readthedocs.io/en/stable/", None), } + +# https://github.com/readthedocs/sphinx_rtd_theme +html_theme = "sphinx_rtd_theme" From 24a09e5c9d8facc1b45cefad257b33bb7bde910c Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 10:44:42 +0100 Subject: [PATCH 15/26] Add deps to requirements.txt based on rtd build errors --- docs/requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 53fc1f32..3f855685 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,6 @@ sphinx==7.1.2 sphinx-rtd-theme==1.3.0rc1 +zarr +dask +numpy +scipy From f1dbe72b65efd5a76d9db33383fd91607f0a750a Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 10:54:03 +0100 Subject: [PATCH 16/26] Add deps to requirements.txt based on rtd build errors --- docs/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3f855685..d8714270 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,4 @@ zarr dask numpy scipy +skimage From 9152f31dac18fc805cde5eda1021d873cbad48c5 Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 25 Aug 2023 11:18:17 +0100 Subject: [PATCH 17/26] Add deps to requirements.txt based on rtd build errors --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index d8714270..c2a20d9f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,4 +4,4 @@ zarr dask numpy scipy -skimage +scikit-image From 36f2b4d3debd018e9af277ea75ef7dd35e18026e Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 12 Sep 2023 11:17:40 +0100 Subject: [PATCH 18/26] Use sphinx-rtd-theme==1.3.0 --- docs/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index c2a20d9f..19884415 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx==7.1.2 -sphinx-rtd-theme==1.3.0rc1 +sphinx-rtd-theme==1.3.0 zarr dask numpy From 17ee62ac553879d269e1486f03710c2505c102e1 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 12 Sep 2023 11:30:46 +0100 Subject: [PATCH 19/26] Temp cap fsspec to avoid https://github.com/fsspec/filesystem_spec/pull/1358 --- docs/requirements.txt | 1 + requirements/requirements-dev.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/requirements.txt b/docs/requirements.txt index 19884415..76aa0da8 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ sphinx==7.1.2 sphinx-rtd-theme==1.3.0 +fsspec==2023.6.0 zarr dask numpy diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index d5a00601..2931335e 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -1,3 +1,4 @@ +fsspec==2023.6.0 black cython >= 0.29.16 numpy >= 1.16.0 From e0d0cdeadde256c200979ebc93f83bf18c04f4e8 Mon Sep 17 00:00:00 2001 From: William Moore Date: Wed, 13 Sep 2023 15:45:22 +0100 Subject: [PATCH 20/26] Exclude fsspec 2023.9.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f5cc028d..e43b3366 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def read(fname): install_requires += (["dask"],) install_requires += (["distributed"],) install_requires += (["zarr>=2.8.1"],) -install_requires += (["fsspec[s3]>=0.8,!=2021.07.0"],) +install_requires += (["fsspec[s3]>=0.8,!=2021.07.0,!=2023.09.0"],) # See https://github.com/fsspec/filesystem_spec/issues/819 install_requires += (["aiohttp<4"],) install_requires += (["requests"],) From 27eb82abe3ed42a8a4811a3c0b90958d78ff8d7a Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 15 Sep 2023 19:54:56 +0200 Subject: [PATCH 21/26] =?UTF-8?q?Bump=20version:=200.8.1.dev0=20=E2=86=92?= =?UTF-8?q?=200.8.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 167faa4c..89363fd4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.1.dev0 +current_version = 0.8.1 commit = True tag = True sign_tags = True diff --git a/setup.py b/setup.py index e43b3366..55932cf3 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(fname): setup( name="ome-zarr", - version="0.8.1.dev0", + version="0.8.1", author="The Open Microscopy Team", url="https://github.com/ome/ome-zarr-py", description="Implementation of images in Zarr files.", From d72a7b01aa18ba42a9c0663c1d961226405b8d84 Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Fri, 15 Sep 2023 19:55:14 +0200 Subject: [PATCH 22/26] =?UTF-8?q?Bump=20version:=200.8.1=20=E2=86=92=200.8?= =?UTF-8?q?.2.dev0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 89363fd4..709335a3 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.1 +current_version = 0.8.2.dev0 commit = True tag = True sign_tags = True diff --git a/setup.py b/setup.py index 55932cf3..feb11e89 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(fname): setup( name="ome-zarr", - version="0.8.1", + version="0.8.2.dev0", author="The Open Microscopy Team", url="https://github.com/ome/ome-zarr-py", description="Implementation of images in Zarr files.", From 1da15eb911d104554c7c3d871d27065b4257075f Mon Sep 17 00:00:00 2001 From: Sean Martin Date: Tue, 19 Sep 2023 12:23:59 +0100 Subject: [PATCH 23/26] Update python.rst to have min, max The OME-NGFF validator complains about window not having the min and max properties in the current version of this script. --- docs/source/python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/python.rst b/docs/source/python.rst index f6bbdc26..e4a399d5 100644 --- a/docs/source/python.rst +++ b/docs/source/python.rst @@ -40,7 +40,7 @@ The following code creates a 3D Image in OME-Zarr with labels:: root.attrs["omero"] = { "channels": [{ "color": "00FFFF", - "window": {"start": 0, "end": 20}, + "window": {"start": 0, "end": 20, "min": 0, "max": 255}, "label": "random", "active": True, }] From 291ada0b99e400937f54a59ce08c926a40bdcdd2 Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos Orfanos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 28 Sep 2023 20:29:55 +0200 Subject: [PATCH 24/26] `factors` is already a tuple There is no need to create a new tuple from a tuple. --- ome_zarr/dask_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ome_zarr/dask_utils.py b/ome_zarr/dask_utils.py index e424476a..12417730 100644 --- a/ome_zarr/dask_utils.py +++ b/ome_zarr/dask_utils.py @@ -81,7 +81,7 @@ def downscale_nearest(image: da.Array, factors: Tuple[int, ...]) -> da.Array: ): raise ValueError( f"All scale factors must not be greater than the dimension length: " - f"({tuple(factors)}) <= ({tuple(image.shape)})" + f"({factors}) <= ({tuple(image.shape)})" ) slices = tuple(slice(None, None, factor) for factor in factors) return image[slices] From 395a3408978d1e2131363ceb85790b6bc87c2a8e Mon Sep 17 00:00:00 2001 From: Dimitri Papadopoulos <3234522+DimitriPapadopoulos@users.noreply.github.com> Date: Thu, 28 Sep 2023 21:48:48 +0200 Subject: [PATCH 25/26] Simpler startswith() and isinstance() calls --- ome_zarr/format.py | 2 +- ome_zarr/reader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ome_zarr/format.py b/ome_zarr/format.py index 6b57b4ff..0ae001bb 100644 --- a/ome_zarr/format.py +++ b/ome_zarr/format.py @@ -187,7 +187,7 @@ def init_store(self, path: str, mode: str = "r") -> FSStore: } mkdir = True - if "r" in mode or path.startswith("http") or path.startswith("s3"): + if "r" in mode or path.startswith(("http", "s3")): # Could be simplified on the fsspec side mkdir = False if mkdir: diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index ccc4ea01..2fe8d607 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -31,7 +31,7 @@ def __init__( self.zarr = zarr self.root = root self.seen: List[ZarrLocation] = [] - if isinstance(root, Node) or isinstance(root, Reader): + if isinstance(root, (Node, Reader)): self.seen = root.seen else: self.seen = cast(List[ZarrLocation], root) @@ -235,7 +235,7 @@ def __init__(self, node: Node) -> None: if rgba: rgba = [x / 255 for x in rgba] - if isinstance(label_value, bool) or isinstance(label_value, int): + if isinstance(label_value, (bool, int)): colors[label_value] = rgba else: raise Exception("not bool or int") From 54177646f2c5aeb841c40611a7b88030af0545cf Mon Sep 17 00:00:00 2001 From: William Moore Date: Fri, 29 Sep 2023 11:40:53 +0100 Subject: [PATCH 26/26] Handle Well grid layout not fully filled with fields --- ome_zarr/reader.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ome_zarr/reader.py b/ome_zarr/reader.py index 1408a110..a2a04f77 100644 --- a/ome_zarr/reader.py +++ b/ome_zarr/reader.py @@ -422,13 +422,16 @@ def __init__(self, node: Node) -> None: def get_field(row: int, col: int, level: int) -> da.core.Array: """tile_name is 'row,col'""" field_index = (column_count * row) + col - image_path = image_paths[field_index] - path = f"{image_path}/{level}" - LOGGER.debug("LOADING tile... %s", path) + data = None try: - data = self.zarr.load(path) + # handle e.g. 2x2 grid with only 3 images/fields + if field_index < len(image_paths): + image_path = image_paths[field_index] + path = f"{image_path}/{level}" + data = self.zarr.load(path) except ValueError: LOGGER.error("Failed to load %s", path) + if data is None: data = da.zeros(self.img_pyramid_shapes[level], dtype=self.numpy_type) return data