Skip to content

Commit

Permalink
Merge pull request #235 from clbarnes/n5_source
Browse files Browse the repository at this point in the history
Use CATMAID stack info to create xarray.DataArrays
clbarnes authored Jan 15, 2024
2 parents 647d688 + 7b402bc commit 0f05a46
Showing 10 changed files with 884 additions and 19 deletions.
16 changes: 15 additions & 1 deletion docs/source/api.rst
Original file line number Diff line number Diff line change
@@ -172,19 +172,33 @@ Functions for reconstruction samplers:
pymaid.get_sampler_domains
pymaid.get_sampler_counts

Image data (tiles)
Image metadata
--------------
Functions to fetch information about the image stacks CATMAID knows about.

.. autosummary::
:toctree: generated/

pymaid.get_stacks
pymaid.get_stack_info
pymaid.get_mirror_info

Image data (tiles and N5 volumes)
------------------
Functions to fetch and process image data. Note that this is not imported at
top level but has to be imported explicitly::

>>> from pymaid import tiles
>>> help(tiles.crop_neuron)
>>> from pymaid.stack import Stack
>>> help(Stack)

.. autosummary::
:toctree: generated/

pymaid.tiles.TileLoader
pymaid.tiles.crop_neuron
pymaid.stack.Stack

.. _api_misc:

2 changes: 2 additions & 0 deletions docs/source/whats_new.rst
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@ What's new?
- BREAKING: Drop python 3.7 support.
- - :class:`pymaid.neuron_label.NeuronLabeller` added for labelling neurons
like in the CATMAID frontend.
- :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data
- :class:`pymaid.stack.Stack` class for accessing N5 and JPEG tile image data as a :class:`xarray.DataArray`
* - 2.4.0
- 27/05/23
- - :func:`pymaid.get_annotation_graph` deprecated in favour of the new
2 changes: 2 additions & 0 deletions pymaid/fetch/__init__.py
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@
from .landmarks import get_landmarks, get_landmark_groups
from .skeletons import get_skeleton_ids
from .annotations import get_annotation_graph, get_entity_graph, get_annotation_id
from .stack import get_stacks, get_stack_info, get_mirror_info


__all__ = ['get_annotation_details', 'get_annotation_id',
@@ -93,6 +94,7 @@
'get_landmarks',
'get_landmark_groups',
'get_skeleton_ids',
'get_stacks', 'get_stack_info', 'get_mirror_info',
]

# Set up logging
2 changes: 1 addition & 1 deletion pymaid/fetch/annotations.py
Original file line number Diff line number Diff line change
@@ -391,7 +391,7 @@ def get_entity_graph(
Neurons additionally have
- skeleton_ids: list[int]
- skeleton_ids: List[int]
Skeletons additionally have
251 changes: 251 additions & 0 deletions pymaid/fetch/stack.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
from typing import Any, Optional, Union, Literal, Sequence, Tuple, Dict, List
import numpy as np
from ..utils import _eval_remote_instance
from ..client import CatmaidInstance
from enum import IntEnum
from dataclasses import dataclass, asdict

Dimension = Literal["x", "y", "z"]


class Orientation(IntEnum):
XY = 0
# todo: check these
XZ = 1
ZY = 2

def __bool__(self) -> bool:
return True

def full_orientation(self, reverse=False) -> Tuple[Dimension, Dimension, Dimension]:
out = [
("x", "y", "z"),
("x", "z", "y"),
("z", "y", "x"),
][self.value]
if reverse:
out = out[::-1]
return out

@classmethod
def from_dims(cls, dims: Sequence[Dimension]):
pair = (dims[0].lower(), dims[1].lower())
out = {
("x", "y"): cls.XY,
("x", "z"): cls.XZ,
("z", "y"): cls.ZY,
}.get(pair)
if out is None:
raise ValueError(f"Unknown dimensions: {dims}")
return out


@dataclass
class StackSummary:
id: int
pid: int
title: str
comment: str


def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> List[StackSummary]:
"""Get summary of all stacks in the project.
Parameters
----------
remote_instance : Optional[CatmaidInstance], optional
By default global instance.
Returns
-------
stacks
List of StackSummary objects.
"""
cm = _eval_remote_instance(remote_instance)
url = cm.make_url(cm.project_id, "stacks")
return [StackSummary(**r) for r in cm.fetch(url)]


@dataclass
class MirrorInfo:
id: int
title: str
image_base: str
tile_width: int
tile_height: int
tile_source_type: int
file_extension: str
position: int

def to_jso(self):
return asdict(self)


@dataclass
class Color:
r: float
g: float
b: float
a: float


@dataclass
class StackInfo:
sid: int
pid: int
ptitle: str
stitle: str
downsample_factors: Optional[List[Dict[Dimension, float]]]
num_zoom_levels: int
translation: Dict[Dimension, float]
resolution: Dict[Dimension, float]
dimension: Dict[Dimension, int]
comment: str
description: str
metadata: Optional[str]
broken_slices: Dict[int, int]
mirrors: List[MirrorInfo]
orientation: Orientation
attribution: str
canary_location: Dict[Dimension, int]
placeholder_color: Color

@classmethod
def from_jso(cls, sinfo: Dict[str, Any]):
sinfo["orientation"] = Orientation(sinfo["orientation"])
sinfo["placeholder_color"] = Color(**sinfo["placeholder_color"])
sinfo["mirrors"] = [MirrorInfo(**m) for m in sinfo["mirrors"]]
return StackInfo(**sinfo)

def to_jso(self):
return asdict(self)

def get_downsample(self, scale_level=0) -> Dict[Dimension, float]:
"""Get the downsample factors for a given scale level.
If the downsample factors are explicit in the stack info,
use that value.
Otherwise, use the CATMAID default:
scale by a factor of 2 per scale level in everything except the slicing dimension.
If number of scale levels is known,
ensure the scale level exists.
Parameters
----------
scale_level : int, optional
Returns
-------
dict[Dimension, float]
Raises
------
IndexError
If the scale level is known not to exist
"""
if self.downsample_factors is not None:
return self.downsample_factors[scale_level]
if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels:
raise IndexError("list index out of range")

first, second, slicing = self.orientation.full_orientation()
return {first: 2**scale_level, second: 2**scale_level, slicing: 1}

def get_coords(self, scale_level: int = 0) -> Dict[Dimension, np.ndarray]:
dims = self.orientation.full_orientation()
dims = dims[::-1]

downsamples = self.get_downsample(scale_level)

out: Dict[Dimension, np.ndarray] = dict()
for d in dims:
c = np.arange(self.dimension[d], dtype=float)
c *= self.resolution[d]
c *= downsamples[d]
c += self.translation[d]
out[d] = c
return out


def get_stack_info(
stack: Union[int, str], remote_instance: Optional[CatmaidInstance] = None
) -> StackInfo:
"""Get information about an image stack.
Parameters
----------
stack : Union[int, str]
Integer ID or string title of the stack.
remote_instance : Optional[CatmaidInstance], optional
By default global.
Returns
-------
StackInfo
Raises
------
ValueError
If an unknown stack title is given.
"""
cm = _eval_remote_instance(remote_instance)
if isinstance(stack, str):
stacks = get_stacks(cm)
for s in stacks:
if s.title == stack:
stack_id = s.id
break
else:
raise ValueError(f"No stack with title '{stack}'")
else:
stack_id = int(stack)

url = cm.make_url(cm.project_id, "stack", stack_id, "info")
sinfo = cm.fetch(url)
return StackInfo.from_jso(sinfo)


def get_mirror_info(
stack: Union[int, str, StackInfo],
mirror: Union[int, str],
remote_instance: Optional[CatmaidInstance] = None,
) -> MirrorInfo:
"""Get information about a stack mirror.
Parameters
----------
stack : Union[int, str, StackInfo]
Integer stack ID, string stack title,
or an existing StackInfo object (avoids server request).
mirror : Union[int, str]
Integer mirror ID, or string mirror title.
remote_instance : Optional[CatmaidInstance]
By default, global.
Returns
-------
MirrorInfo
Raises
------
ValueError
No mirror matching given ID/ title.
"""
if isinstance(stack, StackInfo):
stack_info = stack
else:
stack_info = get_stack_info(stack, remote_instance)

if isinstance(mirror, str):
key = "title"
else:
key = "id"
mirror = int(mirror)

for m in stack_info.mirrors:
if getattr(m, key) == mirror:
return m

raise ValueError(
f"No mirror for stack '{stack_info.stitle}' with {key} {repr(mirror)}"
)
22 changes: 11 additions & 11 deletions pymaid/neuron_label.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from functools import cache
from typing import Optional, Union
from typing import Optional, Union, List, Tuple
import re

import networkx as nx
@@ -38,7 +38,7 @@ def __init__(
self,
skeleton_id: Optional[int] = None,
name: Optional[str] = None,
annotations: Optional[list[str]] = None,
annotations: Optional[List[str]] = None,
remote_instance: Optional[CatmaidInstance] = None,
) -> None:
"""
@@ -51,7 +51,7 @@ def __init__(
If None, determined from name.
name : Optional[str], optional
If None, determined from skeleton ID.
annotations : Optional[list[str]], optional
annotations : Optional[List[str]], optional
If None, determined from skeleton ID or name.
remote_instance : Optional[CatmaidInstance], optional
If None, uses global instance.
@@ -90,7 +90,7 @@ def name(self) -> str:
return self._name

@property
def annotations(self) -> list[str]:
def annotations(self) -> List[str]:
if self._annotations is None:
skid = self.skeleton_id
skid_to_anns = pymaid.get_annotations(skid)
@@ -155,8 +155,8 @@ def __init__(
super().__init__()

def _filter_by_author(
self, annotations: list[str], remote_instance: CatmaidInstance
) -> list[str]:
self, annotations: List[str], remote_instance: CatmaidInstance
) -> List[str]:
if self.annotator_name is None or not annotations:
return annotations

@@ -166,8 +166,8 @@ def _filter_by_author(
return [a for a in annotations if a in allowed]

def _filter_by_annotation(
self, annotations: list[str], remote_instance: CatmaidInstance
) -> list[str]:
self, annotations: List[str], remote_instance: CatmaidInstance
) -> List[str]:
if self.annotated_with is None or not annotations:
return annotations

@@ -193,7 +193,7 @@ def dedup_whitespace(s: str):
@cache
def parse_components(
fmt: str,
) -> tuple[list[str], list[tuple[str, int, Optional[str]]]]:
) -> Tuple[List[str], List[Tuple[str, int, Optional[str]]]]:
joiners = []
components = []
last_end = 0
@@ -264,7 +264,7 @@ class NeuronLabeller:
"""Class for calculating neurons' labels, as used in the CATMAID frontend."""
def __init__(
self,
components: Optional[list[LabelComponent]] = None,
components: Optional[List[LabelComponent]] = None,
fmt="%0",
trim_empty=True,
remove_neighboring_duplicates=True,
@@ -273,7 +273,7 @@ def __init__(
Parameters
----------
components : list[LabelComponent], optional
components : List[LabelComponent], optional
The label components as used in CATMAID's user settings.
See `SkeletonId`, `NeuronName`, and `Annotations`.
First component should be ``SkeletonId()`` for compatibility with CATMAID.
581 changes: 581 additions & 0 deletions pymaid/stack.py

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[build-system]
requires = ["setuptools", "extreqs"]
build-backend = "setuptools.build_meta"
10 changes: 10 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -9,6 +9,16 @@ scipy>=1.3.0
six>=1.11.0
tqdm>=4.50.0
psutil>=5.4.3

#extra: extras
ujson~=1.35

#extra: stack
zarr
fsspec[http]
xarray[parallel]
imageio

# diskcache>=4.0.0

# Below are optional dependencies
14 changes: 8 additions & 6 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import itertools
from setuptools import setup, find_packages
import re
from pathlib import Path

from extreqs import parse_requirement_files


VERSIONFILE = "pymaid/__init__.py"
@@ -11,9 +15,8 @@
else:
raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,))

with open('requirements.txt') as f:
requirements = f.read().splitlines()
requirements = [l for l in requirements if not l.startswith('#')]
install_requires, extras_require = parse_requirement_files(Path("requirements.txt"))
extras_require["all"] = list(set(itertools.chain.from_iterable(extras_require.values())))

setup(
name='python-catmaid',
@@ -45,9 +48,8 @@
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
],
install_requires=requirements,
extras_require={'extras': ['fuzzywuzzy[speedup]~=0.17.0',
'ujson~=1.35']},
install_requires=install_requires,
extras_require=extras_require,
python_requires='>=3.9',
zip_safe=False
)

0 comments on commit 0f05a46

Please sign in to comment.