Skip to content

Commit

Permalink
Merge pull request #235 from clbarnes/n5_source
Browse files Browse the repository at this point in the history
Use CATMAID stack info to create xarray.DataArrays
clbarnes authored Jan 15, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents 647d688 + 7b402bc commit 0f05a46
Showing 10 changed files with 884 additions and 19 deletions.
16 changes: 15 additions & 1 deletion docs/source/api.rst
Original file line number Diff line number Diff line change
@@ -172,19 +172,33 @@ Functions for reconstruction samplers:
pymaid.get_sampler_domains
pymaid.get_sampler_counts

Image data (tiles)
Image metadata
--------------
Functions to fetch information about the image stacks CATMAID knows about.

.. autosummary::
:toctree: generated/

pymaid.get_stacks
pymaid.get_stack_info
pymaid.get_mirror_info

Image data (tiles and N5 volumes)
------------------
Functions to fetch and process image data. Note that this is not imported at
top level but has to be imported explicitly::

>>> from pymaid import tiles
>>> help(tiles.crop_neuron)
>>> from pymaid.stack import Stack
>>> help(Stack)

.. autosummary::
:toctree: generated/

pymaid.tiles.TileLoader
pymaid.tiles.crop_neuron
pymaid.stack.Stack

.. _api_misc:

2 changes: 2 additions & 0 deletions docs/source/whats_new.rst
Original file line number Diff line number Diff line change
@@ -15,6 +15,8 @@ What's new?
- BREAKING: Drop python 3.7 support.
- - :class:`pymaid.neuron_label.NeuronLabeller` added for labelling neurons
like in the CATMAID frontend.
- :func:`pymaid.get_stacks`, :func:`pymaid.get_stack_info`, :func:`pymaid.get_mirror_info` functions for getting information about image data
- :class:`pymaid.stack.Stack` class for accessing N5 and JPEG tile image data as a :class:`xarray.DataArray`
* - 2.4.0
- 27/05/23
- - :func:`pymaid.get_annotation_graph` deprecated in favour of the new
2 changes: 2 additions & 0 deletions pymaid/fetch/__init__.py
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@
from .landmarks import get_landmarks, get_landmark_groups
from .skeletons import get_skeleton_ids
from .annotations import get_annotation_graph, get_entity_graph, get_annotation_id
from .stack import get_stacks, get_stack_info, get_mirror_info


__all__ = ['get_annotation_details', 'get_annotation_id',
@@ -93,6 +94,7 @@
'get_landmarks',
'get_landmark_groups',
'get_skeleton_ids',
'get_stacks', 'get_stack_info', 'get_mirror_info',
]

# Set up logging
2 changes: 1 addition & 1 deletion pymaid/fetch/annotations.py
Original file line number Diff line number Diff line change
@@ -391,7 +391,7 @@ def get_entity_graph(
Neurons additionally have
- skeleton_ids: list[int]
- skeleton_ids: List[int]
Skeletons additionally have
251 changes: 251 additions & 0 deletions pymaid/fetch/stack.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
from typing import Any, Optional, Union, Literal, Sequence, Tuple, Dict, List
import numpy as np
from ..utils import _eval_remote_instance
from ..client import CatmaidInstance
from enum import IntEnum
from dataclasses import dataclass, asdict

Dimension = Literal["x", "y", "z"]


class Orientation(IntEnum):
XY = 0
# todo: check these
XZ = 1
ZY = 2

def __bool__(self) -> bool:
return True

def full_orientation(self, reverse=False) -> Tuple[Dimension, Dimension, Dimension]:
out = [
("x", "y", "z"),
("x", "z", "y"),
("z", "y", "x"),
][self.value]
if reverse:
out = out[::-1]
return out

@classmethod
def from_dims(cls, dims: Sequence[Dimension]):
pair = (dims[0].lower(), dims[1].lower())
out = {
("x", "y"): cls.XY,
("x", "z"): cls.XZ,
("z", "y"): cls.ZY,
}.get(pair)
if out is None:
raise ValueError(f"Unknown dimensions: {dims}")
return out


@dataclass
class StackSummary:
id: int
pid: int
title: str
comment: str


def get_stacks(remote_instance: Optional[CatmaidInstance] = None) -> List[StackSummary]:
"""Get summary of all stacks in the project.
Parameters
----------
remote_instance : Optional[CatmaidInstance], optional
By default global instance.
Returns
-------
stacks
List of StackSummary objects.
"""
cm = _eval_remote_instance(remote_instance)
url = cm.make_url(cm.project_id, "stacks")
return [StackSummary(**r) for r in cm.fetch(url)]


@dataclass
class MirrorInfo:
id: int
title: str
image_base: str
tile_width: int
tile_height: int
tile_source_type: int
file_extension: str
position: int

def to_jso(self):
return asdict(self)


@dataclass
class Color:
r: float
g: float
b: float
a: float


@dataclass
class StackInfo:
sid: int
pid: int
ptitle: str
stitle: str
downsample_factors: Optional[List[Dict[Dimension, float]]]
num_zoom_levels: int
translation: Dict[Dimension, float]
resolution: Dict[Dimension, float]
dimension: Dict[Dimension, int]
comment: str
description: str
metadata: Optional[str]
broken_slices: Dict[int, int]
mirrors: List[MirrorInfo]
orientation: Orientation
attribution: str
canary_location: Dict[Dimension, int]
placeholder_color: Color

@classmethod
def from_jso(cls, sinfo: Dict[str, Any]):
sinfo["orientation"] = Orientation(sinfo["orientation"])
sinfo["placeholder_color"] = Color(**sinfo["placeholder_color"])
sinfo["mirrors"] = [MirrorInfo(**m) for m in sinfo["mirrors"]]
return StackInfo(**sinfo)

def to_jso(self):
return asdict(self)

def get_downsample(self, scale_level=0) -> Dict[Dimension, float]:
"""Get the downsample factors for a given scale level.
If the downsample factors are explicit in the stack info,
use that value.
Otherwise, use the CATMAID default:
scale by a factor of 2 per scale level in everything except the slicing dimension.
If number of scale levels is known,
ensure the scale level exists.
Parameters
----------
scale_level : int, optional
Returns
-------
dict[Dimension, float]
Raises
------
IndexError
If the scale level is known not to exist
"""
if self.downsample_factors is not None:
return self.downsample_factors[scale_level]
if self.num_zoom_levels > 0 and scale_level >= self.num_zoom_levels:
raise IndexError("list index out of range")

first, second, slicing = self.orientation.full_orientation()
return {first: 2**scale_level, second: 2**scale_level, slicing: 1}

def get_coords(self, scale_level: int = 0) -> Dict[Dimension, np.ndarray]:
dims = self.orientation.full_orientation()
dims = dims[::-1]

downsamples = self.get_downsample(scale_level)

out: Dict[Dimension, np.ndarray] = dict()
for d in dims:
c = np.arange(self.dimension[d], dtype=float)
c *= self.resolution[d]
c *= downsamples[d]
c += self.translation[d]
out[d] = c
return out


def get_stack_info(
stack: Union[int, str], remote_instance: Optional[CatmaidInstance] = None
) -> StackInfo:
"""Get information about an image stack.
Parameters
----------
stack : Union[int, str]
Integer ID or string title of the stack.
remote_instance : Optional[CatmaidInstance], optional
By default global.
Returns
-------
StackInfo
Raises
------
ValueError
If an unknown stack title is given.
"""
cm = _eval_remote_instance(remote_instance)
if isinstance(stack, str):
stacks = get_stacks(cm)
for s in stacks:
if s.title == stack:
stack_id = s.id
break
else:
raise ValueError(f"No stack with title '{stack}'")
else:
stack_id = int(stack)

url = cm.make_url(cm.project_id, "stack", stack_id, "info")
sinfo = cm.fetch(url)
return StackInfo.from_jso(sinfo)


def get_mirror_info(
stack: Union[int, str, StackInfo],
mirror: Union[int, str],
remote_instance: Optional[CatmaidInstance] = None,
) -> MirrorInfo:
"""Get information about a stack mirror.
Parameters
----------
stack : Union[int, str, StackInfo]
Integer stack ID, string stack title,
or an existing StackInfo object (avoids server request).
mirror : Union[int, str]
Integer mirror ID, or string mirror title.
remote_instance : Optional[CatmaidInstance]
By default, global.
Returns
-------
MirrorInfo
Raises
------
ValueError
No mirror matching given ID/ title.
"""
if isinstance(stack, StackInfo):
stack_info = stack
else:
stack_info = get_stack_info(stack, remote_instance)

if isinstance(mirror, str):
key = "title"
else:
key = "id"
mirror = int(mirror)

for m in stack_info.mirrors:
if getattr(m, key) == mirror:
return m

raise ValueError(
f"No mirror for stack '{stack_info.stitle}' with {key} {repr(mirror)}"
)
22 changes: 11 additions & 11 deletions pymaid/neuron_label.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from abc import ABC, abstractmethod
from functools import cache
from typing import Optional, Union
from typing import Optional, Union, List, Tuple
import re

import networkx as nx
@@ -38,7 +38,7 @@ def __init__(
self,
skeleton_id: Optional[int] = None,
name: Optional[str] = None,
annotations: Optional[list[str]] = None,
annotations: Optional[List[str]] = None,
remote_instance: Optional[CatmaidInstance] = None,
) -> None:
"""
@@ -51,7 +51,7 @@ def __init__(
If None, determined from name.
name : Optional[str], optional
If None, determined from skeleton ID.
annotations : Optional[list[str]], optional
annotations : Optional[List[str]], optional
If None, determined from skeleton ID or name.
remote_instance : Optional[CatmaidInstance], optional
If None, uses global instance.
@@ -90,7 +90,7 @@ def name(self) -> str:
return self._name

@property
def annotations(self) -> list[str]:
def annotations(self) -> List[str]:
if self._annotations is None:
skid = self.skeleton_id
skid_to_anns = pymaid.get_annotations(skid)
@@ -155,8 +155,8 @@ def __init__(
super().__init__()

def _filter_by_author(
self, annotations: list[str], remote_instance: CatmaidInstance
) -> list[str]:
self, annotations: List[str], remote_instance: CatmaidInstance
) -> List[str]:
if self.annotator_name is None or not annotations:
return annotations

@@ -166,8 +166,8 @@ def _filter_by_author(
return [a for a in annotations if a in allowed]

def _filter_by_annotation(
self, annotations: list[str], remote_instance: CatmaidInstance
) -> list[str]:
self, annotations: List[str], remote_instance: CatmaidInstance
) -> List[str]:
if self.annotated_with is None or not annotations:
return annotations

@@ -193,7 +193,7 @@ def dedup_whitespace(s: str):
@cache
def parse_components(
fmt: str,
) -> tuple[list[str], list[tuple[str, int, Optional[str]]]]:
) -> Tuple[List[str], List[Tuple[str, int, Optional[str]]]]:
joiners = []
components = []
last_end = 0
@@ -264,7 +264,7 @@ class NeuronLabeller:
"""Class for calculating neurons' labels, as used in the CATMAID frontend."""
def __init__(
self,
components: Optional[list[LabelComponent]] = None,
components: Optional[List[LabelComponent]] = None,
fmt="%0",
trim_empty=True,
remove_neighboring_duplicates=True,
@@ -273,7 +273,7 @@ def __init__(
Parameters
----------
components : list[LabelComponent], optional
components : List[LabelComponent], optional
The label components as used in CATMAID's user settings.
See `SkeletonId`, `NeuronName`, and `Annotations`.
First component should be ``SkeletonId()`` for compatibility with CATMAID.
581 changes: 581 additions & 0 deletions pymaid/stack.py

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[build-system]
requires = ["setuptools", "extreqs"]
build-backend = "setuptools.build_meta"
10 changes: 10 additions & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -9,6 +9,16 @@ scipy>=1.3.0
six>=1.11.0
tqdm>=4.50.0
psutil>=5.4.3

#extra: extras
ujson~=1.35

#extra: stack
zarr
fsspec[http]
xarray[parallel]
imageio

# diskcache>=4.0.0

# Below are optional dependencies
14 changes: 8 additions & 6 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
import itertools
from setuptools import setup, find_packages
import re
from pathlib import Path

from extreqs import parse_requirement_files


VERSIONFILE = "pymaid/__init__.py"
@@ -11,9 +15,8 @@
else:
raise RuntimeError("Unable to find version string in %s." % (VERSIONFILE,))

with open('requirements.txt') as f:
requirements = f.read().splitlines()
requirements = [l for l in requirements if not l.startswith('#')]
install_requires, extras_require = parse_requirement_files(Path("requirements.txt"))
extras_require["all"] = list(set(itertools.chain.from_iterable(extras_require.values())))

setup(
name='python-catmaid',
@@ -45,9 +48,8 @@
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
],
install_requires=requirements,
extras_require={'extras': ['fuzzywuzzy[speedup]~=0.17.0',
'ujson~=1.35']},
install_requires=install_requires,
extras_require=extras_require,
python_requires='>=3.9',
zip_safe=False
)

0 comments on commit 0f05a46

Please sign in to comment.