diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/path.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/path.py index 6cd059d4..8330e98b 100644 --- a/src/python/janelia_emrp/msem/ingestion_ibeammsem/path.py +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/path.py @@ -40,12 +40,14 @@ def get_sfov_path(slab_path: Path, mfov: int, sfov: int) -> Path: The microscope numbering is 1-indexed: the sfov with ID=0 points to sfov_001.png """ - return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"sfov_{sfov+1:03}.png" + return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"sfov_{sfov + 1:03}.png" def get_thumbnail_path(slab_path: Path, mfov: int, sfov: int) -> Path: """Returns the path of the SFOV thumbnail given the slab path.""" - return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"thumbnail_{sfov+1:03}.png" + return ( + get_mfov_path(slab_path=slab_path, mfov=mfov) / f"thumbnail_{sfov + 1:03}.png" + ) def get_image_paths( diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/review.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/review.py new file mode 100644 index 00000000..2792ef6d --- /dev/null +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/review.py @@ -0,0 +1,127 @@ +"""Review of IBEAM-MSEM data. + +Before ingesting IBEAM-MSEM data, +the IBEAM-MSEM or data acquisition operators +may add a final review to document non-nominal items. + +This review is contained in an array in the xlog. + +The granularity level of the review array is XDim.MFOV. +Every MFOV has a set of one or more flags that describe the MFOV, +e.g. {Flag.NOMINAL} +or {Flag.DISTORTION_Y_LINEAR_MILD, Flag.OFFSET_SALVAGEABLE}. + +The ingestion operators take actions about MFOVs. +The actions are defined in ReviewAction, +e.g., ReviewAction.USE or ReviewAction.NO_Z_DROP. + +The mapping from a set of ReviewFlags to ReviewActions is called a review strategy. +It defines what ingestion action to take depending on the flags, +e.g. we ReviewAction.USE MFOVs with the flag {Flag.NOMINAL} +e.g. we ReviewAction.DROP_NO_Z MFOVs with the flags {Flag.TEST, Flag.DISTORTION_Y_LINEAR_MILD} + +We can define different strategies. +Strategies are labeled with integers. +E.g., in strategy #0, we are conservative + and decide to use only ReviewFlag.NOMINAL data + and drop all the rest. +E.g., in strategy #1, we are less conservative and ingest more edge cases. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import numpy as np +from janelia_emrp.msem.ingestion_ibeammsem.review.reviewstrategy import REVIEW_STRATEGY +from janelia_emrp.msem.ingestion_ibeammsem.review.reviewerror import ( + FlagSetWithNoActionError, +) +from janelia_emrp.msem.ingestion_ibeammsem.xdim import XDim +from janelia_emrp.msem.ingestion_ibeammsem.xvar import XVar +from janelia_emrp.msem.ingestion_ibeammsem.review.reviewflag import ReviewFlag + +if TYPE_CHECKING: + import xarray as xr + from janelia_emrp.msem.ingestion_ibeammsem.review.reviewaction import ReviewAction + + +def get_review_flag( + xlog: xr.Dataset, + scan: int | list[int] | np.ndarray | slice = slice(0, None), + slab: int | list[int] | np.ndarray | slice = slice(0, None), + mfov: int | list[int] | np.ndarray | slice = slice(0, None), +) -> xr.DataArray: + """Returns the review flags of MFOVs. + + Omit a dimension argument to select all items of the dimension. + E.g. get_review_flag(scan=12) + returns the review flags of all MFOVs in all slabs in scan 12. + """ + return xlog[XVar.REVIEW].sel(scan=scan, slab=slab, mfov=mfov) + + +def get_review_action( + review_flag: xr.DataArray, scan: int, slab: int, mfov: int, review_strategy: int +) -> ReviewAction: + """Gets the review action of an MFOV given a review flag array. + + The review flag array must contain the MFOV of interest. + + Possible use: + review_flag_slab = get_review_flag(slab=0).load() + for scan in scans: + for mfov in mfovs: + action = get_review_action(review_flag_slab, scan=scan, slab=0, mfov=mfov) + if action is Action.USE: + ... + elif action is Action.WITH_Z_MASK: + ... + """ + review_flag_mfov = review_flag.expand_dims( + tuple({XDim.SCAN, XDim.SLAB, XDim.MFOV} - set(review_flag.dims)) + ).sel(scan=scan, slab=slab, mfov=mfov) + key_flags: frozenset[int] = frozenset( + review_flag_mfov.where(review_flag_mfov) + .dropna(XDim.REVIEW_FLAG)[XDim.REVIEW_FLAG] + .values + ) + if key_flags not in REVIEW_STRATEGY[review_strategy]: + raise FlagSetWithNoActionError(set(key_flags)) + return REVIEW_STRATEGY[review_strategy][key_flags] + + +def get_flag_sets_without_action( + review: xr.DataArray, review_strategy: int +) -> list[set[ReviewFlag]]: + """Sets of flags that exist in the review array but miss a defined action. + + When starting a new ingestion for a new dataset, + or when extending the ingestion to more scans, + we may want to check up-front what sets of review flags + are missing an action, + instead of stopping multiple times at runtime with NoActionFlagErrors. + + E.g., the method could return + [ + { + , + , + }, + { + , + , + }, + ] + We should then add two more entries in the review strategy to handle these two cases. + """ + flag_patterns = np.unique( + review.stack(all_mfovs=tuple(set(review.dims) - {XDim.REVIEW_FLAG})).transpose( + "all_mfovs", ... + ), + axis=0, + ) + missing_keys = { + frozenset(np.nonzero(flag_pattern)[0]) for flag_pattern in flag_patterns + } - set(REVIEW_STRATEGY[review_strategy].keys()) + return [{ReviewFlag(flag_int) for flag_int in key} for key in missing_keys] diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewaction.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewaction.py new file mode 100644 index 00000000..ea24f518 --- /dev/null +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewaction.py @@ -0,0 +1,35 @@ +"""ReviewAction.""" + +from enum import IntEnum + + +class ReviewAction(IntEnum): + """Action about an item during data ingestion.""" + + USE = 0 + """Use the item for the ingestion. + + It is the nominal action. + """ + WITH_Z_MASK = 1 + """The item would have filled a Z-slice: mask it. + + WITH_Z means that a nominal image of the item is not available + and it would have filled a Z-slice slot. + """ + WITH_Z_INPAINT = 2 + """The item would have filled a Z-slice: inpaint it. + + WITH_Z means that a nominal image of the item is not available + and it would have filled a Z-slice slot. + """ + NO_Z_DROP = 3 + """The item would not have filled a Z-slice: drop it. + + We typically decide to NO_Z_DROP items with any of these ReviewFlag: + REDEPOSITED_MATERIAL + DEPLETED + NO_SAMPLE_IN_SLAB_NO_LOSS + + The item is not meant to fill a Z-slice. + """ diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewerror.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewerror.py new file mode 100644 index 00000000..6a4c2b2d --- /dev/null +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewerror.py @@ -0,0 +1,12 @@ +"""ReviewError""" + + +class ReviewError(Exception): + """ReviewError.""" + + +class FlagSetWithNoActionError(ReviewError): + """A set of flags is missing a defined action. + + You should add a new entry in the ReviewStrategy. + """ diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewflag.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewflag.py new file mode 100644 index 00000000..20c63322 --- /dev/null +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewflag.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from enum import IntEnum + + +class ReviewFlag(IntEnum): + """Review of the acquired data prior to ingestion. + + The xarray contains a variable REVIEW. + This enum defines the values of this variable. + """ + + NOMINAL = 0 + """nominal data""" + NO_FILE = 1 + """the file is missing""" + OFFSET_SALVAGEABLE = 2 + """data can be salvaged by applying an unusual spatial offset. + + The stage can suffer from lost step events. + After a lost step event, for all stage move requests to (x,y) + the stage moves to (x + dx, y + dy) instead of (x,y) + with the following properties: + 1. either dx or dy is non-zero + 2. (dx,dy) remains constant until + the lost step event is fixed + or until another lost step event occurs. + In most cases, the acquisition pipeline + 1. notices the events + 2. fixes the issue + 3. re-acquires affected items + so that the end data is not affected. + + In rare cases, some of the 3 items above have failed. + + We classify the symptoms at the MFOV level: + OFFSET_SALVAGEABLE + OFFSET_LOSS + + The drawings below show an example slab containing 3 MFOVs. + The imagery of interest is represented by ABCDEFGHIJKLM. + The drawing on the left shows the nominal case. + The drawing on the right shows the problem. + + MFOV #1 is labeled as OFFSET_LOSS: there is no useful data + + MFOVs #2 and #3 are labeled as OFFSET_SALVAGEABLE: + there is useful data + an unusually large spatial offset is needed + to align the data. + + The MFOVs do not cover the sample part "LM": it is a true data loss. + + MFOV#1 MFOV#2 MFOV#3 MFOV#1 MFOV#2 MFOV#3 + +------++------++------+ +------++------++------+ + | ABCD||EFGHIJ||KLM |-->| || ABCDE||FGHIJK|LM + | || || | | || || | + +------++------++------+ +------++------++------+ + """ + OFFSET_LOSS = 3 + """There is no data to salvage from this item. + + See OFFSET_SALVAGEABLE. + OFFSET_LOSS corresponds to MFOV#1 in the example drawing. + """ + DISTORTION_Y_LINEAR_MILD = 4 + """data has a mild y-axis linear distortion. + + Due to a failure of the scan amplifier of the microscope. + Applying linear transforms should fix the distortion. + There might be some true lost data between SFOVs + because they do not overlap. + """ + DISTORTION_Y_LINEAR_SEVERE = 5 + """data has a severe y-axis linear distortion.""" + DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN = 6 + """data has a severe y-axis linear distortion and has been retaken later.""" + REDEPOSITED_MATERIAL = 7 + """image of redeposited material which is not of interest. + + The electron irradiation step of IBEAM-MSEM + produces redeposited material at the surface of slabs. + The redeposited material is not part of the final dataset. + This flag is typically present in the early scans of an experiment, + e.g. scans [0,1,2]. + """ + DEPLETED = 8 + """IBEAM depleted the slab of material of interest. + + When IBEAM depleted a slab of its material of interest + and IBEAM-MSEM operators determined that the slab is finished, + they stop acquiring data for that slab. + This determination is made at the slab level, e.g., + one slab is still being acquired while another one is not acquired any more. + Several factors influence why some slabs require more scans than others. + + The determination from IBEAM-MSEM operators may be too conservative: + more scans of a slab were acquired than necessary, e.g., + starting from scan #75 onwards, the scans of a slab do not contain useful data. + The IBEAM-MSEM operators may, but do not have to, + flag the acquired data after scan #75 as DEPLETED. + """ + NO_SAMPLE_IN_SLAB_NO_LOSS = 9 + """the sample is missing from the slab but there is no lost data. + + For example, when mechanical sectioning of the sample block + is stopped to collect on a new wafer, + the first cut slab after the interruption might be partial, + e.g. slab #1 in drawing below. + That is, only one part of the slab is present + and the rest is physically missing. + The IBEAM-MSEM operator might decide + to still acquire the part that seems missing + in case there is some sample in the apparently missing slab part + that could not be seen in the light micrograph overview. + + The missing thickness does not necessarily rebalance at the next slab, + intead, the missing thickness might rebalance smoothly over several slabs. + + Side view of slabs: + + -----A-----------A----- slab #0 scan #0 + -----B-----------B----- slab #0 scan #1 + -----C-----------C----- + -----D-----------D----- + -----E-----------E----- + + -----F----- slab #1 is partial + -----G----- + + -----H-----------F----- slab #2 + -----I-----------G----- + -----J-----------H----- + (-----I----) the missing thickness does not + (-----J----) necessarily rebalance at the next slab + """ + TEST = 10 + """IBEAM-MSEM operators acquired some test data.""" + DISTORTION_Y_NONLINEAR_MAYBE = 11 + """It is possible that data has a y-axis non-linear distortion. + + Flag likely specific to Janelia wafers #60/#61 only. + A nonlinear distortion along the y axis affected some MFOVs during some scans. + The distortion occurred approximately randomly, + but is restricted to specific scans only. + All SFOVs of an affected MFOV show the same distortion. + """ diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewstrategy.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewstrategy.py new file mode 100644 index 00000000..41d28248 --- /dev/null +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewstrategy.py @@ -0,0 +1,56 @@ +"""Strategy for ingestion depending on review. + +A strategy is a mapping from a set of ReviewFlags to a ReviewAction: +dict[frozenset[Flag], Action] + +An MFOV might have the flags + { + Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN, + Flag.DISTORTION_Y_NONLINEAR_MAYBE, + }. +A strategy defines an action for MFOVs with such a set of flags. +""" + +from janelia_emrp.msem.ingestion_ibeammsem.review.reviewflag import ReviewFlag as Flag +from janelia_emrp.msem.ingestion_ibeammsem.review.reviewaction import ( + ReviewAction as Action, +) + +fset = frozenset + +REVIEW_STRATEGY: dict[int, dict[frozenset[Flag], Action]] = { + 0: { + # Action.USE + fset({Flag.NOMINAL}): Action.USE, + fset({Flag.DISTORTION_Y_LINEAR_MILD}): Action.USE, + fset({Flag.DISTORTION_Y_NONLINEAR_MAYBE}): Action.USE, + # Action.WITH_Z_MASK + fset({Flag.NO_FILE}): Action.WITH_Z_MASK, + fset({Flag.OFFSET_LOSS}): Action.WITH_Z_MASK, + fset({Flag.OFFSET_SALVAGEABLE}): Action.WITH_Z_MASK, + fset({Flag.DISTORTION_Y_LINEAR_SEVERE}): Action.WITH_Z_MASK, + # Action.NO_Z_DROP + fset({Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN}): Action.NO_Z_DROP, + fset( + { + Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN, + Flag.DISTORTION_Y_NONLINEAR_MAYBE, + } + ): Action.NO_Z_DROP, + fset({Flag.REDEPOSITED_MATERIAL}): Action.NO_Z_DROP, + fset( + {Flag.REDEPOSITED_MATERIAL, Flag.DISTORTION_Y_NONLINEAR_MAYBE} + ): Action.NO_Z_DROP, + fset( + { + Flag.REDEPOSITED_MATERIAL, + Flag.DISTORTION_Y_NONLINEAR_MAYBE, + Flag.NO_SAMPLE_IN_SLAB_NO_LOSS, + } + ): Action.NO_Z_DROP, + fset({Flag.DEPLETED}): Action.NO_Z_DROP, + fset({Flag.NO_SAMPLE_IN_SLAB_NO_LOSS}): Action.NO_Z_DROP, + fset({Flag.TEST}): Action.NO_Z_DROP, + }, + # 1: add your custom strategy +} diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/xdim.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/xdim.py index ee715d39..dc58db42 100644 --- a/src/python/janelia_emrp/msem/ingestion_ibeammsem/xdim.py +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/xdim.py @@ -16,3 +16,5 @@ class XDim(StrEnum): """pixels of an SFOV along the X axis""" Y_SFOV = auto() """pixels of an SFOV along the Y axis""" + REVIEW_FLAG = auto() + """review flags. See ReviewFlag for numbering reference.""" diff --git a/src/python/janelia_emrp/msem/ingestion_ibeammsem/xvar.py b/src/python/janelia_emrp/msem/ingestion_ibeammsem/xvar.py index f344f495..99573268 100644 --- a/src/python/janelia_emrp/msem/ingestion_ibeammsem/xvar.py +++ b/src/python/janelia_emrp/msem/ingestion_ibeammsem/xvar.py @@ -143,3 +143,15 @@ class XVar(StrEnum): ROTATION_SIMILARITY = auto() X_REFERENCE = auto() Y_REFERENCE = auto() + + REVIEW = auto() + """Review of acquired data by the IBEAM-MSEM operators. + + This array documents the review of the data prior to data ingestion. + It is an array of bool with MFOV granularity. + For every MFOV, it is an array of bool along the dimension REVIEW_FLAG. + + E.g. if review.sel(scan=0, slab=0, mfov=0, review_flag=10) == True + then this MFOV has the flag ReviewFlag.TEST + (ReviewFlag is an IntEnum and ReviewFlag.TEST == 10). + """ diff --git a/src/python/janelia_emrp/msem/slab_info.py b/src/python/janelia_emrp/msem/slab_info.py index d8f5b04e..4e715a48 100644 --- a/src/python/janelia_emrp/msem/slab_info.py +++ b/src/python/janelia_emrp/msem/slab_info.py @@ -87,10 +87,10 @@ def load_slab_info(xlog: xarray.Dataset, for slab in magc_ids: id_serial=get_serial_ids(xlog=xlog,magc_ids=[slab])[0] mfovs = get_mfovs(xlog=xlog, slab=slab) - region_ids = get_region_ids(xlog=xlog, slab=slab, mfovs=mfovs) - if len(region_ids) == 0: + if mfovs.size == 0: magc_ids_without_regions.append(slab) continue + region_ids = get_region_ids(xlog=xlog, slab=slab, mfovs=mfovs) id_region = region_ids[0] slabs.append(