From c6b8f7643129dc04b1594e0666675422a945917a Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Mon, 22 Jan 2018 16:03:40 +0000 Subject: [PATCH] Add roiimport plugin and readers for idr0012 and idr0016 --- plugins/IDR0012Reader.py | 65 +++++++++++ plugins/IDR0016Reader.py | 118 ++++++++++++++++++++ plugins/ROIReader.py | 21 ++++ plugins/roiimport.py | 226 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 430 insertions(+) create mode 100644 plugins/IDR0012Reader.py create mode 100644 plugins/IDR0016Reader.py create mode 100644 plugins/ROIReader.py create mode 100644 plugins/roiimport.py diff --git a/plugins/IDR0012Reader.py b/plugins/IDR0012Reader.py new file mode 100644 index 000000000..d23bda4d2 --- /dev/null +++ b/plugins/IDR0012Reader.py @@ -0,0 +1,65 @@ +from ROIReader import ROIReader + +from math import isnan + +import omero +from omero.rtypes import rint, rstring + + +class IDR0012Reader(ROIReader): + + # Relevant columns in the HDF5 file + COLUMN_X = "x" + COLUMN_Y = "y" + COLUMN_CLASS = "class" + COLUMN_FIELD = "spot" + + cellClasses = {'AF': 'Actin fibre', + 'BC': 'Big cell', + 'C': 'Condensed cell', + 'D': 'Debris', + 'LA': 'Lamellipodia', + 'M': 'Metaphase', + 'MB': 'Membrane blebbing', + 'N': 'Normal cell', + 'P': 'Protrusion/Elongation', + 'Z': 'Telophase'} + + def nextROIs(self): + + for plate in self.h5f.iter_nodes(self.h5f.root): + for well in self.h5f.iter_nodes(plate): + rois = {} + for row in well.iterrows(): + x = row[self.COLUMN_X] + y = row[self.COLUMN_Y] + cl = row[self.COLUMN_CLASS] + fld = row[self.COLUMN_FIELD] + + if not (isnan(x) or isnan(y)): + roi = omero.model.RoiI() + point = omero.model.PointI() + point.x = x + point.y = y + point.theZ = rint(0) + point.theT = rint(0) + if cl: + if cl in self.cellClasses: + point.textValue = rstring( + self.cellClasses[cl]) + else: + point.textValue = rstring(cl) + + roi.addShape(point) + + if fld not in rois: + rois[fld] = [] + + rois[fld].append(roi) + + for fld in rois: + wella = well._v_name[0] + wellb = "%d" % int(well._v_name[1:]) + wellname = wella + wellb + imgpos = '%s | %s | %s' % (plate._v_name, wellname, fld) + yield imgpos, rois[fld] diff --git a/plugins/IDR0016Reader.py b/plugins/IDR0016Reader.py new file mode 100644 index 000000000..8e31acb61 --- /dev/null +++ b/plugins/IDR0016Reader.py @@ -0,0 +1,118 @@ +from ROIReader import ROIReader + +import sys +from math import isnan + +import omero +from omero.rtypes import rint, rstring + +from tables import open_file + + +class IDR0016Reader(ROIReader): + + # Relevant columns in the HDF5 file + COLUMN_IMAGENUMBER = "ImageNumber" + COLUMN_WELLPOSITION = "Image_Metadata_CPD_WELL_POSITION" + COLUMN_PLATEID = "Image_Metadata_PlateID" + COLUMN_FIELD = "Image_Metadata_Site" + + NUCLEI_LOCATION_X = "Nuclei_Location_Center_X" + NUCLEI_LOCATION_Y = "Nuclei_Location_Center_Y" + CELLS_LOCATION_X = "Cells_Location_Center_X" + CELLS_LOCATION_Y = "Cells_Location_Center_Y" + CYTOPLASM_LOCATION_X = "Cytoplasm_Location_Center_X" + CYTOPLASM_LOCATION_Y = "Cytoplasm_Location_Center_Y" + + def __init__(self, hdfFile): + try: + h5f = open_file(hdfFile, "a") + objs = h5f.get_node("/Objects") + imgnoColumn = objs.colinstances[self.COLUMN_IMAGENUMBER] + if not imgnoColumn.is_indexed: + sys.stdout.write("Create index for the image number column...") + imgnoColumn.create_index() + except Exception: + sys.stderr.write("Could not create index. This will significantly " + "slow down reading performance!") + finally: + h5f.close() + + ROIReader.__init__(self, hdfFile) + + def _mapImageNumberToPosition(self): + """ + Maps the ImageNumber in the HDF5 file to plate positions + :param args: The arguments array + :return: A dictionary mapping the ImageNumber in the HDF5 file to + plate positions (in form 'PlateName | Well | Field') + """ + imgdict = {} + try: + imgs = self.h5f.get_node("/Images") + + # Map image number to image position (in form + # 'PlateName | Well | Field') + for row in imgs: + well = row[self.COLUMN_WELLPOSITION] + # Wells can have leading zero, e.g. A03, have to strip the zero + # to match e.g. A3 + wella = well[0] + wellb = "%d" % int(well[1:]) + well = wella + wellb + imgpos = "%s | %s | %s" % (row[self.COLUMN_PLATEID], well, + row[self.COLUMN_FIELD]) + imgdict[row[self.COLUMN_IMAGENUMBER]] = imgpos + except Exception: + sys.stderr.write("Could not map image numbers to plate positions.") + return imgdict + + def nextROIs(self): + imgNumbers = self._mapImageNumberToPosition() + + objs = self.h5f.get_node("/Objects") + for imgNumber in imgNumbers: + imgpos = imgNumbers[imgNumber] + rois = [] + query = self.COLUMN_IMAGENUMBER + " == " + str(imgNumber) + for row in objs.where(query): + x = row[self.NUCLEI_LOCATION_X] + y = row[self.NUCLEI_LOCATION_Y] + if not (isnan(x) or isnan(y)): + roi = omero.model.RoiI() + point = omero.model.PointI() + point.x = x + point.y = y + point.theZ = rint(0) + point.theT = rint(0) + point.textValue = rstring("Nucleus") + roi.addShape(point) + rois.append(roi) + + x = row[self.CELLS_LOCATION_X] + y = row[self.CELLS_LOCATION_Y] + if not (isnan(x) or isnan(y)): + roi = omero.model.RoiI() + point = omero.model.PointI() + point.x = x + point.y = y + point.theZ = rint(0) + point.theT = rint(0) + point.textValue = rstring("Cell") + roi.addShape(point) + rois.append(roi) + + x = row[self.CYTOPLASM_LOCATION_X] + y = row[self.CYTOPLASM_LOCATION_Y] + if not (isnan(x) or isnan(y)): + roi = omero.model.RoiI() + point = omero.model.PointI() + point.x = x + point.y = y + point.theZ = rint(0) + point.theT = rint(0) + point.textValue = rstring("Cytoplasm") + roi.addShape(point) + rois.append(roi) + + yield imgpos, rois diff --git a/plugins/ROIReader.py b/plugins/ROIReader.py new file mode 100644 index 000000000..9a01ce45b --- /dev/null +++ b/plugins/ROIReader.py @@ -0,0 +1,21 @@ +from tables import open_file + + +class ROIReader: + + def __init__(self, hdfFile): + self.h5f = open_file(hdfFile, "r") + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.h5f.close() + + def nextROIs(self): + """ + Get the next batch of ROIs for an Image + :return: Image position in form 'PlateName | Well | Field' + and a list of ROIs for this image + """ + raise NotImplementedError("Not implemented yet") diff --git a/plugins/roiimport.py b/plugins/roiimport.py new file mode 100644 index 000000000..cdab14add --- /dev/null +++ b/plugins/roiimport.py @@ -0,0 +1,226 @@ +from omero.cli import BaseControl, CLI, ExceptionHandler + +import omero +from omero.rtypes import rlong +from omero.cmd import Delete2 + +from parse import parse + +import signal +from time import time + +import importlib + +HELP = """Plugin for importing IDR ROIs""" + + +class ROIImportControl(BaseControl): + + exitNow = False + + def _configure(self, parser): + signal.signal(signal.SIGINT, self._exitGracefully) + signal.signal(signal.SIGTERM, self._exitGracefully) + + self.exc = ExceptionHandler() + + parser.add_login_arguments() + + parser.add_argument( + "command", nargs="?", + choices=("import", "remove"), + help="The operation to be performed. CAUTION: 'remove' " + "will delete all ROIs of the screen!") + + parser.add_argument( + "file", + help="The HDF5 file") + + parser.add_argument( + "screenId", + help="The screen id") + + parser.add_argument( + "--format", help="Specify the format of the HDF5 file " + "(e.g. idr0012)") + + parser.add_argument( + "--dry-run", action="store_true", + help="Does not write anything to OMERO") + + parser.set_defaults(func=self.process) + + def process(self, args): + if not args.command: + self.ctx.die(100, "No command provided") + + if args.command == "import": + self.importFile(args) + + if args.command == "remove": + self.remove(args) + + def _exitGracefully(self, signum, frame): + self.ctx.out("Caught exit signal, will stop when current transaction " + "is finished.") + self.exitNow = True + + def _mapImagePositionToId(self, queryService, screenid): + """ + Map all image names (in form 'PlateName | Well | Field') + to their OMERO image ids + :param queryService: Reference to the query service + :param screenid: The screen id + :return: A dictionary mapping 'PlateName | Well | Field' + to the image ID + """ + params = omero.sys.Parameters() + params.map = {"sid": rlong(screenid)} + query = "select i.id, i.name from Screen s " \ + "right outer join s.plateLinks as pl " \ + "join pl.child as p " \ + "right outer join p.wells as w " \ + "right outer join w.wellSamples as ws " \ + "join ws.image as i " \ + "where s.id = :sid" + imgdic = {} + for e in queryService.projection(query, params): + imgId = e[0].val + imgName = e[1].val + p = parse("{} [Well {}, Field {}]", imgName) + imgName = "%s | %s | %s" % (p[0], p[1], p[2]) + imgdic[imgName] = imgId + + return imgdic + + def _getROICount(self, queryService, imgId): + try: + params = omero.sys.Parameters() + params.map = {"imageId": rlong(imgId)} + query = "select count(*) from Roi as roi " \ + "where roi.image.id = :imageId" + count = queryService.projection(query, params) + return count[0][0]._val + + except Exception: + self.ctx.err("Could not get ROI count for image %s" % str(imgId)) + return 0 + + def _saveROIs(self, rois, imgId, queryService, updateService): + """ + Save the ROIs back to OMERO + :param rois: A list of ROIs + :param imgId: The image ID to attach the ROIs to + :param queryService: Reference to the query service + :param updateService: Reference to the update service + (can be None to simulate a 'dry-run') + :return: + """ + try: + if self._getROICount(queryService, imgId) > 0: + self.ctx.out("Skipping image %s, already has " + "ROIs attached." % imgId) + + else: + image = queryService.get("Image", imgId) + for roi in rois: + roi.setImage(image) + + if updateService: + updateService.saveCollection(rois) + self.ctx.out("Saved %d ROIs for Image %s" % + (len(rois), imgId)) + else: + self.ctx.out("Dry run - Would save %d ROIs for Image %s" % + (len(rois), imgId)) + + except Exception: + self.ctx.err("WARNING: Could not save the ROIs for Image %s" % + imgId) + + def _getReader(self, args): + try: + name = "%sReader" % args.format.upper() + mod = importlib.import_module(name) + reader = getattr(mod, name) + return reader + except Exception: + return None + + def importFile(self, args): + self.ctx.out("Import ROIs from file %s for screen %s" % + (args.file, args.screenId)) + + conn = self.ctx.conn(args) + self.sf = conn.sf + updateService = conn.sf.getUpdateService() + queryService = conn.sf.getQueryService() + + imgpositions = self._mapImagePositionToId(queryService, args.screenId) + total = len(imgpositions) + print("Mapped %d image ids to plate positions" % total) + + done = 0 + start = time() + readerClass = self._getReader(args) + if readerClass is None: + self.ctx.die(1, "No reader found for format %s" % args.format) + + with readerClass(args.file) as reader: + for imgpos, rois in reader.nextROIs(): + if imgpos in imgpositions: + if args.dry_run: + self._saveROIs(rois, imgpositions[imgpos], + queryService, None) + else: + self._saveROIs(rois, imgpositions[imgpos], + queryService, updateService) + else: + self.ctx.err("WARNING: Could not map image %s to" + " an OMERO image id." % imgpos) + done += 1 + + if done % 100 == 0: + percDone = int(done * 100 / total) + duration = (time() - start) / 100 + left = duration * (total - done) + m, s = divmod(left, 60) + h, m = divmod(m, 60) + start = time() + self.ctx.out("%d %% Done. ETR: %d:%02d:%02d hrs" + % (percDone, h, m, s)) + + if self.exitNow: + self.ctx.die(0, "") + + def remove(self, args): + self.ctx.out("Delete ROIs for screen %s" % args.screenId) + + conn = self.ctx.conn(args) + queryService = conn.sf.getQueryService() + + params = omero.sys.Parameters() + params.map = {"sid": rlong(args.screenId)} + query = "select r.id from Screen s " \ + "right outer join s.plateLinks as pl " \ + "join pl.child as p " \ + "right outer join p.wells as w " \ + "right outer join w.wellSamples as ws " \ + "join ws.image as i " \ + "join i.rois as r " \ + "where s.id = :sid" + roiIds = [] + for e in queryService.projection(query, params): + roiIds.append(e[0].val) + + delete = Delete2(targetObjects={'Roi': roiIds}) + conn.sf.submit(delete) + + +try: + register("roiimport", ROIImportControl, HELP) +except NameError: + if __name__ == "__main__": + cli = CLI() + cli.register("roiimport", ROIImportControl, HELP) + cli.invoke(omero.sys.argv[1:])