From c6ba1c7a2fc0f5ef3a7d05f85b0b17c35ca58eba Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 08:41:25 +0200 Subject: [PATCH 1/6] Initial Min_Max.py commit Script which calculates the min/max for a given number of images and resets the StatsInfo if so directed. --- omero/util_scripts/Min_Max.py | 221 ++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 omero/util_scripts/Min_Max.py diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py new file mode 100644 index 000000000..fa49320db --- /dev/null +++ b/omero/util_scripts/Min_Max.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" + components/tools/OmeroPy/scripts/omero/util_scripts/Min_Max.py + +----------------------------------------------------------------------------- + Copyright (C) 2015 University of Dundee. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +------------------------------------------------------------------------------ +""" + +from omero.gateway import BlitzGateway +from omero.model import StatsInfoI +from omero.rtypes import rdouble, rlong, rstring + +import omero.scripts as scripts +import omero.util.script_utils as script_utils + +from collections import defaultdict + +from numpy import amin, amax, iinfo +from numpy import average as avg + + +def calcStatsInfo(conn, imageId): + """ + Process a single image here: creating a new StatsInfo object + if necessary. + + @param imageId: Original image + """ + + oldImage = conn.getObject("Image", imageId) + if oldImage is None: + raise Exception("Image not found for ID:" % imageId) + + # these dimensions don't change + sizeZ = oldImage.getSizeZ() + sizeC = oldImage.getSizeC() + sizeT = oldImage.getSizeT() + sizeX = oldImage.getSizeX() + sizeY = oldImage.getSizeY() + + # check we're not dealing with Big image. + rps = oldImage.getPrimaryPixels()._prepareRawPixelsStore() + bigImage = rps.requiresPixelsPyramid() + rps.close() + if bigImage: + raise Exception(( + "This script does not support 'BIG' images such as Image ID: " + "%s X: %d Y: %d") % (imageId, sizeX, sizeY)) + + # setup the (z,c,t) list of planes we need + zctMap = defaultdict(list) + for c in range(sizeC): + for z in range(sizeZ): + for t in range(sizeT): + zctMap[c].append((z, c, t)) + + def channelGen(): + pixels = oldImage.getPrimaryPixels() + rv = dict() + first = pixels.getPlane(0, 0, 0) + dt = first.dtype + # hack! TODO: add method to pixels to supply dtype + # get the planes one at a time - exceptions on getPlane() don't affect + # subsequent calls (new RawPixelsStore) + for c, zctList in zctMap.items(): + plane_min = iinfo(dt).max # Everything is less + plane_max = iinfo(dt).min # Everything is more + for i in range(len(zctList)): + if (0, 0, 0) == zctList[i]: + plane = first + else: + plane = pixels.getPlane(*zctList[i]) + plane_min = amin(plane) + plane_max = amax(plane) + rv[c] = (plane_min, plane_max) + yield rv + + statsInfos = dict() + for x in channelGen(): + statsInfos.update(x) + return statsInfos + + +def processImages(conn, scriptParams): + """ + Process the script params to make a list of channel_offsets, then iterate + through the images creating a new image from each with the specified + channel offsets + """ + + message = "" + images, logMessage = script_utils.getObjects(conn, scriptParams) + message += logMessage + if not images: + return None, None, message + imageIds = sorted(set([i.getId() for i in images])) + + globalmin = defaultdict(list) + globalmax = defaultdict(list) + + statsInfos = dict() + for iId in imageIds: + statsInfo = calcStatsInfo(conn, iId) + statsInfos[iId] = statsInfo + if scriptParams["DryRun"]: + print "Image:%s" % iId + for c, si in sorted(statsInfo.items()): + c_min, c_max = si + globalmin[c].append(c_min) + globalmax[c].append(c_max) + if scriptParams["DryRun"]: + print " c=%s, min=%s, max=%s" % (c, c_min, c_max) + + if scriptParams["DryRun"]: + for c in globalmin: + print "="*30 + print "Channel %s" % c + c_min = globalmin[c] + c_max = globalmax[c] + print "Max window: min=%s, max=%s" % (min(c_min), max(c_max)) + print "Min window: min=%s, max=%s" % (max(c_min), min(c_max)) + print "Avg window: min=%s, max=%s" % (avg(c_min), avg(c_max)) + print "="*30 + else: + method = scriptParams["Method"] + for iId in imageIds: + img = conn.getObject("Image", iId) + for c, ch in enumerate(img.getChannels(noRE=True)): + si = ch.getStatsInfo() + if si is None: + si = StatsInfoI() + action = "creating" + else: + si = si._obj + action = "updating" + + if method == "no": + si.globalMin = rdouble(statsInfos[iId][c][0]) + si.globalMax = rdouble(statsInfos[iId][c][1]) + elif method == "outer": + si.globalMin = rdouble(min(globalmin[c])) + si.globalMax = rdouble(max(globalmax[c])) + elif method == "inner": + si.globalMin = rdouble(max(globalmin[c])) + si.globalMax = rdouble(min(globalmax[c])) + elif method == "average": + si.globalMin = rdouble(avg(globalmin[c])) + si.globalMax = rdouble(avg(globalmax[c])) + + print "Image:%s(c=%s) - %s StatsInfo(%s, %s)" % ( + iId, c, action, si.globalMin.val, si.globalMax.val) + ch._obj.statsInfo = si + ch.save() + + count = sum(map(len, statsInfos.values())) + message += "%s stats info object(s) processed" % count + return message + + +def runAsScript(): + dataTypes = [rstring('Image')] + client = scripts.client( + 'MinMax.py', + """Create or reset StatsInfo objects for all channels + +See http://help.openmicroscopy.org/utility-scripts.html""", + + scripts.String( + "Data_Type", optional=False, grouping="1", + description="Pick Images by 'Image' ID or by the ID of their " + "Dataset'", values=dataTypes, default="Image"), + + scripts.List( + "IDs", optional=False, grouping="2", + description="List of Dataset IDs or Image IDs to " + "process.").ofType(rlong(0)), + + scripts.Bool( + "DryRun", optional=True, grouping="3", + description="Whether to print or set values", + default=True), + + scripts.String( + "Method", optional=True, grouping="4", + description="Whether and if so how to combine values", + default="no", + values=("no", "outer", "inner", "average")), + + version="5.1.3", + authors=["Josh Moore", "OME Team"], + institutions=["University of Dundee"], + contact="ome-users@lists.openmicroscopy.org.uk", + ) + + try: + scriptParams = client.getInputs(unwrap=True) + conn = BlitzGateway(client_obj=client) + message = processImages(conn, scriptParams) + client.setOutput("Message", rstring(message)) + + finally: + client.closeSession() + +if __name__ == "__main__": + runAsScript() From 4b10446b0061cfb64f4e9436eabce539ce47f0fe Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 08:50:35 +0200 Subject: [PATCH 2/6] Minor improvements --- omero/util_scripts/Min_Max.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py index fa49320db..9758afbd4 100644 --- a/omero/util_scripts/Min_Max.py +++ b/omero/util_scripts/Min_Max.py @@ -22,6 +22,9 @@ ------------------------------------------------------------------------------ """ +import omero.all + +from omero import MissingPyramidException from omero.gateway import BlitzGateway from omero.model import StatsInfoI from omero.rtypes import rdouble, rlong, rstring @@ -55,9 +58,14 @@ def calcStatsInfo(conn, imageId): sizeY = oldImage.getSizeY() # check we're not dealing with Big image. - rps = oldImage.getPrimaryPixels()._prepareRawPixelsStore() - bigImage = rps.requiresPixelsPyramid() - rps.close() + bigImage = False + try: + rps = oldImage.getPrimaryPixels()._prepareRawPixelsStore() + bigImage = rps.requiresPixelsPyramid() + rps.close() + except MissingPyramidException: + bigImage = True + if bigImage: raise Exception(( "This script does not support 'BIG' images such as Image ID: " @@ -108,7 +116,7 @@ def processImages(conn, scriptParams): images, logMessage = script_utils.getObjects(conn, scriptParams) message += logMessage if not images: - return None, None, message + raise Exception("No images found") imageIds = sorted(set([i.getId() for i in images])) globalmin = defaultdict(list) From 1c2b403df5f1fbf760b9c1ddb56032f0ab9f4bca Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 09:15:35 +0200 Subject: [PATCH 3/6] Add support for big images --- omero/util_scripts/Min_Max.py | 75 ++++++++++++++++------------------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py index 9758afbd4..5899d4e73 100644 --- a/omero/util_scripts/Min_Max.py +++ b/omero/util_scripts/Min_Max.py @@ -22,12 +22,10 @@ ------------------------------------------------------------------------------ """ -import omero.all - -from omero import MissingPyramidException from omero.gateway import BlitzGateway from omero.model import StatsInfoI from omero.rtypes import rdouble, rlong, rstring +from omero.util.tiles import TileLoop, TileLoopIteration import omero.scripts as scripts import omero.util.script_utils as script_utils @@ -50,53 +48,48 @@ def calcStatsInfo(conn, imageId): if oldImage is None: raise Exception("Image not found for ID:" % imageId) - # these dimensions don't change + sizeX = oldImage.getSizeX() + sizeY = oldImage.getSizeY() sizeZ = oldImage.getSizeZ() sizeC = oldImage.getSizeC() sizeT = oldImage.getSizeT() - sizeX = oldImage.getSizeX() - sizeY = oldImage.getSizeY() + tileW = min(256, sizeX) + tileH = min(256, sizeY) - # check we're not dealing with Big image. - bigImage = False - try: - rps = oldImage.getPrimaryPixels()._prepareRawPixelsStore() - bigImage = rps.requiresPixelsPyramid() - rps.close() - except MissingPyramidException: - bigImage = True - - if bigImage: - raise Exception(( - "This script does not support 'BIG' images such as Image ID: " - "%s X: %d Y: %d") % (imageId, sizeX, sizeY)) - - # setup the (z,c,t) list of planes we need zctMap = defaultdict(list) - for c in range(sizeC): - for z in range(sizeZ): - for t in range(sizeT): - zctMap[c].append((z, c, t)) + + class Loop(TileLoop): + + def createData(self): + return self + + def close(self): + pass + + class Iteration(TileLoopIteration): + + def run(self, data, z, c, t, x, y, + tileWidth, tileHeight, tileCount): + zctMap[c].append( + (z, c, t, (x, y, tileWidth, tileHeight))) + + Loop().forEachTile( + sizeX, sizeY, + sizeZ, sizeC, sizeT, + tileW, tileH, Iteration()) def channelGen(): pixels = oldImage.getPrimaryPixels() rv = dict() - first = pixels.getPlane(0, 0, 0) - dt = first.dtype - # hack! TODO: add method to pixels to supply dtype - # get the planes one at a time - exceptions on getPlane() don't affect - # subsequent calls (new RawPixelsStore) - for c, zctList in zctMap.items(): - plane_min = iinfo(dt).max # Everything is less - plane_max = iinfo(dt).min # Everything is more - for i in range(len(zctList)): - if (0, 0, 0) == zctList[i]: - plane = first - else: - plane = pixels.getPlane(*zctList[i]) - plane_min = amin(plane) - plane_max = amax(plane) - rv[c] = (plane_min, plane_max) + dt = pixels.getTile(0, 0, 0, (0, 0, 16, 16)).dtype + tile_min = iinfo(dt).max # Everything is less + tile_max = iinfo(dt).min # Everything is more + for c, zctTileList in zctMap.items(): + for tileInfo in zctTileList: + tile = pixels.getTile(*tileInfo) + tile_min = min(tile_min, amin(tile)) + tile_max = max(tile_max, amax(tile)) + rv[c] = (tile_min, tile_max) yield rv statsInfos = dict() From 9cb1874f2f2adacbbc74e5dace109a4c0ae41047 Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 09:26:53 +0200 Subject: [PATCH 4/6] Use TableBuilder for output --- omero/util_scripts/Min_Max.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py index 5899d4e73..2d07870fb 100644 --- a/omero/util_scripts/Min_Max.py +++ b/omero/util_scripts/Min_Max.py @@ -25,6 +25,7 @@ from omero.gateway import BlitzGateway from omero.model import StatsInfoI from omero.rtypes import rdouble, rlong, rstring +from omero.util.text import TableBuilder from omero.util.tiles import TileLoop, TileLoopIteration import omero.scripts as scripts @@ -115,29 +116,27 @@ def processImages(conn, scriptParams): globalmin = defaultdict(list) globalmax = defaultdict(list) + tb = TableBuilder("Context", "Channel", "Min", "Max") statsInfos = dict() for iId in imageIds: statsInfo = calcStatsInfo(conn, iId) statsInfos[iId] = statsInfo - if scriptParams["DryRun"]: - print "Image:%s" % iId for c, si in sorted(statsInfo.items()): c_min, c_max = si globalmin[c].append(c_min) globalmax[c].append(c_max) - if scriptParams["DryRun"]: - print " c=%s, min=%s, max=%s" % (c, c_min, c_max) + tb.row("Image:%s" % iId, c, c_min, c_max) + + tb.row("", "", "", "") + for c in globalmin: + c_min = globalmin[c] + c_max = globalmax[c] + tb.row("Total: outer ", c, min(c_min), max(c_max)) + tb.row("Total: inner ", c, max(c_min), min(c_max)) + tb.row("Total: average", c, avg(c_min), avg(c_max)) if scriptParams["DryRun"]: - for c in globalmin: - print "="*30 - print "Channel %s" % c - c_min = globalmin[c] - c_max = globalmax[c] - print "Max window: min=%s, max=%s" % (min(c_min), max(c_max)) - print "Min window: min=%s, max=%s" % (max(c_min), min(c_max)) - print "Avg window: min=%s, max=%s" % (avg(c_min), avg(c_max)) - print "="*30 + print str(tb.build()) else: method = scriptParams["Method"] for iId in imageIds: From ccabe3238d77aca30d99dea22bc5dc6699dba09c Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 11:14:48 +0200 Subject: [PATCH 5/6] Add 'default' and 'random' strategies Add options under the 'Choice' argument to reduce the processing overhead. Note: this does not yet search for random tiles across *all* images but rather per image --- omero/util_scripts/Min_Max.py | 56 ++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py index 2d07870fb..44c3ee71c 100644 --- a/omero/util_scripts/Min_Max.py +++ b/omero/util_scripts/Min_Max.py @@ -32,12 +32,13 @@ import omero.util.script_utils as script_utils from collections import defaultdict +from random import shuffle from numpy import amin, amax, iinfo from numpy import average as avg -def calcStatsInfo(conn, imageId): +def calcStatsInfo(conn, imageId, choice, debug=False): """ Process a single image here: creating a new StatsInfo object if necessary. @@ -67,10 +68,16 @@ def createData(self): def close(self): pass + only_default = (choice == "default") + class Iteration(TileLoopIteration): def run(self, data, z, c, t, x, y, tileWidth, tileHeight, tileCount): + + if only_default: + if t != 0 or z != int(sizeZ/2): + return zctMap[c].append( (z, c, t, (x, y, tileWidth, tileHeight))) @@ -79,7 +86,17 @@ def run(self, data, z, c, t, x, y, sizeZ, sizeC, sizeT, tileW, tileH, Iteration()) + if choice == "random": + for c in zctMap: + copy = list(zctMap[c]) + if len(copy) >= 100: + copy = copy[0:100] + shuffle(copy) + zctMap[c] = copy + def channelGen(): + byte_count = 0 + tile_count = 0 pixels = oldImage.getPrimaryPixels() rv = dict() dt = pixels.getTile(0, 0, 0, (0, 0, 16, 16)).dtype @@ -90,12 +107,16 @@ def channelGen(): tile = pixels.getTile(*tileInfo) tile_min = min(tile_min, amin(tile)) tile_max = max(tile_max, amax(tile)) + byte_count += len(tile) + tile_count += 1 rv[c] = (tile_min, tile_max) - yield rv + yield rv, byte_count, tile_count statsInfos = dict() - for x in channelGen(): + for x, byte_count, tile_count in channelGen(): statsInfos.update(x) + + print "Loaded %s tile(s) (%s bytes)" % (tile_count, byte_count) return statsInfos @@ -113,13 +134,15 @@ def processImages(conn, scriptParams): raise Exception("No images found") imageIds = sorted(set([i.getId() for i in images])) + choice = scriptParams["Choice"] + debug = bool(scriptParams.get("Debug", False)) globalmin = defaultdict(list) globalmax = defaultdict(list) tb = TableBuilder("Context", "Channel", "Min", "Max") statsInfos = dict() for iId in imageIds: - statsInfo = calcStatsInfo(conn, iId) + statsInfo = calcStatsInfo(conn, iId, choice, debug) statsInfos[iId] = statsInfo for c, si in sorted(statsInfo.items()): c_min, c_max = si @@ -138,7 +161,7 @@ def processImages(conn, scriptParams): if scriptParams["DryRun"]: print str(tb.build()) else: - method = scriptParams["Method"] + combine = scriptParams["Combine"] for iId in imageIds: img = conn.getObject("Image", iId) for c, ch in enumerate(img.getChannels(noRE=True)): @@ -150,18 +173,20 @@ def processImages(conn, scriptParams): si = si._obj action = "updating" - if method == "no": + if combine == "no": si.globalMin = rdouble(statsInfos[iId][c][0]) si.globalMax = rdouble(statsInfos[iId][c][1]) - elif method == "outer": + elif combine == "outer": si.globalMin = rdouble(min(globalmin[c])) si.globalMax = rdouble(max(globalmax[c])) - elif method == "inner": + elif combine == "inner": si.globalMin = rdouble(max(globalmin[c])) si.globalMax = rdouble(min(globalmax[c])) - elif method == "average": + elif combine == "average": si.globalMin = rdouble(avg(globalmin[c])) si.globalMax = rdouble(avg(globalmax[c])) + else: + raise Exception("unknown combine: %s" % combine) print "Image:%s(c=%s) - %s StatsInfo(%s, %s)" % ( iId, c, action, si.globalMin.val, si.globalMax.val) @@ -197,11 +222,22 @@ def runAsScript(): default=True), scripts.String( - "Method", optional=True, grouping="4", + "Choice", optional=True, grouping="4", + description="How to choose which planes will be chosen", + default="default", + values=("default", "random", "all")), + + scripts.String( + "Combine", optional=True, grouping="5", description="Whether and if so how to combine values", default="no", values=("no", "outer", "inner", "average")), + scripts.Bool( + "Debug", optional=True, grouping="6", + description="Whether to print debug statements", + default=False), + version="5.1.3", authors=["Josh Moore", "OME Team"], institutions=["University of Dundee"], From a5328845065e21f26b38442e4e58d8735c34ff5f Mon Sep 17 00:00:00 2001 From: jmoore Date: Wed, 1 Jul 2015 11:27:58 +0200 Subject: [PATCH 6/6] Improve debugging logic --- omero/util_scripts/Min_Max.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/omero/util_scripts/Min_Max.py b/omero/util_scripts/Min_Max.py index 44c3ee71c..0cb533b20 100644 --- a/omero/util_scripts/Min_Max.py +++ b/omero/util_scripts/Min_Max.py @@ -107,7 +107,7 @@ def channelGen(): tile = pixels.getTile(*tileInfo) tile_min = min(tile_min, amin(tile)) tile_max = max(tile_max, amax(tile)) - byte_count += len(tile) + byte_count += tile.nbytes tile_count += 1 rv[c] = (tile_min, tile_max) yield rv, byte_count, tile_count @@ -116,7 +116,8 @@ def channelGen(): for x, byte_count, tile_count in channelGen(): statsInfos.update(x) - print "Loaded %s tile(s) (%s bytes)" % (tile_count, byte_count) + if debug: + print "Loaded %s tile(s) (%s bytes)" % (tile_count, byte_count) return statsInfos @@ -156,7 +157,7 @@ def processImages(conn, scriptParams): c_max = globalmax[c] tb.row("Total: outer ", c, min(c_min), max(c_max)) tb.row("Total: inner ", c, max(c_min), min(c_max)) - tb.row("Total: average", c, avg(c_min), avg(c_max)) + tb.row("Total: average", c, int(avg(c_min)), int(avg(c_max))) if scriptParams["DryRun"]: print str(tb.build()) @@ -188,8 +189,9 @@ def processImages(conn, scriptParams): else: raise Exception("unknown combine: %s" % combine) - print "Image:%s(c=%s) - %s StatsInfo(%s, %s)" % ( - iId, c, action, si.globalMin.val, si.globalMax.val) + if debug: + print "Image:%s(c=%s) - %s StatsInfo(%s, %s)" % ( + iId, c, action, si.globalMin.val, si.globalMax.val) ch._obj.statsInfo = si ch.save()