diff --git a/ant/java.xml b/ant/java.xml index a9566591401..64058451b55 100644 --- a/ant/java.xml +++ b/ant/java.xml @@ -220,6 +220,30 @@ your FindBugs installation's lib directory. E.g.: + + + ${runtimePaths.string} + + + + + + + + + + + + + + + + + + + + + diff --git a/components/bundles/bioformats_package/assembly.xml b/components/bundles/bioformats_package/assembly.xml index b46896c4601..46c2e893cfd 100644 --- a/components/bundles/bioformats_package/assembly.xml +++ b/components/bundles/bioformats_package/assembly.xml @@ -36,4 +36,9 @@ false + + + metaInf-services + + diff --git a/components/formats-gpl/build.properties b/components/formats-gpl/build.properties index 51586ed4bf5..d49efc442ad 100644 --- a/components/formats-gpl/build.properties +++ b/components/formats-gpl/build.properties @@ -14,7 +14,8 @@ component.deprecation = true component.resources-bin = loci/formats/bio-formats-logo.png \ loci/formats/utests/2008-09.ome \ - lib/**/*.dll + lib/**/*.dll \ + META-INF/services/ucar.nc2.filter.FilterProvider component.resources-text = component.main-class = loci.formats.gui.ImageViewer diff --git a/components/formats-gpl/pom.xml b/components/formats-gpl/pom.xml index 70f7336c8ac..02e136cd640 100644 --- a/components/formats-gpl/pom.xml +++ b/components/formats-gpl/pom.xml @@ -120,6 +120,11 @@ kryo ${kryo.version} + + io.airlift + aircompressor + 0.27 + diff --git a/components/formats-gpl/src/META-INF/services/ucar.nc2.filter.FilterProvider b/components/formats-gpl/src/META-INF/services/ucar.nc2.filter.FilterProvider new file mode 100644 index 00000000000..90fa9817eaa --- /dev/null +++ b/components/formats-gpl/src/META-INF/services/ucar.nc2.filter.FilterProvider @@ -0,0 +1 @@ +loci.formats.filter.LZ4Filter$LZ4FilterProvider diff --git a/components/formats-gpl/src/loci/formats/filter/LZ4Filter.java b/components/formats-gpl/src/loci/formats/filter/LZ4Filter.java new file mode 100644 index 00000000000..c130f347734 --- /dev/null +++ b/components/formats-gpl/src/loci/formats/filter/LZ4Filter.java @@ -0,0 +1,98 @@ +/* + * #%L + * OME Bio-Formats package for reading and converting biological file formats. + * %% + * Copyright (C) 2005 - 2017 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +package loci.formats.filter; + +import ucar.nc2.filter.FilterProvider; +import ucar.nc2.filter.Filter; + +import io.airlift.compress.lz4.Lz4Decompressor; + +import java.util.Map; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class LZ4Filter extends Filter { + + static final String name = "LZ4 filter"; + static final int id = 32004; + private Lz4Decompressor decompressor; + + public LZ4Filter(Map properties) { + decompressor = new Lz4Decompressor(); + } + + @Override + public String getName() { + return name; + } + + @Override + public int getId() { + return id; + } + + @Override + public byte[] encode(byte[] dataIn) throws IOException { + throw new UnsupportedOperationException("LZ4Filter does not support data compression"); + } + + @Override + public byte[] decode(byte[] dataIn) throws IOException { + ByteBuffer byteBuffer = ByteBuffer.wrap(dataIn); + + long totalDecompressedSize = byteBuffer.getLong(); + int decompressedBlockSize = byteBuffer.getInt(); + int compressedBlockSize = byteBuffer.getInt(); + + byte[] decompressedBlock = new byte[Math.toIntExact(totalDecompressedSize)]; + byte[] compressedBlock = new byte[compressedBlockSize]; + + byteBuffer.get(compressedBlock, 0, compressedBlockSize); + decompressor.decompress(compressedBlock, 0, compressedBlockSize, decompressedBlock, 0, decompressedBlockSize); + + return decompressedBlock; + } + + public static class LZ4FilterProvider implements FilterProvider { + + @Override + public String getName() { + return name; + } + + @Override + public int getId() { + return id; + } + + @Override + public Filter create(Map properties) { + return new LZ4Filter(properties); + } + + } + +} diff --git a/components/formats-gpl/src/loci/formats/in/ImarisHDFReader.java b/components/formats-gpl/src/loci/formats/in/ImarisHDFReader.java index 930ce7f80c8..13827c8c2f0 100644 --- a/components/formats-gpl/src/loci/formats/in/ImarisHDFReader.java +++ b/components/formats-gpl/src/loci/formats/in/ImarisHDFReader.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Hashtable; import loci.common.DataTools; import loci.common.Location; @@ -71,6 +72,15 @@ public class ImarisHDFReader extends SubResolutionFormatReader { private List colors; private int lastChannel = 0; + // caching parameters + private long maxBufferSize = 1024 * 1024 * 1024; + private Object[] buffer = null; + private int[] blockSizeZPerResolution; + private int lastMinZ = 0; + private int lastMaxZ = 0; + private int lastRes = 0; + private int lastT = 0; + // -- Constructor -- /** Constructs a new Imaris HDF reader. */ @@ -223,6 +233,10 @@ public void close(boolean fileOnly) throws IOException { pixelSizeX = pixelSizeY = pixelSizeZ = 0; minX = minY = minZ = maxX = maxY = maxZ = 0; + lastRes = lastT = lastMinZ = lastMaxZ = 0; + buffer = null; + blockSizeZPerResolution = null; + if (netcdf != null) netcdf.close(); netcdf = null; @@ -300,12 +314,22 @@ protected void initFile(String id) throws FormatException, IOException { ms0.thumbnail = false; ms0.dimensionOrder = "XYZCT"; + // read block sizes for caching + blockSizeZPerResolution = new int[seriesCount]; + for (int res = 0; res < seriesCount; res++) { + String datasetPath = "DataSet/ResolutionLevel_" + res + "/TimePoint_0/Channel_0/Data"; + Hashtable table = netcdf.getVariableAttributes(datasetPath); + String chunkSizesString = (String) table.get("_ChunkSizes"); + String[] sizes = chunkSizesString.split(" "); + blockSizeZPerResolution[res] = Integer.parseInt(sizes[0]); + } + // determine pixel type - this isn't stored in the metadata, so we need // to check the pixels themselves int type = -1; - Object pix = getImageData(0, 0, 0, 1, 1); + Object pix = getSampleData(); if (pix instanceof byte[][]) type = FormatTools.UINT8; else if (pix instanceof short[][]) type = FormatTools.UINT16; else if (pix instanceof int[][]) type = FormatTools.UINT32; @@ -434,8 +458,7 @@ private Object getImageData(int no, int x, int y, int width, int height) throws FormatException { int[] zct = getZCTCoords(no); - String path = "/DataSet/ResolutionLevel_" + getCoreIndex() + "/TimePoint_" + - zct[2] + "/Channel_" + zct[1] + "/Data"; + int resolutionIndex = getCoreIndex(); Object image = null; // the width and height cannot be 1, because then netCDF will give us a @@ -467,14 +490,110 @@ private Object getImageData(int no, int x, int y, int width, int height) x = (getSizeX() / 2) - 1; } - int[] dimensions = new int[] {1, height, width}; - int[] indices = new int[] {zct[0], y, x}; + // cache dataset blocks to avoid multiple reads. + // use caching for 3D datasets and only if the size of the required buffer is < maxBufferSize + if (getSizeZ() > 1 && getSizeX() * getSizeY() * blockSizeZPerResolution[resolutionIndex] * getSizeC() * FormatTools.getBytesPerPixel(getPixelType()) < maxBufferSize) { + // update buffer if needed + if (zct[0] < lastMinZ || zct[0] > lastMaxZ || zct[2] != lastT || resolutionIndex != lastRes || buffer == null) { + buffer = new Object[getSizeC()]; + if (resolutionIndex != lastRes) { + lastRes = resolutionIndex; + lastT = zct[2]; + } + if (zct[2] != lastT) { + lastT = zct[2]; + } + + int blockNumber = zct[0] / blockSizeZPerResolution[resolutionIndex]; + int[] dims = new int[] {blockSizeZPerResolution[resolutionIndex], getSizeY(), getSizeX()}; + int[] idcs = new int[] {blockNumber * blockSizeZPerResolution[resolutionIndex], 0, 0}; + try { + String path; + for (int ch = 0; ch < getSizeC(); ch++) { + path = "/DataSet/ResolutionLevel_" + resolutionIndex + "/TimePoint_" + zct[2] + "/Channel_" + ch + "/Data"; + buffer[ch] = netcdf.getArray(path, idcs, dims); + } + } + catch (ServiceException e) { + throw new FormatException(e); + } + + lastMinZ = blockNumber * blockSizeZPerResolution[resolutionIndex]; + lastMaxZ = lastMinZ + blockSizeZPerResolution[resolutionIndex] - 1; + } + + // read from buffer + if (buffer[zct[1]] instanceof byte[][][]) { + byte[][][] byteBuffer = (byte[][][]) buffer[zct[1]]; + byte[][] slice = new byte[height][width]; + for (int i = y; i < y + height; i++) { + for (int j = x; j < x + width; j++) { + slice[i - y][j - x] = byteBuffer[zct[0] - lastMinZ][i][j]; + } + } + image = (Object) slice; + } + else if (buffer[zct[1]] instanceof short[][][]) { + short[][][] shortBuffer = (short[][][]) buffer[zct[1]]; + short[][] slice = new short[height][width]; + for (int i = y; i < y + height; i++) { + for (int j = x; j < x + width; j++) { + slice[i - y][j - x] = shortBuffer[zct[0] - lastMinZ][i][j]; + } + } + image = (Object) slice; + } + else if (buffer[zct[1]] instanceof int[][][]) { + int[][][] intBuffer = (int[][][]) buffer[zct[1]]; + int[][] slice = new int[height][width]; + for (int i = y; i < y + height; i++) { + for (int j = x; j < x + width; j++) { + slice[i - y][j - x] = intBuffer[zct[0] - lastMinZ][i][j]; + } + } + image = (Object) slice; + } + else if (buffer[zct[1]] instanceof float[][][]) { + float[][][] floatBuffer = (float[][][]) buffer[zct[1]]; + float[][] slice = new float[height][width]; + for (int i = y; i < y + height; i++) { + for (int j = x; j < x + width; j++) { + slice[i - y][j - x] = floatBuffer[zct[0] - lastMinZ][i][j]; + } + } + image = (Object) slice; + } + } + else { + int[] dimensions = new int[] {1, height, width}; + int[] indices = new int[] {zct[0], y, x}; + try { + String path = "/DataSet/ResolutionLevel_" + resolutionIndex + "/TimePoint_" + zct[2] + "/Channel_" + zct[1] + "/Data"; + image = netcdf.getArray(path, indices, dimensions); + } + catch (ServiceException e) { + throw new FormatException(e); + } + } + + return image; + } + + private Object getSampleData() + throws FormatException + { + int resolutionIndex = getCoreIndex(); + Object image = null; + int[] dimensions = new int[] {1, 2, 2}; + int[] indices = new int[] {0, 0, 0}; try { + String path = "/DataSet/ResolutionLevel_" + resolutionIndex + "/TimePoint_0/Channel_0/Data"; image = netcdf.getArray(path, indices, dimensions); } catch (ServiceException e) { throw new FormatException(e); } + return image; } diff --git a/components/formats-gpl/src/loci/formats/services/NetCDFServiceImpl.java b/components/formats-gpl/src/loci/formats/services/NetCDFServiceImpl.java index bf24c751883..48af84a42fb 100644 --- a/components/formats-gpl/src/loci/formats/services/NetCDFServiceImpl.java +++ b/components/formats-gpl/src/loci/formats/services/NetCDFServiceImpl.java @@ -45,6 +45,7 @@ import ucar.nc2.Attribute; import ucar.nc2.Group; import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; import ucar.nc2.Variable; import com.esotericsoftware.kryo.Kryo; @@ -247,7 +248,7 @@ private void parseAttributesAndVariables(List groups) { if (!groupName.endsWith("/")) variableName = "/" + variableName; variableList.add(variableName); } - groups = group.getGroups(); + groups = (List) group.getGroups(); parseAttributesAndVariables(groups); } } @@ -307,7 +308,7 @@ public void print(String s) { } }; System.setOut(throwaway); throwaway.close(); - netCDFFile = NetcdfFile.open(currentId); + netCDFFile = NetcdfFiles.open(currentId); System.setOut(outStream); root = netCDFFile.getRootGroup(); }