diff --git a/pom.xml b/pom.xml index ca08d16..63af9fa 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,24 @@ dev.zarr jzarr 0.4.2 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + + org.apache.httpcomponents + httpcore + + @@ -91,9 +109,22 @@ 2.7.2 runtime - - - + + com.amazonaws + aws-java-sdk-s3 + 1.12.659 + + + commons-logging + commons-logging + + + + + commons-logging + commons-logging + 1.2 + diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java new file mode 100644 index 0000000..2d5a40f --- /dev/null +++ b/src/loci/formats/S3FileSystemStore.java @@ -0,0 +1,245 @@ +package loci.formats; + +/*- + * #%L + * Implementation of Bio-Formats readers for the next-generation file formats + * %% + * Copyright (C) 2020 - 2022 Open Microscopy Environment + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +import com.bc.zarr.ZarrConstants; +import com.bc.zarr.ZarrUtils; +import com.bc.zarr.storage.Store; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.ListIterator; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; + +public class S3FileSystemStore implements Store { + + private Path root; + AmazonS3 client; + public static final String ENDPOINT_PROTOCOL= "https://"; + protected static final Logger LOGGER = + LoggerFactory.getLogger(S3FileSystemStore.class); + + public S3FileSystemStore(String path, FileSystem fileSystem) { + if (fileSystem == null) { + root = Paths.get(path); + } else { + root = fileSystem.getPath(path); + } + setupClient(); + } + + public void updateRoot(String path) { + root = Paths.get(path); + } + + public String getRoot() { + return root.toString(); + } + + private void setupClient() { + String[] pathSplit = root.toString().split(File.separator); + String endpoint = ENDPOINT_PROTOCOL + pathSplit[1] + File.separator; + try { + client = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) + .withPathStyleAccessEnabled(true) + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); + } catch (Exception e) { + LOGGER.info("Exception caught while constructing S3 client", e); + } + + } + + public void close() { + if (client != null) { + client.shutdown(); + } + } + + public S3FileSystemStore(Path rootPath) { + root = rootPath; + setupClient(); + } + + @Override + public InputStream getInputStream(String key) throws IOException { + // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point + String[] pathSplit = root.toString().split(File.separator); + String bucketName = pathSplit[2]; + + // Append the desired key onto the remaining prefix + String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()) + File.separator + key; + + try { + S3Object o = client.getObject(bucketName, key2); + S3ObjectInputStream responseStream = o.getObjectContent(); + return responseStream; + } catch (Exception e) { + LOGGER.info( "Unable to locate or access key: " + key2, e); + } + + return null; + } + + @Override + public OutputStream getOutputStream(String key) throws IOException { + final Path filePath = root.resolve(key); + final Path dir = filePath.getParent(); + Files.createDirectories(dir); + return Files.newOutputStream(filePath); + } + + @Override + public void delete(String key) throws IOException { + final Path toBeDeleted = root.resolve(key); + if (Files.isDirectory(toBeDeleted)) { + ZarrUtils.deleteDirectoryTreeRecursively(toBeDeleted); + } + if (Files.exists(toBeDeleted)){ + Files.delete(toBeDeleted); + } + if (Files.exists(toBeDeleted)|| Files.isDirectory(toBeDeleted)) { + throw new IOException("Unable to initialize " + toBeDeleted.toAbsolutePath().toString()); + } + } + + @Override + public TreeSet getArrayKeys() throws IOException { + return getKeysFor(ZarrConstants.FILENAME_DOT_ZARRAY); + } + + @Override + public TreeSet getGroupKeys() throws IOException { + return getKeysFor(ZarrConstants.FILENAME_DOT_ZGROUP); + } + + /** + * Copied from {@com.bc.zarr.storage.FileSystemStorage#getKeysEndingWith(String). + * + * @param suffix + * @return + * @throws IOException + */ + public TreeSet getKeysEndingWith(String suffix) throws IOException { + return (TreeSet)Files.walk(this.root).filter((path) -> { + return path.toString().endsWith(suffix); + }).map((path) -> { + return this.root.relativize(path).toString(); + }).collect(Collectors.toCollection(TreeSet::new)); + } + + /** + * Copied from {@com.bc.zarr.storage.FileSystemStorage#getRelativeLeafKeys(String). + * + * @param key + * @return + * @throws IOException + */ + public Stream getRelativeLeafKeys(String key) throws IOException { + Path walkingRoot = this.root.resolve(key); + return Files.walk(walkingRoot).filter((path) -> { + return !Files.isDirectory(path, new LinkOption[0]); + }).map((path) -> { + return walkingRoot.relativize(path).toString(); + }).map(ZarrUtils::normalizeStoragePath).filter((s) -> { + return s.trim().length() > 0; + }); + } + + private TreeSet getKeysFor(String suffix) throws IOException { + TreeSet keys = new TreeSet(); + + // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point + String[] pathSplit = root.toString().split(File.separator); + String bucketName = pathSplit[2]; + + // Append the desired key onto the remaining prefix + String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); + + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(key2) + ; + + ObjectListing listObjectsResponse = null; + String lastKey = null; + + do { + if ( listObjectsResponse != null ) { + listObjectsRequest = listObjectsRequest + .withMarker(lastKey) + ; + } + + listObjectsResponse = client.listObjects(listObjectsRequest); + List objects = listObjectsResponse.getObjectSummaries(); + + // Iterate over results + ListIterator iterVals = objects.listIterator(); + while (iterVals.hasNext()) { + S3ObjectSummary object = (S3ObjectSummary) iterVals.next(); + String k = object.getKey(); + if (k.contains(suffix)) { + String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.indexOf(suffix)); + if (!key.isEmpty()) { + keys.add(key.substring(0, key.length()-1)); + } + } + lastKey = k; + } + } while ( listObjectsResponse.isTruncated() ); + + return keys; + } +} \ No newline at end of file diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 5bfc736..efd2a31 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -97,6 +97,8 @@ public class ZarrReader extends FormatReader { public static final String LIST_PIXELS_ENV_KEY = "OME_ZARR_LIST_PIXELS"; public static final String INCLUDE_LABELS_KEY = "omezarr.include_labels"; public static final boolean INCLUDE_LABELS_DEFAULT = false; + public static final String ALT_STORE_KEY = "omezarr.alt_store"; + public static final String ALT_STORE_DEFAULT = null; protected transient ZarrService zarrService; private ArrayList arrayPaths = new ArrayList(); @@ -189,7 +191,7 @@ protected void initFile(String id) throws FormatException, IOException { Location omeMetaFile = new Location( zarrRootPath + File.separator + "OME", "METADATA.ome.xml" ); String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); - initializeZarrService(canonicalPath); + initializeZarrService(); reloadOptionsFile(zarrRootPath); ArrayList omeSeriesOrder = new ArrayList(); @@ -445,7 +447,7 @@ private int[] get5DShape(int [] originalShape) { * @param size of the shape required by jzarr * @return a 5D shape to be used within the reader */ - private int[] getOriginalShape(int [] shape5D, int size) { + private static int[] getOriginalShape(int [] shape5D, int size) { int [] shape = new int[size]; int shape5DIndex = 4; for (int s = shape.length - 1; s >= 0; s--) { @@ -460,15 +462,15 @@ private int[] getOriginalShape(int [] shape5D, int size) { public void reopenFile() throws IOException { try { String canonicalPath = new Location(currentId).getCanonicalPath(); - initializeZarrService(canonicalPath); + initializeZarrService(); } catch (FormatException e) { throw new IOException(e); } } - protected void initializeZarrService(String rootPath) throws IOException, FormatException { - zarrService = new JZarrServiceImpl(rootPath); + protected void initializeZarrService() throws IOException, FormatException { + zarrService = new JZarrServiceImpl(altStore()); openZarr(); } @@ -1145,6 +1147,7 @@ protected ArrayList getAvailableOptions() { optionsList.add(LIST_PIXELS_KEY); optionsList.add(QUICK_READ_KEY); optionsList.add(INCLUDE_LABELS_KEY); + optionsList.add(ALT_STORE_KEY); return optionsList; } @@ -1200,6 +1203,19 @@ public boolean includeLabels() { } return INCLUDE_LABELS_DEFAULT; } + + /** + * Used to provide the location of an alternative file store where the data is located + * @return String representing the root path of the alternative file store or null if no alternative location exist + */ + public String altStore() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).get( + ALT_STORE_KEY, ALT_STORE_DEFAULT); + } + return ALT_STORE_DEFAULT; + } private boolean systemEnvListPixels() { String value = System.getenv(LIST_PIXELS_ENV_KEY); diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index dce0186..732b8c2 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -1,5 +1,7 @@ package loci.formats.services; +import java.io.File; + /*- * #%L * Implementation of Bio-Formats readers for the next-generation file formats @@ -54,6 +56,7 @@ import loci.common.services.AbstractService; import loci.formats.FormatException; import loci.formats.FormatTools; +import loci.formats.S3FileSystemStore; import loci.formats.meta.IPyramidStore; import loci.formats.meta.MetadataRetrieve; import ucar.ma2.InvalidRangeException; @@ -65,6 +68,7 @@ public class JZarrServiceImpl extends AbstractService public static final String NO_ZARR_MSG = "JZARR is required to read Zarr files."; // -- Fields -- + S3FileSystemStore s3fs; ZarrArray zarrArray; String currentId; Compressor zlibComp = CompressorFactory.create("zlib", "level", 8); // 8 = compression level .. valid values 0 .. 9 @@ -76,21 +80,21 @@ public class JZarrServiceImpl extends AbstractService */ public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); - if (root != null && root.toLowerCase().contains("s3:")) { - LOGGER.warn("S3 access currently not supported"); + if (root != null && (root.toLowerCase().contains("s3:") || root.toLowerCase().contains("s3."))) { + String[] pathSplit = root.toString().split(File.separator); + if (S3FileSystemStore.ENDPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { + s3fs = new S3FileSystemStore(Paths.get(root)); + } + else { + LOGGER.warn("Zarr Reader is not using S3FileSystemStore as this is currently for use with S3 configured with a https endpoint"); + } } } @Override public void open(String file) throws IOException, FormatException { currentId = file; - // TODO: Update s3 location identification - if (!file.toLowerCase().contains("s3:")) { - zarrArray = ZarrArray.open(file); - } - else { - LOGGER.warn("S3 access currently not supported"); - } + zarrArray = getArray(file); } public void open(String id, ZarrArray array) { @@ -99,51 +103,19 @@ public void open(String id, ZarrArray array) { } public Map getGroupAttr(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { - group = ZarrGroup.open(path); - } - else { - LOGGER.warn("S3 access currently not supported"); - return null; - } - return group.getAttributes(); + return getGroup(path).getAttributes(); } public Map getArrayAttr(String path) throws IOException, FormatException { - ZarrArray array = null; - if (!path.toLowerCase().contains("s3:")) { - array = ZarrArray.open(path); - } - else { - LOGGER.warn("S3 access currently not supported"); - return null; - } - return array.getAttributes(); + return getArray(path).getAttributes(); } public Set getGroupKeys(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { - group = ZarrGroup.open(path); - } - else { - LOGGER.warn("S3 access currently not supported"); - return null; - } - return group.getGroupKeys(); + return getGroup(path).getGroupKeys(); } public Set getArrayKeys(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { - group = ZarrGroup.open(path); - } - else { - LOGGER.warn("S3 access currently not supported"); - return null; - } - return group.getArrayKeys(); + return getGroup(path).getArrayKeys(); } public DataType getZarrPixelType(int pixType) { @@ -247,6 +219,9 @@ public boolean isLittleEndian() { public void close() throws IOException { zarrArray = null; currentId = null; + if (s3fs != null) { + s3fs.close(); + } } @Override @@ -358,5 +333,39 @@ public void create(String id, MetadataRetrieve meta, int[] chunks) throws IOExce create(id, meta, chunks, Compression.NONE); } + private String stripZarrRoot(String path) { + return path.substring(path.indexOf(".zarr")+5); + } + + private String getZarrRoot(String path) { + return path.substring(0, path.indexOf(".zarr")+5); + } + private ZarrGroup getGroup(String path) throws IOException { + ZarrGroup group = null; + if (s3fs == null) { + group = ZarrGroup.open(path); + } + else { + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + group = ZarrGroup.open(s3fs); + } + return group; + } + + private ZarrArray getArray(String path) throws IOException { + ZarrArray array = null; + if (s3fs == null) { + array = ZarrArray.open(path); + } + else { + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + array = ZarrArray.open(s3fs); + } + return array; + } + + public boolean usingS3FileSystemStore() { + return s3fs != null; + } } diff --git a/test/loci/formats/utests/ZarrReaderMock.java b/test/loci/formats/utests/ZarrReaderMock.java index 8575e4a..bc0cd23 100644 --- a/test/loci/formats/utests/ZarrReaderMock.java +++ b/test/loci/formats/utests/ZarrReaderMock.java @@ -44,7 +44,7 @@ public ZarrReaderMock(ZarrService zarrService) { } @Override - protected void initializeZarrService(String rootPath) throws IOException, FormatException { + protected void initializeZarrService() throws IOException, FormatException { zarrService = mockService; } }