Unsigned Access to Pubilc S3 Data

CHANGELOG.unreleased.md

@@ -11,6 +11,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 [Commits](https://github.com/scalableminds/webknossos/compare/22.09.0...HEAD)
 
 ### Added
+- Zarr-based remote dataset import now also works for public AWS S3 endpoints with no credentials. [#6421](https://github.com/scalableminds/webknossos/pull/6421)
 
 ### Changed
 

build.sbt

@@ -14,10 +14,15 @@ ThisBuild / scalacOptions ++= Seq(
   "-language:implicitConversions",
   "-language:postfixOps",
   "-Xlint:unused",
+  "-Xlint:deprecation",
   s"-Wconf:src=target/.*:s",
   s"-Wconf:src=webknossos-datastore/target/.*:s",
   s"-Wconf:src=webknossos-tracingstore/target/.*:s"
 )
+ThisBuild / javacOptions ++= Seq(
+  "-Xlint:unchecked",
+  "-Xlint:deprecation"
+)
 
 ThisBuild / dependencyCheckAssemblyAnalyzerEnabled := Some(false)
 

conf/META-INF/services/java.nio.file.spi.FileSystemProvider

@@ -1,3 +1,3 @@
-com.upplication.s3fs.S3FileSystemProvider
+com.scalableminds.webknossos.datastore.s3fs.S3FileSystemProvider
 com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpsFileSystemProvider
 com.scalableminds.webknossos.datastore.storage.httpsfilesystem.HttpFileSystemProvider

frontend/javascripts/admin/dataset/dataset_add_zarr_view.tsx

@@ -206,9 +206,7 @@ function DatasetAddZarrView(props: Props) {
               value={showCredentialsFields ? "show" : "hide"}
               onChange={(e) => setShowCredentialsFields(e.target.value === "show")}
             >
-              <Radio value="hide" disabled={selectedProtocol === "s3"}>
-                {selectedProtocol === "https" ? "None" : "Anonymous"}
-              </Radio>
+              <Radio value="hide">{selectedProtocol === "https" ? "None" : "Anonymous"}</Radio>
               <Radio value="show">
                 {selectedProtocol === "https" ? "Basic authentication" : "With credentials"}
               </Radio>

project/Dependencies.scala

@@ -45,9 +45,13 @@ object Dependencies {
   private val swagger = "io.swagger" %% "swagger-play2" % "1.7.1"
   private val jhdf = "cisd" % "jhdf5" % "19.04.0"
   private val ucarCdm = "edu.ucar" % "cdm-core" % "5.3.3"
-  private val s3fs = "org.lasersonlab" % "s3fs" % "2.2.3"
   private val jblosc = "org.lasersonlab" % "jblosc" % "1.0.1"
   private val scalajHttp = "org.scalaj" %% "scalaj-http" % "2.4.2"
+  private val guava = "com.google.guava" % "guava" % "18.0"
+  private val awsS3 = "com.amazonaws" % "aws-java-sdk-s3" % "1.12.288"
+  private val tika = "org.apache.tika" % "tika-core" % "1.5"
+  private val jackson = "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.12.7"
+
 
   private val sql = Seq(
     "com.typesafe.slick" %% "slick" % "3.2.3",
@@ -91,7 +95,10 @@ object Dependencies {
     redis,
     jhdf,
     ucarCdm,
-    s3fs,
+    jackson,
+    guava,
+    awsS3,
+    tika,
     jblosc,
     scalajHttp
   )

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/AmazonS3Factory.java

@@ -0,0 +1,124 @@
+package com.scalableminds.webknossos.datastore.s3fs;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.S3ClientOptions;
+import com.scalableminds.webknossos.datastore.s3fs.util.AnonymousAWSCredentialsProvider;
+
+import java.net.URI;
+import java.util.Properties;
+
+
+/**
+ * Factory base class to create a new AmazonS3 instance.
+ */
+public class AmazonS3Factory {
+
+    public static final String ACCESS_KEY = "s3fs_access_key";
+    public static final String SECRET_KEY = "s3fs_secret_key";
+    public static final String REQUEST_METRIC_COLLECTOR_CLASS = "s3fs_request_metric_collector_class";
+    public static final String CONNECTION_TIMEOUT = "s3fs_connection_timeout";
+    public static final String MAX_CONNECTIONS = "s3fs_max_connections";
+    public static final String MAX_ERROR_RETRY = "s3fs_max_retry_error";
+    public static final String PROTOCOL = "s3fs_protocol";
+    public static final String PROXY_DOMAIN = "s3fs_proxy_domain";
+    public static final String PROXY_HOST = "s3fs_proxy_host";
+    public static final String PROXY_PASSWORD = "s3fs_proxy_password";
+    public static final String PROXY_PORT = "s3fs_proxy_port";
+    public static final String PROXY_USERNAME = "s3fs_proxy_username";
+    public static final String PROXY_WORKSTATION = "s3fs_proxy_workstation";
+    public static final String SOCKET_SEND_BUFFER_SIZE_HINT = "s3fs_socket_send_buffer_size_hint";
+    public static final String SOCKET_RECEIVE_BUFFER_SIZE_HINT = "s3fs_socket_receive_buffer_size_hint";
+    public static final String SOCKET_TIMEOUT = "s3fs_socket_timeout";
+    public static final String USER_AGENT = "s3fs_user_agent";
+    public static final String SIGNER_OVERRIDE = "s3fs_signer_override";
+    public static final String PATH_STYLE_ACCESS = "s3fs_path_style_access";
+
+    /**
+     * Build a new Amazon S3 instance with the URI and the properties provided
+     * @param uri URI mandatory
+     * @param props Properties with the credentials and others options
+     * @return AmazonS3
+     */
+    public AmazonS3 getAmazonS3Client(URI uri, Properties props) {
+        return AmazonS3ClientBuilder
+            .standard()
+            .withCredentials(getCredentialsProvider(props))
+            .withClientConfiguration(getClientConfiguration(props))
+            .withEndpointConfiguration(getEndpointConfiguration(uri))
+            .build();
+    }
+
+    protected AwsClientBuilder.EndpointConfiguration getEndpointConfiguration(URI uri) {
+        String endpoint = uri.getHost();
+        if (uri.getPort() != -1)
+            endpoint = uri.getHost() + ':' + uri.getPort();
+        return new AwsClientBuilder.EndpointConfiguration(endpoint, Regions.DEFAULT_REGION.toString());
+    }
+
+    protected AWSCredentialsProvider getCredentialsProvider(Properties props) {
+        if (props.getProperty(ACCESS_KEY) == null && props.getProperty(SECRET_KEY) == null) {
+            return new AnonymousAWSCredentialsProvider();
+        }
+        return new AWSStaticCredentialsProvider(getAWSCredentials(props));
+    }
+
+    protected S3ClientOptions getClientOptions(Properties props) {
+        S3ClientOptions.Builder builder = S3ClientOptions.builder();
+        if (props.getProperty(PATH_STYLE_ACCESS) != null &&
+                Boolean.parseBoolean(props.getProperty(PATH_STYLE_ACCESS)))
+            builder.setPathStyleAccess(true);
+
+        return builder.build();
+    }
+
+    protected ClientConfiguration getClientConfiguration(Properties props) {
+        ClientConfiguration clientConfiguration = new ClientConfiguration();
+        if (props.getProperty(CONNECTION_TIMEOUT) != null)
+            clientConfiguration.setConnectionTimeout(Integer.parseInt(props.getProperty(CONNECTION_TIMEOUT)));
+        if (props.getProperty(MAX_CONNECTIONS) != null)
+            clientConfiguration.setMaxConnections(Integer.parseInt(props.getProperty(MAX_CONNECTIONS)));
+        if (props.getProperty(MAX_ERROR_RETRY) != null)
+            clientConfiguration.setMaxErrorRetry(Integer.parseInt(props.getProperty(MAX_ERROR_RETRY)));
+        if (props.getProperty(PROTOCOL) != null)
+            clientConfiguration.setProtocol(Protocol.valueOf(props.getProperty(PROTOCOL)));
+        if (props.getProperty(PROXY_DOMAIN) != null)
+            clientConfiguration.setProxyDomain(props.getProperty(PROXY_DOMAIN));
+        if (props.getProperty(PROXY_HOST) != null)
+            clientConfiguration.setProxyHost(props.getProperty(PROXY_HOST));
+        if (props.getProperty(PROXY_PASSWORD) != null)
+            clientConfiguration.setProxyPassword(props.getProperty(PROXY_PASSWORD));
+        if (props.getProperty(PROXY_PORT) != null)
+            clientConfiguration.setProxyPort(Integer.parseInt(props.getProperty(PROXY_PORT)));
+        if (props.getProperty(PROXY_USERNAME) != null)
+            clientConfiguration.setProxyUsername(props.getProperty(PROXY_USERNAME));
+        if (props.getProperty(PROXY_WORKSTATION) != null)
+            clientConfiguration.setProxyWorkstation(props.getProperty(PROXY_WORKSTATION));
+        int socketSendBufferSizeHint = 0;
+        if (props.getProperty(SOCKET_SEND_BUFFER_SIZE_HINT) != null)
+            socketSendBufferSizeHint = Integer.parseInt(props.getProperty(SOCKET_SEND_BUFFER_SIZE_HINT));
+        int socketReceiveBufferSizeHint = 0;
+        if (props.getProperty(SOCKET_RECEIVE_BUFFER_SIZE_HINT) != null)
+            socketReceiveBufferSizeHint = Integer.parseInt(props.getProperty(SOCKET_RECEIVE_BUFFER_SIZE_HINT));
+        clientConfiguration.setSocketBufferSizeHints(socketSendBufferSizeHint, socketReceiveBufferSizeHint);
+        if (props.getProperty(SOCKET_TIMEOUT) != null)
+            clientConfiguration.setSocketTimeout(Integer.parseInt(props.getProperty(SOCKET_TIMEOUT)));
+        if (props.getProperty(USER_AGENT) != null)
+            clientConfiguration.setUserAgentPrefix(props.getProperty(USER_AGENT));
+        if (props.getProperty(SIGNER_OVERRIDE) != null)
+            clientConfiguration.setSignerOverride(props.getProperty(SIGNER_OVERRIDE));
+        return clientConfiguration;
+    }
+
+    protected BasicAWSCredentials getAWSCredentials(Properties props) {
+        return new BasicAWSCredentials(props.getProperty(ACCESS_KEY), props.getProperty(SECRET_KEY));
+    }
+}

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/LICENSE

@@ -0,0 +1,31 @@
+The s3fs package is based on
+
+https://github.com/lasersonlab/Amazon-S3-FileSystem-NIO2
+
+version 5117da7c5a75a455951d7a1788c1a4b7a0719692
+Aug 25, 2022
+
+Published under:
+
+MIT License (MIT)
+
+Copyright (c) 2014 Javier Arnáiz @arnaix
+Copyright (c) 2014 Better.be Application Services BV
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

webknossos-datastore/app/com/scalableminds/webknossos/datastore/s3fs/S3AccessControlList.java

@@ -0,0 +1,64 @@
+package com.scalableminds.webknossos.datastore.s3fs;
+
+import static java.lang.String.format;
+
+import java.nio.file.AccessDeniedException;
+import java.nio.file.AccessMode;
+import java.util.EnumSet;
+
+import com.amazonaws.services.s3.model.AccessControlList;
+import com.amazonaws.services.s3.model.Grant;
+import com.amazonaws.services.s3.model.Owner;
+import com.amazonaws.services.s3.model.Permission;
+
+public class S3AccessControlList {
+    private String fileStoreName;
+    private String key;
+    private AccessControlList acl;
+    private Owner owner;
+
+    public S3AccessControlList(String fileStoreName, String key, AccessControlList acl, Owner owner) {
+        this.fileStoreName = fileStoreName;
+        this.acl = acl;
+        this.key = key;
+        this.owner = owner;
+    }
+
+    public String getKey() {
+        return key;
+    }
+
+    /**
+     * have almost one of the permission set in the parameter permissions
+     *
+     * @param permissions almost one
+     * @return
+     */
+    private boolean hasPermission(EnumSet<Permission> permissions) {
+        for (Grant grant : acl.getGrantsAsList())
+            if (grant.getGrantee().getIdentifier().equals(owner.getId()) && permissions.contains(grant.getPermission()))
+                return true;
+        return false;
+    }
+
+    public void checkAccess(AccessMode[] modes) throws AccessDeniedException {
+        for (AccessMode accessMode : modes) {
+            switch (accessMode) {
+                case EXECUTE:
+                    throw new AccessDeniedException(fileName(), null, "file is not executable");
+                case READ:
+                    if (!hasPermission(EnumSet.of(Permission.FullControl, Permission.Read)))
+                        throw new AccessDeniedException(fileName(), null, "file is not readable");
+                    break;
+                case WRITE:
+                    if (!hasPermission(EnumSet.of(Permission.FullControl, Permission.Write)))
+                        throw new AccessDeniedException(fileName(), null, format("bucket '%s' is not writable", fileStoreName));
+                    break;
+            }
+        }
+    }
+
+    private String fileName() {
+        return fileStoreName + S3Path.PATH_SEPARATOR + key;
+    }
+}