diff --git a/doc/release-notes/5733-s3-creds-chain.md b/doc/release-notes/5733-s3-creds-chain.md new file mode 100644 index 00000000000..6fa1b5d8273 --- /dev/null +++ b/doc/release-notes/5733-s3-creds-chain.md @@ -0,0 +1,7 @@ +# Providing S3 Storage Credentials via MicroProfile Config + +With this release, you may use two new options to pass an access key identifier and a secret access key for S3-based +storage definitions without creating the files used by the AWS CLI tools (`~/.aws/config` & `~/.aws/credentials`). + +This has been added to ease setups using containers (Docker, Podman, Kubernetes, OpenShift) or testing and developing +installations. Find added [documentation and a word of warning in the installation guide](https://guides.dataverse.org/en/latest/installation/config.html#s3-mpconfig). \ No newline at end of file diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 10a0ae48716..446fadb7868 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -386,6 +386,9 @@ of two methods described below: 1. Manually through creation of the credentials and config files or 2. Automatically via the AWS console commands. +Some usage scenarios might be eased without generating these files. You may also provide :ref:`static credentials via +MicroProfile Config `, see below. + Preparation When Using Amazon's S3 Service ########################################## @@ -526,28 +529,69 @@ been tested already and what other options have been set for a successful integr Lastly, go ahead and restart your Payara server. With Dataverse deployed and the site online, you should be able to upload datasets and data files and see the corresponding files in your S3 bucket. Within a bucket, the folder structure emulates that found in local file storage. -S3 Storage Options -################## - -=========================================== ================== ========================================================================== ============= -JVM Option Value Description Default value -=========================================== ================== ========================================================================== ============= -dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` -dataverse.files..bucket-name The bucket name. See above. (none) -dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` -dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` -dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) -dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 -dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` -dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) -dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` -dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) -dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) -dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` -dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` -dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` -dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` -=========================================== ================== ========================================================================== ============= +List of S3 Storage Options +########################## + +.. table:: + :align: left + + =========================================== ================== ========================================================================== ============= + JVM Option Value Description Default value + =========================================== ================== ========================================================================== ============= + dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` + dataverse.files..type ``s3`` **Required** to mark this storage as S3 based. (none) + dataverse.files..label **Required** label to be shown in the UI for this storage (none) + dataverse.files..bucket-name The bucket name. See above. (none) + dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` + dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` + dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) + dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 + dataverse.files..min-part-size Multipart direct uploads will occur for files larger than this. Optional. ``1024**3`` + dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) + dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` + dataverse.files..profile Allows the use of AWS profiles for storage spanning multiple AWS accounts. (none) + dataverse.files..proxy-url URL of a proxy protecting the S3 store. Optional. (none) + dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` + dataverse.files..payload-signing ``true``/``false`` Enable payload signing. Optional ``false`` + dataverse.files..chunked-encoding ``true``/``false`` Disable chunked encoding. Optional ``true`` + dataverse.files..connection-pool-size The maximum number of open connections to the S3 server ``256`` + =========================================== ================== ========================================================================== ============= + +.. table:: + :align: left + + =========================================== ================== ========================================================================== ============= + MicroProfile Config Option Value Description Default value + =========================================== ================== ========================================================================== ============= + dataverse.files..access-key :ref:`Provide static access key ID. Read before use! ` ``""`` + dataverse.files..secret-key :ref:`Provide static secret access key. Read before use! ` ``""`` + =========================================== ================== ========================================================================== ============= + + +.. _s3-mpconfig: + +Credentials via MicroProfile Config +################################### + +Optionally, you may provide static credentials for each S3 storage using MicroProfile Config options: + +- ``dataverse.files..access-key`` for this storages "access key ID" +- ``dataverse.files..secret-key`` for this storages "secret access key" + +You may provide the values for these via any of the +`supported config sources `_. + +**WARNING:** + +*For security, do not use the sources "environment variable" or "system property" (JVM option) in a production context!* +*Rely on password alias, secrets directory or cloud based sources instead!* + +**NOTE:** + +1. Providing both AWS CLI profile files (as setup in first step) and static keys, credentials from ``~/.aws`` + will win over configured keys when valid! +2. A non-empty ``dataverse.files..profile`` will be ignored when no credentials can be found for this profile name. + Current codebase does not make use of "named profiles" as seen for AWS CLI besides credentials. Reported Working S3-Compatible Storage ###################################### diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 80fbbd103e7..ea19d29b41e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -4,6 +4,9 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.HttpMethod; import com.amazonaws.SdkClientException; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.profile.ProfileCredentialsProvider; import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; @@ -59,6 +62,8 @@ import java.util.Random; import java.util.logging.Logger; import org.apache.commons.io.IOUtils; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; import javax.json.Json; import javax.json.JsonObjectBuilder; @@ -77,6 +82,7 @@ */ public class S3AccessIO extends StorageIO { + private static final Config config = ConfigProvider.getConfig(); private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.dataaccess.S3AccessIO"); private static HashMap driverClientMap = new HashMap(); @@ -1162,8 +1168,20 @@ private static AmazonS3 getClient(String driverId) { * The default is "default" which should work when only one profile exists. */ String s3profile = System.getProperty("dataverse.files." + driverId + ".profile","default"); - - s3CB.setCredentials(new ProfileCredentialsProvider(s3profile)); + ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(s3profile); + + // Try to retrieve credentials via Microprofile Config API, too. For production use, you should not use env + // vars or system properties to provide these, but use the secrets config source provided by Payara. + AWSStaticCredentialsProvider staticCredentials = new AWSStaticCredentialsProvider( + new BasicAWSCredentials( + config.getOptionalValue("dataverse.files." + driverId + ".access-key", String.class).orElse(""), + config.getOptionalValue("dataverse.files." + driverId + ".secret-key", String.class).orElse("") + )); + + // Add both providers to chain - the first working provider will be used (so static credentials are the fallback) + AWSCredentialsProviderChain providerChain = new AWSCredentialsProviderChain(profileCredentials, staticCredentials); + s3CB.setCredentials(providerChain); + // let's build the client :-) AmazonS3 client = s3CB.build(); driverClientMap.put(driverId, client);