From 4bee6953264799712dcdaa9015522e3505b5cdcd Mon Sep 17 00:00:00 2001 From: posac Date: Tue, 15 Oct 2019 16:01:00 +0200 Subject: [PATCH 1/3] AWS Profile configuration + example how to setup region/aws_profile for each server --- S3StorageWagon/README.md | 16 ++++++++ .../maven/cloud/s3/CredentialsFactory.java | 19 ++++++---- .../maven/cloud/s3/S3StorageRepository.java | 4 +- .../maven/cloud/s3/S3StorageWagon.java | 37 +++++++++++-------- .../s3/plugin/download/S3DownloadMojo.java | 9 +++-- .../cloud/s3/plugin/upload/S3UploadMojo.java | 8 +++- .../maven/cloud/s3/utils/S3Connect.java | 9 +++-- .../maven/cloud/s3/S3StorageWagonTest.java | 2 +- 8 files changed, 69 insertions(+), 35 deletions(-) diff --git a/S3StorageWagon/README.md b/S3StorageWagon/README.md index 6595060..4a81f31 100644 --- a/S3StorageWagon/README.md +++ b/S3StorageWagon/README.md @@ -17,6 +17,22 @@ The S3StorageWagon project enables you to upload your artifacts to a google clou ``` Full guide on [wagon](https://egkatzioura.com/2018/04/09/host-your-maven-artifacts-using-amazon-s3/) +## Additional server configuration +In your settings xml you can setup server region, credentials and awsProfile. +```xml + + + bucket-repo + + eu-west-1 + profile-to-your-aws + + + + + +``` + ### Public repos You can specify your artifacts to be public and thus getting downloaded without the need for authorised access to your bucket. diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/CredentialsFactory.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/CredentialsFactory.java index 9b7a711..f09d88f 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/CredentialsFactory.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/CredentialsFactory.java @@ -16,14 +16,14 @@ package com.gkatzioura.maven.cloud.s3; -import java.util.logging.Logger; - -import org.apache.maven.wagon.authentication.AuthenticationInfo; - import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import org.apache.maven.wagon.authentication.AuthenticationInfo; + +import java.util.logging.Logger; public class CredentialsFactory { @@ -38,15 +38,18 @@ public class CredentialsFactory { * for details. * * @param authenticationInfo an {@link AuthenticationInfo} containing the AWS credentials to use + * @param awsProfile * @return a newly-built {@link AWSCredentialsProvider} with the credentials associated to the passed - * {@code authenticationInfo} + * {@code authenticationInfo} */ - public AWSCredentialsProvider create(AuthenticationInfo authenticationInfo) { - if(authenticationInfo==null) { + public AWSCredentialsProvider create(AuthenticationInfo authenticationInfo, String awsProfile) { + if (awsProfile != null) { + return new ProfileCredentialsProvider(awsProfile); + } else if (authenticationInfo == null) { return new DefaultAWSCredentialsProviderChain(); } else { LOGGER.info("Using static credentials provider"); - return new AWSStaticCredentialsProvider(new BasicAWSCredentials(authenticationInfo.getUserName(),authenticationInfo.getPassword())); + return new AWSStaticCredentialsProvider(new BasicAWSCredentials(authenticationInfo.getUserName(), authenticationInfo.getPassword())); } } } diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageRepository.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageRepository.java index 4642f08..2013361 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageRepository.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageRepository.java @@ -83,8 +83,8 @@ public S3StorageRepository(String bucket, String baseDirectory, PublicReadProper } - public void connect(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle) throws AuthenticationException { - this.amazonS3 = S3Connect.connect(authenticationInfo, region, endpoint, pathStyle); + public void connect(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle, String awsProfile) throws AuthenticationException { + this.amazonS3 = S3Connect.connect(authenticationInfo, region, endpoint, pathStyle, awsProfile); } public void copy(String resourceName, File destination, TransferProgress transferProgress) throws TransferFailedException, ResourceDoesNotExistException { diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageWagon.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageWagon.java index 8a3c7ce..2202a4e 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageWagon.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/S3StorageWagon.java @@ -16,16 +16,12 @@ package com.gkatzioura.maven.cloud.s3; -import java.io.File; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - +import com.amazonaws.services.s3.model.AmazonS3Exception; import com.gkatzioura.maven.cloud.resolver.KeyResolver; +import com.gkatzioura.maven.cloud.transfer.TransferProgress; +import com.gkatzioura.maven.cloud.transfer.TransferProgressImpl; +import com.gkatzioura.maven.cloud.wagon.AbstractStorageWagon; +import com.gkatzioura.maven.cloud.wagon.PublicReadProperty; import org.apache.commons.io.FileUtils; import org.apache.maven.wagon.ConnectionException; import org.apache.maven.wagon.PathUtils; @@ -39,11 +35,14 @@ import org.apache.maven.wagon.repository.Repository; import org.apache.maven.wagon.resource.Resource; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.gkatzioura.maven.cloud.transfer.TransferProgress; -import com.gkatzioura.maven.cloud.transfer.TransferProgressImpl; -import com.gkatzioura.maven.cloud.wagon.AbstractStorageWagon; -import com.gkatzioura.maven.cloud.wagon.PublicReadProperty; +import java.io.File; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; public class S3StorageWagon extends AbstractStorageWagon { @@ -51,6 +50,7 @@ public class S3StorageWagon extends AbstractStorageWagon { private final KeyResolver keyResolver = new KeyResolver(); private String region; + private String awsProfile; private Boolean publicRepository; private static final Logger LOGGER = Logger.getLogger(S3StorageWagon.class.getName()); @@ -181,7 +181,7 @@ public void connect(Repository repository, AuthenticationInfo authenticationInfo LOGGER.log(Level.FINER,String.format("Opening connection for bucket %s and directory %s",bucket,directory)); s3StorageRepository = new S3StorageRepository(bucket, directory, new PublicReadProperty(publicRepository)); - s3StorageRepository.connect(authenticationInfo, region, new EndpointProperty(endpoint), new PathStyleEnabledProperty(pathStyleEnabled)); + s3StorageRepository.connect(authenticationInfo, region, new EndpointProperty(endpoint), new PathStyleEnabledProperty(pathStyleEnabled), awsProfile); sessionListenerContainer.fireSessionLoggedIn(); sessionListenerContainer.fireSessionOpened(); @@ -227,4 +227,11 @@ public void setPathStyleAccessEnabled(String pathStyleEnabled) { this.pathStyleEnabled = pathStyleEnabled; } + public void setAwsProfile(String awsProfile) { + this.awsProfile = awsProfile; + } + + public String getAwsProfile() { + return awsProfile; + } } diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/download/S3DownloadMojo.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/download/S3DownloadMojo.java index 8559655..a08e4fb 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/download/S3DownloadMojo.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/download/S3DownloadMojo.java @@ -34,7 +34,6 @@ import org.apache.maven.wagon.authentication.AuthenticationException; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.S3ClientOptions; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; @@ -47,6 +46,9 @@ @Mojo(name = "s3-download") public class S3DownloadMojo extends AbstractMojo { + @Parameter( property = "s3-download.awsProfile") + private String awsProfile; + @Parameter( property = "s3-download.bucket") private String bucket; @@ -66,11 +68,12 @@ public class S3DownloadMojo extends AbstractMojo { public S3DownloadMojo() { } - public S3DownloadMojo(String bucket, List keys, String downloadPath, String region) { + public S3DownloadMojo(String bucket, List keys, String downloadPath, String region, String awsProfile) { this.bucket = bucket; this.keys = keys; this.downloadPath = downloadPath; this.region = region; + this.awsProfile = awsProfile; } @Override @@ -80,7 +83,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { try { //Sending the authenticationInfo as null will make this use the default S3 authentication, which will only //look at the environment Java properties or environment variables - amazonS3 = S3Connect.connect(null, region, EndpointProperty.empty(), new PathStyleEnabledProperty(String.valueOf(S3ClientOptions.DEFAULT_PATH_STYLE_ACCESS))); + amazonS3 = S3Connect.connect(null, region, EndpointProperty.empty(), new PathStyleEnabledProperty(String.valueOf(S3ClientOptions.DEFAULT_PATH_STYLE_ACCESS)), awsProfile); } catch (AuthenticationException e) { throw new MojoExecutionException( String.format("Unable to authenticate to S3 with the available credentials. Make sure to either define the environment variables or System properties defined in https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html.%n" + diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/upload/S3UploadMojo.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/upload/S3UploadMojo.java index 49e0ed3..9f9f3b8 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/upload/S3UploadMojo.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/plugin/upload/S3UploadMojo.java @@ -42,6 +42,9 @@ @Mojo(name = "s3-upload") public class S3UploadMojo extends AbstractMojo { + @Parameter( property = "s3-upload.profile") + private String awsProfile; + @Parameter( property = "s3-upload.bucket") private String bucket; @@ -65,11 +68,12 @@ public S3UploadMojo() { * @param key * @param region */ - public S3UploadMojo(String bucket, String path, String key, String region) { + public S3UploadMojo(String bucket, String path, String key, String region, String awsProfile) { this.bucket = bucket; this.path = path; this.key = key; this.region = region; + this.awsProfile = awsProfile; } /** @@ -87,7 +91,7 @@ public void execute() throws MojoExecutionException, MojoFailureException { try { //Sending the authenticationInfo as null will make this use the default S3 authentication, which will only //look at the environment Java properties or environment variables - amazonS3 = S3Connect.connect(null, region, EndpointProperty.empty(), new PathStyleEnabledProperty(String.valueOf(S3ClientOptions.DEFAULT_PATH_STYLE_ACCESS))); + amazonS3 = S3Connect.connect(null, region, EndpointProperty.empty(), new PathStyleEnabledProperty(String.valueOf(S3ClientOptions.DEFAULT_PATH_STYLE_ACCESS)), awsProfile); } catch (AuthenticationException e) { throw new MojoExecutionException( String.format("Unable to authenticate to S3 with the available credentials. Make sure to either define the environment variables or System properties defined in https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html.%n" + diff --git a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/utils/S3Connect.java b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/utils/S3Connect.java index 432d35c..f427407 100644 --- a/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/utils/S3Connect.java +++ b/S3StorageWagon/src/main/java/com/gkatzioura/maven/cloud/s3/utils/S3Connect.java @@ -45,13 +45,14 @@ public class S3Connect { * @param pathStyle A {@link PathStyleEnabledProperty} indicating whether the endpoint/bucket configuration being * passed is in a path-style configuration. See * Accessing a Bucket in the S3 documentation. + * @param awsProfile * @return An instance of {@link AmazonS3} that can be used to send and receive data to the intended endpoint/bucket. * @throws AuthenticationException if the passed credentials are invalid for connecting to the intended endpoint/bucket. */ - public static AmazonS3 connect(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle) throws AuthenticationException { + public static AmazonS3 connect(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle, String awsProfile) throws AuthenticationException { AmazonS3ClientBuilder builder = null; try { - builder = createAmazonS3ClientBuilder(authenticationInfo, region, endpoint, pathStyle); + builder = createAmazonS3ClientBuilder(authenticationInfo, region, endpoint, pathStyle, awsProfile); AmazonS3 amazonS3 = builder.build(); @@ -77,11 +78,11 @@ public static AmazonS3 connect(AuthenticationInfo authenticationInfo, String reg } } - private static AmazonS3ClientBuilder createAmazonS3ClientBuilder(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle) { + private static AmazonS3ClientBuilder createAmazonS3ClientBuilder(AuthenticationInfo authenticationInfo, String region, EndpointProperty endpoint, PathStyleEnabledProperty pathStyle, String awsProfile) { final S3StorageRegionProviderChain regionProvider = new S3StorageRegionProviderChain(region); AmazonS3ClientBuilder builder; - builder = AmazonS3ClientBuilder.standard().withCredentials(new CredentialsFactory().create(authenticationInfo)); + builder = AmazonS3ClientBuilder.standard().withCredentials(new CredentialsFactory().create(authenticationInfo, awsProfile)); if (endpoint.isPresent()){ builder.setEndpointConfiguration( new AwsClientBuilder.EndpointConfiguration(endpoint.get(), builder.getRegion())); diff --git a/S3StorageWagon/src/test/java/com/gkatzioura/maven/cloud/s3/S3StorageWagonTest.java b/S3StorageWagon/src/test/java/com/gkatzioura/maven/cloud/s3/S3StorageWagonTest.java index ff47f8e..a707947 100644 --- a/S3StorageWagon/src/test/java/com/gkatzioura/maven/cloud/s3/S3StorageWagonTest.java +++ b/S3StorageWagon/src/test/java/com/gkatzioura/maven/cloud/s3/S3StorageWagonTest.java @@ -55,7 +55,7 @@ protected long getExpectedLastModifiedOnGet(Repository repository, Resource reso protected void setUp() throws Exception { super.setUp(); //creates the bucket - amazonS3 = S3Connect.connect(getAuthInfo(), null, new EndpointProperty(null), new PathStyleEnabledProperty(null)); + amazonS3 = S3Connect.connect(getAuthInfo(), null, new EndpointProperty(null), new PathStyleEnabledProperty(null), null); createBucket(); } From f90c0d5419f1f9a48b0b732f2788bcd555da158c Mon Sep 17 00:00:00 2001 From: posac Date: Tue, 15 Oct 2019 16:10:44 +0200 Subject: [PATCH 2/3] AWS Profile configuration + example how to setup region/aws_profile for each server - Readme fix --- S3StorageWagon/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/S3StorageWagon/README.md b/S3StorageWagon/README.md index 4a81f31..4648df0 100644 --- a/S3StorageWagon/README.md +++ b/S3StorageWagon/README.md @@ -46,7 +46,7 @@ To specify a repo as public you can do it through the settings.xml access_secret eu-west-1 - true + true ``` @@ -74,6 +74,19 @@ Then you can use the artifact without any authorised access ``` +### Private repos - access by profile +You can assign aws_profile for your server. You can do it through the settings.xml: +```xml + + bucket-repo + + eu-west-1 + aws_profile_name + + +``` + + ## Upload/download files for ci/cd purposes Apart from giving a solution to use s3 a maven repository the storage s3-storage-wagon can be used as a plugin in order to From 0df0770b71a5edc39d1516d5b8280d9cfa4f23e8 Mon Sep 17 00:00:00 2001 From: posac Date: Tue, 15 Oct 2019 16:11:33 +0200 Subject: [PATCH 3/3] AWS Profile configuration + example how to setup region/aws_profile for each server - Readme fix --- S3StorageWagon/README.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/S3StorageWagon/README.md b/S3StorageWagon/README.md index 4648df0..349e643 100644 --- a/S3StorageWagon/README.md +++ b/S3StorageWagon/README.md @@ -17,22 +17,6 @@ The S3StorageWagon project enables you to upload your artifacts to a google clou ``` Full guide on [wagon](https://egkatzioura.com/2018/04/09/host-your-maven-artifacts-using-amazon-s3/) -## Additional server configuration -In your settings xml you can setup server region, credentials and awsProfile. -```xml - - - bucket-repo - - eu-west-1 - profile-to-your-aws - - - - - -``` - ### Public repos You can specify your artifacts to be public and thus getting downloaded without the need for authorised access to your bucket.