From a89d77b4422f8efbcf013945936c39f2ff423bf5 Mon Sep 17 00:00:00 2001 From: Raphael Azzolini Date: Fri, 7 Jun 2024 11:19:35 -0700 Subject: [PATCH] HADOOP-19197. S3A: Support AWS KMS Encryption Context Add the property fs.s3a.encryption.context that allow users to specify the AWS KMS Encryption Context to be used in S3A. The value of the encryption context is a key/value string that will be Base64 encoded and set in the parameter ssekmsEncryptionContext from the S3 client. Contributed by Raphael Azzolini --- .../fs/CommonConfigurationKeysPublic.java | 1 + .../src/main/resources/core-default.xml | 10 ++ .../org/apache/hadoop/fs/s3a/Constants.java | 10 ++ .../org/apache/hadoop/fs/s3a/S3AUtils.java | 22 +++- .../EncryptionSecretOperations.java | 16 +++ .../auth/delegation/EncryptionSecrets.java | 35 +++++- .../fs/s3a/impl/RequestFactoryImpl.java | 14 +++ .../hadoop/fs/s3a/impl/S3AEncryption.java | 106 ++++++++++++++++++ .../markdown/tools/hadoop-aws/encryption.md | 30 +++++ .../site/markdown/tools/hadoop-aws/index.md | 14 +++ .../fs/s3a/AbstractTestS3AEncryption.java | 2 + ...EncryptionSSEKMSWithEncryptionContext.java | 101 +++++++++++++++++ .../hadoop/fs/s3a/TestSSEConfiguration.java | 69 +++++++++--- .../s3a/auth/TestMarshalledCredentials.java | 3 +- .../ITestSessionDelegationTokens.java | 6 +- .../TestS3ADelegationTokenSupport.java | 24 +++- .../fs/s3a/impl/TestRequestFactory.java | 2 +- .../hadoop/fs/s3a/impl/TestS3AEncryption.java | 77 +++++++++++++ 18 files changed, 513 insertions(+), 29 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java index 24a3167b3db2d..c22c48f1ffe22 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java @@ -1019,6 +1019,7 @@ public class CommonConfigurationKeysPublic { "fs.s3a.*.server-side-encryption.key", "fs.s3a.encryption.algorithm", "fs.s3a.encryption.key", + "fs.s3a.encryption.context", "fs.azure\\.account.key.*", "credential$", "oauth.*secret", diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml index 075c7e02e8111..c31843e8c4a4b 100644 --- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml +++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml @@ -742,6 +742,7 @@ fs.s3a.*.server-side-encryption.key fs.s3a.encryption.algorithm fs.s3a.encryption.key + fs.s3a.encryption.context fs.s3a.secret.key fs.s3a.*.secret.key fs.s3a.session.key @@ -1760,6 +1761,15 @@ + + fs.s3a.encryption.context + Specific encryption context to use if fs.s3a.encryption.algorithm + has been set to 'SSE-KMS' or 'DSSE-KMS'. The value of this property is a set + of non-secret comma-separated key-value pairs of additional contextual + information about the data that are separated by equal operator (=). + + + fs.s3a.signing-algorithm Override the default signing algorithm so legacy diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 8ee65e645c520..87d292386b9b8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -770,6 +770,16 @@ private Constants() { public static final String S3_ENCRYPTION_KEY = "fs.s3a.encryption.key"; + /** + * Set S3-SSE encryption context. + * The value of this property is a set of non-secret comma-separated key-value pairs + * of additional contextual information about the data that are separated by equal + * operator (=). + * value:{@value} + */ + public static final String S3_ENCRYPTION_CONTEXT = + "fs.s3a.encryption.context"; + /** * List of custom Signers. The signer class will be loaded, and the signer * name will be associated with this signer class in the S3 SDK. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java index 1b858f5c1e8a7..6f4a655c5d32a 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java @@ -38,6 +38,7 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.PathIOException; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.s3a.impl.S3AEncryption; import org.apache.hadoop.util.functional.RemoteIterators; import org.apache.hadoop.fs.s3a.auth.delegation.EncryptionSecrets; import org.apache.hadoop.fs.s3a.impl.MultiObjectDeleteException; @@ -1326,7 +1327,7 @@ static void patchSecurityCredentialProviders(Configuration conf) { * @throws IOException on any IO problem * @throws IllegalArgumentException bad arguments */ - private static String lookupBucketSecret( + public static String lookupBucketSecret( String bucket, Configuration conf, String baseKey) @@ -1472,6 +1473,8 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket, int encryptionKeyLen = StringUtils.isBlank(encryptionKey) ? 0 : encryptionKey.length(); String diagnostics = passwordDiagnostics(encryptionKey, "key"); + String encryptionContext = S3AEncryption.getS3EncryptionContextBase64Encoded(bucket, conf, + encryptionMethod.requiresSecret()); switch (encryptionMethod) { case SSE_C: LOG.debug("Using SSE-C with {}", diagnostics); @@ -1507,7 +1510,7 @@ public static EncryptionSecrets buildEncryptionSecrets(String bucket, LOG.debug("Data is unencrypted"); break; } - return new EncryptionSecrets(encryptionMethod, encryptionKey); + return new EncryptionSecrets(encryptionMethod, encryptionKey, encryptionContext); } /** @@ -1700,6 +1703,21 @@ public static Map getTrimmedStringCollectionSplitByEquals( final Configuration configuration, final String name) { String valueString = configuration.get(name); + return getTrimmedStringCollectionSplitByEquals(valueString); + } + + /** + * Get the equal op (=) delimited key-value pairs of the name property as + * a collection of pair of Strings, trimmed of the leading and trailing whitespace + * after delimiting the name by comma and new line separator. + * If no such property is specified then empty Map is returned. + * + * @param valueString the string containing the key-value pairs. + * @return property value as a Map of Strings, or empty + * Map. + */ + public static Map getTrimmedStringCollectionSplitByEquals( + final String valueString) { if (null == valueString) { return new HashMap<>(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java index 8a55a970134f3..ea5c0cf20786d 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecretOperations.java @@ -61,4 +61,20 @@ public static Optional getSSEAwsKMSKey(final EncryptionSecrets secrets) return Optional.empty(); } } + + /** + * Gets the SSE-KMS context if present, else don't set it in the S3 request. + * + * @param secrets source of the encryption secrets. + * @return an optional AWS KMS encryption context to attach to a request. + */ + public static Optional getSSEAwsKMSEncryptionContext(final EncryptionSecrets secrets) { + if ((secrets.getEncryptionMethod() == S3AEncryptionMethods.SSE_KMS + || secrets.getEncryptionMethod() == S3AEncryptionMethods.DSSE_KMS) + && secrets.hasEncryptionContext()) { + return Optional.of(secrets.getEncryptionContext()); + } else { + return Optional.empty(); + } + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java index 092653de557f0..f421ecca24cf9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/delegation/EncryptionSecrets.java @@ -67,6 +67,11 @@ public class EncryptionSecrets implements Writable, Serializable { */ private String encryptionKey = ""; + /** + * Encryption context: base64-encoded UTF-8 string. + */ + private String encryptionContext = ""; + /** * This field isn't serialized/marshalled; it is rebuilt from the * encryptionAlgorithm field. @@ -84,23 +89,28 @@ public EncryptionSecrets() { * Create a pair of secrets. * @param encryptionAlgorithm algorithm enumeration. * @param encryptionKey key/key reference. + * @param encryptionContext base64-encoded string with the encryption context key-value pairs. * @throws IOException failure to initialize. */ public EncryptionSecrets(final S3AEncryptionMethods encryptionAlgorithm, - final String encryptionKey) throws IOException { - this(encryptionAlgorithm.getMethod(), encryptionKey); + final String encryptionKey, + final String encryptionContext) throws IOException { + this(encryptionAlgorithm.getMethod(), encryptionKey, encryptionContext); } /** * Create a pair of secrets. * @param encryptionAlgorithm algorithm name * @param encryptionKey key/key reference. + * @param encryptionContext base64-encoded string with the encryption context key-value pairs. * @throws IOException failure to initialize. */ public EncryptionSecrets(final String encryptionAlgorithm, - final String encryptionKey) throws IOException { + final String encryptionKey, + final String encryptionContext) throws IOException { this.encryptionAlgorithm = encryptionAlgorithm; this.encryptionKey = encryptionKey; + this.encryptionContext = encryptionContext; init(); } @@ -114,6 +124,7 @@ public void write(final DataOutput out) throws IOException { new LongWritable(serialVersionUID).write(out); Text.writeString(out, encryptionAlgorithm); Text.writeString(out, encryptionKey); + Text.writeString(out, encryptionContext); } /** @@ -132,6 +143,7 @@ public void readFields(final DataInput in) throws IOException { } encryptionAlgorithm = Text.readString(in, MAX_SECRET_LENGTH); encryptionKey = Text.readString(in, MAX_SECRET_LENGTH); + encryptionContext = Text.readString(in); init(); } @@ -164,6 +176,10 @@ public String getEncryptionKey() { return encryptionKey; } + public String getEncryptionContext() { + return encryptionContext; + } + /** * Does this instance have encryption options? * That is: is the algorithm non-null. @@ -181,6 +197,14 @@ public boolean hasEncryptionKey() { return StringUtils.isNotEmpty(encryptionKey); } + /** + * Does this instance have an encryption context? + * @return true if there's an encryption context. + */ + public boolean hasEncryptionContext() { + return StringUtils.isNotEmpty(encryptionContext); + } + @Override public boolean equals(final Object o) { if (this == o) { @@ -191,12 +215,13 @@ public boolean equals(final Object o) { } final EncryptionSecrets that = (EncryptionSecrets) o; return Objects.equals(encryptionAlgorithm, that.encryptionAlgorithm) - && Objects.equals(encryptionKey, that.encryptionKey); + && Objects.equals(encryptionKey, that.encryptionKey) + && Objects.equals(encryptionContext, that.encryptionContext); } @Override public int hashCode() { - return Objects.hash(encryptionAlgorithm, encryptionKey); + return Objects.hash(encryptionAlgorithm, encryptionKey, encryptionContext); } /** diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index 3406c60028780..e9f7d707286a8 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -280,6 +280,8 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom, LOG.debug("Propagating SSE-KMS settings from source {}", sourceKMSId); copyObjectRequestBuilder.ssekmsKeyId(sourceKMSId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext); return; } @@ -292,11 +294,15 @@ protected void copyEncryptionParameters(HeadObjectResponse srcom, // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext); break; case DSSE_KMS: copyObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(copyObjectRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(copyObjectRequestBuilder::ssekmsEncryptionContext); break; case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) @@ -386,11 +392,15 @@ private void putEncryptionParameters(PutObjectRequest.Builder putObjectRequestBu // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(putObjectRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext); break; case DSSE_KMS: putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(putObjectRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(putObjectRequestBuilder::ssekmsEncryptionContext); break; case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) @@ -462,11 +472,15 @@ private void multipartUploadEncryptionParameters( // Set the KMS key if present, else S3 uses AWS managed key. EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(mpuRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(mpuRequestBuilder::ssekmsEncryptionContext); break; case DSSE_KMS: mpuRequestBuilder.serverSideEncryption(ServerSideEncryption.AWS_KMS_DSSE); EncryptionSecretOperations.getSSEAwsKMSKey(encryptionSecrets) .ifPresent(mpuRequestBuilder::ssekmsKeyId); + EncryptionSecretOperations.getSSEAwsKMSEncryptionContext(encryptionSecrets) + .ifPresent(mpuRequestBuilder::ssekmsEncryptionContext); break; case SSE_C: EncryptionSecretOperations.getSSECustomerKey(encryptionSecrets) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java new file mode 100644 index 0000000000000..a720d2ca10000 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/S3AEncryption.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.S3AUtils; + +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT; + +/** + * Utility methods for S3A encryption properties. + */ +public final class S3AEncryption { + + private static final Logger LOG = LoggerFactory.getLogger(S3AEncryption.class); + + private S3AEncryption() { + } + + /** + * Get any SSE context from a configuration/credential provider. + * @param bucket bucket to query for + * @param conf configuration to examine + * @return the encryption context value or "" + * @throws IOException if reading a JCEKS file raised an IOE + * @throws IllegalArgumentException bad arguments. + */ + public static String getS3EncryptionContext(String bucket, Configuration conf) + throws IOException { + // look up the per-bucket value of the encryption context + String encryptionContext = S3AUtils.lookupBucketSecret(bucket, conf, S3_ENCRYPTION_CONTEXT); + if (encryptionContext == null) { + // look up the global value of the encryption context + encryptionContext = S3AUtils.lookupPassword(null, conf, S3_ENCRYPTION_CONTEXT); + } + if (encryptionContext == null) { + // no encryption context, return "" + return ""; + } + return encryptionContext; + } + + /** + * Get any SSE context from a configuration/credential provider. + * This includes converting the values to a base64-encoded UTF-8 string + * holding JSON with the encryption context key-value pairs + * @param bucket bucket to query for + * @param conf configuration to examine + * @param propagateExceptions should IO exceptions be rethrown? + * @return the Base64 encryption context or "" + * @throws IllegalArgumentException bad arguments. + * @throws IOException if propagateExceptions==true and reading a JCEKS file raised an IOE + */ + public static String getS3EncryptionContextBase64Encoded( + String bucket, + Configuration conf, + boolean propagateExceptions) throws IOException { + try { + final String encryptionContextValue = getS3EncryptionContext(bucket, conf); + if (StringUtils.isBlank(encryptionContextValue)) { + return ""; + } + final Map encryptionContextMap = S3AUtils + .getTrimmedStringCollectionSplitByEquals(encryptionContextValue); + if (encryptionContextMap.isEmpty()) { + return ""; + } + final String encryptionContextJson = new ObjectMapper().writeValueAsString( + encryptionContextMap); + return Base64.encodeBase64String(encryptionContextJson.getBytes(StandardCharsets.UTF_8)); + } catch (IOException e) { + if (propagateExceptions) { + throw e; + } + LOG.warn("Cannot retrieve {} for bucket {}", + S3_ENCRYPTION_CONTEXT, bucket, e); + return ""; + } + } +} diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md index 42ef91c032ba8..7b9e8d0412efd 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/encryption.md @@ -243,6 +243,21 @@ The ID of the specific key used to encrypt the data should also be set in the pr ``` +Optionally, you can specify the encryption context in the property `fs.s3a.encryption.context`: + +```xml + + fs.s3a.encryption.context + + key1=value1, + key2=value2, + key3=value3, + key4=value4, + key5=value5 + + +``` + Organizations may define a default key in the Amazon KMS; if a default key is set, then it will be used whenever SSE-KMS encryption is chosen and the value of `fs.s3a.encryption.key` is empty. @@ -378,6 +393,21 @@ The ID of the specific key used to encrypt the data should also be set in the pr ``` +Optionally, you can specify the encryption context in the property `fs.s3a.encryption.context`: + +```xml + + fs.s3a.encryption.context + + key1=value1, + key2=value2, + key3=value3, + key4=value4, + key5=value5 + + +``` + Organizations may define a default key in the Amazon KMS; if a default key is set, then it will be used whenever SSE-KMS encryption is chosen and the value of `fs.s3a.encryption.key` is empty. diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index d601e21bbed69..59c1ceac1c52c 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -1006,6 +1006,15 @@ Here are some the S3A properties for use in production. + + fs.s3a.encryption.context + Specific encryption context to use if fs.s3a.encryption.algorithm + has been set to 'SSE-KMS' or 'DSSE-KMS'. The value of this property is a set + of non-secret comma-separated key-value pairs of additional contextual + information about the data that are separated by equal operator (=). + + + fs.s3a.signing-algorithm Override the default signing algorithm so legacy @@ -1675,6 +1684,11 @@ For a site configuration of: unset + + fs.s3a.encryption.context + unset + + ``` diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java index 3a3d82d94ffaf..55cebeab8ef32 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractTestS3AEncryption.java @@ -30,6 +30,7 @@ import static org.apache.hadoop.fs.contract.ContractTestUtils.*; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_ALGORITHM; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT; import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM; import static org.apache.hadoop.fs.s3a.Constants.SERVER_SIDE_ENCRYPTION_KEY; @@ -69,6 +70,7 @@ protected void patchConfigurationEncryptionSettings( removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_ALGORITHM, S3_ENCRYPTION_KEY, + S3_ENCRYPTION_CONTEXT, SERVER_SIDE_ENCRYPTION_ALGORITHM, SERVER_SIDE_ENCRYPTION_KEY); conf.set(S3_ENCRYPTION_ALGORITHM, diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java new file mode 100644 index 0000000000000..c3d4cd41fc5ae --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEKMSWithEncryptionContext.java @@ -0,0 +1,101 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Set; + +import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.impl.S3AEncryption; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT; +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_KEY; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.DSSE_KMS; +import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.SSE_KMS; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.assume; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.getTestBucketName; + +/** + * Concrete class that extends {@link AbstractTestS3AEncryption} + * and tests KMS encryption with encryption context. + * S3's HeadObject doesn't return the object's encryption context. + * Therefore, we don't have a way to assert its value in code. + * In order to properly test if the encryption context is being set, + * the KMS key or the IAM User need to have a deny statements like the one below in the policy: + * 

+ * {
+ *     "Effect": "Deny",
+ *     "Principal": {
+ *         "AWS": "*"
+ *     },
+ *     "Action": "kms:Decrypt",
+ *     "Resource": "*",
+ *     "Condition": {
+ *         "StringNotEquals": {
+ *             "kms:EncryptionContext:project": "hadoop"
+ *         }
+ *     }
+ * }
+ *
+ * With the statement above, S3A will fail to read the object from S3 if it was encrypted + * without the key-pair "project": "hadoop" in the encryption context. + */ +public class ITestS3AEncryptionSSEKMSWithEncryptionContext + extends AbstractTestS3AEncryption { + + private static final Set KMS_ENCRYPTION_ALGORITHMS = ImmutableSet.of( + SSE_KMS, DSSE_KMS); + + private S3AEncryptionMethods encryptionAlgorithm; + + @Override + protected Configuration createConfiguration() { + try { + // get the KMS key and context for this test. + Configuration c = new Configuration(); + final String bucketName = getTestBucketName(c); + String kmsKey = S3AUtils.getS3EncryptionKey(bucketName, c); + String encryptionContext = S3AEncryption.getS3EncryptionContext(bucketName, c); + encryptionAlgorithm = S3AUtils.getEncryptionAlgorithm(bucketName, c); + assume("Expected a KMS encryption algorithm", + KMS_ENCRYPTION_ALGORITHMS.contains(encryptionAlgorithm)); + if (StringUtils.isBlank(encryptionContext)) { + skip(S3_ENCRYPTION_CONTEXT + " is not set."); + } + Configuration conf = super.createConfiguration(); + S3ATestUtils.removeBaseAndBucketOverrides(conf, S3_ENCRYPTION_KEY, S3_ENCRYPTION_CONTEXT); + conf.set(S3_ENCRYPTION_KEY, kmsKey); + conf.set(S3_ENCRYPTION_CONTEXT, encryptionContext); + return conf; + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + protected S3AEncryptionMethods getSSEAlgorithm() { + return encryptionAlgorithm; + } +} diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java index 6985fa44c3bda..dcda68155195e 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestSSEConfiguration.java @@ -29,9 +29,11 @@ import org.junit.rules.Timeout; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.s3a.impl.S3AEncryption; import org.apache.hadoop.security.ProviderUtils; import org.apache.hadoop.security.alias.CredentialProvider; import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.util.StringUtils; import static org.apache.hadoop.fs.s3a.Constants.*; import static org.apache.hadoop.fs.s3a.S3AEncryptionMethods.*; @@ -48,6 +50,9 @@ public class TestSSEConfiguration extends Assert { /** Bucket to use for per-bucket options. */ public static final String BUCKET = "dataset-1"; + /** Valid set of key/value pairs for the encryption context. */ + private static final String VALID_ENCRYPTION_CONTEXT = "key1=value1, key2=value2, key3=value3"; + @Rule public Timeout testTimeout = new Timeout( S3ATestConstants.S3A_TEST_TIMEOUT @@ -58,41 +63,41 @@ public class TestSSEConfiguration extends Assert { @Test public void testSSECNoKey() throws Throwable { - assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), null); + assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), null, null); } @Test public void testSSECBlankKey() throws Throwable { - assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), ""); + assertGetAlgorithmFails(SSE_C_NO_KEY_ERROR, SSE_C.getMethod(), "", null); } @Test public void testSSECGoodKey() throws Throwable { - assertEquals(SSE_C, getAlgorithm(SSE_C, "sseckey")); + assertEquals(SSE_C, getAlgorithm(SSE_C, "sseckey", null)); } @Test public void testKMSGoodKey() throws Throwable { - assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey")); + assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey", null)); } @Test public void testAESKeySet() throws Throwable { assertGetAlgorithmFails(SSE_S3_WITH_KEY_ERROR, - SSE_S3.getMethod(), "setkey"); + SSE_S3.getMethod(), "setkey", null); } @Test public void testSSEEmptyKey() { // test the internal logic of the test setup code - Configuration c = buildConf(SSE_C.getMethod(), ""); + Configuration c = buildConf(SSE_C.getMethod(), "", null); assertEquals("", getS3EncryptionKey(BUCKET, c)); } @Test public void testSSEKeyNull() throws Throwable { // test the internal logic of the test setup code - final Configuration c = buildConf(SSE_C.getMethod(), null); + final Configuration c = buildConf(SSE_C.getMethod(), null, null); assertEquals("", getS3EncryptionKey(BUCKET, c)); intercept(IOException.class, SSE_C_NO_KEY_ERROR, @@ -147,28 +152,30 @@ void setProviderOption(final Configuration conf, } /** - * Assert that the exception text from {@link #getAlgorithm(String, String)} + * Assert that the exception text from {@link #getAlgorithm(String, String, String)} * is as expected. * @param expected expected substring in error * @param alg algorithm to ask for * @param key optional key value + * @param context optional encryption context value * @throws Exception anything else which gets raised */ public void assertGetAlgorithmFails(String expected, - final String alg, final String key) throws Exception { + final String alg, final String key, final String context) throws Exception { intercept(IOException.class, expected, - () -> getAlgorithm(alg, key)); + () -> getAlgorithm(alg, key, context)); } private S3AEncryptionMethods getAlgorithm(S3AEncryptionMethods algorithm, - String key) + String key, + String encryptionContext) throws IOException { - return getAlgorithm(algorithm.getMethod(), key); + return getAlgorithm(algorithm.getMethod(), key, encryptionContext); } - private S3AEncryptionMethods getAlgorithm(String algorithm, String key) + private S3AEncryptionMethods getAlgorithm(String algorithm, String key, String encryptionContext) throws IOException { - return getEncryptionAlgorithm(BUCKET, buildConf(algorithm, key)); + return getEncryptionAlgorithm(BUCKET, buildConf(algorithm, key, encryptionContext)); } /** @@ -176,10 +183,11 @@ private S3AEncryptionMethods getAlgorithm(String algorithm, String key) * and key. * @param algorithm algorithm to use, may be null * @param key key, may be null + * @param encryptionContext encryption context, may be null * @return the new config. */ @SuppressWarnings("deprecation") - private Configuration buildConf(String algorithm, String key) { + private Configuration buildConf(String algorithm, String key, String encryptionContext) { Configuration conf = emptyConf(); if (algorithm != null) { conf.set(Constants.S3_ENCRYPTION_ALGORITHM, algorithm); @@ -193,6 +201,11 @@ private Configuration buildConf(String algorithm, String key) { conf.unset(SERVER_SIDE_ENCRYPTION_KEY); conf.unset(Constants.S3_ENCRYPTION_KEY); } + if (encryptionContext != null) { + conf.set(S3_ENCRYPTION_CONTEXT, encryptionContext); + } else { + conf.unset(S3_ENCRYPTION_CONTEXT); + } return conf; } @@ -308,4 +321,30 @@ public void testNoEncryptionMethod() throws Throwable { assertEquals(NONE, getMethod(" ")); } + @Test + public void testGoodEncryptionContext() throws Throwable { + assertEquals(SSE_KMS, getAlgorithm(SSE_KMS, "kmskey", VALID_ENCRYPTION_CONTEXT)); + } + + @Test + public void testSSEEmptyEncryptionContext() throws Throwable { + // test the internal logic of the test setup code + Configuration c = buildConf(SSE_KMS.getMethod(), "kmskey", ""); + assertEquals("", S3AEncryption.getS3EncryptionContext(BUCKET, c)); + } + + @Test + public void testSSEEncryptionContextNull() throws Throwable { + // test the internal logic of the test setup code + final Configuration c = buildConf(SSE_KMS.getMethod(), "kmskey", null); + assertEquals("", S3AEncryption.getS3EncryptionContext(BUCKET, c)); + } + + @Test + public void testSSEInvalidEncryptionContext() throws Throwable { + intercept(IllegalArgumentException.class, + StringUtils.STRING_COLLECTION_SPLIT_EQUALS_INVALID_ARG, + () -> getAlgorithm(SSE_KMS.getMethod(), "kmskey", "invalid context")); + } + } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java index b9d547635f7f3..71f22f4314f4f 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/TestMarshalledCredentials.java @@ -80,7 +80,8 @@ public void testRoundTripNoSessionData() throws Throwable { public void testRoundTripEncryptionData() throws Throwable { EncryptionSecrets secrets = new EncryptionSecrets( S3AEncryptionMethods.SSE_KMS, - "key"); + "key", + "encryptionContext"); EncryptionSecrets result = S3ATestUtils.roundTrip(secrets, new Configuration()); assertEquals("round trip", secrets, result); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java index efc775966859d..b58ca24aaa832 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/ITestSessionDelegationTokens.java @@ -116,7 +116,7 @@ public void testCanonicalization() throws Throwable { public void testSaveLoadTokens() throws Throwable { File tokenFile = File.createTempFile("token", "bin"); EncryptionSecrets encryptionSecrets = new EncryptionSecrets( - S3AEncryptionMethods.SSE_KMS, KMS_KEY); + S3AEncryptionMethods.SSE_KMS, KMS_KEY, ""); Token dt = delegationTokens.createDelegationToken(encryptionSecrets, null); final SessionTokenIdentifier origIdentifier @@ -171,7 +171,7 @@ public void testCreateAndUseDT() throws Throwable { assertNull("Current User has delegation token", delegationTokens.selectTokenFromFSOwner()); EncryptionSecrets secrets = new EncryptionSecrets( - S3AEncryptionMethods.SSE_KMS, KMS_KEY); + S3AEncryptionMethods.SSE_KMS, KMS_KEY, ""); Token originalDT = delegationTokens.createDelegationToken(secrets, null); assertEquals("Token kind mismatch", getTokenKind(), originalDT.getKind()); @@ -229,7 +229,7 @@ public void testCreateWithRenewer() throws Throwable { assertNull("Current User has delegation token", delegationTokens.selectTokenFromFSOwner()); EncryptionSecrets secrets = new EncryptionSecrets( - S3AEncryptionMethods.SSE_KMS, KMS_KEY); + S3AEncryptionMethods.SSE_KMS, KMS_KEY, ""); Token dt = delegationTokens.createDelegationToken(secrets, renewer); assertEquals("Token kind mismatch", getTokenKind(), dt.getKind()); diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java index af306cc5a9a5f..a06e9ac62ff71 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/auth/delegation/TestS3ADelegationTokenSupport.java @@ -19,10 +19,12 @@ package org.apache.hadoop.fs.s3a.auth.delegation; import java.net.URI; +import java.nio.charset.StandardCharsets; import org.junit.BeforeClass; import org.junit.Test; +import org.apache.commons.codec.binary.Base64; import org.apache.hadoop.fs.s3a.S3AEncryptionMethods; import org.apache.hadoop.fs.s3a.S3ATestUtils; import org.apache.hadoop.fs.s3a.auth.MarshalledCredentialBinding; @@ -70,13 +72,17 @@ public void testSessionTokenIssueDate() throws Throwable { public void testSessionTokenDecode() throws Throwable { Text alice = new Text("alice"); Text renewer = new Text("yarn"); + String encryptionKey = "encryptionKey"; + String encryptionContextJson = "{\"key\":\"value\", \"key2\": \"value3\"}"; + String encryptionContextEncoded = Base64.encodeBase64String(encryptionContextJson.getBytes( + StandardCharsets.UTF_8)); AbstractS3ATokenIdentifier identifier = new SessionTokenIdentifier(SESSION_TOKEN_KIND, alice, renewer, new URI("s3a://anything/"), new MarshalledCredentials("a", "b", ""), - new EncryptionSecrets(S3AEncryptionMethods.SSE_S3, ""), + new EncryptionSecrets(S3AEncryptionMethods.SSE_S3, encryptionKey, encryptionContextEncoded), "origin"); Token t1 = new Token<>(identifier, @@ -100,6 +106,10 @@ public void testSessionTokenDecode() throws Throwable { assertEquals("origin", decoded.getOrigin()); assertEquals("issue date", identifier.getIssueDate(), decoded.getIssueDate()); + EncryptionSecrets encryptionSecrets = decoded.getEncryptionSecrets(); + assertEquals(S3AEncryptionMethods.SSE_S3, encryptionSecrets.getEncryptionMethod()); + assertEquals(encryptionKey, encryptionSecrets.getEncryptionKey()); + assertEquals(encryptionContextEncoded, encryptionSecrets.getEncryptionContext()); } @Test @@ -112,13 +122,19 @@ public void testFullTokenKind() throws Throwable { @Test public void testSessionTokenIdentifierRoundTrip() throws Throwable { Text renewer = new Text("yarn"); + String encryptionKey = "encryptionKey"; + String encryptionContextJson = "{\"key\":\"value\", \"key2\": \"value3\"}"; + String encryptionContextEncoded = Base64.encodeBase64String(encryptionContextJson.getBytes( + StandardCharsets.UTF_8)); SessionTokenIdentifier id = new SessionTokenIdentifier( SESSION_TOKEN_KIND, new Text(), renewer, externalUri, new MarshalledCredentials("a", "b", "c"), - new EncryptionSecrets(), ""); + new EncryptionSecrets(S3AEncryptionMethods.DSSE_KMS, encryptionKey, + encryptionContextEncoded), + ""); SessionTokenIdentifier result = S3ATestUtils.roundTrip(id, null); String ids = id.toString(); @@ -127,6 +143,10 @@ public void testSessionTokenIdentifierRoundTrip() throws Throwable { id.getMarshalledCredentials(), result.getMarshalledCredentials()); assertEquals("renewer in " + ids, renewer, id.getRenewer()); + EncryptionSecrets encryptionSecrets = result.getEncryptionSecrets(); + assertEquals(S3AEncryptionMethods.DSSE_KMS, encryptionSecrets.getEncryptionMethod()); + assertEquals(encryptionKey, encryptionSecrets.getEncryptionKey()); + assertEquals(encryptionContextEncoded, encryptionSecrets.getEncryptionContext()); } @Test diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java index 4c3f0859d8a95..b864cd3b63982 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestRequestFactory.java @@ -76,7 +76,7 @@ public void testRequestFactoryWithEncryption() throws Throwable { .withBucket("bucket") .withEncryptionSecrets( new EncryptionSecrets(S3AEncryptionMethods.SSE_KMS, - "kms:key")) + "kms:key", "")) .build(); createFactoryObjects(factory); } diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java new file mode 100644 index 0000000000000..a9d83819fda56 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/impl/TestS3AEncryption.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.impl; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.Test; + +import org.apache.commons.codec.binary.Base64; +import org.apache.hadoop.conf.Configuration; + +import static org.apache.hadoop.fs.s3a.Constants.S3_ENCRYPTION_CONTEXT; + +public class TestS3AEncryption { + + private static final String GLOBAL_CONTEXT = " project=hadoop, jira=HADOOP-19197 "; + private static final String BUCKET_CONTEXT = "component=fs/s3"; + + @Test + public void testGetS3EncryptionContextPerBucket() throws IOException { + Configuration configuration = new Configuration(false); + configuration.set("fs.s3a.bucket.bucket1.encryption.context", BUCKET_CONTEXT); + configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT); + final String result = S3AEncryption.getS3EncryptionContext("bucket1", configuration); + Assert.assertEquals(BUCKET_CONTEXT, result); + } + + @Test + public void testGetS3EncryptionContextFromGlobal() throws IOException { + Configuration configuration = new Configuration(false); + configuration.set("fs.s3a.bucket.bucket1.encryption.context", BUCKET_CONTEXT); + configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT); + final String result = S3AEncryption.getS3EncryptionContext("bucket2", configuration); + Assert.assertEquals(GLOBAL_CONTEXT.trim(), result); + } + + @Test + public void testGetS3EncryptionContextNoSet() throws IOException { + Configuration configuration = new Configuration(false); + final String result = S3AEncryption.getS3EncryptionContext("bucket1", configuration); + Assert.assertEquals("", result); + } + + @Test + public void testGetS3EncryptionContextBase64Encoded() throws IOException { + Configuration configuration = new Configuration(false); + configuration.set(S3_ENCRYPTION_CONTEXT, GLOBAL_CONTEXT); + final String result = S3AEncryption.getS3EncryptionContextBase64Encoded("bucket", + configuration, true); + final String decoded = new String(Base64.decodeBase64(result), StandardCharsets.UTF_8); + final TypeReference> typeRef = new TypeReference>() {}; + final Map resultMap = new ObjectMapper().readValue(decoded, typeRef); + Assert.assertEquals("hadoop", resultMap.get("project")); + Assert.assertEquals("HADOOP-19197", resultMap.get("jira")); + } +}