rebase with AWS updates from main branch

apache · Dec 13, 2024 · eec522f · eec522f
1 parent ea8ad6f
commit eec522f
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 1 deletion.
diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java
@@ -158,6 +158,7 @@ private StorageConfigInfo getStorageInfo(Map<String, String> internalProperties)
                 s3Config.getS3CredentialsCatalogSecretAccessKey())
             .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey())
             .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId())
+            .setS3Region(s3Config.getS3Region())
             .build();
       }
       if (configInfo instanceof AzureStorageConfigurationInfo) {
@@ -283,6 +284,7 @@ public Builder setStorageConfigurationInfo(
                     s3ConfigModel.getS3CredentialsClientAccessKeyId(),
                     s3ConfigModel.getS3CredentialsClientSecretAccessKey(),
                     s3ConfigModel.getS3PathStyleAccess(),
+                    s3ConfigModel.getS3Region(),
                     new ArrayList<>(allowedLocations));
             break;
           case AZURE:

diff --git a/...g/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java b/...g/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java
@@ -78,6 +78,9 @@ public EnumMap<PolarisCredentialProperty, String> getSubscopedCreds(
     propertiesMap.put(
         PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS,
         storageConfig.getS3PathStyleAccess().toString());
+    if (storageConfig.getS3Region() != null) {
+      propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region());
+    }
 
     if (storageConfig.getSkipCredentialSubscopingIndirection() == true) {
       LOGGER.debug("S3Compatible - skipCredentialSubscopingIndirection !");

diff --git a/...va/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java b/...va/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java
@@ -42,6 +42,7 @@ public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigur
   private @NotNull Boolean skipCredentialSubscopingIndirection;
   private @Nullable String s3CredentialsClientAccessKeyId;
   private @Nullable String s3CredentialsClientSecretAccessKey;
+  private @Nullable String s3Region;
 
   // Constructor
   @JsonCreator
@@ -60,6 +61,7 @@ public S3CompatibleStorageConfigurationInfo(
           String s3CredentialsClientSecretAccessKey,
       @JsonProperty(value = "s3PathStyleAccess", required = false) @Nullable
           Boolean s3PathStyleAccess,
+      @JsonProperty(value = "S3Region", required = false) @Nullable String region,
       @JsonProperty(value = "allowedLocations", required = true) @Nullable
           List<String> allowedLocations) {
 
@@ -73,6 +75,7 @@ public S3CompatibleStorageConfigurationInfo(
     this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId;
     this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey;
     this.skipCredentialSubscopingIndirection = skipCredentialSubscopingIndirection;
+    this.s3Region = region;
   }
 
   public @NotNull String getS3Endpoint() {
@@ -105,6 +108,14 @@ public S3CompatibleStorageConfigurationInfo(
         : this.s3CredentialsClientSecretAccessKey;
   }
 
+  public @Nullable String getS3Region() {
+    return s3Region;
+  }
+
+  public void setS3Region(@Nullable String region) {
+    this.s3Region = region;
+  }
+
   public @Nullable Boolean getSkipCredentialSubscopingIndirection() {
     return this.skipCredentialSubscopingIndirection;
   }

diff --git a/regtests/run_spark_sql_s3compatible.sh b/regtests/run_spark_sql_s3compatible.sh
@@ -212,7 +212,8 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \
               \"s3.path-style-access\": true,
               \"s3.credentials.catalog.access-key-id\": \"CATALOG_ID\",
               \"s3.credentials.catalog.secret-access-key\": \"CATALOG_SECRET\",
-              \"skip-credential-subscoping-indirection\": true
+              \"skip-credential-subscoping-indirection\": true,
+              \"s3.region\": \"rack-1\"
             }
           }"
 
@@ -223,6 +224,7 @@ ${SPARK_HOME}/bin/spark-sql --verbose \
   --conf spark.sql.catalog.polaris.warehouse=manual_spark \
   --conf spark.sql.defaultCatalog=polaris \
   --conf spark.hadoop.hive.cli.print.header=true \
+  --conf spark.hadoop.fs.s3a.aws.region=rack-1 \
   -f "minio/queries-for-spark.sql"
 
 

diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml
@@ -943,6 +943,10 @@ components:
           type: string
           description: Optional - if not empty, it will be used when 'skip-credential-subscoping-indirection' is true
           example: "CLIENT_OF_CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME"
+        s3.region:
+          type: string
+          description: the s3 region where data is stored
+          example: "rack-1"
       required:
         - s3.endpoint