fix

apache · Dec 31, 2024 · 06b192b · 06b192b
1 parent 9b7bf01
commit 06b192b
Show file tree

Hide file tree

Showing 10 changed files with 13 additions and 16 deletions.
diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts
@@ -39,6 +39,7 @@ tasks.withType(ShadowJar::class.java) {
   relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3")
   relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common")
   relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson")
+  mergeServiceFiles()
 }
 
 tasks.jar {

diff --git a/bundles/azure-bundle/build.gradle.kts b/bundles/azure-bundle/build.gradle.kts
@@ -42,6 +42,7 @@ tasks.withType(ShadowJar::class.java) {
   relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml")
   relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common")
   relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty")
+  mergeServiceFiles()
 }
 
 tasks.jar {

diff --git a/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java
@@ -20,6 +20,7 @@
 package org.apache.gravitino.abs.fs;
 
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_AUTH_TYPE_PROPERTY_NAME;
+import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_ACCOUNT_IS_HNS_ENABLED;
 import static org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys.FS_AZURE_SAS_TOKEN_PROVIDER_TYPE;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -91,6 +92,7 @@ public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map<String, String>
           configuration.set(
               FS_AZURE_SAS_TOKEN_PROVIDER_TYPE + "." + accountName,
               AzureSasCredentialProvider.class.getName());
+          configuration.set(FS_AZURE_ACCOUNT_IS_HNS_ENABLED, "true");
         } else if (azureSasCredentialProvider.getAzureStorageAccountKey() != null
             && azureSasCredentialProvider.getAzureStorageAccountName() != null) {
           configuration.set(

diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts
@@ -42,6 +42,7 @@ tasks.withType(ShadowJar::class.java) {
   relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common")
   relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml")
   relocate("org.eclipse.jetty", "org.apache.gravitino.gcp.shaded.org.eclipse.jetty")
+  mergeServiceFiles()
 }
 
 tasks.jar {

diff --git a/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java
@@ -18,13 +18,11 @@
  */
 package org.apache.gravitino.gcs.fs;
 
-import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem;
 import com.google.cloud.hadoop.util.AccessTokenProvider;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
 import java.util.Map;
-import org.apache.commons.lang3.StringUtils;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider;
 import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils;
 import org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration;
@@ -58,10 +56,6 @@ public FileSystem getFileSystem(Path path, Map<String, String> config) throws IO
       }
     }
 
-    if (StringUtils.isBlank(configuration.get("fs.gs.impl"))) {
-      configuration.set("fs.gs.impl", GoogleHadoopFileSystem.class.getName());
-    }
-
     return FileSystem.newInstance(path.toUri(), configuration);
   }
 

diff --git a/clients/client-python/gravitino/dto/requests/model_version_link_request.py b/clients/client-python/gravitino/dto/requests/model_version_link_request.py
@@ -59,7 +59,7 @@ def validate(self):
 
         for alias in self._aliases or []:
             if not self._is_not_blank(alias):
-                raise IllegalArgumentException('Alias must not be null or empty')
+                raise IllegalArgumentException("Alias must not be null or empty")
 
     def _is_not_blank(self, string: str) -> bool:
         return string is not None and string.strip()
diff --git a/...oop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/...oop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java
@@ -412,14 +412,14 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat
                 // https://github.com/apache/gravitino/issues/5609
                 resetFileSystemServiceLoader(scheme);
 
-                Map<String, String> maps = getConfigMap(getConf());
-                // If enable the cloud store credential, we should pass the configuration here.
-                maps.put(GVFS_FILESET_IDENTIFIER, identifier.toString());
+                Map<String, String> catalogProperty = catalog.properties();
+                Map<String, String> totalProperty = Maps.newHashMap(catalogProperty);
 
-                // Should add catalog properties to the configuration
-                maps.putAll(catalog.properties());
+                totalProperty.putAll(getConfigMap(getConf()));
+                // If enable the cloud store credential, we should pass the configuration here.
+                totalProperty.put(GVFS_FILESET_IDENTIFIER, identifier.toString());
 
-                return provider.getFileSystem(filePath, maps);
+                return provider.getFileSystem(filePath, totalProperty);
               } catch (IOException ioe) {
                 throw new GravitinoRuntimeException(
                     "Exception occurs when create new FileSystem for actual uri: %s, msg: %s",

diff --git a/...avitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSCredentialIT.java b/...avitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSCredentialIT.java
@@ -92,7 +92,6 @@ public void startUp() throws Exception {
     conf.set("fs.gvfs.impl.disable.cache", "true");
     conf.set("fs.gravitino.server.uri", serverUri);
     conf.set("fs.gravitino.client.metalake", metalakeName);
-    conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
 
     // Pass this configuration to the real file system
     conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);

diff --git a/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/.../apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java
@@ -88,7 +88,6 @@ public void startUp() throws Exception {
     conf.set("fs.gvfs.impl.disable.cache", "true");
     conf.set("fs.gravitino.server.uri", serverUri);
     conf.set("fs.gravitino.client.metalake", metalakeName);
-    conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem");
 
     // Pass this configuration to the real file system
     conf.set(GCSProperties.GCS_SERVICE_ACCOUNT_JSON_PATH, SERVICE_ACCOUNT_FILE);

diff --git a/core/src/main/java/org/apache/gravitino/credential/config/GCSCredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/GCSCredentialConfig.java
@@ -30,7 +30,7 @@
 public class GCSCredentialConfig extends Config {
 
   @VisibleForTesting
-  public static final String GRAVITINO_GCS_CREDENTIAL_FILE_PATH = "gcs-credential-file-path";
+  public static final String GRAVITINO_GCS_CREDENTIAL_FILE_PATH = "gcs-service-account-file";
 
   public static final ConfigEntry<String> GCS_CREDENTIAL_FILE_PATH =
       new ConfigBuilder(GRAVITINO_GCS_CREDENTIAL_FILE_PATH)