Merge branch 'main' of github.com:datastrato/graviton into issue_5973

apache · Dec 27, 2024 · 44425e9 · 44425e9
2 parents 0c61a48 + 07cdcba
commit 44425e9
Show file tree

Hide file tree

Showing 107 changed files with 4,067 additions and 358 deletions.
diff --git a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java
@@ -93,7 +93,10 @@ Model registerModel(NameIdentifier ident, String comment, Map<String, String> pr
    *
    * @param ident The name identifier of the model.
    * @param uri The model artifact URI.
-   * @param aliases The aliases of the model version. The alias are optional and can be empty.
+   * @param aliases The aliases of the model version. The aliases should be unique in this model,
+   *     otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases
+   *     are optional and can be empty. Also, be aware that the alias cannot be a number or a number
+   *     string.
    * @param comment The comment of the model. The comment is optional and can be null.
    * @param properties The properties of the model. The properties are optional and can be null or
    *     empty.
@@ -198,7 +201,8 @@ default boolean modelVersionExists(NameIdentifier ident, String alias) {
    * @param uri The URI of the model version artifact.
    * @param aliases The aliases of the model version. The aliases should be unique in this model,
    *     otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases
-   *     are optional and can be empty.
+   *     are optional and can be empty. Also, be aware that the alias cannot be a number or a number
+   *     string.
    * @param comment The comment of the model version. The comment is optional and can be null.
    * @param properties The properties of the model version. The properties are optional and can be
    *     null or empty.

diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts
@@ -81,6 +81,7 @@ dependencies {
     exclude("net.java.dev.jna")
     exclude("javax.ws.rs")
     exclude("org.eclipse.jetty")
+    exclude("org.apache.hadoop", "hadoop-common")
   }
   testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
   testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") {
@@ -93,11 +94,10 @@ dependencies {
   testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") {
     exclude("com.sun.jersey")
   }
-  testImplementation(libs.hadoop3.client)
-  testImplementation(libs.hadoop3.common) {
-    exclude("com.sun.jersey")
-    exclude("javax.servlet", "servlet-api")
-  }
+
+  testImplementation(libs.hadoop3.client.api)
+  testImplementation(libs.hadoop3.client.runtime)
+
   testImplementation(libs.hadoop3.hdfs) {
     exclude("com.sun.jersey")
     exclude("javax.servlet", "servlet-api")

diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts
@@ -36,6 +36,7 @@ dependencies {
   }
   implementation(libs.bundles.log4j)
   implementation(libs.commons.lang3)
+  implementation(libs.commons.dbcp2)
   implementation(libs.guava)
   implementation(libs.javax.jaxb.api) {
     exclude("*")

diff --git a/...mmon/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/...mmon/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java
@@ -31,9 +31,9 @@ public AuthorizationProperties(Map<String, String> properties) {
             .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
   }
 
-  abstract String getPropertiesPrefix();
+  public abstract String getPropertiesPrefix();
 
-  abstract void validate();
+  public abstract void validate();
 
   public static void validate(String type, Map<String, String> properties) {
     switch (type) {

diff --git a/...ization/jdbc/JdbcAuthorizationPlugin.java → ...ization/jdbc/JdbcAuthorizationPlugin.java b/...ization/jdbc/JdbcAuthorizationPlugin.java → ...ization/jdbc/JdbcAuthorizationPlugin.java
@@ -40,7 +40,6 @@
 import org.apache.gravitino.authorization.RoleChange;
 import org.apache.gravitino.authorization.SecurableObject;
 import org.apache.gravitino.authorization.User;
-import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties;
 import org.apache.gravitino.connector.authorization.AuthorizationPlugin;
 import org.apache.gravitino.exceptions.AuthorizationPluginException;
 import org.apache.gravitino.meta.AuditInfo;
@@ -55,7 +54,7 @@
  * JDBC-based authorization plugins can inherit this class and implement their own SQL statements.
  */
 @Unstable
-abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL {
+public abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL {
 
   private static final String GROUP_PREFIX = "GRAVITINO_GROUP_";
   private static final Logger LOG = LoggerFactory.getLogger(JdbcAuthorizationPlugin.class);
@@ -351,11 +350,11 @@ public List<String> getRevokeRoleSQL(String roleName, String revokerType, String
   }
 
   @VisibleForTesting
-  Connection getConnection() throws SQLException {
+  public Connection getConnection() throws SQLException {
     return dataSource.getConnection();
   }
 
-  protected void executeUpdateSQL(String sql) {
+  public void executeUpdateSQL(String sql) {
     executeUpdateSQL(sql, null);
   }
 
@@ -382,7 +381,7 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti
         "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode());
   }
 
-  void executeUpdateSQL(String sql, String ignoreErrorMsg) {
+  public void executeUpdateSQL(String sql, String ignoreErrorMsg) {
     try (final Connection connection = getConnection()) {
       try (final Statement statement = connection.createStatement()) {
         statement.executeUpdate(sql);

diff --git a/...n/common/JdbcAuthorizationProperties.java → ...ion/jdbc/JdbcAuthorizationProperties.java b/...n/common/JdbcAuthorizationProperties.java → ...ion/jdbc/JdbcAuthorizationProperties.java
@@ -16,9 +16,10 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.gravitino.authorization.common;
+package org.apache.gravitino.authorization.jdbc;
 
 import java.util.Map;
+import org.apache.gravitino.authorization.common.AuthorizationProperties;
 
 /** The properties for JDBC authorization plugin. */
 public class JdbcAuthorizationProperties extends AuthorizationProperties {
@@ -39,7 +40,7 @@ private void check(String key, String errorMsg) {
   }
 
   @Override
-  String getPropertiesPrefix() {
+  public String getPropertiesPrefix() {
     return CONFIG_PREFIX;
   }
 

diff --git a/...horization/jdbc/JdbcAuthorizationSQL.java → ...horization/jdbc/JdbcAuthorizationSQL.java b/...horization/jdbc/JdbcAuthorizationSQL.java → ...horization/jdbc/JdbcAuthorizationSQL.java
@@ -25,7 +25,7 @@
 
 /** Interface for SQL operations of the underlying access control system. */
 @Unstable
-interface JdbcAuthorizationSQL {
+public interface JdbcAuthorizationSQL {
 
   /**
    * Get SQL statements for creating a user.

diff --git a/...uthorization/jdbc/JdbcMetadataObject.java → ...uthorization/jdbc/JdbcMetadataObject.java b/...uthorization/jdbc/JdbcMetadataObject.java → ...uthorization/jdbc/JdbcMetadataObject.java
diff --git a/...ino/authorization/jdbc/JdbcPrivilege.java → ...ino/authorization/jdbc/JdbcPrivilege.java b/...ino/authorization/jdbc/JdbcPrivilege.java → ...ino/authorization/jdbc/JdbcPrivilege.java
diff --git a/...thorization/jdbc/JdbcSecurableObject.java → ...thorization/jdbc/JdbcSecurableObject.java b/...thorization/jdbc/JdbcSecurableObject.java → ...thorization/jdbc/JdbcSecurableObject.java
@@ -44,7 +44,7 @@ private JdbcSecurableObject(
     this.privileges = privileges;
   }
 
-  static JdbcSecurableObject create(
+  public static JdbcSecurableObject create(
       String schema, String table, List<AuthorizationPrivilege> privileges) {
     String parent = table == null ? null : schema;
     String name = table == null ? schema : table;

diff --git a/...c/JdbcSecurableObjectMappingProvider.java → ...c/JdbcSecurableObjectMappingProvider.java b/...c/JdbcSecurableObjectMappingProvider.java → ...c/JdbcSecurableObjectMappingProvider.java
diff --git a/...ion/jdbc/JdbcAuthorizationPluginTest.java → ...ion/jdbc/TestJdbcAuthorizationPlugin.java b/...ion/jdbc/JdbcAuthorizationPluginTest.java → ...ion/jdbc/TestJdbcAuthorizationPlugin.java
@@ -34,15 +34,14 @@
 import org.apache.gravitino.authorization.SecurableObject;
 import org.apache.gravitino.authorization.SecurableObjects;
 import org.apache.gravitino.authorization.User;
-import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties;
 import org.apache.gravitino.meta.AuditInfo;
 import org.apache.gravitino.meta.GroupEntity;
 import org.apache.gravitino.meta.RoleEntity;
 import org.apache.gravitino.meta.UserEntity;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 
-public class JdbcAuthorizationPluginTest {
+public class TestJdbcAuthorizationPlugin {
   private static List<String> expectSQLs = Lists.newArrayList();
   private static List<MetadataObject.Type> expectTypes = Lists.newArrayList();
   private static List<String> expectObjectNames = Lists.newArrayList();
@@ -75,7 +74,7 @@ public List<String> getSetOwnerSQL(
           return Collections.emptyList();
         }
 
-        void executeUpdateSQL(String sql, String ignoreErrorMsg) {
+        public void executeUpdateSQL(String sql, String ignoreErrorMsg) {
           Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql);
           currentSQLIndex++;
         }

diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts
diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts
@@ -67,7 +67,12 @@ dependencies {
     exclude("net.java.dev.jna")
     exclude("javax.ws.rs")
     exclude("org.eclipse.jetty")
+    // Conflicts with hadoop-client-api used in hadoop-catalog.
+    exclude("org.apache.hadoop", "hadoop-common")
   }
+  implementation(libs.hadoop3.client.api)
+  implementation(libs.hadoop3.client.runtime)
+
   implementation(libs.rome)
   compileOnly(libs.lombok)
   testRuntimeOnly(libs.junit.jupiter.engine)
@@ -92,11 +97,7 @@ dependencies {
   testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") {
     exclude("com.sun.jersey")
   }
-  testImplementation(libs.hadoop3.client)
-  testImplementation(libs.hadoop3.common) {
-    exclude("com.sun.jersey")
-    exclude("javax.servlet", "servlet-api")
-  }
+
   testImplementation(libs.hadoop3.hdfs) {
     exclude("com.sun.jersey")
     exclude("javax.servlet", "servlet-api")

diff --git a/build.gradle.kts b/build.gradle.kts
@@ -779,7 +779,7 @@ tasks {
         !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" &&
         it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") &&
         it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") &&
-        it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && it.name != "hadoop-common"
+        it.parent?.name != "bundles" && it.name != "hadoop-common"
       ) {
         from(it.configurations.runtimeClasspath)
         into("distribution/package/libs")
@@ -799,10 +799,8 @@ tasks {
         !it.name.startsWith("integration-test") &&
         !it.name.startsWith("flink") &&
         !it.name.startsWith("trino-connector") &&
-        it.name != "bundled-catalog" &&
-        it.name != "hive-metastore-common" && it.name != "gcp-bundle" &&
-        it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" &&
-        it.name != "hadoop-common" && it.name != "docs"
+        it.name != "hive-metastore-common" &&
+        it.name != "docs" && it.name != "hadoop-common" && it.parent?.name != "bundles"
       ) {
         dependsOn("${it.name}:build")
         from("${it.name}/build/libs")

diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts
@@ -25,34 +25,11 @@ plugins {
 }
 
 dependencies {
-  compileOnly(project(":api"))
-  compileOnly(project(":core"))
-  compileOnly(project(":catalogs:catalog-common"))
-  compileOnly(project(":catalogs:catalog-hadoop"))
-  compileOnly(project(":catalogs:hadoop-common")) {
-    exclude("*")
-  }
-  compileOnly(libs.hadoop3.common)
-
-  implementation(libs.aliyun.credentials.sdk)
+  implementation(project(":bundles:aliyun"))
+  implementation(libs.commons.collections3)
+  implementation(libs.hadoop3.client.api)
+  implementation(libs.hadoop3.client.runtime)
   implementation(libs.hadoop3.oss)
-
-  // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add
-  // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1
-  implementation(libs.sun.activation)
-
-  // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0
-  // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils
-  // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111)
-  // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323)
-  // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611)
-  implementation(libs.commons.lang3)
-
-  implementation(project(":catalogs:catalog-common")) {
-    exclude("*")
-  }
-  implementation(project(":clients:client-java-runtime", configuration = "shadow"))
-  implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow"))
 }
 
 tasks.withType(ShadowJar::class.java) {
@@ -62,8 +39,12 @@ tasks.withType(ShadowJar::class.java) {
   mergeServiceFiles()
 
   // Relocate dependencies to avoid conflicts
-  relocate("org.jdom", "org.apache.gravitino.shaded.org.jdom")
-  relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3")
+  relocate("org.jdom", "org.apache.gravitino.aliyun.shaded.org.jdom")
+  relocate("org.apache.commons.lang3", "org.apache.gravitino.aliyun.shaded.org.apache.commons.lang3")
+  relocate("com.fasterxml.jackson", "org.apache.gravitino.aliyun.shaded.com.fasterxml.jackson")
+  relocate("com.google.common", "org.apache.gravitino.aliyun.shaded.com.google.common")
+  relocate("org.apache.http", "org.apache.gravitino.aliyun.shaded.org.apache.http")
+  relocate("org.apache.commons.collections", "org.apache.gravitino.aliyun.shaded.org.apache.commons.collections")
 }
 
 tasks.jar {