diff --git a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java index cea2e94e3c7..3fb39c18aea 100644 --- a/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java +++ b/api/src/main/java/org/apache/gravitino/model/ModelCatalog.java @@ -93,7 +93,10 @@ Model registerModel(NameIdentifier ident, String comment, Map pr * * @param ident The name identifier of the model. * @param uri The model artifact URI. - * @param aliases The aliases of the model version. The alias are optional and can be empty. + * @param aliases The aliases of the model version. The aliases should be unique in this model, + * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model. The comment is optional and can be null. * @param properties The properties of the model. The properties are optional and can be null or * empty. @@ -198,7 +201,8 @@ default boolean modelVersionExists(NameIdentifier ident, String alias) { * @param uri The URI of the model version artifact. * @param aliases The aliases of the model version. The aliases should be unique in this model, * otherwise the {@link ModelVersionAliasesAlreadyExistException} will be thrown. The aliases - * are optional and can be empty. + * are optional and can be empty. Also, be aware that the alias cannot be a number or a number + * string. * @param comment The comment of the model version. The comment is optional and can be null. * @param properties The properties of the model version. The properties are optional and can be * null or empty. diff --git a/authorizations/authorization-chain/build.gradle.kts b/authorizations/authorization-chain/build.gradle.kts index d5cd160742c..e14cfa05ba9 100644 --- a/authorizations/authorization-chain/build.gradle.kts +++ b/authorizations/authorization-chain/build.gradle.kts @@ -81,6 +81,7 @@ dependencies { exclude("net.java.dev.jna") exclude("javax.ws.rs") exclude("org.eclipse.jetty") + exclude("org.apache.hadoop", "hadoop-common") } testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion") { @@ -93,11 +94,10 @@ dependencies { testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { exclude("com.sun.jersey") } - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/authorizations/authorization-common/build.gradle.kts b/authorizations/authorization-common/build.gradle.kts index ba64510f2ce..9bab92dac3e 100644 --- a/authorizations/authorization-common/build.gradle.kts +++ b/authorizations/authorization-common/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) + implementation(libs.commons.dbcp2) implementation(libs.guava) implementation(libs.javax.jaxb.api) { exclude("*") diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java index 3005cc5f3e9..3ece6353d6e 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/AuthorizationProperties.java @@ -31,9 +31,9 @@ public AuthorizationProperties(Map properties) { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - abstract String getPropertiesPrefix(); + public abstract String getPropertiesPrefix(); - abstract void validate(); + public abstract void validate(); public static void validate(String type, Map properties) { switch (type) { diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java index d9bc28636c3..d0a1b0897ec 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPlugin.java @@ -40,7 +40,6 @@ import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; @@ -55,7 +54,7 @@ * JDBC-based authorization plugins can inherit this class and implement their own SQL statements. */ @Unstable -abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { +public abstract class JdbcAuthorizationPlugin implements AuthorizationPlugin, JdbcAuthorizationSQL { private static final String GROUP_PREFIX = "GRAVITINO_GROUP_"; private static final Logger LOG = LoggerFactory.getLogger(JdbcAuthorizationPlugin.class); @@ -351,11 +350,11 @@ public List getRevokeRoleSQL(String roleName, String revokerType, String } @VisibleForTesting - Connection getConnection() throws SQLException { + public Connection getConnection() throws SQLException { return dataSource.getConnection(); } - protected void executeUpdateSQL(String sql) { + public void executeUpdateSQL(String sql) { executeUpdateSQL(sql, null); } @@ -382,7 +381,7 @@ protected AuthorizationPluginException toAuthorizationPluginException(SQLExcepti "JDBC authorization plugin fail to execute SQL, error code: %d", se.getErrorCode()); } - void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public void executeUpdateSQL(String sql, String ignoreErrorMsg) { try (final Connection connection = getConnection()) { try (final Statement statement = connection.createStatement()) { statement.executeUpdate(sql); diff --git a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java similarity index 92% rename from authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java index 9a5e7c6cc97..69a12135023 100644 --- a/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/common/JdbcAuthorizationProperties.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationProperties.java @@ -16,9 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.authorization.common; +package org.apache.gravitino.authorization.jdbc; import java.util.Map; +import org.apache.gravitino.authorization.common.AuthorizationProperties; /** The properties for JDBC authorization plugin. */ public class JdbcAuthorizationProperties extends AuthorizationProperties { @@ -39,7 +40,7 @@ private void check(String key, String errorMsg) { } @Override - String getPropertiesPrefix() { + public String getPropertiesPrefix() { return CONFIG_PREFIX; } diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java similarity index 99% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java index f7171ff354a..de031f70e78 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationSQL.java @@ -25,7 +25,7 @@ /** Interface for SQL operations of the underlying access control system. */ @Unstable -interface JdbcAuthorizationSQL { +public interface JdbcAuthorizationSQL { /** * Get SQL statements for creating a user. diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcMetadataObject.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcPrivilege.java diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java similarity index 98% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java index 78b82e2a8da..2c721093e2c 100644 --- a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java +++ b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObject.java @@ -44,7 +44,7 @@ private JdbcSecurableObject( this.privileges = privileges; } - static JdbcSecurableObject create( + public static JdbcSecurableObject create( String schema, String table, List privileges) { String parent = table == null ? null : schema; String name = table == null ? schema : table; diff --git a/authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java b/authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java similarity index 100% rename from authorizations/authorization-jdbc/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java rename to authorizations/authorization-common/src/main/java/org/apache/gravitino/authorization/jdbc/JdbcSecurableObjectMappingProvider.java diff --git a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java similarity index 98% rename from authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java rename to authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java index e261fad78d2..6f9f7e29c37 100644 --- a/authorizations/authorization-jdbc/src/test/java/org/apache/gravitino/authorization/jdbc/JdbcAuthorizationPluginTest.java +++ b/authorizations/authorization-common/src/test/java/org/apache/gravitino/authorization/jdbc/TestJdbcAuthorizationPlugin.java @@ -34,7 +34,6 @@ import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.authorization.User; -import org.apache.gravitino.authorization.common.JdbcAuthorizationProperties; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.RoleEntity; @@ -42,7 +41,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -public class JdbcAuthorizationPluginTest { +public class TestJdbcAuthorizationPlugin { private static List expectSQLs = Lists.newArrayList(); private static List expectTypes = Lists.newArrayList(); private static List expectObjectNames = Lists.newArrayList(); @@ -75,7 +74,7 @@ public List getSetOwnerSQL( return Collections.emptyList(); } - void executeUpdateSQL(String sql, String ignoreErrorMsg) { + public void executeUpdateSQL(String sql, String ignoreErrorMsg) { Assertions.assertEquals(expectSQLs.get(currentSQLIndex), sql); currentSQLIndex++; } diff --git a/authorizations/authorization-jdbc/build.gradle.kts b/authorizations/authorization-jdbc/build.gradle.kts deleted file mode 100644 index 1a61f7c0cf9..00000000000 --- a/authorizations/authorization-jdbc/build.gradle.kts +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -description = "authorization-jdbc" - -plugins { - `maven-publish` - id("java") - id("idea") -} - -dependencies { - implementation(project(":api")) { - exclude(group = "*") - } - implementation(project(":core")) { - exclude(group = "*") - } - implementation(project(":authorizations:authorization-common")) { - exclude(group = "*") - } - implementation(libs.bundles.log4j) - implementation(libs.commons.lang3) - implementation(libs.guava) - implementation(libs.javax.jaxb.api) { - exclude("*") - } - implementation(libs.javax.ws.rs.api) - implementation(libs.jettison) - compileOnly(libs.lombok) - implementation(libs.mail) - implementation(libs.rome) - implementation(libs.commons.dbcp2) - - testImplementation(project(":common")) - testImplementation(project(":clients:client-java")) - testImplementation(project(":server")) - testImplementation(project(":catalogs:catalog-common")) - testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(libs.junit.jupiter.api) - testImplementation(libs.mockito.core) - testImplementation(libs.testcontainers) - testRuntimeOnly(libs.junit.jupiter.engine) -} - -tasks { - val runtimeJars by registering(Copy::class) { - from(configurations.runtimeClasspath) - into("build/libs") - } - - val copyAuthorizationLibs by registering(Copy::class) { - dependsOn("jar", runtimeJars) - from("build/libs") { - exclude("guava-*.jar") - exclude("log4j-*.jar") - exclude("slf4j-*.jar") - } - into("$rootDir/distribution/package/authorizations/ranger/libs") - } - - register("copyLibAndConfig", Copy::class) { - dependsOn(copyAuthorizationLibs) - } - - jar { - dependsOn(runtimeJars) - } -} - -tasks.test { - dependsOn(":catalogs:catalog-hive:jar", ":catalogs:catalog-hive:runtimeJars") - - val skipITs = project.hasProperty("skipITs") - if (skipITs) { - // Exclude integration tests - exclude("**/integration/test/**") - } else { - dependsOn(tasks.jar) - } -} diff --git a/authorizations/authorization-ranger/build.gradle.kts b/authorizations/authorization-ranger/build.gradle.kts index d410b1ee8d4..8cc82250c23 100644 --- a/authorizations/authorization-ranger/build.gradle.kts +++ b/authorizations/authorization-ranger/build.gradle.kts @@ -67,7 +67,12 @@ dependencies { exclude("net.java.dev.jna") exclude("javax.ws.rs") exclude("org.eclipse.jetty") + // Conflicts with hadoop-client-api used in hadoop-catalog. + exclude("org.apache.hadoop", "hadoop-common") } + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) + implementation(libs.rome) compileOnly(libs.lombok) testRuntimeOnly(libs.junit.jupiter.engine) @@ -92,11 +97,7 @@ dependencies { testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion") { exclude("com.sun.jersey") } - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/build.gradle.kts b/build.gradle.kts index c64997f3a90..154b4e7f776 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -779,7 +779,7 @@ tasks { !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") && it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && - it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && it.name != "hadoop-common" + it.parent?.name != "bundles" && it.name != "hadoop-common" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -799,10 +799,8 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && - it.name != "bundled-catalog" && - it.name != "hive-metastore-common" && it.name != "gcp-bundle" && - it.name != "aliyun-bundle" && it.name != "aws-bundle" && it.name != "azure-bundle" && - it.name != "hadoop-common" && it.name != "docs" + it.name != "hive-metastore-common" && + it.name != "docs" && it.name != "hadoop-common" && it.parent?.name != "bundles" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/aliyun-bundle/build.gradle.kts b/bundles/aliyun-bundle/build.gradle.kts index 39883feef7a..bd0841ecb0d 100644 --- a/bundles/aliyun-bundle/build.gradle.kts +++ b/bundles/aliyun-bundle/build.gradle.kts @@ -25,34 +25,11 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - compileOnly(libs.hadoop3.common) - - implementation(libs.aliyun.credentials.sdk) + implementation(project(":bundles:aliyun")) + implementation(libs.commons.collections3) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.oss) - - // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add - // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 - implementation(libs.sun.activation) - - // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 - // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils - // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) - // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) - // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) - implementation(libs.commons.lang3) - - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } - implementation(project(":clients:client-java-runtime", configuration = "shadow")) - implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) } tasks.withType(ShadowJar::class.java) { @@ -62,8 +39,12 @@ tasks.withType(ShadowJar::class.java) { mergeServiceFiles() // Relocate dependencies to avoid conflicts - relocate("org.jdom", "org.apache.gravitino.shaded.org.jdom") - relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3") + relocate("org.jdom", "org.apache.gravitino.aliyun.shaded.org.jdom") + relocate("org.apache.commons.lang3", "org.apache.gravitino.aliyun.shaded.org.apache.commons.lang3") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aliyun.shaded.com.fasterxml.jackson") + relocate("com.google.common", "org.apache.gravitino.aliyun.shaded.com.google.common") + relocate("org.apache.http", "org.apache.gravitino.aliyun.shaded.org.apache.http") + relocate("org.apache.commons.collections", "org.apache.gravitino.aliyun.shaded.org.apache.commons.collections") } tasks.jar { diff --git a/bundles/aliyun/build.gradle.kts b/bundles/aliyun/build.gradle.kts new file mode 100644 index 00000000000..88425a90d92 --- /dev/null +++ b/bundles/aliyun/build.gradle.kts @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + compileOnly(libs.hadoop3.oss) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + + implementation(libs.aliyun.credentials.sdk) + implementation(libs.commons.collections3) + + // oss needs StringUtils from commons-lang3 or the following error will occur in 3.3.0 + // java.lang.NoClassDefFoundError: org/apache/commons/lang3/StringUtils + // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystemStore.initialize(AliyunOSSFileSystemStore.java:111) + // org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem.initialize(AliyunOSSFileSystem.java:323) + // org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:3611) + implementation(libs.commons.lang3) + implementation(libs.guava) + + implementation(libs.httpclient) + implementation(libs.jackson.databind) + implementation(libs.jackson.annotations) + implementation(libs.jackson.datatype.jdk8) + implementation(libs.jackson.datatype.jsr310) + + // Aliyun oss SDK depends on this package, and JDK >= 9 requires manual add + // https://www.alibabacloud.com/help/en/oss/developer-reference/java-installation?spm=a2c63.p38356.0.i1 + implementation(libs.sun.activation) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + mergeServiceFiles() + + // Relocate dependencies to avoid conflicts + relocate("org.jdom", "org.apache.gravitino.aliyun.shaded.org.jdom") + relocate("org.apache.commons.lang3", "org.apache.gravitino.aliyun.shaded.org.apache.commons.lang3") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aliyun.shaded.com.fasterxml.jackson") + relocate("com.google.common", "org.apache.gravitino.aliyun.shaded.com.google.common") + relocate("org.apache.http", "org.apache.gravitino.aliyun.shaded.org.apache.http") + relocate("org.apache.commons.collections", "org.apache.gravitino.aliyun.shaded.org.apache.commons.collections") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSSecretKeyProvider.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/OSSTokenProvider.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Condition.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Effect.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Policy.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/Statement.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/credential/policy/StringLike.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSFileSystemProvider.java diff --git a/bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSSessionCredentialProvider.java b/bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSSessionCredentialProvider.java similarity index 100% rename from bundles/aliyun-bundle/src/main/java/org/apache/gravitino/oss/fs/OSSSessionCredentialProvider.java rename to bundles/aliyun/src/main/java/org/apache/gravitino/oss/fs/OSSSessionCredentialProvider.java diff --git a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/aliyun-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/aliyun/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts index 3c2a6a867c1..35b1e22a4f6 100644 --- a/bundles/aws-bundle/build.gradle.kts +++ b/bundles/aws-bundle/build.gradle.kts @@ -25,32 +25,20 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - compileOnly(libs.hadoop3.common) - - implementation(libs.aws.iam) - implementation(libs.aws.policy) - implementation(libs.aws.sts) - implementation(libs.commons.lang3) + implementation(project(":bundles:aws")) implementation(libs.hadoop3.aws) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } - implementation(project(":clients:client-java-runtime", configuration = "shadow")) - implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) } tasks.withType(ShadowJar::class.java) { isZip64 = true configurations = listOf(project.configurations.runtimeClasspath.get()) - relocate("org.apache.commons", "org.apache.gravitino.aws.shaded.org.apache.commons") archiveClassifier.set("") + + relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3") + relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson") } tasks.jar { diff --git a/bundles/aws/build.gradle.kts b/bundles/aws/build.gradle.kts new file mode 100644 index 00000000000..f4a58d7dd81 --- /dev/null +++ b/bundles/aws/build.gradle.kts @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + compileOnly(libs.hadoop3.aws) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + + implementation(libs.aws.iam) + implementation(libs.aws.policy) + implementation(libs.aws.sts) + implementation(libs.commons.lang3) + implementation(libs.hadoop3.aws) + implementation(libs.guava) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + relocate("org.apache.commons.lang3", "org.apache.gravitino.aws.shaded.org.apache.commons.lang3") + relocate("com.google.common", "org.apache.gravitino.aws.shaded.com.google.common") + relocate("com.fasterxml.jackson", "org.apache.gravitino.aws.shaded.com.fasterxml.jackson") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java similarity index 100% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3SecretKeyProvider.java diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java similarity index 100% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/credential/S3TokenProvider.java diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java similarity index 56% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java index 152442a86d4..625ba462ee1 100644 --- a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java +++ b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java @@ -19,9 +19,14 @@ package org.apache.gravitino.s3.fs; +import com.amazonaws.auth.AWSCredentialsProvider; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; import java.io.IOException; +import java.util.List; import java.util.Map; import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; import org.apache.gravitino.catalog.hadoop.fs.FileSystemUtils; @@ -32,9 +37,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.s3a.Constants; import org.apache.hadoop.fs.s3a.S3AFileSystem; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class S3FileSystemProvider implements FileSystemProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(S3FileSystemProvider.class); + @VisibleForTesting public static final Map GRAVITINO_KEY_TO_S3_HADOOP_KEY = ImmutableMap.of( @@ -42,13 +51,18 @@ public class S3FileSystemProvider implements FileSystemProvider { S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, Constants.ACCESS_KEY, S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, Constants.SECRET_KEY); + // We can't use Constants.AWS_CREDENTIALS_PROVIDER directly, as in 2.7, this key does not exist. + private static final String S3_CREDENTIAL_KEY = "fs.s3a.aws.credentials.provider"; + private static final String S3_SIMPLE_CREDENTIAL = + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"; + @Override public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); Map hadoopConfMap = FileSystemUtils.toHadoopConfigMap(config, GRAVITINO_KEY_TO_S3_HADOOP_KEY); - if (!hadoopConfMap.containsKey(Constants.AWS_CREDENTIALS_PROVIDER) + if (!hadoopConfMap.containsKey(S3_CREDENTIAL_KEY) && config.containsKey( GravitinoVirtualFileSystemConfiguration.FS_GRAVITINO_SERVER_URI_KEY)) { hadoopConfMap.put( @@ -56,9 +70,51 @@ public FileSystem getFileSystem(Path path, Map config) throws IO } hadoopConfMap.forEach(configuration::set); + + // Hadoop-aws 2 does not support IAMInstanceCredentialsProvider + checkAndSetCredentialProvider(configuration); + return S3AFileSystem.newInstance(path.toUri(), configuration); } + private void checkAndSetCredentialProvider(Configuration configuration) { + String provides = configuration.get(S3_CREDENTIAL_KEY); + if (provides == null) { + return; + } + + Splitter splitter = Splitter.on(',').trimResults().omitEmptyStrings(); + Joiner joiner = Joiner.on(",").skipNulls(); + // Split the list of providers + List providers = splitter.splitToList(provides); + List validProviders = Lists.newArrayList(); + + for (String provider : providers) { + try { + Class c = Class.forName(provider); + if (AWSCredentialsProvider.class.isAssignableFrom(c)) { + validProviders.add(provider); + } else { + LOGGER.warn( + "Credential provider {} is not a subclass of AWSCredentialsProvider, skipping", + provider); + } + } catch (Exception e) { + LOGGER.warn( + "Credential provider {} not found in the Hadoop runtime, falling back to default", + provider); + configuration.set(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL); + return; + } + } + + if (validProviders.isEmpty()) { + configuration.set(S3_CREDENTIAL_KEY, S3_SIMPLE_CREDENTIAL); + } else { + configuration.set(S3_CREDENTIAL_KEY, joiner.join(validProviders)); + } + } + /** * Get the scheme of the FileSystem. Attention, for S3 the schema is "s3a", not "s3". Users should * use "s3a://..." to access S3. diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3SessionCredentialProvider.java b/bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3SessionCredentialProvider.java similarity index 100% rename from bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3SessionCredentialProvider.java rename to bundles/aws/src/main/java/org/apache/gravitino/s3/fs/S3SessionCredentialProvider.java diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/aws/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/azure-bundle/build.gradle.kts b/bundles/azure-bundle/build.gradle.kts index f9ce722b807..7d9e253ac8a 100644 --- a/bundles/azure-bundle/build.gradle.kts +++ b/bundles/azure-bundle/build.gradle.kts @@ -25,29 +25,10 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - - compileOnly(libs.hadoop3.common) - - implementation(libs.azure.identity) - implementation(libs.azure.storage.file.datalake) - - implementation(libs.commons.lang3) - // runtime used - implementation(libs.commons.logging) + implementation(project(":bundles:azure")) implementation(libs.hadoop3.abs) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } - implementation(project(":clients:client-java-runtime", configuration = "shadow")) - - implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) } tasks.withType(ShadowJar::class.java) { @@ -59,7 +40,8 @@ tasks.withType(ShadowJar::class.java) { relocate("org.apache.httpcomponents", "org.apache.gravitino.azure.shaded.org.apache.httpcomponents") relocate("org.apache.commons", "org.apache.gravitino.azure.shaded.org.apache.commons") relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml") - relocate("com.google.guava", "org.apache.gravitino.azure.shaded.com.google.guava") + relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common") + relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty") } tasks.jar { diff --git a/bundles/azure/build.gradle.kts b/bundles/azure/build.gradle.kts new file mode 100644 index 00000000000..3513cc02d16 --- /dev/null +++ b/bundles/azure/build.gradle.kts @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + + compileOnly(libs.hadoop3.abs) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + + implementation(libs.azure.identity) + implementation(libs.azure.storage.file.datalake) + + implementation(libs.commons.lang3) + // runtime used + implementation(libs.commons.logging) + implementation(libs.guava) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("org.apache.httpcomponents", "org.apache.gravitino.azure.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.azure.shaded.org.apache.commons") + relocate("com.fasterxml", "org.apache.gravitino.azure.shaded.com.fasterxml") + relocate("com.google.common", "org.apache.gravitino.azure.shaded.com.google.common") + relocate("org.eclipse.jetty", "org.apache.gravitino.azure.shaded.org.eclipse.jetty") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSLocationUtils.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureFileSystemProvider.java diff --git a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureSasCredentialProvider.java b/bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureSasCredentialProvider.java similarity index 100% rename from bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/fs/AzureSasCredentialProvider.java rename to bundles/azure/src/main/java/org/apache/gravitino/abs/fs/AzureSasCredentialProvider.java diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/azure/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/bundles/gcp-bundle/build.gradle.kts b/bundles/gcp-bundle/build.gradle.kts index 05c0c8d9146..73efaf9f22c 100644 --- a/bundles/gcp-bundle/build.gradle.kts +++ b/bundles/gcp-bundle/build.gradle.kts @@ -25,26 +25,10 @@ plugins { } dependencies { - compileOnly(project(":api")) - compileOnly(project(":core")) - compileOnly(project(":catalogs:catalog-common")) - compileOnly(project(":catalogs:catalog-hadoop")) - compileOnly(project(":catalogs:hadoop-common")) { - exclude("*") - } - - compileOnly(libs.hadoop3.common) - - implementation(libs.commons.lang3) - // runtime used - implementation(libs.commons.logging) + implementation(project(":bundles:gcp")) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.gcs) - implementation(project(":catalogs:catalog-common")) { - exclude("*") - } - implementation(libs.google.auth.http) - implementation(libs.google.auth.credentials) - implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) } tasks.withType(ShadowJar::class.java) { @@ -55,8 +39,9 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") - relocate("com.google", "org.apache.gravitino.gcp.shaded.com.google") + relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common") relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml") + relocate("org.eclipse.jetty", "org.apache.gravitino.gcp.shaded.org.eclipse.jetty") } tasks.jar { diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem b/bundles/gcp-bundle/src/main/resources/org.apache.hadoop.fs.FileSystem similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem rename to bundles/gcp-bundle/src/main/resources/org.apache.hadoop.fs.FileSystem diff --git a/bundles/gcp/build.gradle.kts b/bundles/gcp/build.gradle.kts new file mode 100644 index 00000000000..bd6cc605d52 --- /dev/null +++ b/bundles/gcp/build.gradle.kts @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":api")) + compileOnly(project(":catalogs:catalog-common")) + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(project(":core")) + + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) + compileOnly(libs.hadoop3.gcs) + + implementation(project(":catalogs:catalog-common")) { + exclude("*") + } + implementation(project(":catalogs:hadoop-common")) { + exclude("*") + } + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(project(":clients:filesystem-hadoop3-runtime", configuration = "shadow")) + + implementation(libs.commons.lang3) + // runtime used + implementation(libs.commons.logging) + implementation(libs.google.auth.credentials) + implementation(libs.google.auth.http) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") + + // Relocate dependencies to avoid conflicts + relocate("org.apache.httpcomponents", "org.apache.gravitino.gcp.shaded.org.apache.httpcomponents") + relocate("org.apache.commons", "org.apache.gravitino.gcp.shaded.org.apache.commons") + relocate("com.google.common", "org.apache.gravitino.gcp.shaded.com.google.common") + relocate("com.fasterxml", "org.apache.gravitino.gcp.shaded.com.fasterxml") + relocate("com.fasterxml.jackson", "org.apache.gravitino.gcp.shaded.com.fasterxml.jackson") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java similarity index 100% rename from bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java rename to bundles/gcp/src/main/java/org/apache/gravitino/gcs/credential/GCSTokenProvider.java diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSCredentialProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSCredentialProvider.java similarity index 100% rename from bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSCredentialProvider.java rename to bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSCredentialProvider.java diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java similarity index 88% rename from bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java rename to bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java index fa26015971f..3ac0fc4a63a 100644 --- a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java +++ b/bundles/gcp/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java @@ -18,7 +18,6 @@ */ package org.apache.gravitino.gcs.fs; -import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; import com.google.cloud.hadoop.util.AccessTokenProvider; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; @@ -31,11 +30,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; public class GCSFileSystemProvider implements FileSystemProvider { - private static final Logger LOGGER = LoggerFactory.getLogger(GCSFileSystemProvider.class); private static final String GCS_SERVICE_ACCOUNT_JSON_FILE = "fs.gs.auth.service.account.json.keyfile"; @@ -55,12 +51,11 @@ public FileSystem getFileSystem(Path path, Map config) throws IO // Why is this check necessary?, if Gravitino fails to get any credentials, we fall back to // the default behavior of the GoogleHadoopFileSystem to use service account credentials. if (accessTokenProvider.getAccessToken() != null) { - LOGGER.info("Creating GCS file system with credential provider: {}", config); configuration.set( "fs.gs.auth.access.token.provider.impl", GCSCredentialProvider.class.getName()); } } - return GoogleHadoopFileSystem.newInstance(path.toUri(), configuration); + return FileSystem.newInstance(path.toUri(), configuration); } @Override diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider rename to bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider diff --git a/bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider b/bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider similarity index 100% rename from bundles/gcp-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider rename to bundles/gcp/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java new file mode 100644 index 00000000000..468728362bb --- /dev/null +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/hadoop/Constants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.hadoop; + +public class Constants { + + public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local"; + public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs"; +} diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java index d2753f24b5e..c766a86c141 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java @@ -22,6 +22,8 @@ public class CredentialConstants { public static final String CREDENTIAL_PROVIDER_TYPE = "credential-provider-type"; public static final String CREDENTIAL_PROVIDERS = "credential-providers"; + public static final String CREDENTIAL_CACHE_EXPIRE_RATIO = "credential-cache-expire-ratio"; + public static final String CREDENTIAL_CACHE_MAX_SIZE = "credential-cache-max-size"; public static final String S3_TOKEN_CREDENTIAL_PROVIDER = "s3-token"; public static final String S3_TOKEN_EXPIRE_IN_SECS = "s3-token-expire-in-secs"; diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 8873b795046..d599a5e72f1 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -28,43 +28,22 @@ dependencies { implementation(project(":api")) { exclude(group = "*") } - - implementation(project(":core")) { - exclude(group = "*") - } - - implementation(project(":common")) { - exclude(group = "*") - } - implementation(project(":catalogs:catalog-common")) { exclude(group = "*") } - implementation(project(":catalogs:hadoop-common")) { exclude(group = "*") } - - implementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - exclude("org.eclipse.jetty", "*") - exclude("org.apache.hadoop", "hadoop-auth") - exclude("org.apache.curator", "curator-client") - exclude("org.apache.curator", "curator-framework") - exclude("org.apache.curator", "curator-recipes") - exclude("org.apache.avro", "avro") - exclude("com.sun.jersey", "jersey-servlet") + implementation(project(":common")) { + exclude(group = "*") } - - implementation(libs.hadoop3.client) { - exclude("org.apache.hadoop", "hadoop-mapreduce-client-core") - exclude("org.apache.hadoop", "hadoop-mapreduce-client-jobclient") - exclude("org.apache.hadoop", "hadoop-yarn-api") - exclude("org.apache.hadoop", "hadoop-yarn-client") - exclude("com.squareup.okhttp", "okhttp") + implementation(project(":core")) { + exclude(group = "*") } - + implementation(libs.commons.lang3) + implementation(libs.commons.io) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) implementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") @@ -74,20 +53,18 @@ dependencies { exclude("io.netty") exclude("org.fusesource.leveldbjni") } - implementation(libs.slf4j.api) compileOnly(libs.guava) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:gcp-bundle")) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:azure-bundle")) testImplementation(project(":clients:client-java")) + testImplementation(project(":bundles:aws-bundle", configuration = "shadow")) + testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) + testImplementation(project(":bundles:aliyun-bundle", configuration = "shadow")) + testImplementation(project(":bundles:azure-bundle", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) - testImplementation(libs.bundles.log4j) testImplementation(libs.hadoop3.gcs) testImplementation(libs.hadoop3.minicluster) diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index b471fccead1..6a8b815ab97 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -96,6 +96,9 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) + testImplementation(project(":catalogs:hadoop-common")) { + exclude("*") + } testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) diff --git a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java index 5b90eab7265..545f6482a3f 100644 --- a/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java +++ b/catalogs/catalog-model/src/main/java/org/apache/gravitino/catalog/model/ModelCatalogImpl.java @@ -19,7 +19,6 @@ package org.apache.gravitino.catalog.model; import java.util.Map; -import org.apache.gravitino.CatalogProvider; import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.connector.BaseCatalog; @@ -40,7 +39,7 @@ public class ModelCatalogImpl extends BaseCatalog { @Override public String shortName() { - return CatalogProvider.shortNameForManagedCatalog(super.type()); + return "model"; } @Override diff --git a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider index 37c682aa745..e43f995ea7d 100644 --- a/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider +++ b/catalogs/catalog-model/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider @@ -16,4 +16,4 @@ # specific language governing permissions and limitations # under the License. # -org.apache.gravitino.catalog.model.ModelCatalog +org.apache.gravitino.catalog.model.ModelCatalogImpl diff --git a/catalogs/hadoop-common/build.gradle.kts b/catalogs/hadoop-common/build.gradle.kts index ab768cb1f11..566ce5986e3 100644 --- a/catalogs/hadoop-common/build.gradle.kts +++ b/catalogs/hadoop-common/build.gradle.kts @@ -23,6 +23,9 @@ plugins { // try to avoid adding extra dependencies because it is used by catalogs and connectors. dependencies { + implementation(project(":catalogs:catalog-common")) implementation(libs.commons.lang3) - implementation(libs.hadoop3.common) + implementation(libs.hadoop3.client.api) + implementation(libs.hadoop3.client.runtime) + implementation(libs.guava) } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java similarity index 95% rename from catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java rename to catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java index 129a8e88274..a1434e85c3e 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java +++ b/catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/FileSystemUtils.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.catalog.hadoop.fs; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_HDFS_FS_PROVIDER; -import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.BUILTIN_LOCAL_FS_PROVIDER; +import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_HDFS_FS_PROVIDER; +import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_LOCAL_FS_PROVIDER; import static org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider.GRAVITINO_BYPASS; import com.google.common.collect.Maps; @@ -45,7 +45,7 @@ public static Map getFileSystemProviders(String file fileSystemProviders != null ? Arrays.stream(fileSystemProviders.split(",")) .map(f -> f.trim().toLowerCase(Locale.ROOT)) - .collect(java.util.stream.Collectors.toSet()) + .collect(Collectors.toSet()) : Sets.newHashSet(); // Add built-in file system providers to the use list automatically. diff --git a/clients/filesystem-hadoop3-runtime/build.gradle.kts b/clients/filesystem-hadoop3-runtime/build.gradle.kts index 8081a55604e..db439a4981e 100644 --- a/clients/filesystem-hadoop3-runtime/build.gradle.kts +++ b/clients/filesystem-hadoop3-runtime/build.gradle.kts @@ -28,6 +28,7 @@ plugins { dependencies { implementation(project(":clients:filesystem-hadoop3")) implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(libs.commons.lang3) } tasks.withType(ShadowJar::class.java) { @@ -38,6 +39,8 @@ tasks.withType(ShadowJar::class.java) { // Relocate dependencies to avoid conflicts relocate("com.google", "org.apache.gravitino.shaded.com.google") relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine") + // relocate common lang3 package + relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3") } tasks.jar { diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index d24eb4efdf2..424f6a11406 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -25,7 +25,8 @@ plugins { dependencies { compileOnly(project(":clients:client-java-runtime", configuration = "shadow")) - compileOnly(libs.hadoop3.common) + compileOnly(libs.hadoop3.client.api) + compileOnly(libs.hadoop3.client.runtime) implementation(project(":catalogs:catalog-common")) { exclude(group = "*") @@ -35,32 +36,31 @@ dependencies { } implementation(libs.caffeine) + implementation(libs.guava) + implementation(libs.commons.lang3) testImplementation(project(":api")) testImplementation(project(":core")) + testImplementation(project(":catalogs:catalog-hadoop")) testImplementation(project(":common")) testImplementation(project(":server")) testImplementation(project(":server-common")) testImplementation(project(":clients:client-java")) testImplementation(project(":integration-test-common", "testArtifacts")) - testImplementation(project(":catalogs:catalog-hadoop")) - testImplementation(project(":bundles:gcp-bundle")) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:azure-bundle")) - testImplementation(project(":bundles:gcp-bundle")) + + testImplementation(project(":bundles:aws-bundle", configuration = "shadow")) + testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) + testImplementation(project(":bundles:aliyun-bundle", configuration = "shadow")) + testImplementation(project(":bundles:azure-bundle", configuration = "shadow")) testImplementation(libs.awaitility) testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) testImplementation(libs.bundles.jwt) - testImplementation(libs.guava) - testImplementation(libs.hadoop3.client) - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(libs.hadoop3.hdfs) { exclude("com.sun.jersey") exclude("javax.servlet", "servlet-api") diff --git a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java index a9a4c8ac621..5b422da1327 100644 --- a/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java +++ b/clients/filesystem-hadoop3/src/main/java/org/apache/gravitino/filesystem/hadoop/GravitinoVirtualFileSystem.java @@ -42,6 +42,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.audit.CallerContext; import org.apache.gravitino.audit.FilesetAuditConstants; @@ -397,6 +398,11 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat scheme, GravitinoVirtualFileSystemConfiguration.GVFS_SCHEME); } + // Reset the FileSystem service loader to make sure the FileSystem will reload the + // service file systems, this is a temporary solution to fix the issue + // https://github.com/apache/gravitino/issues/5609 + resetFileSystemServiceLoader(scheme); + Map maps = getConfigMap(getConf()); // If enable the cloud store credential, we should pass the configuration here. maps.put(GVFS_FILESET_IDENTIFIER, identifier.toString()); @@ -412,6 +418,24 @@ private FilesetContextPair getFilesetContext(Path virtualPath, FilesetDataOperat return new FilesetContextPair(new Path(actualFileLocation), fs); } + private void resetFileSystemServiceLoader(String fsScheme) { + try { + Map> serviceFileSystems = + (Map>) + FieldUtils.getField(FileSystem.class, "SERVICE_FILE_SYSTEMS", true).get(null); + + if (serviceFileSystems.containsKey(fsScheme)) { + return; + } + + // Set this value to false so that FileSystem will reload the service file systems when + // needed. + FieldUtils.getField(FileSystem.class, "FILE_SYSTEMS_LOADED", true).set(null, false); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private Map getConfigMap(Configuration configuration) { Map maps = Maps.newHashMap(); configuration.forEach(entry -> maps.put(entry.getKey(), entry.getValue())); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java index 02f9499a34c..3d8fd457611 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemABSCredentialIT.java @@ -171,6 +171,9 @@ public void testAppend() throws IOException {} private static boolean absIsConfigured() { return StringUtils.isNotBlank(System.getenv("ABS_STS_ACCOUNT_NAME")) && StringUtils.isNotBlank(System.getenv("ABS_STS_ACCOUNT_KEY")) - && StringUtils.isNotBlank(System.getenv("ABS_STS_CONTAINER_NAME")); + && StringUtils.isNotBlank(System.getenv("ABS_STS_CONTAINER_NAME")) + && StringUtils.isNotBlank(System.getenv("ABS_STS_TENANT_ID")) + && StringUtils.isNotBlank(System.getenv("ABS_STS_CLIENT_ID")) + && StringUtils.isNotBlank(System.getenv("ABS_STS_CLIENT_SECRET")); } } diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java index b5e1a5418a3..6fe3938a1d3 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSCredentialIT.java @@ -161,6 +161,8 @@ protected static boolean ossIsConfigured() { return StringUtils.isNotBlank(System.getenv("OSS_STS_ACCESS_KEY_ID")) && StringUtils.isNotBlank(System.getenv("OSS_STS_SECRET_ACCESS_KEY")) && StringUtils.isNotBlank(System.getenv("OSS_STS_ENDPOINT")) - && StringUtils.isNotBlank(System.getenv("OSS_STS_BUCKET_NAME")); + && StringUtils.isNotBlank(System.getenv("OSS_STS_BUCKET_NAME")) + && StringUtils.isNotBlank(System.getenv("OSS_STS_REGION")) + && StringUtils.isNotBlank(System.getenv("OSS_STS_ROLE_ARN")); } } diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java index 0490399bc55..19297b131ab 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3CredentialIT.java @@ -37,9 +37,12 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.condition.EnabledIf; +import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@EnabledIf(value = "s3IsConfigured", disabledReason = "s3 with credential is not prepared") public class GravitinoVirtualFileSystemS3CredentialIT extends GravitinoVirtualFileSystemIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemS3CredentialIT.class); @@ -155,4 +158,13 @@ protected String genStorageLocation(String fileset) { @Disabled( "GCS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} + + protected static boolean s3IsConfigured() { + return StringUtils.isNotBlank(System.getenv("S3_STS_ACCESS_KEY_ID")) + && StringUtils.isNotBlank(System.getenv("S3_STS_SECRET_ACCESS_KEY")) + && StringUtils.isNotBlank(System.getenv("S3_STS_ENDPOINT")) + && StringUtils.isNotBlank(System.getenv("S3_STS_BUCKET_NAME")) + && StringUtils.isNotBlank(System.getenv("S3_STS_REGION")) + && StringUtils.isNotBlank(System.getenv("S3_STS_ROLE_ARN")); + } } diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java new file mode 100644 index 00000000000..44688399335 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelDTO.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.Model; + +/** Represents a model DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelDTO implements Model { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("latestVersion") + private int latestVersion; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public String name() { + return name; + } + + @Override + public String comment() { + return comment; + } + + @Override + public Map properties() { + return properties; + } + + @Override + public int latestVersion() { + return latestVersion; + } + + @Override + public AuditDTO auditInfo() { + return audit; + } + + /** + * Creates a new builder for constructing a Model DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model DTO. */ + public static class Builder { + private String name; + private String comment; + private Map properties; + private int latestVersion; + private AuditDTO audit; + + /** + * Sets the name of the model. + * + * @param name The name of the model. + * @return The builder. + */ + public Builder withName(String name) { + this.name = name; + return this; + } + + /** + * Sets the comment associated with the model. + * + * @param comment The comment associated with the model. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the properties associated with the model. + * + * @param properties The properties associated with the model. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the latest version of the model. + * + * @param latestVersion The latest version of the model. + * @return The builder. + */ + public Builder withLatestVersion(int latestVersion) { + this.latestVersion = latestVersion; + return this; + } + + /** + * Sets the audit information associated with the model. + * + * @param audit The audit information associated with the model. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the model DTO. + * + * @return The model DTO. + */ + public ModelDTO build() { + Preconditions.checkArgument(StringUtils.isNotBlank(name), "name cannot be null or empty"); + Preconditions.checkArgument(latestVersion >= 0, "latestVersion cannot be negative"); + Preconditions.checkArgument(audit != null, "audit cannot be null"); + + return new ModelDTO(name, comment, properties, latestVersion, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java new file mode 100644 index 00000000000..e887ba5bdb2 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/model/ModelVersionDTO.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.Audit; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.model.ModelVersion; + +/** Represents a model version DTO (Data Transfer Object). */ +@NoArgsConstructor(access = AccessLevel.PRIVATE, force = true) +@AllArgsConstructor(access = AccessLevel.PRIVATE) +@EqualsAndHashCode +public class ModelVersionDTO implements ModelVersion { + + @JsonProperty("version") + private int version; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("uri") + private String uri; + + @JsonProperty("properties") + private Map properties; + + @JsonProperty("audit") + private AuditDTO audit; + + @Override + public Audit auditInfo() { + return audit; + } + + @Override + public int version() { + return version; + } + + @Override + public String comment() { + return comment; + } + + @Override + public String[] aliases() { + return aliases; + } + + @Override + public String uri() { + return uri; + } + + @Override + public Map properties() { + return properties; + } + + /** + * Creates a new builder for constructing a Model Version DTO. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** Builder for constructing a Model Version DTO. */ + public static class Builder { + private int version; + private String comment; + private String[] aliases; + private String uri; + private Map properties; + private AuditDTO audit; + + /** + * Sets the version number of the model version. + * + * @param version The version number. + * @return The builder. + */ + public Builder withVersion(int version) { + this.version = version; + return this; + } + + /** + * Sets the comment of the model version. + * + * @param comment The comment. + * @return The builder. + */ + public Builder withComment(String comment) { + this.comment = comment; + return this; + } + + /** + * Sets the aliases of the model version. + * + * @param aliases The aliases. + * @return The builder. + */ + public Builder withAliases(String[] aliases) { + this.aliases = aliases; + return this; + } + + /** + * Sets the URI of the model version. + * + * @param uri The URI. + * @return The builder. + */ + public Builder withUri(String uri) { + this.uri = uri; + return this; + } + + /** + * Sets the properties of the model version. + * + * @param properties The properties. + * @return The builder. + */ + public Builder withProperties(Map properties) { + this.properties = properties; + return this; + } + + /** + * Sets the audit information of the model version. + * + * @param audit The audit information. + * @return The builder. + */ + public Builder withAudit(AuditDTO audit) { + this.audit = audit; + return this; + } + + /** + * Builds the Model Version DTO. + * + * @return The Model Version DTO. + */ + public ModelVersionDTO build() { + Preconditions.checkArgument(version >= 0, "Version must be non-negative"); + Preconditions.checkArgument(StringUtils.isNotBlank(uri), "URI cannot be null or empty"); + Preconditions.checkArgument(audit != null, "Audit cannot be null"); + + return new ModelVersionDTO(version, comment, aliases, uri, properties, audit); + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java index 3da6579676d..d543ddb1649 100644 --- a/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java +++ b/common/src/main/java/org/apache/gravitino/dto/requests/CatalogCreateRequest.java @@ -18,8 +18,8 @@ */ package org.apache.gravitino.dto.requests; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; import com.google.common.base.Preconditions; import java.util.Map; import javax.annotation.Nullable; @@ -54,11 +54,6 @@ public class CatalogCreateRequest implements RESTRequest { @JsonProperty("properties") private final Map properties; - /** Default constructor for CatalogCreateRequest. */ - public CatalogCreateRequest() { - this(null, null, null, null, null); - } - /** * Constructor for CatalogCreateRequest. * @@ -68,34 +63,24 @@ public CatalogCreateRequest() { * @param comment The comment for the catalog. * @param properties The properties for the catalog. */ + @JsonCreator public CatalogCreateRequest( - String name, - Catalog.Type type, - String provider, - String comment, - Map properties) { + @JsonProperty("name") String name, + @JsonProperty("type") Catalog.Type type, + @JsonProperty("provider") String provider, + @JsonProperty("comment") String comment, + @JsonProperty("properties") Map properties) { this.name = name; this.type = type; - this.provider = provider; this.comment = comment; this.properties = properties; - } - /** - * Sets the provider of the catalog if it is null. The value of provider in the request can be - * null if the catalog is a managed catalog. For such request, the value will be set when it is - * deserialized. - * - * @param provider The provider of the catalog. - */ - @JsonSetter(value = "provider") - public void setProvider(String provider) { if (StringUtils.isNotBlank(provider)) { this.provider = provider; } else if (type != null && type.supportsManagedCatalog()) { this.provider = CatalogProvider.shortNameForManagedCatalog(type); } else { - throw new IllegalStateException( + throw new IllegalArgumentException( "Provider cannot be null for catalog type " + type + " that doesn't support managed catalog"); diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java new file mode 100644 index 00000000000..b9cd1916174 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelRegisterRequest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to register a model. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelRegisterRequest implements RESTRequest { + + @JsonProperty("name") + private String name; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(name), "\"name\" field is required and cannot be empty"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java new file mode 100644 index 00000000000..24e5932932c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/ModelVersionLinkRequest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.math.NumberUtils; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to link a model version. */ +@Getter +@ToString +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ModelVersionLinkRequest implements RESTRequest { + + @JsonProperty("uri") + private String uri; + + @JsonProperty("aliases") + private String[] aliases; + + @JsonProperty("comment") + private String comment; + + @JsonProperty("properties") + private Map properties; + + @Override + public void validate() throws IllegalArgumentException { + Preconditions.checkArgument( + StringUtils.isNotBlank(uri), "\"uri\" field is required and cannot be empty"); + + if (aliases != null && aliases.length > 0) { + for (String alias : aliases) { + Preconditions.checkArgument( + StringUtils.isNotBlank(alias), "alias must not be null or empty"); + Preconditions.checkArgument( + !NumberUtils.isCreatable(alias), "alias must not be a number or a number string"); + } + } + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java new file mode 100644 index 00000000000..ac51cdd647c --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelDTO; + +/** Response for model response. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelResponse extends BaseResponse { + + @JsonProperty("model") + private final ModelDTO model; + + /** + * Constructor for ModelResponse. + * + * @param model The model DTO object. + */ + public ModelResponse(ModelDTO model) { + super(0); + this.model = model; + } + + /** Default constructor for ModelResponse. (Used for Jackson deserialization.) */ + public ModelResponse() { + super(); + this.model = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(model != null, "model must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java new file mode 100644 index 00000000000..4d3551e1397 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionListResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; + +/** Represents a response for a list of model versions. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionListResponse extends BaseResponse { + + @JsonProperty("versions") + private int[] versions; + + /** + * Constructor for ModelVersionListResponse. + * + * @param versions The list of model versions. + */ + public ModelVersionListResponse(int[] versions) { + super(0); + this.versions = versions; + } + + /** Default constructor for ModelVersionListResponse. (Used for Jackson deserialization.) */ + public ModelVersionListResponse() { + super(); + this.versions = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + Preconditions.checkArgument(versions != null, "versions cannot be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java new file mode 100644 index 00000000000..8b21472833b --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/responses/ModelVersionResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.responses; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.dto.model.ModelVersionDTO; + +/** Represents a response for a model version. */ +@Getter +@ToString +@EqualsAndHashCode(callSuper = true) +public class ModelVersionResponse extends BaseResponse { + + @JsonProperty("modelVersion") + private final ModelVersionDTO modelVersion; + + /** + * Constructor for ModelVersionResponse. + * + * @param modelVersion The model version DTO object. + */ + public ModelVersionResponse(ModelVersionDTO modelVersion) { + super(0); + this.modelVersion = modelVersion; + } + + /** Default constructor for ModelVersionResponse. (Used for Jackson deserialization.) */ + public ModelVersionResponse() { + super(); + this.modelVersion = null; + } + + @Override + public void validate() throws IllegalArgumentException { + super.validate(); + + Preconditions.checkArgument(modelVersion != null, "modelVersion must not be null"); + } +} diff --git a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java index 254de8c3245..ce63398e605 100644 --- a/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java +++ b/common/src/main/java/org/apache/gravitino/dto/util/DTOConverters.java @@ -51,6 +51,8 @@ import org.apache.gravitino.dto.credential.CredentialDTO; import org.apache.gravitino.dto.file.FilesetDTO; import org.apache.gravitino.dto.messaging.TopicDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.DistributionDTO; import org.apache.gravitino.dto.rel.SortOrderDTO; @@ -80,6 +82,8 @@ import org.apache.gravitino.dto.tag.TagDTO; import org.apache.gravitino.file.Fileset; import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; import org.apache.gravitino.rel.Column; import org.apache.gravitino.rel.Table; import org.apache.gravitino.rel.expressions.Expression; @@ -629,6 +633,39 @@ public static TopicDTO toDTO(Topic topic) { .build(); } + /** + * Converts a Model to a ModelDTO. + * + * @param model The model to be converted. + * @return The model DTO. + */ + public static ModelDTO toDTO(Model model) { + return ModelDTO.builder() + .withName(model.name()) + .withComment(model.comment()) + .withProperties(model.properties()) + .withLatestVersion(model.latestVersion()) + .withAudit(toDTO(model.auditInfo())) + .build(); + } + + /** + * Converts a ModelVersion to a ModelVersionDTO. + * + * @param modelVersion The model version to be converted. + * @return The model version DTO. + */ + public static ModelVersionDTO toDTO(ModelVersion modelVersion) { + return ModelVersionDTO.builder() + .withVersion(modelVersion.version()) + .withComment(modelVersion.comment()) + .withAliases(modelVersion.aliases()) + .withUri(modelVersion.uri()) + .withProperties(modelVersion.properties()) + .withAudit(toDTO(modelVersion.auditInfo())) + .build(); + } + /** * Converts an array of Columns to an array of ColumnDTOs. * diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java new file mode 100644 index 00000000000..39e4628eca2 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelDTO.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelDTO { + + @Test + public void testModelSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model_test") + .withComment("model comment") + .withLatestVersion(0) + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelDTO); + ModelDTO deserModelDTO = JsonUtils.objectMapper().readValue(serJson, ModelDTO.class); + Assertions.assertEquals(modelDTO, deserModelDTO); + + // Test with null comment and properties + ModelDTO modelDTO1 = + ModelDTO.builder().withName("model_test").withLatestVersion(0).withAudit(audit).build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelDTO1); + ModelDTO deserModelDTO1 = JsonUtils.objectMapper().readValue(serJson1, ModelDTO.class); + Assertions.assertEquals(modelDTO1, deserModelDTO1); + Assertions.assertNull(deserModelDTO1.comment()); + Assertions.assertNull(deserModelDTO1.properties()); + } + + @Test + public void testInvalidModelDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelDTO.builder().withName("model_test").withLatestVersion(0).build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java new file mode 100644 index 00000000000..5251246c377 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/model/TestModelVersionDTO.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.model; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.time.Instant; +import java.util.Map; +import org.apache.gravitino.dto.AuditDTO; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionDTO { + + @Test + public void testModelVersionSerDe() throws JsonProcessingException { + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + Map props = ImmutableMap.of("key", "value"); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO); + ModelVersionDTO deserModelVersionDTO = + JsonUtils.objectMapper().readValue(serJson, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO, deserModelVersionDTO); + + // Test with null aliases + ModelVersionDTO modelVersionDTO1 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO1); + ModelVersionDTO deserModelVersionDTO1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO1, deserModelVersionDTO1); + Assertions.assertNull(deserModelVersionDTO1.aliases()); + + // Test with empty aliases + ModelVersionDTO modelVersionDTO2 = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO2); + ModelVersionDTO deserModelVersionDTO2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO2, deserModelVersionDTO2); + Assertions.assertArrayEquals(new String[] {}, deserModelVersionDTO2.aliases()); + + // Test with null comment and properties + ModelVersionDTO modelVersionDTO3 = + ModelVersionDTO.builder().withVersion(0).withUri("uri").withAudit(audit).build(); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(modelVersionDTO3); + ModelVersionDTO deserModelVersionDTO3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionDTO.class); + + Assertions.assertEquals(modelVersionDTO3, deserModelVersionDTO3); + Assertions.assertNull(deserModelVersionDTO3.comment()); + Assertions.assertNull(deserModelVersionDTO3.properties()); + } + + @Test + public void testInvalidModelVersionDTO() { + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(-1).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("").build(); + }); + + Assertions.assertThrows( + IllegalArgumentException.class, + () -> { + ModelVersionDTO.builder().withVersion(0).withUri("uri").build(); + }); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java index b4b7383a7ea..3b5221ff1a7 100644 --- a/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestCatalogCreateRequest.java @@ -57,19 +57,24 @@ public void testCatalogCreateRequestSerDe() throws JsonProcessingException { String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); CatalogCreateRequest deserRequest1 = JsonUtils.objectMapper().readValue(serJson1, CatalogCreateRequest.class); - Assertions.assertEquals( deserRequest1.getType().name().toLowerCase(Locale.ROOT), deserRequest1.getProvider()); Assertions.assertNull(deserRequest1.getComment()); Assertions.assertNull(deserRequest1.getProperties()); + String json = "{\"name\":\"catalog_test\",\"type\":\"model\"}"; + CatalogCreateRequest deserRequest2 = + JsonUtils.objectMapper().readValue(json, CatalogCreateRequest.class); + Assertions.assertEquals("model", deserRequest2.getProvider()); + // Test using null provider with catalog type doesn't support managed catalog - CatalogCreateRequest request2 = - new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> new CatalogCreateRequest("catalog_test", Catalog.Type.RELATIONAL, null, null, null)); - String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + String json1 = "{\"name\":\"catalog_test\",\"type\":\"relational\"}"; Assertions.assertThrows( JsonMappingException.class, - () -> JsonUtils.objectMapper().readValue(serJson2, CatalogCreateRequest.class)); + () -> JsonUtils.objectMapper().readValue(json1, CatalogCreateRequest.class)); } } diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java new file mode 100644 index 00000000000..09dbdb8c312 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelRegisterRequest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelRegisterRequest { + + @Test + public void testModelRegisterRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelRegisterRequest req = new ModelRegisterRequest("model", "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(req); + ModelRegisterRequest deserReq = + JsonUtils.objectMapper().readValue(serJson, ModelRegisterRequest.class); + Assertions.assertEquals(req, deserReq); + + // Test with null comment and properties + ModelRegisterRequest req1 = new ModelRegisterRequest("model", null, null); + String serJson1 = JsonUtils.objectMapper().writeValueAsString(req1); + ModelRegisterRequest deserReq1 = + JsonUtils.objectMapper().readValue(serJson1, ModelRegisterRequest.class); + Assertions.assertEquals(req1, deserReq1); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java new file mode 100644 index 00000000000..4c0df6d73e8 --- /dev/null +++ b/common/src/test/java/org/apache/gravitino/dto/requests/TestModelVersionLinkRequest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.json.JsonUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelVersionLinkRequest { + + @Test + public void testModelVersionLinkRequestSerDe() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + ModelVersionLinkRequest request = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, "comment", props); + + String serJson = JsonUtils.objectMapper().writeValueAsString(request); + ModelVersionLinkRequest deserRequest = + JsonUtils.objectMapper().readValue(serJson, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request, deserRequest); + + // Test with null aliases + ModelVersionLinkRequest request1 = new ModelVersionLinkRequest("uri", null, "comment", props); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(request1); + ModelVersionLinkRequest deserRequest1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request1, deserRequest1); + Assertions.assertNull(deserRequest1.getAliases()); + + // Test with empty aliases + ModelVersionLinkRequest request2 = + new ModelVersionLinkRequest("uri", new String[] {}, "comment", props); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(request2); + ModelVersionLinkRequest deserRequest2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request2, deserRequest2); + Assertions.assertEquals(0, deserRequest2.getAliases().length); + + // Test with null comment and properties + ModelVersionLinkRequest request3 = + new ModelVersionLinkRequest("uri", new String[] {"alias1", "alias2"}, null, null); + + String serJson3 = JsonUtils.objectMapper().writeValueAsString(request3); + ModelVersionLinkRequest deserRequest3 = + JsonUtils.objectMapper().readValue(serJson3, ModelVersionLinkRequest.class); + + Assertions.assertEquals(request3, deserRequest3); + Assertions.assertNull(deserRequest3.getComment()); + Assertions.assertNull(deserRequest3.getProperties()); + } +} diff --git a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java index 5f947820222..57813c0bc64 100644 --- a/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java +++ b/common/src/test/java/org/apache/gravitino/dto/responses/TestResponses.java @@ -26,8 +26,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.core.JsonProcessingException; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import java.time.Instant; +import java.util.Map; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.authorization.Privileges; @@ -41,6 +43,8 @@ import org.apache.gravitino.dto.authorization.RoleDTO; import org.apache.gravitino.dto.authorization.SecurableObjectDTO; import org.apache.gravitino.dto.authorization.UserDTO; +import org.apache.gravitino.dto.model.ModelDTO; +import org.apache.gravitino.dto.model.ModelVersionDTO; import org.apache.gravitino.dto.rel.ColumnDTO; import org.apache.gravitino.dto.rel.TableDTO; import org.apache.gravitino.dto.rel.partitioning.Partitioning; @@ -390,4 +394,76 @@ void testFileLocationResponseException() { FileLocationResponse response = new FileLocationResponse(); assertThrows(IllegalArgumentException.class, () -> response.validate()); } + + @Test + void testModelResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelDTO modelDTO = + ModelDTO.builder() + .withName("model1") + .withLatestVersion(0) + .withComment("comment1") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelResponse response = new ModelResponse(modelDTO); + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelResponse deserResponse = JsonUtils.objectMapper().readValue(serJson, ModelResponse.class); + + assertEquals(response, deserResponse); + + ModelResponse response1 = new ModelResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } + + @Test + void testModelVersionListResponse() throws JsonProcessingException { + ModelVersionListResponse response1 = new ModelVersionListResponse(new int[] {}); + assertDoesNotThrow(response1::validate); + + String serJson1 = JsonUtils.objectMapper().writeValueAsString(response1); + ModelVersionListResponse deserResponse1 = + JsonUtils.objectMapper().readValue(serJson1, ModelVersionListResponse.class); + assertEquals(response1, deserResponse1); + assertArrayEquals(new int[] {}, deserResponse1.getVersions()); + + ModelVersionListResponse response2 = new ModelVersionListResponse(new int[] {1, 2}); + assertDoesNotThrow(response2::validate); + + String serJson2 = JsonUtils.objectMapper().writeValueAsString(response2); + ModelVersionListResponse deserResponse2 = + JsonUtils.objectMapper().readValue(serJson2, ModelVersionListResponse.class); + assertEquals(response2, deserResponse2); + assertArrayEquals(new int[] {1, 2}, deserResponse2.getVersions()); + } + + @Test + void testModelVersionResponse() throws JsonProcessingException { + Map props = ImmutableMap.of("key", "value"); + AuditDTO audit = AuditDTO.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + ModelVersionDTO modelVersionDTO = + ModelVersionDTO.builder() + .withVersion(0) + .withComment("model version comment") + .withAliases(new String[] {"alias1", "alias2"}) + .withUri("uri") + .withProperties(props) + .withAudit(audit) + .build(); + + ModelVersionResponse response = new ModelVersionResponse(modelVersionDTO); + response.validate(); // No exception thrown + + String serJson = JsonUtils.objectMapper().writeValueAsString(response); + ModelVersionResponse deserResponse = + JsonUtils.objectMapper().readValue(serJson, ModelVersionResponse.class); + assertEquals(response, deserResponse); + + ModelVersionResponse response1 = new ModelVersionResponse(); + assertThrows(IllegalArgumentException.class, response1::validate); + } } diff --git a/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java b/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java index 148cda4fa70..d42f445e10a 100644 --- a/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java +++ b/core/src/main/java/org/apache/gravitino/config/ConfigBuilder.java @@ -170,6 +170,31 @@ public ConfigEntry longConf() { return conf; } + /** + * Creates a configuration entry for Double data type. + * + * @return The created ConfigEntry instance for Double data type. + */ + public ConfigEntry doubleConf() { + ConfigEntry conf = + new ConfigEntry<>(key, version, doc, alternatives, isPublic, isDeprecated); + Function func = + s -> { + if (s == null || s.isEmpty()) { + return null; + } else { + return Double.parseDouble(s); + } + }; + conf.setValueConverter(func); + + Function stringFunc = + t -> Optional.ofNullable(t).map(String::valueOf).orElse(null); + conf.setStringConverter(stringFunc); + + return conf; + } + /** * Creates a configuration entry for Boolean data type. * diff --git a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java index b4c788a60d8..a32c2fff21d 100644 --- a/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java +++ b/core/src/main/java/org/apache/gravitino/connector/PropertyEntry.java @@ -29,6 +29,7 @@ @Getter public final class PropertyEntry { + private final String name; private final String description; private final boolean required; @@ -90,6 +91,7 @@ private PropertyEntry( } public static class Builder { + private String name; private String description; private boolean required; @@ -214,6 +216,28 @@ public static PropertyEntry longPropertyEntry( .build(); } + public static PropertyEntry doublePropertyEntry( + String name, + String description, + boolean required, + boolean immutable, + double defaultValue, + boolean hidden, + boolean reserved) { + return new Builder() + .withName(name) + .withDescription(description) + .withRequired(required) + .withImmutable(immutable) + .withJavaType(Double.class) + .withDefaultValue(defaultValue) + .withDecoder(Double::parseDouble) + .withEncoder(String::valueOf) + .withHidden(hidden) + .withReserved(reserved) + .build(); + } + public static PropertyEntry integerPropertyEntry( String name, String description, diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java index a39dbba01bd..6ac0c498c02 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java @@ -19,6 +19,7 @@ package org.apache.gravitino.credential; +import com.google.common.base.Objects; import com.google.common.base.Preconditions; import javax.validation.constraints.NotNull; @@ -35,4 +36,27 @@ public CatalogCredentialContext(String userName) { public String getUserName() { return userName; } + + @Override + public int hashCode() { + return Objects.hashCode(userName); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof CatalogCredentialContext)) { + return false; + } + return Objects.equal(userName, ((CatalogCredentialContext) o).userName); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder.append("User name: ").append(userName); + return stringBuilder.toString(); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java index 2fe6fedccd9..0e407a399b4 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialManager.java @@ -34,20 +34,21 @@ public class CatalogCredentialManager implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(CatalogCredentialManager.class); + private final CredentialCache credentialCache; + private final String catalogName; private final Map credentialProviders; public CatalogCredentialManager(String catalogName, Map catalogProperties) { this.catalogName = catalogName; this.credentialProviders = CredentialUtils.loadCredentialProviders(catalogProperties); + this.credentialCache = new CredentialCache(); + credentialCache.initialize(catalogProperties); } public Credential getCredential(String credentialType, CredentialContext context) { - // todo: add credential cache - Preconditions.checkState( - credentialProviders.containsKey(credentialType), - String.format("Credential %s not found", credentialType)); - return credentialProviders.get(credentialType).getCredential(context); + CredentialCacheKey credentialCacheKey = new CredentialCacheKey(credentialType, context); + return credentialCache.getCredential(credentialCacheKey, cacheKey -> doGetCredential(cacheKey)); } @Override @@ -67,4 +68,14 @@ public void close() { } }); } + + private Credential doGetCredential(CredentialCacheKey credentialCacheKey) { + String credentialType = credentialCacheKey.getCredentialType(); + CredentialContext context = credentialCacheKey.getCredentialContext(); + LOG.debug("Try get credential, credential type: {}, context: {}.", credentialType, context); + Preconditions.checkState( + credentialProviders.containsKey(credentialType), + String.format("Credential %s not found", credentialType)); + return credentialProviders.get(credentialType).getCredential(context); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java b/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java new file mode 100644 index 00000000000..afbb09d50ed --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialCache.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.Expiry; +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.apache.gravitino.credential.config.CredentialConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CredentialCache implements Closeable { + + private static final Logger LOG = LoggerFactory.getLogger(CredentialCache.class); + + // Calculates the credential expire time in the cache. + static class CredentialExpireTimeCalculator implements Expiry { + + private double credentialCacheExpireRatio; + + public CredentialExpireTimeCalculator(double credentialCacheExpireRatio) { + this.credentialCacheExpireRatio = credentialCacheExpireRatio; + } + + // Set expire time after add a credential in the cache. + @Override + public long expireAfterCreate(T key, Credential credential, long currentTime) { + long credentialExpireTime = credential.expireTimeInMs(); + long timeToExpire = credentialExpireTime - System.currentTimeMillis(); + if (timeToExpire <= 0) { + return 0; + } + + timeToExpire = (long) (timeToExpire * credentialCacheExpireRatio); + return TimeUnit.MILLISECONDS.toNanos(timeToExpire); + } + + // Not change expire time after update credential, this should not happen. + @Override + public long expireAfterUpdate(T key, Credential value, long currentTime, long currentDuration) { + return currentDuration; + } + + // Not change expire time after read credential. + @Override + public long expireAfterRead(T key, Credential value, long currentTime, long currentDuration) { + return currentDuration; + } + } + + private Cache credentialCache; + + public void initialize(Map catalogProperties) { + CredentialConfig credentialConfig = new CredentialConfig(catalogProperties); + long cacheSize = credentialConfig.get(CredentialConfig.CREDENTIAL_CACHE_MAX_SIZE); + double cacheExpireRatio = credentialConfig.get(CredentialConfig.CREDENTIAL_CACHE_EXPIRE_RATIO); + + this.credentialCache = + Caffeine.newBuilder() + .expireAfter(new CredentialExpireTimeCalculator(cacheExpireRatio)) + .maximumSize(cacheSize) + .removalListener( + (cacheKey, credential, c) -> + LOG.debug("Credential expire, cache key: {}.", cacheKey)) + .build(); + } + + public Credential getCredential(T cacheKey, Function credentialSupplier) { + return credentialCache.get(cacheKey, key -> credentialSupplier.apply(cacheKey)); + } + + @Override + public void close() throws IOException { + if (credentialCache != null) { + credentialCache.invalidateAll(); + credentialCache = null; + } + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java new file mode 100644 index 00000000000..1d0d8f7b3b6 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Objects; +import lombok.Getter; + +@Getter +public class CredentialCacheKey { + + private final String credentialType; + private final CredentialContext credentialContext; + + public CredentialCacheKey(String credentialType, CredentialContext credentialContext) { + this.credentialType = credentialType; + this.credentialContext = credentialContext; + } + + @Override + public int hashCode() { + return Objects.hash(credentialType, credentialContext); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof CredentialCacheKey)) { + return false; + } + CredentialCacheKey that = (CredentialCacheKey) o; + return Objects.equals(credentialType, that.credentialType) + && Objects.equals(credentialContext, that.credentialContext); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder + .append("credentialType: ") + .append(credentialType) + .append("credentialContext: ") + .append(credentialContext); + return stringBuilder.toString(); + } +} diff --git a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java index 03e7bbe0e31..06d17b134ba 100644 --- a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java @@ -20,6 +20,7 @@ package org.apache.gravitino.credential; import com.google.common.base.Preconditions; +import java.util.Objects; import java.util.Set; import javax.validation.constraints.NotNull; @@ -55,4 +56,36 @@ public Set getWritePaths() { public Set getReadPaths() { return readPaths; } + + @Override + public int hashCode() { + return Objects.hash(userName, writePaths, readPaths); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !(o instanceof PathBasedCredentialContext)) { + return false; + } + PathBasedCredentialContext that = (PathBasedCredentialContext) o; + return Objects.equals(userName, that.userName) + && Objects.equals(writePaths, that.writePaths) + && Objects.equals(readPaths, that.readPaths); + } + + @Override + public String toString() { + StringBuilder stringBuilder = new StringBuilder(); + stringBuilder + .append("User name: ") + .append(userName) + .append(", write path: ") + .append(writePaths) + .append(", read path: ") + .append(readPaths); + return stringBuilder.toString(); + } } diff --git a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java index d8823417cda..31a5183cc22 100644 --- a/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java +++ b/core/src/main/java/org/apache/gravitino/credential/config/CredentialConfig.java @@ -21,16 +21,23 @@ import com.google.common.collect.ImmutableMap; import java.util.Map; +import org.apache.gravitino.Config; +import org.apache.gravitino.config.ConfigBuilder; +import org.apache.gravitino.config.ConfigConstants; +import org.apache.gravitino.config.ConfigEntry; import org.apache.gravitino.connector.PropertyEntry; import org.apache.gravitino.credential.CredentialConstants; -public class CredentialConfig { +public class CredentialConfig extends Config { + + private static final long DEFAULT_CREDENTIAL_CACHE_MAX_SIZE = 10_000L; + private static final double DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO = 0.15d; public static final Map> CREDENTIAL_PROPERTY_ENTRIES = new ImmutableMap.Builder>() .put( CredentialConstants.CREDENTIAL_PROVIDERS, - PropertyEntry.booleanPropertyEntry( + PropertyEntry.stringPropertyEntry( CredentialConstants.CREDENTIAL_PROVIDERS, "Credential providers for the Gravitino catalog, schema, fileset, table, etc.", false /* required */, @@ -38,5 +45,50 @@ public class CredentialConfig { null /* default value */, false /* hidden */, false /* reserved */)) + .put( + CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO, + PropertyEntry.doublePropertyEntry( + CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO, + "Ratio of the credential's expiration time when Gravitino remove credential from the cache.", + false /* required */, + false /* immutable */, + DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO /* default value */, + false /* hidden */, + false /* reserved */)) + .put( + CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE, + PropertyEntry.longPropertyEntry( + CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE, + "Max size for the credential cache.", + false /* required */, + false /* immutable */, + DEFAULT_CREDENTIAL_CACHE_MAX_SIZE /* default value */, + false /* hidden */, + false /* reserved */)) .build(); + + public static final ConfigEntry CREDENTIAL_CACHE_EXPIRE_RATIO = + new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_EXPIRE_RATIO) + .doc( + "Ratio of the credential's expiration time when Gravitino remove credential from the " + + "cache.") + .version(ConfigConstants.VERSION_0_8_0) + .doubleConf() + .checkValue( + ratio -> ratio >= 0 && ratio < 1, + "Ratio of the credential's expiration time should greater than or equal to 0 " + + "and less than 1.") + .createWithDefault(DEFAULT_CREDENTIAL_CACHE_EXPIRE_RATIO); + + public static final ConfigEntry CREDENTIAL_CACHE_MAX_SIZE = + new ConfigBuilder(CredentialConstants.CREDENTIAL_CACHE_MAX_SIZE) + .doc("Max cache size for the credential.") + .version(ConfigConstants.VERSION_0_8_0) + .longConf() + .createWithDefault(DEFAULT_CREDENTIAL_CACHE_MAX_SIZE); + + public CredentialConfig(Map properties) { + super(false); + loadFromMap(properties, k -> true); + } } diff --git a/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java b/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java new file mode 100644 index 00000000000..b29c660a97d --- /dev/null +++ b/core/src/test/java/org/apache/gravitino/credential/TestCredentialCacheKey.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.credential; + +import java.util.Set; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableSet; + +public class TestCredentialCacheKey { + + @Test + void testCredentialCacheKey() { + + PathBasedCredentialContext context = + new PathBasedCredentialContext("user1", ImmutableSet.of("path1"), ImmutableSet.of("path2")); + PathBasedCredentialContext contextWithDiffUser = + new PathBasedCredentialContext("user2", ImmutableSet.of("path1"), ImmutableSet.of("path2")); + PathBasedCredentialContext contextWithDiffPath = + new PathBasedCredentialContext("user1", ImmutableSet.of("path3"), ImmutableSet.of("path4")); + + CredentialCacheKey key1 = new CredentialCacheKey("s3-token", context); + + Set cache = ImmutableSet.of(key1); + Assertions.assertTrue(cache.contains(key1)); + + // different user + CredentialCacheKey key2 = new CredentialCacheKey("s3-token", contextWithDiffUser); + Assertions.assertFalse(cache.contains(key2)); + + // different path + CredentialCacheKey key3 = new CredentialCacheKey("s3-token", contextWithDiffPath); + Assertions.assertFalse(cache.contains(key3)); + + // different credential type + CredentialCacheKey key4 = new CredentialCacheKey("s3-token1", context); + Assertions.assertFalse(cache.contains(key4)); + } +} diff --git a/docs/hadoop-catalog.md b/docs/hadoop-catalog.md index ce58826cb93..9048556ffa5 100644 --- a/docs/hadoop-catalog.md +++ b/docs/hadoop-catalog.md @@ -10,10 +10,8 @@ license: "This software is licensed under the Apache License version 2." Hadoop catalog is a fileset catalog that using Hadoop Compatible File System (HCFS) to manage the storage location of the fileset. Currently, it supports local filesystem and HDFS. For -object storage like S3, GCS, and Azure Blob Storage, you can put the hadoop object store jar like -hadoop-aws into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory to enable the support. -Gravitino itself hasn't yet tested the object storage support, so if you have any issue, -please create an [issue](https://github.com/apache/gravitino/issues). +object storage like S3, GCS, Azure Blob Storage and OSS, you can put the hadoop object store jar like +`gravitino-aws-bundle-{gravitino-version}.jar` into the `$GRAVITINO_HOME/catalogs/hadoop/libs` directory to enable the support. Note that Gravitino uses Hadoop 3 dependencies to build Hadoop catalog. Theoretically, it should be compatible with both Hadoop 2.x and 3.x, since Gravitino doesn't leverage any new features in @@ -52,7 +50,7 @@ Apart from the above properties, to access fileset like HDFS, S3, GCS, OSS or cu | `s3-access-key-id` | The access key of the AWS S3. | (none) | Yes if it's a S3 fileset. | 0.7.0-incubating | | `s3-secret-access-key` | The secret key of the AWS S3. | (none) | Yes if it's a S3 fileset. | 0.7.0-incubating | -At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### GCS fileset @@ -62,7 +60,7 @@ At the same time, you need to place the corresponding bundle jar [`gravitino-aws | `default-filesystem-provider` | The name default filesystem providers of this Hadoop catalog if users do not specify the scheme in the URI. Default value is `builtin-local`, for GCS, if we set this value, we can omit the prefix 'gs://' in the location. | `builtin-local` | No | 0.7.0-incubating | | `gcs-service-account-file` | The path of GCS service account JSON file. | (none) | Yes if it's a GCS fileset. | 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### OSS fileset @@ -74,7 +72,7 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp- | `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset. | 0.7.0-incubating | | `oss-secret-access-key` | The secret key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset. | 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aliyun-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. #### Azure Blob Storage fileset @@ -86,7 +84,7 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-aliy | `azure-storage-account-name ` | The account name of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | | `azure-storage-account-key` | The account key of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | -Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. +Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure-bundle/) in the directory `${GRAVITINO_HOME}/catalogs/hadoop/libs`. :::note - Gravitino contains builtin file system providers for local file system(`builtin-local`) and HDFS(`builtin-hdfs`), that is to say if `filesystem-providers` is not set, Gravitino will still support local file system and HDFS. Apart from that, you can set the `filesystem-providers` to support other file systems like S3, GCS, OSS or custom file system. diff --git a/docs/how-to-use-gvfs.md b/docs/how-to-use-gvfs.md index 162d535be11..0dbfd867a3d 100644 --- a/docs/how-to-use-gvfs.md +++ b/docs/how-to-use-gvfs.md @@ -77,7 +77,9 @@ Apart from the above properties, to access fileset like S3, GCS, OSS and custom | `s3-access-key-id` | The access key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | | `s3-secret-access-key` | The secret key of the AWS S3. | (none) | Yes if it's a S3 fileset.| 0.7.0-incubating | -At the same time, you need to place the corresponding bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +At the same time, you need to add the corresponding bundle jar +1. [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-aws-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws/) and hadoop-aws jar and other necessary dependencies in the classpath. #### GCS fileset @@ -86,7 +88,9 @@ At the same time, you need to place the corresponding bundle jar [`gravitino-aws |--------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|---------------------------|------------------| | `gcs-service-account-file` | The path of GCS service account JSON file. | (none) | Yes if it's a GCS fileset.| 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gcp-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +In the meantime, you need to add the corresponding bundle jar +1. [`gravitino-gcp-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp-bundle/) in the classpath if no hadoop environment is available, or +2. or [`gravitino-gcp-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-gcp/) and [gcs-connector jar](https://github.com/GoogleCloudDataproc/hadoop-connectors/releases) and other necessary dependencies in the classpath. #### OSS fileset @@ -97,7 +101,10 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-gcp- | `oss-access-key-id` | The access key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | | `oss-secret-access-key` | The secret key of the Aliyun OSS. | (none) | Yes if it's a OSS fileset.| 0.7.0-incubating | -In the meantime, you need to place the corresponding bundle jar [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aliyun-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). + +In the meantime, you need to place the corresponding bundle jar +1. [`gravitino-aliyun-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-aliyun-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aliyun/) and hadoop-aliyun jar and other necessary dependencies in the classpath. #### Azure Blob Storage fileset @@ -106,7 +113,9 @@ In the meantime, you need to place the corresponding bundle jar [`gravitino-aliy | `azure-storage-account-name` | The account name of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | | `azure-storage-account-key` | The account key of Azure Blob Storage. | (none) | Yes if it's a Azure Blob Storage fileset. | 0.8.0-incubating | -Similar to the above, you need to place the corresponding bundle jar [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/azure-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`). +Similar to the above, you need to place the corresponding bundle jar +1. [`gravitino-azure-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure-bundle/) in the classpath if no hadoop environment is available, or +2. [`gravitino-azure-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-azure/) and hadoop-azure jar and other necessary dependencies in the classpath. #### Custom fileset Since 0.7.0-incubating, users can define their own fileset type and configure the corresponding properties, for more, please refer to [Custom Fileset](./hadoop-catalog.md#how-to-custom-your-own-hcfs-file-system-fileset). @@ -137,8 +146,13 @@ You can configure these properties in two ways: ``` :::note -If you want to access the S3, GCS, OSS or custom fileset through GVFS, apart from the above properties, you need to place the corresponding bundle jar in the Hadoop environment. -For example if you want to access the S3 fileset, you need to place the S3 bundle jar [`gravitino-aws-bundle-${version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/aws-bundle/) in the Hadoop environment(typically located in `${HADOOP_HOME}/share/hadoop/common/lib/`) or add it to the classpath. +If you want to access the S3, GCS, OSS or custom fileset through GVFS, apart from the above properties, you need to place the corresponding bundle jars in the Hadoop environment. +For example, if you want to access the S3 fileset, you need to place +1. The aws hadoop bundle jar [`gravitino-aws-bundle-${gravitino-version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws-bundle/) +2. or [`gravitino-aws-${gravitino-version}.jar`](https://repo1.maven.org/maven2/org/apache/gravitino/gravitino-aws/), and hadoop-aws jar and other necessary dependencies + +to the classpath, it typically locates in `${HADOOP_HOME}/share/hadoop/common/lib/`). + ::: 2. Configure the properties in the `core-site.xml` file of the Hadoop environment: @@ -212,6 +226,12 @@ cp gravitino-filesystem-hadoop3-runtime-{version}.jar ${HADOOP_HOME}/share/hadoo # You need to ensure that the Kerberos has permission on the HDFS directory. kinit -kt your_kerberos.keytab your_kerberos@xxx.com + +# 4. Copy other dependencies to the Hadoop environment if you want to access the S3 fileset via GVFS +cp bundles/aws-bundle/build/libs/gravitino-aws-bundle-{version}.jar ${HADOOP_HOME}/share/hadoop/common/lib/ +cp clients/filesystem-hadoop3-runtime/build/libs/gravitino-filesystem-hadoop3-runtime-{version}-SNAPSHOT.jar ${HADOOP_HOME}/share/hadoop/common/lib/ +cp ${HADOOP_HOME}/share/hadoop/tools/lib/* ${HADOOP_HOME}/share/hadoop/common/lib/ + # 4. Try to list the fileset ./${HADOOP_HOME}/bin/hadoop dfs -ls gvfs://fileset/test_catalog/test_schema/test_fileset_1 ``` @@ -222,6 +242,36 @@ You can also perform operations on the files or directories managed by fileset t Make sure that your code is using the correct Hadoop environment, and that your environment has the `gravitino-filesystem-hadoop3-runtime-{version}.jar` dependency. +```xml + + + org.apache.gravitino + filesystem-hadoop3-runtime + {gravitino-version} + + + + + org.apache.gravitino + gravitino-aws-bundle + {gravitino-version} + + + + + org.apache.gravitino + gravitino-aws + {gravitino-version} + + + + org.apache.hadoop + hadoop-aws + {hadoop-version} + + +``` + For example: ```java @@ -462,8 +512,7 @@ from gravitino import gvfs options = { "cache_size": 20, "cache_expired_time": 3600, - "auth_type": "simple" - + "auth_type": "simple", # Optional, the following properties are required if you want to access the S3 fileset via GVFS python client, for GCS and OSS fileset, you should set the corresponding properties. "s3_endpoint": "http://localhost:9000", "s3_access_key_id": "minio", diff --git a/docs/open-api/models.yaml b/docs/open-api/models.yaml new file mode 100644 index 00000000000..713a7037cd6 --- /dev/null +++ b/docs/open-api/models.yaml @@ -0,0 +1,561 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- + +paths: + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + + get: + tags: + - model + summary: List models + operationId: listModels + responses: + "200": + $ref: "./openapi.yaml#/components/responses/EntityListResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: Register model + operationId: registerModel + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelRegisterRequest" + examples: + ModelRegisterRequest: + $ref: "#/components/examples/ModelRegisterRequest" + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "409": + description: Conflict - The target model already exists + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelAlreadyExistsErrorResponse: + $ref: "#/components/examples/ModelAlreadyExistsException" + "404": + description: Not Found - The schema does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: Get model + operationId: getModel + description: Returns the specified model object + responses: + "200": + $ref: "#/components/responses/ModelResponse" + "404": + description: Not Found - The target fileset does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchMetalakeException: + $ref: "./metalakes.yaml#/components/examples/NoSuchMetalakeException" + NoSuchCatalogException: + $ref: "./catalogs.yaml#/components/examples/NoSuchCatalogException" + NoSuchSchemaException: + $ref: "./schemas.yaml#/components/examples/NoSuchSchemaException" + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model + operationId: deleteModel + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + post: + tags: + - model + summary: link model version + operationId: linkModelVersion + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ModelVersionLinkRequest" + examples: + ModelVersionLinkRequest: + $ref: "#/components/examples/ModelVersionLinkRequest" + responses: + "200": + $ref: "./openapi.yaml#/components/responses/BaseResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "409": + description: Conflict - The model version aliases already exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + ModelVersionAliasesAlreadyExistException: + $ref: "#/components/examples/ModelVersionAliasesAlreadyExistException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + + get: + tags: + - model + summary: List model versions + operationId: listModelVersions + responses: + "200": + $ref: "#/components/responses/ModelVersionListResponse" + "404": + description: Not Found - The target model does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelException: + $ref: "#/components/examples/NoSuchModelException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/version" + + get: + tags: + - model + summary: Get model version + operationId: getModelVersion + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version + operationId: deleteModelVersion + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + parameters: + - $ref: "./openapi.yaml#/components/parameters/metalake" + - $ref: "./openapi.yaml#/components/parameters/catalog" + - $ref: "./openapi.yaml#/components/parameters/schema" + - $ref: "./openapi.yaml#/components/parameters/model" + - $ref: "#/components/parameters/alias" + + get: + tags: + - model + summary: Get model version by alias + operationId: getModelVersionByAlias + responses: + "200": + $ref: "#/components/responses/ModelVersionResponse" + "404": + description: Not Found - The target model version does not exist + content: + application/vnd.gravitino.v1+json: + schema: + $ref: "./openapi.yaml#/components/schemas/ErrorModel" + examples: + NoSuchModelVersionException: + $ref: "#/components/examples/NoSuchModelVersionException" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + + delete: + tags: + - model + summary: delete model version by alias + operationId: deleteModelVersionByAlias + responses: + "200": + $ref: "./openapi.yaml#/components/responses/DropResponse" + "400": + $ref: "./openapi.yaml#/components/responses/BadRequestErrorResponse" + "5xx": + $ref: "./openapi.yaml#/components/responses/ServerErrorResponse" + +components: + parameters: + version: + name: version + in: path + required: true + description: The version of the model + schema: + type: integer + alias: + name: alias + in: path + required: true + description: The alias of the model version + schema: + type: string + + schemas: + Model: + type: object + required: + - name + - audit + - latestVersion + properties: + name: + type: string + description: The name of the model + latestVersion: + type: integer + description: The latest version of the model + comment: + type: string + description: The comment of the fileset + nullable: true + properties: + type: object + description: The properties of the fileset + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelVersion: + type: object + required: + - uri + - version + - audit + properties: + uri: + type: string + description: The uri of the model version + version: + type: integer + description: The version of the model + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + audit: + $ref: "./openapi.yaml#/components/schemas/Audit" + + ModelRegisterRequest: + type: object + required: + - name + properties: + name: + type: string + description: The name of the model. Can not be empty. + comment: + type: string + description: The comment of the model. Can be empty. + nullable: true + properties: + type: object + description: The properties of the model. Can be empty. + nullable: true + default: {} + additionalProperties: + type: string + + ModelVersionLinkRequest: + type: object + required: + - uri + properties: + uri: + type: string + description: The uri of the model version + aliases: + type: array + description: The aliases of the model version + nullable: true + items: + type: string + comment: + type: string + description: The comment of the model version + nullable: true + properties: + type: object + description: The properties of the model version + nullable: true + default: {} + additionalProperties: + type: string + + responses: + ModelResponse: + description: The response of model object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + model: + $ref: "#/components/schemas/Model" + examples: + ModelResponse: + $ref: "#/components/examples/ModelResponse" + ModelVersionListResponse: + description: The response of model version list + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + versions: + type: array + description: The list of model versions + items: + format: int32 + examples: + ModelVersionListResponse: + $ref: "#/components/examples/ModelVersionListResponse" + ModelVersionResponse: + description: The response of model version object + content: + application/vnd.gravitino.v1+json: + schema: + type: object + properties: + code: + type: integer + format: int32 + description: Status code of the response + enum: + - 0 + modelVersion: + $ref: "#/components/schemas/ModelVersion" + examples: + ModelResponse: + $ref: "#/components/examples/ModelVersionResponse" + + examples: + ModelRegisterRequest: + value: { + "name": "model1", + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelVersionLinkRequest: + value: { + "uri": "hdfs://path/to/model", + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + } + } + + ModelResponse: + value: { + "code": 0, + "model" : { + "name": "model1", + "latestVersion": 0, + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelVersionListResponse: + value: { + "code": 0, + "versions": [0, 1, 2] + } + + ModelVersionResponse: + value: { + "code": 0, + "modelVersion" : { + "uri": "hdfs://path/to/model", + "version": 0, + "aliases": ["alias1", "alias2"], + "comment": "This is a comment", + "properties": { + "key1": "value1", + "key2": "value2" + }, + "audit": { + "creator": "user1", + "createTime": "2021-01-01T00:00:00Z", + "lastModifier": "user1", + "lastModifiedTime": "2021-01-01T00:00:00Z" + } + } + } + + ModelAlreadyExistsException: + value: { + "code": 1004, + "type": "ModelAlreadyExistsException", + "message": "Model already exists", + "stack": [ + "org.apache.gravitino.exceptions.ModelAlreadyExistsException: Model already exists" + ] + } + + NoSuchModelException: + value: { + "code": 1003, + "type": "NoSuchModelException", + "message": "Model does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelException: Model does not exist" + ] + } + + ModelVersionAliasesAlreadyExistException: + value: { + "code": 1004, + "type": "ModelVersionAliasesAlreadyExistException", + "message": "Model version aliases already exist", + "stack": [ + "org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException: Model version aliases already exist" + ] + } + + NoSuchModelVersionException: + value: { + "code": 1003, + "type": "NoSuchModelVersionException", + "message": "Model version does not exist", + "stack": [ + "org.apache.gravitino.exceptions.NoSuchModelVersionException: Model version does not exist" + ] + } diff --git a/docs/open-api/openapi.yaml b/docs/open-api/openapi.yaml index dd0564a7f9c..d0c941ab471 100644 --- a/docs/open-api/openapi.yaml +++ b/docs/open-api/openapi.yaml @@ -113,6 +113,20 @@ paths: /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/topics/{topic}: $ref: "./topics.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1topics~1%7Btopic%7D" + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/versions/{version}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1versions~1%7Bversion%7D" + + /metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models/{model}/aliases/{alias}: + $ref: "./models.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1catalogs~1%7Bcatalog%7D~1schemas~1%7Bschema%7D~1models~1%7Bmodel%7D~1aliases~1%7Balias%7D" /metalakes/{metalake}/users: $ref: "./users.yaml#/paths/~1metalakes~1%7Bmetalake%7D~1users" @@ -430,6 +444,14 @@ components: schema: type: string + model: + name: model + in: path + description: The name of the model + required: true + schema: + type: string + tag: name: tag in: path @@ -476,6 +498,7 @@ components: - "COLUMN" - "FILESET" - "TOPIC" + - "MODEL" - "ROLE" metadataObjectFullName: name: metadataObjectFullName diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a33c300ee88..52bccd9b480 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -36,12 +36,13 @@ airlift-json = "237" airlift-resolver = "1.6" hive2 = "2.3.9" hadoop2 = "2.10.2" -hadoop3 = "3.3.0" +hadoop3 = "3.3.1" hadoop3-gcs = "1.9.4-hadoop3" -hadoop3-abs = "3.3.0" -hadoop3-aliyun = "3.3.0" -hadoop-minikdc = "3.3.0" +hadoop3-abs = "3.3.1" +hadoop3-aliyun = "3.3.1" +hadoop-minikdc = "3.3.1" htrace-core4 = "4.1.0-incubating" +httpclient = "4.4.1" httpclient5 = "5.2.1" mockserver = "5.15.0" commons-csv = "1.12.0" @@ -177,6 +178,8 @@ hadoop3-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" } hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} +hadoop3-client-api = { group = "org.apache.hadoop", name = "hadoop-client-api", version.ref = "hadoop3"} +hadoop3-client-runtime = { group = "org.apache.hadoop", name = "hadoop-client-runtime", version.ref = "hadoop3"} hadoop3-minicluster = { group = "org.apache.hadoop", name = "hadoop-minicluster", version.ref = "hadoop-minikdc"} hadoop3-gcs = { group = "com.google.cloud.bigdataoss", name = "gcs-connector", version.ref = "hadoop3-gcs"} hadoop3-oss = { group = "org.apache.hadoop", name = "hadoop-aliyun", version.ref = "hadoop3-aliyun"} @@ -184,6 +187,7 @@ hadoop3-abs = { group = "org.apache.hadoop", name = "hadoop-azure", version.ref htrace-core4 = { group = "org.apache.htrace", name = "htrace-core4", version.ref = "htrace-core4" } airlift-json = { group = "io.airlift", name = "json", version.ref = "airlift-json"} airlift-resolver = { group = "io.airlift.resolver", name = "resolver", version.ref = "airlift-resolver"} +httpclient = { group = "org.apache.httpcomponents", name = "httpclient", version.ref = "httpclient" } httpclient5 = { group = "org.apache.httpcomponents.client5", name = "httpclient5", version.ref = "httpclient5" } mockserver-netty = { group = "org.mock-server", name = "mockserver-netty", version.ref = "mockserver" } mockserver-client-java = { group = "org.mock-server", name = "mockserver-client-java", version.ref = "mockserver" } diff --git a/iceberg/iceberg-rest-server/build.gradle.kts b/iceberg/iceberg-rest-server/build.gradle.kts index 03fe32c92a9..fe35c4e7789 100644 --- a/iceberg/iceberg-rest-server/build.gradle.kts +++ b/iceberg/iceberg-rest-server/build.gradle.kts @@ -62,10 +62,10 @@ dependencies { annotationProcessor(libs.lombok) compileOnly(libs.lombok) - testImplementation(project(":bundles:aliyun-bundle")) - testImplementation(project(":bundles:aws-bundle")) - testImplementation(project(":bundles:gcp-bundle", configuration = "shadow")) - testImplementation(project(":bundles:azure-bundle")) + testImplementation(project(":bundles:aliyun")) + testImplementation(project(":bundles:aws")) + testImplementation(project(":bundles:gcp", configuration = "shadow")) + testImplementation(project(":bundles:azure", configuration = "shadow")) testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation("org.scala-lang.modules:scala-collection-compat_$scalaVersion:$scalaCollectionCompatVersion") diff --git a/integration-test-common/build.gradle.kts b/integration-test-common/build.gradle.kts index 283169a76a9..bd15dc2a34f 100644 --- a/integration-test-common/build.gradle.kts +++ b/integration-test-common/build.gradle.kts @@ -53,11 +53,11 @@ dependencies { exclude("org.elasticsearch") exclude("org.elasticsearch.client") exclude("org.elasticsearch.plugin") + exclude("org.apache.hadoop", "hadoop-common") } - testImplementation(libs.hadoop3.common) { - exclude("com.sun.jersey") - exclude("javax.servlet", "servlet-api") - } + testImplementation(libs.hadoop3.client.api) + testImplementation(libs.hadoop3.client.runtime) + testImplementation(platform("org.junit:junit-bom:5.9.1")) testImplementation("org.junit.jupiter:junit-jupiter") } diff --git a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java index 63e53aefd59..2afc65482b3 100644 --- a/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java +++ b/server/src/main/java/org/apache/gravitino/server/GravitinoServer.java @@ -27,6 +27,7 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.catalog.CatalogDispatcher; import org.apache.gravitino.catalog.FilesetDispatcher; +import org.apache.gravitino.catalog.ModelDispatcher; import org.apache.gravitino.catalog.PartitionDispatcher; import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; @@ -118,6 +119,7 @@ protected void configure() { bind(gravitinoEnv.credentialOperationDispatcher()) .to(CredentialOperationDispatcher.class) .ranked(1); + bind(gravitinoEnv.modelDispatcher()).to(ModelDispatcher.class).ranked(1); } }); register(JsonProcessingExceptionMapper.class); diff --git a/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java b/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java index cdce0a0b125..345a555cd9e 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java +++ b/server/src/main/java/org/apache/gravitino/server/web/ConfigServlet.java @@ -42,22 +42,20 @@ public class ConfigServlet extends HttpServlet { ImmutableSet.of(OAuthConfig.DEFAULT_SERVER_URI, OAuthConfig.DEFAULT_TOKEN_PATH); private static final ImmutableSet> basicConfigEntries = - ImmutableSet.of(Configs.AUTHENTICATORS); + ImmutableSet.of(Configs.AUTHENTICATORS, Configs.ENABLE_AUTHORIZATION); - private final Map configs = Maps.newHashMap(); + private final Map configs = Maps.newHashMap(); public ConfigServlet(ServerConfig serverConfig) { for (ConfigEntry key : basicConfigEntries) { - String config = String.valueOf(serverConfig.get(key)); - configs.put(key.getKey(), config); + configs.put(key.getKey(), serverConfig.get(key)); } if (serverConfig .get(Configs.AUTHENTICATORS) .contains(AuthenticatorType.OAUTH.name().toLowerCase())) { for (ConfigEntry key : oauthConfigEntries) { - String config = String.valueOf(serverConfig.get(key)); - configs.put(key.getKey(), config); + configs.put(key.getKey(), serverConfig.get(key)); } } } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index faf94f50648..b71219b0453 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -32,6 +32,8 @@ import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.MetalakeNotInUseException; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.ModelVersionAliasesAlreadyExistException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NonEmptyCatalogException; import org.apache.gravitino.exceptions.NonEmptyMetalakeException; @@ -126,6 +128,11 @@ public static Response handleCredentialException( return CredentialExceptionHandler.INSTANCE.handle(op, metadataObjectName, "", e); } + public static Response handleModelException( + OperationType op, String model, String schema, Exception e) { + return ModelExceptionHandler.INSTANCE.handle(op, model, schema, e); + } + public static Response handleTestConnectionException(Exception e) { ErrorResponse response; if (e instanceof IllegalArgumentException) { @@ -729,6 +736,44 @@ public Response handle(OperationType op, String name, String parent, Exception e } } + private static class ModelExceptionHandler extends BaseExceptionHandler { + private static final ExceptionHandler INSTANCE = new ModelExceptionHandler(); + + private static String getModelErrorMsg( + String model, String operation, String schema, String reason) { + return String.format( + "Failed to operate model(s)%s operation [%s] under schema [%s], reason [%s]", + model, operation, schema, reason); + } + + @Override + public Response handle(OperationType op, String model, String schema, Exception e) { + String formatted = StringUtil.isBlank(model) ? "" : " [" + model + "]"; + String errorMsg = getModelErrorMsg(formatted, op.name(), schema, getErrorMsg(e)); + LOG.warn(errorMsg, e); + + if (e instanceof IllegalArgumentException) { + return Utils.illegalArguments(errorMsg, e); + + } else if (e instanceof NotFoundException) { + return Utils.notFound(errorMsg, e); + + } else if (e instanceof ModelAlreadyExistsException + || e instanceof ModelVersionAliasesAlreadyExistException) { + return Utils.alreadyExists(errorMsg, e); + + } else if (e instanceof ForbiddenException) { + return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else { + return super.handle(op, model, schema, e); + } + } + } + @VisibleForTesting static class BaseExceptionHandler extends ExceptionHandler { diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java new file mode 100644 index 00000000000..fd507821086 --- /dev/null +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ModelOperations.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import com.codahale.metrics.annotation.ResponseMetered; +import com.codahale.metrics.annotation.Timed; +import javax.inject.Inject; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.dto.util.DTOConverters; +import org.apache.gravitino.metrics.MetricNames; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.server.web.Utils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Path("metalakes/{metalake}/catalogs/{catalog}/schemas/{schema}/models") +public class ModelOperations { + + private static final Logger LOG = LoggerFactory.getLogger(ModelOperations.class); + + private final ModelDispatcher modelDispatcher; + + @Context private HttpServletRequest httpRequest; + + @Inject + public ModelOperations(ModelDispatcher modelDispatcher) { + this.modelDispatcher = modelDispatcher; + } + + @GET + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model", absolute = true) + public Response listModels( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema) { + LOG.info("Received list models request for schema: {}.{}.{}", metalake, catalog, schema); + Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier[] modelIds = modelDispatcher.listModels(modelNs); + modelIds = modelIds == null ? new NameIdentifier[0] : modelIds; + LOG.info("List {} models under schema {}", modelIds.length, modelNs); + return Utils.ok(new EntityListResponse(modelIds)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST, "", schema, e); + } + } + + @GET + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model", absolute = true) + public Response getModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received get model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + Model m = modelDispatcher.getModel(modelId); + LOG.info("Model got: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.GET, model, schema, e); + } + } + + @POST + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "register-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "register-model", absolute = true) + public Response registerModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + ModelRegisterRequest request) { + LOG.info( + "Received register model request: {}.{}.{}.{}", + metalake, + catalog, + schema, + request.getName()); + + try { + request.validate(); + NameIdentifier modelId = + NameIdentifierUtil.ofModel(metalake, catalog, schema, request.getName()); + + return Utils.doAs( + httpRequest, + () -> { + Model m = + modelDispatcher.registerModel( + modelId, request.getComment(), request.getProperties()); + LOG.info("Model registered: {}", modelId); + return Utils.ok(new ModelResponse(DTOConverters.toDTO(m))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.REGISTER, request.getName(), schema, e); + } + } + + @DELETE + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model", absolute = true) + public Response deleteModel( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received delete model request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModel(modelId); + if (!deleted) { + LOG.warn("Cannot find to be deleted model {} under schema {}", model, schema); + } else { + LOG.info("Model deleted: {}", modelId); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.DELETE, model, schema, e); + } + } + + @GET + @Path("{model}/versions") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "list-model-versions." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "list-model-versions", absolute = true) + public Response listModelVersions( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model) { + LOG.info("Received list model versions request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + int[] versions = modelDispatcher.listModelVersions(modelId); + versions = versions == null ? new int[0] : versions; + LOG.info("List {} versions of model {}", versions.length, modelId); + return Utils.ok(new ModelVersionListResponse(versions)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LIST_VERSIONS, model, schema, e); + } + } + + @GET + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-version", absolute = true) + public Response getModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received get model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, version); + LOG.info("Model version got: {}.{}", modelId, version); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, versionString(model, version), schema, e); + } + } + + @GET + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "get-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "get-model-alias", absolute = true) + public Response getModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received get model version alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + ModelVersion mv = modelDispatcher.getModelVersion(modelId, alias); + LOG.info("Model version alias got: {}.{}", modelId, alias); + return Utils.ok(new ModelVersionResponse(DTOConverters.toDTO(mv))); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.GET, aliasString(model, alias), schema, e); + } + } + + @POST + @Path("{model}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "link-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "link-model-version", absolute = true) + public Response linkModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + ModelVersionLinkRequest request) { + LOG.info("Received link model version request: {}.{}.{}.{}", metalake, catalog, schema, model); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + request.validate(); + + return Utils.doAs( + httpRequest, + () -> { + modelDispatcher.linkModelVersion( + modelId, + request.getUri(), + request.getAliases(), + request.getComment(), + request.getProperties()); + LOG.info("Model version linked: {}", modelId); + return Utils.ok(new BaseResponse()); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException(OperationType.LINK, model, schema, e); + } + } + + @DELETE + @Path("{model}/versions/{version}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-version." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-version", absolute = true) + public Response deleteModelVersion( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("version") int version) { + LOG.info( + "Received delete model version request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + version); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, version); + if (!deleted) { + LOG.warn("Cannot find to be deleted version {} in model {}", version, model); + } else { + LOG.info("Model version deleted: {}.{}", modelId, version); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, versionString(model, version), schema, e); + } + } + + @DELETE + @Path("{model}/aliases/{alias}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "delete-model-alias." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "delete-model-alias", absolute = true) + public Response deleteModelVersionByAlias( + @PathParam("metalake") String metalake, + @PathParam("catalog") String catalog, + @PathParam("schema") String schema, + @PathParam("model") String model, + @PathParam("alias") String alias) { + LOG.info( + "Received delete model version by alias request: {}.{}.{}.{}.{}", + metalake, + catalog, + schema, + model, + alias); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, model); + + try { + return Utils.doAs( + httpRequest, + () -> { + boolean deleted = modelDispatcher.deleteModelVersion(modelId, alias); + if (!deleted) { + LOG.warn( + "Cannot find to be deleted model version by alias {} in model {}", alias, model); + } else { + LOG.info("Model version by alias deleted: {}.{}", modelId, alias); + } + + return Utils.ok(new DropResponse(deleted)); + }); + + } catch (Exception e) { + return ExceptionHandlers.handleModelException( + OperationType.DELETE, aliasString(model, alias), schema, e); + } + } + + private String versionString(String model, int version) { + return model + " version(" + version + ")"; + } + + private String aliasString(String model, String alias) { + return model + " alias(" + alias + ")"; + } +} diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java index 8d4bc322ae7..2b8abd91f1d 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/OperationType.java @@ -35,4 +35,7 @@ public enum OperationType { REVOKE, ASSOCIATE, SET, + REGISTER, // An operation to register a model + LIST_VERSIONS, // An operation to list versions of a model + LINK // An operation to link a version to a model } diff --git a/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java b/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java new file mode 100644 index 00000000000..c76587d0397 --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/TestConfigServlet.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.PrintWriter; +import javax.servlet.http.HttpServletResponse; +import org.apache.gravitino.server.ServerConfig; +import org.junit.jupiter.api.Test; + +public class TestConfigServlet { + + @Test + public void testConfigServlet() throws Exception { + ServerConfig serverConfig = new ServerConfig(); + ConfigServlet configServlet = new ConfigServlet(serverConfig); + configServlet.init(); + HttpServletResponse res = mock(HttpServletResponse.class); + PrintWriter writer = mock(PrintWriter.class); + when(res.getWriter()).thenReturn(writer); + configServlet.doGet(null, res); + verify(writer) + .write( + "{\"gravitino.authorization.enable\":false,\"gravitino.authenticators\":[\"simple\"]}"); + configServlet.destroy(); + } +} diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java new file mode 100644 index 00000000000..42e48d0302f --- /dev/null +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestModelOperations.java @@ -0,0 +1,843 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.server.web.rest; + +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.time.Instant; +import java.util.Map; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.Application; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.catalog.ModelDispatcher; +import org.apache.gravitino.dto.requests.ModelRegisterRequest; +import org.apache.gravitino.dto.requests.ModelVersionLinkRequest; +import org.apache.gravitino.dto.responses.BaseResponse; +import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.EntityListResponse; +import org.apache.gravitino.dto.responses.ErrorConstants; +import org.apache.gravitino.dto.responses.ErrorResponse; +import org.apache.gravitino.dto.responses.ModelResponse; +import org.apache.gravitino.dto.responses.ModelVersionListResponse; +import org.apache.gravitino.dto.responses.ModelVersionResponse; +import org.apache.gravitino.exceptions.ModelAlreadyExistsException; +import org.apache.gravitino.exceptions.NoSuchModelException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.meta.AuditInfo; +import org.apache.gravitino.model.Model; +import org.apache.gravitino.model.ModelVersion; +import org.apache.gravitino.rest.RESTUtils; +import org.apache.gravitino.utils.NameIdentifierUtil; +import org.apache.gravitino.utils.NamespaceUtil; +import org.glassfish.jersey.internal.inject.AbstractBinder; +import org.glassfish.jersey.server.ResourceConfig; +import org.glassfish.jersey.test.JerseyTest; +import org.glassfish.jersey.test.TestProperties; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class TestModelOperations extends JerseyTest { + + private static class MockServletRequestFactory extends ServletRequestFactoryBase { + + @Override + public HttpServletRequest get() { + HttpServletRequest request = mock(HttpServletRequest.class); + when(request.getRemoteUser()).thenReturn(null); + return request; + } + } + + private ModelDispatcher modelDispatcher = mock(ModelDispatcher.class); + + private AuditInfo testAuditInfo = + AuditInfo.builder().withCreator("user1").withCreateTime(Instant.now()).build(); + + private Map properties = ImmutableMap.of("key1", "value"); + + private String metalake = "metalake_for_model_test"; + + private String catalog = "catalog_for_model_test"; + + private String schema = "schema_for_model_test"; + + private Namespace modelNs = NamespaceUtil.ofModel(metalake, catalog, schema); + + @Override + protected Application configure() { + try { + forceSet( + TestProperties.CONTAINER_PORT, String.valueOf(RESTUtils.findAvailablePort(2000, 3000))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + ResourceConfig resourceConfig = new ResourceConfig(); + resourceConfig.register(ModelOperations.class); + resourceConfig.register( + new AbstractBinder() { + @Override + protected void configure() { + bind(modelDispatcher).to(ModelDispatcher.class).ranked(2); + bindFactory(TestModelOperations.MockServletRequestFactory.class) + .to(HttpServletRequest.class); + } + }); + + return resourceConfig; + } + + @Test + public void testListModels() { + NameIdentifier modelId1 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + NameIdentifier modelId2 = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model2"); + NameIdentifier[] modelIds = new NameIdentifier[] {modelId1, modelId2}; + when(modelDispatcher.listModels(modelNs)).thenReturn(modelIds); + + Response response = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getMediaType()); + + EntityListResponse resp = response.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp.getCode()); + Assertions.assertArrayEquals(modelIds, resp.identifiers()); + + // Test mock return null for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(null); + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + EntityListResponse resp2 = resp1.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp2.getCode()); + Assertions.assertEquals(0, resp2.identifiers().length); + + // Test mock return empty array for listModels + when(modelDispatcher.listModels(modelNs)).thenReturn(new NameIdentifier[0]); + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + EntityListResponse resp4 = resp3.readEntity(EntityListResponse.class); + Assertions.assertEquals(0, resp4.getCode()); + Assertions.assertEquals(0, resp4.identifiers().length); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp5 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModels(modelNs); + Response resp6 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp6.getStatus()); + + ErrorResponse errorResp1 = resp6.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModel() { + Model mockModel = mockModel("model1", "comment1", 0); + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.getModel(modelId)).thenReturn(mockModel); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + + Model resultModel = modelResp.getModel(); + compare(mockModel, resultModel); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).getModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testRegisterModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + Model mockModel = mockModel("model1", "comment1", 0); + when(modelDispatcher.registerModel(modelId, "comment1", properties)).thenReturn(mockModel); + + ModelRegisterRequest req = new ModelRegisterRequest("model1", "comment1", properties); + Response resp = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelResponse modelResp = resp.readEntity(ModelResponse.class); + Assertions.assertEquals(0, modelResp.getCode()); + compare(mockModel, modelResp.getModel()); + + // Test mock throw NoSuchSchemaException + doThrow(new NoSuchSchemaException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp1 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchSchemaException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelAlreadyExistsException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp2 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .registerModel(modelId, "comment1", properties); + + Response resp3 = + target(modelPath()) + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModel() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModel(modelId)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModel + when(modelDispatcher.deleteModel(modelId)).thenReturn(false); + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).deleteModel(modelId); + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testListModelVersions() { + NameIdentifier modelId = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + int[] versions = new int[] {0, 1, 2}; + when(modelDispatcher.listModelVersions(modelId)).thenReturn(versions); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionListResponse versionListResp = resp.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp.getCode()); + Assertions.assertArrayEquals(versions, versionListResp.getVersions()); + + // Test mock return null for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(null); + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + ModelVersionListResponse versionListResp1 = resp1.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp1.getCode()); + Assertions.assertEquals(0, versionListResp1.getVersions().length); + + // Test mock return empty array for listModelVersions + when(modelDispatcher.listModelVersions(modelId)).thenReturn(new int[0]); + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + ModelVersionListResponse versionListResp2 = resp2.readEntity(ModelVersionListResponse.class); + Assertions.assertEquals(0, versionListResp2.getCode()); + Assertions.assertEquals(0, versionListResp2.getVersions().length); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .listModelVersions(modelId); + Response resp3 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")).when(modelDispatcher).listModelVersions(modelId); + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + } + + @Test + public void testGetModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + ModelVersion mockModelVersion = + mockModelVersion(0, "uri1", new String[] {"alias1"}, "comment1"); + when(modelDispatcher.getModelVersion(modelIdent, 0)).thenReturn(mockModelVersion); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + ModelVersionResponse versionResp = resp.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp.getCode()); + compare(mockModelVersion, versionResp.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, 0); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test get model version by alias + when(modelDispatcher.getModelVersion(modelIdent, "alias1")).thenReturn(mockModelVersion); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + ModelVersionResponse versionResp1 = resp3.readEntity(ModelVersionResponse.class); + Assertions.assertEquals(0, versionResp1.getCode()); + compare(mockModelVersion, versionResp1.getModelVersion()); + + // Test mock throw NoSuchModelVersionException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp2 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp2.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp2.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .getModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .get(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp3 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp3.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp3.getType()); + } + + @Test + public void testLinkModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + doNothing() + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + ModelVersionLinkRequest req = + new ModelVersionLinkRequest("uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + BaseResponse baseResponse = resp.readEntity(BaseResponse.class); + Assertions.assertEquals(0, baseResponse.getCode()); + + // Test mock throw NoSuchModelException + doThrow(new NoSuchModelException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp1 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.NOT_FOUND.getStatusCode(), resp1.getStatus()); + + ErrorResponse errorResp = resp1.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.NOT_FOUND_CODE, errorResp.getCode()); + Assertions.assertEquals(NoSuchModelException.class.getSimpleName(), errorResp.getType()); + + // Test mock throw ModelVersionAliasesAlreadyExistException + doThrow(new ModelAlreadyExistsException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp2 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals(Response.Status.CONFLICT.getStatusCode(), resp2.getStatus()); + + ErrorResponse errorResp1 = resp2.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.ALREADY_EXISTS_CODE, errorResp1.getCode()); + Assertions.assertEquals( + ModelAlreadyExistsException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .linkModelVersion(modelIdent, "uri1", new String[] {"alias1"}, "comment1", properties); + + Response resp3 = + target(modelPath()) + .path("model1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .post(Entity.entity(req, MediaType.APPLICATION_JSON_TYPE)); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp3.getStatus()); + + ErrorResponse errorResp2 = resp3.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + @Test + public void testDeleteModelVersion() { + NameIdentifier modelIdent = NameIdentifierUtil.ofModel(metalake, catalog, schema, "model1"); + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(true); + + Response resp = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getMediaType()); + + DropResponse dropResp = resp.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp.getCode()); + Assertions.assertTrue(dropResp.dropped()); + + // Test mock return false for deleteModelVersion + when(modelDispatcher.deleteModelVersion(modelIdent, 0)).thenReturn(false); + + Response resp1 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp1.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp1.getMediaType()); + + DropResponse dropResp1 = resp1.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp1.getCode()); + Assertions.assertFalse(dropResp1.dropped()); + + // Test mock return true for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(true); + + Response resp2 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp2.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp2.getMediaType()); + + DropResponse dropResp2 = resp2.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp2.getCode()); + + // Test mock return false for deleteModelVersion using alias + when(modelDispatcher.deleteModelVersion(modelIdent, "alias1")).thenReturn(false); + + Response resp3 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals(Response.Status.OK.getStatusCode(), resp3.getStatus()); + Assertions.assertEquals(MediaType.APPLICATION_JSON_TYPE, resp3.getMediaType()); + + DropResponse dropResp3 = resp3.readEntity(DropResponse.class); + Assertions.assertEquals(0, dropResp3.getCode()); + Assertions.assertFalse(dropResp3.dropped()); + + // Test mock throw RuntimeException + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, 0); + + Response resp4 = + target(modelPath()) + .path("model1") + .path("versions") + .path("0") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp4.getStatus()); + + ErrorResponse errorResp1 = resp4.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp1.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp1.getType()); + + // Test mock throw RuntimeException using alias + doThrow(new RuntimeException("mock error")) + .when(modelDispatcher) + .deleteModelVersion(modelIdent, "alias1"); + + Response resp5 = + target(modelPath()) + .path("model1") + .path("aliases") + .path("alias1") + .request(MediaType.APPLICATION_JSON_TYPE) + .accept("application/vnd.gravitino.v1+json") + .delete(); + + Assertions.assertEquals( + Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), resp5.getStatus()); + + ErrorResponse errorResp2 = resp5.readEntity(ErrorResponse.class); + Assertions.assertEquals(ErrorConstants.INTERNAL_ERROR_CODE, errorResp2.getCode()); + Assertions.assertEquals(RuntimeException.class.getSimpleName(), errorResp2.getType()); + } + + private String modelPath() { + return "/metalakes/" + metalake + "/catalogs/" + catalog + "/schemas/" + schema + "/models"; + } + + private Model mockModel(String modelName, String comment, int latestVersion) { + Model mockModel = mock(Model.class); + when(mockModel.name()).thenReturn(modelName); + when(mockModel.comment()).thenReturn(comment); + when(mockModel.latestVersion()).thenReturn(latestVersion); + when(mockModel.properties()).thenReturn(properties); + when(mockModel.auditInfo()).thenReturn(testAuditInfo); + return mockModel; + } + + private ModelVersion mockModelVersion(int version, String uri, String[] aliases, String comment) { + ModelVersion mockModelVersion = mock(ModelVersion.class); + when(mockModelVersion.version()).thenReturn(version); + when(mockModelVersion.uri()).thenReturn(uri); + when(mockModelVersion.aliases()).thenReturn(aliases); + when(mockModelVersion.comment()).thenReturn(comment); + when(mockModelVersion.properties()).thenReturn(properties); + when(mockModelVersion.auditInfo()).thenReturn(testAuditInfo); + return mockModelVersion; + } + + private void compare(Model left, Model right) { + Assertions.assertEquals(left.name(), right.name()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } + + private void compare(ModelVersion left, ModelVersion right) { + Assertions.assertEquals(left.version(), right.version()); + Assertions.assertEquals(left.uri(), right.uri()); + Assertions.assertArrayEquals(left.aliases(), right.aliases()); + Assertions.assertEquals(left.comment(), right.comment()); + Assertions.assertEquals(left.properties(), right.properties()); + + Assertions.assertNotNull(right.auditInfo()); + Assertions.assertEquals(left.auditInfo().creator(), right.auditInfo().creator()); + Assertions.assertEquals(left.auditInfo().createTime(), right.auditInfo().createTime()); + Assertions.assertEquals(left.auditInfo().lastModifier(), right.auditInfo().lastModifier()); + Assertions.assertEquals( + left.auditInfo().lastModifiedTime(), right.auditInfo().lastModifiedTime()); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index f38443db206..c865e14e7a2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -57,7 +57,7 @@ if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) } include("iceberg:iceberg-common") include("iceberg:iceberg-rest-server") -include("authorizations:authorization-ranger", "authorizations:authorization-jdbc", "authorizations:authorization-common", "authorizations:authorization-chain") +include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain") include("trino-connector:trino-connector", "trino-connector:integration-test") include("spark-connector:spark-common") // kyuubi hive connector doesn't support 2.13 for Spark3.3 @@ -77,8 +77,8 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") -include(":bundles:aws-bundle") -include(":bundles:gcp-bundle") -include(":bundles:aliyun-bundle") -include(":bundles:azure-bundle") -include("catalogs:hadoop-common") +include(":bundles:aws", ":bundles:aws-bundle") +include(":bundles:gcp", ":bundles:gcp-bundle") +include(":bundles:aliyun", ":bundles:aliyun-bundle") +include(":bundles:azure", ":bundles:azure-bundle") +include(":catalogs:hadoop-common") diff --git a/web/web/src/lib/store/auth/index.js b/web/web/src/lib/store/auth/index.js index 83d58394c90..ec0f1f52122 100644 --- a/web/web/src/lib/store/auth/index.js +++ b/web/web/src/lib/store/auth/index.js @@ -40,7 +40,7 @@ export const getAuthConfigs = createAsyncThunk('auth/getAuthConfigs', async () = oauthUrl = `${res['gravitino.authenticator.oauth.serverUri']}${res['gravitino.authenticator.oauth.tokenPath']}` // ** get the first authenticator from the response. response example: "[simple, oauth]" - authType = res['gravitino.authenticators'].slice(1, -1).split(',')[0].trim() + authType = res['gravitino.authenticators'][0].trim() localStorage.setItem('oauthUrl', oauthUrl)