diff --git a/.github/workflows/access-control-integration-test.yml b/.github/workflows/access-control-integration-test.yml index e9c55728ab8..ad347aa7e60 100644 --- a/.github/workflows/access-control-integration-test.yml +++ b/.github/workflows/access-control-integration-test.yml @@ -87,9 +87,9 @@ jobs: - name: Authorization Integration Test (JDK${{ matrix.java-version }}) id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" - ./gradlew -PskipTests -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.integration.test.**" + ./gradlew -PtestMode=embedded -PjdbcBackend=h2 -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" + ./gradlew -PtestMode=deploy -PjdbcBackend=mysql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" + ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:authorization-ranger:test --tests "org.apache.gravitino.authorization.ranger.**" - name: Upload integrate tests reports uses: actions/upload-artifact@v3 diff --git a/LICENSE.bin b/LICENSE.bin index ee65d4d6952..e922f936771 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -284,6 +284,7 @@ Apache Hadoop Apache Hadoop Aliyun connector Apache Hadoop GCS connector + Apache Hadoop AWS connector Apache Hadoop Annotatations Apache Hadoop Auth Apache Hadoop Client Aggregator diff --git a/README.md b/README.md index 94f1a1b9fe2..5feb69ed6ab 100644 --- a/README.md +++ b/README.md @@ -81,9 +81,13 @@ Please see [How to build Gravitino](docs/how-to-build.md) for details on buildin ## Quick start -### Configure and start the Apache Gravitino server +### Use Gravitino playground -If you already have a binary distribution package, go to the decompressed package directory. +This is the most recommended way: Gravitino provides a docker-compose based playground to quickly experience the whole system together with other components. Clone or download the [Gravitino playground repository](https://github.com/apache/gravitino-playground) and then follow the [README](https://github.com/apache/gravitino-playground/blob/main/README.md), you will have all then. + +### Configure and start Gravitino server in local + +If you want to start Gravitino in your machine, download a binary package from the [download page](https://gravitino.apache.org/downloads), and then decompressed the package. Before starting the Gravitino server, please configure the Gravitino server configuration file. The configuration file, `gravitino.conf`, is in the `conf` directory and follows the standard property file format. You can modify the configuration within this file. diff --git a/api/src/main/java/org/apache/gravitino/MetadataObjects.java b/api/src/main/java/org/apache/gravitino/MetadataObjects.java index 9fb54eada2b..a1303649707 100644 --- a/api/src/main/java/org/apache/gravitino/MetadataObjects.java +++ b/api/src/main/java/org/apache/gravitino/MetadataObjects.java @@ -153,7 +153,13 @@ public static MetadataObject parse(String fullName, MetadataObject.Type type) { return MetadataObjects.of(parts, type); } - private static String getParentFullName(List names) { + /** + * Get the parent full name of the given full name. + * + * @param names The names of the metadata object + * @return The parent full name if it exists, otherwise null + */ + public static String getParentFullName(List names) { if (names.size() <= 1) { return null; } diff --git a/api/src/main/java/org/apache/gravitino/Metalake.java b/api/src/main/java/org/apache/gravitino/Metalake.java index fb6fdbee094..4f95a8a2365 100644 --- a/api/src/main/java/org/apache/gravitino/Metalake.java +++ b/api/src/main/java/org/apache/gravitino/Metalake.java @@ -29,6 +29,9 @@ @Evolving public interface Metalake extends Auditable { + /** The property indicating the metalake is in use. */ + String PROPERTY_IN_USE = "in-use"; + /** * The name of the metalake. * diff --git a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java index 47cce7d3a4a..515b2d836cf 100644 --- a/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java +++ b/api/src/main/java/org/apache/gravitino/SupportsMetalakes.java @@ -21,7 +21,10 @@ import java.util.Map; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Client interface for supporting metalakes. It includes methods for listing, loading, creating, @@ -38,7 +41,7 @@ public interface SupportsMetalakes { Metalake[] listMetalakes(); /** - * Load a metalake by its identifier. + * Load a metalake by its name. * * @param name the name of the metalake. * @return The metalake. @@ -62,7 +65,7 @@ default boolean metalakeExists(String name) { } /** - * Create a metalake with specified identifier. + * Create a metalake with specified name, comment and properties. * * @param name The name of the metalake. * @param comment The comment of the metalake. @@ -74,7 +77,7 @@ Metalake createMetalake(String name, String comment, Map propert throws MetalakeAlreadyExistsException; /** - * Alter a metalake with specified identifier. + * Alter a metalake with specified metalake name and changes. * * @param name The name of the metalake. * @param changes The changes to apply. @@ -86,10 +89,69 @@ Metalake alterMetalake(String name, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified name. Please make sure: * - * @param name The identifier of the metalake. + *
    + *
  • There is no catalog in the metalake. Otherwise, a {@link NonEmptyEntityException} will be + * thrown. + *
  • The method {@link #disableMetalake(String)} has been called before dropping the metalake. + * Otherwise, a {@link MetalakeInUseException} will be thrown. + *
+ * + * It is equivalent to calling {@code dropMetalake(name, false)}. + * + * @param name The name of the metalake. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws MetalakeInUseException If the metalake is in use. + */ + default boolean dropMetalake(String name) throws NonEmptyEntityException, MetalakeInUseException { + return dropMetalake(name, false); + } + + /** + * Drop a metalake with specified name. If the force flag is true, it will: + * + *
    + *
  • Cascade drop all sub-entities (tags, catalogs, schemas, tables, etc.) of the metalake in + * Gravitino store. + *
  • Drop the metalake even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * deleted unless it is managed (such as managed fileset). + *
+ * + * If the force flag is false, it is equivalent to calling {@link #dropMetalake(String)}. + * + * @param name The name of the metalake. + * @param force Whether to force the drop. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, MetalakeInUseException; + + /** + * Enable a metalake. If the metalake is already in use, this method does nothing. + * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void enableMetalake(String name) throws NoSuchMetalakeException; + + /** + * Disable a metalake. If the metalake is already disabled, this method does nothing. Once a + * metalake is disable: + * + *
    + *
  • It can only be listed, loaded, dropped, or enable. + *
  • Any other operations on the metalake will throw an {@link MetalakeNotInUseException}. + *
  • Any operation on the sub-entities (catalogs, schemas, tables, etc.) will throw an {@link + * MetalakeNotInUseException}. + *
+ * + * @param name The name of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(String name); + void disableMetalake(String name) throws NoSuchMetalakeException; } diff --git a/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java new file mode 100644 index 00000000000..5184116d907 --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** Exception thrown when a metalake is in use and cannot be deleted. */ +public class MetalakeInUseException extends InUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public MetalakeInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java new file mode 100644 index 00000000000..4ad6cc33a6c --- /dev/null +++ b/api/src/main/java/org/apache/gravitino/exceptions/MetalakeNotInUseException.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.exceptions; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; + +/** An exception thrown when operating on a metalake that is not in use. */ +public class MetalakeNotInUseException extends NotInUseException { + /** + * Constructs a new exception with the specified detail message. + * + * @param message the detail message. + * @param args the arguments to the message. + */ + @FormatMethod + public MetalakeNotInUseException(@FormatString String message, Object... args) { + super(message, args); + } +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java index a9b08c8669f..838385c8a95 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHivePlugin.java @@ -18,17 +18,31 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.ranger.RangerPrivilege.RangerHivePrivilege; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.ranger.RangerPrivileges.RangerHivePrivilege; import org.apache.gravitino.authorization.ranger.reference.RangerDefines.PolicyResource; +import org.apache.gravitino.exceptions.AuthorizationPluginException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class RangerAuthorizationHivePlugin extends RangerAuthorizationPlugin { + private static final Logger LOG = LoggerFactory.getLogger(RangerAuthorizationHivePlugin.class); private static volatile RangerAuthorizationHivePlugin instance = null; private RangerAuthorizationHivePlugin(Map config) { @@ -46,11 +60,18 @@ public static synchronized RangerAuthorizationHivePlugin getInstance(Map> privilegesMappingRule() { return ImmutableMap.of( + Privilege.Name.CREATE_CATALOG, + ImmutableSet.of(RangerHivePrivilege.CREATE), + Privilege.Name.USE_CATALOG, + ImmutableSet.of(RangerHivePrivilege.SELECT), Privilege.Name.CREATE_SCHEMA, ImmutableSet.of(RangerHivePrivilege.CREATE), + Privilege.Name.USE_SCHEMA, + ImmutableSet.of(RangerHivePrivilege.SELECT), Privilege.Name.CREATE_TABLE, ImmutableSet.of(RangerHivePrivilege.CREATE), Privilege.Name.MODIFY_TABLE, @@ -60,11 +81,13 @@ public Map> privilegesMappingRule() { ImmutableSet.of(RangerHivePrivilege.READ, RangerHivePrivilege.SELECT)); } + @Override /** Set the default owner rule. */ public Set ownerMappingRule() { return ImmutableSet.of(RangerHivePrivilege.ALL); } + @Override /** Set Ranger policy resource rule. */ public List policyResourceDefinesRule() { return ImmutableList.of( @@ -72,4 +95,272 @@ public List policyResourceDefinesRule() { PolicyResource.TABLE.getName(), PolicyResource.COLUMN.getName()); } + + @Override + /** Allow privilege operation defines rule. */ + public Set allowPrivilegesRule() { + return ImmutableSet.of( + Privilege.Name.CREATE_CATALOG, + Privilege.Name.USE_CATALOG, + Privilege.Name.CREATE_SCHEMA, + Privilege.Name.USE_SCHEMA, + Privilege.Name.CREATE_TABLE, + Privilege.Name.MODIFY_TABLE, + Privilege.Name.SELECT_TABLE); + } + + /** Translate the Gravitino securable object to the Ranger owner securable object. */ + public List translateOwner(MetadataObject metadataObject) { + List rangerSecurableObjects = new ArrayList<>(); + + switch (metadataObject.type()) { + case METALAKE: + case CATALOG: + // Add `*` for the SCHEMA permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(RangerHelper.RESOURCE_ALL), + MetadataObject.Type.SCHEMA, + ownerMappingRule())); + // Add `*.*` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), + MetadataObject.Type.TABLE, + ownerMappingRule())); + // Add `*.*.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL), + MetadataObject.Type.COLUMN, + ownerMappingRule())); + break; + case SCHEMA: + // Add `{schema}` for the SCHEMA permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(metadataObject.name() /*Schema name*/), + MetadataObject.Type.SCHEMA, + ownerMappingRule())); + // Add `{schema}.*` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(metadataObject.name() /*Schema name*/, RangerHelper.RESOURCE_ALL), + MetadataObject.Type.TABLE, + ownerMappingRule())); + // Add `{schema}.*.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + metadataObject.name() /*Schema name*/, + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL), + MetadataObject.Type.COLUMN, + ownerMappingRule())); + break; + case TABLE: + // Add `{schema}.{table}` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + convertToRangerMetadataObject(metadataObject), + MetadataObject.Type.TABLE, + ownerMappingRule())); + // Add `{schema}.{table}.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + Stream.concat( + convertToRangerMetadataObject(metadataObject).stream(), + Stream.of(RangerHelper.RESOURCE_ALL)) + .collect(Collectors.toList()), + MetadataObject.Type.COLUMN, + ownerMappingRule())); + break; + default: + throw new AuthorizationPluginException( + "The owner privilege is not supported for the securable object: %s", + metadataObject.type()); + } + + return rangerSecurableObjects; + } + + /** Translate the Gravitino securable object to the Ranger securable object. */ + public List translatePrivilege(SecurableObject securableObject) { + List rangerSecurableObjects = new ArrayList<>(); + + securableObject.privileges().stream() + .filter(Objects::nonNull) + .forEach( + gravitinoPrivilege -> { + Set rangerPrivileges = new HashSet<>(); + privilegesMappingRule().get(gravitinoPrivilege.name()).stream() + .forEach( + rangerPrivilege -> + rangerPrivileges.add( + new RangerPrivileges.RangerHivePrivilegeImpl( + rangerPrivilege, gravitinoPrivilege.condition()))); + + switch (gravitinoPrivilege.name()) { + case CREATE_CATALOG: + // Ignore the Gravitino privilege `CREATE_CATALOG` in the + // RangerAuthorizationHivePlugin + break; + case USE_CATALOG: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(RangerHelper.RESOURCE_ALL), + MetadataObject.Type.SCHEMA, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + case CREATE_SCHEMA: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + // Add Ranger privilege(`CREATE`) to SCHEMA(`*`) + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(RangerHelper.RESOURCE_ALL), + MetadataObject.Type.SCHEMA, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + case USE_SCHEMA: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + // Add Ranger privilege(`SELECT`) to SCHEMA(`*`) + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(RangerHelper.RESOURCE_ALL), + MetadataObject.Type.SCHEMA, + rangerPrivileges)); + break; + case SCHEMA: + // Add Ranger privilege(`SELECT`) to SCHEMA(`{schema}`) + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of(securableObject.name() /*Schema name*/), + MetadataObject.Type.SCHEMA, + rangerPrivileges)); + break; + default: + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } + break; + case CREATE_TABLE: + case MODIFY_TABLE: + case SELECT_TABLE: + switch (securableObject.type()) { + case METALAKE: + case CATALOG: + // Add `*.*` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + RangerHelper.RESOURCE_ALL, RangerHelper.RESOURCE_ALL), + MetadataObject.Type.TABLE, + rangerPrivileges)); + // Add `*.*.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL), + MetadataObject.Type.COLUMN, + rangerPrivileges)); + break; + case SCHEMA: + // Add `{schema}.*` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + securableObject.name() /*Schema name*/, + RangerHelper.RESOURCE_ALL), + MetadataObject.Type.TABLE, + rangerPrivileges)); + // Add `{schema}.*.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + ImmutableList.of( + securableObject.name() /*Schema name*/, + RangerHelper.RESOURCE_ALL, + RangerHelper.RESOURCE_ALL), + MetadataObject.Type.COLUMN, + rangerPrivileges)); + break; + case TABLE: + if (gravitinoPrivilege.name() == Privilege.Name.CREATE_TABLE) { + throw new AuthorizationPluginException( + "The privilege %s is not supported for the securable object: %s", + gravitinoPrivilege.name(), securableObject.type()); + } else { + // Add `{schema}.{table}` for the TABLE permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + convertToRangerMetadataObject(securableObject), + MetadataObject.Type.TABLE, + rangerPrivileges)); + // Add `{schema}.{table}.*` for the COLUMN permission + rangerSecurableObjects.add( + RangerSecurableObjects.of( + Stream.concat( + convertToRangerMetadataObject(securableObject).stream(), + Stream.of(RangerHelper.RESOURCE_ALL)) + .collect(Collectors.toList()), + MetadataObject.Type.COLUMN, + rangerPrivileges)); + } + break; + default: + LOG.warn( + "RangerAuthorizationHivePlugin -> privilege {} is not supported for the securable object: {}", + gravitinoPrivilege.name(), + securableObject.type()); + } + break; + default: + LOG.warn( + "RangerAuthorizationHivePlugin -> privilege {} is not supported for the securable object: {}", + gravitinoPrivilege.name(), + securableObject.type()); + } + }); + + return rangerSecurableObjects; + } + + /** + * Because the Ranger securable object is different from the Gravitino securable object, we need + * to convert the Gravitino securable object to the Ranger securable object. + */ + List convertToRangerMetadataObject(MetadataObject metadataObject) { + Preconditions.checkArgument( + !(metadataObject instanceof RangerPrivileges), + "The metadata object must be not a RangerPrivileges object."); + List nsMetadataObject = + Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); + nsMetadataObject.remove(0); // remove the catalog name + return nsMetadataObject; + } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java index 021fca2a996..b188b55dd57 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java @@ -19,21 +19,24 @@ package org.apache.gravitino.authorization.ranger; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.time.Instant; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Group; import org.apache.gravitino.authorization.Owner; -import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; +import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.User; import org.apache.gravitino.authorization.ranger.reference.VXGroup; import org.apache.gravitino.authorization.ranger.reference.VXGroupList; @@ -41,6 +44,7 @@ import org.apache.gravitino.authorization.ranger.reference.VXUserList; import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.connector.authorization.AuthorizationPlugin; +import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; import org.apache.gravitino.meta.UserEntity; @@ -78,11 +82,11 @@ protected RangerAuthorizationPlugin(Map config) { // Apache Ranger Password should be minimum 8 characters with min one alphabet and one numeric. String password = config.get(AuthorizationPropertiesMeta.RANGER_PASSWORD); rangerServiceName = config.get(AuthorizationPropertiesMeta.RANGER_SERVICE_NAME); - RangerHelper.check(rangerUrl != null, "Ranger admin URL is required"); - RangerHelper.check(authType != null, "Ranger auth type is required"); - RangerHelper.check(rangerAdminName != null, "Ranger username is required"); - RangerHelper.check(password != null, "Ranger password is required"); - RangerHelper.check(rangerServiceName != null, "Ranger service name is required"); + Preconditions.checkArgument(rangerUrl != null, "Ranger admin URL is required"); + Preconditions.checkArgument(authType != null, "Ranger auth type is required"); + Preconditions.checkArgument(rangerAdminName != null, "Ranger username is required"); + Preconditions.checkArgument(password != null, "Ranger password is required"); + Preconditions.checkArgument(rangerServiceName != null, "Ranger service name is required"); rangerClient = new RangerClientExtension(rangerUrl, authType, rangerAdminName, password); rangerHelper = @@ -90,21 +94,10 @@ protected RangerAuthorizationPlugin(Map config) { rangerClient, rangerAdminName, rangerServiceName, - privilegesMappingRule(), ownerMappingRule(), policyResourceDefinesRule()); } - /** - * Translate the privilege name to the corresponding privilege name in the Ranger - * - * @param name The privilege name to translate - * @return The corresponding Ranger privilege name in the underlying permission system - */ - public Set translatePrivilege(Privilege.Name name) { - return rangerHelper.translatePrivilege(name); - } - /** * Create a new role in the Ranger.
* 1. Create a policy for metadata object.
@@ -112,7 +105,11 @@ public Set translatePrivilege(Privilege.Name name) { */ @Override public Boolean onRoleCreated(Role role) throws RuntimeException { - rangerHelper.createRangerRoleIfNotExists(role.name()); + if (!validAuthorizationOperation(role.securableObjects())) { + return false; + } + + rangerHelper.createRangerRoleIfNotExists(role.name(), false); return onRoleUpdated( role, role.securableObjects().stream() @@ -122,12 +119,18 @@ public Boolean onRoleCreated(Role role) throws RuntimeException { @Override public Boolean onRoleAcquired(Role role) throws RuntimeException { + if (!validAuthorizationOperation(role.securableObjects())) { + return false; + } return rangerHelper.checkRangerRole(role.name()); } /** Remove the role name from the Ranger policy item, and delete this Role in the Ranger.
*/ @Override public Boolean onRoleDeleted(Role role) throws RuntimeException { + if (!validAuthorizationOperation(role.securableObjects())) { + return false; + } // First, remove the role in the Ranger policy onRoleUpdated( role, @@ -147,21 +150,73 @@ public Boolean onRoleDeleted(Role role) throws RuntimeException { @Override public Boolean onRoleUpdated(Role role, RoleChange... changes) throws RuntimeException { for (RoleChange change : changes) { - boolean execResult; if (change instanceof RoleChange.AddSecurableObject) { - execResult = doAddSecurableObject((RoleChange.AddSecurableObject) change); + SecurableObject securableObject = + ((RoleChange.AddSecurableObject) change).getSecurableObject(); + if (!validAuthorizationOperation(Arrays.asList(securableObject))) { + return false; + } + + List rangerSecurableObjects = translatePrivilege(securableObject); + rangerSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + if (!doAddSecurableObject(role.name(), rangerSecurableObject)) { + throw new RuntimeException( + "Failed to add the securable object to the Ranger policy!"); + } + }); } else if (change instanceof RoleChange.RemoveSecurableObject) { - execResult = doRemoveSecurableObject((RoleChange.RemoveSecurableObject) change); + SecurableObject securableObject = + ((RoleChange.RemoveSecurableObject) change).getSecurableObject(); + if (!validAuthorizationOperation(Arrays.asList(securableObject))) { + return false; + } + + List rangerSecurableObjects = translatePrivilege(securableObject); + rangerSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + if (!doRemoveSecurableObject(role.name(), rangerSecurableObject)) { + throw new RuntimeException( + "Failed to add the securable object to the Ranger policy!"); + } + }); } else if (change instanceof RoleChange.UpdateSecurableObject) { - execResult = doUpdateSecurableObject((RoleChange.UpdateSecurableObject) change); + SecurableObject oldSecurableObject = + ((RoleChange.UpdateSecurableObject) change).getSecurableObject(); + if (!validAuthorizationOperation(Arrays.asList(oldSecurableObject))) { + return false; + } + SecurableObject newSecurableObject = + ((RoleChange.UpdateSecurableObject) change).getNewSecurableObject(); + if (!validAuthorizationOperation(Arrays.asList(newSecurableObject))) { + return false; + } + + Preconditions.checkArgument( + (oldSecurableObject.fullName().equals(newSecurableObject.fullName()) + && oldSecurableObject.type().equals(newSecurableObject.type())), + "The old and new securable objects metadata must be equal!"); + List rangerOldSecurableObjects = + translatePrivilege(oldSecurableObject); + List rangerNewSecurableObjects = + translatePrivilege(newSecurableObject); + rangerOldSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + doRemoveSecurableObject(role.name(), rangerSecurableObject); + }); + rangerNewSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + doAddSecurableObject(role.name(), rangerSecurableObject); + }); } else { throw new IllegalArgumentException( "Unsupported role change type: " + (change == null ? "null" : change.getClass().getSimpleName())); } - if (!execResult) { - return Boolean.FALSE; - } } return Boolean.TRUE; @@ -181,29 +236,42 @@ public Boolean onRoleUpdated(Role role, RoleChange... changes) throws RuntimeExc @Override public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner newOwner) throws RuntimeException { - RangerHelper.check(newOwner != null, "The newOwner must be not null"); + Preconditions.checkArgument(newOwner != null, "The newOwner must be not null"); // Add the user or group to the Ranger + String preOwnerUserName = null, + preOwnerGroupName = null, + newOwnerUserName = null, + newOwnerGroupName = null; AuditInfo auditInfo = AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) .withCreateTime(Instant.now()) .build(); + if (preOwner != null) { + if (preOwner.type() == Owner.Type.USER) { + preOwnerUserName = newOwner.name(); + } else { + preOwnerGroupName = newOwner.name(); + } + } if (newOwner.type() == Owner.Type.USER) { + newOwnerUserName = newOwner.name(); UserEntity userEntity = UserEntity.builder() .withId(1L) - .withName(newOwner.name()) + .withName(newOwnerUserName) .withRoleNames(Collections.emptyList()) .withRoleIds(Collections.emptyList()) .withAuditInfo(auditInfo) .build(); onUserAdded(userEntity); } else { + newOwnerGroupName = newOwner.name(); GroupEntity groupEntity = GroupEntity.builder() .withId(1L) - .withName(newOwner.name()) + .withName(newOwnerGroupName) .withRoleNames(Collections.emptyList()) .withRoleIds(Collections.emptyList()) .withAuditInfo(auditInfo) @@ -211,17 +279,79 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n onGroupAdded(groupEntity); } - RangerPolicy policy = rangerHelper.findManagedPolicy(metadataObject); - try { - if (policy == null) { - policy = rangerHelper.addOwnerToNewPolicy(metadataObject, newOwner); - rangerClient.createPolicy(policy); - } else { - rangerHelper.updatePolicyOwner(policy, preOwner, newOwner); - rangerClient.updatePolicy(policy.getId(), policy); - } - } catch (RangerServiceException e) { - throw new RuntimeException(e); + List rangerSecurableObjects = translateOwner(metadataObject); + String ownerRoleName; + switch (metadataObject.type()) { + case METALAKE: + case CATALOG: + // The metalake and catalog use role to manage the owner + if (metadataObject.type() == MetadataObject.Type.METALAKE) { + ownerRoleName = RangerHelper.GRAVITINO_METALAKE_OWNER_ROLE; + } else { + ownerRoleName = RangerHelper.GRAVITINO_CATALOG_OWNER_ROLE; + } + rangerHelper.createRangerRoleIfNotExists(ownerRoleName, true); + try { + if (preOwnerUserName != null || preOwnerGroupName != null) { + GrantRevokeRoleRequest revokeRoleRequest = + rangerHelper.createGrantRevokeRoleRequest( + ownerRoleName, preOwnerUserName, preOwnerGroupName); + rangerClient.revokeRole(rangerServiceName, revokeRoleRequest); + } + if (newOwnerUserName != null || newOwnerGroupName != null) { + GrantRevokeRoleRequest grantRoleRequest = + rangerHelper.createGrantRevokeRoleRequest( + ownerRoleName, newOwnerUserName, newOwnerGroupName); + rangerClient.grantRole(rangerServiceName, grantRoleRequest); + } + } catch (RangerServiceException e) { + // Ignore exception, support idempotent operation + LOG.warn("Grant owner role: {} failed!", ownerRoleName, e); + } + + rangerSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); + try { + if (policy == null) { + policy = + rangerHelper.addOwnerRoleToNewPolicy( + rangerSecurableObject, ownerRoleName); + rangerClient.createPolicy(policy); + } else { + rangerHelper.updatePolicyOwnerRole(policy, ownerRoleName); + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + }); + break; + case SCHEMA: + case TABLE: + // The schema and table use user/group to manage the owner + rangerSecurableObjects.stream() + .forEach( + rangerSecurableObject -> { + RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); + try { + if (policy == null) { + policy = rangerHelper.addOwnerToNewPolicy(rangerSecurableObject, newOwner); + rangerClient.createPolicy(policy); + } else { + rangerHelper.updatePolicyOwner(policy, preOwner, newOwner); + rangerClient.updatePolicy(policy.getId(), policy); + } + } catch (RangerServiceException e) { + throw new RuntimeException(e); + } + }); + break; + default: + throw new AuthorizationPluginException( + "The owner privilege is not supported for the securable object: %s", + metadataObject.type()); } return Boolean.TRUE; @@ -238,13 +368,17 @@ public Boolean onOwnerSet(MetadataObject metadataObject, Owner preOwner, Owner n */ @Override public Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeException { + if (roles.stream().anyMatch(role -> !validAuthorizationOperation(role.securableObjects()))) { + return false; + } + // If the user does not exist, then create it. onUserAdded(user); roles.stream() .forEach( role -> { - rangerHelper.createRangerRoleIfNotExists(role.name()); + rangerHelper.createRangerRoleIfNotExists(role.name(), false); GrantRevokeRoleRequest grantRevokeRoleRequest = rangerHelper.createGrantRevokeRoleRequest(role.name(), user.name(), null); try { @@ -269,6 +403,9 @@ public Boolean onGrantedRolesToUser(List roles, User user) throws RuntimeE */ @Override public Boolean onRevokedRolesFromUser(List roles, User user) throws RuntimeException { + if (roles.stream().anyMatch(role -> !validAuthorizationOperation(role.securableObjects()))) { + return false; + } // If the user does not exist, then create it. onUserAdded(user); @@ -300,13 +437,16 @@ public Boolean onRevokedRolesFromUser(List roles, User user) throws Runtim */ @Override public Boolean onGrantedRolesToGroup(List roles, Group group) throws RuntimeException { + if (roles.stream().anyMatch(role -> !validAuthorizationOperation(role.securableObjects()))) { + return false; + } // If the group does not exist, then create it. onGroupAdded(group); roles.stream() .forEach( role -> { - rangerHelper.createRangerRoleIfNotExists(role.name()); + rangerHelper.createRangerRoleIfNotExists(role.name(), false); GrantRevokeRoleRequest grantRevokeRoleRequest = rangerHelper.createGrantRevokeRoleRequest(role.name(), null, group.name()); try { @@ -330,6 +470,9 @@ public Boolean onGrantedRolesToGroup(List roles, Group group) throws Runti */ @Override public Boolean onRevokedRolesFromGroup(List roles, Group group) throws RuntimeException { + if (roles.stream().anyMatch(role -> !validAuthorizationOperation(role.securableObjects()))) { + return false; + } onGroupAdded(group); roles.stream() .forEach( @@ -414,37 +557,32 @@ public Boolean onGroupAcquired(Group group) { * return true.
* 3. If the policy does not exist, then create a new policy.
*/ - private boolean doAddSecurableObject(RoleChange.AddSecurableObject change) { - RangerPolicy policy = rangerHelper.findManagedPolicy(change.getSecurableObject()); + private boolean doAddSecurableObject(String roleName, RangerSecurableObject securableObject) { + RangerPolicy policy = rangerHelper.findManagedPolicy(securableObject); if (policy != null) { - // Check the policy item's accesses and roles equal the Gravitino securable object's privilege - Set policyPrivileges = + // Check the policy item's accesses and roles equal the Ranger securable object's privilege + Set policyPrivileges = policy.getPolicyItems().stream() - .filter(policyItem -> policyItem.getRoles().contains(change.getRoleName())) + .filter(policyItem -> policyItem.getRoles().contains(roleName)) .flatMap(policyItem -> policyItem.getAccesses().stream()) .map(RangerPolicy.RangerPolicyItemAccess::getType) + .map(RangerPrivileges::valueOf) .collect(Collectors.toSet()); - Set newPrivileges = - change.getSecurableObject().privileges().stream() - .filter(Objects::nonNull) - .flatMap(privilege -> translatePrivilege(privilege.name()).stream()) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); - if (policyPrivileges.containsAll(newPrivileges)) { + if (policyPrivileges.containsAll(securableObject.privileges())) { LOG.info( "The privilege({}) already added to Ranger policy({})!", policy.getName(), - change.getSecurableObject().fullName()); + securableObject.fullName()); // If it exists policy with the same privilege, then directly return true, because support // idempotent operation. return true; } } else { - policy = rangerHelper.createPolicyAddResources(change.getSecurableObject()); + policy = rangerHelper.createPolicyAddResources(securableObject); } - rangerHelper.addPolicyItem(policy, change.getRoleName(), change.getSecurableObject()); + rangerHelper.addPolicyItem(policy, roleName, securableObject); try { if (policy.getId() == null) { rangerClient.createPolicy(policy); @@ -465,13 +603,13 @@ private boolean doAddSecurableObject(RoleChange.AddSecurableObject change) { *
* 3. If policy does not contain any policy item, then delete this policy.
*/ - private boolean doRemoveSecurableObject(RoleChange.RemoveSecurableObject change) { - RangerPolicy policy = rangerHelper.findManagedPolicy(change.getSecurableObject()); + private boolean doRemoveSecurableObject( + String roleName, RangerSecurableObject rangerSecurableObject) { + RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); if (policy == null) { LOG.warn( - "Cannot find the Ranger policy({}) for the Gravitino securable object({})!", - change.getRoleName(), - change.getSecurableObject().fullName()); + "Cannot find the Ranger policy for the Ranger securable object({})!", + rangerSecurableObject.fullName()); // Don't throw exception or return false, because need support immutable operation. return true; } @@ -487,16 +625,13 @@ private boolean doRemoveSecurableObject(RoleChange.RemoveSecurableObject change) access -> { // Use Gravitino privilege to search the Ranger policy item's access boolean matchPrivilege = - change.getSecurableObject().privileges().stream() + rangerSecurableObject.privileges().stream() .filter(Objects::nonNull) - .flatMap( - privilege -> translatePrivilege(privilege.name()).stream()) - .filter(Objects::nonNull) - .anyMatch(privilege -> privilege.equals(access.getType())); + .anyMatch(privilege -> privilege.equalsTo(access.getType())); return matchPrivilege; }); if (match) { - policyItem.getRoles().removeIf(change.getRoleName()::equals); + policyItem.getRoles().removeIf(roleName::equals); } }); @@ -522,36 +657,35 @@ private boolean doRemoveSecurableObject(RoleChange.RemoveSecurableObject change) return true; } - /** - * 1. Find the policy base the metadata object.
- * 2. If the policy exists, then user new securable object's privilege to update.
- * 3. If the policy does not exist, return false.
- */ - private boolean doUpdateSecurableObject(RoleChange.UpdateSecurableObject change) { - RangerPolicy policy = rangerHelper.findManagedPolicy(change.getSecurableObject()); - if (policy == null) { - LOG.warn( - "Cannot find the Ranger policy({}) for the Gravitino securable object({})!", - change.getRoleName(), - change.getSecurableObject().fullName()); - // Don't throw exception or return false, because need support immutable operation. - return false; - } - - rangerHelper.removePolicyItem(policy, change.getRoleName(), change.getSecurableObject()); - rangerHelper.addPolicyItem(policy, change.getRoleName(), change.getNewSecurableObject()); - try { - if (policy.getId() == null) { - rangerClient.createPolicy(policy); - } else { - rangerClient.updatePolicy(policy.getId(), policy); - } - } catch (RangerServiceException e) { - throw new RuntimeException(e); - } - return true; - } - @Override public void close() throws IOException {} + + boolean validAuthorizationOperation(List securableObjects) { + return securableObjects.stream() + .allMatch( + securableObject -> { + AtomicBoolean match = new AtomicBoolean(true); + securableObject.privileges().stream() + .forEach( + privilege -> { + if (!allowPrivilegesRule().contains(privilege.name())) { + LOG.error( + "Authorization to ignore privilege({}) on metadata object({})!", + privilege.name(), + securableObject.fullName()); + match.set(false); + return; + } + + if (!privilege.canBindTo(securableObject.type())) { + LOG.error( + "The privilege({}) is not supported for the metadata object({})!", + privilege.name(), + securableObject.fullName()); + match.set(false); + } + }); + return match.get(); + }); + } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java index fd822559da6..a554559ea5c 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.sun.jersey.api.client.GenericType; import com.sun.jersey.api.client.UniformInterfaceException; @@ -32,6 +33,7 @@ import org.apache.gravitino.authorization.ranger.reference.VXUserList; import org.apache.ranger.RangerClient; import org.apache.ranger.RangerServiceException; +import org.apache.ranger.plugin.model.RangerPolicy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -98,6 +100,19 @@ public RangerClientExtension(String hostName, String authType, String username, } } + public RangerPolicy createPolicy(RangerPolicy policy) throws RangerServiceException { + Preconditions.checkArgument( + policy.getResources().size() > 0, "Ranger policy resources can not be empty!"); + return super.createPolicy(policy); + } + + public RangerPolicy updatePolicy(long policyId, RangerPolicy policy) + throws RangerServiceException { + Preconditions.checkArgument( + policy.getResources().size() > 0, "Ranger policy resources can not be empty!"); + return super.updatePolicy(policyId, policy); + } + public Boolean createUser(VXUser user) throws RuntimeException { try { callAPIMethodClassResponseType.invoke(this, CREATE_EXTERNAL_USER, null, user, VXUser.class); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java index 13f5a5cba30..12a99fd6161 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java @@ -18,22 +18,19 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.errorprone.annotations.FormatMethod; -import com.google.errorprone.annotations.FormatString; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import org.apache.commons.lang.StringUtils; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.Privilege; -import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; import org.apache.gravitino.exceptions.AuthorizationPluginException; import org.apache.ranger.RangerClient; @@ -53,9 +50,8 @@ public class RangerHelper { private static final Logger LOG = LoggerFactory.getLogger(RangerHelper.class); public static final String MANAGED_BY_GRAVITINO = "MANAGED_BY_GRAVITINO"; - - /** Mapping Gravitino privilege name to the underlying authorization system privileges. */ - private final Map> privilegesMapping; + /** The `*` gives access to all resources */ + public static final String RESOURCE_ALL = "*"; /** The owner privileges, the owner can do anything on the metadata object */ private final Set ownerPrivileges; /** The policy search keys */ @@ -64,34 +60,22 @@ public class RangerHelper { private final RangerClient rangerClient; private final String rangerAdminName; private final String rangerServiceName; + public static final String GRAVITINO_METALAKE_OWNER_ROLE = "GRAVITINO_METALAKE_OWNER_ROLE"; + public static final String GRAVITINO_CATALOG_OWNER_ROLE = "GRAVITINO_CATALOG_OWNER_ROLE"; public RangerHelper( RangerClient rangerClient, String rangerAdminName, String rangerServiceName, - Map> privilegesMapping, Set ownerPrivileges, List resourceDefines) { this.rangerClient = rangerClient; this.rangerAdminName = rangerAdminName; this.rangerServiceName = rangerServiceName; - this.privilegesMapping = privilegesMapping; this.ownerPrivileges = ownerPrivileges; this.policyResourceDefines = resourceDefines; } - /** - * Translate the privilege name to the corresponding privilege name in the Ranger - * - * @param name The privilege name to translate - * @return The corresponding Ranger privilege name in the underlying permission system - */ - public Set translatePrivilege(Privilege.Name name) { - return privilegesMapping.get(name).stream() - .map(RangerPrivilege::getName) - .collect(Collectors.toSet()); - } - /** * For easy management, each privilege will create one RangerPolicyItemAccess in the policy. * @@ -123,56 +107,47 @@ void checkPolicyItemAccess(RangerPolicy.RangerPolicyItem policyItem) * We cannot clean the policy items because one Ranger policy maybe contains multiple Gravitino * securable objects.
*/ - void addPolicyItem(RangerPolicy policy, String roleName, SecurableObject securableObject) { - // First check the privilege if support in the Ranger Hive - checkPrivileges(securableObject); - + void addPolicyItem(RangerPolicy policy, String roleName, RangerSecurableObject securableObject) { // Add the policy items by the securable object's privileges securableObject .privileges() .forEach( - gravitinoPrivilege -> { - // Translate the Gravitino privilege to map Ranger privilege - translatePrivilege(gravitinoPrivilege.name()) - .forEach( - rangerPrivilege -> { - // Find the policy item that matches Gravitino privilege - List matchPolicyItems = - policy.getPolicyItems().stream() - .filter( - policyItem -> { - return policyItem.getAccesses().stream() - .anyMatch( - access -> access.getType().equals(rangerPrivilege)); - }) - .collect(Collectors.toList()); - - if (matchPolicyItems.size() == 0) { - // If the policy item does not exist, then create a new policy item - RangerPolicy.RangerPolicyItem policyItem = - new RangerPolicy.RangerPolicyItem(); - RangerPolicy.RangerPolicyItemAccess access = - new RangerPolicy.RangerPolicyItemAccess(); - access.setType(rangerPrivilege); - policyItem.getAccesses().add(access); - policyItem.getRoles().add(roleName); - if (Privilege.Condition.ALLOW == gravitinoPrivilege.condition()) { - policy.getPolicyItems().add(policyItem); - } else { - policy.getDenyPolicyItems().add(policyItem); + rangerPrivilege -> { + // Find the policy item that matches Gravitino privilege + List matchPolicyItems = + policy.getPolicyItems().stream() + .filter( + policyItem -> { + return policyItem.getAccesses().stream() + .anyMatch( + access -> access.getType().equals(rangerPrivilege.getName())); + }) + .collect(Collectors.toList()); + + if (matchPolicyItems.isEmpty()) { + // If the policy item does not exist, then create a new policy item + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + RangerPolicy.RangerPolicyItemAccess access = + new RangerPolicy.RangerPolicyItemAccess(); + access.setType(rangerPrivilege.getName()); + policyItem.getAccesses().add(access); + policyItem.getRoles().add(roleName); + if (Privilege.Condition.ALLOW == rangerPrivilege.condition()) { + policy.getPolicyItems().add(policyItem); + } else { + policy.getDenyPolicyItems().add(policyItem); + } + } else { + // If the policy item exists, then add the role to the policy item + matchPolicyItems.stream() + .forEach( + policyItem -> { + // If the role is not in the policy item, then add it + if (!policyItem.getRoles().contains(roleName)) { + policyItem.getRoles().add(roleName); } - } else { - // If the policy item exists, then add the role to the policy item - matchPolicyItems.stream() - .forEach( - policyItem -> { - // If the role is not in the policy item, then add it - if (!policyItem.getRoles().contains(roleName)) { - policyItem.getRoles().add(roleName); - } - }); - } - }); + }); + } }); } @@ -181,10 +156,8 @@ void addPolicyItem(RangerPolicy policy, String roleName, SecurableObject securab * We cannot directly clean the policy items because one Ranger policy maybe contains multiple * Gravitino privilege objects.
*/ - void removePolicyItem(RangerPolicy policy, String roleName, SecurableObject securableObject) { - // First check the privilege if support in the Ranger Hive - checkPrivileges(securableObject); - + void removePolicyItem( + RangerPolicy policy, String roleName, RangerSecurableObject securableObject) { // Delete the policy role base the securable object's privileges policy.getPolicyItems().stream() .forEach( @@ -195,12 +168,9 @@ void removePolicyItem(RangerPolicy policy, String roleName, SecurableObject secu access -> { boolean matchPrivilege = securableObject.privileges().stream() - .filter(Objects::nonNull) - .flatMap(privilege -> translatePrivilege(privilege.name()).stream()) - .filter(Objects::nonNull) .anyMatch( privilege -> { - return access.getType().equals(privilege); + return access.getType().equals(privilege.getName()); }); if (matchPrivilege) { policyItem.getRoles().removeIf(roleName::equals); @@ -220,33 +190,14 @@ void removePolicyItem(RangerPolicy policy, String roleName, SecurableObject secu } /** - * Whether this privilege is underlying permission system supported - * - * @param securableObject The securable object to check - * @return true if the privilege is supported, otherwise false - */ - private boolean checkPrivileges(SecurableObject securableObject) { - securableObject - .privileges() - .forEach( - privilege -> { - check( - privilegesMapping.containsKey(privilege.name()), - "This privilege %s is not supported in the Ranger hive authorization", - privilege.name()); - }); - return true; - } - - /** - * Find the managed policy for the metadata object. + * Find the managed policy for the ranger securable object. * - * @param metadataObject The metadata object to find the managed policy. + * @param rangerSecurableObject The ranger securable object to find the managed policy. * @return The managed policy for the metadata object. */ - public RangerPolicy findManagedPolicy(MetadataObject metadataObject) + public RangerPolicy findManagedPolicy(RangerSecurableObject rangerSecurableObject) throws AuthorizationPluginException { - List nsMetadataObj = getMetadataObjectNames(metadataObject); + List nsMetadataObj = getMetadataObjectNames(rangerSecurableObject); Map searchFilters = new HashMap<>(); Map preciseFilters = new HashMap<>(); @@ -323,6 +274,10 @@ protected GrantRevokeRoleRequest createGrantRevokeRoleRequest( Set groups = StringUtils.isEmpty(groupName) ? Sets.newHashSet() : Sets.newHashSet(groupName); + if (users.size() == 0 && groups.size() == 0) { + throw new AuthorizationPluginException("The user and group cannot be empty!"); + } + GrantRevokeRoleRequest roleRequest = new GrantRevokeRoleRequest(); roleRequest.setUsers(users); roleRequest.setGroups(groups); @@ -331,8 +286,25 @@ protected GrantRevokeRoleRequest createGrantRevokeRoleRequest( return roleRequest; } - /** Create a Ranger role if the role does not exist. */ - protected RangerRole createRangerRoleIfNotExists(String roleName) { + /** + * Create a Ranger role if the role does not exist. + * + * @param roleName The role name to create + * @param isOwnerRole The role is owner role or not + */ + protected RangerRole createRangerRoleIfNotExists(String roleName, boolean isOwnerRole) { + if (isOwnerRole) { + Preconditions.checkArgument( + roleName.equalsIgnoreCase(GRAVITINO_METALAKE_OWNER_ROLE) + || roleName.equalsIgnoreCase(GRAVITINO_CATALOG_OWNER_ROLE), + "The role name should be GRAVITINO_METALAKE_OWNER_ROLE or GRAVITINO_CATALOG_OWNER_ROLE"); + } else { + Preconditions.checkArgument( + !roleName.equalsIgnoreCase(GRAVITINO_METALAKE_OWNER_ROLE) + && !roleName.equalsIgnoreCase(GRAVITINO_CATALOG_OWNER_ROLE), + "The role name should not be GRAVITINO_METALAKE_OWNER_ROLE or GRAVITINO_CATALOG_OWNER_ROLE"); + } + RangerRole rangerRole = null; try { rangerRole = rangerClient.getRole(roleName, rangerAdminName, rangerServiceName); @@ -423,14 +395,13 @@ protected void updatePolicyOwner(RangerPolicy policy, Owner preOwner, Owner newO }); } - private List getMetadataObjectNames(MetadataObject metadataObject) { + private static List getMetadataObjectNames(MetadataObject metadataObject) { List nsMetadataObject = Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); if (nsMetadataObject.size() > 4) { // The max level of the securable object is `catalog.db.table.column` throw new RuntimeException("The length of the securable object should not be greater than 4"); } - nsMetadataObject.remove(0); // remove `catalog` return nsMetadataObject; } @@ -472,10 +443,73 @@ protected RangerPolicy addOwnerToNewPolicy(MetadataObject metadataObject, Owner return policy; } - @FormatMethod - protected static void check(boolean condition, @FormatString String message, Object... args) { - if (!condition) { - throw new AuthorizationPluginException(message, args); - } + protected RangerPolicy addOwnerRoleToNewPolicy( + MetadataObject metadataObject, String ownerRoleName) { + RangerPolicy policy = createPolicyAddResources(metadataObject); + + ownerPrivileges.forEach( + ownerPrivilege -> { + // Each owner's privilege will create one RangerPolicyItemAccess in the policy + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem + .getAccesses() + .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); + policyItem.getRoles().add(ownerRoleName); + policy.getPolicyItems().add(policyItem); + }); + return policy; + } + + protected void updatePolicyOwnerRole(RangerPolicy policy, String ownerRoleName) { + // Find matching policy items based on the owner's privileges + List matchPolicyItems = + policy.getPolicyItems().stream() + .filter( + policyItem -> { + return policyItem.getAccesses().stream() + .allMatch( + policyItemAccess -> { + return ownerPrivileges.stream() + .anyMatch( + ownerPrivilege -> { + return ownerPrivilege.equalsTo(policyItemAccess.getType()); + }); + }); + }) + .collect(Collectors.toList()); + // Add or remove the owner role in the policy item + matchPolicyItems.forEach( + policyItem -> { + if (!policyItem.getRoles().contains(ownerRoleName)) { + policyItem.getRoles().add(ownerRoleName); + } + }); + + // If the policy item is not equal to owner's privileges, then update the policy + ownerPrivileges.stream() + .filter( + ownerPrivilege -> { + return matchPolicyItems.stream() + .noneMatch( + policyItem -> { + return policyItem.getAccesses().stream() + .anyMatch( + policyItemAccess -> { + return ownerPrivilege.equalsTo(policyItemAccess.getType()); + }); + }); + }) + .forEach( + // Add lost owner's privilege to the policy + ownerPrivilege -> { + RangerPolicy.RangerPolicyItem policyItem = new RangerPolicy.RangerPolicyItem(); + policyItem + .getAccesses() + .add(new RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName())); + if (!policyItem.getRoles().contains(ownerRoleName)) { + policyItem.getRoles().add(ownerRoleName); + } + policy.getPolicyItems().add(policyItem); + }); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilege.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilege.java index 0953ac9a518..04774b417e1 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilege.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilege.java @@ -18,64 +18,13 @@ */ package org.apache.gravitino.authorization.ranger; +import org.apache.gravitino.authorization.Privilege; + /** RangerPrivilege interface is used to define the Ranger privileges. */ public interface RangerPrivilege { String getName(); - boolean equalsTo(String value); - - /** Ranger Hive privileges enumeration. */ - enum RangerHivePrivilege implements RangerPrivilege { - ALL("all"), - SELECT("select"), - UPDATE("update"), - CREATE("create"), - DROP("drop"), - ALTER("alter"), - INDEX("index"), - LOCK("lock"), - READ("read"), - WRITE("write"), - REPLADMIN("repladmin"), - SERVICEADMIN("serviceadmin"); - - private final String name; // Access a type in the Ranger policy item - - RangerHivePrivilege(String name) { - this.name = name; - } - - @Override - public String getName() { - return name; - } + Privilege.Condition condition(); - @Override - public boolean equalsTo(String value) { - return name.equalsIgnoreCase(value); - } - } - - /** Ranger HDFS privileges enumeration. */ - enum RangerHdfsPrivilege implements RangerPrivilege { - READ("read"), - WRITE("write"), - EXECUTE("execute"); - - private final String name; // Access a type in the Ranger policy item - - RangerHdfsPrivilege(String name) { - this.name = name; - } - - @Override - public String getName() { - return name; - } - - @Override - public boolean equalsTo(String value) { - return name.equalsIgnoreCase(value); - } - } + boolean equalsTo(String value); } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java index 1dff01dc8f3..c547781678c 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivileges.java @@ -18,18 +18,111 @@ */ package org.apache.gravitino.authorization.ranger; +import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import java.util.List; +import org.apache.gravitino.authorization.Privilege; public class RangerPrivileges { + /** Ranger Hive privileges enumeration. */ + public enum RangerHivePrivilege implements RangerPrivilege { + ALL("all"), + SELECT("select"), + UPDATE("update"), + CREATE("create"), + DROP("drop"), + ALTER("alter"), + INDEX("index"), + LOCK("lock"), + READ("read"), + WRITE("write"), + REPLADMIN("repladmin"), + SERVICEADMIN("serviceadmin"); + + private final String name; // Access a type in the Ranger policy item + + RangerHivePrivilege(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public Privilege.Condition condition() { + return null; + } + + @Override + public boolean equalsTo(String value) { + return name.equalsIgnoreCase(value); + } + } + + public static class RangerHivePrivilegeImpl implements RangerPrivilege { + private RangerPrivilege rangerHivePrivilege; + private Privilege.Condition condition; + + public RangerHivePrivilegeImpl( + RangerPrivilege rangerHivePrivilege, Privilege.Condition condition) { + this.rangerHivePrivilege = rangerHivePrivilege; + this.condition = condition; + } + + @Override + public String getName() { + return rangerHivePrivilege.getName(); + } + + @Override + public Privilege.Condition condition() { + return condition; + } + + @Override + public boolean equalsTo(String value) { + return rangerHivePrivilege.equalsTo(value); + } + } + + /** Ranger HDFS privileges enumeration. */ + public enum RangerHdfsPrivilege implements RangerPrivilege { + READ("read"), + WRITE("write"), + EXECUTE("execute"); + + private final String name; // Access a type in the Ranger policy item + + RangerHdfsPrivilege(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public Privilege.Condition condition() { + return null; + } + + @Override + public boolean equalsTo(String value) { + return name.equalsIgnoreCase(value); + } + } + static List>> allRangerPrivileges = Lists.newArrayList( - RangerPrivilege.RangerHivePrivilege.class, RangerPrivilege.RangerHdfsPrivilege.class); + RangerPrivileges.RangerHivePrivilege.class, RangerPrivileges.RangerHdfsPrivilege.class); - public static RangerPrivilege valueOf(String string) { - RangerHelper.check(string != null, "Privilege name string cannot be null!"); + public static RangerPrivilege valueOf(String name) { + Preconditions.checkArgument(name != null, "Privilege name string cannot be null!"); - String strPrivilege = string.trim().toLowerCase(); + String strPrivilege = name.trim().toLowerCase(); for (Class> enumClass : allRangerPrivileges) { for (Enum privilege : enumClass.getEnumConstants()) { if (((RangerPrivilege) privilege).equalsTo(strPrivilege)) { @@ -37,6 +130,6 @@ public static RangerPrivilege valueOf(String string) { } } } - throw new IllegalArgumentException("Unknown privilege string: " + string); + throw new IllegalArgumentException("Unknown privilege name: " + name); } } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java index c6a154d2218..6b30b4bb1c6 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerPrivilegesMappingProvider.java @@ -21,7 +21,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.gravitino.MetadataObject; import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.SecurableObject; /** * Ranger authorization use this provider to mapping Gravitino privilege to the Ranger privileges. @@ -29,12 +31,47 @@ * HBase, etc. */ public interface RangerPrivilegesMappingProvider { - /** Set the mapping Gravitino privilege name to the Ranger privileges rule. */ + /** + * Set the mapping Gravitino privilege name to the Ranger privileges rule. + * + * @return The mapping Gravitino privilege name to the Ranger privileges rule. + */ Map> privilegesMappingRule(); - /** Set the owner privileges rule. */ + /** + * Set the owner Ranger privileges rule. + * + * @return The owner Ranger privileges rule. + */ Set ownerMappingRule(); - /** Set the policy resource defines rule. */ + /** + * Set the Ranger policy resource defines rule. + * + * @return The policy resource defines rule. + */ List policyResourceDefinesRule(); + + /** + * Allow Gravitino privilege operation defines rule. + * + * @return The allow Gravitino privilege operation defines rule. + */ + Set allowPrivilegesRule(); + + /** + * Translate the Gravitino securable object to the Ranger securable object. + * + * @param securableObject The Gravitino securable object. + * @return The Ranger securable object list. + */ + List translatePrivilege(SecurableObject securableObject); + + /** + * Translate the Gravitino securable object to the Ranger owner securable object. + * + * @param metadataObject The Gravitino metadata object. + * @return The Ranger owner securable object list. + */ + List translateOwner(MetadataObject metadataObject); } diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java new file mode 100644 index 00000000000..3cdc4e514f5 --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObject.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.annotation.Unstable; + +/** + * The Ranger securable object is the entity which access can be granted. Unless allowed by a grant, + * access is denied.
+ * You can use the helper class `RangerSecurableObjects` to create the Ranger securable object which + * you need.
+ * There is a clear difference between Ranger's Securable Object and Gravitino's Securable Object, + * Ranger's Securable Object does not have the concept of `METALAKE`, so it needs to be defined + * specifically. + */ +@Unstable +public interface RangerSecurableObject extends MetadataObject { + /** + * The privileges of the Ranger securable object. + * + * @return The privileges of the securable object. + */ + List privileges(); +} diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObjects.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObjects.java new file mode 100644 index 00000000000..6405e3e4c7d --- /dev/null +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerSecurableObjects.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Sets; +import java.util.List; +import java.util.Set; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.MetadataObjects.MetadataObjectImpl; + +/** The helper class for {@link RangerSecurableObject}. */ +public class RangerSecurableObjects { + public static RangerSecurableObject of( + List names, MetadataObject.Type type, Set privileges) { + MetadataObject metadataObject = + MetadataObjects.of( + MetadataObjects.getParentFullName(names), names.get(names.size() - 1), type); + return new RangerSecurableObjectImpl( + metadataObject.parent(), metadataObject.name(), type, privileges); + } + + private static class RangerSecurableObjectImpl extends MetadataObjectImpl + implements RangerSecurableObject { + private final List privileges; + + /** + * Create the Ranger securable object with the given name, parent and type. + * + * @param parent The parent of the metadata object + * @param name The name of the metadata object + * @param type The type of the metadata object + */ + public RangerSecurableObjectImpl( + String parent, String name, Type type, Set privileges) { + super(parent, name, type); + this.privileges = ImmutableList.copyOf(Sets.newHashSet(privileges)); + } + + @Override + public List privileges() { + return privileges; + } + } +} diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationPlugin.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationPlugin.java new file mode 100644 index 00000000000..d4f314dd7b7 --- /dev/null +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerAuthorizationPlugin.java @@ -0,0 +1,666 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.authorization.ranger; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import java.util.Arrays; +import java.util.List; +import org.apache.gravitino.MetadataObject; +import org.apache.gravitino.MetadataObjects; +import org.apache.gravitino.authorization.Privilege; +import org.apache.gravitino.authorization.Privileges; +import org.apache.gravitino.authorization.SecurableObject; +import org.apache.gravitino.authorization.SecurableObjects; +import org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class TestRangerAuthorizationPlugin { + private static RangerAuthorizationPlugin rangerAuthPlugin; + + @BeforeAll + public static void setup() { + RangerITEnv.init(); + rangerAuthPlugin = RangerITEnv.rangerAuthHivePlugin; + } + + @Test + public void testTranslatePrivilege() { + SecurableObject createSchemaInMetalake = + SecurableObjects.parse( + String.format("metalake1"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateSchema.allow())); + List createSchemaInMetalake1 = + rangerAuthPlugin.translatePrivilege(createSchemaInMetalake); + Assertions.assertEquals(1, createSchemaInMetalake1.size()); + Assertions.assertEquals("*", createSchemaInMetalake1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.SCHEMA, createSchemaInMetalake1.get(0).type()); + + SecurableObject createSchemaInCatalog = + SecurableObjects.parse( + String.format("catalog1"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateSchema.allow())); + List createSchemaInCatalog1 = + rangerAuthPlugin.translatePrivilege(createSchemaInCatalog); + Assertions.assertEquals(1, createSchemaInCatalog1.size()); + Assertions.assertEquals("*", createSchemaInCatalog1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.SCHEMA, createSchemaInCatalog1.get(0).type()); + + for (Privilege privilege : + ImmutableList.of( + Privileges.CreateTable.allow(), + Privileges.ModifyTable.allow(), + Privileges.SelectTable.allow())) { + SecurableObject metalake = + SecurableObjects.parse( + String.format("metalake1"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateTable.allow())); + List metalake1 = rangerAuthPlugin.translatePrivilege(metalake); + Assertions.assertEquals(2, metalake1.size()); + Assertions.assertEquals("*.*", metalake1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, metalake1.get(0).type()); + Assertions.assertEquals("*.*.*", metalake1.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, metalake1.get(1).type()); + + SecurableObject catalog = + SecurableObjects.parse( + String.format("catalog1"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateTable.allow())); + List catalog1 = rangerAuthPlugin.translatePrivilege(catalog); + Assertions.assertEquals(2, catalog1.size()); + Assertions.assertEquals("*.*", catalog1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, catalog1.get(0).type()); + Assertions.assertEquals("*.*.*", catalog1.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, catalog1.get(1).type()); + + SecurableObject schema = + SecurableObjects.parse( + String.format("catalog1.schema1"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(privilege)); + List schema1 = rangerAuthPlugin.translatePrivilege(schema); + Assertions.assertEquals(2, schema1.size()); + Assertions.assertEquals("schema1.*", schema1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, schema1.get(0).type()); + Assertions.assertEquals("schema1.*.*", schema1.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, schema1.get(1).type()); + + if (!privilege.equals(Privileges.CreateTable.allow())) { + // `CREATE_TABLE` not support securable object for table, So ignore check for table. + SecurableObject table = + SecurableObjects.parse( + String.format("catalog1.schema1.table1"), + MetadataObject.Type.TABLE, + Lists.newArrayList(privilege)); + List table1 = rangerAuthPlugin.translatePrivilege(table); + Assertions.assertEquals(2, table1.size()); + Assertions.assertEquals("schema1.table1", table1.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, table1.get(0).type()); + Assertions.assertEquals("schema1.table1.*", table1.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, table1.get(1).type()); + } + } + } + + @Test + public void testTranslateOwner() { + for (MetadataObject.Type type : + ImmutableList.of(MetadataObject.Type.METALAKE, MetadataObject.Type.CATALOG)) { + MetadataObject metalake = MetadataObjects.parse("metalake_or_catalog", type); + List metalakeOwner = rangerAuthPlugin.translateOwner(metalake); + Assertions.assertEquals(3, metalakeOwner.size()); + Assertions.assertEquals("*", metalakeOwner.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.SCHEMA, metalakeOwner.get(0).type()); + Assertions.assertEquals("*.*", metalakeOwner.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, metalakeOwner.get(1).type()); + Assertions.assertEquals("*.*.*", metalakeOwner.get(2).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, metalakeOwner.get(2).type()); + } + + MetadataObject schema = MetadataObjects.parse("catalog1.schema1", MetadataObject.Type.SCHEMA); + List schemaOwner = rangerAuthPlugin.translateOwner(schema); + Assertions.assertEquals(3, schemaOwner.size()); + Assertions.assertEquals("schema1", schemaOwner.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.SCHEMA, schemaOwner.get(0).type()); + Assertions.assertEquals("schema1.*", schemaOwner.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, schemaOwner.get(1).type()); + Assertions.assertEquals("schema1.*.*", schemaOwner.get(2).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, schemaOwner.get(2).type()); + + MetadataObject table = + MetadataObjects.parse("catalog1.schema1.table1", MetadataObject.Type.TABLE); + List tableOwner = rangerAuthPlugin.translateOwner(table); + Assertions.assertEquals(2, tableOwner.size()); + Assertions.assertEquals("schema1.table1", tableOwner.get(0).fullName()); + Assertions.assertEquals(MetadataObject.Type.TABLE, tableOwner.get(0).type()); + Assertions.assertEquals("schema1.table1.*", tableOwner.get(1).fullName()); + Assertions.assertEquals(MetadataObject.Type.COLUMN, tableOwner.get(1).type()); + } + + @Test + public void testValidAuthorizationOperation() { + // Create Catalog + SecurableObject createCatalog = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateCatalog.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createCatalog))); + + // Use catalog operation + SecurableObject useCatalogInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.UseCatalog.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useCatalogInMetalake))); + SecurableObject useCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.UseCatalog.allow())); + Assertions.assertTrue(rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useCatalog))); + + // Create schema + SecurableObject createSchemaInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateSchema.allow())); + SecurableObject createSchemaInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateSchema.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation( + Arrays.asList(createSchemaInMetalake, createSchemaInCatalog))); + + // Use schema operation + SecurableObject useSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.UseSchema.allow())); + Assertions.assertTrue(rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useSchema))); + + // Table + SecurableObject createTableInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateTable.allow())); + SecurableObject createTableInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateTable.allow())); + SecurableObject createTableInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.CreateTable.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation( + Arrays.asList(createTableInMetalake, createTableInCatalog, createTableInSchema))); + + SecurableObject modifyTableInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.ModifyTable.allow())); + SecurableObject modifyTableInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.ModifyTable.allow())); + SecurableObject modifyTableInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.ModifyTable.allow())); + SecurableObject modifyTable = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.ModifyTable.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation( + Arrays.asList( + modifyTableInMetalake, modifyTableInCatalog, modifyTableInSchema, modifyTable))); + + SecurableObject selectTableInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject selectTableInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject selectTableInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.SelectTable.allow())); + SecurableObject selectTable = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.SelectTable.allow())); + Assertions.assertTrue( + rangerAuthPlugin.validAuthorizationOperation( + Arrays.asList( + selectTableInMetalake, selectTableInCatalog, selectTableInSchema, selectTable))); + + // Ignore the Fileset operation + SecurableObject createFilesetInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateFileset.allow())); + SecurableObject createFilesetInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateFileset.allow())); + SecurableObject createFilesetInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.ReadFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFilesetInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFilesetInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFilesetInSchema))); + + // Ignore the Topic operation + SecurableObject writeFilesetInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.WriteFileset.allow())); + SecurableObject writeFilesetInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.WriteFileset.allow())); + SecurableObject writeFilesetInScheam = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.WriteFileset.allow())); + SecurableObject writeFileset = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.WriteFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFilesetInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFilesetInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFilesetInScheam))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFileset))); + + // Ignore the Fileset operation + SecurableObject readFilesetInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.ReadFileset.allow())); + SecurableObject readFilesetInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.ReadFileset.allow())); + SecurableObject readFilesetInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.ReadFileset.allow())); + SecurableObject readFileset = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ReadFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFilesetInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFilesetInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFilesetInSchema))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFileset))); + + // Ignore the Topic operation + SecurableObject createTopicInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateTopic.allow())); + SecurableObject createTopicInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateTopic.allow())); + SecurableObject createTopicInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopicInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopicInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopicInSchema))); + + SecurableObject produceTopicInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + SecurableObject produceTopicInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + SecurableObject produceTopicInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + SecurableObject produceTopic = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopicInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopicInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopicInSchema))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopic))); + + SecurableObject consumeTopicInMetalake = + SecurableObjects.parse( + String.format("metalake"), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + SecurableObject consumeTopicInCatalog = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + SecurableObject consumeTopicInSchema = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + SecurableObject consumeTopic = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopicInMetalake))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopicInCatalog))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopicInSchema))); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopic))); + } + + @Test + public void testInvalidAuthorizationOperation() { + // Catalog + SecurableObject createCatalog1 = + SecurableObjects.parse( + String.format("catalog"), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.CreateCatalog.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createCatalog1))); + SecurableObject createCatalog2 = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.CreateCatalog.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createCatalog2))); + SecurableObject createCatalog3 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateCatalog.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createCatalog3))); + + SecurableObject useCatalog1 = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.UseCatalog.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useCatalog1))); + SecurableObject useCatalog2 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.UseCatalog.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useCatalog2))); + + // Schema + SecurableObject createSchema1 = + SecurableObjects.parse( + String.format("catalog.schema"), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.CreateSchema.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createSchema1))); + SecurableObject createSchema2 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateSchema.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createSchema2))); + + SecurableObject useSchema1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.UseSchema.allow())); + Assertions.assertFalse(rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(useSchema1))); + + // Table + SecurableObject createTable1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTable1))); + SecurableObject createTable2 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.CreateTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTable2))); + SecurableObject createTable3 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.CreateTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTable3))); + + SecurableObject modifyTable1 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ModifyTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(modifyTable1))); + SecurableObject modifyTable2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.ModifyTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(modifyTable2))); + + SecurableObject selectTable1 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.SelectTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(selectTable1))); + SecurableObject selectTable2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.SelectTable.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(selectTable2))); + + // Topic + SecurableObject createTopic1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopic1))); + SecurableObject createTopic2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopic2))); + SecurableObject createTopic3 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createTopic3))); + SecurableObject produceTopic1 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopic1))); + SecurableObject produceTopic2 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.ProduceTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(produceTopic2))); + + SecurableObject consumeTopic1 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopic1))); + SecurableObject consumeTopic2 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.ConsumeTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(consumeTopic2))); + + // Fileset + SecurableObject createFileset1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.CreateFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFileset1))); + SecurableObject createFileset2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFileset2))); + SecurableObject createFileset3 = + SecurableObjects.parse( + String.format("catalog.schema.fileset"), + MetadataObject.Type.FILESET, + Lists.newArrayList(Privileges.CreateTopic.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(createFileset3))); + SecurableObject writeFileset1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.WriteFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFileset1))); + SecurableObject writeFileset2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.WriteFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(writeFileset2))); + + SecurableObject readFileset1 = + SecurableObjects.parse( + String.format("catalog.schema.table"), + MetadataObject.Type.TABLE, + Lists.newArrayList(Privileges.ReadFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFileset1))); + SecurableObject readFileset2 = + SecurableObjects.parse( + String.format("catalog.schema.topic"), + MetadataObject.Type.TOPIC, + Lists.newArrayList(Privileges.ReadFileset.allow())); + Assertions.assertFalse( + rangerAuthPlugin.validAuthorizationOperation(Arrays.asList(readFileset2))); + } +} diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java index 0f1f4c05abd..91597ce8e52 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveE2EIT.java @@ -47,8 +47,6 @@ import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationHivePlugin; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.catalog.hive.HiveConstants; import org.apache.gravitino.client.GravitinoMetalake; import org.apache.gravitino.connector.AuthorizationPropertiesMeta; @@ -74,7 +72,6 @@ public class RangerHiveE2EIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(RangerHiveE2EIT.class); - private static RangerAuthorizationPlugin rangerAuthPlugin; public static final String metalakeName = GravitinoITUtils.genRandomName("RangerHiveE2EIT_metalake").toLowerCase(); public static final String catalogName = @@ -109,7 +106,7 @@ public void startIntegrationTest() throws Exception { registerCustomConfigs(configs); super.startIntegrationTest(); - RangerITEnv.setup(); + RangerITEnv.init(); RangerITEnv.startHiveRangerContainer(); RANGER_ADMIN_URL = @@ -145,7 +142,7 @@ public void startIntegrationTest() throws Exception { .getOrCreate(); createMetalake(); - createCatalogAndRangerAuthPlugin(); + createCatalog(); } private static void generateRangerSparkSecurityXML() throws IOException { @@ -180,8 +177,8 @@ private static void generateRangerSparkSecurityXML() throws IOException { } @AfterAll - public void stop() throws IOException { - client = null; + public void stop() { + RangerITEnv.cleanup(); if (client != null) { Arrays.stream(catalog.asSchemas().listSchemas()) .filter(schema -> !schema.equals("default")) @@ -191,6 +188,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); } if (sparkSession != null) { @@ -221,8 +219,8 @@ void testAllowUseSchemaPrivilege() throws InterruptedException { // Create a role with CREATE_SCHEMA privilege SecurableObject securableObject1 = SecurableObjects.parse( - String.format("%s.%s", catalogName, schemaName), - MetadataObject.Type.SCHEMA, + String.format("%s", catalogName), + MetadataObject.Type.CATALOG, Lists.newArrayList(Privileges.CreateSchema.allow())); RoleEntity role = RoleEntity.builder() @@ -231,7 +229,7 @@ void testAllowUseSchemaPrivilege() throws InterruptedException { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(securableObject1)) .build(); - rangerAuthPlugin.onRoleCreated(role); + RangerITEnv.rangerAuthHivePlugin.onRoleCreated(role); // Granted this role to the spark execution user `HADOOP_USER_NAME` String userName1 = System.getenv(HADOOP_USER_NAME); @@ -244,7 +242,8 @@ void testAllowUseSchemaPrivilege() throws InterruptedException { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + RangerITEnv.rangerAuthHivePlugin.onGrantedRolesToUser( + Lists.newArrayList(role), userEntity1)); // After Ranger Authorization, Must wait a period of time for the Ranger Spark plugin to update // the policy Sleep time must be greater than the policy update interval @@ -269,29 +268,14 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } - private static void createCatalogAndRangerAuthPlugin() { - rangerAuthPlugin = - RangerAuthorizationHivePlugin.getInstance( - ImmutableMap.of( - AuthorizationPropertiesMeta.RANGER_ADMIN_URL, - RANGER_ADMIN_URL, - RANGER_AUTH_TYPE, - RangerContainer.authType, - RANGER_USERNAME, - RangerContainer.rangerUserName, - RANGER_PASSWORD, - RangerContainer.rangerPassword, - RANGER_SERVICE_NAME, - RangerITEnv.RANGER_HIVE_REPO_NAME)); - + private static void createCatalog() { Map properties = ImmutableMap.of( HiveConstants.METASTORE_URIS, diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java index 97005e58c15..eba00a18864 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java @@ -20,10 +20,10 @@ import static org.apache.gravitino.authorization.SecurableObjects.DOT_SPLITTER; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.currentFunName; -import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.rangerClient; import static org.apache.gravitino.authorization.ranger.integration.test.RangerITEnv.verifyRoleInRanger; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import java.time.Instant; import java.util.ArrayList; @@ -35,21 +35,18 @@ import org.apache.gravitino.MetadataObject; import org.apache.gravitino.MetadataObjects; import org.apache.gravitino.authorization.Owner; +import org.apache.gravitino.authorization.Privilege; import org.apache.gravitino.authorization.Privileges; import org.apache.gravitino.authorization.Role; import org.apache.gravitino.authorization.RoleChange; import org.apache.gravitino.authorization.SecurableObject; import org.apache.gravitino.authorization.SecurableObjects; -import org.apache.gravitino.authorization.ranger.RangerAuthorizationHivePlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHelper; -import org.apache.gravitino.authorization.ranger.RangerPrivilege; import org.apache.gravitino.authorization.ranger.RangerPrivileges; +import org.apache.gravitino.authorization.ranger.RangerSecurableObject; +import org.apache.gravitino.authorization.ranger.RangerSecurableObjects; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; -import org.apache.gravitino.connector.AuthorizationPropertiesMeta; -import org.apache.gravitino.integration.test.container.ContainerSuite; -import org.apache.gravitino.integration.test.container.HiveContainer; -import org.apache.gravitino.integration.test.container.RangerContainer; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.GroupEntity; @@ -57,6 +54,8 @@ import org.apache.gravitino.meta.UserEntity; import org.apache.ranger.RangerServiceException; import org.apache.ranger.plugin.model.RangerPolicy; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Tag; @@ -68,58 +67,46 @@ public class RangerHiveIT { private static final Logger LOG = LoggerFactory.getLogger(RangerHiveIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - private static final String adminUser = "gravitino"; - private static RangerAuthorizationPlugin rangerAuthPlugin; + private static RangerAuthorizationPlugin rangerAuthHivePlugin; private static RangerHelper rangerHelper; private final AuditInfo auditInfo = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); @BeforeAll public static void setup() { - RangerITEnv.setup(); - - containerSuite.startHiveRangerContainer( - new HashMap<>( - ImmutableMap.of( - HiveContainer.HIVE_RUNTIME_VERSION, - HiveContainer.HIVE3, - RangerContainer.DOCKER_ENV_RANGER_SERVER_URL, - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), - RangerContainer.RANGER_SERVER_PORT), - RangerContainer.DOCKER_ENV_RANGER_HIVE_REPOSITORY_NAME, - RangerITEnv.RANGER_HIVE_REPO_NAME, - RangerContainer.DOCKER_ENV_RANGER_HDFS_REPOSITORY_NAME, - RangerITEnv.RANGER_HDFS_REPO_NAME, - HiveContainer.HADOOP_USER_NAME, - adminUser))); - - rangerAuthPlugin = - RangerAuthorizationHivePlugin.getInstance( - ImmutableMap.of( - AuthorizationPropertiesMeta.RANGER_ADMIN_URL, - String.format( - "http://%s:%d", - containerSuite.getRangerContainer().getContainerIpAddress(), - RangerContainer.RANGER_SERVER_PORT), - AuthorizationPropertiesMeta.RANGER_AUTH_TYPE, - RangerContainer.authType, - AuthorizationPropertiesMeta.RANGER_USERNAME, - RangerContainer.rangerUserName, - AuthorizationPropertiesMeta.RANGER_PASSWORD, - RangerContainer.rangerPassword, - AuthorizationPropertiesMeta.RANGER_SERVICE_NAME, - RangerITEnv.RANGER_HIVE_REPO_NAME)); - rangerHelper = - new RangerHelper( - rangerClient, - RangerContainer.rangerUserName, - RangerITEnv.RANGER_HIVE_REPO_NAME, - rangerAuthPlugin.privilegesMappingRule(), - rangerAuthPlugin.ownerMappingRule(), - rangerAuthPlugin.policyResourceDefinesRule()); + RangerITEnv.init(); + + rangerAuthHivePlugin = RangerITEnv.rangerAuthHivePlugin; + rangerHelper = RangerITEnv.rangerHelper; + } + + @AfterAll + public static void stop() { + RangerITEnv.cleanup(); + } + + @AfterEach + public void clean() { + // Clean up the test Ranger policy + Role mockCatalogRole = mockCatalogRole(currentFunName()); + mockCatalogRole.securableObjects().stream() + .forEach( + securableObject -> { + rangerAuthHivePlugin.translatePrivilege(securableObject).stream() + .forEach( + rangerSecurableObject -> { + deleteHivePolicy(rangerSecurableObject); + }); + }); + mockCatalogRole.securableObjects().stream() + .forEach( + securableObject -> { + rangerAuthHivePlugin.translateOwner(securableObject).stream() + .forEach( + rangerSecurableObject -> { + deleteHivePolicy(rangerSecurableObject); + }); + }); } /** @@ -133,8 +120,8 @@ public static void setup() { public RoleEntity mock3TableRole(String roleName) { SecurableObject securableObject1 = SecurableObjects.parse( - String.format("catalog.%s.tab1", roleName), // use unique db name to avoid conflict - MetadataObject.Type.TABLE, + String.format("catalog.%s", roleName), // use unique db name to avoid conflict + MetadataObject.Type.SCHEMA, Lists.newArrayList(Privileges.CreateTable.allow())); SecurableObject securableObject2 = @@ -162,23 +149,42 @@ public RoleEntity mock3TableRole(String roleName) { @Test public void testOnRoleCreated() { RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); - verifyRoleInRanger(rangerAuthPlugin, role); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + verifyRoleInRanger(rangerAuthHivePlugin, role); + } + + @Test + public void testOnRoleCreatedCatalog() { + Role mockCatalogRole = mockCatalogRole(currentFunName()); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(mockCatalogRole)); + // Check if exist this policy + assertFindManagedPolicy(mockCatalogRole, true); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(mockCatalogRole)); + assertFindManagedPolicy(mockCatalogRole, false); } @Test public void testOnRoleDeleted() { // prepare to create a role RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + assertFindManagedPolicy(role, true); // delete this role - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(role)); // Check if the policy is deleted - role.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNull(rangerHelper.findManagedPolicy(securableObject))); + assertFindManagedPolicy(role, false); + } + + @Test + public void testOnRoleDeletedCatalog() { + // prepare to create a role + Role mockCatalogRole = mockCatalogRole(currentFunName()); + + // delete this role + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(mockCatalogRole)); + // Check if exist this policy + assertFindManagedPolicy(mockCatalogRole, false); } @Test @@ -190,24 +196,21 @@ public void testOnRoleDeleted2() { .forEach( securableObject -> { Assertions.assertTrue( - rangerAuthPlugin.onOwnerSet( + rangerAuthHivePlugin.onOwnerSet( securableObject, null, new MockOwner("user1", Owner.Type.USER))); }); // delete this role - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(role)); // Because this metaobject has owner, so the policy should not be deleted - role.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject))); + assertFindManagedPolicy(role, true); } @Test public void testOnRoleAcquired() { RoleEntity role = mock3TableRole(GravitinoITUtils.genRandomName(currentFunName())); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); - Assertions.assertTrue(rangerAuthPlugin.onRoleAcquired(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleAcquired(role)); } /** The metalake role does not to create Ranger policy. Only use it to help test */ @@ -216,7 +219,7 @@ public RoleEntity mockCatalogRole(String roleName) { SecurableObjects.parse( "catalog", SecurableObject.Type.CATALOG, - Lists.newArrayList(Privileges.UseCatalog.allow())); + Lists.newArrayList(Privileges.CreateSchema.allow())); RoleEntity role = RoleEntity.builder() .withId(1L) @@ -245,19 +248,21 @@ public void testFindManagedPolicy() { Lists.newArrayList(String.format("%s3", dbName), "tab*"), GravitinoITUtils.genRandomName(currentFunName())); // findManagedPolicy function use precise search, so return null - SecurableObject securableObject1 = - SecurableObjects.parse( - String.format("catalog.%s3.tab1", dbName), + RangerSecurableObject rangerSecurableObject = + RangerSecurableObjects.of( + ImmutableList.of(String.format("%s3", dbName), "tab1"), MetadataObject.Type.TABLE, - Lists.newArrayList(Privileges.CreateTable.allow())); - Assertions.assertNull(rangerHelper.findManagedPolicy(securableObject1)); + ImmutableSet.of( + new RangerPrivileges.RangerHivePrivilegeImpl( + RangerPrivileges.RangerHivePrivilege.ALL, Privilege.Condition.ALLOW))); + Assertions.assertNull(rangerHelper.findManagedPolicy(rangerSecurableObject)); // Add a policy for `db3.tab1` createHivePolicy( Lists.newArrayList(String.format("%s3", dbName), "tab1"), GravitinoITUtils.genRandomName(currentFunName())); // findManagedPolicy function use precise search, so return not null - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject1)); + Assertions.assertNotNull(rangerHelper.findManagedPolicy(rangerSecurableObject)); } static void createHivePolicy(List metaObjects, String roleName) { @@ -278,7 +283,7 @@ static void createHivePolicy(List metaObjects, String roleName) { policyItem.setAccesses( Arrays.asList( new RangerPolicy.RangerPolicyItemAccess( - RangerPrivilege.RangerHivePrivilege.SELECT.toString()))); + RangerPrivileges.RangerHivePrivilege.SELECT.toString()))); RangerITEnv.updateOrCreateRangerPolicy( RangerDefines.SERVICE_TYPE_HIVE, RangerITEnv.RANGER_HIVE_REPO_NAME, @@ -287,18 +292,30 @@ static void createHivePolicy(List metaObjects, String roleName) { Collections.singletonList(policyItem)); } + static boolean deleteHivePolicy(RangerSecurableObject rangerSecurableObject) { + RangerPolicy policy = rangerHelper.findManagedPolicy(rangerSecurableObject); + if (policy != null) { + try { + RangerITEnv.rangerClient.deletePolicy(policy.getId()); + } catch (RangerServiceException e) { + return false; + } + } + return true; + } + @Test public void testRoleChangeAddSecurableObject() { SecurableObject securableObject1 = SecurableObjects.parse( String.format("catalog.%s.tab1", currentFunName()), SecurableObject.Type.TABLE, - Lists.newArrayList(Privileges.CreateTable.allow())); + Lists.newArrayList(Privileges.SelectTable.allow())); Role mockCatalogRole = mockCatalogRole(currentFunName()); // 1. Add a securable object to the role Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( + rangerAuthHivePlugin.onRoleUpdated( mockCatalogRole, RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject1))); @@ -310,15 +327,15 @@ public void testRoleChangeAddSecurableObject() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(securableObject1)) .build(); - verifyRoleInRanger(rangerAuthPlugin, verifyRole1); + verifyRoleInRanger(rangerAuthHivePlugin, verifyRole1); // 2. Multi-call Add a same entity and privilege to the role, because support idempotent // operation, so return true Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( + rangerAuthHivePlugin.onRoleUpdated( mockCatalogRole, RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject1))); - verifyRoleInRanger(rangerAuthPlugin, verifyRole1); + verifyRoleInRanger(rangerAuthHivePlugin, verifyRole1); // 3. Add the same metadata but have different privileges to the role SecurableObject securableObject3 = @@ -327,17 +344,28 @@ public void testRoleChangeAddSecurableObject() { SecurableObject.Type.TABLE, Lists.newArrayList(Privileges.SelectTable.allow(), Privileges.ModifyTable.allow())); Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( + rangerAuthHivePlugin.onRoleUpdated( mockCatalogRole, RoleChange.addSecurableObject(mockCatalogRole.name(), securableObject3))); } + @Test + public void testRoleChangeAddCatalogSecurableObject() { + Role mockCatalogRole = mockCatalogRole(currentFunName()); + Assertions.assertTrue( + rangerAuthHivePlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.addSecurableObject( + mockCatalogRole.name(), mockCatalogRole.securableObjects().get(0)))); + assertFindManagedPolicy(mockCatalogRole, true); + } + @Test public void testRoleChangeRemoveSecurableObject() { // Prepare a role contain 3 securable objects String currentFunName = currentFunName(); RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); Role mockCatalogRole = mockCatalogRole(currentFunName); // remove a securable object from role @@ -345,7 +373,7 @@ public void testRoleChangeRemoveSecurableObject() { while (!securableObjects.isEmpty()) { SecurableObject removeSecurableObject = securableObjects.remove(0); Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( + rangerAuthHivePlugin.onRoleUpdated( mockCatalogRole, RoleChange.removeSecurableObject(mockCatalogRole.name(), removeSecurableObject))); @@ -358,17 +386,30 @@ public void testRoleChangeRemoveSecurableObject() { .withAuditInfo(auditInfo) .withSecurableObjects(securableObjects) .build(); - verifyRoleInRanger(rangerAuthPlugin, verifyRole); + verifyRoleInRanger(rangerAuthHivePlugin, verifyRole); } } } + @Test + public void testRoleChangeRemoveCatalogSecurableObject() { + String currentFunName = currentFunName(); + Role mockCatalogRole = mockCatalogRole(currentFunName); + + Assertions.assertTrue( + rangerAuthHivePlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.removeSecurableObject( + mockCatalogRole.name(), mockCatalogRole.securableObjects().get(0)))); + assertFindManagedPolicy(mockCatalogRole, false); + } + @Test public void testRoleChangeUpdateSecurableObject() { SecurableObject oldSecurableObject = SecurableObjects.parse( - String.format("catalog.%s.tab1", currentFunName()), - MetadataObject.Type.TABLE, + String.format("catalog1.%s", currentFunName()), + MetadataObject.Type.SCHEMA, Lists.newArrayList(Privileges.CreateTable.allow())); RoleEntity role = RoleEntity.builder() @@ -377,20 +418,18 @@ public void testRoleChangeUpdateSecurableObject() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(oldSecurableObject)) .build(); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); - Role mockCatalogRole = mockCatalogRole(currentFunName()); // Keep the same matedata namespace and type, but change privileges SecurableObject newSecurableObject = SecurableObjects.parse( oldSecurableObject.fullName(), oldSecurableObject.type(), - Lists.newArrayList(Privileges.SelectTable.allow())); + Lists.newArrayList(Privileges.ModifyTable.allow())); Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( - mockCatalogRole, - RoleChange.updateSecurableObject( - mockCatalogRole.name(), oldSecurableObject, newSecurableObject))); + rangerAuthHivePlugin.onRoleUpdated( + role, + RoleChange.updateSecurableObject(role.name(), oldSecurableObject, newSecurableObject))); // construct a verified role to check if the role and Ranger policy are created correctly RoleEntity verifyRole = @@ -400,22 +439,89 @@ public void testRoleChangeUpdateSecurableObject() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(newSecurableObject)) .build(); - verifyRoleInRanger(rangerAuthPlugin, verifyRole); + verifyRoleInRanger(rangerAuthHivePlugin, verifyRole); } @Test + public void testRoleChangeUpdateDifferentMetadata() { + // Test same metadata namespace and different type + SecurableObject metalake = + SecurableObjects.parse( + String.format("%s", currentFunName()), + MetadataObject.Type.METALAKE, + Lists.newArrayList(Privileges.CreateTable.allow())); + SecurableObject catalog = + SecurableObjects.parse( + metalake.fullName(), + MetadataObject.Type.CATALOG, + Lists.newArrayList(Privileges.ModifyTable.allow())); + RoleEntity role = + RoleEntity.builder() + .withId(1L) + .withName(currentFunName()) + .withAuditInfo(auditInfo) + .withSecurableObjects(Lists.newArrayList(metalake)) + .build(); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + rangerAuthHivePlugin.onRoleUpdated( + role, RoleChange.updateSecurableObject(role.name(), metalake, catalog))); + + // Test different metadata namespace and same type + SecurableObject schema1 = + SecurableObjects.parse( + String.format("catalog1.%s", currentFunName()), + MetadataObject.Type.SCHEMA, + Lists.newArrayList(Privileges.CreateTable.allow())); + SecurableObject schema2 = + SecurableObjects.parse( + String.format("catalog1-diff.%s", currentFunName()), + schema1.type(), + Lists.newArrayList(Privileges.ModifyTable.allow())); + Assertions.assertThrows( + IllegalArgumentException.class, + () -> + rangerAuthHivePlugin.onRoleUpdated( + role, RoleChange.updateSecurableObject(role.name(), schema1, schema2))); + } + + @Test + public void testRoleChangeUpdateCatalogSecurableObject() { + String currentFunName = currentFunName(); + Role mockCatalogRole = mockCatalogRole(currentFunName); + + // Keep the same matedata namespace and type, but change privileges + SecurableObject newSecurableObject = + SecurableObjects.parse( + mockCatalogRole.securableObjects().get(0).fullName(), + mockCatalogRole.securableObjects().get(0).type(), + Lists.newArrayList(Privileges.SelectTable.allow())); + + Assertions.assertTrue( + rangerAuthHivePlugin.onRoleUpdated( + mockCatalogRole, + RoleChange.updateSecurableObject( + mockCatalogRole.name(), + mockCatalogRole.securableObjects().get(0), + newSecurableObject))); + assertFindManagedPolicy(mockCatalogRole, false); + } + public void testRoleChangeCombinedOperation() { MetadataObject oldMetadataObject = MetadataObjects.parse( String.format("catalog.%s.tab1", currentFunName()), MetadataObject.Type.TABLE); String userName = "user1"; - rangerAuthPlugin.onOwnerSet(oldMetadataObject, null, new MockOwner(userName, Owner.Type.USER)); + rangerAuthHivePlugin.onOwnerSet( + oldMetadataObject, null, new MockOwner(userName, Owner.Type.USER)); + verifyOwnerInRanger(oldMetadataObject, Lists.newArrayList(userName)); SecurableObject oldSecurableObject = SecurableObjects.parse( oldMetadataObject.fullName(), MetadataObject.Type.TABLE, - Lists.newArrayList(Privileges.CreateTable.allow())); + Lists.newArrayList(Privileges.ModifyTable.allow())); RoleEntity role = RoleEntity.builder() @@ -424,9 +530,8 @@ public void testRoleChangeCombinedOperation() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(oldSecurableObject)) .build(); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); - Role mockCatalogRole = mockCatalogRole(currentFunName()); // Keep the same matedata namespace and type, but change privileges SecurableObject newSecurableObject = SecurableObjects.parse( @@ -434,10 +539,9 @@ public void testRoleChangeCombinedOperation() { oldSecurableObject.type(), Lists.newArrayList(Privileges.SelectTable.allow())); Assertions.assertTrue( - rangerAuthPlugin.onRoleUpdated( - mockCatalogRole, - RoleChange.updateSecurableObject( - mockCatalogRole.name(), oldSecurableObject, newSecurableObject))); + rangerAuthHivePlugin.onRoleUpdated( + role, + RoleChange.updateSecurableObject(role.name(), oldSecurableObject, newSecurableObject))); // construct a verified role to check if the role and Ranger policy are created correctly RoleEntity verifyRole = @@ -447,16 +551,13 @@ public void testRoleChangeCombinedOperation() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(newSecurableObject)) .build(); - verifyRoleInRanger(rangerAuthPlugin, verifyRole); + verifyRoleInRanger(rangerAuthHivePlugin, verifyRole); verifyOwnerInRanger(oldMetadataObject, Lists.newArrayList(userName)); // Delete the role - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(verifyRole)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(verifyRole)); // Because these metaobjects have an owner, so the policy will not be deleted. - role.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject))); + assertFindManagedPolicy(role, true); verifyOwnerInRanger(oldMetadataObject, Lists.newArrayList(userName)); } @@ -464,7 +565,7 @@ public void testRoleChangeCombinedOperation() { public void testOnGrantedRolesToUser() { // prepare to create a role RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); // granted a role to the user1 String userName1 = "user1"; @@ -477,13 +578,13 @@ public void testOnGrantedRolesToUser() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, Lists.newArrayList(userName1)); // multi-call to granted role to the user1 Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, Lists.newArrayList(userName1)); // granted a role to the user2 String userName2 = "user2"; @@ -496,17 +597,17 @@ public void testOnGrantedRolesToUser() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity2)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity2)); // Same to verify user1 and user2 - verifyRoleInRanger(rangerAuthPlugin, role, Lists.newArrayList(userName1, userName2)); + verifyRoleInRanger(rangerAuthHivePlugin, role, Lists.newArrayList(userName1, userName2)); } @Test public void testOnRevokedRolesFromUser() { // prepare to create a role RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); // granted a role to the user1 String userName1 = "user1"; @@ -519,24 +620,24 @@ public void testOnRevokedRolesFromUser() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, Lists.newArrayList(userName1)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, null, Lists.newArrayList(userName1)); // multi-call to revoked role from user1 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, null, Lists.newArrayList(userName1)); } @Test public void testOnGrantedRolesToGroup() { // prepare to create a role RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); // granted a role to the group1 String groupName1 = "group1"; @@ -549,13 +650,13 @@ public void testOnGrantedRolesToGroup() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, null, Lists.newArrayList(groupName1)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, null, null, Lists.newArrayList(groupName1)); // multi-call to grant a role to the group1 test idempotent operation Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, null, Lists.newArrayList(groupName1)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, null, null, Lists.newArrayList(groupName1)); // granted a role to the group2 String groupName2 = "group2"; @@ -568,18 +669,18 @@ public void testOnGrantedRolesToGroup() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity2)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity2)); // Same to verify group1 and group2 verifyRoleInRanger( - rangerAuthPlugin, role, null, null, Lists.newArrayList(groupName1, groupName2)); + rangerAuthHivePlugin, role, null, null, Lists.newArrayList(groupName1, groupName2)); } @Test public void testOnRevokedRolesFromGroup() { // prepare to create a role RoleEntity role = mock3TableRole(currentFunName()); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role)); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role)); // granted a role to the group1 String groupName1 = "group1"; @@ -592,17 +693,37 @@ public void testOnRevokedRolesFromGroup() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, null, Lists.newArrayList(groupName1)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role, null, null, Lists.newArrayList(groupName1)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, null, null, Lists.newArrayList(groupName1)); + rangerAuthHivePlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger( + rangerAuthHivePlugin, role, null, null, null, Lists.newArrayList(groupName1)); // multi-call to revoke from the group1 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role, null, null, null, Lists.newArrayList(groupName1)); + rangerAuthHivePlugin.onRevokedRolesFromGroup(Lists.newArrayList(role), groupEntity1)); + verifyRoleInRanger( + rangerAuthHivePlugin, role, null, null, null, Lists.newArrayList(groupName1)); + } + + private void assertFindManagedPolicy(Role role, boolean policyExist) { + role.securableObjects().stream() + .forEach( + securableObject -> + rangerAuthHivePlugin.translatePrivilege(securableObject).stream() + .forEach( + rangerSecurableObject -> { + LOG.info("rangerSecurableObject: " + rangerSecurableObject); + if (policyExist) { + Assertions.assertNotNull( + rangerHelper.findManagedPolicy(rangerSecurableObject)); + } else { + Assertions.assertNull( + rangerHelper.findManagedPolicy(rangerSecurableObject)); + } + })); } private static class MockOwner implements Owner { @@ -627,39 +748,109 @@ public Type type() { @Test public void testOnOwnerSet() { - MetadataObject metadataObject = + MetadataObject schema = MetadataObjects.parse( - String.format("catalog.%s.tab1", currentFunName()), MetadataObject.Type.TABLE); + String.format("catalog.%s", currentFunName()), MetadataObject.Type.SCHEMA); String userName1 = "user1"; Owner owner1 = new MockOwner(userName1, Owner.Type.USER); - Assertions.assertTrue(rangerAuthPlugin.onOwnerSet(metadataObject, null, owner1)); - verifyOwnerInRanger(metadataObject, Lists.newArrayList(userName1), null, null, null); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(schema, null, owner1)); + rangerAuthHivePlugin.translateOwner(schema).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger(rangerSecurableObject, Lists.newArrayList(userName1)); + }); + MetadataObject table = + MetadataObjects.parse( + String.format("catalog.%s.tab1", currentFunName()), MetadataObject.Type.TABLE); String userName2 = "user2"; Owner owner2 = new MockOwner(userName2, Owner.Type.USER); - Assertions.assertTrue(rangerAuthPlugin.onOwnerSet(metadataObject, owner1, owner2)); - verifyOwnerInRanger( - metadataObject, Lists.newArrayList(userName2), Lists.newArrayList(userName1), null, null); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(table, null, owner2)); + rangerAuthHivePlugin.translateOwner(table).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger(rangerSecurableObject, Lists.newArrayList(userName2)); + }); + + String userName3 = "user3"; + Owner owner3 = new MockOwner(userName3, Owner.Type.USER); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(table, owner2, owner3)); + rangerAuthHivePlugin.translateOwner(table).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger( + rangerSecurableObject, + Lists.newArrayList(userName3), + Lists.newArrayList(userName2)); + }); String groupName1 = "group1"; - Owner owner3 = new MockOwner(groupName1, Owner.Type.GROUP); - Assertions.assertTrue(rangerAuthPlugin.onOwnerSet(metadataObject, owner2, owner3)); - verifyOwnerInRanger( - metadataObject, - null, - Lists.newArrayList(userName1, userName2), - Lists.newArrayList(groupName1), - null); + Owner owner4 = new MockOwner(groupName1, Owner.Type.GROUP); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(table, owner3, owner4)); + rangerAuthHivePlugin.translateOwner(table).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger( + rangerSecurableObject, + null, + Lists.newArrayList(userName1, userName2), + Lists.newArrayList(groupName1)); + }); String groupName2 = "group2"; - Owner owner4 = new MockOwner(groupName2, Owner.Type.GROUP); - Assertions.assertTrue(rangerAuthPlugin.onOwnerSet(metadataObject, owner3, owner4)); - verifyOwnerInRanger( - metadataObject, - null, - Lists.newArrayList(userName1, userName2), - Lists.newArrayList(groupName2), - Lists.newArrayList(groupName1)); + Owner owner5 = new MockOwner(groupName2, Owner.Type.GROUP); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(table, owner4, owner5)); + rangerAuthHivePlugin.translateOwner(table).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger( + rangerSecurableObject, + null, + Lists.newArrayList(userName1, userName2), + Lists.newArrayList(groupName2), + Lists.newArrayList(groupName1)); + }); + } + + @Test + public void testOnOwnerSetCatalog() { + MetadataObject metalake = + MetadataObjects.parse( + String.format("metalake-%s", currentFunName()), MetadataObject.Type.METALAKE); + String userName1 = "user1"; + Owner owner1 = new MockOwner(userName1, Owner.Type.USER); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(metalake, null, owner1)); + rangerAuthHivePlugin.translateOwner(metalake).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger( + rangerSecurableObject, + null, + null, + null, + null, + Lists.newArrayList(RangerHelper.GRAVITINO_METALAKE_OWNER_ROLE)); + }); + + MetadataObject catalog = + MetadataObjects.parse( + String.format("catalog-%s", currentFunName()), MetadataObject.Type.CATALOG); + String userName2 = "user2"; + Owner owner2 = new MockOwner(userName2, Owner.Type.USER); + Assertions.assertTrue(rangerAuthHivePlugin.onOwnerSet(catalog, null, owner2)); + rangerAuthHivePlugin.translateOwner(catalog).stream() + .forEach( + rangerSecurableObject -> { + verifyOwnerInRanger( + rangerSecurableObject, + null, + null, + null, + null, + Lists.newArrayList( + RangerHelper.GRAVITINO_METALAKE_OWNER_ROLE, + RangerHelper.GRAVITINO_CATALOG_OWNER_ROLE)); + }); } @Test @@ -672,10 +863,10 @@ public void testCreateUser() { .withRoleIds(null) .withRoleNames(null) .build(); - Assertions.assertTrue(rangerAuthPlugin.onUserAdded(user)); - Assertions.assertTrue(rangerAuthPlugin.onUserAcquired(user)); - Assertions.assertTrue(rangerAuthPlugin.onUserRemoved(user)); - Assertions.assertFalse(rangerAuthPlugin.onUserAcquired(user)); + Assertions.assertTrue(rangerAuthHivePlugin.onUserAdded(user)); + Assertions.assertTrue(rangerAuthHivePlugin.onUserAcquired(user)); + Assertions.assertTrue(rangerAuthHivePlugin.onUserRemoved(user)); + Assertions.assertFalse(rangerAuthHivePlugin.onUserAcquired(user)); } @Test @@ -689,24 +880,25 @@ public void testCreateGroup() { .withRoleNames(null) .build(); - Assertions.assertTrue(rangerAuthPlugin.onGroupAdded(group)); - Assertions.assertTrue(rangerAuthPlugin.onGroupAcquired(group)); - Assertions.assertTrue(rangerAuthPlugin.onGroupRemoved(group)); - Assertions.assertFalse(rangerAuthPlugin.onGroupAcquired(group)); + Assertions.assertTrue(rangerAuthHivePlugin.onGroupAdded(group)); + Assertions.assertTrue(rangerAuthHivePlugin.onGroupAcquired(group)); + Assertions.assertTrue(rangerAuthHivePlugin.onGroupRemoved(group)); + Assertions.assertFalse(rangerAuthHivePlugin.onGroupAcquired(group)); } @Test public void testCombinationOperation() { - // Create a `CreateTable` privilege role + // Create a `ModifyTable` privilege role SecurableObject securableObject1 = SecurableObjects.parse( String.format( "catalog.%s.tab1", currentFunName()), // use unique db name to avoid conflict MetadataObject.Type.TABLE, - Lists.newArrayList(Privileges.CreateTable.allow())); + Lists.newArrayList(Privileges.ModifyTable.allow())); String ownerName = "owner1"; - rangerAuthPlugin.onOwnerSet(securableObject1, null, new MockOwner(ownerName, Owner.Type.USER)); + rangerAuthHivePlugin.onOwnerSet( + securableObject1, null, new MockOwner(ownerName, Owner.Type.USER)); verifyOwnerInRanger(securableObject1, Lists.newArrayList(ownerName), null, null, null); RoleEntity role1 = @@ -716,8 +908,8 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(securableObject1)) .build(); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role1)); - verifyRoleInRanger(rangerAuthPlugin, role1); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role1)); + verifyRoleInRanger(rangerAuthHivePlugin, role1); // Create a `SelectTable` privilege role SecurableObject securableObject2 = @@ -732,8 +924,8 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(securableObject2)) .build(); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role2)); - verifyRoleInRanger(rangerAuthPlugin, role2); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role2)); + verifyRoleInRanger(rangerAuthHivePlugin, role2); // Create a `ModifyTable` privilege role SecurableObject securableObject3 = @@ -748,8 +940,8 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .withSecurableObjects(Lists.newArrayList(securableObject3)) .build(); - Assertions.assertTrue(rangerAuthPlugin.onRoleCreated(role3)); - verifyRoleInRanger(rangerAuthPlugin, role3); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleCreated(role3)); + verifyRoleInRanger(rangerAuthHivePlugin, role3); /** Test grant to user */ // granted role1 to the user1 @@ -763,11 +955,11 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); // multiple call to grant role1 to the user1 to test idempotent operation Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role1, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role1, Lists.newArrayList(userName1)); // granted role1 to the user2 String userName2 = "user2"; @@ -780,8 +972,8 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity2)); - verifyRoleInRanger(rangerAuthPlugin, role1, Lists.newArrayList(userName1, userName2)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity2)); + verifyRoleInRanger(rangerAuthHivePlugin, role1, Lists.newArrayList(userName1, userName2)); // granted role1 to the user3 String userName3 = "user3"; @@ -794,26 +986,26 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity3)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role1), userEntity3)); verifyRoleInRanger( - rangerAuthPlugin, role1, Lists.newArrayList(userName1, userName2, userName3)); + rangerAuthHivePlugin, role1, Lists.newArrayList(userName1, userName2, userName3)); // Same granted role2 and role3 to the user1 and user2 and user3 Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity1)); - verifyRoleInRanger(rangerAuthPlugin, role2, Lists.newArrayList(userName1)); - verifyRoleInRanger(rangerAuthPlugin, role3, Lists.newArrayList(userName1)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity1)); + verifyRoleInRanger(rangerAuthHivePlugin, role2, Lists.newArrayList(userName1)); + verifyRoleInRanger(rangerAuthHivePlugin, role3, Lists.newArrayList(userName1)); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity2)); - verifyRoleInRanger(rangerAuthPlugin, role2, Lists.newArrayList(userName1, userName2)); - verifyRoleInRanger(rangerAuthPlugin, role3, Lists.newArrayList(userName1, userName2)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity2)); + verifyRoleInRanger(rangerAuthHivePlugin, role2, Lists.newArrayList(userName1, userName2)); + verifyRoleInRanger(rangerAuthHivePlugin, role3, Lists.newArrayList(userName1, userName2)); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity3)); + rangerAuthHivePlugin.onGrantedRolesToUser(Lists.newArrayList(role2, role3), userEntity3)); verifyRoleInRanger( - rangerAuthPlugin, role2, Lists.newArrayList(userName1, userName2, userName3)); + rangerAuthHivePlugin, role2, Lists.newArrayList(userName1, userName2, userName3)); verifyRoleInRanger( - rangerAuthPlugin, role3, Lists.newArrayList(userName1, userName2, userName3)); + rangerAuthHivePlugin, role3, Lists.newArrayList(userName1, userName2, userName3)); /** Test grant to group */ // granted role1 to the group1 @@ -827,9 +1019,9 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity1)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, Lists.newArrayList(userName1, userName2, userName3), null, @@ -846,9 +1038,9 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity2)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, Lists.newArrayList(userName1, userName2, userName3), null, @@ -865,9 +1057,9 @@ public void testCombinationOperation() { .withAuditInfo(auditInfo) .build(); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity3)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role1), groupEntity3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, Lists.newArrayList(userName1, userName2, userName3), null, @@ -875,44 +1067,44 @@ public void testCombinationOperation() { // Same granted role2 and role3 to the group1 and group2 and group3 Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity1)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1)); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity2)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1, groupName2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1, groupName2)); Assertions.assertTrue( - rangerAuthPlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity3)); + rangerAuthHivePlugin.onGrantedRolesToGroup(Lists.newArrayList(role2, role3), groupEntity3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1, groupName2, groupName3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, Lists.newArrayList(userName1, userName2, userName3), null, @@ -921,9 +1113,9 @@ public void testCombinationOperation() { /** Test revoke from user */ // revoke role1 from the user1 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity1)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, Lists.newArrayList(userName2, userName3), Lists.newArrayList(userName1), @@ -931,9 +1123,9 @@ public void testCombinationOperation() { // revoke role1 from the user2 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity2)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, Lists.newArrayList(userName3), Lists.newArrayList(userName1, userName2), @@ -941,9 +1133,9 @@ public void testCombinationOperation() { // revoke role1 from the user3 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity3)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role1), userEntity3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, null, Lists.newArrayList(userName1, userName2, userName3), @@ -951,46 +1143,46 @@ public void testCombinationOperation() { // Same revoke role2 and role3 from the user1 and user2 and user3 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity1)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, Lists.newArrayList(userName2, userName3), Lists.newArrayList(userName1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, Lists.newArrayList(userName2, userName3), Lists.newArrayList(userName1)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity2)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, Lists.newArrayList(userName3), Lists.newArrayList(userName1, userName2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, Lists.newArrayList(userName3), Lists.newArrayList(userName1, userName2)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); + rangerAuthHivePlugin.onRevokedRolesFromUser(Lists.newArrayList(role2, role3), userEntity3)); verifyRoleInRanger( - rangerAuthPlugin, role2, null, Lists.newArrayList(userName1, userName2, userName3)); + rangerAuthHivePlugin, role2, null, Lists.newArrayList(userName1, userName2, userName3)); verifyRoleInRanger( - rangerAuthPlugin, role3, null, Lists.newArrayList(userName1, userName2, userName3)); + rangerAuthHivePlugin, role3, null, Lists.newArrayList(userName1, userName2, userName3)); /** Test revoke from group */ // revoke role1 from the group1 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity1)); + rangerAuthHivePlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, null, Lists.newArrayList(userName1, userName2, userName3), @@ -999,9 +1191,9 @@ public void testCombinationOperation() { // revoke role1 from the group2 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity2)); + rangerAuthHivePlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, null, Lists.newArrayList(userName1, userName2, userName3), @@ -1010,9 +1202,9 @@ public void testCombinationOperation() { // revoke role1 from the group3 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity3)); + rangerAuthHivePlugin.onRevokedRolesFromGroup(Lists.newArrayList(role1), groupEntity3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role1, null, Lists.newArrayList(userName1, userName2, userName3), @@ -1021,16 +1213,17 @@ public void testCombinationOperation() { // Same revoke role2 and role3 from the group1 and group2 and group3 Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role2, role3), groupEntity1)); + rangerAuthHivePlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, null, Lists.newArrayList(userName1, userName2, userName3), Lists.newArrayList(groupName2, groupName3), Lists.newArrayList(groupName1)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, null, Lists.newArrayList(userName1, userName2, userName3), @@ -1038,16 +1231,17 @@ public void testCombinationOperation() { Lists.newArrayList(groupName1)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role2, role3), groupEntity2)); + rangerAuthHivePlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, null, Lists.newArrayList(userName1, userName2, userName3), Lists.newArrayList(groupName3), Lists.newArrayList(groupName1, groupName2)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, null, Lists.newArrayList(userName1, userName2, userName3), @@ -1055,49 +1249,59 @@ public void testCombinationOperation() { Lists.newArrayList(groupName1, groupName2)); Assertions.assertTrue( - rangerAuthPlugin.onRevokedRolesFromGroup(Lists.newArrayList(role2, role3), groupEntity3)); + rangerAuthHivePlugin.onRevokedRolesFromGroup( + Lists.newArrayList(role2, role3), groupEntity3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role2, null, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1, groupName2, groupName3)); verifyRoleInRanger( - rangerAuthPlugin, + rangerAuthHivePlugin, role3, null, Lists.newArrayList(userName1, userName2, userName3), null, Lists.newArrayList(groupName1, groupName2, groupName3)); - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(role1)); - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(role2)); - Assertions.assertTrue(rangerAuthPlugin.onRoleDeleted(role3)); - // Because these metaobjects have owner, so the policy will not be deleted. - role1.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject))); - role2.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject))); - role3.securableObjects().stream() - .forEach( - securableObject -> - Assertions.assertNotNull(rangerHelper.findManagedPolicy(securableObject))); + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(role1)); + // Because role1's secrable object have owner, so the policy will not be deleted. + assertFindManagedPolicy(role1, true); + + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(role2)); + assertFindManagedPolicy(role2, true); + + Assertions.assertTrue(rangerAuthHivePlugin.onRoleDeleted(role3)); + assertFindManagedPolicy(role3, true); } - /** Verify the Gravitino role in Ranger service */ + private static String generatePolicyName(MetadataObject metadataObject) { + List nsMetadataObject = + Lists.newArrayList(SecurableObjects.DOT_SPLITTER.splitToList(metadataObject.fullName())); + if (!(metadataObject instanceof RangerSecurableObject) + && metadataObject.type() != MetadataObject.Type.METALAKE) { + nsMetadataObject.remove(0); // remove `catalog` + } + return String.join(".", nsMetadataObject); + } + + /** + * Verify the Gravitino role in Ranger service + * + *

metadataObject: the Gravitino securable object to be verified + */ private void verifyOwnerInRanger( MetadataObject metadataObject, List includeUsers, List excludeUsers, List includeGroups, - List excludeGroups) { + List excludeGroups, + List includeRoles, + List excludeRoles) { // Find policy by each metadata Object - String policyName = metadataObject.fullName(); + String policyName = generatePolicyName(metadataObject); RangerPolicy policy; try { policy = RangerITEnv.rangerClient.getPolicy(RangerITEnv.RANGER_HIVE_REPO_NAME, policyName); @@ -1113,7 +1317,11 @@ private void verifyOwnerInRanger( // verify namespace List metaObjNamespaces = Lists.newArrayList(DOT_SPLITTER.splitToList(metadataObject.fullName())); - metaObjNamespaces.remove(0); // skip catalog + if (!(metadataObject instanceof RangerSecurableObject) + && metadataObject.type() != MetadataObject.Type.METALAKE) { + metaObjNamespaces.remove(0); // skip catalog + } + List rolePolicies = new ArrayList<>(); for (int i = 0; i < metaObjNamespaces.size(); i++) { rolePolicies.add( @@ -1135,7 +1343,7 @@ private void verifyOwnerInRanger( return policyItem.getAccesses().stream() .anyMatch( access -> { - return rangerAuthPlugin + return rangerAuthHivePlugin .ownerMappingRule() .contains(RangerPrivileges.valueOf(access.getType())); }); @@ -1174,15 +1382,72 @@ private void verifyOwnerInRanger( return false; } } + if (includeRoles != null && !includeRoles.isEmpty()) { + if (!policyItem.getRoles().containsAll(includeRoles)) { + return false; + } + } + if (excludeRoles != null && !excludeRoles.isEmpty()) { + boolean containExcludeRole = + policyItem.getRoles().stream() + .anyMatch( + role -> { + return excludeRoles.contains(role); + }); + if (containExcludeRole) { + return false; + } + } return true; }); } private void verifyOwnerInRanger(MetadataObject metadataObject) { - verifyOwnerInRanger(metadataObject, null, null, null, null); + verifyOwnerInRanger(metadataObject, null, null, null, null, null, null); } private void verifyOwnerInRanger(MetadataObject metadataObject, List includeUsers) { - verifyOwnerInRanger(metadataObject, includeUsers, null, null, null); + verifyOwnerInRanger(metadataObject, includeUsers, null, null, null, null, null); + } + + private void verifyOwnerInRanger( + MetadataObject metadataObject, List includeUsers, List excludeUsers) { + verifyOwnerInRanger(metadataObject, includeUsers, excludeUsers, null, null, null, null); + } + + private void verifyOwnerInRanger( + MetadataObject metadataObject, + List includeUsers, + List excludeUsers, + List includeGroups) { + verifyOwnerInRanger( + metadataObject, includeUsers, excludeUsers, includeGroups, null, null, null); + } + + private void verifyOwnerInRanger( + MetadataObject metadataObject, + List includeUsers, + List excludeUsers, + List includeGroups, + List excludeGroups) { + verifyOwnerInRanger( + metadataObject, includeUsers, excludeUsers, includeGroups, excludeGroups, null, null); + } + + private void verifyOwnerInRanger( + MetadataObject metadataObject, + List includeUsers, + List excludeUsers, + List includeGroups, + List excludeGroups, + List includeRoles) { + verifyOwnerInRanger( + metadataObject, + includeUsers, + excludeUsers, + includeGroups, + excludeGroups, + includeRoles, + null); } } diff --git a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java index 563ec7fd1f7..1a5218b08e7 100644 --- a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java +++ b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java @@ -25,13 +25,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import org.apache.gravitino.authorization.Role; +import org.apache.gravitino.authorization.ranger.RangerAuthorizationHivePlugin; import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin; import org.apache.gravitino.authorization.ranger.RangerHelper; -import org.apache.gravitino.authorization.ranger.RangerPrivilege; +import org.apache.gravitino.authorization.ranger.RangerPrivileges; +import org.apache.gravitino.authorization.ranger.RangerSecurableObject; import org.apache.gravitino.authorization.ranger.reference.RangerDefines; +import org.apache.gravitino.connector.AuthorizationPropertiesMeta; import org.apache.gravitino.integration.test.container.ContainerSuite; import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.gravitino.integration.test.container.RangerContainer; @@ -76,11 +78,37 @@ public class RangerITEnv { public static final String SEARCH_FILTER_COLUMN = SearchFilter.RESOURCE_PREFIX + RESOURCE_COLUMN; // Search filter prefix file path constants public static final String SEARCH_FILTER_PATH = SearchFilter.RESOURCE_PREFIX + RESOURCE_PATH; + public static RangerAuthorizationPlugin rangerAuthHivePlugin; + protected static RangerHelper rangerHelper; - public static void setup() { + public static void init() { containerSuite.startRangerContainer(); rangerClient = containerSuite.getRangerContainer().rangerClient; + rangerAuthHivePlugin = + RangerAuthorizationHivePlugin.getInstance( + ImmutableMap.of( + AuthorizationPropertiesMeta.RANGER_ADMIN_URL, + String.format( + "http://%s:%d", + containerSuite.getRangerContainer().getContainerIpAddress(), + RangerContainer.RANGER_SERVER_PORT), + AuthorizationPropertiesMeta.RANGER_AUTH_TYPE, + RangerContainer.authType, + AuthorizationPropertiesMeta.RANGER_USERNAME, + RangerContainer.rangerUserName, + AuthorizationPropertiesMeta.RANGER_PASSWORD, + RangerContainer.rangerPassword, + AuthorizationPropertiesMeta.RANGER_SERVICE_NAME, + RangerITEnv.RANGER_HIVE_REPO_NAME)); + rangerHelper = + new RangerHelper( + rangerClient, + RangerContainer.rangerUserName, + RangerITEnv.RANGER_HIVE_REPO_NAME, + rangerAuthHivePlugin.ownerMappingRule(), + rangerAuthHivePlugin.policyResourceDefinesRule()); + if (!initRangerService) { synchronized (RangerITEnv.class) { // No IP address set, no impact on testing @@ -146,11 +174,11 @@ static void allowAnyoneAccessHDFS() { policyItem.setAccesses( Arrays.asList( new RangerPolicy.RangerPolicyItemAccess( - RangerPrivilege.RangerHdfsPrivilege.READ.toString()), + RangerPrivileges.RangerHdfsPrivilege.READ.toString()), new RangerPolicy.RangerPolicyItemAccess( - RangerPrivilege.RangerHdfsPrivilege.WRITE.toString()), + RangerPrivileges.RangerHdfsPrivilege.WRITE.toString()), new RangerPolicy.RangerPolicyItemAccess( - RangerPrivilege.RangerHdfsPrivilege.EXECUTE.toString()))); + RangerPrivileges.RangerHdfsPrivilege.EXECUTE.toString()))); updateOrCreateRangerPolicy( RANGER_HDFS_TYPE, RANGER_HDFS_REPO_NAME, @@ -187,7 +215,7 @@ static void allowAnyoneAccessInformationSchema() { policyItem.setAccesses( Arrays.asList( new RangerPolicy.RangerPolicyItemAccess( - RangerPrivilege.RangerHivePrivilege.SELECT.toString()))); + RangerPrivileges.RangerHivePrivilege.SELECT.toString()))); updateOrCreateRangerPolicy( RANGER_HIVE_TYPE, RANGER_HIVE_REPO_NAME, @@ -400,42 +428,42 @@ protected static void verifyRoleInRanger( role.securableObjects() .forEach( securableObject -> { - RangerPolicy policy; - try { - policy = - RangerITEnv.rangerClient.getPolicy( - RangerITEnv.RANGER_HIVE_REPO_NAME, securableObject.fullName()); - LOG.info("policy: " + policy.toString()); - } catch (RangerServiceException e) { - LOG.error("Failed to get policy: " + securableObject.fullName()); - throw new RuntimeException(e); - } - - securableObject - .privileges() - .forEach( - gravitinoPrivilege -> { - Set mappedPrivileges = - rangerAuthPlugin.translatePrivilege(gravitinoPrivilege.name()); - - boolean match = - policy.getPolicyItems().stream() - .filter( - policyItem -> { - // Filter Ranger policy item by Gravitino privilege - return policyItem.getAccesses().stream() - .anyMatch( - access -> { - return mappedPrivileges.contains(access.getType()); - }); - }) - .allMatch( - policyItem -> { - // Verify role name in Ranger policy item - return policyItem.getRoles().contains(role.name()); - }); - Assertions.assertTrue(match); - }); + List rangerSecurableObjects = + rangerAuthPlugin.translatePrivilege(securableObject); + + rangerSecurableObjects.forEach( + rangerSecurableObject -> { + RangerPolicy policy; + try { + policy = + RangerITEnv.rangerClient.getPolicy( + RangerITEnv.RANGER_HIVE_REPO_NAME, rangerSecurableObject.fullName()); + LOG.info("policy: " + policy.toString()); + } catch (RangerServiceException e) { + LOG.error("Failed to get policy: " + securableObject.fullName()); + throw new RuntimeException(e); + } + boolean match = + policy.getPolicyItems().stream() + .filter( + policyItem -> { + // Filter Ranger policy item by Gravitino privilege + return policyItem.getAccesses().stream() + .anyMatch( + access -> { + return rangerSecurableObject + .privileges() + .contains( + RangerPrivileges.valueOf(access.getType())); + }); + }) + .allMatch( + policyItem -> { + // Verify role name in Ranger policy item + return policyItem.getRoles().contains(role.name()); + }); + Assertions.assertTrue(match); + }); }); } diff --git a/build.gradle.kts b/build.gradle.kts index ff287db9110..b954aaf10e7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -512,6 +512,7 @@ tasks.rat { "DISCLAIMER.txt", "ROADMAP.md", "clients/client-python/.pytest_cache/*", + "clients/client-python/**/__pycache__", "clients/client-python/.venv/*", "clients/client-python/venv/*", "clients/client-python/apache_gravitino.egg-info/*", @@ -745,7 +746,9 @@ tasks { if (!it.name.startsWith("catalog") && !it.name.startsWith("authorization") && !it.name.startsWith("client") && !it.name.startsWith("filesystem") && !it.name.startsWith("spark") && !it.name.startsWith("iceberg") && it.name != "trino-connector" && - it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && it.name != "gcp-bundle" && it.name != "aliyun-bundle" + it.name != "integration-test" && it.name != "bundled-catalog" && !it.name.startsWith("flink") && + it.name != "integration-test" && it.name != "hive-metastore-common" && !it.name.startsWith("flink") && + it.name != "gcp-bundle" && it.name != "aliyun-bundle" && it.name != "aws-bundle" ) { from(it.configurations.runtimeClasspath) into("distribution/package/libs") @@ -764,8 +767,9 @@ tasks { !it.name.startsWith("integration-test") && !it.name.startsWith("flink") && !it.name.startsWith("trino-connector") && + it.name != "bundled-catalog" && it.name != "hive-metastore-common" && it.name != "gcp-bundle" && - it.name != "aliyun-bundle" + it.name != "aliyun-bundle" && it.name != "aws-bundle" ) { dependsOn("${it.name}:build") from("${it.name}/build/libs") diff --git a/bundles/aws-bundle/build.gradle.kts b/bundles/aws-bundle/build.gradle.kts new file mode 100644 index 00000000000..741bdc414e1 --- /dev/null +++ b/bundles/aws-bundle/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar + +plugins { + `maven-publish` + id("java") + alias(libs.plugins.shadow) +} + +dependencies { + compileOnly(project(":catalogs:catalog-hadoop")) + compileOnly(libs.hadoop3.common) + implementation(libs.hadoop3.aws) +} + +tasks.withType(ShadowJar::class.java) { + isZip64 = true + configurations = listOf(project.configurations.runtimeClasspath.get()) + archiveClassifier.set("") +} + +tasks.jar { + dependsOn(tasks.named("shadowJar")) + archiveClassifier.set("empty") +} + +tasks.compileJava { + dependsOn(":catalogs:catalog-hadoop:runtimeJars") +} diff --git a/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java new file mode 100644 index 00000000000..4ab1ca24212 --- /dev/null +++ b/bundles/aws-bundle/src/main/java/org/apache/gravitino/s3/fs/S3FileSystemProvider.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.s3.fs; + +import java.io.IOException; +import java.util.Map; +import org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.s3a.S3AFileSystem; + +public class S3FileSystemProvider implements FileSystemProvider { + @Override + public FileSystem getFileSystem(Path path, Map config) throws IOException { + Configuration configuration = new Configuration(); + config.forEach( + (k, v) -> { + configuration.set(k.replace("gravitino.bypass.", ""), v); + }); + + return S3AFileSystem.newInstance(path.toUri(), configuration); + } + + @Override + public String scheme() { + return "s3a"; + } + + @Override + public String name() { + return "s3"; + } +} diff --git a/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider b/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider new file mode 100644 index 00000000000..37a1a84c7ee --- /dev/null +++ b/bundles/aws-bundle/src/main/resources/META-INF/services/org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.gravitino.s3.fs.S3FileSystemProvider \ No newline at end of file diff --git a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java b/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java index 919baa03b19..74a70f0830c 100644 --- a/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java +++ b/bundles/gcp-bundle/src/main/java/org/apache/gravitino/gcs/fs/GCSFileSystemProvider.java @@ -25,8 +25,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class GCSFileSystemProvider implements FileSystemProvider { + private static final Logger LOGGER = LoggerFactory.getLogger(GCSFileSystemProvider.class); + @Override public FileSystem getFileSystem(Path path, Map config) throws IOException { Configuration configuration = new Configuration(); @@ -35,6 +39,7 @@ public FileSystem getFileSystem(Path path, Map config) throws IO configuration.set(k.replace("gravitino.bypass.", ""), v); }); + LOGGER.info("Creating GCS file system with config: {}", config); return GoogleHadoopFileSystem.newInstance(path.toUri(), configuration); } diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/auth/AuthProperties.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/auth/AuthProperties.java new file mode 100644 index 00000000000..2984ee2ba02 --- /dev/null +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/auth/AuthProperties.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.auth; + +public class AuthProperties { + + /** The configuration key for the Gravitino client auth type. */ + public static final String GRAVITINO_CLIENT_AUTH_TYPE = "authType"; + + public static final String SIMPLE_AUTH_TYPE = "simple"; + public static final String OAUTH2_AUTH_TYPE = "oauth2"; + public static final String KERBEROS_AUTH_TYPE = "kerberos"; + + // oauth2 + /** The configuration key for the URI of the default OAuth server. */ + public static final String GRAVITINO_OAUTH2_SERVER_URI = "oauth2.serverUri"; + + /** The configuration key for the client credential. */ + public static final String GRAVITINO_OAUTH2_CREDENTIAL = "oauth2.credential"; + + /** The configuration key for the path which to get the token. */ + public static final String GRAVITINO_OAUTH2_TOKEN_PATH = "oauth2.tokenPath"; + + /** The configuration key for the scope of the token. */ + public static final String GRAVITINO_OAUTH2_SCOPE = "oauth2.scope"; + + public static boolean isKerberos(String authType) { + return KERBEROS_AUTH_TYPE.equalsIgnoreCase(authType); + } + + public static boolean isOAuth2(String authType) { + return OAUTH2_AUTH_TYPE.equalsIgnoreCase(authType); + } + + public static boolean isSimple(String authType) { + return authType == null || SIMPLE_AUTH_TYPE.equalsIgnoreCase(authType); + } + + private AuthProperties() {} +} diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 4c091b14946..62a48656c39 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -73,6 +73,7 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":server")) testImplementation(project(":server-common")) + testImplementation(project(":bundles:aws-bundle")) testImplementation(project(":bundles:gcp-bundle")) testImplementation(project(":bundles:aliyun-bundle")) @@ -161,6 +162,11 @@ tasks.test { } else { dependsOn(tasks.jar) } + + // this task depends on :bundles:aws-bundle:jar + dependsOn(":bundles:aws-bundle:jar") + dependsOn(":bundles:aliyun-bundle:jar") + dependsOn(":bundles:gcp-bundle:jar") } tasks.getByName("generateMetadataFileForMavenJavaPublication") { diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java index ef8f37187e1..b272bd7a889 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopCatalogIT.java @@ -61,19 +61,24 @@ public class HadoopCatalogIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(HadoopCatalogIT.class); protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); - public String metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); - public String catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); - public static final String SCHEMA_PREFIX = "CatalogFilesetIT_schema"; - public String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); + protected String metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); + protected String catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); + public final String SCHEMA_PREFIX = "CatalogFilesetIT_schema"; + protected String schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); protected static final String provider = "hadoop"; - protected static GravitinoMetalake metalake; - protected static Catalog catalog; - protected static FileSystem fileSystem; - protected static String defaultBaseLocation; + protected GravitinoMetalake metalake; + protected Catalog catalog; + protected FileSystem fileSystem; + protected String defaultBaseLocation; + + protected void startNecessaryContainer() { + containerSuite.startHiveContainer(); + } @BeforeAll public void setup() throws IOException { - containerSuite.startHiveContainer(); + startNecessaryContainer(); + Configuration conf = new Configuration(); conf.set("fs.defaultFS", defaultBaseLocation()); fileSystem = FileSystem.get(conf); @@ -88,7 +93,7 @@ public void stop() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (fileSystem != null) { fileSystem.close(); } @@ -104,10 +109,9 @@ protected void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java index db1d01336ca..cca13b77047 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopGCSCatalogIT.java @@ -44,7 +44,7 @@ public class HadoopGCSCatalogIT extends HadoopCatalogIT { @Override public void startIntegrationTest() throws Exception { - // Do nothing. + // Just overwrite super, do nothing. } @BeforeAll diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopS3CatalogIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopS3CatalogIT.java new file mode 100644 index 00000000000..90b44139241 --- /dev/null +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopS3CatalogIT.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.hadoop.integration.test; + +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.net.URI; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +@Tag("gravitino-docker-test") +public class HadoopS3CatalogIT extends HadoopCatalogIT { + private static final Logger LOG = LoggerFactory.getLogger(HadoopOSSCatalogIT.class); + private String bucketName = "s3-bucket-" + UUID.randomUUID().toString().replace("-", ""); + private String accessKey; + private String secretKey; + private String s3Endpoint; + + private GravitinoLocalStackContainer gravitinoLocalStackContainer; + + @VisibleForTesting + public void startIntegrationTest() throws Exception {} + + @Override + protected void startNecessaryContainer() { + + containerSuite.startLocalStackContainer(); + gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> { + try { + Container.ExecResult result = + gravitinoLocalStackContainer.executeInContainer( + "awslocal", "iam", "create-user", "--user-name", "anonymous"); + return result.getExitCode() == 0; + } catch (Exception e) { + LOG.info("LocalStack is not ready yet for: ", e); + return false; + } + }); + + gravitinoLocalStackContainer.executeInContainer("awslocal", "s3", "mb", "s3://" + bucketName); + + Container.ExecResult result = + gravitinoLocalStackContainer.executeInContainer( + "awslocal", "iam", "create-access-key", "--user-name", "anonymous"); + + gravitinoLocalStackContainer.executeInContainer( + "awslocal", + "s3api", + "put-bucket-acl", + "--bucket", + "my-test-bucket", + "--acl", + "public-read-write"); + + // Get access key and secret key from result + String[] lines = result.getStdout().split("\n"); + accessKey = lines[3].split(":")[1].trim().substring(1, 21); + secretKey = lines[5].split(":")[1].trim().substring(1, 41); + + LOG.info("Access key: " + accessKey); + LOG.info("Secret key: " + secretKey); + + s3Endpoint = + String.format("http://%s:%d", gravitinoLocalStackContainer.getContainerIpAddress(), 4566); + } + + @BeforeAll + public void setup() throws IOException { + copyBundleJarsToHadoop("aws-bundle"); + + try { + super.startIntegrationTest(); + } catch (Exception e) { + throw new RuntimeException("Failed to start integration test", e); + } + + startNecessaryContainer(); + + metalakeName = GravitinoITUtils.genRandomName("CatalogFilesetIT_metalake"); + catalogName = GravitinoITUtils.genRandomName("CatalogFilesetIT_catalog"); + schemaName = GravitinoITUtils.genRandomName("CatalogFilesetIT_schema"); + + schemaName = GravitinoITUtils.genRandomName(SCHEMA_PREFIX); + Configuration conf = new Configuration(); + + conf.set("fs.s3a.access.key", accessKey); + conf.set("fs.s3a.secret.key", secretKey); + conf.set("fs.s3a.endpoint", s3Endpoint); + conf.set( + "fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + fileSystem = FileSystem.get(URI.create(String.format("s3a://%s", bucketName)), conf); + + createMetalake(); + createCatalog(); + createSchema(); + } + + @AfterAll + public void stop() throws IOException { + Catalog catalog = metalake.loadCatalog(catalogName); + catalog.asSchemas().dropSchema(schemaName, true); + metalake.dropCatalog(catalogName, true); + client.dropMetalake(metalakeName, true); + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Failed to close CloseableGroup", e); + } + } + + protected String defaultBaseLocation() { + if (defaultBaseLocation == null) { + try { + Path bucket = + new Path( + String.format( + "s3a://%s/%s", bucketName, GravitinoITUtils.genRandomName("CatalogFilesetIT"))); + if (!fileSystem.exists(bucket)) { + fileSystem.mkdirs(bucket); + } + + defaultBaseLocation = bucket.toString(); + } catch (IOException e) { + throw new RuntimeException("Failed to create default base location", e); + } + } + + return defaultBaseLocation; + } + + protected void createCatalog() { + Map map = Maps.newHashMap(); + map.put("gravitino.bypass.fs.s3a.access.key", accessKey); + map.put("gravitino.bypass.fs.s3a.secret.key", secretKey); + map.put("gravitino.bypass.fs.s3a.endpoint", s3Endpoint); + map.put( + "gravitino.bypass.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + map.put(FILESYSTEM_PROVIDERS, "s3"); + + metalake.createCatalog(catalogName, Catalog.Type.FILESET, provider, "comment", map); + + catalog = metalake.loadCatalog(catalogName); + } + + protected String generateLocation(String filesetName) { + return String.format("%s/%s", defaultBaseLocation, filesetName); + } +} diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java index d0de2972742..d074709bd80 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserAuthenticationIT.java @@ -654,6 +654,6 @@ void testUserImpersonation() { catalog.asFilesetCatalog().dropFileset(NameIdentifier.of(SCHEMA_NAME, filesetName)); catalog.asSchemas().dropSchema(SCHEMA_NAME, true); gravitinoMetalake.dropCatalog(catalogName, true); - adminClient.dropMetalake(metalakeName); + adminClient.dropMetalake(metalakeName, true); } } diff --git a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java index 9515b45b5dd..808aafbaef5 100644 --- a/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java +++ b/catalogs/catalog-hadoop/src/test/java/org/apache/gravitino/catalog/hadoop/integration/test/HadoopUserImpersonationIT.java @@ -258,10 +258,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-hive/build.gradle.kts b/catalogs/catalog-hive/build.gradle.kts index aca8959df13..f7d6e60c14d 100644 --- a/catalogs/catalog-hive/build.gradle.kts +++ b/catalogs/catalog-hive/build.gradle.kts @@ -128,7 +128,7 @@ dependencies { testImplementation(libs.testcontainers) testImplementation(libs.testcontainers.mysql) testImplementation(libs.testcontainers.localstack) - testImplementation(libs.hadoop2.s3) + testImplementation(libs.hadoop2.aws) testRuntimeOnly(libs.junit.jupiter.engine) } diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java index 31493d54ba2..903f3fe8a41 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/CatalogHiveIT.java @@ -229,7 +229,7 @@ public void stop() throws IOException { })); Arrays.stream(metalake.listCatalogs()) .forEach((catalogName -> metalake.dropCatalog(catalogName, true))); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } if (hiveClientPool != null) { hiveClientPool.close(); @@ -264,10 +264,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } @@ -1429,8 +1428,8 @@ void testDropAndRename() { client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); client.createMetalake(metalakeName2, "comment", Collections.emptyMap()); - client.dropMetalake(metalakeName1); - client.dropMetalake(metalakeName2); + client.dropMetalake(metalakeName1, true); + client.dropMetalake(metalakeName2, true); client.createMetalake(metalakeName1, "comment", Collections.emptyMap()); diff --git a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java index d328a44dc64..b7d61582efb 100644 --- a/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java +++ b/catalogs/catalog-hive/src/test/java/org/apache/gravitino/catalog/hive/integration/test/ProxyCatalogHiveIT.java @@ -389,10 +389,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); + client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(METALAKE_NAME); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(METALAKE_NAME, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index b08aa491600..96b92b6969d 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -126,7 +126,7 @@ public void startup() throws IOException { public void stop() { clearTableAndSchema(); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); } @AfterEach @@ -143,10 +143,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetaLakes = client.listMetalakes(); assertEquals(0, gravitinoMetaLakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - assertEquals(createdMetalake, loadMetalake); + assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java index 784361d6407..b65d82c2103 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/AuditCatalogMysqlIT.java @@ -77,7 +77,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void stopIntegrationTest() throws IOException, InterruptedException { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); mysqlService.close(); super.stopIntegrationTest(); } @@ -169,10 +169,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index 60af07011e8..f6b91b00ee4 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -144,6 +144,7 @@ public void stop() { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); mysqlService.close(); } @@ -167,10 +168,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java index 0d1292c67a2..f22073c7815 100644 --- a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java +++ b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java @@ -130,6 +130,7 @@ public void stop() { } metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); postgreSqlService.close(); } @@ -153,10 +154,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java index 907b00733bc..dc91a3dda57 100644 --- a/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java +++ b/catalogs/catalog-kafka/src/test/java/org/apache/gravitino/catalog/kafka/integration/test/CatalogKafkaIT.java @@ -126,6 +126,7 @@ public void shutdown() { metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); })); + client.disableMetalake(METALAKE_NAME); client.dropMetalake(METALAKE_NAME); if (adminClient != null) { adminClient.close(); @@ -554,10 +555,9 @@ private TopicDescription getTopicDesc(String topicName) } private void createMetalake() { - GravitinoMetalake createdMetalake = - client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); + client.createMetalake(METALAKE_NAME, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(METALAKE_NAME); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(METALAKE_NAME, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index 9fcc93451ca..7c5d93362f6 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -140,6 +140,7 @@ public void stop() throws Exception { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); } finally { if (spark != null) { @@ -197,10 +198,9 @@ private void createMetalake() { GravitinoMetalake[] gravitinoMetalakes = client.listMetalakes(); Assertions.assertEquals(0, gravitinoMetalakes.length); - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts b/catalogs/catalog-lakehouse-paimon/build.gradle.kts index 16a3382cfc5..6839cc163bc 100644 --- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts +++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts @@ -46,10 +46,9 @@ dependencies { exclude("com.sun.jersey") exclude("javax.servlet") exclude("org.apache.curator") - exclude("org.apache.hive") exclude("org.apache.hbase") exclude("org.apache.zookeeper") - exclude("org.eclipse.jetty.aggregate:jetty-all") + exclude("org.eclipse.jetty.aggregate") exclude("org.mortbay.jetty") exclude("org.mortbay.jetty:jetty") exclude("org.mortbay.jetty:jetty-util") @@ -67,9 +66,40 @@ dependencies { exclude("org.apache.parquet:parquet-encoding") exclude("org.apache.parquet:parquet-common") exclude("org.apache.parquet:parquet-hadoop") + exclude("org.apache.parquet:parquet-hadoop-bundle") exclude("org.apache.paimon:paimon-codegen-loader") exclude("org.apache.paimon:paimon-shade-caffeine-2") exclude("org.apache.paimon:paimon-shade-guava-30") + exclude("org.apache.hive:hive-service-rpc") + exclude("org.apache.logging.log4j") + exclude("com.google.guava") + exclude("commons-lang") + exclude("org.slf4j") + exclude("org.apache.orc") + exclude("org.apache.httpcomponents") + exclude("jline") + exclude("org.eclipse.jetty.orbit") + exclude("org.apache.ant") + exclude("com.tdunning") + exclude("io.dropwizard.metrics") + exclude("com.github.joshelser") + exclude("commons-codec") + exclude("commons-cli") + exclude("tomcat") + exclude("org.apache.avro") + exclude("net.sf.opencsv") + exclude("javolution") + exclude("com.jolbox") + exclude("com.zaxxer") + exclude("org.apache.derby") + exclude("org.datanucleus") + exclude("commons-pool") + exclude("commons-dbcp") + exclude("javax.jdo") + exclude("org.antlr") + exclude("co.cask.tephra") + exclude("com.google.code.findbugs") + exclude("com.github.spotbugs") } implementation(libs.bundles.log4j) implementation(libs.commons.lang3) diff --git a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java index 355a79f5850..7371c5be36f 100644 --- a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java +++ b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java @@ -21,5 +21,6 @@ /** The type of Apache Paimon catalog backend. */ public enum PaimonCatalogBackend { FILESYSTEM, - JDBC + JDBC, + HIVE } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java index ed90745a785..668cd404e91 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonBaseIT.java @@ -99,6 +99,7 @@ public abstract class CatalogPaimonBaseIT extends BaseIT { protected String jdbcPassword; protected Catalog catalog; protected org.apache.paimon.catalog.Catalog paimonCatalog; + protected SparkSession spark; protected String metalakeName = GravitinoITUtils.genRandomName("paimon_it_metalake"); protected String catalogName = GravitinoITUtils.genRandomName("paimon_it_catalog"); protected String schemaName = GravitinoITUtils.genRandomName("paimon_it_schema"); @@ -115,8 +116,8 @@ public abstract class CatalogPaimonBaseIT extends BaseIT { private static final String alertTableName = "alert_table_name"; private static String INSERT_BATCH_WITHOUT_PARTITION_TEMPLATE = "INSERT INTO paimon.%s VALUES %s"; private static final String SELECT_ALL_TEMPLATE = "SELECT * FROM paimon.%s"; + private static final String DEFAULT_DB = "default"; private GravitinoMetalake metalake; - protected SparkSession spark; private Map catalogProperties; @BeforeAll @@ -138,6 +139,7 @@ public void stop() { clearTableAndSchema(); metalake.disableCatalog(catalogName); metalake.dropCatalog(catalogName); + client.disableMetalake(metalakeName); client.dropMetalake(metalakeName); if (spark != null) { spark.close(); @@ -727,7 +729,7 @@ public void testAlterPaimonTable() { // update column position Column col1 = Column.of("name", Types.StringType.get(), "comment"); Column col2 = Column.of("address", Types.StringType.get(), "comment"); - Column col3 = Column.of("date_of_birth", Types.DateType.get(), "comment"); + Column col3 = Column.of("date_of_birth", Types.StringType.get(), "comment"); Column[] newColumns = new Column[] {col1, col2, col3}; NameIdentifier tableIdentifier = @@ -874,14 +876,19 @@ void testOperationDataOfPaimonTable() { private void clearTableAndSchema() { SupportsSchemas supportsSchema = catalog.asSchemas(); Arrays.stream(supportsSchema.listSchemas()) - .forEach(schema -> supportsSchema.dropSchema(schema, true)); + .forEach( + schema -> { + // can not drop default database for hive backend. + if (!DEFAULT_DB.equalsIgnoreCase(schema)) { + supportsSchema.dropSchema(schema, true); + } + }); } private void createMetalake() { - GravitinoMetalake createdMetalake = - client.createMetalake(metalakeName, "comment", Collections.emptyMap()); + client.createMetalake(metalakeName, "comment", Collections.emptyMap()); GravitinoMetalake loadMetalake = client.loadMetalake(metalakeName); - Assertions.assertEquals(createdMetalake, loadMetalake); + Assertions.assertEquals(metalakeName, loadMetalake.name()); metalake = loadMetalake; } diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java new file mode 100644 index 00000000000..fcb220a8806 --- /dev/null +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonHiveIT.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse.paimon.integration.test; + +import com.google.common.collect.Maps; +import java.util.Map; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Schema; +import org.apache.gravitino.SupportsSchemas; +import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata; +import org.apache.gravitino.integration.test.container.HiveContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.paimon.catalog.Catalog; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +@Tag("gravitino-docker-test") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class CatalogPaimonHiveIT extends CatalogPaimonBaseIT { + + @Override + protected Map initPaimonCatalogProperties() { + Map catalogProperties = Maps.newHashMap(); + catalogProperties.put("key1", "val1"); + catalogProperties.put("key2", "val2"); + + TYPE = "hive"; + WAREHOUSE = + String.format( + "hdfs://%s:%d/user/hive/warehouse-catalog-paimon/", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HDFS_DEFAULTFS_PORT); + URI = + String.format( + "thrift://%s:%d", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT); + + catalogProperties.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND, TYPE); + catalogProperties.put(PaimonCatalogPropertiesMetadata.WAREHOUSE, WAREHOUSE); + catalogProperties.put(PaimonCatalogPropertiesMetadata.URI, URI); + + return catalogProperties; + } + + @Test + void testPaimonSchemaProperties() throws Catalog.DatabaseNotExistException { + SupportsSchemas schemas = catalog.asSchemas(); + + // create schema check. + String testSchemaName = GravitinoITUtils.genRandomName("test_schema_1"); + NameIdentifier schemaIdent = NameIdentifier.of(metalakeName, catalogName, testSchemaName); + Map schemaProperties = Maps.newHashMap(); + schemaProperties.put("key", "hive"); + Schema createdSchema = + schemas.createSchema(schemaIdent.name(), schema_comment, schemaProperties); + Assertions.assertEquals(createdSchema.properties().get("key"), "hive"); + + // load schema check. + Schema schema = schemas.loadSchema(schemaIdent.name()); + Assertions.assertEquals(schema.properties().get("key"), "hive"); + Map loadedProps = paimonCatalog.loadDatabaseProperties(schemaIdent.name()); + Assertions.assertEquals(loadedProps.get("key"), "hive"); + } +} diff --git a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java index e8fe66551ba..d1b50d52073 100644 --- a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java +++ b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestCatalogUtils.java @@ -29,9 +29,12 @@ import java.util.function.Consumer; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogBackend; import org.apache.gravitino.catalog.lakehouse.paimon.PaimonConfig; +import org.apache.gravitino.integration.test.container.ContainerSuite; +import org.apache.gravitino.integration.test.container.HiveContainer; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.FileSystemCatalog; import org.apache.paimon.factories.FactoryException; +import org.apache.paimon.hive.HiveCatalog; import org.apache.paimon.jdbc.JdbcCatalog; import org.junit.jupiter.api.Test; @@ -44,6 +47,8 @@ void testLoadCatalogBackend() throws Exception { assertCatalog(PaimonCatalogBackend.FILESYSTEM.name(), FileSystemCatalog.class); // Test load JdbcCatalog for jdbc metastore. assertCatalog(PaimonCatalogBackend.JDBC.name(), JdbcCatalog.class); + // Test load HiveCatalog for hive metastore. + assertCatalog(PaimonCatalogBackend.HIVE.name(), HiveCatalog.class); // Test load catalog exception for other metastore. assertThrowsExactly(FactoryException.class, () -> assertCatalog("other", catalog -> {})); } @@ -66,7 +71,7 @@ private void assertCatalog(String metastore, Consumer consumer) throws System.getProperty("java.io.tmpdir"), "paimon_catalog_warehouse"), PaimonConfig.CATALOG_URI.getKey(), - "jdbc:h2:mem:testdb", + generateUri(metastore), PaimonConfig.CATALOG_JDBC_USER.getKey(), "user", PaimonConfig.CATALOG_JDBC_PASSWORD.getKey(), @@ -75,4 +80,20 @@ private void assertCatalog(String metastore, Consumer consumer) throws consumer.accept(catalog); } } + + private static String generateUri(String metastore) { + String uri = "uri"; + if (PaimonCatalogBackend.JDBC.name().equalsIgnoreCase(metastore)) { + uri = "jdbc:h2:mem:testdb"; + } else if (PaimonCatalogBackend.HIVE.name().equalsIgnoreCase(metastore)) { + ContainerSuite containerSuite = ContainerSuite.getInstance(); + containerSuite.startHiveContainer(); + uri = + String.format( + "thrift://%s:%d", + containerSuite.getHiveContainer().getContainerIpAddress(), + HiveContainer.HIVE_METASTORE_PORT); + } + return uri; + } } diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java index db45b643612..3cc8df1d242 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/ErrorHandlers.java @@ -39,6 +39,8 @@ import org.apache.gravitino.exceptions.IllegalRoleException; import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchFilesetException; import org.apache.gravitino.exceptions.NoSuchGroupException; @@ -281,6 +283,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -329,6 +336,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -377,6 +389,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -428,6 +445,9 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogInUseException.class.getSimpleName())) { throw new CatalogInUseException(errorMessage); + } else if (errorResponse.getType().equals(MetalakeInUseException.class.getSimpleName())) { + throw new MetalakeInUseException(errorMessage); + } else { throw new InUseException(errorMessage); } @@ -436,6 +456,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -468,6 +493,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); + case ErrorConstants.IN_USE_CODE: + throw new MetalakeInUseException(errorMessage); + default: super.accept(errorResponse); } @@ -562,6 +590,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -608,6 +641,11 @@ public void accept(ErrorResponse errorResponse) { if (errorResponse.getType().equals(CatalogNotInUseException.class.getSimpleName())) { throw new CatalogNotInUseException(errorMessage); + } else if (errorResponse + .getType() + .equals(MetalakeNotInUseException.class.getSimpleName())) { + throw new MetalakeNotInUseException(errorMessage); + } else { throw new NotInUseException(errorMessage); } @@ -647,6 +685,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -685,6 +726,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -738,6 +782,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.FORBIDDEN_CODE: throw new ForbiddenException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -788,6 +835,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -831,6 +881,9 @@ public void accept(ErrorResponse errorResponse) { throw new AlreadyExistsException(errorMessage); } + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); @@ -864,6 +917,9 @@ public void accept(ErrorResponse errorResponse) { case ErrorConstants.UNSUPPORTED_OPERATION_CODE: throw new UnsupportedOperationException(errorMessage); + case ErrorConstants.NOT_IN_USE_CODE: + throw new MetalakeNotInUseException(errorMessage); + case ErrorConstants.INTERNAL_ERROR_CODE: throw new RuntimeException(errorMessage); diff --git a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java index 936edcf3c2a..7a4530235c9 100644 --- a/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java +++ b/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoAdminClient.java @@ -22,19 +22,24 @@ import com.google.common.base.Preconditions; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.SupportsMetalakes; import org.apache.gravitino.dto.requests.MetalakeCreateRequest; +import org.apache.gravitino.dto.requests.MetalakeSetRequest; import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; import org.apache.gravitino.dto.responses.DropResponse; +import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Apache Gravitino Client for the administrator to interact with the Gravitino API, allowing the @@ -145,17 +150,35 @@ public GravitinoMetalake alterMetalake(String name, MetalakeChange... changes) } /** - * Drops a specific Metalake using the Gravitino API. + * Drop a metalake with specified name. If the force flag is true, it will: * - * @param name The name of the Metalake to be dropped. - * @return True if the Metalake was successfully dropped, false if the Metalake does not exist. + *

    + *
  • Cascade drop all sub-entities (tags, catalogs, schemas, tables, etc.) of the metalake in + * Gravitino store. + *
  • Drop the metalake even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * deleted unless it is managed (such as managed fileset). + *
+ * + * If the force flag is false, it is equivalent to calling {@link #dropMetalake(String)}. + * + * @param name The name of the metalake. + * @param force Whether to force the drop. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. */ @Override - public boolean dropMetalake(String name) { + public boolean dropMetalake(String name, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { checkMetalakeName(name); + Map params = new HashMap<>(); + params.put("force", String.valueOf(force)); + DropResponse resp = restClient.delete( API_METALAKES_IDENTIFIER_PATH + name, + params, DropResponse.class, Collections.emptyMap(), ErrorHandlers.metalakeErrorHandler()); @@ -163,6 +186,44 @@ public boolean dropMetalake(String name) { return resp.dropped(); } + @Override + public void enableMetalake(String name) throws NoSuchMetalakeException { + MetalakeSetRequest req = new MetalakeSetRequest(true); + + ErrorResponse resp = + restClient.patch( + API_METALAKES_IDENTIFIER_PATH + name, + req, + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + + @Override + public void disableMetalake(String name) throws NoSuchMetalakeException { + MetalakeSetRequest req = new MetalakeSetRequest(false); + + ErrorResponse resp = + restClient.patch( + API_METALAKES_IDENTIFIER_PATH + name, + req, + ErrorResponse.class, + Collections.emptyMap(), + ErrorHandlers.metalakeErrorHandler()); + + if (resp.getCode() == 0) { + return; + } + + ErrorHandlers.metalakeErrorHandler().accept(resp); + } + /** * Creates a new builder for constructing a GravitinoClient. * diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java index c438e0fca1e..fd1fac54f2f 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/AuditIT.java @@ -60,6 +60,7 @@ public void testAuditMetalake() { metaLake = client.alterMetalake(metalakeAuditName, changes); Assertions.assertEquals(expectUser, metaLake.auditInfo().creator()); Assertions.assertEquals(expectUser, metaLake.auditInfo().lastModifier()); + Assertions.assertDoesNotThrow(() -> client.disableMetalake(newName)); Assertions.assertTrue(client.dropMetalake(newName), "metaLake should be dropped"); Assertions.assertFalse(client.dropMetalake(newName), "metalake should be non-existent"); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java index a29ef732094..5360f9f7816 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/CatalogIT.java @@ -81,7 +81,7 @@ public void startUp() { @AfterAll public void tearDown() { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java index fb9efd2ca7f..2922154f319 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/MetalakeIT.java @@ -18,6 +18,7 @@ */ package org.apache.gravitino.client.integration.test; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -182,6 +183,7 @@ public void testCreateMetalakeWithChinese() { public void testDropMetalakes() { GravitinoMetalake metalakeA = client.createMetalake(metalakeNameA, "metalake A comment", Collections.emptyMap()); + assertDoesNotThrow(() -> client.disableMetalake(metalakeA.name())); assertTrue(client.dropMetalake(metalakeA.name()), "metaLake should be dropped"); NameIdentifier id = NameIdentifier.of(metalakeNameA); assertThrows( @@ -205,12 +207,13 @@ public void testUpdateMetalakeWithNullableComment() { new MetalakeChange[] {MetalakeChange.updateComment("new metalake comment")}; GravitinoMetalake updatedMetalake = client.alterMetalake(metalakeNameA, changes); assertEquals("new metalake comment", updatedMetalake.comment()); - client.dropMetalake(metalakeNameA); + assertTrue(client.dropMetalake(metalakeNameA, true)); } public void dropMetalakes() { GravitinoMetalake[] metaLakes = client.listMetalakes(); for (GravitinoMetalake metalake : metaLakes) { + assertDoesNotThrow(() -> client.disableMetalake(metalake.name())); assertTrue(client.dropMetalake(metalake.name())); } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java index f5b5e1dab56..dc82dfd67df 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/TagIT.java @@ -112,7 +112,7 @@ public void tearDown() { relationalCatalog.asTableCatalog().dropTable(NameIdentifier.of(schema.name(), table.name())); relationalCatalog.asSchemas().dropSchema(schema.name(), true); metalake.dropCatalog(relationalCatalog.name(), true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java index a6817b27418..1dcdd27a3e0 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/AccessControlNotAllowIT.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.gravitino.integration.test.authorization; +package org.apache.gravitino.client.integration.test.authorization; import com.google.common.collect.Lists; import java.util.Collections; @@ -148,6 +148,7 @@ public void testNotAllowFilter() { Assertions.assertTrue( e.getMessage().contains("You should set 'gravitino.authorization.enable'")); + client.disableMetalake(metalakeTestName); client.dropMetalake(metalakeTestName); } } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java index f5615beae06..a7339ba0db1 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/CheckCurrentUserIT.java @@ -98,7 +98,7 @@ public void startIntegrationTest() throws Exception { @AfterAll public void tearDown() { if (client != null) { - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); client.close(); client = null; } diff --git a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java index 99f1e830692..daac8002ab9 100644 --- a/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java +++ b/clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/OwnerIT.java @@ -169,7 +169,7 @@ public void testCreateFileset() { catalog.asFilesetCatalog().dropFileset(fileIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameA, true); - client.dropMetalake(metalakeNameA); + client.dropMetalake(metalakeNameA, true); } @Test @@ -220,7 +220,7 @@ public void testCreateTopic() { // Clean up catalogB.asTopicCatalog().dropTopic(topicIdent); metalake.dropCatalog(catalogNameB, true); - client.dropMetalake(metalakeNameB); + client.dropMetalake(metalakeNameB, true); } @Test @@ -255,7 +255,7 @@ public void testCreateRole() { // Clean up metalake.deleteRole("role_owner"); - client.dropMetalake(metalakeNameC); + client.dropMetalake(metalakeNameC, true); } @Test @@ -321,7 +321,7 @@ public void testCreateTable() { catalog.asTableCatalog().dropTable(tableIdent); catalog.asSchemas().dropSchema("schema_owner", true); metalake.dropCatalog(catalogNameD, true); - client.dropMetalake(metalakeNameD); + client.dropMetalake(metalakeNameD, true); } @Test @@ -352,6 +352,6 @@ public void testOwnerWithException() { () -> metalake.setOwner(metalakeObject, "not-existed", Owner.Type.USER)); // Cleanup - client.dropMetalake(metalakeNameE); + client.dropMetalake(metalakeNameE, true); } } diff --git a/clients/client-python/gravitino/client/gravitino_admin_client.py b/clients/client-python/gravitino/client/gravitino_admin_client.py index b730a765b21..85d9ff2f047 100644 --- a/clients/client-python/gravitino/client/gravitino_admin_client.py +++ b/clients/client-python/gravitino/client/gravitino_admin_client.py @@ -22,6 +22,7 @@ from gravitino.client.gravitino_metalake import GravitinoMetalake from gravitino.dto.dto_converters import DTOConverters from gravitino.dto.requests.metalake_create_request import MetalakeCreateRequest +from gravitino.dto.requests.metalake_set_request import MetalakeSetRequest from gravitino.dto.requests.metalake_updates_request import MetalakeUpdatesRequest from gravitino.dto.responses.drop_response import DropResponse from gravitino.dto.responses.metalake_list_response import MetalakeListResponse @@ -112,20 +113,59 @@ def alter_metalake(self, name: str, *changes: MetalakeChange) -> GravitinoMetala return GravitinoMetalake(metalake, self._rest_client) - def drop_metalake(self, name: str) -> bool: + def drop_metalake(self, name: str, force: bool = False) -> bool: """Drops a specific Metalake using the Gravitino API. Args: name: The name of the Metalake to be dropped. + force: Whether to force the drop operation. Returns: - True if the Metalake was successfully dropped, false otherwise. + True if the Metalake was successfully dropped, false if the Metalake does not exist. """ + params = {"force": str(force)} resp = self._rest_client.delete( self.API_METALAKES_IDENTIFIER_PATH + name, + params=params, error_handler=METALAKE_ERROR_HANDLER, ) drop_response = DropResponse.from_json(resp.body, infer_missing=True) return drop_response.dropped() + + def enable_metalake(self, name: str): + """Enable the metalake with specified name. If the metalake is already in use, this method does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + + metalake_enable_request = MetalakeSetRequest(in_use=True) + metalake_enable_request.validate() + + url = self.API_METALAKES_IDENTIFIER_PATH + name + self._rest_client.patch( + url, json=metalake_enable_request, error_handler=METALAKE_ERROR_HANDLER + ) + + def disable_metalake(self, name: str): + """Disable the metalake with specified name. If the metalake is already disabled, does nothing. + + Args: + name: the name of the metalake. + + Raises: + NoSuchMetalakeException if the metalake with specified name does not exist. + """ + + metalake_disable_request = MetalakeSetRequest(in_use=False) + metalake_disable_request.validate() + + url = self.API_METALAKES_IDENTIFIER_PATH + name + self._rest_client.patch( + url, json=metalake_disable_request, error_handler=METALAKE_ERROR_HANDLER + ) diff --git a/clients/client-python/gravitino/dto/requests/metalake_set_request.py b/clients/client-python/gravitino/dto/requests/metalake_set_request.py new file mode 100644 index 00000000000..a7663b0ed6c --- /dev/null +++ b/clients/client-python/gravitino/dto/requests/metalake_set_request.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field + +from dataclasses_json import config + +from gravitino.rest.rest_message import RESTRequest + + +@dataclass +class MetalakeSetRequest(RESTRequest): + """Represents a request to set a metalake in use status.""" + + _in_use: bool = field(metadata=config(field_name="inUse")) + + def __init__(self, in_use: bool): + self._in_use = in_use + + def validate(self): + """Validates the fields of the request. + + Raises: + IllegalArgumentException if in_use is not set. + """ + if self._in_use is None: + raise ValueError('"in_use" field is required and cannot be empty') diff --git a/clients/client-python/gravitino/exceptions/base.py b/clients/client-python/gravitino/exceptions/base.py index 2dff76ac4ad..cd71de2368c 100644 --- a/clients/client-python/gravitino/exceptions/base.py +++ b/clients/client-python/gravitino/exceptions/base.py @@ -97,6 +97,10 @@ class InUseException(GravitinoRuntimeException): """Base class for all exceptions thrown when an entity is in use and cannot be deleted.""" +class MetalakeInUseException(InUseException): + """An exception thrown when a metalake is in use and cannot be deleted.""" + + class CatalogInUseException(InUseException): """An Exception thrown when a catalog is in use and cannot be deleted.""" @@ -105,6 +109,10 @@ class NotInUseException(GravitinoRuntimeException): """Base class for all exceptions thrown when an entity is not in use.""" +class MetalakeNotInUseException(NotInUseException): + """An exception thrown when operating on not in use metalake.""" + + class CatalogNotInUseException(NotInUseException): """An exception thrown when operating on not in use catalog.""" diff --git a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py index 86dc575702e..60a7dd0ac36 100644 --- a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py +++ b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py @@ -21,6 +21,8 @@ from gravitino.exceptions.base import ( NoSuchMetalakeException, MetalakeAlreadyExistsException, + MetalakeInUseException, + MetalakeNotInUseException, ) @@ -33,9 +35,16 @@ def handle(self, error_response: ErrorResponse): if code == ErrorConstants.NOT_FOUND_CODE: raise NoSuchMetalakeException(error_message) + if code == ErrorConstants.ALREADY_EXISTS_CODE: raise MetalakeAlreadyExistsException(error_message) + if code == ErrorConstants.IN_USE_CODE: + raise MetalakeInUseException(error_message) + + if code == ErrorConstants.NOT_IN_USE_CODE: + raise MetalakeNotInUseException(error_message) + super().handle(error_response) diff --git a/clients/client-python/gravitino/filesystem/gvfs.py b/clients/client-python/gravitino/filesystem/gvfs.py index e5a565ce0d6..8f1b2008ab9 100644 --- a/clients/client-python/gravitino/filesystem/gvfs.py +++ b/clients/client-python/gravitino/filesystem/gvfs.py @@ -14,11 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import os from enum import Enum from pathlib import PurePosixPath from typing import Dict, Tuple import re +import importlib import fsspec from cachetools import TTLCache, LRUCache @@ -26,7 +27,7 @@ from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.arrow import ArrowFSWrapper from fsspec.utils import infer_storage_options -from pyarrow.fs import HadoopFileSystem + from readerwriterlock import rwlock from gravitino.audit.caller_context import CallerContext, CallerContextHolder from gravitino.audit.fileset_audit_constants import FilesetAuditConstants @@ -47,6 +48,7 @@ class StorageType(Enum): HDFS = "hdfs" LOCAL = "file" + GCS = "gs" class FilesetContextPair: @@ -66,7 +68,7 @@ def filesystem(self): class GravitinoVirtualFileSystem(fsspec.AbstractFileSystem): - """This is a virtual file system which users can access `fileset` and + """This is a virtual file system that users can access `fileset` and other resources. It obtains the actual storage location corresponding to the resource from the @@ -149,6 +151,7 @@ def __init__( self._cache_lock = rwlock.RWLockFair() self._catalog_cache = LRUCache(maxsize=100) self._catalog_cache_lock = rwlock.RWLockFair() + self._options = options super().__init__(**kwargs) @@ -309,7 +312,9 @@ def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): ) dst_actual_path = dst_context_pair.actual_file_location() - if storage_type == StorageType.HDFS: + # convert the following to in + + if storage_type in [StorageType.HDFS, StorageType.GCS]: src_context_pair.filesystem().mv( self._strip_storage_protocol(storage_type, src_actual_path), self._strip_storage_protocol(storage_type, dst_actual_path), @@ -540,7 +545,11 @@ def _convert_actual_path( :param virtual_location: Virtual location :return A virtual path """ - if storage_location.startswith(f"{StorageType.HDFS.value}://"): + + # If the storage path starts with hdfs, gcs, we should use the path as the prefix. + if storage_location.startswith( + f"{StorageType.HDFS.value}://" + ) or storage_location.startswith(f"{StorageType.GCS.value}://"): actual_prefix = infer_storage_options(storage_location)["path"] elif storage_location.startswith(f"{StorageType.LOCAL.value}:/"): actual_prefix = storage_location[len(f"{StorageType.LOCAL.value}:") :] @@ -681,6 +690,8 @@ def _recognize_storage_type(path: str): return StorageType.HDFS if path.startswith(f"{StorageType.LOCAL.value}:/"): return StorageType.LOCAL + if path.startswith(f"{StorageType.GCS.value}://"): + return StorageType.GCS raise GravitinoRuntimeException( f"Storage type doesn't support now. Path:{path}" ) @@ -705,10 +716,11 @@ def _strip_storage_protocol(storage_type: StorageType, path: str): :param path: The path :return: The stripped path """ - if storage_type == StorageType.HDFS: + if storage_type in (StorageType.HDFS, StorageType.GCS): return path if storage_type == StorageType.LOCAL: return path[len(f"{StorageType.LOCAL.value}:") :] + raise GravitinoRuntimeException( f"Storage type:{storage_type} doesn't support now." ) @@ -774,9 +786,12 @@ def _get_filesystem(self, actual_file_location: str): if cache_value is not None: return cache_value if storage_type == StorageType.HDFS: - fs = ArrowFSWrapper(HadoopFileSystem.from_uri(actual_file_location)) + fs_class = importlib.import_module("pyarrow.fs").HadoopFileSystem + fs = ArrowFSWrapper(fs_class.from_uri(actual_file_location)) elif storage_type == StorageType.LOCAL: fs = LocalFileSystem() + elif storage_type == StorageType.GCS: + fs = ArrowFSWrapper(self._get_gcs_filesystem()) else: raise GravitinoRuntimeException( f"Storage type: `{storage_type}` doesn't support now." @@ -786,5 +801,23 @@ def _get_filesystem(self, actual_file_location: str): finally: write_lock.release() + def _get_gcs_filesystem(self): + # get All keys from the options that start with 'gravitino.bypass.gcs.' and remove the prefix + gcs_options = { + key[len(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_GCS) :]: value + for key, value in self._options.items() + if key.startswith(GVFSConfig.GVFS_FILESYSTEM_BY_PASS_GCS) + } + + # get 'service-account-key' from gcs_options, if the key is not found, throw an exception + service_account_key_path = gcs_options.get(GVFSConfig.GVFS_FILESYSTEM_KEY_FILE) + if service_account_key_path is None: + raise GravitinoRuntimeException( + "Service account key is not found in the options." + ) + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = service_account_key_path + + return importlib.import_module("pyarrow.fs").GcsFileSystem() + fsspec.register_implementation(PROTOCOL_NAME, GravitinoVirtualFileSystem) diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index eb5733b56be..618565c70eb 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -31,3 +31,7 @@ class GVFSConfig: OAUTH2_CREDENTIAL = "oauth2_credential" OAUTH2_PATH = "oauth2_path" OAUTH2_SCOPE = "oauth2_scope" + + GVFS_FILESYSTEM_BY_PASS = "gravitino.bypass" + GVFS_FILESYSTEM_BY_PASS_GCS = "gravitino.bypass.gcs." + GVFS_FILESYSTEM_KEY_FILE = "service-account-key-path" diff --git a/clients/client-python/requirements.txt b/clients/client-python/requirements.txt index 7242082b77c..a330f738a1f 100644 --- a/clients/client-python/requirements.txt +++ b/clients/client-python/requirements.txt @@ -22,4 +22,5 @@ dataclasses-json==0.6.6 readerwriterlock==1.0.9 fsspec==2024.3.1 pyarrow==15.0.2 -cachetools==5.3.3 \ No newline at end of file +cachetools==5.3.3 +google-auth==2.35.0 \ No newline at end of file diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py b/clients/client-python/tests/integration/auth/test_auth_common.py index 2f1506319a7..ede3a4e2ad2 100644 --- a/clients/client-python/tests/integration/auth/test_auth_common.py +++ b/clients/client-python/tests/integration/auth/test_auth_common.py @@ -101,7 +101,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/integration_test_env.py b/clients/client-python/tests/integration/integration_test_env.py index cfe6c0eda09..d0d39a06da7 100644 --- a/clients/client-python/tests/integration/integration_test_env.py +++ b/clients/client-python/tests/integration/integration_test_env.py @@ -21,6 +21,7 @@ import subprocess import time import sys +import shutil import requests @@ -80,6 +81,12 @@ def setUpClass(cls): ) sys.exit(0) + # remove data dir under gravitino_home + data_dir = os.path.join(cls.gravitino_home, "data") + if os.path.exists(data_dir): + logger.info("Remove Gravitino data directory: %s", data_dir) + shutil.rmtree(data_dir) + logger.info("Starting integration test environment...") # Start Gravitino Server @@ -141,6 +148,12 @@ def restart_server(cls): "project root directory." ) + # remove data dir under gravitino_home + data_dir = os.path.join(gravitino_home, "data") + if os.path.exists(data_dir): + logger.info("Remove Gravitino data directory: %s", data_dir) + shutil.rmtree(data_dir) + # Restart Gravitino Server env_vars = os.environ.copy() env_vars["HADOOP_USER_NAME"] = "anonymous" diff --git a/clients/client-python/tests/integration/test_catalog.py b/clients/client-python/tests/integration/test_catalog.py index 58580b8c518..64208315e6e 100644 --- a/clients/client-python/tests/integration/test_catalog.py +++ b/clients/client-python/tests/integration/test_catalog.py @@ -39,7 +39,8 @@ class TestCatalog(IntegrationTestEnv): metalake_name: str = "TestSchema_metalake" + str(randint(1, 10000)) - catalog_name: str = "testCatalog" + catalog_name: str = "testCatalog" + str(randint(1, 10000)) + catalog_name_bak = catalog_name catalog_comment: str = "catalogComment" catalog_location_prop: str = "location" # Fileset Catalog must set `location` catalog_provider: str = "hadoop" @@ -81,21 +82,27 @@ def clean_test_data(self): ) try: logger.info( - "Drop catalog %s[%s]", + "TestCatalog: drop catalog %s[%s]", self.catalog_ident, self.gravitino_client.drop_catalog(name=self.catalog_name, force=True), ) except GravitinoRuntimeException: - logger.warning("Failed to drop catalog %s", self.catalog_name) + logger.warning("TestCatalog: failed to drop catalog %s", self.catalog_name) try: logger.info( - "Drop metalake %s[%s]", + "TestCatalog: drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: - logger.warning("Failed to drop metalake %s", self.metalake_name) + logger.warning( + "TestCatalog: failed to drop metalake %s", self.metalake_name + ) + + self.catalog_name = self.catalog_name_bak def test_list_catalogs(self): self.create_catalog(self.catalog_name) diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py b/clients/client-python/tests/integration/test_fileset_catalog.py index 2696c170a0d..754735b16e8 100644 --- a/clients/client-python/tests/integration/test_fileset_catalog.py +++ b/clients/client-python/tests/integration/test_fileset_catalog.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/client-python/tests/integration/test_gvfs_with_gcs.py b/clients/client-python/tests/integration/test_gvfs_with_gcs.py new file mode 100644 index 00000000000..16f84dff3b1 --- /dev/null +++ b/clients/client-python/tests/integration/test_gvfs_with_gcs.py @@ -0,0 +1,173 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import logging +import os +from random import randint +import unittest + +from fsspec.implementations.arrow import ArrowFSWrapper +from pyarrow.fs import GcsFileSystem + +from tests.integration.test_gvfs_with_hdfs import TestGvfsWithHDFS +from gravitino import ( + gvfs, + GravitinoClient, + Catalog, + Fileset, +) +from gravitino.exceptions.base import GravitinoRuntimeException + + +logger = logging.getLogger(__name__) + + +@unittest.skip("This test require GCS service account key file") +class TestGvfsWithGCS(TestGvfsWithHDFS): + # Before running this test, please set the make sure gcp-bundle-x.jar has been + # copy to the $GRAVITINO_HOME/catalogs/hadoop/libs/ directory + key_file = "your_key_file.json" + bucket_name = "your_bucket_name" + metalake_name: str = "TestGvfsWithGCS_metalake" + str(randint(1, 10000)) + + def setUp(self): + self.options = {"gravitino.bypass.gcs.service-account-key-path": self.key_file} + + def tearDown(self): + self.options = {} + + @classmethod + def setUpClass(cls): + cls._get_gravitino_home() + + cls.hadoop_conf_path = f"{cls.gravitino_home}/catalogs/hadoop/conf/hadoop.conf" + # restart the server + cls.restart_server() + # create entity + cls._init_test_entities() + + @classmethod + def tearDownClass(cls): + cls._clean_test_data() + # reset server conf in case of other ITs like HDFS has changed it and fail + # to reset it + cls._reset_conf(cls.config, cls.hadoop_conf_path) + # restart server + cls.restart_server() + + # clear all config in the conf_path + @classmethod + def _reset_conf(cls, config, conf_path): + logger.info("Reset %s.", conf_path) + if not os.path.exists(conf_path): + raise GravitinoRuntimeException(f"Conf file is not found at `{conf_path}`.") + filtered_lines = [] + with open(conf_path, mode="r", encoding="utf-8") as file: + origin_lines = file.readlines() + + for line in origin_lines: + line = line.strip() + if line.startswith("#"): + # append annotations directly + filtered_lines.append(line + "\n") + + with open(conf_path, mode="w", encoding="utf-8") as file: + for line in filtered_lines: + file.write(line) + + @classmethod + def _init_test_entities(cls): + cls.gravitino_admin_client.create_metalake( + name=cls.metalake_name, comment="", properties={} + ) + cls.gravitino_client = GravitinoClient( + uri="http://localhost:8090", metalake_name=cls.metalake_name + ) + + cls.config = {} + cls.conf = {} + catalog = cls.gravitino_client.create_catalog( + name=cls.catalog_name, + catalog_type=Catalog.Type.FILESET, + provider=cls.catalog_provider, + comment="", + properties={ + "filesystem-providers": "gcs", + "gravitino.bypass.fs.gs.auth.service.account.enable": "true", + "gravitino.bypass.fs.gs.auth.service.account.json.keyfile": cls.key_file, + }, + ) + catalog.as_schemas().create_schema( + schema_name=cls.schema_name, comment="", properties={} + ) + + cls.fileset_storage_location: str = ( + f"gs://{cls.bucket_name}/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}" + ) + cls.fileset_gvfs_location = ( + f"gvfs://fileset/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}" + ) + catalog.as_fileset_catalog().create_fileset( + ident=cls.fileset_ident, + fileset_type=Fileset.Type.MANAGED, + comment=cls.fileset_comment, + storage_location=cls.fileset_storage_location, + properties=cls.fileset_properties, + ) + + os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = cls.key_file + arrow_gcs_fs = GcsFileSystem() + cls.fs = ArrowFSWrapper(arrow_gcs_fs) + + def test_modified(self): + modified_dir = self.fileset_gvfs_location + "/test_modified" + modified_actual_dir = self.fileset_storage_location + "/test_modified" + fs = gvfs.GravitinoVirtualFileSystem( + server_uri="http://localhost:8090", + metalake_name=self.metalake_name, + options=self.options, + **self.conf, + ) + self.fs.mkdir(modified_actual_dir) + self.assertTrue(self.fs.exists(modified_actual_dir)) + self.assertTrue(fs.exists(modified_dir)) + + # GCP only supports getting the `object` modify time, so the modified time will be None + # if it's a directory. + # >>> gcs.mkdir('example_qazwsx/catalog/schema/fileset3') + # >>> r = gcs.modified('example_qazwsx/catalog/schema/fileset3') + # >>> print(r) + # None + self.assertIsNone(fs.modified(modified_dir)) + + # create a file under the dir 'modified_dir'. + file_path = modified_dir + "/test.txt" + fs.touch(file_path) + self.assertTrue(fs.exists(file_path)) + self.assertIsNotNone(fs.modified(file_path)) + + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) + def test_pandas(self): + pass + + @unittest.skip( + "This test will fail for https://github.com/apache/arrow/issues/44438" + ) + def test_pyarrow(self): + pass diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py index 8bc6597b455..5be5914f15b 100644 --- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py +++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py @@ -89,6 +89,7 @@ class TestGvfsWithHDFS(IntegrationTestEnv): uri="http://localhost:8090" ) gravitino_client: GravitinoClient = None + options = {} @classmethod def setUpClass(cls): @@ -124,7 +125,8 @@ def tearDownClass(cls): BaseHadoopEnvironment.clear_hadoop_env() finally: # close hdfs container - cls.hdfs_container.close() + if cls.hdfs_container is not None: + cls.hdfs_container.close() @classmethod def _init_test_entities(cls): @@ -159,7 +161,7 @@ def _init_test_entities(cls): properties=cls.fileset_properties, ) arrow_hadoop_fs = HadoopFileSystem(host=cls.hdfs_container.get_ip(), port=9000) - cls.hdfs = ArrowFSWrapper(arrow_hadoop_fs) + cls.fs = ArrowFSWrapper(arrow_hadoop_fs) cls.conf: Dict = {"fs.defaultFS": f"hdfs://{cls.hdfs_container.get_ip()}:9000/"} @classmethod @@ -234,15 +236,16 @@ def test_ls(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(ls_actual_dir) - self.assertTrue(self.hdfs.exists(ls_actual_dir)) + self.fs.mkdir(ls_actual_dir) + self.assertTrue(self.fs.exists(ls_actual_dir)) ls_file = self.fileset_gvfs_location + "/test_ls/test.file" ls_actual_file = self.fileset_storage_location + "/test_ls/test.file" - self.hdfs.touch(ls_actual_file) - self.assertTrue(self.hdfs.exists(ls_actual_file)) + self.fs.touch(ls_actual_file) + self.assertTrue(self.fs.exists(ls_actual_file)) # test detail = false file_list_without_detail = fs.ls(ls_dir, detail=False) @@ -260,15 +263,16 @@ def test_info(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(info_actual_dir) - self.assertTrue(self.hdfs.exists(info_actual_dir)) + self.fs.mkdir(info_actual_dir) + self.assertTrue(self.fs.exists(info_actual_dir)) info_file = self.fileset_gvfs_location + "/test_info/test.file" info_actual_file = self.fileset_storage_location + "/test_info/test.file" - self.hdfs.touch(info_actual_file) - self.assertTrue(self.hdfs.exists(info_actual_file)) + self.fs.touch(info_actual_file) + self.assertTrue(self.fs.exists(info_actual_file)) dir_info = fs.info(info_dir) self.assertEqual(dir_info["name"], info_dir[len("gvfs://") :]) @@ -282,16 +286,17 @@ def test_exist(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(exist_actual_dir) - self.assertTrue(self.hdfs.exists(exist_actual_dir)) + self.fs.mkdir(exist_actual_dir) + self.assertTrue(self.fs.exists(exist_actual_dir)) self.assertTrue(fs.exists(exist_dir)) exist_file = self.fileset_gvfs_location + "/test_exist/test.file" exist_actual_file = self.fileset_storage_location + "/test_exist/test.file" - self.hdfs.touch(exist_actual_file) - self.assertTrue(self.hdfs.exists(exist_actual_file)) + self.fs.touch(exist_actual_file) + self.assertTrue(self.fs.exists(exist_actual_file)) self.assertTrue(fs.exists(exist_file)) def test_cp_file(self): @@ -300,19 +305,20 @@ def test_cp_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(cp_file_actual_dir) - self.assertTrue(self.hdfs.exists(cp_file_actual_dir)) + self.fs.mkdir(cp_file_actual_dir) + self.assertTrue(self.fs.exists(cp_file_actual_dir)) self.assertTrue(fs.exists(cp_file_dir)) cp_file_file = self.fileset_gvfs_location + "/test_cp_file/test.file" cp_file_actual_file = self.fileset_storage_location + "/test_cp_file/test.file" - self.hdfs.touch(cp_file_actual_file) - self.assertTrue(self.hdfs.exists(cp_file_actual_file)) + self.fs.touch(cp_file_actual_file) + self.assertTrue(self.fs.exists(cp_file_actual_file)) self.assertTrue(fs.exists(cp_file_file)) - with self.hdfs.open(cp_file_actual_file, "wb") as f: + with self.fs.open(cp_file_actual_file, "wb") as f: f.write(b"test_file_1") cp_file_new_file = self.fileset_gvfs_location + "/test_cp_file/test_cp.file" @@ -322,7 +328,7 @@ def test_cp_file(self): fs.cp_file(cp_file_file, cp_file_new_file) self.assertTrue(fs.exists(cp_file_new_file)) - with self.hdfs.open(cp_file_new_actual_file, "rb") as f: + with self.fs.open(cp_file_new_actual_file, "rb") as f: result = f.read() self.assertEqual(b"test_file_1", result) @@ -332,10 +338,11 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(mv_actual_dir) - self.assertTrue(self.hdfs.exists(mv_actual_dir)) + self.fs.mkdir(mv_actual_dir) + self.assertTrue(self.fs.exists(mv_actual_dir)) self.assertTrue(fs.exists(mv_dir)) mv_new_dir = self.fileset_gvfs_location + "/test_mv_new" @@ -343,16 +350,17 @@ def test_mv(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(mv_new_actual_dir) - self.assertTrue(self.hdfs.exists(mv_new_actual_dir)) + self.fs.mkdir(mv_new_actual_dir) + self.assertTrue(self.fs.exists(mv_new_actual_dir)) self.assertTrue(fs.exists(mv_new_dir)) mv_file = self.fileset_gvfs_location + "/test_mv/test.file" mv_actual_file = self.fileset_storage_location + "/test_mv/test.file" - self.hdfs.touch(mv_actual_file) - self.assertTrue(self.hdfs.exists(mv_actual_file)) + self.fs.touch(mv_actual_file) + self.assertTrue(self.fs.exists(mv_actual_file)) self.assertTrue(fs.exists(mv_file)) mv_new_file = self.fileset_gvfs_location + "/test_mv_new/test_new.file" @@ -362,7 +370,7 @@ def test_mv(self): fs.mv(mv_file, mv_new_file) self.assertTrue(fs.exists(mv_new_file)) - self.assertTrue(self.hdfs.exists(mv_new_actual_file)) + self.assertTrue(self.fs.exists(mv_new_actual_file)) # test rename without sub path, which should throw an exception with self.assertRaises(GravitinoRuntimeException): @@ -374,16 +382,17 @@ def test_rm(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rm_actual_dir) - self.assertTrue(self.hdfs.exists(rm_actual_dir)) + self.fs.mkdir(rm_actual_dir) + self.assertTrue(self.fs.exists(rm_actual_dir)) self.assertTrue(fs.exists(rm_dir)) rm_file = self.fileset_gvfs_location + "/test_rm/test.file" rm_actual_file = self.fileset_storage_location + "/test_rm/test.file" - self.hdfs.touch(rm_file) - self.assertTrue(self.hdfs.exists(rm_actual_file)) + fs.touch(rm_file) + self.assertTrue(self.fs.exists(rm_actual_file)) self.assertTrue(fs.exists(rm_file)) # test delete file @@ -393,8 +402,8 @@ def test_rm(self): # test delete dir with recursive = false rm_new_file = self.fileset_gvfs_location + "/test_rm/test_new.file" rm_new_actual_file = self.fileset_storage_location + "/test_rm/test_new.file" - self.hdfs.touch(rm_new_actual_file) - self.assertTrue(self.hdfs.exists(rm_new_actual_file)) + self.fs.touch(rm_new_actual_file) + self.assertTrue(self.fs.exists(rm_new_actual_file)) self.assertTrue(fs.exists(rm_new_file)) with self.assertRaises(ValueError): fs.rm(rm_dir, recursive=False) @@ -409,16 +418,17 @@ def test_rm_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rm_file_actual_dir) - self.assertTrue(self.hdfs.exists(rm_file_actual_dir)) + self.fs.mkdir(rm_file_actual_dir) + self.assertTrue(self.fs.exists(rm_file_actual_dir)) self.assertTrue(fs.exists(rm_file_dir)) rm_file_file = self.fileset_gvfs_location + "/test_rm_file/test.file" rm_file_actual_file = self.fileset_storage_location + "/test_rm_file/test.file" - self.hdfs.touch(rm_file_actual_file) - self.assertTrue(self.hdfs.exists(rm_file_actual_file)) + self.fs.touch(rm_file_actual_file) + self.assertTrue(self.fs.exists(rm_file_actual_file)) self.assertTrue(fs.exists(rm_file_file)) # test delete file @@ -435,16 +445,17 @@ def test_rmdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(rmdir_actual_dir) - self.assertTrue(self.hdfs.exists(rmdir_actual_dir)) + self.fs.mkdir(rmdir_actual_dir) + self.assertTrue(self.fs.exists(rmdir_actual_dir)) self.assertTrue(fs.exists(rmdir_dir)) rmdir_file = self.fileset_gvfs_location + "/test_rmdir/test.file" rmdir_actual_file = self.fileset_storage_location + "/test_rmdir/test.file" - self.hdfs.touch(rmdir_actual_file) - self.assertTrue(self.hdfs.exists(rmdir_actual_file)) + self.fs.touch(rmdir_actual_file) + self.assertTrue(self.fs.exists(rmdir_actual_file)) self.assertTrue(fs.exists(rmdir_file)) # test delete file @@ -461,16 +472,17 @@ def test_open(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(open_actual_dir) - self.assertTrue(self.hdfs.exists(open_actual_dir)) + self.fs.mkdir(open_actual_dir) + self.assertTrue(self.fs.exists(open_actual_dir)) self.assertTrue(fs.exists(open_dir)) open_file = self.fileset_gvfs_location + "/test_open/test.file" open_actual_file = self.fileset_storage_location + "/test_open/test.file" - self.hdfs.touch(open_actual_file) - self.assertTrue(self.hdfs.exists(open_actual_file)) + self.fs.touch(open_actual_file) + self.assertTrue(self.fs.exists(open_actual_file)) self.assertTrue(fs.exists(open_file)) # test open and write file @@ -488,11 +500,12 @@ def test_mkdir(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.mkdir(mkdir_dir) self.assertTrue(fs.exists(mkdir_dir)) - self.assertTrue(self.hdfs.exists(mkdir_actual_dir)) + self.assertTrue(self.fs.exists(mkdir_actual_dir)) # test mkdir dir with create_parents = false parent_not_exist_virtual_path = mkdir_dir + "/not_exist/sub_dir" @@ -514,11 +527,12 @@ def test_makedirs(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) fs.makedirs(makedirs_dir) self.assertTrue(fs.exists(makedirs_dir)) - self.assertTrue(self.hdfs.exists(makedirs_actual_dir)) + self.assertTrue(self.fs.exists(makedirs_actual_dir)) # test mkdir dir not exist parent_not_exist_virtual_path = makedirs_dir + "/not_exist/sub_dir" @@ -532,10 +546,11 @@ def test_created(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(created_actual_dir) - self.assertTrue(self.hdfs.exists(created_actual_dir)) + self.fs.mkdir(created_actual_dir) + self.assertTrue(self.fs.exists(created_actual_dir)) self.assertTrue(fs.exists(created_dir)) with self.assertRaises(GravitinoRuntimeException): @@ -547,10 +562,11 @@ def test_modified(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(modified_actual_dir) - self.assertTrue(self.hdfs.exists(modified_actual_dir)) + self.fs.mkdir(modified_actual_dir) + self.assertTrue(self.fs.exists(modified_actual_dir)) self.assertTrue(fs.exists(modified_dir)) # test mkdir dir which exists @@ -562,16 +578,17 @@ def test_cat_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(cat_actual_dir) - self.assertTrue(self.hdfs.exists(cat_actual_dir)) + self.fs.mkdir(cat_actual_dir) + self.assertTrue(self.fs.exists(cat_actual_dir)) self.assertTrue(fs.exists(cat_dir)) cat_file = self.fileset_gvfs_location + "/test_cat/test.file" cat_actual_file = self.fileset_storage_location + "/test_cat/test.file" - self.hdfs.touch(cat_actual_file) - self.assertTrue(self.hdfs.exists(cat_actual_file)) + self.fs.touch(cat_actual_file) + self.assertTrue(self.fs.exists(cat_actual_file)) self.assertTrue(fs.exists(cat_file)) # test open and write file @@ -589,16 +606,17 @@ def test_get_file(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(get_actual_dir) - self.assertTrue(self.hdfs.exists(get_actual_dir)) + self.fs.mkdir(get_actual_dir) + self.assertTrue(self.fs.exists(get_actual_dir)) self.assertTrue(fs.exists(get_dir)) get_file = self.fileset_gvfs_location + "/test_get/test.file" get_actual_file = self.fileset_storage_location + "/test_get/test.file" - self.hdfs.touch(get_actual_file) - self.assertTrue(self.hdfs.exists(get_actual_file)) + self.fs.touch(get_actual_file) + self.assertTrue(self.fs.exists(get_actual_file)) self.assertTrue(fs.exists(get_file)) # test open and write file @@ -628,10 +646,11 @@ def test_pandas(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(pands_actual_dir) - self.assertTrue(self.hdfs.exists(pands_actual_dir)) + self.fs.mkdir(pands_actual_dir) + self.assertTrue(self.fs.exists(pands_actual_dir)) self.assertTrue(fs.exists(pands_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) @@ -642,7 +661,7 @@ def test_pandas(self): ) data.to_parquet(parquet_file, filesystem=fs) self.assertTrue(fs.exists(parquet_file)) - self.assertTrue(self.hdfs.exists(parquet_actual_file)) + self.assertTrue(self.fs.exists(parquet_actual_file)) # read parquet ds1 = pandas.read_parquet(path=parquet_file, filesystem=fs) @@ -650,6 +669,7 @@ def test_pandas(self): storage_options = { "server_uri": "http://localhost:8090", "metalake_name": self.metalake_name, + "options": self.options, } # to csv csv_file = self.fileset_gvfs_location + "/test_pandas/test.csv" @@ -660,7 +680,7 @@ def test_pandas(self): storage_options=storage_options, ) self.assertTrue(fs.exists(csv_file)) - self.assertTrue(self.hdfs.exists(csv_actual_file)) + self.assertTrue(self.fs.exists(csv_actual_file)) # read csv ds2 = pandas.read_csv(csv_file, storage_options=storage_options) @@ -672,10 +692,11 @@ def test_pyarrow(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(pyarrow_actual_dir) - self.assertTrue(self.hdfs.exists(pyarrow_actual_dir)) + self.fs.mkdir(pyarrow_actual_dir) + self.assertTrue(self.fs.exists(pyarrow_actual_dir)) self.assertTrue(fs.exists(pyarrow_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) @@ -701,16 +722,18 @@ def test_llama_index(self): fs = gvfs.GravitinoVirtualFileSystem( server_uri="http://localhost:8090", metalake_name=self.metalake_name, + options=self.options, **self.conf, ) - self.hdfs.mkdir(llama_actual_dir) - self.assertTrue(self.hdfs.exists(llama_actual_dir)) + self.fs.mkdir(llama_actual_dir) + self.assertTrue(self.fs.exists(llama_actual_dir)) self.assertTrue(fs.exists(llama_dir)) data = pandas.DataFrame({"Name": ["A", "B", "C", "D"], "ID": [20, 21, 19, 18]}) storage_options = { "server_uri": "http://localhost:8090", "metalake_name": self.metalake_name, + "options": self.options, } csv_file = llama_dir + "/test.csv" # to csv diff --git a/clients/client-python/tests/integration/test_metalake.py b/clients/client-python/tests/integration/test_metalake.py index 794cb894d97..75d3a06f26c 100644 --- a/clients/client-python/tests/integration/test_metalake.py +++ b/clients/client-python/tests/integration/test_metalake.py @@ -118,7 +118,7 @@ def test_alter_metalake(self): self.assertTrue(self.metalake_properties_key1 not in metalake.properties()) def drop_metalake(self, metalake_name: str) -> bool: - return self.gravitino_admin_client.drop_metalake(metalake_name) + return self.gravitino_admin_client.drop_metalake(metalake_name, True) def test_drop_metalake(self): self.create_metalake(self.metalake_name) @@ -152,7 +152,9 @@ def test_load_metalakes(self): self.assertIsNotNone(metalake) self.assertEqual(metalake.name(), self.metalake_name) self.assertEqual(metalake.comment(), self.metalake_comment) - self.assertEqual(metalake.properties(), self.metalake_properties) + self.assertEqual( + metalake.properties(), {**self.metalake_properties, "in-use": "true"} + ) self.assertEqual(metalake.audit_info().creator(), "anonymous") def test_failed_load_metalakes(self): diff --git a/clients/client-python/tests/integration/test_schema.py b/clients/client-python/tests/integration/test_schema.py index 269693dcf25..c8a6b270b84 100644 --- a/clients/client-python/tests/integration/test_schema.py +++ b/clients/client-python/tests/integration/test_schema.py @@ -128,7 +128,9 @@ def clean_test_data(self): logger.info( "Drop metalake %s[%s]", self.metalake_name, - self.gravitino_admin_client.drop_metalake(self.metalake_name), + self.gravitino_admin_client.drop_metalake( + self.metalake_name, force=True + ), ) except GravitinoRuntimeException: logger.warning("Failed to drop metalake %s", self.metalake_name) diff --git a/clients/filesystem-hadoop3/build.gradle.kts b/clients/filesystem-hadoop3/build.gradle.kts index c3f8c6d7bcf..7f21c700d6e 100644 --- a/clients/filesystem-hadoop3/build.gradle.kts +++ b/clients/filesystem-hadoop3/build.gradle.kts @@ -41,6 +41,7 @@ dependencies { testImplementation(project(":integration-test-common", "testArtifacts")) testImplementation(project(":bundles:gcp-bundle")) testImplementation(project(":bundles:aliyun-bundle")) + testImplementation(project(":bundles:aws-bundle")) testImplementation(libs.awaitility) testImplementation(libs.bundles.jetty) testImplementation(libs.bundles.jersey) @@ -89,6 +90,11 @@ tasks.test { } else { dependsOn(":catalogs:catalog-hadoop:jar", ":catalogs:catalog-hadoop:runtimeJars") } + + // this task depends on :bundles:aws-bundle:shadowJar + dependsOn(":bundles:aws-bundle:jar") + dependsOn(":bundles:aliyun-bundle:jar") + dependsOn(":bundles:gcp-bundle:jar") } tasks.javadoc { diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java index 73a45006f03..312236fe5da 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemGCSIT.java @@ -52,6 +52,7 @@ public void startIntegrationTest() { @BeforeAll public void startUp() throws Exception { + // Copy the GCP jars to the gravitino server if in deploy mode. copyBundleJarsToHadoop("gcp-bundle"); // Need to download jars to gravitino server super.startIntegrationTest(); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java index ef41637d6f9..b971ab918d2 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemIT.java @@ -56,13 +56,14 @@ @Tag("gravitino-docker-test") public class GravitinoVirtualFileSystemIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemIT.class); - private static final ContainerSuite containerSuite = ContainerSuite.getInstance(); + protected static final ContainerSuite containerSuite = ContainerSuite.getInstance(); protected String metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); protected String catalogName = GravitinoITUtils.genRandomName("catalog"); protected String schemaName = GravitinoITUtils.genRandomName("schema"); protected GravitinoMetalake metalake; protected Configuration conf = new Configuration(); protected int defaultBockSize = 128 * 1024 * 1024; + protected int defaultReplication = 3; @BeforeAll public void startUp() throws Exception { @@ -92,7 +93,7 @@ public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); metalake.dropCatalog(catalogName, true); - client.dropMetalake(metalakeName); + client.dropMetalake(metalakeName, true); if (client != null) { client.close(); @@ -459,7 +460,7 @@ public void testGetDefaultReplications() throws IOException { Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); Path gvfsPath = genGvfsPath(filesetName); try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { - assertEquals(3, gvfs.getDefaultReplication(gvfsPath)); + assertEquals(defaultReplication, gvfs.getDefaultReplication(gvfsPath)); } catalog.asFilesetCatalog().dropFileset(filesetIdent); diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java index 67c76be3db5..6a6557c6c55 100644 --- a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemOSSIT.java @@ -21,25 +21,18 @@ import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; -import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.common.collect.Maps; import java.io.IOException; import java.util.Collections; -import java.util.HashMap; import java.util.Map; import org.apache.gravitino.Catalog; -import org.apache.gravitino.NameIdentifier; -import org.apache.gravitino.file.Fileset; import org.apache.gravitino.integration.test.util.GravitinoITUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +61,9 @@ public void startUp() throws Exception { // This value can be by tune by the user, please change it accordingly. defaultBockSize = 64 * 1024 * 1024; + // The default replication factor is 1. + defaultReplication = 1; + metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); catalogName = GravitinoITUtils.genRandomName("catalog"); schemaName = GravitinoITUtils.genRandomName("schema"); @@ -111,7 +107,7 @@ public void startUp() throws Exception { public void tearDown() throws IOException { Catalog catalog = metalake.loadCatalog(catalogName); catalog.asSchemas().dropSchema(schemaName, true); - metalake.dropCatalog(catalogName); + metalake.dropCatalog(catalogName, true); client.dropMetalake(metalakeName); if (client != null) { @@ -145,30 +141,6 @@ protected String genStorageLocation(String fileset) { return String.format("oss://%s/%s", BUCKET_NAME, fileset); } - @Test - public void testGetDefaultReplications() throws IOException { - String filesetName = GravitinoITUtils.genRandomName("test_get_default_replications"); - NameIdentifier filesetIdent = NameIdentifier.of(schemaName, filesetName); - Catalog catalog = metalake.loadCatalog(catalogName); - String storageLocation = genStorageLocation(filesetName); - catalog - .asFilesetCatalog() - .createFileset( - filesetIdent, - "fileset comment", - Fileset.Type.MANAGED, - storageLocation, - new HashMap<>()); - Assertions.assertTrue(catalog.asFilesetCatalog().filesetExists(filesetIdent)); - Path gvfsPath = genGvfsPath(filesetName); - try (FileSystem gvfs = gvfsPath.getFileSystem(conf)) { - // Here HDFS is 3, but for oss is 1. - assertEquals(1, gvfs.getDefaultReplication(gvfsPath)); - } - - catalog.asFilesetCatalog().dropFileset(filesetIdent); - } - @Disabled( "OSS does not support append, java.io.IOException: The append operation is not supported") public void testAppend() throws IOException {} diff --git a/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java new file mode 100644 index 00000000000..2f9952f28d2 --- /dev/null +++ b/clients/filesystem-hadoop3/src/test/java/org/apache/gravitino/filesystem/hadoop/integration/test/GravitinoVirtualFileSystemS3IT.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.filesystem.hadoop.integration.test; + +import static org.apache.gravitino.catalog.hadoop.HadoopCatalogPropertiesMetadata.FILESYSTEM_PROVIDERS; +import static org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystemConfiguration.FS_FILESYSTEM_PROVIDERS; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.apache.gravitino.Catalog; +import org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer; +import org.apache.gravitino.integration.test.util.GravitinoITUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.shaded.org.awaitility.Awaitility; + +public class GravitinoVirtualFileSystemS3IT extends GravitinoVirtualFileSystemIT { + private static final Logger LOG = LoggerFactory.getLogger(GravitinoVirtualFileSystemS3IT.class); + + private String bucketName = "s3-bucket-" + UUID.randomUUID().toString().replace("-", ""); + private String accessKey; + private String secretKey; + private String s3Endpoint; + + private GravitinoLocalStackContainer gravitinoLocalStackContainer; + + @BeforeAll + public void startIntegrationTest() { + // Do nothing + } + + private void startS3Mocker() { + containerSuite.startLocalStackContainer(); + gravitinoLocalStackContainer = containerSuite.getLocalStackContainer(); + + Awaitility.await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(1, TimeUnit.SECONDS) + .until( + () -> { + try { + Container.ExecResult result = + gravitinoLocalStackContainer.executeInContainer( + "awslocal", "iam", "create-user", "--user-name", "anonymous"); + return result.getExitCode() == 0; + } catch (Exception e) { + LOG.info("LocalStack is not ready yet for: ", e); + return false; + } + }); + + gravitinoLocalStackContainer.executeInContainer("awslocal", "s3", "mb", "s3://" + bucketName); + + Container.ExecResult result = + gravitinoLocalStackContainer.executeInContainer( + "awslocal", "iam", "create-access-key", "--user-name", "anonymous"); + + gravitinoLocalStackContainer.executeInContainer( + "awslocal", + "s3api", + "put-bucket-acl", + "--bucket", + "my-test-bucket", + "--acl", + "public-read-write"); + + // Get access key and secret key from result + String[] lines = result.getStdout().split("\n"); + accessKey = lines[3].split(":")[1].trim().substring(1, 21); + secretKey = lines[5].split(":")[1].trim().substring(1, 41); + + LOG.info("Access key: " + accessKey); + LOG.info("Secret key: " + secretKey); + + s3Endpoint = + String.format("http://%s:%d", gravitinoLocalStackContainer.getContainerIpAddress(), 4566); + } + + @BeforeAll + public void startUp() throws Exception { + copyBundleJarsToHadoop("aws-bundle"); + + // Start s3 simulator + startS3Mocker(); + + // Need to download jars to gravitino server + super.startIntegrationTest(); + + // This value can be by tune by the user, please change it accordingly. + defaultBockSize = 32 * 1024 * 1024; + + // The value is 1 for S3 + defaultReplication = 1; + + metalakeName = GravitinoITUtils.genRandomName("gvfs_it_metalake"); + catalogName = GravitinoITUtils.genRandomName("catalog"); + schemaName = GravitinoITUtils.genRandomName("schema"); + + Assertions.assertFalse(client.metalakeExists(metalakeName)); + metalake = client.createMetalake(metalakeName, "metalake comment", Collections.emptyMap()); + Assertions.assertTrue(client.metalakeExists(metalakeName)); + + Map properties = Maps.newHashMap(); + properties.put("gravitino.bypass.fs.s3a.access.key", accessKey); + properties.put("gravitino.bypass.fs.s3a.secret.key", secretKey); + properties.put("gravitino.bypass.fs.s3a.endpoint", s3Endpoint); + properties.put( + "gravitino.bypass.fs.s3a.aws.credentials.provider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + properties.put(FILESYSTEM_PROVIDERS, "s3"); + + Catalog catalog = + metalake.createCatalog( + catalogName, Catalog.Type.FILESET, "hadoop", "catalog comment", properties); + Assertions.assertTrue(metalake.catalogExists(catalogName)); + + catalog.asSchemas().createSchema(schemaName, "schema comment", properties); + Assertions.assertTrue(catalog.asSchemas().schemaExists(schemaName)); + + conf.set("fs.gvfs.impl", "org.apache.gravitino.filesystem.hadoop.GravitinoVirtualFileSystem"); + conf.set("fs.AbstractFileSystem.gvfs.impl", "org.apache.gravitino.filesystem.hadoop.Gvfs"); + conf.set("fs.gvfs.impl.disable.cache", "true"); + conf.set("fs.gravitino.server.uri", serverUri); + conf.set("fs.gravitino.client.metalake", metalakeName); + + // Pass this configuration to the real file system + conf.set("fs.s3a.access.key", accessKey); + conf.set("fs.s3a.secret.key", secretKey); + conf.set("fs.s3a.endpoint", s3Endpoint); + conf.set( + "fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"); + conf.set(FS_FILESYSTEM_PROVIDERS, "s3"); + } + + @AfterAll + public void tearDown() throws IOException { + Catalog catalog = metalake.loadCatalog(catalogName); + catalog.asSchemas().dropSchema(schemaName, true); + metalake.dropCatalog(catalogName, true); + client.dropMetalake(metalakeName, true); + + if (client != null) { + client.close(); + client = null; + } + + try { + closer.close(); + } catch (Exception e) { + LOG.error("Exception in closing CloseableGroup", e); + } + } + + /** + * Remove the `gravitino.bypass` prefix from the configuration and pass it to the real file system + * This method corresponds to the method org.apache.gravitino.filesystem.hadoop + * .GravitinoVirtualFileSystem#getConfigMap(Configuration) in the original code. + */ + protected Configuration convertGvfsConfigToRealFileSystemConfig(Configuration gvfsConf) { + Configuration gcsConf = new Configuration(); + gvfsConf.forEach( + entry -> { + gcsConf.set(entry.getKey().replace("gravitino.bypass.", ""), entry.getValue()); + }); + + return gcsConf; + } + + protected String genStorageLocation(String fileset) { + return String.format("s3a://%s/%s", bucketName, fileset); + } + + @Disabled( + "GCS does not support append, java.io.IOException: The append operation is not supported") + public void testAppend() throws IOException {} +} diff --git a/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java b/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java new file mode 100644 index 00000000000..be75b2d4a21 --- /dev/null +++ b/common/src/main/java/org/apache/gravitino/dto/requests/MetalakeSetRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.dto.requests; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.apache.gravitino.rest.RESTRequest; + +/** Represents a request to set a Metalake in use. */ +@Getter +@EqualsAndHashCode +@ToString +public class MetalakeSetRequest implements RESTRequest { + + private final boolean inUse; + + /** Default constructor for MetalakeSetRequest. */ + public MetalakeSetRequest() { + this(false); + } + + /** + * Constructor for MetalakeSetRequest. + * + * @param inUse The in use status to set. + */ + public MetalakeSetRequest(boolean inUse) { + this.inUse = inUse; + } + + /** + * Validates the request. No validation needed. + * + * @throws IllegalArgumentException If the request is invalid. + */ + @Override + public void validate() throws IllegalArgumentException { + // No validation needed + } +} diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 81447abfd3b..1d5a2acf034 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -19,14 +19,12 @@ package org.apache.gravitino.authorization; import com.google.common.collect.Sets; -import java.io.IOException; import java.util.Collection; import java.util.List; import java.util.Set; import java.util.function.Consumer; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; -import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.MetadataObject; import org.apache.gravitino.NameIdentifier; @@ -40,12 +38,9 @@ import org.apache.gravitino.exceptions.IllegalPrivilegeException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchUserException; import org.apache.gravitino.utils.MetadataObjectUtil; import org.apache.gravitino.utils.NameIdentifierUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /* The utilization class of authorization module*/ public class AuthorizationUtils { @@ -53,8 +48,6 @@ public class AuthorizationUtils { static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in th metalake %s"; static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in th metalake %s"; static final String ROLE_DOES_NOT_EXIST_MSG = "Role %s does not exist in th metalake %s"; - private static final Logger LOG = LoggerFactory.getLogger(AuthorizationUtils.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final Set FILESET_PRIVILEGES = Sets.immutableEnumSet( @@ -68,21 +61,6 @@ public class AuthorizationUtils { private AuthorizationUtils() {} - static void checkMetalakeExists(String metalake) throws NoSuchMetalakeException { - try { - EntityStore store = GravitinoEnv.getInstance().entityStore(); - - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!store.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - public static void checkCurrentUser(String metalake, String user) { try { AccessControlDispatcher dispatcher = GravitinoEnv.getInstance().accessControlDispatcher(); diff --git a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java index dc675fdcef5..11c24102bca 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/RoleManager.java @@ -19,6 +19,8 @@ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Sets; import java.io.IOException; import java.time.Instant; @@ -33,7 +35,6 @@ import org.apache.gravitino.SupportsRelationOperations; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchRoleException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.meta.AuditInfo; @@ -52,7 +53,6 @@ class RoleManager { private static final Logger LOG = LoggerFactory.getLogger(RoleManager.class); - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private final EntityStore store; private final IdGenerator idGenerator; @@ -67,7 +67,7 @@ RoleEntity createRole( Map properties, List securableObjects) throws RoleAlreadyExistsException { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); RoleEntity roleEntity = RoleEntity.builder() .withId(idGenerator.nextId()) @@ -104,7 +104,7 @@ RoleEntity createRole( RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return getRoleEntity(AuthorizationUtils.ofRole(metalake, role)); } catch (NoSuchEntityException e) { LOG.warn("Role {} does not exist in the metalake {}", role, metalake, e); @@ -114,7 +114,7 @@ RoleEntity getRole(String metalake, String role) throws NoSuchRoleException { boolean deleteRole(String metalake, String role) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); NameIdentifier ident = AuthorizationUtils.ofRole(metalake, role); try { @@ -138,14 +138,11 @@ boolean deleteRole(String metalake, String role) { String[] listRoleNames(String metalake) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofRoleNamespace(metalake); return store.list(namespace, RoleEntity.class, Entity.EntityType.ROLE).stream() .map(Role::name) .toArray(String[]::new); - } catch (NoSuchEntityException e) { - LOG.warn("Metalake {} does not exist", metalake, e); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalake); } catch (IOException ioe) { LOG.error("Listing user under metalake {} failed due to storage issues", metalake, ioe); throw new RuntimeException(ioe); @@ -154,7 +151,7 @@ String[] listRoleNames(String metalake) { String[] listRoleNamesByObject(String metalake, MetadataObject object) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.relationOperations() .listEntitiesByRelation( diff --git a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java index cd852ab66a7..905b100a652 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java +++ b/core/src/main/java/org/apache/gravitino/authorization/UserGroupManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.authorization; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.collect.Lists; import java.io.IOException; import java.time.Instant; @@ -27,6 +29,7 @@ import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; import org.apache.gravitino.EntityStore; +import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; import org.apache.gravitino.exceptions.NoSuchEntityException; @@ -62,7 +65,7 @@ class UserGroupManager { User addUser(String metalake, String name) throws UserAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); UserEntity userEntity = UserEntity.builder() .withId(idGenerator.nextId()) @@ -90,7 +93,7 @@ User addUser(String metalake, String name) throws UserAlreadyExistsException { boolean removeUser(String metalake, String user) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER); } catch (IOException ioe) { LOG.error( @@ -101,7 +104,7 @@ boolean removeUser(String metalake, String user) { User getUser(String metalake, String user) throws NoSuchUserException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofUser(metalake, user), Entity.EntityType.USER, UserEntity.class); @@ -127,7 +130,7 @@ User[] listUsers(String metalake) { Group addGroup(String metalake, String group) throws GroupAlreadyExistsException { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); GroupEntity groupEntity = GroupEntity.builder() .withId(idGenerator.nextId()) @@ -155,7 +158,7 @@ Group addGroup(String metalake, String group) throws GroupAlreadyExistsException boolean removeGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.delete(AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP); } catch (IOException ioe) { LOG.error( @@ -169,7 +172,7 @@ boolean removeGroup(String metalake, String group) { Group getGroup(String metalake, String group) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); return store.get( AuthorizationUtils.ofGroup(metalake, group), Entity.EntityType.GROUP, GroupEntity.class); @@ -194,7 +197,7 @@ String[] listGroupNames(String metalake) { private User[] listUsersInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofUserNamespace(metalake); return store @@ -211,7 +214,7 @@ private User[] listUsersInternal(String metalake, boolean allFields) { private Group[] listGroupInternal(String metalake, boolean allFields) { try { - AuthorizationUtils.checkMetalakeExists(metalake); + checkMetalake(NameIdentifier.of(metalake), store); Namespace namespace = AuthorizationUtils.ofGroupNamespace(metalake); return store .list(namespace, GroupEntity.class, EntityType.GROUP, allFields) diff --git a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java index f03b500bb2d..6af759c6142 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java +++ b/core/src/main/java/org/apache/gravitino/catalog/CatalogManager.java @@ -23,6 +23,8 @@ import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; import static org.apache.gravitino.connector.BaseCatalogPropertiesMetadata.BASIC_CATALOG_PROPERTIES_METADATA; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; +import static org.apache.gravitino.metalake.MetalakeManager.metalakeInUse; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -81,6 +83,7 @@ import org.apache.gravitino.exceptions.CatalogInUseException; import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.GravitinoRuntimeException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; @@ -104,14 +107,18 @@ public class CatalogManager implements CatalogDispatcher, Closeable { private static final String CATALOG_DOES_NOT_EXIST_MSG = "Catalog %s does not exist"; - private static final String METALAKE_DOES_NOT_EXIST_MSG = "Metalake %s does not exist"; private static final Logger LOG = LoggerFactory.getLogger(CatalogManager.class); - public static boolean catalogInUse(EntityStore store, NameIdentifier ident) - throws NoSuchMetalakeException, NoSuchCatalogException { - // todo: check if the metalake is in use - return getInUseValue(store, ident); + public static void checkCatalogInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException, NoSuchCatalogException, CatalogNotInUseException, + MetalakeNotInUseException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + + if (!getCatalogInUseValue(store, ident)) { + throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); + } } /** Wrapper class for a catalog instance and its class loader. */ @@ -284,7 +291,7 @@ public void close() { @Override public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(NameIdentifier.of(namespace.level(0)), store); try { return store.list(namespace, CatalogEntity.class, EntityType.CATALOG).stream() @@ -300,7 +307,7 @@ public NameIdentifier[] listCatalogs(Namespace namespace) throws NoSuchMetalakeE @Override public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeException { NameIdentifier metalakeIdent = NameIdentifier.of(namespace.levels()); - checkMetalakeExists(metalakeIdent); + checkMetalake(metalakeIdent, store); try { List catalogEntities = @@ -325,6 +332,9 @@ public Catalog[] listCatalogsInfo(Namespace namespace) throws NoSuchMetalakeExce */ @Override public Catalog loadCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + return loadCatalogAndWrap(ident).catalog; } @@ -348,6 +358,9 @@ public Catalog createCatalog( String comment, Map properties) throws NoSuchMetalakeException, CatalogAlreadyExistsException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + Map mergedConfig = buildCatalogConf(provider, properties); long uid = idGenerator.nextId(); @@ -374,12 +387,6 @@ public Catalog createCatalog( boolean needClean = true; try { - NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } - store.put(e, false /* overwrite */); CatalogWrapper wrapper = catalogCache.get(ident, id -> createCatalogWrapper(e, mergedConfig)); @@ -433,11 +440,9 @@ public void testConnection( String comment, Map properties) { NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); - try { - if (!store.exists(metalakeIdent, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, metalakeIdent); - } + checkMetalake(metalakeIdent, store); + try { if (store.exists(ident, EntityType.CATALOG)) { throw new CatalogAlreadyExistsException("Catalog %s already exists", ident); } @@ -483,6 +488,9 @@ public void testConnection( @Override public void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException, CatalogNotInUseException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { if (catalogInUse(store, ident)) { return; @@ -511,6 +519,9 @@ public void enableCatalog(NameIdentifier ident) @Override public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + checkMetalake(metalakeIdent, store); + try { if (!catalogInUse(store, ident)) { return; @@ -550,9 +561,7 @@ public void disableCatalog(NameIdentifier ident) throws NoSuchCatalogException { @Override public Catalog alterCatalog(NameIdentifier ident, CatalogChange... changes) throws NoSuchCatalogException, IllegalArgumentException { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); // There could be a race issue that someone is using the catalog from cache while we are // updating it. @@ -665,7 +674,13 @@ public CatalogWrapper loadCatalogAndWrap(NameIdentifier ident) throws NoSuchCata return catalogCache.get(ident, this::loadCatalogInternal); } - private static boolean getInUseValue(EntityStore store, NameIdentifier catalogIdent) { + private static boolean catalogInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException, NoSuchCatalogException { + NameIdentifier metalakeIdent = NameIdentifier.of(ident.namespace().levels()); + return metalakeInUse(store, metalakeIdent) && getCatalogInUseValue(store, ident); + } + + private static boolean getCatalogInUseValue(EntityStore store, NameIdentifier catalogIdent) { try { CatalogEntity catalogEntity = store.get(catalogIdent, EntityType.CATALOG, CatalogEntity.class); @@ -730,17 +745,6 @@ private Pair, Map> getCatalogAlterProperty( return Pair.of(upserts, deletes); } - private void checkMetalakeExists(NameIdentifier ident) throws NoSuchMetalakeException { - try { - if (!store.exists(ident, EntityType.METALAKE)) { - throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); - } - } catch (IOException e) { - LOG.error("Failed to do storage operation", e); - throw new RuntimeException(e); - } - } - private CatalogWrapper loadCatalogInternal(NameIdentifier ident) throws NoSuchCatalogException { try { CatalogEntity entity = store.get(ident, EntityType.CATALOG, CatalogEntity.class); diff --git a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java index 88add624870..3e2ed6c1b18 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/OperationDispatcher.java @@ -18,7 +18,7 @@ */ package org.apache.gravitino.catalog; -import static org.apache.gravitino.catalog.CatalogManager.catalogInUse; +import static org.apache.gravitino.catalog.CatalogManager.checkCatalogInUse; import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; import static org.apache.gravitino.utils.NameIdentifierUtil.getCatalogIdentifier; @@ -35,7 +35,6 @@ import org.apache.gravitino.connector.HasPropertyMetadata; import org.apache.gravitino.connector.PropertiesMetadata; import org.apache.gravitino.connector.capability.Capability; -import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.file.FilesetChange; import org.apache.gravitino.messaging.TopicChange; @@ -94,9 +93,7 @@ protected R doWithTable( protected R doWithCatalog( NameIdentifier ident, ThrowableFunction fn, Class ex) throws E { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); @@ -118,9 +115,7 @@ protected R doWithCatalog( Class ex1, Class ex2) throws E1, E2 { - if (!catalogInUse(store, ident)) { - throw new CatalogNotInUseException("Catalog %s is not in use, please enable it first", ident); - } + checkCatalogInUse(store, ident); try { CatalogManager.CatalogWrapper c = catalogManager.loadCatalogAndWrap(ident); diff --git a/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java b/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java index b54f0688715..da869d65f6a 100644 --- a/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/catalog/TableOperationDispatcher.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.EntityStore; import org.apache.gravitino.GravitinoEnv; @@ -499,8 +500,8 @@ private Table internalCreateTable( .withCreateTime(Instant.now()) .build(); List columnEntityList = - Arrays.stream(columns) - .map(c -> ColumnEntity.toColumnEntity(c, idGenerator.nextId(), audit)) + IntStream.range(0, columns.length) + .mapToObj(i -> ColumnEntity.toColumnEntity(columns[i], i, idGenerator.nextId(), audit)) .collect(Collectors.toList()); TableEntity tableEntity = @@ -531,13 +532,14 @@ private Table internalCreateTable( private List toColumnEntities(Column[] columns, AuditInfo audit) { return columns == null ? Collections.emptyList() - : Arrays.stream(columns) - .map(c -> ColumnEntity.toColumnEntity(c, idGenerator.nextId(), audit)) + : IntStream.range(0, columns.length) + .mapToObj(i -> ColumnEntity.toColumnEntity(columns[i], i, idGenerator.nextId(), audit)) .collect(Collectors.toList()); } - private boolean isSameColumn(Column left, ColumnEntity right) { + private boolean isSameColumn(Column left, int columnPosition, ColumnEntity right) { return Objects.equal(left.name(), right.name()) + && columnPosition == right.position() && Objects.equal(left.dataType(), right.dataType()) && Objects.equal(left.comment(), right.comment()) && left.nullable() == right.nullable() @@ -554,11 +556,12 @@ private Pair> updateColumnsIfNecessary( return Pair.of(false, Collections.emptyList()); } - Map columnsFromCatalogTable = + Map> columnsFromCatalogTable = tableFromCatalog.columns() == null ? Collections.emptyMap() - : Arrays.stream(tableFromCatalog.columns()) - .collect(Collectors.toMap(Column::name, Function.identity())); + : IntStream.range(0, tableFromCatalog.columns().length) + .mapToObj(i -> Pair.of(i, tableFromCatalog.columns()[i])) + .collect(Collectors.toMap(p -> p.getRight().name(), Function.identity())); Map columnsFromTableEntity = tableFromGravitino.columns() == null ? Collections.emptyMap() @@ -569,25 +572,27 @@ private Pair> updateColumnsIfNecessary( List columnsToInsert = Lists.newArrayList(); boolean columnsNeedsUpdate = false; for (Map.Entry entry : columnsFromTableEntity.entrySet()) { - Column column = columnsFromCatalogTable.get(entry.getKey()); - if (column == null) { + Pair columnPair = columnsFromCatalogTable.get(entry.getKey()); + if (columnPair == null) { LOG.debug( "Column {} is not found in the table from underlying source, it will be removed" + " from the table entity", entry.getKey()); columnsNeedsUpdate = true; - } else if (!isSameColumn(column, entry.getValue())) { + } else if (!isSameColumn(columnPair.getRight(), columnPair.getLeft(), entry.getValue())) { // If the column need to be updated, we create a new ColumnEntity with the same id LOG.debug( "Column {} is found in the table from underlying source, but it is different " + "from the one in the table entity, it will be updated", entry.getKey()); + Column column = columnPair.getRight(); ColumnEntity updatedColumnEntity = ColumnEntity.builder() .withId(entry.getValue().id()) .withName(column.name()) + .withPosition(columnPair.getLeft()) .withDataType(column.dataType()) .withComment(column.comment()) .withNullable(column.nullable()) @@ -612,7 +617,7 @@ private Pair> updateColumnsIfNecessary( } // Check if there are new columns in the table from the underlying source - for (Map.Entry entry : columnsFromCatalogTable.entrySet()) { + for (Map.Entry> entry : columnsFromCatalogTable.entrySet()) { if (!columnsFromTableEntity.containsKey(entry.getKey())) { LOG.debug( "Column {} is found in the table from underlying source but not in the table " @@ -620,7 +625,8 @@ private Pair> updateColumnsIfNecessary( entry.getKey()); ColumnEntity newColumnEntity = ColumnEntity.toColumnEntity( - entry.getValue(), + entry.getValue().getRight(), + entry.getValue().getLeft(), idGenerator.nextId(), AuditInfo.builder() .withCreator(PrincipalUtils.getCurrentPrincipal().getName()) diff --git a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java index 830a94b4f60..d4778b2ff90 100644 --- a/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java +++ b/core/src/main/java/org/apache/gravitino/connector/PropertiesMetadata.java @@ -89,7 +89,7 @@ default Object getOrDefault(Map properties, String propertyName) throw new IllegalArgumentException("Property is not defined: " + propertyName); } - if (properties.containsKey(propertyName)) { + if (properties != null && properties.containsKey(propertyName)) { return propertyEntries().get(propertyName).decode(properties.get(propertyName)); } return propertyEntries().get(propertyName).getDefaultValue(); diff --git a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java index 3c242bd56fe..95554857a95 100644 --- a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java @@ -28,7 +28,9 @@ import org.apache.gravitino.authorization.Owner; import org.apache.gravitino.authorization.OwnerManager; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.metalake.MetalakeDispatcher; import org.apache.gravitino.utils.NameIdentifierUtil; import org.apache.gravitino.utils.PrincipalUtils; @@ -87,8 +89,19 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { - return dispatcher.dropMetalake(ident); + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.disableMetalake(ident); } @Override diff --git a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java index 04a2600d8ec..ff70d4cffda 100644 --- a/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/CatalogEventDispatcher.java @@ -177,7 +177,7 @@ public void testConnection( @Override public void enableCatalog(NameIdentifier ident) throws NoSuchCatalogException, CatalogNotInUseException { - // todo: support activate catalog event + // todo: support enable catalog event dispatcher.enableCatalog(ident); } diff --git a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java index 33005893f4c..535d337b303 100644 --- a/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/listener/MetalakeEventDispatcher.java @@ -24,7 +24,9 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.listener.api.event.AlterMetalakeEvent; import org.apache.gravitino.listener.api.event.AlterMetalakeFailureEvent; import org.apache.gravitino.listener.api.event.CreateMetalakeEvent; @@ -129,9 +131,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) } @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { try { - boolean isExists = dispatcher.dropMetalake(ident); + boolean isExists = dispatcher.dropMetalake(ident, force); eventBus.dispatchEvent( new DropMetalakeEvent(PrincipalUtils.getCurrentUserName(), ident, isExists)); return isExists; @@ -141,4 +144,16 @@ public boolean dropMetalake(NameIdentifier ident) { throw e; } } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support enable metalake event + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + // todo: support disable metalake event + dispatcher.disableMetalake(ident); + } } diff --git a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java index bab042d378f..4e2ca045f2e 100644 --- a/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java +++ b/core/src/main/java/org/apache/gravitino/meta/BaseMetalake.java @@ -25,13 +25,13 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.ToString; -import org.apache.gravitino.Audit; import org.apache.gravitino.Auditable; import org.apache.gravitino.Entity; import org.apache.gravitino.Field; import org.apache.gravitino.HasIdentifier; import org.apache.gravitino.Metalake; -import org.apache.gravitino.StringIdentifier; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.metalake.MetalakePropertiesMetadata; /** Base implementation of a Metalake entity. */ @EqualsAndHashCode @@ -52,6 +52,8 @@ public class BaseMetalake implements Metalake, Entity, Auditable, HasIdentifier public static final Field SCHEMA_VERSION = Field.required("version", SchemaVersion.class, "The version of the schema for the metalake"); + public static final PropertiesMetadata PROPERTIES_METADATA = new MetalakePropertiesMetadata(); + private Long id; private String name; @@ -87,10 +89,10 @@ public Map fields() { /** * The audit information of the metalake. * - * @return The audit information as an {@link Audit} instance. + * @return The audit information as an {@link AuditInfo} instance. */ @Override - public Audit auditInfo() { + public AuditInfo auditInfo() { return auditInfo; } @@ -141,7 +143,11 @@ public EntityType type() { */ @Override public Map properties() { - return StringIdentifier.newPropertiesWithoutId(properties); + return properties; + } + + public PropertiesMetadata propertiesMetadata() { + return PROPERTIES_METADATA; } /** Builder class for creating instances of {@link BaseMetalake}. */ diff --git a/core/src/main/java/org/apache/gravitino/meta/ColumnEntity.java b/core/src/main/java/org/apache/gravitino/meta/ColumnEntity.java index 37904426013..5e68e48744f 100644 --- a/core/src/main/java/org/apache/gravitino/meta/ColumnEntity.java +++ b/core/src/main/java/org/apache/gravitino/meta/ColumnEntity.java @@ -41,6 +41,8 @@ public class ColumnEntity implements Entity, Auditable { public static final Field ID = Field.required("id", Long.class, "The column's unique identifier"); public static final Field NAME = Field.required("name", String.class, "The column's name"); + public static final Field POSITION = + Field.required("position", Integer.class, "The column's position"); public static final Field TYPE = Field.required("dataType", Type.class, "The column's data type"); public static final Field COMMENT = Field.optional("comment", String.class, "The column's comment"); @@ -57,6 +59,8 @@ public class ColumnEntity implements Entity, Auditable { private String name; + private Integer position; + private Type dataType; private String comment; @@ -76,6 +80,7 @@ public Map fields() { Map fields = Maps.newHashMap(); fields.put(ID, id); fields.put(NAME, name); + fields.put(POSITION, position); fields.put(TYPE, dataType); fields.put(COMMENT, comment); fields.put(NULLABLE, nullable); @@ -104,6 +109,10 @@ public String name() { return name; } + public Integer position() { + return position; + } + public Type dataType() { return dataType; } @@ -132,6 +141,7 @@ public boolean equals(Object o) { ColumnEntity that = (ColumnEntity) o; return Objects.equal(id, that.id) && Objects.equal(name, that.name) + && Objects.equal(position, that.position) && Objects.equal(dataType, that.dataType) && Objects.equal(comment, that.comment) && Objects.equal(nullable, that.nullable) @@ -143,17 +153,19 @@ public boolean equals(Object o) { @Override public int hashCode() { return Objects.hashCode( - id, name, dataType, comment, nullable, autoIncrement, defaultValue, auditInfo); + id, name, position, dataType, comment, nullable, autoIncrement, defaultValue, auditInfo); } public static Builder builder() { return new Builder(); } - public static ColumnEntity toColumnEntity(Column column, long uid, AuditInfo audit) { + public static ColumnEntity toColumnEntity( + Column column, int position, long uid, AuditInfo audit) { return builder() .withId(uid) .withName(column.name()) + .withPosition(position) .withComment(column.comment()) .withDataType(column.dataType()) .withNullable(column.nullable()) @@ -180,6 +192,11 @@ public Builder withName(String name) { return this; } + public Builder withPosition(Integer position) { + columnEntity.position = position; + return this; + } + public Builder withDataType(Type dataType) { columnEntity.dataType = dataType; return this; diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java index 54298c0efdd..33498665a2d 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeManager.java @@ -18,9 +18,12 @@ */ package org.apache.gravitino.metalake; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + import com.google.common.collect.Maps; import java.io.IOException; import java.time.Instant; +import java.util.List; import java.util.Map; import org.apache.gravitino.Entity.EntityType; import org.apache.gravitino.EntityAlreadyExistsException; @@ -28,13 +31,16 @@ import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; -import org.apache.gravitino.StringIdentifier; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; import org.apache.gravitino.meta.AuditInfo; import org.apache.gravitino.meta.BaseMetalake; +import org.apache.gravitino.meta.CatalogEntity; import org.apache.gravitino.meta.SchemaVersion; import org.apache.gravitino.storage.IdGenerator; import org.apache.gravitino.utils.PrincipalUtils; @@ -63,6 +69,48 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { this.idGenerator = idGenerator; } + /** + * Check whether the metalake is available + * + * @param ident The identifier of the Metalake to check. + * @param store The EntityStore to use for managing Metalakes. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + * @throws MetalakeNotInUseException If the Metalake is not in use. + */ + public static void checkMetalake(NameIdentifier ident, EntityStore store) + throws NoSuchMetalakeException, MetalakeNotInUseException { + boolean metalakeInUse = metalakeInUse(store, ident); + if (!metalakeInUse) { + throw new MetalakeNotInUseException( + "Metalake %s is not in use, please enable it first", ident); + } + } + + /** + * Return true if the metalake is in used, false otherwise. + * + * @param store The EntityStore to use for managing Metalakes. + * @param ident The identifier of the Metalake to check. + * @return True if the metalake is in use, false otherwise. + * @throws NoSuchMetalakeException If the Metalake with the given identifier does not exist. + */ + public static boolean metalakeInUse(EntityStore store, NameIdentifier ident) + throws NoSuchMetalakeException { + try { + BaseMetalake metalake = store.get(ident, EntityType.METALAKE, BaseMetalake.class); + return (boolean) + metalake.propertiesMetadata().getOrDefault(metalake.properties(), PROPERTY_IN_USE); + + } catch (NoSuchEntityException e) { + LOG.warn("Metalake {} does not exist", ident, e); + throw new NoSuchMetalakeException(METALAKE_DOES_NOT_EXIST_MSG, ident); + + } catch (IOException e) { + LOG.error("Failed to do store operation", e); + throw new RuntimeException(e); + } + } + /** * Lists all available Metalakes. * @@ -72,8 +120,9 @@ public MetalakeManager(EntityStore store, IdGenerator idGenerator) { @Override public BaseMetalake[] listMetalakes() { try { - return store.list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE).stream() - .toArray(BaseMetalake[]::new); + return store + .list(Namespace.empty(), BaseMetalake.class, EntityType.METALAKE) + .toArray(new BaseMetalake[0]); } catch (IOException ioe) { LOG.error("Listing Metalakes failed due to storage issues.", ioe); throw new RuntimeException(ioe); @@ -116,14 +165,13 @@ public BaseMetalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { long uid = idGenerator.nextId(); - StringIdentifier stringId = StringIdentifier.fromId(uid); BaseMetalake metalake = BaseMetalake.builder() .withId(uid) .withName(ident.name()) .withComment(comment) - .withProperties(StringIdentifier.newPropertiesWithId(stringId, properties)) + .withProperties(properties) .withVersion(SchemaVersion.V_0_1) .withAuditInfo( AuditInfo.builder() @@ -158,28 +206,17 @@ public BaseMetalake createMetalake( public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException { try { + if (!metalakeInUse(store, ident)) { + throw new MetalakeNotInUseException( + "Metalake %s is not in use, please enable it first", ident); + } + return store.update( ident, BaseMetalake.class, EntityType.METALAKE, metalake -> { - BaseMetalake.Builder builder = - BaseMetalake.builder() - .withId(metalake.id()) - .withName(metalake.name()) - .withComment(metalake.comment()) - .withProperties(metalake.properties()) - .withVersion(metalake.getVersion()); - - AuditInfo newInfo = - AuditInfo.builder() - .withCreator(metalake.auditInfo().creator()) - .withCreateTime(metalake.auditInfo().createTime()) - .withLastModifier( - metalake.auditInfo().creator()) /*TODO: Use real user later on. */ - .withLastModifiedTime(Instant.now()) - .build(); - builder.withAuditInfo(newInfo); + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); Map newProps = metalake.properties() == null @@ -204,23 +241,106 @@ public BaseMetalake alterMetalake(NameIdentifier ident, MetalakeChange... change } } - /** - * Deletes a Metalake. - * - * @param ident The identifier of the Metalake to be deleted. - * @return `true` if the Metalake was successfully deleted, `false` otherwise. - * @throws RuntimeException If deleting the Metalake encounters storage issues. - */ @Override - public boolean dropMetalake(NameIdentifier ident) { + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { try { - return store.delete(ident, EntityType.METALAKE); - } catch (IOException ioe) { - LOG.error("Deleting metalake {} failed due to storage issues", ident, ioe); - throw new RuntimeException(ioe); + boolean inUse = metalakeInUse(store, ident); + if (inUse && !force) { + throw new MetalakeInUseException( + "Metalake %s is in use, please disable it first or use force option", ident); + } + + List catalogEntities = + store.list(Namespace.of(ident.name()), CatalogEntity.class, EntityType.CATALOG); + if (!catalogEntities.isEmpty() && !force) { + throw new NonEmptyEntityException( + "Metalake %s has catalogs, please drop them first or use force option", ident); + } + + return store.delete(ident, EntityType.METALAKE, true); + } catch (NoSuchMetalakeException e) { + return false; + + } catch (IOException e) { + throw new RuntimeException(e); } } + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + + boolean inUse = metalakeInUse(store, ident); + if (!inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "true"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + try { + boolean inUse = metalakeInUse(store, ident); + if (inUse) { + store.update( + ident, + BaseMetalake.class, + EntityType.METALAKE, + metalake -> { + BaseMetalake.Builder builder = newMetalakeBuilder(metalake); + + Map newProps = + metalake.properties() == null + ? Maps.newHashMap() + : Maps.newHashMap(metalake.properties()); + newProps.put(PROPERTY_IN_USE, "false"); + builder.withProperties(newProps); + + return builder.build(); + }); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private BaseMetalake.Builder newMetalakeBuilder(BaseMetalake metalake) { + BaseMetalake.Builder builder = + BaseMetalake.builder() + .withId(metalake.id()) + .withName(metalake.name()) + .withComment(metalake.comment()) + .withProperties(metalake.properties()) + .withVersion(metalake.getVersion()); + + AuditInfo newInfo = + AuditInfo.builder() + .withCreator(metalake.auditInfo().creator()) + .withCreateTime(metalake.auditInfo().createTime()) + .withLastModifier(metalake.auditInfo().creator()) /*TODO: Use real user later on. */ + .withLastModifiedTime(Instant.now()) + .build(); + return builder.withAuditInfo(newInfo); + } + /** * Updates an entity with the provided changes. * diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java index b0c8cbf1766..dbc9d6bdc29 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakeNormalizeDispatcher.java @@ -19,16 +19,25 @@ package org.apache.gravitino.metalake; import static org.apache.gravitino.Entity.SYSTEM_METALAKE_RESERVED_NAME; +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForAlter; +import static org.apache.gravitino.catalog.PropertiesMetadataHelpers.validatePropertyForCreate; +import static org.apache.gravitino.meta.BaseMetalake.PROPERTIES_METADATA; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import java.util.Arrays; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; import org.apache.gravitino.Metalake; import org.apache.gravitino.MetalakeChange; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; +import org.apache.gravitino.meta.BaseMetalake; public class MetalakeNormalizeDispatcher implements MetalakeDispatcher { private static final Set RESERVED_WORDS = ImmutableSet.of(SYSTEM_METALAKE_RESERVED_NAME); @@ -57,7 +66,7 @@ public Metalake[] listMetalakes() { @Override public Metalake loadMetalake(NameIdentifier ident) throws NoSuchMetalakeException { - return dispatcher.loadMetalake(ident); + return newMetalakeWithResolvedProperties((BaseMetalake) dispatcher.loadMetalake(ident)); } @Override @@ -70,6 +79,7 @@ public Metalake createMetalake( NameIdentifier ident, String comment, Map properties) throws MetalakeAlreadyExistsException { validateMetalakeName(ident.name()); + validatePropertyForCreate(PROPERTIES_METADATA, properties); return dispatcher.createMetalake(ident, comment, properties); } @@ -83,6 +93,10 @@ public Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) validateMetalakeName(((MetalakeChange.RenameMetalake) change).getNewName()); } }); + Pair, Map> alterProperty = + getMetalakeAlterProperty(changes); + validatePropertyForAlter( + PROPERTIES_METADATA, alterProperty.getLeft(), alterProperty.getRight()); return dispatcher.alterMetalake(ident, changes); } @@ -93,6 +107,22 @@ public boolean dropMetalake(NameIdentifier ident) { return dispatcher.dropMetalake(ident); } + @Override + public boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException { + return dispatcher.dropMetalake(ident, force); + } + + @Override + public void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.enableMetalake(ident); + } + + @Override + public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException { + dispatcher.disableMetalake(ident); + } + private void validateMetalakeName(String name) { if (RESERVED_WORDS.contains(name)) { throw new IllegalArgumentException("The metalake name '" + name + "' is reserved."); @@ -101,4 +131,45 @@ private void validateMetalakeName(String name) { throw new IllegalArgumentException("The metalake name '" + name + "' is illegal."); } } + + private BaseMetalake newMetalakeWithResolvedProperties(BaseMetalake metalakeEntity) { + Map newProps = Maps.newHashMap(metalakeEntity.properties()); + newProps + .entrySet() + .removeIf(e -> metalakeEntity.propertiesMetadata().isHiddenProperty(e.getKey())); + newProps.putIfAbsent( + PROPERTY_IN_USE, + metalakeEntity.propertiesMetadata().getDefaultValue(PROPERTY_IN_USE).toString()); + + return BaseMetalake.builder() + .withId(metalakeEntity.id()) + .withName(metalakeEntity.name()) + .withComment(metalakeEntity.comment()) + .withProperties(newProps) + .withVersion(metalakeEntity.getVersion()) + .withAuditInfo(metalakeEntity.auditInfo()) + .build(); + } + + private Pair, Map> getMetalakeAlterProperty( + MetalakeChange... metalakeChanges) { + Map upserts = Maps.newHashMap(); + Map deletes = Maps.newHashMap(); + + Arrays.stream(metalakeChanges) + .forEach( + metalakeChange -> { + if (metalakeChange instanceof MetalakeChange.SetProperty) { + MetalakeChange.SetProperty setProperty = + (MetalakeChange.SetProperty) metalakeChange; + upserts.put(setProperty.getProperty(), setProperty.getValue()); + } else if (metalakeChange instanceof MetalakeChange.RemoveProperty) { + MetalakeChange.RemoveProperty removeProperty = + (MetalakeChange.RemoveProperty) metalakeChange; + deletes.put(removeProperty.getProperty(), removeProperty.getProperty()); + } + }); + + return Pair.of(upserts, deletes); + } } diff --git a/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java new file mode 100644 index 00000000000..7a39b0b63f8 --- /dev/null +++ b/core/src/main/java/org/apache/gravitino/metalake/MetalakePropertiesMetadata.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.metalake; + +import static org.apache.gravitino.Metalake.PROPERTY_IN_USE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class MetalakePropertiesMetadata implements PropertiesMetadata { + + private static final Map> PROPERTY_ENTRIES; + + static { + List> propertyEntries = + ImmutableList.of( + PropertyEntry.booleanReservedPropertyEntry( + PROPERTY_IN_USE, + "The property indicating the catalog is in use", + true /* default value */, + false /* hidden */)); + + PROPERTY_ENTRIES = Maps.uniqueIndex(propertyEntries, PropertyEntry::getName); + } + + @Override + public Map> propertyEntries() { + return PROPERTY_ENTRIES; + } +} diff --git a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java index bd00c7ff5e8..481bfe84476 100644 --- a/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java +++ b/core/src/main/java/org/apache/gravitino/metalake/SupportsMetalakes.java @@ -24,7 +24,10 @@ import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.annotation.Evolving; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NonEmptyEntityException; /** * Interface for supporting metalakes. It includes methods for listing, loading, creating, altering @@ -89,10 +92,68 @@ Metalake alterMetalake(NameIdentifier ident, MetalakeChange... changes) throws NoSuchMetalakeException, IllegalArgumentException; /** - * Drop a metalake with specified identifier. + * Drop a metalake with specified identifier. Please make sure: + * + *
    + *
  • There is no catalog in the metalake. Otherwise, a {@link NonEmptyEntityException} will be + * thrown. + *
  • The method {@link #disableMetalake(NameIdentifier)} has been called before dropping the + * metalake. Otherwise, a {@link MetalakeInUseException} will be thrown. + *
+ * + * It is equivalent to calling {@code dropMetalake(ident, false)}. + * + * @param ident The identifier of the metalake. + * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty. + * @throws MetalakeInUseException If the metalake is in use. + */ + default boolean dropMetalake(NameIdentifier ident) + throws NonEmptyEntityException, MetalakeInUseException { + return dropMetalake(ident, false); + } + + /** + * Drop a metalake with specified identifier. If the force flag is true, it will: + * + *
    + *
  • Cascade drop all sub-entities (tags, catalogs, schemas, tables, etc.) of the metalake in + * Gravitino store. + *
  • Drop the metalake even if it is in use. + *
  • External resources (e.g. database, table, etc.) associated with sub-entities will not be + * deleted unless it is managed (such as managed fileset). + *
* * @param ident The identifier of the metalake. + * @param force Whether to force the drop. * @return True if the metalake was dropped, false if the metalake does not exist. + * @throws NonEmptyEntityException If the metalake is not empty and force is false. + * @throws MetalakeInUseException If the metalake is in use and force is false. + */ + boolean dropMetalake(NameIdentifier ident, boolean force) + throws NonEmptyEntityException, MetalakeInUseException; + + /** + * Enable a metalake. If the metalake is already in use, this method does nothing. + * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. + */ + void enableMetalake(NameIdentifier ident) throws NoSuchMetalakeException; + + /** + * Disable a metalake. If the metalake is already disabled, this method does nothing. Once a + * metalake is disable: + * + *
    + *
  • It can only be listed, loaded, dropped, or enable. + *
  • Any other operations on the metalake will throw an {@link MetalakeNotInUseException}. + *
  • Any operation on the sub-entities (catalogs, schemas, tables, etc.) will throw an {@link + * MetalakeNotInUseException}. + *
+ * + * @param ident The identifier of the metalake. + * @throws NoSuchMetalakeException If the metalake does not exist. */ - boolean dropMetalake(NameIdentifier ident); + void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException; } diff --git a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java index d8536fffbdb..e18b280e691 100644 --- a/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java +++ b/core/src/main/java/org/apache/gravitino/proto/BaseMetalakeSerDe.java @@ -20,7 +20,6 @@ package org.apache.gravitino.proto; import org.apache.gravitino.Namespace; -import org.apache.gravitino.meta.AuditInfo; /** A class for serializing and deserializing BaseMetalake objects. */ class BaseMetalakeSerDe implements ProtoSerDe { @@ -38,7 +37,7 @@ public Metalake serialize(org.apache.gravitino.meta.BaseMetalake baseMetalake) { Metalake.newBuilder() .setId(baseMetalake.id()) .setName(baseMetalake.name()) - .setAuditInfo(new AuditInfoSerDe().serialize((AuditInfo) baseMetalake.auditInfo())); + .setAuditInfo(new AuditInfoSerDe().serialize(baseMetalake.auditInfo())); if (baseMetalake.comment() != null) { builder.setComment(baseMetalake.comment()); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java index 0af9889ba6a..cdc32425b6f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/base/TableColumnBaseSQLProvider.java @@ -28,6 +28,7 @@ public class TableColumnBaseSQLProvider { public String listColumnPOsByTableIdAndVersion( @Param("tableId") Long tableId, @Param("tableVersion") Long tableVersion) { return "SELECT t1.column_id AS columnId, t1.column_name AS columnName," + + " t1.column_position AS columnPosition," + " t1.metalake_id AS metalakeId, t1.catalog_id AS catalogId," + " t1.schema_id AS schemaId, t1.table_id AS tableId," + " t1.table_version AS tableVersion, t1.column_type AS columnType," @@ -50,15 +51,16 @@ public String insertColumnPOs(@Param("columnPOs") List columnPOs) { return ""; } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/po/ColumnPO.java b/core/src/main/java/org/apache/gravitino/storage/relational/po/ColumnPO.java index 46c79f97320..9238d88503b 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/po/ColumnPO.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/po/ColumnPO.java @@ -127,6 +127,8 @@ public static AutoIncrement fromBoolean(boolean autoIncrement) { private String columnName; + private Integer columnPosition; + private Long metalakeId; private Long catalogId; @@ -177,6 +179,11 @@ public Builder withColumnName(String columnName) { return this; } + public Builder withColumnPosition(Integer columnPosition) { + columnPO.columnPosition = columnPosition; + return this; + } + public Builder withMetalakeId(Long metalakeId) { columnPO.metalakeId = metalakeId; return this; @@ -247,6 +254,7 @@ public ColumnPO build() { Preconditions.checkArgument( StringUtils.isNotBlank(columnPO.columnName), "Column name is required and cannot be blank"); + Preconditions.checkArgument(columnPO.columnPosition != null, "Column position is required"); Preconditions.checkArgument(columnPO.metalakeId != null, "Metalake id is required"); Preconditions.checkArgument(columnPO.catalogId != null, "Catalog id is required"); Preconditions.checkArgument(columnPO.schemaId != null, "Schema id is required"); diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java index 7ec975d45f8..f881602bc53 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/service/TableColumnMetaService.java @@ -110,12 +110,14 @@ void updateColumnPOsFromTableDiff( List columnPOsToInsert = Lists.newArrayList(); for (ColumnEntity newColumn : newColumns.values()) { ColumnEntity oldColumn = oldColumns.get(newColumn.id()); + // If the column is not existed in old columns, or if the column is updated, mark it as UPDATE if (oldColumn == null || !oldColumn.equals(newColumn)) { columnPOsToInsert.add( POConverters.initializeColumnPO(newTablePO, newColumn, ColumnPO.ColumnOpType.UPDATE)); } } + // Mark the columns to DELETE if they are not existed in new columns. for (ColumnEntity oldColumn : oldColumns.values()) { if (!newColumns.containsKey(oldColumn.id())) { columnPOsToInsert.add( diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java index dabce09cb2f..4cccd06759f 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/utils/POConverters.java @@ -452,6 +452,7 @@ public static ColumnEntity fromColumnPO(ColumnPO columnPO) { return ColumnEntity.builder() .withId(columnPO.getColumnId()) .withName(columnPO.getColumnName()) + .withPosition(columnPO.getColumnPosition()) .withDataType(JsonUtils.anyFieldMapper().readValue(columnPO.getColumnType(), Type.class)) .withComment(columnPO.getColumnComment()) .withAutoIncrement( @@ -482,6 +483,7 @@ public static ColumnPO initializeColumnPO( return ColumnPO.builder() .withColumnId(columnEntity.id()) .withColumnName(columnEntity.name()) + .withColumnPosition(columnEntity.position()) .withMetalakeId(tablePO.getMetalakeId()) .withCatalogId(tablePO.getCatalogId()) .withSchemaId(tablePO.getSchemaId()) diff --git a/core/src/main/java/org/apache/gravitino/tag/TagManager.java b/core/src/main/java/org/apache/gravitino/tag/TagManager.java index aaffd35b50a..1b1626de00e 100644 --- a/core/src/main/java/org/apache/gravitino/tag/TagManager.java +++ b/core/src/main/java/org/apache/gravitino/tag/TagManager.java @@ -18,6 +18,8 @@ */ package org.apache.gravitino.tag; +import static org.apache.gravitino.metalake.MetalakeManager.checkMetalake; + import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -35,7 +37,6 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.exceptions.NoSuchEntityException; import org.apache.gravitino.exceptions.NoSuchMetadataObjectException; -import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchTagException; import org.apache.gravitino.exceptions.NotFoundException; import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; @@ -93,7 +94,7 @@ public Tag[] listTagsInfo(String metalake) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore @@ -114,7 +115,7 @@ public Tag createTag(String metalake, String name, String comment, Map { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); TagEntity tagEntity = TagEntity.builder() @@ -148,7 +149,7 @@ public Tag getTag(String metalake, String name) throws NoSuchTagException { ofTagIdent(metalake, name), LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.get( @@ -169,7 +170,7 @@ public Tag alterTag(String metalake, String name, TagChange... changes) NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.update( @@ -195,7 +196,7 @@ public boolean deleteTag(String metalake, String name) { NameIdentifier.of(ofTagNamespace(metalake).levels()), LockType.WRITE, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { return entityStore.delete(ofTagIdent(metalake, name), Entity.EntityType.TAG); @@ -213,7 +214,7 @@ public MetadataObject[] listMetadataObjectsForTag(String metalake, String name) tagId, LockType.READ, () -> { - checkMetalakeExists(metalake, entityStore); + checkMetalake(NameIdentifier.of(metalake), entityStore); try { if (!entityStore.exists(tagId, Entity.EntityType.TAG)) { @@ -356,19 +357,6 @@ public String[] associateTagsForMetadataObject( })); } - private static void checkMetalakeExists(String metalake, EntityStore entityStore) { - try { - NameIdentifier metalakeIdent = NameIdentifier.of(metalake); - if (!entityStore.exists(metalakeIdent, Entity.EntityType.METALAKE)) { - LOG.warn("Metalake {} does not exist", metalakeIdent); - throw new NoSuchMetalakeException("Metalake %s does not exist", metalakeIdent); - } - } catch (IOException ioe) { - LOG.error("Failed to check if metalake exists", ioe); - throw new RuntimeException(ioe); - } - } - public static Namespace ofTagNamespace(String metalake) { return Namespace.of(metalake, Entity.SYSTEM_CATALOG_RESERVED_NAME, Entity.TAG_SCHEMA_NAME); } diff --git a/core/src/test/java/org/apache/gravitino/TestColumn.java b/core/src/test/java/org/apache/gravitino/TestColumn.java index 7085da6d3fb..77410652465 100644 --- a/core/src/test/java/org/apache/gravitino/TestColumn.java +++ b/core/src/test/java/org/apache/gravitino/TestColumn.java @@ -26,17 +26,40 @@ @ToString public class TestColumn extends BaseColumn { + private int position; + private TestColumn() {} + public int position() { + return position; + } + + public void setPosition(int position) { + this.position = position; + } + public static class Builder extends BaseColumn.BaseColumnBuilder { + + private Integer position; + /** Creates a new instance of {@link Builder}. */ private Builder() {} + public Builder withPosition(int position) { + this.position = position; + return this; + } + @Override protected TestColumn internalBuild() { TestColumn column = new TestColumn(); + if (position == null) { + throw new IllegalArgumentException("Position is required"); + } + column.name = name; + column.position = position; column.comment = comment; column.dataType = dataType; column.nullable = nullable; diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestTableNormalizeDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestTableNormalizeDispatcher.java index c45f5cab27d..1401f990ec2 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestTableNormalizeDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestTableNormalizeDispatcher.java @@ -75,8 +75,16 @@ public void testNameCaseInsensitive() { NameIdentifier tableIdent = NameIdentifier.of(tableNs, "tableNAME"); Column[] columns = new Column[] { - TestColumn.builder().withName("colNAME1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("colNAME2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("colNAME1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("colNAME2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; RangePartition assignedPartition = Partitions.range( @@ -143,8 +151,16 @@ public void testNameSpec() { NameIdentifier.of(tableNs, MetadataObjects.METADATA_OBJECT_RESERVED_NAME); Column[] columns = new Column[] { - TestColumn.builder().withName("colNAME1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("colNAME2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("colNAME1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("colNAME2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; Exception exception = Assertions.assertThrows( @@ -166,6 +182,7 @@ public void testNameSpec() { new Column[] { TestColumn.builder() .withName(MetadataObjects.METADATA_OBJECT_RESERVED_NAME) + .withPosition(0) .withType(Types.StringType.get()) .build() }; diff --git a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java index 29eb655a354..6acec229ef2 100644 --- a/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java +++ b/core/src/test/java/org/apache/gravitino/catalog/TestTableOperationDispatcher.java @@ -98,7 +98,16 @@ public void testCreateAndListTables() throws IOException { NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, "table1"); Column[] columns = new Column[] { - Column.of("col1", Types.StringType.get()), Column.of("col2", Types.StringType.get()) + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; Table table1 = @@ -177,8 +186,16 @@ public void testCreateAndLoadTable() throws IOException { NameIdentifier tableIdent1 = NameIdentifier.of(tableNs, "table11"); Column[] columns = new Column[] { - TestColumn.builder().withName("col1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("col2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; Table table1 = @@ -248,8 +265,16 @@ public void testCreateAndAlterTable() throws IOException { NameIdentifier tableIdent = NameIdentifier.of(tableNs, "table21"); Column[] columns = new Column[] { - TestColumn.builder().withName("col1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("col2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; Table table = @@ -314,8 +339,16 @@ public void testCreateAndDropTable() throws IOException { Map props = ImmutableMap.of("k1", "v1", "k2", "v2"); Column[] columns = new Column[] { - TestColumn.builder().withName("col1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("col2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; schemaOperationDispatcher.createSchema( @@ -347,8 +380,16 @@ public void testCreateTableNeedImportingSchema() throws IOException { NameIdentifier.of(tableNs.levels()), "", Collections.emptyMap()); Column[] columns = new Column[] { - TestColumn.builder().withName("col1").withType(Types.StringType.get()).build(), - TestColumn.builder().withName("col2").withType(Types.StringType.get()).build() + TestColumn.builder() + .withName("col1") + .withPosition(0) + .withType(Types.StringType.get()) + .build(), + TestColumn.builder() + .withName("col2") + .withPosition(1) + .withType(Types.StringType.get()) + .build() }; tableOperationDispatcher.createTable(tableIdent, columns, "comment", props); Assertions.assertTrue(entityStore.exists(NameIdentifier.of(tableNs.levels()), SCHEMA)); @@ -366,6 +407,7 @@ public void testCreateAndLoadTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col1") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment1") .withNullable(true) @@ -374,6 +416,7 @@ public void testCreateAndLoadTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col2") + .withPosition(1) .withType(Types.StringType.get()) .withComment("comment2") .withNullable(false) @@ -494,6 +537,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col1") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment1") .withNullable(true) @@ -502,6 +546,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col2") + .withPosition(1) .withType(Types.StringType.get()) .withComment("comment2") .withNullable(false) @@ -523,6 +568,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col3") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment1") .withNullable(true) @@ -531,6 +577,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col2") + .withPosition(1) .withType(Types.StringType.get()) .withComment("comment2") .withNullable(false) @@ -559,6 +606,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment4") .withNullable(true) @@ -567,6 +615,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col3") + .withPosition(1) .withType(Types.StringType.get()) .withComment("comment1") .withNullable(true) @@ -575,6 +624,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col2") + .withPosition(2) .withType(Types.StringType.get()) .withComment("comment2") .withNullable(false) @@ -598,6 +648,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment4") .withNullable(true) @@ -621,6 +672,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment4") .withNullable(true) @@ -643,6 +695,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.IntegerType.get()) .withComment("comment4") .withNullable(true) @@ -665,6 +718,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.IntegerType.get()) .withComment("new comment") .withNullable(true) @@ -687,6 +741,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.IntegerType.get()) .withComment("new comment") .withNullable(false) @@ -709,6 +764,7 @@ public void testCreateAndAlterTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col4") + .withPosition(0) .withType(Types.IntegerType.get()) .withComment("new comment") .withNullable(false) @@ -734,6 +790,7 @@ public void testCreateAndDropTableWithColumn() throws IOException { new Column[] { TestColumn.builder() .withName("col1") + .withPosition(0) .withType(Types.StringType.get()) .withComment("comment1") .withNullable(true) @@ -742,6 +799,7 @@ public void testCreateAndDropTableWithColumn() throws IOException { .build(), TestColumn.builder() .withName("col2") + .withPosition(1) .withType(Types.StringType.get()) .withComment("comment2") .withNullable(false) @@ -776,14 +834,16 @@ private static void testColumns(Column[] expectedColumns, Column[] actualColumns Assertions.assertEquals(expectedColumnMap.size(), actualColumnMap.size()); expectedColumnMap.forEach( (name, expectedColumn) -> { - Column actualColumn = actualColumnMap.get(name); + TestColumn actualColumn = (TestColumn) actualColumnMap.get(name); + TestColumn e = (TestColumn) expectedColumn; Assertions.assertNotNull(actualColumn); - Assertions.assertEquals(expectedColumn.name().toLowerCase(), actualColumn.name()); - Assertions.assertEquals(expectedColumn.dataType(), actualColumn.dataType()); - Assertions.assertEquals(expectedColumn.comment(), actualColumn.comment()); - Assertions.assertEquals(expectedColumn.nullable(), actualColumn.nullable()); - Assertions.assertEquals(expectedColumn.autoIncrement(), actualColumn.autoIncrement()); - Assertions.assertEquals(expectedColumn.defaultValue(), actualColumn.defaultValue()); + Assertions.assertEquals(e.name().toLowerCase(), actualColumn.name()); + Assertions.assertEquals(e.position(), actualColumn.position()); + Assertions.assertEquals(e.dataType(), actualColumn.dataType()); + Assertions.assertEquals(e.comment(), actualColumn.comment()); + Assertions.assertEquals(e.nullable(), actualColumn.nullable()); + Assertions.assertEquals(e.autoIncrement(), actualColumn.autoIncrement()); + Assertions.assertEquals(e.defaultValue(), actualColumn.defaultValue()); }); } @@ -804,13 +864,15 @@ private static void testColumnAndColumnEntities( expectedColumnMap.forEach( (name, expectedColumn) -> { ColumnEntity actualColumn = actualColumnMap.get(name); + TestColumn e = (TestColumn) expectedColumn; Assertions.assertNotNull(actualColumn); - Assertions.assertEquals(expectedColumn.name(), actualColumn.name()); - Assertions.assertEquals(expectedColumn.dataType(), actualColumn.dataType()); - Assertions.assertEquals(expectedColumn.comment(), actualColumn.comment()); - Assertions.assertEquals(expectedColumn.nullable(), actualColumn.nullable()); - Assertions.assertEquals(expectedColumn.autoIncrement(), actualColumn.autoIncrement()); - Assertions.assertEquals(expectedColumn.defaultValue(), actualColumn.defaultValue()); + Assertions.assertEquals(e.name(), actualColumn.name()); + Assertions.assertEquals(e.position(), actualColumn.position()); + Assertions.assertEquals(e.dataType(), actualColumn.dataType()); + Assertions.assertEquals(e.comment(), actualColumn.comment()); + Assertions.assertEquals(e.nullable(), actualColumn.nullable()); + Assertions.assertEquals(e.autoIncrement(), actualColumn.autoIncrement()); + Assertions.assertEquals(e.defaultValue(), actualColumn.defaultValue()); }); } } diff --git a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java index 13c4652058a..4fb98c596b8 100644 --- a/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java +++ b/core/src/test/java/org/apache/gravitino/connector/TestCatalogOperations.java @@ -24,10 +24,12 @@ import java.io.IOException; import java.time.Instant; import java.util.Arrays; +import java.util.Comparator; import java.util.HashMap; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.NameIdentifier; @@ -136,13 +138,29 @@ public Table createTable( AuditInfo auditInfo = AuditInfo.builder().withCreator("test").withCreateTime(Instant.now()).build(); + TestColumn[] sortedColumns = + IntStream.range(0, columns.length) + .mapToObj( + i -> + TestColumn.builder() + .withName(columns[i].name()) + .withPosition(i) + .withComment(columns[i].comment()) + .withType(columns[i].dataType()) + .withNullable(columns[i].nullable()) + .withAutoIncrement(columns[i].autoIncrement()) + .withDefaultValue(columns[i].defaultValue()) + .build()) + .sorted(Comparator.comparingInt(TestColumn::position)) + .toArray(TestColumn[]::new); + TestTable table = TestTable.builder() .withName(ident.name()) .withComment(comment) .withProperties(new HashMap<>(properties)) .withAuditInfo(auditInfo) - .withColumns(columns) + .withColumns(sortedColumns) .withDistribution(distribution) .withSortOrders(sortOrders) .withPartitioning(partitions) @@ -160,7 +178,7 @@ public Table createTable( .withComment(comment) .withProperties(new HashMap<>(properties)) .withAuditInfo(auditInfo) - .withColumns(columns) + .withColumns(sortedColumns) .withDistribution(distribution) .withSortOrders(sortOrders) .withPartitioning(partitions) @@ -646,6 +664,39 @@ private boolean checkSingleFile(Fileset fileset) { } } + private Map updateColumnPositionsAfterColumnUpdate( + String updatedColumnName, + TableChange.ColumnPosition newColumnPosition, + Map allColumns) { + TestColumn updatedColumn = (TestColumn) allColumns.get(updatedColumnName); + int newPosition; + if (newColumnPosition instanceof TableChange.First) { + newPosition = 0; + } else if (newColumnPosition instanceof TableChange.Default) { + newPosition = allColumns.size() - 1; + } else if (newColumnPosition instanceof TableChange.After) { + String afterColumnName = ((TableChange.After) newColumnPosition).getColumn(); + Column afterColumn = allColumns.get(afterColumnName); + newPosition = ((TestColumn) afterColumn).position() + 1; + } else { + throw new IllegalArgumentException("Unsupported column position: " + newColumnPosition); + } + updatedColumn.setPosition(newPosition); + + allColumns.forEach( + (columnName, column) -> { + if (columnName.equals(updatedColumnName)) { + return; + } + TestColumn testColumn = (TestColumn) column; + if (testColumn.position() >= newPosition) { + testColumn.setPosition(testColumn.position() + 1); + } + }); + + return allColumns; + } + private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] columnChanges) { Map columnMap = Arrays.stream(columns).collect(Collectors.toMap(Column::name, Function.identity())); @@ -656,6 +707,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn column = TestColumn.builder() .withName(String.join(".", addColumn.fieldName())) + .withPosition(columnMap.size()) .withComment(addColumn.getComment()) .withType(addColumn.getDataType()) .withNullable(addColumn.isNullable()) @@ -663,9 +715,18 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu .withDefaultValue(addColumn.getDefaultValue()) .build(); columnMap.put(column.name(), column); + updateColumnPositionsAfterColumnUpdate(column.name(), addColumn.getPosition(), columnMap); } else if (columnChange instanceof TableChange.DeleteColumn) { - columnMap.remove(String.join(".", columnChange.fieldName())); + TestColumn removedColumn = + (TestColumn) columnMap.remove(String.join(".", columnChange.fieldName())); + columnMap.forEach( + (columnName, column) -> { + TestColumn testColumn = (TestColumn) column; + if (testColumn.position() > removedColumn.position()) { + testColumn.setPosition(testColumn.position() - 1); + } + }); } else if (columnChange instanceof TableChange.RenameColumn) { String oldName = String.join(".", columnChange.fieldName()); @@ -674,6 +735,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(newName) + .withPosition(((TestColumn) column).position()) .withComment(column.comment()) .withType(column.dataType()) .withNullable(column.nullable()) @@ -690,6 +752,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(columnName) + .withPosition(((TestColumn) oldColumn).position()) .withComment(oldColumn.comment()) .withType(oldColumn.dataType()) .withNullable(oldColumn.nullable()) @@ -705,6 +768,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(columnName) + .withPosition(((TestColumn) oldColumn).position()) .withComment(oldColumn.comment()) .withType(updateColumnType.getNewDataType()) .withNullable(oldColumn.nullable()) @@ -721,6 +785,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(columnName) + .withPosition(((TestColumn) oldColumn).position()) .withComment(updateColumnComment.getNewComment()) .withType(oldColumn.dataType()) .withNullable(oldColumn.nullable()) @@ -737,6 +802,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(columnName) + .withPosition(((TestColumn) oldColumn).position()) .withComment(oldColumn.comment()) .withType(oldColumn.dataType()) .withNullable(updateColumnNullable.nullable()) @@ -753,6 +819,7 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu TestColumn newColumn = TestColumn.builder() .withName(columnName) + .withPosition(((TestColumn) oldColumn).position()) .withComment(oldColumn.comment()) .withType(oldColumn.dataType()) .withNullable(oldColumn.nullable()) @@ -761,10 +828,22 @@ private Column[] updateColumns(Column[] columns, TableChange.ColumnChange[] colu .build(); columnMap.put(columnName, newColumn); + } else if (columnChange instanceof TableChange.UpdateColumnPosition) { + String columnName = String.join(".", columnChange.fieldName()); + TableChange.UpdateColumnPosition updateColumnPosition = + (TableChange.UpdateColumnPosition) columnChange; + columnMap = + updateColumnPositionsAfterColumnUpdate( + columnName, updateColumnPosition.getPosition(), columnMap); + } else { - // do nothing + throw new IllegalArgumentException("Unsupported column change: " + columnChange); } } - return columnMap.values().toArray(new Column[0]); + + return columnMap.values().stream() + .map(TestColumn.class::cast) + .sorted(Comparator.comparingInt(TestColumn::position)) + .toArray(TestColumn[]::new); } } diff --git a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java index 319ac641f61..2b5377f7598 100644 --- a/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java +++ b/core/src/test/java/org/apache/gravitino/listener/api/event/TestMetalakeEvent.java @@ -19,6 +19,7 @@ package org.apache.gravitino.listener.api.event; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -205,7 +206,7 @@ private MetalakeDispatcher mockMetalakeDispatcher() { when(dispatcher.createMetalake(any(NameIdentifier.class), any(String.class), any(Map.class))) .thenReturn(metalake); when(dispatcher.loadMetalake(any(NameIdentifier.class))).thenReturn(metalake); - when(dispatcher.dropMetalake(any(NameIdentifier.class))).thenReturn(true); + when(dispatcher.dropMetalake(any(NameIdentifier.class), anyBoolean())).thenReturn(true); when(dispatcher.listMetalakes()).thenReturn(null); when(dispatcher.alterMetalake(any(NameIdentifier.class), any(MetalakeChange.class))) .thenReturn(metalake); diff --git a/core/src/test/java/org/apache/gravitino/meta/TestColumnEntity.java b/core/src/test/java/org/apache/gravitino/meta/TestColumnEntity.java index e7ad7e7d076..519bcfca362 100644 --- a/core/src/test/java/org/apache/gravitino/meta/TestColumnEntity.java +++ b/core/src/test/java/org/apache/gravitino/meta/TestColumnEntity.java @@ -36,6 +36,7 @@ public void testColumnEntityFields() { ColumnEntity.builder() .withId(1L) .withName("test") + .withPosition(1) .withComment("test comment") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -47,6 +48,7 @@ public void testColumnEntityFields() { Assertions.assertEquals(1L, columnEntity.id()); Assertions.assertEquals("test", columnEntity.name()); + Assertions.assertEquals(1, columnEntity.position()); Assertions.assertEquals("test comment", columnEntity.comment()); Assertions.assertEquals(Types.IntegerType.get(), columnEntity.dataType()); Assertions.assertTrue(columnEntity.nullable()); @@ -57,6 +59,7 @@ public void testColumnEntityFields() { ColumnEntity.builder() .withId(1L) .withName("test") + .withPosition(1) .withDataType(Types.IntegerType.get()) .withNullable(true) .withAutoIncrement(true) @@ -70,6 +73,7 @@ public void testColumnEntityFields() { ColumnEntity.builder() .withId(1L) .withName("test") + .withPosition(1) .withComment("test comment") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -87,6 +91,7 @@ public void testWithoutRequiredFields() { () -> { ColumnEntity.builder() .withId(1L) + .withPosition(1) .withName("test") .withNullable(true) .withAutoIncrement(true) @@ -101,6 +106,7 @@ public void testWithoutRequiredFields() { () -> { ColumnEntity.builder() .withId(1L) + .withPosition(1) .withComment("test comment") .withDataType(Types.IntegerType.get()) .withAutoIncrement(true) @@ -115,6 +121,7 @@ public void testWithoutRequiredFields() { () -> { ColumnEntity.builder() .withId(1L) + .withName("test") .withComment("test comment") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -143,6 +150,7 @@ public void testTableColumnEntity() { ColumnEntity.builder() .withId(1L) .withName("test") + .withPosition(1) .withComment("test comment") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -156,6 +164,7 @@ public void testTableColumnEntity() { ColumnEntity.builder() .withId(2L) .withName("test2") + .withPosition(2) .withComment("test comment2") .withDataType(Types.StringType.get()) .withNullable(true) @@ -169,6 +178,7 @@ public void testTableColumnEntity() { ColumnEntity.builder() .withId(3L) .withName("test3") + .withPosition(3) .withComment("test comment3") .withDataType(Types.BooleanType.get()) .withNullable(true) diff --git a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java index 96bc1ebdd82..3b0f796e979 100644 --- a/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java +++ b/core/src/test/java/org/apache/gravitino/metalake/TestMetalakeManager.java @@ -173,6 +173,7 @@ public void testDropMetalake() { Assertions.assertEquals("comment", metalake.comment()); testProperties(props, metalake.properties()); + metalakeManager.disableMetalake(ident); boolean dropped = metalakeManager.dropMetalake(ident); Assertions.assertTrue(dropped, "metalake should be dropped"); diff --git a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java index 6502b6931ea..01d919676e7 100644 --- a/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java +++ b/core/src/test/java/org/apache/gravitino/storage/TestEntityStorage.java @@ -578,10 +578,10 @@ void testEntityDelete(String type) throws IOException { createSchemaEntity(1L, Namespace.of("metalake", "catalog"), "schema1", auditInfo); ColumnEntity column1 = createColumnEntity( - RandomIdGenerator.INSTANCE.nextId(), "column1", Types.StringType.get(), auditInfo); + RandomIdGenerator.INSTANCE.nextId(), "column1", 0, Types.StringType.get(), auditInfo); ColumnEntity column2 = createColumnEntity( - RandomIdGenerator.INSTANCE.nextId(), "column2", Types.StringType.get(), auditInfo); + RandomIdGenerator.INSTANCE.nextId(), "column2", 1, Types.StringType.get(), auditInfo); TableEntity table1 = createTableEntityWithColumns( 1L, @@ -600,10 +600,10 @@ void testEntityDelete(String type) throws IOException { createSchemaEntity(2L, Namespace.of("metalake", "catalog"), "schema2", auditInfo); ColumnEntity column3 = createColumnEntity( - RandomIdGenerator.INSTANCE.nextId(), "column3", Types.StringType.get(), auditInfo); + RandomIdGenerator.INSTANCE.nextId(), "column3", 2, Types.StringType.get(), auditInfo); ColumnEntity column4 = createColumnEntity( - RandomIdGenerator.INSTANCE.nextId(), "column4", Types.StringType.get(), auditInfo); + RandomIdGenerator.INSTANCE.nextId(), "column4", 3, Types.StringType.get(), auditInfo); TableEntity table1InSchema2 = createTableEntityWithColumns( 2L, @@ -695,9 +695,7 @@ void testEntityDelete(String type) throws IOException { // metalake BaseMetalake metalakeNew = createBaseMakeLake( - RandomIdGenerator.INSTANCE.nextId(), - metalake.name(), - (AuditInfo) metalake.auditInfo()); + RandomIdGenerator.INSTANCE.nextId(), metalake.name(), metalake.auditInfo()); store.put(metalakeNew); // catalog CatalogEntity catalogNew = @@ -976,7 +974,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake1"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake1New.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake1New.id(), "metalake2", e.auditInfo())); // Rename metalake3 --> metalake1 BaseMetalake metalake3New1 = @@ -986,7 +984,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New1.id(), "metalake1", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New1.id(), "metalake1", e.auditInfo())); // Rename metalake3 --> metalake2 BaseMetalake metalake3New2 = @@ -998,7 +996,7 @@ void testDeleteAndRename(String type) throws IOException { NameIdentifier.of("metalake3"), BaseMetalake.class, Entity.EntityType.METALAKE, - e -> createBaseMakeLake(metalake3New2.id(), "metalake2", (AuditInfo) e.auditInfo())); + e -> createBaseMakeLake(metalake3New2.id(), "metalake2", e.auditInfo())); // Finally, only metalake2 and metalake1 are left. Assertions.assertDoesNotThrow( @@ -1257,10 +1255,11 @@ public static SchemaEntity createSchemaEntity( } public static ColumnEntity createColumnEntity( - Long id, String name, Type dataType, AuditInfo auditInfo) { + Long id, String name, int position, Type dataType, AuditInfo auditInfo) { return ColumnEntity.builder() .withId(id) .withName(name) + .withPosition(position) .withComment("") .withDataType(dataType) .withNullable(true) diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java index 0b15d1e0f5c..8d61d357cc7 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/service/TestTableColumnMetaService.java @@ -76,6 +76,7 @@ public void testInsertAndGetTableColumns() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column1") + .withPosition(0) .withComment("comment1") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -87,6 +88,7 @@ public void testInsertAndGetTableColumns() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column2") + .withPosition(1) .withComment("comment2") .withDataType(Types.StringType.get()) .withNullable(false) @@ -118,6 +120,7 @@ public void testInsertAndGetTableColumns() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column3") + .withPosition(0) .withComment("comment3") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -187,6 +190,7 @@ public void testUpdateTable() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column1") + .withPosition(0) .withComment("comment1") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -222,6 +226,7 @@ public void testUpdateTable() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column2") + .withPosition(1) .withComment("comment2") .withDataType(Types.StringType.get()) .withNullable(false) @@ -257,6 +262,7 @@ public void testUpdateTable() throws IOException { ColumnEntity.builder() .withId(column1.id()) .withName(column1.name()) + .withPosition(column1.position()) .withComment("comment1_updated") .withDataType(Types.LongType.get()) .withNullable(column1.nullable()) @@ -331,6 +337,7 @@ public void testCreateAndDeleteTable() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column1") + .withPosition(0) .withComment("comment1") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -377,6 +384,7 @@ public void testDeleteMetalake() throws IOException { ColumnEntity.builder() .withId(RandomIdGenerator.INSTANCE.nextId()) .withName("column1") + .withPosition(0) .withComment("comment1") .withDataType(Types.IntegerType.get()) .withNullable(true) @@ -423,6 +431,7 @@ private void compareTwoColumns( Assertions.assertNotNull(expectedColumn); Assertions.assertEquals(expectedColumn.id(), column.id()); Assertions.assertEquals(expectedColumn.name(), column.name()); + Assertions.assertEquals(expectedColumn.position(), column.position()); Assertions.assertEquals(expectedColumn.comment(), column.comment()); Assertions.assertEquals(expectedColumn.dataType(), column.dataType()); Assertions.assertEquals(expectedColumn.nullable(), column.nullable()); diff --git a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java index e97e650a144..76a2c35d317 100644 --- a/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java +++ b/core/src/test/java/org/apache/gravitino/storage/relational/utils/TestPOConverters.java @@ -160,6 +160,7 @@ public void testFromColumnPO() throws JsonProcessingException { createColumnPO( 1L, "test", + 0, 1L, 1L, 1L, @@ -173,7 +174,7 @@ public void testFromColumnPO() throws JsonProcessingException { ColumnEntity expectedColumn = createColumn( - 1L, "test", Types.IntegerType.get(), "test", true, true, Literals.integerLiteral(1)); + 1L, "test", 0, Types.IntegerType.get(), "test", true, true, Literals.integerLiteral(1)); ColumnEntity convertedColumn = POConverters.fromColumnPO(columnPO); assertEquals(expectedColumn.id(), convertedColumn.id()); @@ -189,6 +190,7 @@ public void testFromColumnPO() throws JsonProcessingException { createColumnPO( 1L, "test", + 0, 1L, 1L, 1L, @@ -202,7 +204,7 @@ public void testFromColumnPO() throws JsonProcessingException { ColumnEntity expectedColumn1 = createColumn( - 1L, "test", Types.IntegerType.get(), null, true, true, Literals.integerLiteral(1)); + 1L, "test", 0, Types.IntegerType.get(), null, true, true, Literals.integerLiteral(1)); ColumnEntity convertedColumn1 = POConverters.fromColumnPO(columnPO1); assertEquals(expectedColumn1.comment(), convertedColumn1.comment()); @@ -215,6 +217,7 @@ public void testFromTableColumnPOs() throws JsonProcessingException { createColumnPO( 1L, "test1", + 0, 1L, 1L, 1L, @@ -230,6 +233,7 @@ public void testFromTableColumnPOs() throws JsonProcessingException { createColumnPO( 2L, "test2", + 1, 1L, 1L, 1L, @@ -243,11 +247,25 @@ public void testFromTableColumnPOs() throws JsonProcessingException { ColumnEntity expectedColumn1 = createColumn( - 1L, "test1", Types.IntegerType.get(), "test1", true, true, Literals.integerLiteral(1)); + 1L, + "test1", + 0, + Types.IntegerType.get(), + "test1", + true, + true, + Literals.integerLiteral(1)); ColumnEntity expectedColumn2 = createColumn( - 2L, "test2", Types.StringType.get(), "test2", true, true, Literals.stringLiteral("1")); + 2L, + "test2", + 1, + Types.StringType.get(), + "test2", + true, + true, + Literals.stringLiteral("1")); TableEntity expectedTable = createTableWithColumns( @@ -962,6 +980,7 @@ private static TablePO createTablePO( private static ColumnPO createColumnPO( Long id, String columnName, + Integer columnPosition, Long metalakeId, Long catalogId, Long schemaId, @@ -978,6 +997,7 @@ private static ColumnPO createColumnPO( return ColumnPO.builder() .withColumnId(id) .withColumnName(columnName) + .withColumnPosition(columnPosition) .withMetalakeId(metalakeId) .withCatalogId(catalogId) .withSchemaId(schemaId) @@ -999,6 +1019,7 @@ private static ColumnPO createColumnPO( private static ColumnEntity createColumn( Long id, String columnName, + Integer columnPosition, Type columnType, String columnComment, boolean columnNullable, @@ -1009,6 +1030,7 @@ private static ColumnEntity createColumn( return ColumnEntity.builder() .withId(id) .withName(columnName) + .withPosition(columnPosition) .withDataType(columnType) .withComment(columnComment) .withNullable(columnNullable) diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md index 4c336f3d323..14595f06cbf 100644 --- a/docs/lakehouse-paimon-catalog.md +++ b/docs/lakehouse-paimon-catalog.md @@ -22,17 +22,16 @@ Builds with Apache Paimon `0.8.0`. ### Catalog capabilities -- Works as a catalog proxy, supporting `FilesystemCatalog` and `JdbcCatalog`. +- Works as a catalog proxy, supporting `FilesystemCatalog`, `JdbcCatalog` and `HiveCatalog`. - Supports DDL operations for Paimon schemas and tables. -- Doesn't support `HiveCatalog` catalog backend now. - Doesn't support alterSchema. ### Catalog properties | Property name | Description | Default value | Required | Since Version | |----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------|-----------------------------------------------------------------|-------------------| -| `catalog-backend` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem` and `jdbc` now. | (none) | Yes | 0.6.0-incubating | +| `catalog-backend` | Catalog backend of Gravitino Paimon catalog. Supports `filesystem`, `jdbc` and `hive`. | (none) | Yes | 0.6.0-incubating | | `uri` | The URI configuration of the Paimon catalog. `thrift://127.0.0.1:9083` or `jdbc:postgresql://127.0.0.1:5432/db_name` or `jdbc:mysql://127.0.0.1:3306/metastore_db`. It is optional for `FilesystemCatalog`. | (none) | required if the value of `catalog-backend` is not `filesystem`. | 0.6.0-incubating | | `warehouse` | Warehouse directory of catalog. `file:///user/hive/warehouse-paimon/` for local fs, `hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or `oss://{bucket-name}/path` for Aliyun OSS | (none) | Yes | 0.6.0-incubating | | `authentication.type` | The type of authentication for Paimon catalog backend, currently Gravitino only supports `Kerberos` and `simple`. | `simple` | No | 0.6.0-incubating | @@ -51,6 +50,9 @@ Builds with Apache Paimon `0.8.0`. If you want to use the `oss` or `s3` warehouse, you need to place related jars in the `catalogs/lakehouse-paimon/lib` directory, more information can be found in the [Paimon S3](https://paimon.apache.org/docs/master/filesystems/s3/). ::: +:::note +The hive backend does not support the kerberos authentication now. +::: Any properties not defined by Gravitino with `gravitino.bypass.` prefix will pass to Paimon catalog properties and HDFS configuration. For example, if specify `gravitino.bypass.table.type`, `table.type` will pass to Paimon catalog properties. diff --git a/docs/spark-connector/spark-authentication-with-gravitino.md b/docs/spark-connector/spark-authentication-with-gravitino.md new file mode 100644 index 00000000000..9ec4a949ae0 --- /dev/null +++ b/docs/spark-connector/spark-authentication-with-gravitino.md @@ -0,0 +1,39 @@ +--- +title: "Spark authentication with Gravitino server" +slug: /spark-connector/spark-authentication +keyword: spark connector authentication oauth2 kerberos +license: "This software is licensed under the Apache License version 2." +--- + +## Overview + +Spark connector supports `simple` `oauth2` and `kerberos` authentication when accessing Gravitino server. + +| Property | Type | Default Value | Description | Required | Since Version | +|------------------------------|--------|---------------|---------------------------------------------------------------------------------------------------------------------|----------|------------------| +| spark.sql.gravitino.authType | string | `simple` | The authentication mechanisms when communicating with Gravitino server, supports `simple`, `oauth2` and `kerberos`. | No | 0.7.0-incubating | + +## Simple mode + +In the simple mode, the username originates from Spark, and is obtained using the following sequences: +1. The environment variable of `SPARK_USER` +2. The environment variable of `HADOOP_USER_NAME` +3. The user login in the machine + +## OAuth2 mode + +In the OAuth2 mode, you could use the following configuration to fetch an OAuth2 token to access Gravitino server. + +| Property | Type | Default Value | Description | Required | Since Version | +|---------------------------------------|--------|---------------|-----------------------------------------------|----------------------|------------------| +| spark.sql.gravitino.oauth2.serverUri | string | None | The OAuth2 server uri address. | Yes, for OAuth2 mode | 0.7.0-incubating | +| spark.sql.gravitino.oauth2.tokenPath | string | None | The path of token interface in OAuth2 server. | Yes, for OAuth2 mode | 0.7.0-incubating | +| spark.sql.gravitino.oauth2.credential | string | None | The credential to request the OAuth2 token. | Yes, for OAuth2 mode | 0.7.0-incubating | +| spark.sql.gravitino.oauth2.scope | string | None | The scope to request the OAuth2 token. | Yes, for OAuth2 mode | 0.7.0-incubating | + +## Kerberos mode + +In kerberos mode, you could use the Spark kerberos configuration to fetch a kerberos ticket to access Gravitino server, use `spark.kerberos.principal`, `spark.kerberos.keytab` to specify kerberos principal and keytab. + +The principal of Gravitino server is like `HTTP/$host@$realm`, please keep the `$host` consistent with the host in Gravitino server uri address. +Please make sure `krb5.conf` is accessible by Spark, like by specifying the configuration `spark.driver.extraJavaOptions="-Djava.security.krb5.conf=/xx/krb5.conf"`. diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fc06975c19d..6a50fc2b455 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -150,7 +150,8 @@ hadoop2-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref hadoop2-hdfs-client = { group = "org.apache.hadoop", name = "hadoop-hdfs-client", version.ref = "hadoop2" } hadoop2-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop2"} hadoop2-mapreduce-client-core = { group = "org.apache.hadoop", name = "hadoop-mapreduce-client-core", version.ref = "hadoop2"} -hadoop2-s3 = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = "hadoop2"} +hadoop2-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = "hadoop2"} +hadoop3-aws = { group = "org.apache.hadoop", name = "hadoop-aws", version.ref = "hadoop3"} hadoop3-hdfs = { group = "org.apache.hadoop", name = "hadoop-hdfs", version.ref = "hadoop3" } hadoop3-common = { group = "org.apache.hadoop", name = "hadoop-common", version.ref = "hadoop3"} hadoop3-client = { group = "org.apache.hadoop", name = "hadoop-client", version.ref = "hadoop3"} diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java index 192ba2c0cb5..517d6e877ee 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/BaseContainer.java @@ -24,6 +24,7 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.InspectContainerResponse; import com.github.dockerjava.api.model.ContainerNetwork; +import com.github.dockerjava.api.model.Ulimit; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -79,8 +80,8 @@ protected BaseContainer( cmd -> cmd.getHostConfig() .withSysctls( - Collections.singletonMap( - "net.ipv4.ip_local_port_range", "20000 40000"))); + Collections.singletonMap("net.ipv4.ip_local_port_range", "20000 40000")) + .withUlimits(new Ulimit[] {new Ulimit("nproc", 120000L, 120000L)})); this.ports = requireNonNull(ports, "ports is null"); this.hostName = requireNonNull(hostName, "hostName is null"); this.extraHosts = extraHosts; diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/HttpUtils.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/HttpUtils.java index 6ccac7dd76e..0fe4d728c73 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/HttpUtils.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/HttpUtils.java @@ -29,7 +29,7 @@ public class HttpUtils { private static final Logger LOG = LoggerFactory.getLogger(HttpUtils.class); /** - * Check if the http server is up, If http response status code is 200, then we're assuming the + * Check if the http server is up. If http response status code is 200, then we're assuming the * server is up. Or else we assume the server is not ready. * *

Note: The method will ignore the response body and only check the status code. diff --git a/scripts/h2/schema-0.7.0-h2.sql b/scripts/h2/schema-0.7.0-h2.sql index bada37abc32..87253c502dd 100644 --- a/scripts/h2/schema-0.7.0-h2.sql +++ b/scripts/h2/schema-0.7.0-h2.sql @@ -93,6 +93,7 @@ CREATE TABLE IF NOT EXISTS `table_column_version_info` ( `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', + `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', `column_type` TEXT NOT NULL COMMENT 'column type', `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', diff --git a/scripts/h2/upgrade-0.6.0-to-0.7.0-h2.sql b/scripts/h2/upgrade-0.6.0-to-0.7.0-h2.sql index cdf1bbdc432..4143b7ad6fb 100644 --- a/scripts/h2/upgrade-0.6.0-to-0.7.0-h2.sql +++ b/scripts/h2/upgrade-0.6.0-to-0.7.0-h2.sql @@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS `table_column_version_info` ( `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', + `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', `column_type` TEXT NOT NULL COMMENT 'column type', `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', diff --git a/scripts/mysql/schema-0.7.0-mysql.sql b/scripts/mysql/schema-0.7.0-mysql.sql index 13f46debc0d..79bdce4c0d5 100644 --- a/scripts/mysql/schema-0.7.0-mysql.sql +++ b/scripts/mysql/schema-0.7.0-mysql.sql @@ -88,6 +88,7 @@ CREATE TABLE IF NOT EXISTS `table_column_version_info` ( `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', + `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', `column_type` TEXT NOT NULL COMMENT 'column type', `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', diff --git a/scripts/mysql/upgrade-0.6.0-to-0.7.0-mysql.sql b/scripts/mysql/upgrade-0.6.0-to-0.7.0-mysql.sql index 0afe5607841..a20b980cd34 100644 --- a/scripts/mysql/upgrade-0.6.0-to-0.7.0-mysql.sql +++ b/scripts/mysql/upgrade-0.6.0-to-0.7.0-mysql.sql @@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS `table_column_version_info` ( `table_version` INT UNSIGNED NOT NULL COMMENT 'table version', `column_id` BIGINT(20) UNSIGNED NOT NULL COMMENT 'column id', `column_name` VARCHAR(128) NOT NULL COMMENT 'column name', + `column_position` INT UNSIGNED NOT NULL COMMENT 'column position, starting from 0', `column_type` TEXT NOT NULL COMMENT 'column type', `column_comment` VARCHAR(256) DEFAULT '' COMMENT 'column comment', `column_nullable` TINYINT(1) NOT NULL DEFAULT 1 COMMENT 'column nullable, 0 is not nullable, 1 is nullable', diff --git a/scripts/postgresql/schema-0.7.0-postgresql.sql b/scripts/postgresql/schema-0.7.0-postgresql.sql index d377c57b556..da13cfd9116 100644 --- a/scripts/postgresql/schema-0.7.0-postgresql.sql +++ b/scripts/postgresql/schema-0.7.0-postgresql.sql @@ -149,6 +149,7 @@ CREATE TABLE IF NOT EXISTS table_column_version_info ( table_version INT NOT NULL, column_id BIGINT NOT NULL, column_name VARCHAR(128) NOT NULL, + column_position INT NOT NULL, column_type TEXT NOT NULL, column_comment VARCHAR(256) DEFAULT '', column_nullable SMALLINT NOT NULL DEFAULT 1, @@ -173,6 +174,7 @@ COMMENT ON COLUMN table_column_version_info.table_id IS 'table id'; COMMENT ON COLUMN table_column_version_info.table_version IS 'table version'; COMMENT ON COLUMN table_column_version_info.column_id IS 'column id'; COMMENT ON COLUMN table_column_version_info.column_name IS 'column name'; +COMMENT ON COLUMN table_column_version_info.column_position IS 'column position, starting from 0'; COMMENT ON COLUMN table_column_version_info.column_type IS 'column type'; COMMENT ON COLUMN table_column_version_info.column_comment IS 'column comment'; COMMENT ON COLUMN table_column_version_info.column_nullable IS 'column nullable, 0 is not nullable, 1 is nullable'; diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java index 2fe844d8c02..284a07b849f 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/ExceptionHandlers.java @@ -24,16 +24,18 @@ import org.apache.gravitino.dto.responses.ErrorResponse; import org.apache.gravitino.exceptions.AlreadyExistsException; import org.apache.gravitino.exceptions.CatalogAlreadyExistsException; -import org.apache.gravitino.exceptions.CatalogInUseException; -import org.apache.gravitino.exceptions.CatalogNotInUseException; import org.apache.gravitino.exceptions.ConnectionFailedException; import org.apache.gravitino.exceptions.FilesetAlreadyExistsException; import org.apache.gravitino.exceptions.ForbiddenException; import org.apache.gravitino.exceptions.GroupAlreadyExistsException; +import org.apache.gravitino.exceptions.InUseException; import org.apache.gravitino.exceptions.MetalakeAlreadyExistsException; +import org.apache.gravitino.exceptions.MetalakeInUseException; +import org.apache.gravitino.exceptions.MetalakeNotInUseException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NonEmptySchemaException; import org.apache.gravitino.exceptions.NotFoundException; +import org.apache.gravitino.exceptions.NotInUseException; import org.apache.gravitino.exceptions.PartitionAlreadyExistsException; import org.apache.gravitino.exceptions.RoleAlreadyExistsException; import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; @@ -131,6 +133,9 @@ public static Response handleTestConnectionException(Exception e) { } else if (e instanceof AlreadyExistsException) { response = ErrorResponse.alreadyExists(e.getClass().getSimpleName(), e.getMessage(), e); + } else if (e instanceof NotInUseException) { + response = ErrorResponse.notInUse(e.getClass().getSimpleName(), e.getMessage(), e); + } else { return Utils.internalError(e.getMessage(), e); } @@ -180,7 +185,7 @@ public Response handle(OperationType op, String partition, String table, Excepti } else if (e instanceof UnsupportedOperationException) { return Utils.unsupportedOperation(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -221,7 +226,7 @@ public Response handle(OperationType op, String table, String schema, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -265,7 +270,7 @@ public Response handle(OperationType op, String schema, String catalog, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -306,10 +311,10 @@ public Response handle(OperationType op, String catalog, String metalake, Except } else if (e instanceof CatalogAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); - } else if (e instanceof CatalogInUseException) { + } else if (e instanceof InUseException) { return Utils.inUse(errorMsg, e); } else { @@ -343,6 +348,12 @@ public Response handle(OperationType op, String metalake, String parent, Excepti } else if (e instanceof NoSuchMetalakeException) { return Utils.notFound(errorMsg, e); + } else if (e instanceof MetalakeNotInUseException) { + return Utils.notInUse(errorMsg, e); + + } else if (e instanceof MetalakeInUseException) { + return Utils.inUse(errorMsg, e); + } else { return super.handle(op, metalake, parent, e); } @@ -377,7 +388,7 @@ public Response handle(OperationType op, String fileset, String schema, Exceptio } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); - } else if (e instanceof CatalogNotInUseException) { + } else if (e instanceof NotInUseException) { return Utils.notInUse(errorMsg, e); } else { @@ -418,6 +429,9 @@ public Response handle(OperationType op, String user, String metalake, Exception } else if (e instanceof UserAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, user, metalake, e); } @@ -450,6 +464,9 @@ public Response handle(OperationType op, String group, String metalake, Exceptio } else if (e instanceof GroupAlreadyExistsException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, group, metalake, e); } @@ -485,6 +502,9 @@ public Response handle(OperationType op, String role, String metalake, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, role, metalake, e); } @@ -518,6 +538,10 @@ public Response handle(OperationType op, String topic, String schema, Exception } else if (e instanceof ForbiddenException) { return Utils.forbidden(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, topic, schema, e); } @@ -581,6 +605,9 @@ public Response handle(OperationType op, String roles, String parent, Exception } else if (e instanceof NotFoundException) { return Utils.notFound(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, roles, parent, e); } @@ -616,6 +643,9 @@ public Response handle(OperationType op, String tag, String parent, Exception e) } else if (e instanceof TagAlreadyAssociatedException) { return Utils.alreadyExists(errorMsg, e); + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, tag, parent, e); } @@ -643,6 +673,10 @@ public Response handle(OperationType op, String name, String parent, Exception e } else if (e instanceof NotFoundException) { return Utils.notFound(errorMsg, e); + + } else if (e instanceof NotInUseException) { + return Utils.notInUse(errorMsg, e); + } else { return super.handle(op, name, parent, e); } diff --git a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java index fff86cf2e87..e28f28f15e5 100644 --- a/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java +++ b/server/src/main/java/org/apache/gravitino/server/web/rest/MetalakeOperations.java @@ -25,12 +25,15 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; +import javax.ws.rs.PATCH; import javax.ws.rs.POST; import javax.ws.rs.PUT; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -40,8 +43,10 @@ import org.apache.gravitino.Namespace; import org.apache.gravitino.dto.MetalakeDTO; import org.apache.gravitino.dto.requests.MetalakeCreateRequest; +import org.apache.gravitino.dto.requests.MetalakeSetRequest; import org.apache.gravitino.dto.requests.MetalakeUpdateRequest; import org.apache.gravitino.dto.requests.MetalakeUpdatesRequest; +import org.apache.gravitino.dto.responses.BaseResponse; import org.apache.gravitino.dto.responses.DropResponse; import org.apache.gravitino.dto.responses.MetalakeListResponse; import org.apache.gravitino.dto.responses.MetalakeResponse; @@ -149,6 +154,43 @@ public Response loadMetalake(@PathParam("name") String metalakeName) { } } + @PATCH + @Path("{name}") + @Produces("application/vnd.gravitino.v1+json") + @Timed(name = "set-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) + @ResponseMetered(name = "set-metalake", absolute = true) + public Response setMetalake(@PathParam("name") String metalakeName, MetalakeSetRequest request) { + LOG.info("Received set request for metalake: {}", metalakeName); + try { + return Utils.doAs( + httpRequest, + () -> { + NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); + TreeLockUtils.doWithTreeLock( + identifier, + LockType.WRITE, + () -> { + if (request.isInUse()) { + metalakeDispatcher.enableMetalake(identifier); + } else { + metalakeDispatcher.disableMetalake(identifier); + } + return null; + }); + Response response = Utils.ok(new BaseResponse()); + LOG.info( + "Successfully {} metalake: {}", + request.isInUse() ? "enable" : "disable", + metalakeName); + return response; + }); + + } catch (Exception e) { + LOG.info("Failed to {} metalake: {}", request.isInUse() ? "enable" : "disable", metalakeName); + return ExceptionHandlers.handleMetalakeException(OperationType.LOAD, metalakeName, e); + } + } + @PUT @Path("{name}") @Produces("application/vnd.gravitino.v1+json") @@ -186,7 +228,9 @@ public Response alterMetalake( @Produces("application/vnd.gravitino.v1+json") @Timed(name = "drop-metalake." + MetricNames.HTTP_PROCESS_DURATION, absolute = true) @ResponseMetered(name = "drop-metalake", absolute = true) - public Response dropMetalake(@PathParam("name") String metalakeName) { + public Response dropMetalake( + @PathParam("name") String metalakeName, + @DefaultValue("false") @QueryParam("force") boolean force) { LOG.info("Received drop metalake request for metalake: {}", metalakeName); try { return Utils.doAs( @@ -195,7 +239,7 @@ public Response dropMetalake(@PathParam("name") String metalakeName) { NameIdentifier identifier = NameIdentifierUtil.ofMetalake(metalakeName); boolean dropped = TreeLockUtils.doWithRootTreeLock( - LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier)); + LockType.WRITE, () -> metalakeDispatcher.dropMetalake(identifier, force)); if (!dropped) { LOG.warn("Failed to drop metalake by name {}", metalakeName); } diff --git a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java index 3b73cf035be..9b265e016d2 100644 --- a/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java +++ b/server/src/test/java/org/apache/gravitino/server/web/rest/TestMetalakeOperations.java @@ -22,6 +22,7 @@ import static org.apache.gravitino.Configs.TREE_LOCK_MAX_NODE_IN_MEMORY; import static org.apache.gravitino.Configs.TREE_LOCK_MIN_NODE_IN_MEMORY; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -381,7 +382,7 @@ public void testAlterMetalake() { @Test public void testDropMetalake() { - when(metalakeManager.dropMetalake(any())).thenReturn(true); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(true); Response resp = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -396,7 +397,7 @@ public void testDropMetalake() { Assertions.assertTrue(dropped); // Test when failed to drop metalake - when(metalakeManager.dropMetalake(any())).thenReturn(false); + when(metalakeManager.dropMetalake(any(), anyBoolean())).thenReturn(false); Response resp2 = target("/metalakes/test") .request(MediaType.APPLICATION_JSON_TYPE) @@ -408,7 +409,9 @@ public void testDropMetalake() { Assertions.assertFalse(dropResponse2.dropped()); // Test throw an exception when deleting tenant. - doThrow(new RuntimeException("Internal error")).when(metalakeManager).dropMetalake(any()); + doThrow(new RuntimeException("Internal error")) + .when(metalakeManager) + .dropMetalake(any(), anyBoolean()); Response resp1 = target("/metalakes/test") diff --git a/settings.gradle.kts b/settings.gradle.kts index 10cf107497c..6d08431f0fb 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -70,6 +70,7 @@ project(":spark-connector:spark-runtime-3.5").projectDir = file("spark-connector include("web:web", "web:integration-test") include("docs") include("integration-test-common") +include(":bundles:aws-bundle") include(":bundles:gcp-bundle") include("bundles:aliyun-bundle") findProject(":bundles:aliyun-bundle")?.name = "aliyun-bundle" diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/GravitinoSparkConfig.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/GravitinoSparkConfig.java index 160e6f6ec96..630ae5c33a1 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/GravitinoSparkConfig.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/GravitinoSparkConfig.java @@ -19,6 +19,8 @@ package org.apache.gravitino.spark.connector; +import org.apache.gravitino.auth.AuthProperties; + public class GravitinoSparkConfig { private static final String GRAVITINO_PREFIX = "spark.sql.gravitino."; @@ -26,6 +28,20 @@ public class GravitinoSparkConfig { public static final String GRAVITINO_METALAKE = GRAVITINO_PREFIX + "metalake"; public static final String GRAVITINO_ENABLE_ICEBERG_SUPPORT = GRAVITINO_PREFIX + "enableIcebergSupport"; + + public static final String GRAVITINO_AUTH_TYPE = + GRAVITINO_PREFIX + AuthProperties.GRAVITINO_CLIENT_AUTH_TYPE; + public static final String GRAVITINO_OAUTH2_URI = + GRAVITINO_PREFIX + AuthProperties.GRAVITINO_OAUTH2_SERVER_URI; + public static final String GRAVITINO_OAUTH2_PATH = + GRAVITINO_PREFIX + AuthProperties.GRAVITINO_OAUTH2_TOKEN_PATH; + public static final String GRAVITINO_OAUTH2_CREDENTIAL = + GRAVITINO_PREFIX + AuthProperties.GRAVITINO_OAUTH2_CREDENTIAL; + public static final String GRAVITINO_OAUTH2_SCOPE = + GRAVITINO_PREFIX + AuthProperties.GRAVITINO_OAUTH2_SCOPE; + public static final String GRAVITINO_KERBEROS_PRINCIPAL = "spark.kerberos.principal"; + public static final String GRAVITINO_KERBEROS_KEYTAB_FILE_PATH = "spark.kerberos.keytab"; + public static final String GRAVITINO_HIVE_METASTORE_URI = "metastore.uris"; public static final String SPARK_HIVE_METASTORE_URI = "hive.metastore.uris"; diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java index cd3bd9cc201..f04193f33a3 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/catalog/GravitinoCatalogManager.java @@ -19,14 +19,14 @@ package org.apache.gravitino.spark.connector.catalog; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import java.util.Arrays; import java.util.Map; import java.util.concurrent.ExecutionException; import org.apache.gravitino.Catalog; -import org.apache.gravitino.client.GravitinoAdminClient; -import org.apache.gravitino.client.GravitinoMetalake; +import org.apache.gravitino.client.GravitinoClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,22 +37,18 @@ public class GravitinoCatalogManager { private volatile boolean isClosed = false; private final Cache gravitinoCatalogs; - private final String metalakeName; - private final GravitinoMetalake metalake; - private final GravitinoAdminClient gravitinoClient; + private final GravitinoClient gravitinoClient; - private GravitinoCatalogManager(String gravitinoUri, String metalakeName) { - this.metalakeName = metalakeName; - this.gravitinoClient = GravitinoAdminClient.builder(gravitinoUri).build(); + private GravitinoCatalogManager(Supplier clientBuilder) { + this.gravitinoClient = clientBuilder.get(); // Will not evict catalog by default this.gravitinoCatalogs = CacheBuilder.newBuilder().build(); - this.metalake = gravitinoClient.loadMetalake(metalakeName); } - public static GravitinoCatalogManager create(String gravitinoUrl, String metalakeName) { + public static GravitinoCatalogManager create(Supplier clientBuilder) { Preconditions.checkState( gravitinoCatalogManager == null, "Should not create duplicate GravitinoCatalogManager"); - gravitinoCatalogManager = new GravitinoCatalogManager(gravitinoUrl, metalakeName); + gravitinoCatalogManager = new GravitinoCatalogManager(clientBuilder); return gravitinoCatalogManager; } @@ -80,12 +76,8 @@ public Catalog getGravitinoCatalogInfo(String name) { } } - public String getMetalakeName() { - return metalakeName; - } - public void loadRelationalCatalogs() { - Catalog[] catalogs = metalake.listCatalogsInfo(); + Catalog[] catalogs = gravitinoClient.listCatalogsInfo(); Arrays.stream(catalogs) .filter(catalog -> Catalog.Type.RELATIONAL.equals(catalog.type())) .forEach(catalog -> gravitinoCatalogs.put(catalog.name(), catalog)); @@ -96,7 +88,7 @@ public Map getCatalogs() { } private Catalog loadCatalog(String catalogName) { - Catalog catalog = metalake.loadCatalog(catalogName); + Catalog catalog = gravitinoClient.loadCatalog(catalogName); Preconditions.checkArgument( Catalog.Type.RELATIONAL.equals(catalog.type()), "Only support relational catalog"); LOG.info("Load catalog {} from Gravitino successfully.", catalogName); diff --git a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java index 335e85a3880..d46ef3d9f18 100644 --- a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java +++ b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/plugin/GravitinoDriverPlugin.java @@ -24,18 +24,26 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; +import javax.annotation.Nullable; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; +import org.apache.gravitino.auth.AuthProperties; +import org.apache.gravitino.client.DefaultOAuth2TokenProvider; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.client.GravitinoClient.ClientBuilder; +import org.apache.gravitino.client.KerberosTokenProvider; import org.apache.gravitino.spark.connector.GravitinoSparkConfig; import org.apache.gravitino.spark.connector.catalog.GravitinoCatalogManager; import org.apache.gravitino.spark.connector.iceberg.extensions.GravitinoIcebergSparkSessionExtensions; import org.apache.gravitino.spark.connector.version.CatalogNameAdaptor; +import org.apache.hadoop.security.UserGroupInformation; import org.apache.spark.SparkConf; import org.apache.spark.SparkContext; import org.apache.spark.api.plugin.DriverPlugin; @@ -83,7 +91,9 @@ public Map init(SparkContext sc, PluginContext pluginContext) { gravitinoDriverExtensions.addAll(gravitinoIcebergExtensions); } - this.catalogManager = GravitinoCatalogManager.create(gravitinoUri, metalake); + this.catalogManager = + GravitinoCatalogManager.create( + () -> createGravitinoClient(gravitinoUri, metalake, conf, sc.sparkUser())); catalogManager.loadRelationalCatalogs(); registerGravitinoCatalogs(conf, catalogManager.getCatalogs()); registerSqlExtensions(conf); @@ -155,4 +165,57 @@ private void registerSqlExtensions(SparkConf conf) { conf.set(StaticSQLConf.SPARK_SESSION_EXTENSIONS().key(), extensionString); } } + + private static GravitinoClient createGravitinoClient( + String uri, String metalake, SparkConf sparkConf, String sparkUser) { + ClientBuilder builder = GravitinoClient.builder(uri).withMetalake(metalake); + String authType = + sparkConf.get(GravitinoSparkConfig.GRAVITINO_AUTH_TYPE, AuthProperties.SIMPLE_AUTH_TYPE); + if (AuthProperties.isSimple(authType)) { + Preconditions.checkArgument( + !UserGroupInformation.isSecurityEnabled(), + "Spark simple auth mode doesn't support setting kerberos configurations"); + builder.withSimpleAuth(sparkUser); + } else if (AuthProperties.isOAuth2(authType)) { + String oAuthUri = getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_OAUTH2_URI); + String credential = + getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_OAUTH2_CREDENTIAL); + String path = getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_OAUTH2_PATH); + String scope = getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_OAUTH2_SCOPE); + DefaultOAuth2TokenProvider oAuth2TokenProvider = + DefaultOAuth2TokenProvider.builder() + .withUri(oAuthUri) + .withCredential(credential) + .withPath(path) + .withScope(scope) + .build(); + builder.withOAuth(oAuth2TokenProvider); + } else if (AuthProperties.isKerberos(authType)) { + String principal = + getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_KERBEROS_PRINCIPAL); + String keyTabFile = + getRequiredConfig(sparkConf, GravitinoSparkConfig.GRAVITINO_KERBEROS_KEYTAB_FILE_PATH); + KerberosTokenProvider kerberosTokenProvider = + KerberosTokenProvider.builder() + .withClientPrincipal(principal) + .withKeyTabFile(new File(keyTabFile)) + .build(); + builder.withKerberosAuth(kerberosTokenProvider); + } else { + throw new UnsupportedOperationException("Unsupported auth type: " + authType); + } + return builder.build(); + } + + private static String getRequiredConfig(SparkConf sparkConf, String configKey) { + String configValue = sparkConf.get(configKey, null); + Preconditions.checkArgument( + StringUtils.isNotBlank(configValue), configKey + " should not be empty"); + return configValue; + } + + @Nullable + private static String getOptionalConfig(SparkConf sparkConf, String configKey) { + return sparkConf.get(configKey, null); + } } diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index eabd7732194..5271197d2d3 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -156,7 +156,7 @@ private static void dropMetalake() { if (!exists) { return; } - gravitinoClient.dropMetalake(metalakeName); + gravitinoClient.dropMetalake(metalakeName, true); } private static void createCatalog( diff --git a/web/web/src/app/metalakes/metalake/MetalakeTree.js b/web/web/src/app/metalakes/metalake/MetalakeTree.js index 58b7c138bfb..74065f93336 100644 --- a/web/web/src/app/metalakes/metalake/MetalakeTree.js +++ b/web/web/src/app/metalakes/metalake/MetalakeTree.js @@ -70,6 +70,8 @@ const MetalakeTree = props => { return 'custom-icons-doris' case 'lakehouse-paimon': return 'custom-icons-paimon' + case 'lakehouse-hudi': + return 'custom-icons-hudi' default: return 'bx:book' } diff --git a/web/web/src/lib/api/metalakes/index.js b/web/web/src/lib/api/metalakes/index.js index a34fad5a4bf..5b7b142284b 100644 --- a/web/web/src/lib/api/metalakes/index.js +++ b/web/web/src/lib/api/metalakes/index.js @@ -47,7 +47,7 @@ export const createMetalakeApi = data => { export const deleteMetalakeApi = name => { return defHttp.delete({ - url: `${Apis.DELETE}/${name}` + url: `${Apis.DELETE}/${name}?force=true` }) } diff --git a/web/web/src/lib/icons/iconify-icons.css b/web/web/src/lib/icons/iconify-icons.css index 9fbd7ad75ac..f5803fdf93d 100644 --- a/web/web/src/lib/icons/iconify-icons.css +++ b/web/web/src/lib/icons/iconify-icons.css @@ -19,6 +19,7 @@ .custom-icons-doris, .custom-icons-hive, +.custom-icons-hudi, .custom-icons-paimon { display: inline-block; width: 1em; @@ -36,6 +37,10 @@ background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1000 900' width='1000' height='900'%3E%3Cpath fill-rule='evenodd' stroke='%23fdee21' stroke-width='5.129' d='M596.797 704.59v192.85h47.346v-88.632l9.767.183v88.45h47.881V704.59h-47.88v71.552h-9.768V704.59zm163.92 0h-46.116v192.85h46.116zm7.55 0 36.429 192.85h54.984l36.429-192.85h-48.951l-12.067 82.517c-1.006 3.16-4.402 3.16-5.009 0l-12.06-82.52zm229.17 0h-91.538v192.85h91.538v-50.569l-50.898-.434v-21.679h24.248v-50.204H946.54v-19.397h50.898z'/%3E%3Cpath fill='%23fdee21' fill-rule='evenodd' d='M503.127 119.752c-7.71-21.115-272.92-124.22-276.31-112.9-48.7 12.695-68.52 68.826-102.02 104.38l-74.393-4.261C17.633 149.113.437 192.17 12.146 236.921c42.002 56.906 90.76 105.33 121.15 176.81 2.402 33.692 145.82 3.533 176.56-3.136-41.992 30.059-78.561 76.65-62.846 210.84 14.346 63.014 24.159 133.37 151.4 204.64 16.75 9.381 51.407 20.207 72.838 28.098 20.495 9.4 44.461 13.264 112.07-7.413 39.124-16.863 81.365-27.022 119.65-43.844l-46.017 2.16c-63.369 1.378-112.29 6.105-127.38-11.6l-58.32-100.68 34-66.04c47.056 4.826 62.675 42.986 104.15 57.518l48.885-36.215c141.99 83.816 198.48-53.12 214.67-159.77-1.728-43.392-93.952 13.61-88.362-6.68 2.166-46.643-35.854-107.67-60.42-155.22l28.43-110.01c12.9-11.49-59.72-133.86-119.02-149.12-52.03-13.39-130.46 52.493-130.46 52.493z'/%3E%3Cg fill-rule='evenodd'%3E%3Cg fill='%23c8c037'%3E%3Cpath d='M736.387 353.55c.51 11.816.452 26.187 1.594 35.15-.197 6.838-5.323 7.089-9.564 8.521l27.63 10.12c5.486 9.23 9.895 18.462 14.348 27.693 5.165 22.708 1.31 23.357-2.126 25.031-10.027.102-20.025.121-29.358-1.864 4.181 2.207 5.1 3.815 5.543 6.642.816 5.212-2.54 12.33-8.067 19.188 8.172 4.534 23.071 8.97 34.14 13.181l12.62-28.093c-5.715-41.27-23.045-79.309-46.76-115.57zm51.57 138.43c15.324 6.387 36.495 4.515 65.848-10.613 4.365-3.104 8.431-1.248.62 3.858-38.5 34.547-56.928 17.265-66.468 6.755'/%3E%3Cpath d='M856.747 476.84c4.552 8.717 1.794 15.805.266 23.167-4.754 37.449-16.423 67.056-39.321 97.994-74.478 119.74-149.67 44.817-209.62-3.945l-24.798 62.137c-1.317 5.446-6.79 9.688 31.937 26.738l39.452-27.867c150.46 114.48 257.08-160.24 202.08-178.22zm-230.53 107c-10.151.724-40.361 10.997-41.706 16.946l13.902-20.712zm-19.74-126.97c1.86 0 10.788 2.397 9.83 1.864-1.37-.763-.153 12.519 2.657 19.439l-10.893 23.7c20.865-24.063 53.858-22.462 84.486-25.564l-14.612-9.054c2.538-8.466-.45-15.165-1.86-22.368zm51.27-71.36c-22.387 6.863-43.816 18.075-58.203 41.976 10.976-39.395 13.213-37.899 16.757-39.846 15.351-6.865 27.922-2.586 41.446-2.13'/%3E%3C/g%3E%3Cg fill='%23fcf6a0'%3E%3Cpath d='M538.197 847.25c-27.07 29.87-87.47-2.26-137.63-18.64-127.46-81.05-152.36-157.35-154.09-232.2-6.553-107.22 26.37-169.52 68.014-184.27-27.561 53.129-40.41 148.5-27.631 219.42 10.315 39.455 10.529 106 76.545 141.62 32.327 18.199 23.571 32.368 45.431 49.413 23.955 18.679 90.747 36.437 129.36 24.658zm40.44-495.51c-45.831-64.991-110.29-89.387-182.98-92.053 14.568-4.749 29.136-7.222 43.704-14.246 3.567-3.748 2.401-10.344 1.063-17.042-69.97-18.27-114.13-40.85-170.03-61.781l150.91 36.215c101.91 3.872 93.158 29.032 157.34 148.91z'/%3E%3Cpath d='M627.707 320.4c-33.87-48.35-66.72-107.03-111.31-144.35-107.64-48.584-214.12-84.666-338.91-117.5l39.827-51.216c132.42 30.227 256.8 80.808 368.21 164.19 18.849 47.66 31.419 95.383 42.173 148.87zm68.28-85.44s-18.988-42.599-28.233-58.866c-21.446-23.766-32.29-68.054-76.213-88.922 13.892 3.752 23.399-.718 51.759 24.992l44.669 84.734z'/%3E%3Cpath d='M719.607 308.76c4.38-36.908 12.695-96.11 3.152-119.5-26.55-35.624-53.29-72.446-80.04-106.63-4.43-4.123-7.62-9.719-11.11-14.084 37.61 9.744 76.86 35.894 129.79 139.13z'/%3E%3C/g%3E%3Cpath d='M561.487 313.12c-14.98-11.79-28.33-53.49-51.03-62.16-21.011-1.758-28.715-8.368-56.344-2.739 10.021-5.02 19.482-11.446 30.134-14.885 7.215-1.703 13.983.184 20.867 1.134 1.7-.804 2.814-1.803 2.398-3.313-27.812-17.187-84.866-17.432-123.38-27.243 44.7 1.496 94.156-.982 127.75 9.79 27.013 23.974 35.788 69.12 49.605 99.416m81.91-201.093c6.201.563 39.574 53.353 41.142 62.137 2.775 19.202 9.62 40.728 11.46 60.819-5.497-19.269-11.486-38.537-18.974-57.806-2.19-5.306-7.724-17.302-23.107-34.269-7.163-11.959-8.71-21.359-10.52-30.88zm113.09 287.523h-14.278l15.781 4.142zm-97.42-14.26c-14.358-1.765-29.091-2-43.448 1.157-5.852 7.86-6.255 15.856-8.813 23.245 17.454-19.236 24.38-20.412 52.261-24.401zm195.82 99.28c-7.99 6.377-8.788 12.379-32.413 19.572-17.228 4.074-26.961-2.831-35.07-12.116 12.294 3.522 14.259 12.81 47.225 2.397zm-71.6 3.99c-5.75 15.812-11.949 31.791-18.199 46.6-14.883 17.454-8.54 7.508-30.819 34.617 7.129-10.998 16.4-21.566 21.254-33.02 3.41-7.542 7.653-15.543 9.532-22.244-5.907-2.95-17.999-3.183-19.639-2.125-19.808 11.683-23.382 24.047-35.059 36.086 8.464-13.53 15.693-28.298 25.505-40.476 1.118-1.58 8.383-1.886 12.85-2.819-6.941-1.19-19.787-3.684-20.82-3.572-12.923 3.106-20.164 14.361-29.757 21.836 6.994-10.098 13.436-20.612 22.317-29.292.879-1.02 24.883 1.083 37.324 6.047l.247 4.95 7.063 2.85 5.314-13.314zm-156.75 95.33c-3.188-1.09-31.35-4.711-31.35-3.861-10.813 7.696-12.453 15.059-12.221 21.436 17.989-17.697 17.319-17.747 43.572-17.575zm-21.39-127.28s20.322-2.963 19.926-1.065c-1.472 7.05 2.488 24.233 2.923 21.836l-11.424 9.32c23.926-12.03 49.113-9.612 74.656-11.45 0 0-21.507-5.938-20.192-6.125 4.797-.68-1.278-22.43-2.923-22.9 4.091-1.385 8.087-2.43 12.221-3.462-30.756-8.049-48.541-3.306-75.188 13.847z'/%3E%3Cpath d='m781.027 471.78-4.517 2.263c-.64-13.23-14.85-12.26-26.834-12.382l-9.83-2.396c2.99 2.766 12.23 2.55 8.502 8.521-3.137 1.677-4.58 8.87-6.376 14.912l-5.314.266c9.387 3.669 18.095 6.884 28.428 11.184l6.11.932 14.613-7.855z'/%3E%3Cpath fill='%23fff' d='M646.327 462.86c-2.287-4.525-13.794-3.882-16.275-.955-2.016 2.379-.125 17.642 2.46 15.068 3.736-3.285 8.617-4.659 14.48-4.394.739-2.82.477-5.864-.665-9.719m110.35 22.74c1.082-.791 3.369-8.443 2.63-10.733-1.512-5.403-12.212-4.266-12.212-4.266-2.61 1.536-4.899 9.96-3.945 12.74.456 2.352 11.81 3.102 13.526 2.259z'/%3E%3Cpath d='m299.107 749.64 6.76.661c40.07 19.675 106.05 48.847 201.57 33.171l14.988 23.584c-42.067 20.442-87.312 15.738-129.71 17.172zm-57.53-155.54 20.073 84.268c53.74 33.593 145.54 72.07 222.06 68.965l-16.91-38.8c-159.73-33.92-173.24-76.8-225.22-114.44zm49.95-172.61c9.81 65.302 22.907 114.81 79.69 156.28 34.804 25.03 69.351 49.954 111.37 70.524 0 0-3.684 19.171-6.763 18.829-125.05-13.89-216.6-117.58-227.69-164.94 10.74-36.329 26.149-59.021 43.389-80.693zm60.11-37c16.664 72.376 56.397 145.68 95.889 212.18 14.477 18.369 18.266 26.475 40.579 37.659 30.159 9.55 51.486 7.112 73.377 5.676-5.998-11.102-11.329-22.706-18.217-33.14-49.018-38.765-26.364-74.06-13.344-96.406-27.256-6.887-63.37-21.525-68.345-40.778-8-62.271-3.937-82.025 4.068-114.01-36.909 7.782-74.309 15.494-114.01 28.82z'/%3E%3Cpath stroke='%23000' stroke-width='.114' d='M216.237.058c-12.131 7.925-23.818 19.256-36.391 32.135-20.843 21.575-34.699 42.621-55.915 58.224-4.243 3.659-16.151 9.551-31.121 10.425-7.052.352-11.646 1.519-24.604-.108-11.428-6.188-22.285-2.164-32.881 10.51-11.619 16.709-26.334 48.437-32.453 68.41-12.611 50.937 19.541 92.905 50.374 125.25 27.429 26.796 43.243 43.542 54.505 68.107 8.527 15.933 14.844 37.799 21.683 53.13 2.466 4.86 1.953 4.835 8.591 6.3 14.333 3.059 34.215 3.083 51.915 4.604 7.659.107 18.175-.178 28.14-1.217 13.731-2.592 29.863-5.133 43.384-9.81 13.213-3.253 24.985-7.759 35.597-11.906-1.368 4.644-11.478 9.115-15.268 15.002-35.97 51.49-45.823 97.308-39.56 169.6 3.542 32.042 10.895 58.605 21.991 88.997 5.078 13.908 15.882 35.934 26.236 50.565 30.79 43.506 99.672 99.374 195.56 120.88 16.73 2.286 35.715 1.067 53.571-3.689 47.326-14.346 143.78-48.275 143.78-48.275s-85.619 7.083-124.83 3.206c-9.078-1.42-19.08-1.901-25.405-8.087-1.06-1.37-4.914-9.132-2.425-9.202 3.395-.095 13.142-4.14 28.19-5.482-32.128-3.459-31.67-3.418-34.015-9.27-3.648-9.085-9.23-21.502-14.977-32.367 14.118 1.19 45.376 2.945 55.984-8.003 0 0-18.497 2.136-34.843.219-5.508-.646-14.891-3.995-17.71-5.03-7.339-2.858-13.38-3.746-14.788-5.61-2.549-6.485-4.276-8.666-7.351-17.355-4.166-11.504-4.496-24.337-5.29-36.083 10.693 13.18 24.318 24.275 42.356 29.94.232-.487 23.4 10.073 40.226 4.643l3.489-1.126c0 .312-11.148.977-15.237-1.047-34.219-14.545-39.37-27.276-44.895-33.599l-14.777-22.133c4.494-8.96 7.035-9.301 12.989-9.384 18.011 2 25.848 3.54 36.807.896 7.405 15.24 9.406 30.57 26.265 41.662 56.166 16.672 68.369-4.951 81.973-24.057 40.412 29.658 106.2 38.803 151.89.514 58.086-67.212 76.476-173.17 71.325-179.22-7.254-12.307-16.772-24.999-24.945-23.245-29.13 7.952-39.871 22.73-68.735 19.36 3.438-.195 9.219-.288 9.263-.616 2.224-24.44-.179-36.267-1.252-38.297-8.759-19.317-20.261-39.646-28.278-54.88-2.067-3.161-8.137-27.166-18.308-36.649-4.362-3.724-15.039-13.546-15.039-13.546l-.905 10.44s4.174.63 5.763 7.097c6.022 24.517 36.687 82.485 38.938 85.263 10.868 17.545 1.104 39.4 9.367 51.663.794 1.557 16.857-.137 29.307.628 20.473-4.403 19.609-13.426 37.294-14.782 11.913-.913 13.108 21.46 12.95 23.196-2.22 24.704-9.734 53.427-21.235 79.658-23.912 46.07-50.691 87.446-88.67 93.216-46.326 8.095-70.4-12.158-95.512-25.055l-9.607 8.194c-32.637 32.5-71.389 29.614-87.239-12.851-7.89-16.473-18.172-26.57-26.976-40.7l-46.688 33.627c-3.886 7.97-8.665 20.54-14.47 34.58-4.036 9.76-7.42 26.525-7.2 40.456-5.984 10.195 20.73 51.542 37.61 76.692 4.98 7.42 14.279 20.385 14.59 21.193 3.347 8.683 10.679 16.349 11.124 17.284 31.337 40.872-39.531 31.46-54.167 28.99-28.876-4.6-57.121-16.662-83.67-32.791a339 339 0 01-4.605-2.848c-31.551-19.866-60.581-45.283-85.264-70.423-14.718-16.666-28.732-50.827-39.083-75.045-15.513-58.226-37.776-159.15 22.532-235.63 3.823-4.372 7.948-11.65 11.46-13.09 17.918-12.12 37.375-20.38 58.298-24.966l-2.112-13.563c-10.45 2.374-45.633 15.243-55.581 20.629-22.623 6.558-41.388 13.406-70.22 20.625-9.373 1.146-18.592 1.167-27.58-.054-20.74-2.818-56.093-.3-58.348-2.396-14.044-19.585-17.89-54.676-30.703-73.456l-.152-.187-.162-.178c-7.47-10.04-16.177-18.23-24.752-26.75C54.652 291.5 28.37 262 21.012 230.74c-1.85-9.23-6.614-18.79-4.082-46.1l.047-.134.044-.133c7.388-25.513 19.407-46.31 39.806-67.153 21.228.342 42.24.809 58.607 3.935 7.51 1.256 23.124 3.296 39.252 9.304 40.822 15.207 94.76 40.209 94.76 40.209-40.488-22.236-85.76-51.115-114.35-56.943-4.253-.602-6.868-2.543-8.13-5.93 42.998-25.264 50.864-55.009 79.146-81.686 12.925-5.626 17.965-8.498 28.947-9.692 101.57 16.06 165.51 56.515 216.09 83.566 20.53 11.164 39.191 19.606 57.015 29.773 15.624 5.14 62.771 40.734 76.57 59.925 14.016 29.595 24.25 61.563 33.584 92.411 6.693 31.375 12.564 44.102 12.564 44.102s-5.688-26.243-4.74-30.958c5.898 2.204 19.84 6.558 25.62 5.881 0 0-25.817-13.346-29.2-25.14-10.856-37.848-21.744-96.766-24.68-99.838-8.274-10.41-42.777-36.816-63.941-49.03-8.006-4.62-12.333-7.486-12.671-9.518 6.76-6.83 15.11-15.865 22.598-21.606 7.177-5.503 13.71-11.709 23.713-15.423 43.963-19.8 69.1 7.618 75.066 2.066 0 0-9.458-10.841-5.257-9.056 4.302 2.33 18.323 5.078 19.862 6.509 16.023 12.534 57.913 58.344 83.16 106.68 6.065 11.84 8.53 19.636 5.717 33.893-2.82 14.27-5.001 22.117-8.077 31.647-2.779 6.371-18.498 49.988-18.437 55.848-3.167 23.932 10.264 53.893 10.264 53.893.127-8.133-.5-12.466.236-18.282l.904-10.393s-.573-2.764-.469-3.86c.682-7.187 2.445-13.229 2.963-17.345 5.047-31.255 13.822-53.755 23.775-81.182 2.958-6.922 6.828-10.771 6.63-16.041.163-9.352-8.205-21.904-14.252-34.163-6.1-12.39-13.39-26.202-22.96-40.76-21.82-31.376-40.42-55.925-74.48-71.289-9.53-4.182-47.03-8.27-59.98-5.841-15.71 3.273-29.22 6.5-39.99 13.398-16.95 10.854-30.27 27.659-45.89 37.56-34.56-17.281-51.24-30.215-54.35-31.998-20.54-11.006-45.2-23.778-71.73-35.89-12.72-11.862-91.71-40.233-164.04-45.892zm442.08 644.59c-21.3-16.65-39.23-33.79-51.05-51.44-3.858 20.758-17.864 35.542-28.688 50.083-2.155 3.41-3.708 8.06 6.902 22.879 2.86 3.948 13.207 4.623 20.192 4.26-7.106-5.371-17.918-11.052-19.66-15.976 12.501 8.51 24.076 10.957 34.538 9.586 2.39-.269 5.36-2.804 7.656-6.706 4.663-10.02 8.315-12.433 12.005-15.13l8.501 10.651z' color='%23000'/%3E%3C/g%3E%3C/svg%3E"); } +.custom-icons-hudi { + background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 2000 2000' width='2000' height='2000'%3E%3Cpath fill='%23FFF' d='M0 1000V0h2000v2000H0zm1529 553c0-5-102-212-227-461l-228-453-22 46c-20 42-25 45-60 45h-38l-54-110c-30-61-57-110-60-110-4 0-470 924-470 933 0 1 59 1 130-1l130-4v41c0 23-5 51-10 62-11 19-2 19 450 19 253 0 460-3 459-7m290-154 103-84-47-14-47-15 6-111c8-127-5-191-56-288-82-154-302-287-433-263-32 6-33 7-23 45l10 38 82 5c102 7 177 41 242 110 68 73 96 142 102 254 2 50 2 108-2 128-6 31-11 36-35 36-16 0-47-7-70-15-39-13-42-13-37 3 3 9 19 67 36 127 17 61 34 118 36 128 4 12 10 15 17 9 7-5 59-46 116-93M232 1248c13-19 11-27-19-87-66-129-67-286-3-418 46-94 146-173 218-173 27 0 31 4 42 48 7 26 16 50 19 54 7 8 171-204 171-220 0-6-243-92-263-92-3 0-2 12 3 26 6 14 10 42 10 62 0 34-5 39-48 63-225 125-326 365-249 588 24 68 83 171 98 171 4 0 13-10 21-22'/%3E%3Cpath fill='%2300B3EF' d='M600 1479c0-9-36-10-132-5-73 3-138 3-144-1-8-6 71-173 248-526 226-453 261-517 274-506 9 7 43 67 76 133 33 67 62 123 65 126s11-4 19-14c13-18 14-17 14 17 0 28-46 129-187 412-164 327-191 375-210 375-13 0-23-5-23-11m-434-195c-148-189-149-478-2-665 38-48 165-147 206-161 9-3 6-24-10-76-12-40-20-75-18-77 5-5 369 114 381 124 7 7-230 331-242 331-4 0-19-36-32-80l-24-79-33 16c-93 46-163 143-188 264-22 105 5 218 72 307l26 33-26 37c-14 20-35 48-45 61l-18 24z'/%3E%3Cpath fill='%23004E7F' d='M596 1493c6-21 3-22-78-26l-83-3 90-2 90-2 198-395c108-217 201-395 205-394 4 0 92 172 195 382 185 377 210 440 175 452-7 2-189 6-405 7l-392 3zm724-778c0-22 4-25 35-25s35 3 35 25-4 25-35 25-35-3-35-25'/%3E%3Cpath fill='%232F5B7F' d='m577 1538 28-53 375-3c206-1 378-6 383-11 4-4-76-177-180-384L996 711l39-81 39-81 258 513c141 282 257 516 257 521 1 4-233 7-520 7H549z'/%3E%3Cpath fill='%23013868' d='M1616 1369c-32-110-55-203-52-206 2-3 39 6 80 20l75 25 7-35c11-57-4-161-30-219-51-111-162-195-277-210-30-4-57-12-61-18-5-8-8-7-8 2 0 6-11 12-24 12-20 0-25-8-41-65-9-36-15-68-12-71 13-13 131-16 189-4 142 27 286 138 352 270 47 95 60 163 53 286l-6 100 70 23c38 12 69 24 69 25 0 3-272 224-316 258-8 6-29-54-68-193'/%3E%3C/svg%3E"); +} + .custom-icons-paimon { background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 218.61 218.61' width='218.61' height='218.61'%3E%3Cdefs%3E%3Cstyle%3E.cls-2{fill:%230163e1}%3C/style%3E%3C/defs%3E%3Cg id='svgID0'%3E%3Cpath d='M55.63 111.28a35.01 35.01 0 00-21.95-21.65l-12.74-4.26h80.4c19.85-7.47 43.6-13.02 72.56-17.39l19.6-3.48 13.7-3.85C189.3 24.7 152.19 0 109.3 0 48.94 0 0 48.94 0 109.3c0 20.41 2.93 39.59 17.2 58.79 8.15-17.84 23.42-40.44 38.43-56.82Z' class='cls-2'/%3E%3Cpath d='M182.4 72.57A430.5 430.5 0 0023.48 177c20.02 25.34 51.02 41.61 85.82 41.61 60.37 0 109.3-48.94 109.3-109.3 0-16.36-3.6-31.88-10.04-45.81l-26.17 9.08Z' class='cls-2'/%3E%3Cpath fill='%23e7e7e7' d='m182.4 72.57 26.17-9.08c-.44-.95-.9-1.9-1.36-2.84l-13.7 3.85-17.66 1.02c-29.79 2.42-54.8 8.05-86.19 19.85H20.94l9 3.87c9.89 3.88 22.18 12 25.69 22.04-16.77 16.69-30.7 39.55-38.43 56.82 2.46 4.13 5.32 7.62 8.29 11.37 48.68-53.33 96.8-84.37 156.91-106.89Z'/%3E%3C/g%3E%3C/svg%3E"); } diff --git a/web/web/src/lib/icons/svg/hudi.svg b/web/web/src/lib/icons/svg/hudi.svg new file mode 100644 index 00000000000..8dfc3cb27b6 --- /dev/null +++ b/web/web/src/lib/icons/svg/hudi.svg @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/web/web/src/lib/utils/initial.js b/web/web/src/lib/utils/initial.js index 8d85779ab6b..e916fcd5ef3 100644 --- a/web/web/src/lib/utils/initial.js +++ b/web/web/src/lib/utils/initial.js @@ -274,5 +274,24 @@ export const providers = [ hide: 'simple' } ] + }, + { + label: 'Apache Hudi', + value: 'lakehouse-hudi', + defaultProps: [ + { + key: 'catalog-backend', + value: 'hms', + defaultValue: 'hms', + required: true, + description: 'Apache Hudi catalog type choose properties' + }, + { + key: 'uri', + value: '', + required: true, + description: 'Apache Hudi catalog uri config' + } + ] } ]