[#5029] improvement(storage): Delete the related relations about the …

…metadata object when it is deleted. (#5426) ### What changes were proposed in this pull request? Delete the related relations about the metadata object when it is deleted. ### Why are the changes needed? Fix: #5029 ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Add UTs.
apache · Nov 6, 2024 · facfe83 · facfe83
1 parent d20b535
commit facfe83
Show file tree

Hide file tree

Showing 20 changed files with 1,474 additions and 58 deletions.
diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/SecurableObjectMapper.java
@@ -58,8 +58,25 @@ void batchSoftDeleteSecurableObjects(
 
   @UpdateProvider(
       type = SecurableObjectSQLProviderFactory.class,
-      method = "softDeleteRoleMetasByMetalakeId")
-  void softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId);
+      method = "softDeleteSecurableObjectsByMetalakeId")
+  void softDeleteSecurableObjectsByMetalakeId(@Param("metalakeId") Long metalakeId);
+
+  @UpdateProvider(
+      type = SecurableObjectSQLProviderFactory.class,
+      method = "softDeleteObjectRelsByMetadataObject")
+  void softDeleteObjectRelsByMetadataObject(
+      @Param("metadataObjectId") Long metadataObjectId,
+      @Param("metadataObjectType") String metadataObjectType);
+
+  @UpdateProvider(
+      type = SecurableObjectSQLProviderFactory.class,
+      method = "softDeleteObjectRelsByCatalogId")
+  void softDeleteObjectRelsByCatalogId(@Param("catalogId") Long catalogId);
+
+  @UpdateProvider(
+      type = SecurableObjectSQLProviderFactory.class,
+      method = "softDeleteObjectRelsBySchemaId")
+  void softDeleteObjectRelsBySchemaId(@Param("schemaId") Long schemaId);
 
   @SelectProvider(
       type = SecurableObjectSQLProviderFactory.class,

diff --git a/...ava/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java b/...ava/org/apache/gravitino/storage/relational/mapper/SecurableObjectSQLProviderFactory.java
@@ -66,8 +66,23 @@ public static String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long ro
     return getProvider().softDeleteSecurableObjectsByRoleId(roleId);
   }
 
-  public static String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) {
-    return getProvider().softDeleteRoleMetasByMetalakeId(metalakeId);
+  public static String softDeleteSecurableObjectsByMetalakeId(
+      @Param("metalakeId") Long metalakeId) {
+    return getProvider().softDeleteSecurableObjectsByMetalakeId(metalakeId);
+  }
+
+  public static String softDeleteObjectRelsByMetadataObject(
+      @Param("metadataObjectId") Long metadataObjectId,
+      @Param("metadataObjectType") String metadataObjectType) {
+    return getProvider().softDeleteObjectRelsByMetadataObject(metadataObjectId, metadataObjectType);
+  }
+
+  public static String softDeleteObjectRelsByCatalogId(@Param("catalogId") Long catalogId) {
+    return getProvider().softDeleteObjectRelsByCatalogId(catalogId);
+  }
+
+  public static String softDeleteObjectRelsBySchemaId(@Param("schemaId") Long schemaId) {
+    return getProvider().softDeleteObjectRelsBySchemaId(schemaId);
   }
 
   public static String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) {

diff --git a/.../main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java b/.../main/java/org/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelMapper.java
@@ -75,6 +75,28 @@ Integer softDeleteTagMetadataObjectRelsByMetalakeAndTagName(
       method = "softDeleteTagMetadataObjectRelsByMetalakeId")
   void softDeleteTagMetadataObjectRelsByMetalakeId(@Param("metalakeId") Long metalakeId);
 
+  @UpdateProvider(
+      type = TagMetadataObjectRelSQLProviderFactory.class,
+      method = "softDeleteTagMetadataObjectRelsByMetadataObject")
+  void softDeleteTagMetadataObjectRelsByMetadataObject(
+      @Param("metadataObjectId") Long metadataObjectId,
+      @Param("metadataObjectType") String metadataObjectType);
+
+  @UpdateProvider(
+      type = TagMetadataObjectRelSQLProviderFactory.class,
+      method = "softDeleteTagMetadataObjectRelsByCatalogId")
+  void softDeleteTagMetadataObjectRelsByCatalogId(@Param("catalogId") Long catalogId);
+
+  @UpdateProvider(
+      type = TagMetadataObjectRelSQLProviderFactory.class,
+      method = "softDeleteTagMetadataObjectRelsBySchemaId")
+  void softDeleteTagMetadataObjectRelsBySchemaId(@Param("schemaId") Long schemaId);
+
+  @UpdateProvider(
+      type = TagMetadataObjectRelSQLProviderFactory.class,
+      method = "softDeleteTagMetadataObjectRelsByTableId")
+  void softDeleteTagMetadataObjectRelsByTableId(@Param("tableId") Long tableId);
+
   @DeleteProvider(
       type = TagMetadataObjectRelSQLProviderFactory.class,
       method = "deleteTagEntityRelsByLegacyTimeline")

diff --git a/...rg/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java b/...rg/apache/gravitino/storage/relational/mapper/TagMetadataObjectRelSQLProviderFactory.java
@@ -95,6 +95,26 @@ public static String softDeleteTagMetadataObjectRelsByMetalakeId(
     return getProvider().softDeleteTagMetadataObjectRelsByMetalakeId(metalakeId);
   }
 
+  public static String softDeleteTagMetadataObjectRelsByMetadataObject(
+      @Param("metadataObjectId") Long metadataObjectId,
+      @Param("metadataObjectType") String metadataObjectType) {
+    return getProvider()
+        .softDeleteTagMetadataObjectRelsByMetadataObject(metadataObjectId, metadataObjectType);
+  }
+
+  public static String softDeleteTagMetadataObjectRelsByCatalogId(
+      @Param("catalogId") Long catalogId) {
+    return getProvider().softDeleteTagMetadataObjectRelsByCatalogId(catalogId);
+  }
+
+  public static String softDeleteTagMetadataObjectRelsBySchemaId(@Param("schemaId") Long schemaId) {
+    return getProvider().softDeleteTagMetadataObjectRelsBySchemaId(schemaId);
+  }
+
+  public static String softDeleteTagMetadataObjectRelsByTableId(@Param("tableId") Long tableId) {
+    return getProvider().softDeleteTagMetadataObjectRelsByTableId(tableId);
+  }
+
   public static String deleteTagEntityRelsByLegacyTimeline(
       @Param("legacyTimeline") Long legacyTimeline, @Param("limit") int limit) {
     return getProvider().deleteTagEntityRelsByLegacyTimeline(legacyTimeline, limit);

diff --git a/...rg/apache/gravitino/storage/relational/mapper/provider/base/OwnerMetaBaseSQLProvider.java b/...rg/apache/gravitino/storage/relational/mapper/provider/base/OwnerMetaBaseSQLProvider.java
@@ -123,30 +123,30 @@ public String softDeleteOwnerRelByCatalogId(@Param("catalogId") Long catalogId)
         + OWNER_TABLE_NAME
         + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
         + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
-        + " WHERE EXISTS ("
+        + " WHERE ot.deleted_at = 0 AND EXISTS ("
         + " SELECT ct.catalog_id FROM "
         + CatalogMetaMapper.TABLE_NAME
-        + " ct WHERE ct.catalog_id = #{catalogId}  AND ct.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'"
+        + " ct WHERE ct.catalog_id = #{catalogId} AND "
+        + " ct.catalog_id = ot.metadata_object_id AND ot.metadata_object_type = 'CATALOG'"
         + " UNION "
         + " SELECT st.catalog_id FROM "
         + SchemaMetaMapper.TABLE_NAME
-        + " st WHERE st.catalog_id = #{catalogId} AND st.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'"
+        + " st WHERE st.catalog_id = #{catalogId} AND "
+        + " st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'"
         + " UNION "
         + " SELECT tt.catalog_id FROM "
         + TopicMetaMapper.TABLE_NAME
-        + " tt WHERE tt.catalog_id = #{catalogId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'"
+        + " tt WHERE tt.catalog_id = #{catalogId} AND "
+        + " tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'"
         + " UNION "
         + " SELECT tat.catalog_id FROM "
         + TableMetaMapper.TABLE_NAME
-        + " tat WHERE tat.catalog_id = #{catalogId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'"
+        + " tat WHERE tat.catalog_id = #{catalogId} AND "
+        + " tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'"
         + " UNION "
         + " SELECT ft.catalog_id FROM "
         + FilesetMetaMapper.META_TABLE_NAME
-        + " ft WHERE ft.catalog_id = #{catalogId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND"
+        + " ft WHERE ft.catalog_id = #{catalogId} AND"
         + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'"
         + ")";
   }
@@ -156,26 +156,26 @@ public String sotDeleteOwnerRelBySchemaId(@Param("schemaId") Long schemaId) {
         + OWNER_TABLE_NAME
         + " ot SET ot.deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
         + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
-        + " WHERE EXISTS ("
+        + " WHERE ot.deleted_at = 0 AND EXISTS ("
         + " SELECT st.schema_id FROM "
         + SchemaMetaMapper.TABLE_NAME
-        + " st WHERE st.schema_id = #{schemaId} AND st.deleted_at = 0 AND ot.deleted_at = 0 "
-        + "AND st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'"
+        + " st WHERE st.schema_id = #{schemaId} AND"
+        + " st.schema_id = ot.metadata_object_id AND ot.metadata_object_type = 'SCHEMA'"
         + " UNION "
         + " SELECT tt.schema_id FROM "
         + TopicMetaMapper.TABLE_NAME
-        + " tt WHERE tt.schema_id = #{schemaId} AND tt.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'"
+        + " tt WHERE tt.schema_id = #{schemaId} AND "
+        + " tt.topic_id = ot.metadata_object_id AND ot.metadata_object_type = 'TOPIC'"
         + " UNION "
         + " SELECT tat.schema_id FROM "
         + TableMetaMapper.TABLE_NAME
-        + " tat WHERE tat.schema_id = #{schemaId} AND tat.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'"
+        + " tat WHERE tat.schema_id = #{schemaId} AND "
+        + " tat.table_id = ot.metadata_object_id AND ot.metadata_object_type = 'TABLE'"
         + " UNION "
         + " SELECT ft.schema_id FROM "
         + FilesetMetaMapper.META_TABLE_NAME
-        + " ft WHERE ft.schema_id = #{schemaId} AND ft.deleted_at = 0 AND ot.deleted_at = 0 AND "
-        + "ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'"
+        + " ft WHERE ft.schema_id = #{schemaId} AND "
+        + " ft.fileset_id = ot.metadata_object_id AND ot.metadata_object_type = 'FILESET'"
         + ")";
   }
 

diff --git a/...che/gravitino/storage/relational/mapper/provider/base/SecurableObjectBaseSQLProvider.java b/...che/gravitino/storage/relational/mapper/provider/base/SecurableObjectBaseSQLProvider.java
@@ -22,6 +22,11 @@
 import static org.apache.gravitino.storage.relational.mapper.SecurableObjectMapper.SECURABLE_OBJECT_TABLE_NAME;
 
 import java.util.List;
+import org.apache.gravitino.storage.relational.mapper.CatalogMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.FilesetMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.SchemaMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TableMetaMapper;
+import org.apache.gravitino.storage.relational.mapper.TopicMetaMapper;
 import org.apache.gravitino.storage.relational.po.SecurableObjectPO;
 import org.apache.ibatis.annotations.Param;
 
@@ -73,7 +78,7 @@ public String softDeleteSecurableObjectsByRoleId(@Param("roleId") Long roleId) {
         + " WHERE role_id = #{roleId} AND deleted_at = 0";
   }
 
-  public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalakeId) {
+  public String softDeleteSecurableObjectsByMetalakeId(@Param("metalakeId") Long metalakeId) {
     return "UPDATE "
         + SECURABLE_OBJECT_TABLE_NAME
         + " ob SET ob.deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
@@ -84,6 +89,78 @@ public String softDeleteRoleMetasByMetalakeId(@Param("metalakeId") Long metalake
         + " AND ro.deleted_at = 0) AND ob.deleted_at = 0";
   }
 
+  public String softDeleteObjectRelsByMetadataObject(
+      @Param("metadataObjectId") Long metadataObjectId,
+      @Param("metadataObjectType") String metadataObjectType) {
+    return "UPDATE "
+        + SECURABLE_OBJECT_TABLE_NAME
+        + " SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+        + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+        + " WHERE metadata_object_id = #{metadataObjectId} AND deleted_at = 0"
+        + " AND type = #{metadataObjectType}";
+  }
+
+  public String softDeleteObjectRelsByCatalogId(@Param("catalogId") Long catalogId) {
+    return "UPDATE "
+        + SECURABLE_OBJECT_TABLE_NAME
+        + " sect SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+        + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+        + " WHERE sect.deleted_at = 0 AND EXISTS ("
+        + " SELECT ct.catalog_id FROM "
+        + CatalogMetaMapper.TABLE_NAME
+        + " ct WHERE ct.catalog_id = #{catalogId}  AND "
+        + " ct.catalog_id = sect.metadata_object_id AND sect.type = 'CATALOG'"
+        + " UNION "
+        + " SELECT st.catalog_id FROM "
+        + SchemaMetaMapper.TABLE_NAME
+        + " st WHERE st.catalog_id = #{catalogId} AND "
+        + " st.schema_id = sect.metadata_object_id AND sect.type = 'SCHEMA'"
+        + " UNION "
+        + " SELECT tt.catalog_id FROM "
+        + TopicMetaMapper.TABLE_NAME
+        + " tt WHERE tt.catalog_id = #{catalogId} AND "
+        + " tt.topic_id = sect.metadata_object_id AND sect.type = 'TOPIC'"
+        + " UNION "
+        + " SELECT tat.catalog_id FROM "
+        + TableMetaMapper.TABLE_NAME
+        + " tat WHERE tat.catalog_id = #{catalogId} AND "
+        + " tat.table_id = sect.metadata_object_id AND sect.type = 'TABLE'"
+        + " UNION "
+        + " SELECT ft.catalog_id FROM "
+        + FilesetMetaMapper.META_TABLE_NAME
+        + " ft WHERE ft.catalog_id = #{catalogId} AND"
+        + " ft.fileset_id = sect.metadata_object_id AND sect.type = 'FILESET'"
+        + ")";
+  }
+
+  public String softDeleteObjectRelsBySchemaId(@Param("schemaId") Long schemaId) {
+    return "UPDATE "
+        + SECURABLE_OBJECT_TABLE_NAME
+        + " sect SET deleted_at = (UNIX_TIMESTAMP() * 1000.0)"
+        + " + EXTRACT(MICROSECOND FROM CURRENT_TIMESTAMP(3)) / 1000"
+        + " WHERE sect.deleted_at = 0 AND EXISTS ("
+        + " SELECT st.schema_id FROM "
+        + SchemaMetaMapper.TABLE_NAME
+        + " st WHERE st.schema_id = #{schemaId} "
+        + " AND st.schema_id = sect.metadata_object_id AND sect.type = 'SCHEMA'"
+        + " UNION "
+        + " SELECT tt.schema_id FROM "
+        + TopicMetaMapper.TABLE_NAME
+        + " tt WHERE tt.schema_id = #{schemaId} AND "
+        + " tt.topic_id = sect.metadata_object_id AND sect.type = 'TOPIC'"
+        + " UNION "
+        + " SELECT tat.schema_id FROM "
+        + TableMetaMapper.TABLE_NAME
+        + " tat WHERE tat.schema_id = #{schemaId} AND "
+        + " tat.table_id = sect.metadata_object_id AND sect.type = 'TABLE'"
+        + " UNION "
+        + " SELECT ft.schema_id FROM "
+        + FilesetMetaMapper.META_TABLE_NAME
+        + " ft WHERE ft.schema_id = #{schemaId} AND "
+        + " ft.fileset_id = sect.metadata_object_id AND sect.type = 'FILESET'"
+        + ")";
+  }
+
   public String listSecurableObjectsByRoleId(@Param("roleId") Long roleId) {
     return "SELECT role_id as roleId, metadata_object_id as metadataObjectId,"
         + " type as type, privilege_names as privilegeNames,"