From f6ccc4d80fc925d51540595c3b18556c220dd70e Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Tue, 17 Oct 2023 17:33:07 +0800 Subject: [PATCH] [KYUUBI #5359] [AUTHZ] Support Create Table Commands for Hudi ### _Why are the changes needed?_ To close #5359. Kyuubi authz support hudi create table commands - [CreateHoodieTableCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala): https://hudi.apache.org/docs/sql_ddl#create-table - [CreateHoodieTableAsSelectCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala): https://hudi.apache.org/docs/sql_ddl#create-table-as-select-ctas - [CreateHoodieTableLikeCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableLikeCommand.scala): https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableLikeCommand.scala ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No Closes #5439 from AngersZhuuuu/KYUUBI-5359. Closes #5359 d07010913 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala f0bc79a77 [Angerszhuuuu] [KYUUBI #5284] Support Hudi Creeate Table Command in Authz Authored-by: Angerszhuuuu Signed-off-by: Cheng Pan --- .../main/resources/table_command_spec.json | 54 +++++++++++++ .../plugin/spark/authz/gen/HudiCommands.scala | 33 +++++++- ...HudiCatalogRangerSparkExtensionSuite.scala | 77 +++++++++++++++++++ 3 files changed, 163 insertions(+), 1 deletion(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 513259e1375..913fef016ac 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -1513,6 +1513,60 @@ } ], "opType" : "ALTERTABLE_PROPERTIES", "queryDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE_AS_SELECT", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand", + "tableDescs" : [ { + "fieldName" : "targetTable", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : true + }, { + "fieldName" : "sourceTable", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : true + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ] }, { "classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala index 6e3237d2a72..c4488edbf66 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala @@ -72,11 +72,42 @@ object HudiCommands { TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES) } + val CreateHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE) + } + + val CreateHoodieTableAsSelectCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand" + CreateHoodieTableCommand.copy( + classname = cmd, + opType = CREATETABLE_AS_SELECT, + queryDescs = Seq(QueryDesc("query"))) + } + + val CreateHoodieTableLikeCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand" + val tableDesc1 = TableDesc( + "targetTable", + classOf[TableIdentifierTableExtractor], + setCurrentDatabaseIfMissing = true) + val tableDesc2 = TableDesc( + "sourceTable", + classOf[TableIdentifierTableExtractor], + isInput = true, + setCurrentDatabaseIfMissing = true) + TableCommandSpec(cmd, Seq(tableDesc1, tableDesc2), CREATETABLE) + } + val data: Array[TableCommandSpec] = Array( AlterHoodieTableAddColumnsCommand, AlterHoodieTableChangeColumnCommand, AlterHoodieTableDropPartitionCommand, AlterHoodieTableRenameCommand, AlterTableCommand, - Spark31AlterTableCommand) + Spark31AlterTableCommand, + CreateHoodieTableCommand, + CreateHoodieTableAsSelectCommand, + CreateHoodieTableLikeCommand) } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala index 8fcae6cf941..ac1e357a22f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala @@ -129,4 +129,81 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { s"does not have [alter] privilege on [$namespace1/$table1]") } } + + test("CreateHoodieTableCommand") { + withCleanTmpResources(Seq((namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)))(s"does not have [create] privilege on [$namespace1/$table1]") + } + } + + test("CreateHoodieTableAsSelectCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2 + |USING HUDI + |AS + |SELECT id FROM $namespace1.$table1 + |""".stripMargin)))(s"does not have [select] privilege on [$namespace1/$table1/id]") + } + } + + test("CreateHoodieTableLikeCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2 + |LIKE $namespace1.$table1 + |USING HUDI + |""".stripMargin)))(s"does not have [select] privilege on [$namespace1/$table1]") + } + } }