Skip to content

Commit

Permalink
[KYUUBI #5535][AUTHZ] Support vacuum table command for Delta Lake
Browse files Browse the repository at this point in the history
### _Why are the changes needed?_
To close #5535 .
Support vacuum table command for Delta Lake.
https://docs.delta.io/latest/delta-utility.html#remove-files-no-longer-referenced-by-a-delta-table

== Analyzed Logical Plan ==

Delta Lake 3.0/Spark 3.5
```
== Analyzed Logical Plan ==
path: string
VacuumTableCommand false
+- ResolvedTable org.apache.spark.sql.delta.catalog.DeltaCatalog3947bad4, delta_ns.table1_delta, DeltaTableV2(org.apache.spark.sql.SparkSession2e7075d4,file:/var/folders/gc/c__qhntd7s502txfp0ltxh880000gn/T/spark-warehouse-ba741e85-4c66-46f7-b1a7-10a1f32cd20c/delta_ns.db/table1_delta,Some(CatalogTable(
Catalog: spark_catalog
Database: delta_ns
Table: table1_delta
Owner: default_table_owner
Created Time: Thu Nov 09 10:45:20 CST 2023
Last Access: UNKNOWN
Created By: Spark 3.5.0
Type: MANAGED
Provider: delta
Location: file:/var/folders/gc/c__qhntd7s502txfp0ltxh880000gn/T/spark-warehouse-ba741e85-4c66-46f7-b1a7-10a1f32cd20c/delta_ns.db/table1_delta
Serde Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
InputFormat: org.apache.hadoop.mapred.SequenceFileInputFormat
OutputFormat: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
Partition Provider: Catalog)),Some(delta_ns.table1_delta),None,Map()), [id#818, name#819, gender#820, birthDate#821]
```
Before Delta Lake 3.0/Spark 3.5
```
== Analyzed Logical Plan ==
path: string
VacuumTableCommand `delta_ns`.`table1_delta`, false
```
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request

### _Was this patch authored or co-authored using generative AI tooling?_
No.

Closes #5655 from zml1206/KYUUBI-5535.

Closes #5535

9500ee7 [zml1206] Support vacuum table command for Delta Lake

Authored-by: zml1206 <[email protected]>
Signed-off-by: Kent Yao <[email protected]>
  • Loading branch information
zml1206 authored and yaooqinn committed Nov 9, 2023
1 parent aa71e08 commit e8e981d
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -2042,6 +2042,30 @@
"opType" : "QUERY",
"queryDescs" : [ ],
"uriDescs" : [ ]
}, {
"classname" : "io.delta.tables.execution.VacuumTableCommand",
"tableDescs" : [ {
"fieldName" : "child",
"fieldExtractor" : "ResolvedTableTableExtractor",
"columnDesc" : null,
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
"setCurrentDatabaseIfMissing" : false
}, {
"fieldName" : "table",
"fieldExtractor" : "TableIdentifierOptionTableExtractor",
"columnDesc" : null,
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
"setCurrentDatabaseIfMissing" : false
} ],
"opType" : "MSCK",
"queryDescs" : [ ],
"uriDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.delta.commands.DeleteCommand",
"tableDescs" : [ {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,17 @@ object DeltaCommands extends CommandSpecs[TableCommandSpec] {
TableCommandSpec(cmd, Seq(childDesc, tableDesc), ALTERTABLE_COMPACT)
}

val VacuumTableCommand = {
val cmd = "io.delta.tables.execution.VacuumTableCommand"
val childDesc = TableDesc("child", classOf[ResolvedTableTableExtractor])
val tableDesc = TableDesc("table", classOf[TableIdentifierOptionTableExtractor])
TableCommandSpec(cmd, Seq(childDesc, tableDesc), MSCK)
}

override def specs: Seq[TableCommandSpec] = Seq(
DeleteCommand,
MergeIntoCommand,
OptimizeTableCommand,
UpdateCommand)
UpdateCommand,
VacuumTableCommand)
}
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,18 @@ class DeltaCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
doAs(admin, sql(optimizeTableSql))
}
}

test("vacuum table") {
withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (s"$namespace1", "database"))) {
doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
doAs(admin, sql(createTableSql(namespace1, table1)))
val vacuumTableSql = s"VACUUM $namespace1.$table1"
interceptContains[AccessControlException](
doAs(someone, sql(vacuumTableSql)))(
s"does not have [alter] privilege on [$namespace1/$table1]")
doAs(admin, sql(vacuumTableSql))
}
}
}

object DeltaCatalogRangerSparkExtensionSuite {
Expand Down

0 comments on commit e8e981d

Please sign in to comment.