Add table/columns/view remarks support - update Readme with progress

README.md

@@ -1,19 +1,63 @@
 # liquibase-databricks
 
 
-## Current Summary 
-Base and Foundational Change types should be supported at this stage. Change types such as procedures, triggers, merge column, indexes are not supported. 
+## Summary 
+This is the Liquibase Extension for Managing Delta Tables with DatabricksSQL. 
+
+Base/Contributed and Foundational Change types should be supported at this stage. Change types such as procedures, triggers, sequences, indexes are not supported. 
 Databricks specific change types that are added are listed below along with their completion status. 
+Databricks tables creates with liquibase are automatically created with the Delta configs / versions that are required for all passing change types including: 'delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name'
+
+
+## NOTE! ONLY TABLES CREATED WITH UNITY CATALOG ARE SUPPORTED FOR MOST ADVANCED OPERATIONS
+This extension utilizes Unity Catalog System tables for many advanced operations such as snapshotting, idedentifying various constraints (PK/FK/NOT NULL, etc.)
+If hive_metastore is used, this is not tested and may not provide all the below functionality.
+
+
+## TO DO: 
+
+1. Add unit tests with liquibase test harness - Cody Davis - DONE
+2. Pass Foundational Test Harness - Cody Davis - DONE 4/1/2023
+3. Pass Contributed Test Harness - Cody Davis - IN PROGRESS 9/15/2023
+4. Pass Advanced Test Harness - <> - NOT STARTED
+
+
+## Currently Supported Change Types:
+1. createTable/dropTable
+2. addColumn/dropColumn
+3. addPrimaryKey/dropPrimaryKey
+4. addForeignKey/dropForeignKey
+5. addNotNullConstraint/dropNotNullConstraint
+6. createTable/createTableDataTypeText/createTableTimestamp/dropTable
+7. createView/dropView
+8. dropAllForeignKeyConstraints
+9. createView/dropView
+10. setTableRemarks
+11. setColumnRemarks
+12. setViewRemarks (set in TBLPROPERTIES ('comment' = '<comment>'))
+13. executeCommand
+14. mergeColumns
+15. modifySql
+16. renameColumn
+17. renameView
+18. sql
+19. sqlFile
+20. Change Data Test: apply delete
+21. Change Data Test: apply insert
+22. Change Data Test: apply loadData
+
+
+## Remaining Relevant Change Types to Finish in Base/Contributed
+1. createFunction/dropFunction - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
+2. addCheckConstraint/dropCheckConstraint - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
+3. addLookupTable (executing out of order/dropping FK before creation)
+4. Change Data Test: apply loadUpdateData
 
 
-## To Do: 
+The remaining other change types are not relevant to Databricks and have been marked with INVALID TEST
 
-1. Add unit tests with liquibase test harness - Cody Davis - Done
-2. Pass Foundational Test Harness - Cody Davis - Done
-3. Pass Contributed Test Harness - Cody Davis - In Progress - ETA May 15, 2023
-4. Pass Advanced Test Harness - Unassigned - Not Started
 
-## Change Types to Add: 
+## Aspirational Roadmap - Databricks Specific Additional Change Types to Add: 
 
 1. COPY INTO
 2. MERGE
@@ -23,8 +67,10 @@ Databricks specific change types that are added are listed below along with thei
 6. CLONE
 7. BLOOM FILTERS
 8. OPTIMIZE / ZORDER - Code Complete - No Test Yet - Cody Davis
-9. VACUUM - Code Complete - Cody Davis
-10. SYNC IDENTITY - Not Started -
+9. VACUUM - Code Complete - No Test Yet - Cody Davis
+10. SYNC IDENTITY
+11. VOLUMES
+12. GRANT / REVOKE statements
 
 
 

src/main/java/liquibase/ext/databricks/sqlgenerator/SetColumnRemarksGeneratorDatabricks.java

@@ -0,0 +1,68 @@
+package liquibase.ext.databricks.sqlgenerator;
+
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.database.Database;
+import liquibase.database.core.*;
+import liquibase.datatype.DataTypeFactory;
+import liquibase.exception.ValidationErrors;
+import liquibase.exception.Warnings;
+import liquibase.sql.Sql;
+import liquibase.sql.UnparsedSql;
+import liquibase.sqlgenerator.SqlGeneratorChain;
+import liquibase.statement.core.SetColumnRemarksStatement;
+import liquibase.structure.core.Column;
+import liquibase.structure.core.Data;
+import liquibase.structure.core.Table;
+import liquibase.util.ColumnParentType;
+import liquibase.util.StringUtil;
+import liquibase.sqlgenerator.core.SetColumnRemarksGenerator;
+
+public class SetColumnRemarksGeneratorDatabricks extends SetColumnRemarksGenerator {
+    @Override
+    public int getPriority() {
+        return DatabricksDatabase.PRIORITY_DATABASE;
+    }
+
+    @Override
+    public boolean supports(SetColumnRemarksStatement statement, Database database) {
+        return (database instanceof DatabricksDatabase);
+    }
+
+    @Override
+    public ValidationErrors validate(SetColumnRemarksStatement setColumnRemarksStatement, Database database, SqlGeneratorChain sqlGeneratorChain) {
+        ValidationErrors validationErrors = new ValidationErrors();
+        validationErrors.checkRequiredField("tableName", setColumnRemarksStatement.getTableName());
+        validationErrors.checkRequiredField("columnName", setColumnRemarksStatement.getColumnName());
+        validationErrors.checkDisallowedField("catalogName", setColumnRemarksStatement.getCatalogName(), database, MSSQLDatabase.class);
+        if (database instanceof MySQLDatabase) {
+            validationErrors.checkRequiredField("columnDataType", StringUtil.trimToNull(setColumnRemarksStatement.getColumnDataType()));
+        }
+        return validationErrors;
+    }
+
+    @Override
+    public Warnings warn(SetColumnRemarksStatement statementType, Database database, SqlGeneratorChain<SetColumnRemarksStatement> sqlGeneratorChain) {
+        final Warnings warnings = super.warn(statementType, database, sqlGeneratorChain);
+        if (database instanceof MySQLDatabase) {
+            ((MySQLDatabase) database).warnAboutAlterColumn("setColumnRemarks", warnings);
+        }
+
+        return warnings;
+    }
+
+    @Override
+    public Sql[] generateSql(SetColumnRemarksStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) {
+
+        String remarksEscaped = database.escapeStringForDatabase(StringUtil.trimToEmpty(statement.getRemarks()));
+
+
+        return new Sql[]{new UnparsedSql("ALTER TABLE " + database.escapeTableName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName())
+                    + " ALTER COLUMN " + database.escapeColumnName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName(), statement.getColumnName()) + " COMMENT '"
+                    + remarksEscaped + "'", getAffectedColumn(statement))};
+
+    }
+
+    protected Column getAffectedColumn(SetColumnRemarksStatement statement) {
+        return new Column().setName(statement.getColumnName()).setRelation(new Table().setName(statement.getTableName()).setSchema(statement.getCatalogName(), statement.getSchemaName()));
+    }
+}

src/main/java/liquibase/ext/databricks/sqlgenerator/SetTableRemarksGeneratorDatabricks.java

@@ -0,0 +1,44 @@
+package liquibase.ext.databricks.sqlgenerator;
+
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.database.Database;
+import liquibase.database.core.*;
+import liquibase.exception.ValidationErrors;
+import liquibase.sql.Sql;
+import liquibase.sql.UnparsedSql;
+import liquibase.sqlgenerator.SqlGeneratorChain;
+import liquibase.statement.core.SetTableRemarksStatement;
+import liquibase.structure.core.Relation;
+import liquibase.structure.core.Table;
+import liquibase.util.StringUtil;
+import liquibase.sqlgenerator.core.SetTableRemarksGenerator;
+
+public class SetTableRemarksGeneratorDatabricks extends SetTableRemarksGenerator {
+
+    @Override
+    public boolean supports(SetTableRemarksStatement statement, Database database) {
+        return (database instanceof DatabricksDatabase);
+    }
+
+    @Override
+    public ValidationErrors validate(SetTableRemarksStatement setTableRemarksStatement, Database database, SqlGeneratorChain sqlGeneratorChain) {
+        ValidationErrors validationErrors = new ValidationErrors();
+        validationErrors.checkRequiredField("tableName", setTableRemarksStatement.getTableName());
+        return validationErrors;
+    }
+
+    @Override
+    public Sql[] generateSql(SetTableRemarksStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) {
+        String sql;
+        String remarksEscaped = database.escapeStringForDatabase(StringUtil.trimToEmpty(statement.getRemarks()));
+
+        sql = "COMMENT ON TABLE " + database.escapeTableName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName()) + " IS '"
+                    + remarksEscaped + "'";
+
+        return new Sql[] { new UnparsedSql(sql, getAffectedTable(statement)) };
+    }
+
+    protected Relation getAffectedTable(SetTableRemarksStatement statement) {
+        return new Table().setName(statement.getTableName()).setSchema(statement.getCatalogName(), statement.getSchemaName());
+    }
+}

src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator

@@ -9,4 +9,6 @@ liquibase.ext.databricks.sqlgenerator.GetViewDefinitionGeneratorDatabricks
 liquibase.ext.databricks.sqlgenerator.RenameTableGeneratorDatabricks
 liquibase.ext.databricks.sqlgenerator.RenameViewGeneratorDatabricks
 liquibase.ext.databricks.sqlgenerator.RenameColumnGeneratorDatabricks
-liquibase.ext.databricks.sqlgenerator.AddPrimaryKeyGeneratorDatabricks
+liquibase.ext.databricks.sqlgenerator.AddPrimaryKeyGeneratorDatabricks
+liquibase.ext.databricks.sqlgenerator.SetTableRemarksGeneratorDatabricks
+liquibase.ext.databricks.sqlgenerator.SetColumnRemarksGeneratorDatabricks

src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/setTableRemarks.json

@@ -0,0 +1,14 @@
+{
+  "snapshot": {
+    "objects": {
+      "liquibase.structure.core.Table": [
+        {
+          "table": {
+            "name": "authors",
+            "remarks": "A Test Remark"
+          }
+        }
+      ]
+    }
+  }
+}

src/test/resources/liquibase/harness/change/expectedSql/databricks/renameColumn.sql

@@ -0,0 +1 @@
+ALTER TABLE main.liquibase_harness_test_ds.authors RENAME COLUMN first_name TO first_name_renameColumn_test

src/test/resources/liquibase/harness/change/expectedSql/databricks/renameView.sql

@@ -0,0 +1,2 @@
+CREATE VIEW main.liquibase_harness_test_ds.test_view AS select id, first_name, last_name, email from authors
+ALTER VIEW main.liquibase_harness_test_ds.test_view RENAME TO test_view_new

src/test/resources/liquibase/harness/change/expectedSql/databricks/setColumnRemarks.sql

@@ -0,0 +1 @@
+ALTER TABLE main.liquibase_harness_test_ds.authors ALTER COLUMN id COMMENT 'A Test Column Remark'