Finish Contributed Testing

1. Finished Contributed Tests (loadUpdateData, etc.)
2. Make GitActions do each testing level in series to not disturb the others

.github/workflows/lth.yml

@@ -21,6 +21,7 @@ jobs:
       WORKSPACE_ID: ${{ secrets.TH_DATABRICKS_WORKSPACE_ID }}
 
     strategy:
+      max-parallel: 1
       matrix:
         liquibase-support-level: [Foundational, Contributed, Advanced] # Define the different test levels to run
       fail-fast: false # Set fail-fast to false to run all test levels even if some of them fail

README.md

@@ -18,40 +18,45 @@ If hive_metastore is used, this is not tested and may not provide all the below
 
 1. Add unit tests with liquibase test harness - Cody Davis - DONE
 2. Pass Foundational Test Harness - Cody Davis - DONE 4/1/2023
-3. Pass Contributed Test Harness - Cody Davis - IN PROGRESS 9/15/2023
-4. Pass Advanced Test Harness - Cody Davis - IN PROGRESS
+3. Pass Contributed Test Harness - Cody Davis - DONE 9/15/2023
+4. Pass Advanced Test Harness - Cody Davis - IN PROGRESS (3/6 testing passing)
 
 
 ## Currently Supported Change Types:
-1. createTable/dropTable
-2. addColumn/dropColumn
-3. addPrimaryKey/dropPrimaryKey
-4. addForeignKey/dropForeignKey
-5. addNotNullConstraint/dropNotNullConstraint
-6. createTable/createTableDataTypeText/createTableTimestamp/dropTable
-7. createView/dropView
-8. dropAllForeignKeyConstraints
-9. createView/dropView
-10. setTableRemarks
-11. setColumnRemarks
-12. setViewRemarks (set in TBLPROPERTIES ('comment' = '<comment>'))
-13. executeCommand
-14. mergeColumns
-15. modifySql
-16. renameColumn
-17. renameView
-18. sql
-19. sqlFile
-20. Change Data Test: apply delete
-21. Change Data Test: apply insert
-22. Change Data Test: apply loadData
-
+1. [x] createTable/dropTable 
+2. [x] addColumn/dropColumn
+3. [x] addPrimaryKey/dropPrimaryKey
+4. [x] addForeignKey/dropForeignKey
+5. [x] addNotNullConstraint/dropNotNullConstraint
+6. [x] createTable/createTableDataTypeText/createTableTimestamp/dropTable
+7. [x] createView/dropView
+8. [x] dropAllForeignKeyConstraints
+9. [x] createView/dropView
+10. [x] setTableRemarks
+11. [x] setColumnRemarks
+12. [x] setViewRemarks (set in TBLPROPERTIES ('comment' = '<comment>'))
+13. [x] executeCommand
+14. [x] mergeColumns
+15. [x] modifySql
+16. [x] renameColumn
+17. [x] renameView
+18. [x] sql
+19. [x] sqlFile
+20. [x] Change Data Test: apply delete
+21. [x] Change Data Test: apply insert
+22. [x] Change Data Test: apply loadData
+23. [x] Change Data Test: apply loadDataUpdate
+
+
+## Remaining Required Change Types to Finish in Advanced
+1. [ ]
+2. [ ]
+3. [ ]
 
 ## Remaining Required Change Types to Finish in Base/Contributed
-1. createFunction/dropFunction - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
-2. addCheckConstraint/dropCheckConstraint - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
-3. addLookupTable (executing out of order/dropping FK before creation)
-4. Change Data Test: apply loadUpdateData
+1. [ ] (nice to have, not required) createFunction/dropFunction - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
+2. [ ] (nice to have, not required) addCheckConstraint/dropCheckConstraint - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core
+
 
 
 The remaining other change types are not relevant to Databricks and have been marked with INVALID TEST

src/main/java/liquibase/ext/databricks/change/addLookupTable/AddLookupTableChangeDatabricks.java

@@ -4,35 +4,39 @@
 import java.util.Arrays;
 import java.util.List;
 
+import liquibase.change.core.*;
 import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.ext.databricks.change.createTable.CreateTableStatementDatabricks;
+import liquibase.ext.databricks.change.createTable.CreateTableChangeDatabricks;
 import liquibase.Scope;
 import liquibase.change.*;
 import liquibase.database.Database;
+import liquibase.database.core.DB2Database;
 import liquibase.database.core.Db2zDatabase;
+import liquibase.database.core.HsqlDatabase;
+import liquibase.database.core.InformixDatabase;
+import liquibase.database.core.MSSQLDatabase;
+import liquibase.database.core.OracleDatabase;
+import liquibase.database.core.SybaseASADatabase;
+import liquibase.datatype.DataTypeFactory;
 import liquibase.exception.ValidationErrors;
 import liquibase.snapshot.SnapshotGeneratorFactory;
 import liquibase.statement.NotNullConstraint;
 import liquibase.statement.SqlStatement;
 import liquibase.statement.core.CreateTableStatement;
 import liquibase.statement.core.RawSqlStatement;
+import liquibase.statement.core.ReorganizeTableStatement;
 import liquibase.structure.core.Column;
 import liquibase.structure.core.ForeignKey;
 import liquibase.structure.core.Table;
 import liquibase.change.core.AddLookupTableChange;
-import liquibase.change.core.DropForeignKeyConstraintChange;
-import liquibase.change.core.DropTableChange;
-import liquibase.change.core.AddNotNullConstraintChange;
-import liquibase.change.core.AddPrimaryKeyChange;
-import liquibase.change.core.AddForeignKeyConstraintChange;
-
 import static liquibase.statement.SqlStatement.EMPTY_SQL_STATEMENT;
 
 /**
  * Extracts data from an existing column to create a lookup table.
  * A foreign key is created between the old column and the new lookup table.
  */
-
-@DatabaseChange(name = "addLookupTable", priority = ChangeMetaData.PRIORITY_DATABASE +500, appliesTo = "column",
+@DatabaseChange(name = "addLookupTable", priority = DatabricksDatabase.PRIORITY_DATABASE + 500, appliesTo = "column",
         description = "Creates a lookup table containing values stored in a column and creates a foreign key to the new table.")
 public class AddLookupTableChangeDatabricks extends AddLookupTableChange {
 
@@ -97,7 +101,7 @@ public void setExistingColumnName(String existingColumnName) {
         this.existingColumnName = existingColumnName;
     }
 
-    @DatabaseChangeProperty(since = "3.0", description = "Name of the database catalog for the lookup table")
+    @DatabaseChangeProperty(description = "Name of the database catalog for the lookup table")
     public String getNewTableCatalogName() {
         return newTableCatalogName;
     }
@@ -150,7 +154,7 @@ public String getConstraintName() {
 
     public String getFinalConstraintName() {
         if (constraintName == null) {
-            return ("FK_" + getExistingTableName() + "_" + getNewTableName()).toUpperCase();
+            return ("fk_" + getExistingTableName() + "_" + getNewTableName()).toLowerCase();
         } else {
             return constraintName;
         }
@@ -162,10 +166,7 @@ public void setConstraintName(String constraintName) {
 
     @Override
     public boolean supports(Database database) {
-        if (database instanceof DatabricksDatabase) {
-            return true;
-        }
-        return super.supports(database);
+       return (database instanceof DatabricksDatabase);
     }
 
     @Override
@@ -187,43 +188,40 @@ protected Change[] createInverses() {
 
     @Override
     public SqlStatement[] generateStatements(Database database) {
-        List<SqlStatement> statements = new ArrayList<>();
 
         String newTableCatalogName = getNewTableCatalogName();
         String newTableSchemaName = getNewTableSchemaName();
 
         String existingTableCatalogName = getExistingTableCatalogName();
         String existingTableSchemaName = getExistingTableSchemaName();
 
-        // Step 1: Create table statement CTAS as lookup table
-        SqlStatement[] createTablesSQL = {new RawSqlStatement("CREATE TABLE " + database.escapeTableName(newTableCatalogName, newTableSchemaName, getNewTableName()) + " AS SELECT DISTINCT " + database.escapeObjectName(getExistingColumnName(), Column.class) + " AS " + database.escapeObjectName(getNewColumnName(), Column.class) + " FROM " + database.escapeTableName(existingTableCatalogName, existingTableSchemaName, getExistingTableName()) + " WHERE " + database.escapeObjectName(getExistingColumnName(), Column.class) + " IS NOT NULL")};
-
-        statements.addAll(Arrays.asList(createTablesSQL));
-
-        // Step 2: Add not null constraint to lookup table
-        AddNotNullConstraintChange addNotNullChange = new AddNotNullConstraintChange();
-        addNotNullChange.setSchemaName(newTableSchemaName);
-        addNotNullChange.setTableName(getNewTableName());
-        addNotNullChange.setColumnName(getNewColumnName());
-        addNotNullChange.setColumnDataType(getNewColumnDataType());
-        statements.addAll(Arrays.asList(addNotNullChange.generateStatements(database)));
-
+        SqlStatement[] createTablesSQL = {new RawSqlStatement("CREATE TABLE " + database.escapeTableName(newTableCatalogName, newTableSchemaName, getNewTableName())
+               + " USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') "
+               + " AS SELECT DISTINCT " + database.escapeObjectName(getExistingColumnName(), Column.class)
+                + " AS " + database.escapeObjectName(getNewColumnName(), Column.class)
+                + " FROM " + database.escapeTableName(existingTableCatalogName, existingTableSchemaName, getExistingTableName())
+                + " WHERE " + database.escapeObjectName(getExistingColumnName(), Column.class)
+                + " IS NOT NULL")
+        };
 
-        // Step 3: Add Primary Key Constraint to Lookup table
-        // Add a properly named primary key with just the column name + "_pk"
+        List<SqlStatement> statements = new ArrayList<>(Arrays.asList(createTablesSQL));
 
-        String inferred_pk_name = "pk_" + getNewColumnName();
+        if (!(database instanceof OracleDatabase) && !(database instanceof Db2zDatabase)) {
+            AddNotNullConstraintChange addNotNullChange = new AddNotNullConstraintChange();
+            addNotNullChange.setSchemaName(newTableSchemaName);
+            addNotNullChange.setTableName(getNewTableName());
+            addNotNullChange.setColumnName(getNewColumnName());
+            addNotNullChange.setColumnDataType(getNewColumnDataType());
+            statements.addAll(Arrays.asList(addNotNullChange.generateStatements(database)));
+        }
 
         AddPrimaryKeyChange addPKChange = new AddPrimaryKeyChange();
         addPKChange.setSchemaName(newTableSchemaName);
         addPKChange.setTableName(getNewTableName());
         addPKChange.setColumnNames(getNewColumnName());
-        addPKChange.setConstraintName(inferred_pk_name);
-
         statements.addAll(Arrays.asList(addPKChange.generateStatements(database)));
 
 
-        // Step 4: Add FK constraint to original table, referencing lookup table
         AddForeignKeyConstraintChange addFKChange = new AddForeignKeyConstraintChange();
         addFKChange.setBaseTableSchemaName(existingTableSchemaName);
         addFKChange.setBaseTableName(getExistingTableName());
@@ -235,7 +233,7 @@ public SqlStatement[] generateStatements(Database database) {
         addFKChange.setConstraintName(getFinalConstraintName());
         statements.addAll(Arrays.asList(addFKChange.generateStatements(database)));
 
-        return statements.toArray(EMPTY_SQL_STATEMENT);
+        return statements.toArray(new SqlStatement[0]);
     }
 
     @Override
@@ -270,4 +268,4 @@ public String getConfirmationMessage() {
     public String getSerializedObjectNamespace() {
         return STANDARD_CHANGELOG_NAMESPACE;
     }
-}
+}

src/main/java/liquibase/ext/databricks/database/DatabricksDatabase.java

@@ -108,6 +108,8 @@ public boolean supportsTablespaces() {
         return false;
     }
 
+    @Override
+    public boolean supportsSequences() { return false; }
 
     @Override
     public String getAutoIncrementClause(final BigInteger startWith, final BigInteger incrementBy, final String generationType, final Boolean defaultOnNull) {

src/main/java/liquibase/ext/databricks/sqlgenerator/AddAutoIncrementGeneratorDatabricks.java

@@ -0,0 +1,28 @@
+package liquibase.ext.databricks.sqlgenerator;
+
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.database.Database;
+import liquibase.exception.ValidationErrors;
+import liquibase.sqlgenerator.SqlGeneratorChain;
+import liquibase.sqlgenerator.core.AddAutoIncrementGenerator;
+import liquibase.statement.core.AddAutoIncrementStatement;
+
+public class AddAutoIncrementGeneratorDatabricks extends AddAutoIncrementGenerator {
+
+    @Override
+    public int getPriority() {
+        return DatabricksDatabase.PRIORITY_DATABASE;
+    }
+
+    @Override
+    public boolean supports(AddAutoIncrementStatement statement, Database database) {
+        return super.supports(statement, database)
+                && database instanceof DatabricksDatabase;
+    }
+
+    @Override
+    public ValidationErrors validate(AddAutoIncrementStatement statement,
+                                     Database database, SqlGeneratorChain sqlGeneratorChain) {
+        return new ValidationErrors().addError("Databricks does not support adding AUTO_INCREMENT.");
+    }
+}

src/main/java/liquibase/ext/databricks/sqlgenerator/AddUniqueConstraintGeneratorDatabricks.java

@@ -0,0 +1,30 @@
+package liquibase.ext.databricks.sqlgenerator;
+
+
+import liquibase.ext.databricks.database.DatabricksDatabase;
+import liquibase.database.Database;
+import liquibase.exception.ValidationErrors;
+import liquibase.sqlgenerator.SqlGeneratorChain;
+import liquibase.sqlgenerator.core.AddUniqueConstraintGenerator;
+import liquibase.statement.core.AddUniqueConstraintStatement;
+
+public class AddUniqueConstraintGeneratorDatabricks extends AddUniqueConstraintGenerator {
+
+    @Override
+    public int getPriority() {
+        return DatabricksDatabase.PRIORITY_DATABASE;
+    }
+
+    @Override
+    public boolean supports(AddUniqueConstraintStatement statement, Database database) {
+        return super.supports(statement, database)
+                && database instanceof DatabricksDatabase;
+    }
+
+    @Override
+    public ValidationErrors validate(AddUniqueConstraintStatement statement,
+                                     Database database, SqlGeneratorChain sqlGeneratorChain) {
+        return new ValidationErrors().addError(
+                "Databricks does not support altering unique constraint key.");
+    }
+}