diff --git a/README.md b/README.md index c4d18449..4be98846 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Databricks tables creates with liquibase are automatically created with the Delt ## NOTE! ONLY TABLES CREATED WITH UNITY CATALOG ARE SUPPORTED FOR MOST ADVANCED OPERATIONS -This extension utilizes Unity Catalog System tables for many advanced operations such as snapshotting, idedentifying various constraints (PK/FK/NOT NULL, etc.) +This extension utilizes Unity Catalog System tables for many advanced operations such as snapshotting, identifying various constraints (PK/FK/NOT NULL, etc.) If hive_metastore is used, this is not tested and may not provide all the below functionality. @@ -19,7 +19,7 @@ If hive_metastore is used, this is not tested and may not provide all the below 1. Add unit tests with liquibase test harness - Cody Davis - DONE 2. Pass Foundational Test Harness - Cody Davis - DONE 4/1/2023 3. Pass Contributed Test Harness - Cody Davis - IN PROGRESS 9/15/2023 -4. Pass Advanced Test Harness - <> - NOT STARTED +4. Pass Advanced Test Harness - Cody Davis - IN PROGRESS ## Currently Supported Change Types: @@ -47,7 +47,7 @@ If hive_metastore is used, this is not tested and may not provide all the below 22. Change Data Test: apply loadData -## Remaining Relevant Change Types to Finish in Base/Contributed +## Remaining Required Change Types to Finish in Base/Contributed 1. createFunction/dropFunction - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core 2. addCheckConstraint/dropCheckConstraint - in Liquibase Pro, should work in Databricks, but change type not accessible from Liquibase Core 3. addLookupTable (executing out of order/dropping FK before creation) diff --git a/pom.xml b/pom.xml index 8f1619e8..84672c84 100644 --- a/pom.xml +++ b/pom.xml @@ -168,7 +168,7 @@ true false - Liquibase Databricks ${version} API + Liquibase Databricks ${project.version} API true none UTF-8 diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ForeignKeySnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ForeignKeySnapshotGeneratorDatabricks.java index 40be1af9..071a63c4 100644 --- a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ForeignKeySnapshotGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ForeignKeySnapshotGeneratorDatabricks.java @@ -1,6 +1,7 @@ package liquibase.ext.databricks.snapshot.jvm; import liquibase.ext.databricks.database.DatabricksDatabase; +import liquibase.snapshot.*; import liquibase.snapshot.jvm.JdbcSnapshotGenerator; import liquibase.CatalogAndSchema; import liquibase.database.AbstractJdbcDatabase; @@ -13,10 +14,8 @@ import liquibase.database.jvm.JdbcConnection; import liquibase.diff.compare.DatabaseObjectComparatorFactory; import liquibase.exception.DatabaseException; -import liquibase.snapshot.CachedRow; -import liquibase.snapshot.DatabaseSnapshot; -import liquibase.snapshot.InvalidExampleException; -import liquibase.snapshot.JdbcDatabaseSnapshot; +import liquibase.snapshot.jvm.SequenceSnapshotGenerator; +import liquibase.snapshot.jvm.UniqueConstraintSnapshotGenerator; import liquibase.structure.DatabaseObject; import liquibase.structure.core.*; import java.sql.DatabaseMetaData; @@ -24,8 +23,9 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import liquibase.snapshot.jvm.ForeignKeySnapshotGenerator; -public class ForeignKeySnapshotGeneratorDatabricks extends JdbcSnapshotGenerator { +public class ForeignKeySnapshotGeneratorDatabricks extends ForeignKeySnapshotGenerator { protected static final String METADATA_DEFERRABILITY = "DEFERRABILITY"; public static final String METADATA_FKTABLE_CAT = "FKTABLE_CAT"; @@ -40,8 +40,9 @@ public class ForeignKeySnapshotGeneratorDatabricks extends JdbcSnapshotGenerator public static final String METADATA_DELETE_RULE = "DELETE_RULE"; - public ForeignKeySnapshotGeneratorDatabricks() { - super(ForeignKey.class, new Class[]{Table.class}); + @Override + public Class[] replaces() { + return new Class[]{ForeignKeySnapshotGenerator.class}; } @@ -167,11 +168,16 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot } setValidateOptionIfAvailable(database, foreignKey, row); + /* + DATABRICKS does not support indexes, see OPTIMIZE / CLUSTER BY change types for that behavior + Index exampleIndex = new Index().setRelation(foreignKey.getForeignKeyTable()); exampleIndex.getColumns().addAll(foreignKey.getForeignKeyColumns()); exampleIndex.addAssociatedWith(Index.MARK_FOREIGN_KEY); foreignKey.setBackingIndex(exampleIndex); + */ + } if (snapshot.get(ForeignKey.class).contains(foreignKey)) { return null; diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/AddForeignKeyConstraintGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/AddForeignKeyConstraintGeneratorDatabricks.java index 4b9fe118..d9ac16e7 100644 --- a/src/main/java/liquibase/ext/databricks/sqlgenerator/AddForeignKeyConstraintGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/AddForeignKeyConstraintGeneratorDatabricks.java @@ -12,10 +12,8 @@ import liquibase.structure.core.ForeignKey; import liquibase.structure.core.Table; import liquibase.sqlgenerator.core.AddForeignKeyConstraintGenerator; -import liquibase.sqlgenerator.core.AbstractSqlGenerator; - -public class AddForeignKeyConstraintGeneratorDatabricks extends AbstractSqlGenerator { +public class AddForeignKeyConstraintGeneratorDatabricks extends AddForeignKeyConstraintGenerator { @Override @SuppressWarnings({"SimplifiableIfStatement"}) diff --git a/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/addUniqueConstraint.sql b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/addUniqueConstraint.sql new file mode 100644 index 00000000..c5061573 --- /dev/null +++ b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/addUniqueConstraint.sql @@ -0,0 +1,2 @@ +INVALID TEST +-- Databricks does not support unique constraints diff --git a/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createIndex.sql b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createIndex.sql new file mode 100644 index 00000000..6647b408 --- /dev/null +++ b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createIndex.sql @@ -0,0 +1,2 @@ +INVALID TEST +-- Databricks does not have classical indexes, see CLUSTER BY or OPTIMIZE / ZORDER \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createTable.sql b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createTable.sql new file mode 100644 index 00000000..aef9738a --- /dev/null +++ b/src/test/resources/liquibase/harness/compatibility/advanced/expectedSql/generateChangelog/createTable.sql @@ -0,0 +1,5 @@ +CREATE TABLE authors_data (authors_email STRING(255) NOT NULL, CONSTRAINT pk_authors_email PRIMARY KEY (authors_email)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table (test_column INT(10), varcharColumn STRING(255), intColumn INT(10), dateColumn date) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table_base (id INT(10) NOT NULL, CONSTRAINT pk_test_table_base PRIMARY KEY (id)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table_reference (id INT(10), test_column INT(10) NOT NULL, CONSTRAINT pk_test_table_reference PRIMARY KEY (test_column)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table_xml (test_column INT(10)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/generateChangelog/expectedSql/databricks/createTable.sql b/src/test/resources/liquibase/harness/generateChangelog/expectedSql/databricks/createTable.sql new file mode 100644 index 00000000..aa91a4a2 --- /dev/null +++ b/src/test/resources/liquibase/harness/generateChangelog/expectedSql/databricks/createTable.sql @@ -0,0 +1,3 @@ +CREATE TABLE authors_data (authors_email STRING(255) NOT NULL, CONSTRAINT pk_authors_email PRIMARY KEY (authors_email)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table (test_column INT(10), varcharColumn STRING(255), intColumn INT(10), dateColumn date) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') +CREATE TABLE test_table_xml (test_column INT(10)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name') \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addColumn.json b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addColumn.json new file mode 100644 index 00000000..96eda625 --- /dev/null +++ b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addColumn.json @@ -0,0 +1,51 @@ +{ + "snapshot": { + "objects": { + "liquibase.structure.core.Table": [ + { + "table": { + "name": "test_table" + } + } + ], + "liquibase.structure.core.Column": [ + { + "column": { + "name": "test_column", + "nullable": true, + "type": { + "typeName": "INT" + } + } + }, + { + "column": { + "name": "varcharColumn", + "nullable": true, + "type": { + "typeName": "STRING" + } + } + }, + { + "column": { + "name": "intColumn", + "nullable": true, + "type": { + "typeName": "INT" + } + } + }, + { + "column": { + "name": "dateColumn", + "nullable": true, + "type": { + "typeName": "DATE" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addForeignKey.json b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addForeignKey.json new file mode 100644 index 00000000..8cdc40e3 --- /dev/null +++ b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addForeignKey.json @@ -0,0 +1,27 @@ +{ + "snapshot": { + "objects": { + "liquibase.structure.core.Table": [ + { + "table": { + "name": "test_table_base" + } + }, + { + "table": { + "name": "test_table_reference" + } + } + ], + "liquibase.structure.core.ForeignKey": [ + { + "foreignKey": { + "deferrable": true, + "initiallyDeferred": true, + "name": "test_fk" + } + } + ] + } + } +} \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addPrimaryKey.json b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addPrimaryKey.json new file mode 100644 index 00000000..38b2c854 --- /dev/null +++ b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/addPrimaryKey.json @@ -0,0 +1,20 @@ +{ + "snapshot": { + "objects": { + "liquibase.structure.core.Table": [ + { + "table": { + "name": "test_table" + } + } + ], + "liquibase.structure.core.PrimaryKey": [ + { + "primaryKey": { + "name": "test_pk" + } + } + ] + } + } +} \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/snapshotCaalogAndSchema.json b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/snapshotCaalogAndSchema.json new file mode 100644 index 00000000..f9909b00 --- /dev/null +++ b/src/test/resources/liquibase/harness/snapshot/expectedSnapshot/snapshotCaalogAndSchema.json @@ -0,0 +1,20 @@ +{ + "snapshot": { + "objects": { + "liquibase.structure.core.Catalog": [ + { + "catalog": { + "name": "main" + } + } + ], + "liquibase.structure.core.Schema": [ + { + "schema": { + "name": "liquibase_harness_test_ds" + } + } + ] + } + } +} \ No newline at end of file