From 41366277893c9190575cceb6d46eec5719056432 Mon Sep 17 00:00:00 2001 From: Vitalii Makarchenkov Date: Tue, 23 Jul 2024 12:11:30 +0300 Subject: [PATCH 1/3] Add AlterCluster change --- .../AlterClusterChangeDatabricks.java | 107 ++++++++++++++++++ .../AlterClusterDatabricksStatement.java | 60 ++++++++++ .../change/alterCluster/ColumnConfig.java | 26 +++++ .../change/alterCluster/NoneConfig.java | 26 +++++ .../AlterClusterGeneratorDatabricks.java | 49 ++++++++ .../META-INF/services/liquibase.change.Change | 3 +- .../liquibase.sqlgenerator.SqlGenerator | 3 +- .../databricks/liquibase-databricks-1.0.xsd | 44 +++++++ .../liquibase-databricks-latest.xsd | 44 +++++++ .../changelogs/databricks/alterCluster.xml | 47 ++++++++ .../databricks/alterCluster.json | 2 + .../expectedSql/databricks/alterCluster.sql | 3 + 12 files changed, 412 insertions(+), 2 deletions(-) create mode 100644 src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterChangeDatabricks.java create mode 100644 src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterDatabricksStatement.java create mode 100644 src/main/java/liquibase/ext/databricks/change/alterCluster/ColumnConfig.java create mode 100644 src/main/java/liquibase/ext/databricks/change/alterCluster/NoneConfig.java create mode 100644 src/main/java/liquibase/ext/databricks/sqlgenerator/AlterClusterGeneratorDatabricks.java create mode 100644 src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml create mode 100644 src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/alterCluster.json create mode 100644 src/test/resources/liquibase/harness/change/expectedSql/databricks/alterCluster.sql diff --git a/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterChangeDatabricks.java b/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterChangeDatabricks.java new file mode 100644 index 00000000..65f64084 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterChangeDatabricks.java @@ -0,0 +1,107 @@ +package liquibase.ext.databricks.change.alterCluster; + +import liquibase.change.AbstractChange; +import liquibase.change.DatabaseChange; +import liquibase.change.DatabaseChangeProperty; +import liquibase.database.Database; +import liquibase.exception.ValidationErrors; +import liquibase.servicelocator.PrioritizedService; +import liquibase.statement.SqlStatement; + +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.List; + +@DatabaseChange(name = "alterCluster", description = "Alter Cluster", priority = PrioritizedService.PRIORITY_DATABASE +500) +public class AlterClusterChangeDatabricks extends AbstractChange { + + private String tableName; + private String catalogName; + private String schemaName; + private List columns; + private List clusterBy; + + @Override + public ValidationErrors validate(Database database) { + ValidationErrors validationErrors = new ValidationErrors(); + validationErrors.addAll(super.validate(database)); + + if (columns == null && clusterBy == null) { + validationErrors.addError("Alter Cluster change require list of columns or element 'ClusterBy', please add at least one option."); + } + return validationErrors; + } + + public AlterClusterChangeDatabricks() { + super(); + columns = new ArrayList<>(); + clusterBy = new ArrayList<>(); + } + + @Override + public String getConfirmationMessage() { + return MessageFormat.format("{0}.{1}.{2} successfully altered.", getCatalogName(), getSchemaName(), getTableName()); + } + + @Override + public SqlStatement[] generateStatements(Database database) { + AlterClusterDatabricksStatement statement = new AlterClusterDatabricksStatement(tableName, catalogName, schemaName); + if (getColumns() != null && !getColumns().isEmpty()) { + statement.setColumns(getColumns()); + } else if (getClusterBy() != null && !getClusterBy().isEmpty()) { + statement.setClusterBy(getClusterBy()); + } + return new SqlStatement[]{statement}; + } + + @DatabaseChangeProperty + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + @DatabaseChangeProperty + public List getColumns() { + if (columns == null) { + return new ArrayList<>(); + } + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + @DatabaseChangeProperty + public String getCatalogName() { + return catalogName; + } + + public void setCatalogName(String catalogName) { + this.catalogName = catalogName; + } + + @DatabaseChangeProperty + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + @DatabaseChangeProperty + public List getClusterBy() { + if (clusterBy == null) { + return new ArrayList<>(); + } + return clusterBy; + } + + public void setClusterBy(List clusterBy) { + this.clusterBy = clusterBy; + } +} diff --git a/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterDatabricksStatement.java b/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterDatabricksStatement.java new file mode 100644 index 00000000..3d644af4 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/alterCluster/AlterClusterDatabricksStatement.java @@ -0,0 +1,60 @@ +package liquibase.ext.databricks.change.alterCluster; + +import liquibase.statement.AbstractSqlStatement; + +import java.util.List; + +public class AlterClusterDatabricksStatement extends AbstractSqlStatement { + + private String tableName; + private String catalogName; + private String schemaName; + private List columns; + private List clusterBy; + + public AlterClusterDatabricksStatement(String tableName, String catalogName, String schemaName) { + this.tableName = tableName; + this.catalogName = catalogName; + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public List getColumns() { + return columns; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + public String getCatalogName() { + return catalogName; + } + + public void setCatalogName(String catalogName) { + this.catalogName = catalogName; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public List getClusterBy() { + return clusterBy; + } + + public void setClusterBy(List clusterBy) { + this.clusterBy = clusterBy; + } +} diff --git a/src/main/java/liquibase/ext/databricks/change/alterCluster/ColumnConfig.java b/src/main/java/liquibase/ext/databricks/change/alterCluster/ColumnConfig.java new file mode 100644 index 00000000..ecf53504 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/alterCluster/ColumnConfig.java @@ -0,0 +1,26 @@ +package liquibase.ext.databricks.change.alterCluster; + +import liquibase.serializer.AbstractLiquibaseSerializable; + +public class ColumnConfig extends AbstractLiquibaseSerializable { + + private String name; + + @Override + public String getSerializedObjectName() { + return "column"; + } + + @Override + public String getSerializedObjectNamespace() { + return "http://www.liquibase.org/xml/ns/databricks"; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/src/main/java/liquibase/ext/databricks/change/alterCluster/NoneConfig.java b/src/main/java/liquibase/ext/databricks/change/alterCluster/NoneConfig.java new file mode 100644 index 00000000..448a4db6 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/change/alterCluster/NoneConfig.java @@ -0,0 +1,26 @@ +package liquibase.ext.databricks.change.alterCluster; + +import liquibase.serializer.AbstractLiquibaseSerializable; + +public class NoneConfig extends AbstractLiquibaseSerializable { + + private String none; + + @Override + public String getSerializedObjectName() { + return "clusterBy"; + } + + @Override + public String getSerializedObjectNamespace() { + return "http://www.liquibase.org/xml/ns/databricks"; + } + + public String getNone() { + return none; + } + + public void setNone(String none) { + this.none = none; + } +} diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/AlterClusterGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/AlterClusterGeneratorDatabricks.java new file mode 100644 index 00000000..3d3a2f16 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/AlterClusterGeneratorDatabricks.java @@ -0,0 +1,49 @@ +package liquibase.ext.databricks.sqlgenerator; + +import liquibase.database.Database; +import liquibase.exception.ValidationErrors; +import liquibase.ext.databricks.change.alterCluster.AlterClusterDatabricksStatement; +import liquibase.ext.databricks.change.alterCluster.ColumnConfig; +import liquibase.sql.Sql; +import liquibase.sql.UnparsedSql; +import liquibase.sqlgenerator.SqlGeneratorChain; +import liquibase.sqlgenerator.core.AbstractSqlGenerator; + +public class AlterClusterGeneratorDatabricks extends AbstractSqlGenerator { + @Override + public ValidationErrors validate(AlterClusterDatabricksStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) { + ValidationErrors validationErrors = new ValidationErrors(); + if (statement.getClusterBy() == null && statement.getColumns() == null){ + validationErrors.addError("WARNING! Alter Cluster change require list of columns or element 'ClusterBy', please add at least one option."); + } + if (statement.getClusterBy() != null && (statement.getClusterBy().isEmpty() || !statement.getClusterBy().get(0).getNone().equals("true"))) { + validationErrors.addError("WARNING! ClusterBy attribute require attribute 'none=\"true\"'"); + } + return validationErrors; + } + + @Override + public Sql[] generateSql(AlterClusterDatabricksStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) { + StringBuilder buffer = new StringBuilder(); + + buffer.append("ALTER TABLE "); + buffer.append(database.escapeTableName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName())); + buffer.append(" CLUSTER BY "); + if (statement.getColumns() != null && !statement.getColumns().isEmpty()) { + buffer.append("("); + for (ColumnConfig column : statement.getColumns()) { + buffer.append(column.getName()); + buffer.append(","); + } + buffer.deleteCharAt(buffer.length() - 1); + buffer.append(")"); + } else if (statement.getClusterBy() != null && !statement.getClusterBy().isEmpty()) { + buffer.append("NONE"); + } + + return new Sql[]{ + new UnparsedSql(buffer.toString()) + }; + } + +} diff --git a/src/main/resources/META-INF/services/liquibase.change.Change b/src/main/resources/META-INF/services/liquibase.change.Change index 93aa7f79..59a3a1e3 100644 --- a/src/main/resources/META-INF/services/liquibase.change.Change +++ b/src/main/resources/META-INF/services/liquibase.change.Change @@ -5,4 +5,5 @@ liquibase.ext.databricks.change.vacuumTable.VacuumTableChange liquibase.ext.databricks.change.addLookupTable.AddLookupTableChangeDatabricks liquibase.ext.databricks.change.addCheckConstraint.AddCheckConstraintChangeDatabricks liquibase.ext.databricks.change.dropCheckConstraint.DropCheckConstraintChangeDatabricks -liquibase.ext.databricks.change.alterTableProperties.AlterTablePropertiesChangeDatabricks \ No newline at end of file +liquibase.ext.databricks.change.alterTableProperties.AlterTablePropertiesChangeDatabricks +liquibase.ext.databricks.change.alterCluster.AlterClusterChangeDatabricks \ No newline at end of file diff --git a/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator b/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator index 0af35927..ea7bbb24 100644 --- a/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator +++ b/src/main/resources/META-INF/services/liquibase.sqlgenerator.SqlGenerator @@ -18,4 +18,5 @@ liquibase.ext.databricks.sqlgenerator.InsertOrUpdateGeneratorDatabricks liquibase.ext.databricks.sqlgenerator.UpdateGeneratorDatabricks liquibase.ext.databricks.change.addCheckConstraint.AddCheckConstraintGeneratorDatabricks liquibase.ext.databricks.change.dropCheckConstraint.DropCheckConstraintGeneratorDatabricks -liquibase.ext.databricks.sqlgenerator.AlterTablePropertiesGeneratorDatabricks \ No newline at end of file +liquibase.ext.databricks.sqlgenerator.AlterTablePropertiesGeneratorDatabricks +liquibase.ext.databricks.sqlgenerator.AlterClusterGeneratorDatabricks \ No newline at end of file diff --git a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd index 4e61b353..415abea9 100644 --- a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd +++ b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-1.0.xsd @@ -37,4 +37,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd index 4e61b353..415abea9 100644 --- a/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd +++ b/src/main/resources/www.liquibase.org/xml/ns/databricks/liquibase-databricks-latest.xsd @@ -37,4 +37,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml new file mode 100644 index 00000000..3f400e6f --- /dev/null +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/alterCluster.json b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/alterCluster.json new file mode 100644 index 00000000..7a73a41b --- /dev/null +++ b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/alterCluster.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/src/test/resources/liquibase/harness/change/expectedSql/databricks/alterCluster.sql b/src/test/resources/liquibase/harness/change/expectedSql/databricks/alterCluster.sql new file mode 100644 index 00000000..f20e7b33 --- /dev/null +++ b/src/test/resources/liquibase/harness/change/expectedSql/databricks/alterCluster.sql @@ -0,0 +1,3 @@ +CREATE TABLE main.liquibase_harness_test_ds.test_table_alter_cluster (test_id INT NOT NULL, test_new INT, CONSTRAINT PK_TEST_TABLE_ALTER_CLUSTER PRIMARY KEY (test_id)) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true) +ALTER TABLE main.liquibase_harness_test_ds.test_table_alter_cluster CLUSTER BY (test_id) +ALTER TABLE main.liquibase_harness_test_ds.test_table_alter_cluster CLUSTER BY (test_id,test_new) From fc855cd4119bd9a60ae7baf349501b5b8b757f59 Mon Sep 17 00:00:00 2001 From: Vitalii Makarchenkov Date: Tue, 23 Jul 2024 17:29:03 +0300 Subject: [PATCH 2/3] Fix test data --- .../harness/change/changelogs/databricks/alterCluster.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml index 3f400e6f..706b58b8 100644 --- a/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.xml @@ -24,12 +24,12 @@ - - + - - + + + From 56ad8fb6650c6f230de185e29a6601a8c0430960 Mon Sep 17 00:00:00 2001 From: Vitalii Makarchenkov Date: Wed, 31 Jul 2024 15:59:58 +0300 Subject: [PATCH 3/3] Add YAML and JSON changelogs --- .../changelogs/databricks/alterCluster.json | 55 +++++++++++++++++++ .../changelogs/databricks/alterCluster.yaml | 25 +++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.json create mode 100644 src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.yaml diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.json b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.json new file mode 100644 index 00000000..8445058c --- /dev/null +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.json @@ -0,0 +1,55 @@ +{ + "databaseChangeLog": [ + { + "changeSet": { + "id": "1", + "author": "your.name", + "changes": [ + { + "createTable": { + "tableName": "test_table_alter_cluster", + "columns": [ + { + "column": { + "name":"test_id", + "type": "int" + } + } + ] + } + } + ] + } + }, + { + "changeSet": { + "id": "2", + "author": "your.name", + "changes": [ + { + "alterCluster": { + "tableName": "test_table_alter_cluster", + "columns": [ + { + "column": { + "name": "test_id" + } + } + ] + } + } + ], + "rollback": [ + { + "alterCluster": { + "tableName": "test_table_alter_cluster", + "clusterBy": { + "none": "true" + } + } + } + ] + } + } + ] +} diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.yaml b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.yaml new file mode 100644 index 00000000..8482ff00 --- /dev/null +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/alterCluster.yaml @@ -0,0 +1,25 @@ +databaseChangeLog: + - changeSet: + id: 1 + author: your.name + changes: + - createTable: + tableName: test_table_alter_cluster + columns: + - column: + name: test_id + type: int + - changeSet: + id: 2 + author: your.name + changes: + - alterCluster: + tableName: test_table_alter_cluster + columns: + - column: + name: test_id + rollback: + - alterCluster: + tableName: test_table_alter_cluster + clusterBy: + none: "true"