diff --git a/pom.xml b/pom.xml index f2409958..bf2031ad 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ UTF-8 1.8 1.8 - 4.27.0 + 4.28.0 liquibase ${sonar.organization}_${project.artifactId} ${project.name} @@ -119,6 +119,12 @@ ${dependency.spock.version} test + + com.databricks + databricks-jdbc + 2.6.38 + compile + diff --git a/src/main/java/liquibase/ext/databricks/database/DatabricksConnection.java b/src/main/java/liquibase/ext/databricks/database/DatabricksConnection.java index 08e9b277..302f3e24 100644 --- a/src/main/java/liquibase/ext/databricks/database/DatabricksConnection.java +++ b/src/main/java/liquibase/ext/databricks/database/DatabricksConnection.java @@ -3,6 +3,7 @@ import com.databricks.client.jdbc.jdbc42.S42Connection; import com.databricks.client.spark.core.SparkJDBCConnection; import liquibase.Scope; +import liquibase.database.DatabaseConnection; import liquibase.database.jvm.JdbcConnection; import liquibase.exception.DatabaseException; import liquibase.exception.UnexpectedLiquibaseException; @@ -55,12 +56,18 @@ public Connection getUnderlyingConnection() { public void open(String url, Driver driverObject, Properties driverProperties) throws DatabaseException { driverProperties.setProperty("UserAgentEntry", "Liquibase"); + driverProperties.setProperty("EnableArrow", "0"); // Set UserAgent to specify to Databricks that liquibase is the tool running these commands // Set EnableArrow because the arrow results break everything. And the JDBC release notes say to just disable it. - //String updatedUrl = url + "UserAgentEntry=Liquibase;EnableArrow=0"; - // This is done in getConnectionUrl() - this.openConn(url, driverObject, driverProperties); + // Ensure there's a terminating semicolon for consistent parsing + if (!url.endsWith(";")) { + url += ";"; + } + + String updatedUrl = url + "UserAgentEntry=Liquibase;EnableArrow=0"; + + this.openConn(updatedUrl, driverObject, driverProperties); } public void openConn(String url, Driver driverObject, Properties driverProperties) throws DatabaseException { @@ -96,25 +103,34 @@ public void setAutoCommit(boolean autoCommit) throws DatabaseException { } protected static String getUrlParamValue(String url, String paramName, String defaultValue) { + + //System.out.println("PARSE URL - url" + url); + if (url == null) { return null; } - - // Get catalog of connection and schema of connection + // Ensure there's a terminating semicolon for consistent parsing + if (!url.endsWith(";")) { + url += ";"; + } + // Remove spaces and split by semicolon String[] uriArgs = url.replace(" ", "").split(";"); + + // System.out.println("PARSE URL - url args" + uriArgs.toString()); + + // Use Java Streams to find the parameter value Optional paramString = Arrays.stream(uriArgs) .filter(x -> x.startsWith(paramName + "=")) .findFirst(); - + // Return the parameter value if found, otherwise return the default value if (!paramString.isPresent()) { return defaultValue; } String[] defaultParamsArr = paramString.get().split("="); - return defaultParamsArr[1]; + return defaultParamsArr.length > 1 ? defaultParamsArr[1] : defaultValue; // Check to avoid index out of bound } - @Override public String getDatabaseProductVersion() throws DatabaseException { try { diff --git a/src/main/java/liquibase/ext/databricks/database/DatabricksDatabase.java b/src/main/java/liquibase/ext/databricks/database/DatabricksDatabase.java index 7bc8ec67..85b0ce3d 100644 --- a/src/main/java/liquibase/ext/databricks/database/DatabricksDatabase.java +++ b/src/main/java/liquibase/ext/databricks/database/DatabricksDatabase.java @@ -2,6 +2,7 @@ import liquibase.Scope; import liquibase.database.AbstractJdbcDatabase; +import liquibase.database.Database; import liquibase.database.DatabaseConnection; import liquibase.database.jvm.JdbcConnection; import liquibase.exception.DatabaseException; @@ -11,6 +12,7 @@ import liquibase.structure.core.Catalog; import liquibase.structure.core.Schema; import java.math.BigInteger; +import java.sql.Connection; import java.sql.ResultSet; import java.util.Arrays; import java.util.HashSet; @@ -206,6 +208,7 @@ protected String getConnectionSchemaName() { } try { + String foundSchema = parseUrlForSchema(connection.getURL()); Scope.getCurrentScope().getLog(getClass()).info("SCHEMA IDENTIFIED: " + foundSchema); @@ -338,8 +341,6 @@ private Set getDatabricksReservedWords() { )); } - - @Override public void setConnection(DatabaseConnection conn) { DatabaseConnection dbConn; diff --git a/src/main/java/liquibase/ext/databricks/datatype/BinaryDataTypeDatabricks.java b/src/main/java/liquibase/ext/databricks/datatype/BinaryDataTypeDatabricks.java new file mode 100644 index 00000000..589316ff --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/datatype/BinaryDataTypeDatabricks.java @@ -0,0 +1,35 @@ +package liquibase.ext.databricks.datatype; + +import liquibase.change.core.LoadDataChange; +import liquibase.database.Database; +import liquibase.datatype.DataTypeInfo; +import liquibase.datatype.DatabaseDataType; +import liquibase.datatype.LiquibaseDataType; +import liquibase.datatype.core.BlobType; +import liquibase.ext.databricks.database.DatabricksDatabase; + + +public class BinaryDataTypeDatabricks extends BlobType { + + + @Override + public DatabaseDataType toDatabaseDataType(Database database) { + + if (database instanceof DatabricksDatabase) { + return new DatabaseDataType("BINARY"); + } + + return super.toDatabaseDataType(database); + } + + @Override + public int getPriority() { + return DatabricksDatabase.DATABRICKS_PRIORITY_DATABASE; + } + + + @Override + public boolean supports(Database database) { + return database instanceof DatabricksDatabase; + } +} diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ViewSnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ViewSnapshotGeneratorDatabricks.java index f6bb67f1..9f4e0187 100644 --- a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ViewSnapshotGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ViewSnapshotGeneratorDatabricks.java @@ -4,8 +4,10 @@ import liquibase.Scope; import liquibase.database.AbstractJdbcDatabase; import liquibase.database.Database; +import liquibase.database.DatabaseConnection; import liquibase.exception.DatabaseException; import liquibase.executor.ExecutorService; +import liquibase.ext.databricks.database.DatabricksConnection; import liquibase.snapshot.DatabaseSnapshot; import liquibase.snapshot.jvm.ViewSnapshotGenerator; import liquibase.statement.core.RawSqlStatement; @@ -14,10 +16,12 @@ import liquibase.structure.core.View; import liquibase.util.StringUtil; +import java.sql.ResultSet; import java.util.List; import java.util.Map; import liquibase.ext.databricks.database.DatabricksDatabase; + public class ViewSnapshotGeneratorDatabricks extends ViewSnapshotGenerator { @@ -37,31 +41,57 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot } else { Database database = snapshot.getDatabase(); Schema schema = example.getSchema(); + DatabaseConnection connection = database.getConnection(); CatalogAndSchema catalogAndSchema = (new CatalogAndSchema(schema.getCatalogName(), schema.getName())).customize(database); String jdbcSchemaName = database.correctObjectName(((AbstractJdbcDatabase) database).getJdbcSchemaName(catalogAndSchema), Schema.class); String query = String.format("SELECT view_definition FROM %s.%s.VIEWS WHERE table_name='%s' AND table_schema='%s' AND table_catalog='%s';", schema.getCatalogName(), database.getSystemSchema(), example.getName(), schema.getName(), schema.getCatalogName()); + // DEBUG + //System.out.println("Snapshot Database Connection URL : " + database.getConnection().getURL()); + //System.out.println("Snapshot Database Connection Class : " + database.getConnection().getClass().getName()); + + List> viewsMetadataRs = Scope.getCurrentScope().getSingleton(ExecutorService.class) .getExecutor("jdbc", database).queryForList(new RawSqlStatement(query)); + // New Code, likely superfluous, was used for testing + /// This should use our existing DatabaseConnection url processing + String rawViewDefinition = null; + + try (ResultSet viewMetadataResultSet = ((DatabricksConnection) connection).createStatement().executeQuery(query)) { + //System.out.println("Raw Result VIEW " + viewMetadataResultSet); + + viewMetadataResultSet.next(); + rawViewDefinition = viewMetadataResultSet.getString(1); + + + } catch (Exception e) { + Scope.getCurrentScope().getLog(getClass()).info("Error getting View Definiton via existing context, going to pull from URL", e); + } + + /// Old Code + if (viewsMetadataRs.isEmpty()) { return null; } else { + Map row = viewsMetadataRs.get(0); String rawViewName = example.getName(); String rawSchemaName = schema.getName(); String rawCatalogName = schema.getCatalogName(); + View view = (new View()).setName(this.cleanNameFromDatabase(rawViewName, database)); CatalogAndSchema schemaFromJdbcInfo = ((AbstractJdbcDatabase) database).getSchemaFromJdbcInfo(rawCatalogName, rawSchemaName); view.setSchema(new Schema(schemaFromJdbcInfo.getCatalogName(), schemaFromJdbcInfo.getSchemaName())); - String definition = (String) row.get("VIEW_DEFINITION"); - if (definition.startsWith("FULL_DEFINITION: ")) { - definition = definition.replaceFirst("^FULL_DEFINITION: ", ""); - view.setContainsFullDefinition(true); + String definition = rawViewDefinition; + + if (definition.isEmpty()) { + definition = (String) row.get("view_definition"); + } int length = definition.length(); @@ -81,5 +111,4 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot } } - -} \ No newline at end of file +} diff --git a/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType b/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType index 62cd2432..14bb01cc 100644 --- a/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType +++ b/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType @@ -6,4 +6,5 @@ liquibase.ext.databricks.datatype.BooleanDatatypeDatabricks liquibase.ext.databricks.datatype.FloatDatatypeDatabricks liquibase.ext.databricks.datatype.DoubleDatatypeDatabricks liquibase.ext.databricks.datatype.TinyintDatatypeDatabricks -liquibase.ext.databricks.datatype.SmallintDatatypeDatabricks \ No newline at end of file +liquibase.ext.databricks.datatype.SmallintDatatypeDatabricks +liquibase.ext.databricks.datatype.BinaryDataTypeDatabricks \ No newline at end of file