diff --git a/engine/pom.xml b/engine/pom.xml index d35b8f3..f21a050 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -570,13 +570,13 @@ true - /spark + /databricks - com.simba - spark - 2.6.22.1040 + com.databricks + client + 2.6.40.1071 @@ -593,11 +593,11 @@ install-file - com.simba - spark - 2.6.22.1040 + com.databricks + client + 2.6.40.1071 jar - ${spark.classpath}/spark-2.6.22.1040.jar + ${spark.classpath}/DatabricksJDBC42.jar diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java index c9972bb..fdfdce3 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java @@ -61,6 +61,14 @@ public static Connection getConnectionWithAutoCommit(DataSourceUnsecuredDTO data if (dataSource.getType().equals(DBMSType.SNOWFLAKE)) { info.put("CLIENT_RESULT_COLUMN_CASE_INSENSITIVE", "true"); } + + // Set the Databricks JDBC driver to mimic Spark's behavior + if (dataSource.getType().equals(DBMSType.SPARK)) { + info.put("driver", "com.databricks.client.jdbc.Driver"); + // Replace "spark" with "databricks" in the URL + url = url.replace("spark", "databricks"); + } + Connection conn = DriverManager.getConnection(url, info); return conn;