diff --git a/engine/pom.xml b/engine/pom.xml
index d35b8f3..f21a050 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -570,13 +570,13 @@
true
- /spark
+ /databricks
- com.simba
- spark
- 2.6.22.1040
+ com.databricks
+ client
+ 2.6.40.1071
@@ -593,11 +593,11 @@
install-file
- com.simba
- spark
- 2.6.22.1040
+ com.databricks
+ client
+ 2.6.40.1071
jar
- ${spark.classpath}/spark-2.6.22.1040.jar
+ ${spark.classpath}/DatabricksJDBC42.jar
diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java
index c9972bb..fdfdce3 100644
--- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java
+++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java
@@ -61,6 +61,14 @@ public static Connection getConnectionWithAutoCommit(DataSourceUnsecuredDTO data
if (dataSource.getType().equals(DBMSType.SNOWFLAKE)) {
info.put("CLIENT_RESULT_COLUMN_CASE_INSENSITIVE", "true");
}
+
+ // Set the Databricks JDBC driver to mimic Spark's behavior
+ if (dataSource.getType().equals(DBMSType.SPARK)) {
+ info.put("driver", "com.databricks.client.jdbc.Driver");
+ // Replace "spark" with "databricks" in the URL
+ url = url.replace("spark", "databricks");
+ }
+
Connection conn = DriverManager.getConnection(url, info);
return conn;