From f975ff60bb9239ae4109c85ab68a45aa00f58bce Mon Sep 17 00:00:00 2001 From: dmitrys Date: Fri, 1 Nov 2024 13:51:03 +0100 Subject: [PATCH 1/5] Remove HikariCP to avoid spinning up on-demand DBMS targets due to automatic connection rotation in background --- .../arachne/executionengine/execution/r/RService.java | 6 ++---- .../arachne/executionengine/execution/sql/SQLService.java | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RService.java index 36d9ef0..1b25085 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RService.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/r/RService.java @@ -8,7 +8,7 @@ import com.odysseusinc.arachne.executionengine.config.properties.HiveBulkLoadProperties; import com.odysseusinc.arachne.executionengine.execution.DriverLocations; import com.odysseusinc.arachne.executionengine.execution.Overseer; -import com.odysseusinc.arachne.executionengine.service.ConnectionPoolService; +import com.odysseusinc.arachne.executionengine.util.SQLUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -57,8 +57,6 @@ public abstract class RService { @Autowired private HiveBulkLoadProperties hiveBulkLoadProperties; @Autowired - private ConnectionPoolService poolService; - @Autowired private DriverLocations drivers; private static String sanitizeFilename(String filename) { @@ -70,7 +68,7 @@ public Overseer analyze(AnalysisSyncRequestDTO analysis, File analysisDir, BiCon DataSourceUnsecuredDTO dataSource = analysis.getDataSource(); log.info("Execution [{}] checking connection to [{}]", id, dataSource.getConnectionString()); - try (Connection conn = poolService.getDataSource(dataSource).getConnection()) { + try (Connection conn = SQLUtils.getConnectionWithAutoCommit(dataSource)) { String name = conn.getMetaData().getDatabaseProductName(); log.info("Execution [{}] connection verified, engine: [{}]", id, name); } catch (SQLException | UncheckedExecutionException e) { diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/sql/SQLService.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/sql/SQLService.java index bc89b1c..f440614 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/sql/SQLService.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/execution/sql/SQLService.java @@ -32,8 +32,8 @@ import com.odysseusinc.arachne.executionengine.execution.AbstractOverseer; import com.odysseusinc.arachne.executionengine.execution.ExecutionService; import com.odysseusinc.arachne.executionengine.execution.Overseer; -import com.odysseusinc.arachne.executionengine.service.ConnectionPoolService; import com.odysseusinc.arachne.executionengine.util.AnalisysUtils; +import com.odysseusinc.arachne.executionengine.util.SQLUtils; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -79,8 +79,6 @@ public class SQLService implements ExecutionService { @Autowired @Qualifier("analysisTaskExecutor") private ThreadPoolTaskExecutor taskExecutor; - @Autowired - private ConnectionPoolService poolService; @Value("${csv.separator}") private char csvSeparator; @@ -104,7 +102,7 @@ public Overseer analyze(AnalysisSyncRequestDTO analysis, File dir, BiConsumer callback, DataSourceUnsecuredDTO dataSource, Long id, StringBuffer stdout) { - try (Connection conn = poolService.getDataSource(dataSource).getConnection()) { + try (Connection conn = SQLUtils.getConnectionWithAutoCommit(dataSource)) { String name = conn.getMetaData().getDatabaseProductName(); boolean singleStatement = SINGLE_STATEMENT_TYPES.contains(dataSource.getType()); log.info("Execution [{}] connected to [{}], single statement: {}", id, name, singleStatement); From 8273780ed94a86c02c897fe074aafc29b9b45dc2 Mon Sep 17 00:00:00 2001 From: konstantin Date: Mon, 9 Dec 2024 18:23:33 +0100 Subject: [PATCH 2/5] Raising Spark/Databricks driver --- engine/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/pom.xml b/engine/pom.xml index f3468a2..c35baeb 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -590,9 +590,9 @@ - com.simba - spark - 2.6.22.1040 + com.databricks + client + 2.6.40.1071 From 7e5693f04bff32b5cbbb6f24ee624da92eddeccf Mon Sep 17 00:00:00 2001 From: konstantin Date: Mon, 9 Dec 2024 18:29:43 +0100 Subject: [PATCH 3/5] Raising Spark/Databricks driver --- engine/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/engine/pom.xml b/engine/pom.xml index 9268134..f21a050 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -570,7 +570,7 @@ true - /spark + /databricks @@ -593,11 +593,11 @@ install-file - com.simba - spark - 2.6.22.1040 + com.databricks + client + 2.6.40.1071 jar - ${spark.classpath}/spark-2.6.22.1040.jar + ${spark.classpath}/DatabricksJDBC42.jar From 13f2db63080cbdb7f3fa6d9c0688761f2285e9b9 Mon Sep 17 00:00:00 2001 From: konstantin Date: Mon, 9 Dec 2024 18:36:39 +0100 Subject: [PATCH 4/5] Mimic for Spark driver --- .../odysseusinc/arachne/executionengine/util/SQLUtils.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java index c9972bb..d76a71b 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java @@ -61,6 +61,12 @@ public static Connection getConnectionWithAutoCommit(DataSourceUnsecuredDTO data if (dataSource.getType().equals(DBMSType.SNOWFLAKE)) { info.put("CLIENT_RESULT_COLUMN_CASE_INSENSITIVE", "true"); } + + // Set the Databricks JDBC driver to mimic Spark's behavior + if (dataSource.getType().equals(DBMSType.SPARK)) { + info.put("driver", "com.databricks.client.jdbc.Driver"); + } + Connection conn = DriverManager.getConnection(url, info); return conn; From b37407eea9e36b46c9d9a65f50e51d676f30009b Mon Sep 17 00:00:00 2001 From: konstantin Date: Mon, 9 Dec 2024 18:48:15 +0100 Subject: [PATCH 5/5] Update connection string for Spark --- .../com/odysseusinc/arachne/executionengine/util/SQLUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java index d76a71b..fdfdce3 100644 --- a/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java +++ b/engine/src/main/java/com/odysseusinc/arachne/executionengine/util/SQLUtils.java @@ -65,6 +65,8 @@ public static Connection getConnectionWithAutoCommit(DataSourceUnsecuredDTO data // Set the Databricks JDBC driver to mimic Spark's behavior if (dataSource.getType().equals(DBMSType.SPARK)) { info.put("driver", "com.databricks.client.jdbc.Driver"); + // Replace "spark" with "databricks" in the URL + url = url.replace("spark", "databricks"); } Connection conn = DriverManager.getConnection(url, info);