From 9f74e9f576a38c68fe761127d44511e9f6172a72 Mon Sep 17 00:00:00 2001 From: Mathieu Gabelle <54168385+mgabelle@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:45:56 +0100 Subject: [PATCH] test(queries): add additional tests (#440) tests for Queries : - Redshift - Snowflake - Sybase - Vertica --- .../jdbc/redshift/RedshiftQueriesTest.java | 103 +++++++++++++++ .../resources/scripts/redshift_queries.sql | 28 ++++ .../plugin/jdbc/snowflake/Download.java | 2 +- .../kestra/plugin/jdbc/snowflake/Upload.java | 2 +- .../jdbc/snowflake/SnowflakeQueriesTest.java | 122 ++++++++++++++++++ .../plugin/jdbc/snowflake/SnowflakeTest.java | 7 +- .../jdbc/snowflake/SnowflakeTriggerTest.java | 2 +- .../jdbc/snowflake/UploadDownloadTest.java | 2 +- .../resources/scripts/snowflake_queries.sql | 27 ++++ .../io/kestra/plugin/jdbc/sybase/Query.java | 5 +- .../plugin/jdbc/sybase/SybaseQueriesTest.java | 91 +++++++++++++ .../kestra/plugin/jdbc/sybase/SybaseTest.java | 4 +- .../plugin/jdbc/sybase/SybaseTriggerTest.java | 2 +- .../test/resources/scripts/sybase_queries.sql | 28 ++++ .../io/kestra/plugin/jdbc/vertica/Query.java | 12 +- .../kestra/plugin/jdbc/vertica/BatchTest.java | 2 +- .../jdbc/vertica/VerticaQueriesTest.java | 95 ++++++++++++++ .../plugin/jdbc/vertica/VerticaTest.java | 4 +- .../jdbc/vertica/VerticaTriggerTest.java | 3 +- .../resources/scripts/vertica_queries.sql | 29 +++++ .../plugin/jdbc/AbstractJdbcQueries.java | 12 +- 21 files changed, 557 insertions(+), 25 deletions(-) create mode 100644 plugin-jdbc-redshift/src/test/java/io/kestra/plugin/jdbc/redshift/RedshiftQueriesTest.java create mode 100644 plugin-jdbc-redshift/src/test/resources/scripts/redshift_queries.sql create mode 100644 plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeQueriesTest.java create mode 100644 plugin-jdbc-snowflake/src/test/resources/scripts/snowflake_queries.sql create mode 100644 plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseQueriesTest.java create mode 100644 plugin-jdbc-sybase/src/test/resources/scripts/sybase_queries.sql create mode 100644 plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaQueriesTest.java create mode 100644 plugin-jdbc-vertica/src/test/resources/scripts/vertica_queries.sql diff --git a/plugin-jdbc-redshift/src/test/java/io/kestra/plugin/jdbc/redshift/RedshiftQueriesTest.java b/plugin-jdbc-redshift/src/test/java/io/kestra/plugin/jdbc/redshift/RedshiftQueriesTest.java new file mode 100644 index 00000000..826d0cfe --- /dev/null +++ b/plugin-jdbc-redshift/src/test/java/io/kestra/plugin/jdbc/redshift/RedshiftQueriesTest.java @@ -0,0 +1,103 @@ +package io.kestra.plugin.jdbc.redshift; + +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQueries; +import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import io.micronaut.context.annotation.Value; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.math.BigDecimal; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static io.kestra.core.models.tasks.common.FetchType.FETCH; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; + +@KestraTest +@Disabled("no server for unit test") +public class RedshiftQueriesTest extends AbstractRdbmsTest { + @Value("${redshift.url}") + protected String url; + + + @Value("${redshift.user}") + protected String user; + + @Value("${redshift.password}") + protected String password; + + @Test + void testMultiSelectWithParameters() throws Exception { + RunContext runContext = runContextFactory.of(Collections.emptyMap()); + + Map parameters = Map.of( + "age", 40, + "brand", "Apple", + "cpu_frequency", 1.5 + ); + + Queries taskGet = Queries.builder() + .url(getUrl()) + .username(getUsername()) + .password(getPassword()) + .fetchType(FETCH) + .timeZoneId("Europe/Paris") + .sql(""" + SELECT firstName, lastName, age FROM employee where age > :age and age < :age + 10; + SELECT brand, model FROM laptop where brand = :brand and cpu_frequency > :cpu_frequency; + """) + .parameters(Property.of(parameters)) + .build(); + + AbstractJdbcQueries.MultiQueryOutput runOutput = taskGet.run(runContext); + assertThat(runOutput.getOutputs().size(), is(2)); + + List> employees = runOutput.getOutputs().getFirst().getRows(); + assertThat("employees", employees, notNullValue()); + assertThat("employees", employees.size(), is(1)); + assertThat("employee selected", employees.getFirst().get("age"), is(BigDecimal.valueOf(45))); + assertThat("employee selected", employees.getFirst().get("firstname"), is("John")); + assertThat("employee selected", employees.getFirst().get("lastname"), is("Doe")); + + List>laptops = runOutput.getOutputs().getLast().getRows(); + assertThat("laptops", laptops, notNullValue()); + assertThat("laptops", laptops.size(), is(1)); + assertThat("selected laptop", laptops.getFirst().get("brand"), is("Apple")); + } + + @Override + protected String getUrl() { + return this.url; + } + + @Override + protected String getUsername() { + return this.user; + } + + @Override + protected String getPassword() { + return this.password; + } + + @Override + @BeforeEach + public void init() throws IOException, URISyntaxException, SQLException { + initDatabase(); + } + + @Override + protected void initDatabase() throws SQLException, FileNotFoundException, URISyntaxException { + executeSqlScript("scripts/redshift_queries.sql"); + } +} diff --git a/plugin-jdbc-redshift/src/test/resources/scripts/redshift_queries.sql b/plugin-jdbc-redshift/src/test/resources/scripts/redshift_queries.sql new file mode 100644 index 00000000..8bc0ded8 --- /dev/null +++ b/plugin-jdbc-redshift/src/test/resources/scripts/redshift_queries.sql @@ -0,0 +1,28 @@ +-- Create table employee +DROP TABLE IF EXISTS employee; +CREATE TABLE employee ( + firstName VARCHAR(100), + lastName VARCHAR(100), + age INTEGER +); + +INSERT INTO employee(firstName, lastName, age) VALUES ('John', 'Doe', 45); +INSERT INTO employee(firstName, lastName, age) VALUES ('Bryan', 'Grant', 33); +INSERT INTO employee(firstName, lastName, age) VALUES ('Jude', 'Philips', 25); +INSERT INTO employee(firstName, lastName, age) VALUES ('Michael', 'Page', 62); + + +-- Create table laptop +DROP TABLE IF EXISTS laptop; +CREATE TABLE laptop +( + brand VARCHAR(100), + model VARCHAR(100), + cpu_frequency DECIMAL +); + +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M1 13', 2.2); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M3 16', 1.5); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('LG', 'Gram', 1.95); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Lenovo', 'ThinkPad', 1.05); + diff --git a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Download.java b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Download.java index 264c07d6..2b0cfe18 100644 --- a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Download.java +++ b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Download.java @@ -6,6 +6,7 @@ import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; import lombok.*; import lombok.experimental.SuperBuilder; import net.snowflake.client.jdbc.SnowflakeConnection; @@ -18,7 +19,6 @@ import java.io.InputStream; import java.net.URI; import java.sql.Connection; -import jakarta.validation.constraints.NotNull; @SuperBuilder @ToString diff --git a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Upload.java b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Upload.java index 03104f50..10c4977b 100644 --- a/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Upload.java +++ b/plugin-jdbc-snowflake/src/main/java/io/kestra/plugin/jdbc/snowflake/Upload.java @@ -6,6 +6,7 @@ import io.kestra.core.models.tasks.RunnableTask; import io.kestra.core.runners.RunContext; import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotNull; import lombok.*; import lombok.experimental.SuperBuilder; import net.snowflake.client.jdbc.SnowflakeConnection; @@ -15,7 +16,6 @@ import java.io.InputStream; import java.net.URI; import java.sql.Connection; -import jakarta.validation.constraints.NotNull; @SuperBuilder @ToString diff --git a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeQueriesTest.java b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeQueriesTest.java new file mode 100644 index 00000000..e46ad03f --- /dev/null +++ b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeQueriesTest.java @@ -0,0 +1,122 @@ +package io.kestra.plugin.jdbc.snowflake; + +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQueries; +import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.math.BigDecimal; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static io.kestra.core.models.tasks.common.FetchType.FETCH; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * After creating a Snowflake account, run this SQL query to obtain the host: + * use role acountadmin; + * select system$allowlist(); + * Then find "type":"SNOWFLAKE_DEPLOYMENT" and the associated "host" will be + * like .snowflakecomputing.com under + */ +@KestraTest +@Disabled("Create a Snowflake account for unit testing") +public class SnowflakeQueriesTest extends AbstractRdbmsTest { + protected String host = ""; + protected String username = ""; + protected String password = ""; + protected String database = "KESTRA"; + + @Test + void testMultiSelectWithParameters() throws Exception { + RunContext runContext = runContextFactory.of(Collections.emptyMap()); + + Map parameters = Map.of( + "age", 40, + "brand", "Apple", + "cpu_frequency", 1.5 + ); + + Queries taskGet = Queries.builder() + .url(getUrl()) + .username(getUsername()) + .password(getPassword()) + .warehouse("COMPUTE_WH") + .database(database) + .fetchType(FETCH) + .timeZoneId("Europe/Paris") + .sql(""" + SELECT firstName, lastName, age FROM employee where age > :age and age < :age + 10; + SELECT brand, model FROM laptop where brand = :brand and cpu_frequency > :cpu_frequency; + """) + .parameters(Property.of(parameters)) + .build(); + + AbstractJdbcQueries.MultiQueryOutput runOutput = taskGet.run(runContext); + assertThat(runOutput.getOutputs().size(), is(2)); + + List> employees = runOutput.getOutputs().getFirst().getRows(); + assertThat("employees", employees, notNullValue()); + assertThat("employees", employees.size(), is(1)); + assertThat("employee selected", employees.getFirst().get("AGE"), is(BigDecimal.valueOf(45))); + assertThat("employee selected", employees.getFirst().get("FIRSTNAME"), is("John")); + assertThat("employee selected", employees.getFirst().get("LASTNAME"), is("Doe")); + + List>laptops = runOutput.getOutputs().getLast().getRows(); + assertThat("laptops", laptops, notNullValue()); + assertThat("laptops", laptops.size(), is(1)); + assertThat("selected laptop", laptops.getFirst().get("BRAND"), is("Apple")); + } + + @Override + protected String getUrl() { + return "jdbc:snowflake://" + this.host + "/?loginTimeout=3"; + } + + @Override + protected String getUsername() { + return this.username; + } + + @Override + protected String getPassword() { + return this.password; + } + + @Override + protected Connection getConnection() throws SQLException { + Properties properties = new Properties(); + properties.put("user", getUsername()); + properties.put("password", getPassword()); + properties.put("warehouse", "COMPUTE_WH"); + properties.put("db", "UNITTEST"); + properties.put("schema", "public"); + + return DriverManager.getConnection(getUrl(), properties); + } + + @Override + @BeforeEach + public void init() throws IOException, URISyntaxException, SQLException { + initDatabase(); + } + + @Override + protected void initDatabase() throws SQLException, FileNotFoundException, URISyntaxException { + executeSqlScript("scripts/snowflake_queries.sql"); + } +} diff --git a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTest.java b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTest.java index 903cbd29..c3a389e9 100644 --- a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTest.java +++ b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTest.java @@ -1,11 +1,11 @@ package io.kestra.plugin.jdbc.snowflake; import com.google.common.collect.ImmutableMap; +import io.kestra.core.junit.annotations.KestraTest; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AbstractRdbmsTest; import io.micronaut.context.annotation.Value; -import io.kestra.core.junit.annotations.KestraTest; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -15,7 +15,10 @@ import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.time.*; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTriggerTest.java b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTriggerTest.java index ede29eb9..4160fdcd 100644 --- a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTriggerTest.java +++ b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/SnowflakeTriggerTest.java @@ -1,8 +1,8 @@ package io.kestra.plugin.jdbc.snowflake; +import io.kestra.core.junit.annotations.KestraTest; import io.kestra.plugin.jdbc.AbstractJdbcTriggerTest; import io.micronaut.context.annotation.Value; -import io.kestra.core.junit.annotations.KestraTest; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java index 508b6628..1cd1aaf4 100644 --- a/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java +++ b/plugin-jdbc-snowflake/src/test/java/io/kestra/plugin/jdbc/snowflake/UploadDownloadTest.java @@ -2,12 +2,12 @@ import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; +import io.kestra.core.junit.annotations.KestraTest; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; import io.kestra.core.storages.StorageInterface; import io.kestra.core.utils.IdUtils; import io.micronaut.context.annotation.Value; -import io.kestra.core.junit.annotations.KestraTest; import jakarta.inject.Inject; import org.apache.commons.io.IOUtils; import org.junit.jupiter.api.Disabled; diff --git a/plugin-jdbc-snowflake/src/test/resources/scripts/snowflake_queries.sql b/plugin-jdbc-snowflake/src/test/resources/scripts/snowflake_queries.sql new file mode 100644 index 00000000..852eceb5 --- /dev/null +++ b/plugin-jdbc-snowflake/src/test/resources/scripts/snowflake_queries.sql @@ -0,0 +1,27 @@ +-- Create table employee +USE KESTRA; +CREATE OR REPLACE TABLE employee ( + firstName VARCHAR(100), + lastName VARCHAR(100), + age INTEGER +); + +INSERT INTO employee(firstName, lastName, age) VALUES ('John', 'Doe', 45); +INSERT INTO employee(firstName, lastName, age) VALUES ('Bryan', 'Grant', 33); +INSERT INTO employee(firstName, lastName, age) VALUES ('Jude', 'Philips', 25); +INSERT INTO employee(firstName, lastName, age) VALUES ('Michael', 'Page', 62); + + +-- Create table laptop +CREATE OR REPLACE TABLE laptop +( + brand VARCHAR(100), + model VARCHAR(100), + cpu_frequency DOUBLE PRECISION +); + +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M1 13', 2.2); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M3 16', 1.5); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('LG', 'Gram', 1.95); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Lenovo', 'ThinkPad', 1.05); + diff --git a/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java b/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java index 480e83bc..1eed6c4e 100644 --- a/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java +++ b/plugin-jdbc-sybase/src/main/java/io/kestra/plugin/jdbc/sybase/Query.java @@ -8,7 +8,10 @@ import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; -import lombok.*; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; import lombok.experimental.SuperBuilder; import java.sql.DriverManager; diff --git a/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseQueriesTest.java b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseQueriesTest.java new file mode 100644 index 00000000..9f641d94 --- /dev/null +++ b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseQueriesTest.java @@ -0,0 +1,91 @@ +package io.kestra.plugin.jdbc.sybase; + +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQueries; +import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static io.kestra.core.models.tasks.common.FetchType.FETCH; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +@KestraTest +@Disabled("Lauching Sybase via our docker-compose crash the CI") +public class SybaseQueriesTest extends AbstractRdbmsTest { + @Test + void testMultiSelectWithParameters() throws Exception { + RunContext runContext = runContextFactory.of(Collections.emptyMap()); + + Map parameters = Map.of( + "age", 40, + "brand", "Apple", + "cpu_frequency", 1.5 + ); + + Queries taskGet = Queries.builder() + .url(getUrl()) + .username(getUsername()) + .password(getPassword()) + .fetchType(FETCH) + .timeZoneId("Europe/Paris") + .sql(""" + SELECT firstName, lastName, age FROM employee where age > :age and age < :age + 10; + SELECT brand, model FROM laptop where brand = :brand and cpu_frequency > :cpu_frequency; + """) + .parameters(Property.of(parameters)) + .build(); + + AbstractJdbcQueries.MultiQueryOutput runOutput = taskGet.run(runContext); + assertThat(runOutput.getOutputs().size(), is(2)); + + List> employees = runOutput.getOutputs().getFirst().getRows(); + assertThat("employees", employees, notNullValue()); + assertThat("employees", employees.size(), is(1)); + assertThat("employee selected", employees.getFirst().get("age"), is(45)); + assertThat("employee selected", employees.getFirst().get("firstName"), is("John")); + assertThat("employee selected", employees.getFirst().get("lastName"), is("Doe")); + + List>laptops = runOutput.getOutputs().getLast().getRows(); + assertThat("laptops", laptops, notNullValue()); + assertThat("laptops", laptops.size(), is(1)); + assertThat("selected laptop", laptops.getFirst().get("brand"), is("Apple")); + } + + @Override + protected String getUrl() { + return "jdbc:sybase:Tds:127.0.0.1:5000/kestra"; + } + + @Override + protected String getUsername() { + return "sa"; + } + + @Override + protected String getPassword() { + return "myPassword"; + } + + @Override + @BeforeEach + public void init() throws SQLException, FileNotFoundException, URISyntaxException { + initDatabase(); + } + @Override + protected void initDatabase() throws SQLException, FileNotFoundException, URISyntaxException { + executeSqlScript("scripts/sybase_queries.sql"); + } + +} diff --git a/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTest.java b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTest.java index d7ec37e2..b38358da 100644 --- a/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTest.java +++ b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTest.java @@ -2,11 +2,11 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.junit.annotations.KestraTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import java.io.FileNotFoundException; import java.math.BigDecimal; diff --git a/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTriggerTest.java b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTriggerTest.java index 1a892fc0..f87387fa 100644 --- a/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTriggerTest.java +++ b/plugin-jdbc-sybase/src/test/java/io/kestra/plugin/jdbc/sybase/SybaseTriggerTest.java @@ -1,7 +1,7 @@ package io.kestra.plugin.jdbc.sybase; -import io.kestra.plugin.jdbc.AbstractJdbcTriggerTest; import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.plugin.jdbc.AbstractJdbcTriggerTest; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/plugin-jdbc-sybase/src/test/resources/scripts/sybase_queries.sql b/plugin-jdbc-sybase/src/test/resources/scripts/sybase_queries.sql new file mode 100644 index 00000000..9e01c017 --- /dev/null +++ b/plugin-jdbc-sybase/src/test/resources/scripts/sybase_queries.sql @@ -0,0 +1,28 @@ +IF EXISTS(select 1 from sysobjects where name='employee' and type='U') DROP TABLE employee; + +CREATE TABLE employee ( + employee_id INT IDENTITY PRIMARY KEY, + firstName VARCHAR(30), + lastName VARCHAR(30), + age INT +); + +INSERT INTO employee (firstName, lastName, age) VALUES ('John', 'Doe', 45); +INSERT INTO employee (firstName, lastName, age) VALUES ('Bryan', 'Grant', 33); +INSERT INTO employee (firstName, lastName, age) VALUES ('Jude', 'Philips', 25); +INSERT INTO employee (firstName, lastName, age) VALUES ('Michael', 'Page', 62); + +IF EXISTS(select 1 from sysobjects where name='laptop' and type='U') DROP TABLE laptop; + +CREATE TABLE laptop +( + laptop_id INT IDENTITY PRIMARY KEY, + brand VARCHAR(30), + model VARCHAR(30), + cpu_frequency REAL +); + +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M1 13', 2.2); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M3 16', 1.5); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('LG', 'Gram', 1.95); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Lenovo', 'ThinkPad', 1.05); \ No newline at end of file diff --git a/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java b/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java index ca05d49f..182c30ac 100644 --- a/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java +++ b/plugin-jdbc-vertica/src/main/java/io/kestra/plugin/jdbc/vertica/Query.java @@ -1,5 +1,11 @@ package io.kestra.plugin.jdbc.vertica; +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.tasks.RunnableTask; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractCellConverter; +import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AutoCommitInterface; import io.swagger.v3.oas.annotations.media.Schema; import lombok.EqualsAndHashCode; @@ -7,12 +13,6 @@ import lombok.NoArgsConstructor; import lombok.ToString; import lombok.experimental.SuperBuilder; -import io.kestra.core.models.annotations.Example; -import io.kestra.core.models.annotations.Plugin; -import io.kestra.core.models.tasks.RunnableTask; -import io.kestra.core.runners.RunContext; -import io.kestra.plugin.jdbc.AbstractCellConverter; -import io.kestra.plugin.jdbc.AbstractJdbcQuery; import java.sql.DriverManager; import java.sql.SQLException; diff --git a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java index baad751b..86000e67 100644 --- a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java +++ b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/BatchTest.java @@ -1,12 +1,12 @@ package io.kestra.plugin.jdbc.vertica; import com.google.common.collect.ImmutableMap; +import io.kestra.core.junit.annotations.KestraTest; import io.kestra.core.runners.RunContext; import io.kestra.core.serializers.FileSerde; import io.kestra.core.utils.IdUtils; import io.kestra.plugin.jdbc.AbstractJdbcBatch; import io.kestra.plugin.jdbc.AbstractRdbmsTest; -import io.kestra.core.junit.annotations.KestraTest; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaQueriesTest.java b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaQueriesTest.java new file mode 100644 index 00000000..e7e7f366 --- /dev/null +++ b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaQueriesTest.java @@ -0,0 +1,95 @@ +package io.kestra.plugin.jdbc.vertica; + +import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.core.models.property.Property; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.jdbc.AbstractJdbcQueries; +import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.FileNotFoundException; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static io.kestra.core.models.tasks.common.FetchType.FETCH; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; + +/** + * See : https://dev.mysql.com/doc/connector-j/8.0/en/connector-j-reference-type-conversions.html + */ +@KestraTest +@Disabled("Issue with EOFException") +public class VerticaQueriesTest extends AbstractRdbmsTest { + + @Test + void testMultiSelectWithParameters() throws Exception { + RunContext runContext = runContextFactory.of(Collections.emptyMap()); + + Map parameters = Map.of( + "age", 40, + "brand", "Apple", + "cpu_frequency", 1.5 + ); + + Queries taskGet = Queries.builder() + .url(getUrl()) + .username(getUsername()) + .password(getPassword()) + .fetchType(FETCH) + .timeZoneId("Europe/Paris") + .sql(""" + SELECT firstName, lastName, age FROM employee where age > :age and age < :age + 10; + SELECT brand, model FROM laptop where brand = :brand and cpu_frequency > :cpu_frequency; + """) + .parameters(Property.of(parameters)) + .build(); + + AbstractJdbcQueries.MultiQueryOutput runOutput = taskGet.run(runContext); + assertThat(runOutput.getOutputs().size(), is(2)); + + List> employees = runOutput.getOutputs().getFirst().getRows(); + assertThat("employees", employees, notNullValue()); + assertThat("employees", employees.size(), is(1)); + assertThat("employee selected", employees.getFirst().get("age"), is(45L)); + assertThat("employee selected", employees.getFirst().get("firstName"), is("John")); + assertThat("employee selected", employees.getFirst().get("lastName"), is("Doe")); + + List>laptops = runOutput.getOutputs().getLast().getRows(); + assertThat("laptops", laptops, notNullValue()); + assertThat("laptops", laptops.size(), is(1)); + assertThat("selected laptop", laptops.getFirst().get("brand"), is("Apple")); + } + + @Override + protected String getUrl() { + return "jdbc:vertica://127.0.0.1:25433/docker"; + } + + @Override + protected String getUsername() { + return "dbadmin"; + } + + @Override + protected String getPassword() { + return "vertica_passwd"; + } + + @Override + @BeforeEach + public void init() throws SQLException, FileNotFoundException, URISyntaxException { + initDatabase(); + } + + @Override + protected void initDatabase() throws SQLException, FileNotFoundException, URISyntaxException { + executeSqlScript("scripts/vertica_queries.sql"); + } +} diff --git a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTest.java b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTest.java index 40ad4c79..026d7db8 100644 --- a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTest.java +++ b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTest.java @@ -2,11 +2,11 @@ import com.google.common.collect.ImmutableMap; import io.kestra.core.junit.annotations.KestraTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; import io.kestra.core.runners.RunContext; import io.kestra.plugin.jdbc.AbstractJdbcQuery; import io.kestra.plugin.jdbc.AbstractRdbmsTest; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; import java.io.FileNotFoundException; import java.math.BigDecimal; diff --git a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTriggerTest.java b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTriggerTest.java index c4d4433a..ceec079f 100644 --- a/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTriggerTest.java +++ b/plugin-jdbc-vertica/src/test/java/io/kestra/plugin/jdbc/vertica/VerticaTriggerTest.java @@ -1,8 +1,7 @@ package io.kestra.plugin.jdbc.vertica; -import io.kestra.plugin.jdbc.AbstractJdbcTriggerTest; -import io.micronaut.context.annotation.Value; import io.kestra.core.junit.annotations.KestraTest; +import io.kestra.plugin.jdbc.AbstractJdbcTriggerTest; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; diff --git a/plugin-jdbc-vertica/src/test/resources/scripts/vertica_queries.sql b/plugin-jdbc-vertica/src/test/resources/scripts/vertica_queries.sql new file mode 100644 index 00000000..a7d404a3 --- /dev/null +++ b/plugin-jdbc-vertica/src/test/resources/scripts/vertica_queries.sql @@ -0,0 +1,29 @@ +-- Create table employee +DROP TABLE IF EXISTS employee; + +CREATE TABLE employee ( + firstName VARCHAR, + lastName VARCHAR, + age INTEGER +); + +INSERT INTO employee(firstName, lastName, age) VALUES ('John', 'Doe', 45); +INSERT INTO employee(firstName, lastName, age) VALUES ('Bryan', 'Grant', 33); +INSERT INTO employee(firstName, lastName, age) VALUES ('Jude', 'Philips', 25); +INSERT INTO employee(firstName, lastName, age) VALUES ('Michael', 'Page', 62); + + +-- Create table laptop +DROP TABLE IF EXISTS laptop; + +CREATE TABLE laptop +( + brand VARCHAR, + model VARCHAR, + cpu_frequency DOUBLE PRECISION +); + +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M1 13', 2.2); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Apple', 'MacBookPro M3 16', 1.5); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('LG', 'Gram', 1.95); +INSERT INTO laptop (brand, model, cpu_frequency) VALUES ('Lenovo', 'ThinkPad', 1.05); \ No newline at end of file diff --git a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQueries.java b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQueries.java index efebdccb..6e9741b5 100644 --- a/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQueries.java +++ b/plugin-jdbc/src/main/java/io/kestra/plugin/jdbc/AbstractJdbcQueries.java @@ -77,7 +77,7 @@ public AbstractJdbcQueries.MultiQueryOutput run(RunContext runContext) throws Ex rollbackIfTransactional(isTransactional); throw new RuntimeException(e); } finally { - closeConnectionAndStatement(); + closeConnectionAndStatement(runContext); } } @@ -130,9 +130,13 @@ private void rollbackIfTransactional(boolean isTransactional) throws SQLExceptio } } - private void closeConnectionAndStatement() throws SQLException { - if(conn != null) { conn.close(); } - if(stmt != null) { stmt.close(); } + private void closeConnectionAndStatement(RunContext runContext) { + try { + if(conn != null && !conn.isClosed()) { conn.close(); } + if(stmt != null && !stmt.isClosed()) { stmt.close(); } + } catch (SQLException e) { + runContext.logger().warn("Issue when closing the connection : {}", e.getMessage()); + } } private Savepoint initializeSavepoint(Connection conn) throws SQLException {