From 71cf625b6069e1b32c558a0aca6803f793d208ea Mon Sep 17 00:00:00 2001 From: Stephane Geneix Date: Tue, 28 Nov 2023 11:23:30 -0800 Subject: [PATCH] introduce BaseImage to MySql source --- .../sources/CDCMySqlDatatypeAccuracyTest.java | 3 ++- .../CdcBinlogsMySqlSourceDatatypeTest.java | 3 ++- ...cInitialSnapshotMySqlSourceDatatypeTest.java | 3 ++- .../sources/CdcMySqlSourceAcceptanceTest.java | 3 ++- .../sources/MySqlSourceAcceptanceTest.java | 3 ++- .../SshPasswordMySqlSourceAcceptanceTest.java | 3 ++- .../source/mysql/CdcMysqlSourceTest.java | 3 ++- .../mysql/MySqlJdbcSourceAcceptanceTest.java | 3 ++- .../source/mysql/MySqlSourceOperationsTest.java | 3 ++- .../source/mysql/MySqlSourceTests.java | 7 ++++--- .../source/mysql/MySQLTestDatabase.java | 17 +++++++++++++++-- 11 files changed, 37 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java index 357ccc336ace3..fd7ea961688ab 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java @@ -7,6 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.Database; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; public class CDCMySqlDatatypeAccuracyTest extends MySqlDatatypeAccuracyTest { @@ -21,7 +22,7 @@ protected JsonNode getConfig() { @Override protected Database setupDatabase() { - testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions(); + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); return testdb.getDatabase(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java index 54f2ea9c1ca1a..04fdacb09b504 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java @@ -12,6 +12,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -38,7 +39,7 @@ protected JsonNode getConfig() { @Override protected Database setupDatabase() { - testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions(); + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); return testdb.getDatabase(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java index 230f34ca13fea..6a3550b561cc7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java @@ -9,6 +9,7 @@ import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; public class CdcInitialSnapshotMySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { @@ -28,7 +29,7 @@ protected JsonNode getConfig() { @Override protected Database setupDatabase() { - testdb = MySQLTestDatabase.in("mysql:8.0").withoutStrictMode().withCdcPermissions(); + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); return testdb.getDatabase(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java index 1db8613696c52..1da27179cb40e 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java @@ -19,6 +19,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -101,7 +102,7 @@ protected JsonNode getState() { @Override protected void setupEnvironment(final TestDestinationEnv environment) { - testdb = MySQLTestDatabase.in("mysql:8.0", extraContainerFactoryMethods().toArray(String[]::new)) + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, extraContainerFactoryMethods().toArray(String[]::new)) .withCdcPermissions() .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java index 3ec7d4ab67404..52945b7a444e7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java @@ -13,6 +13,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -33,7 +34,7 @@ public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest { @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - testdb = MySQLTestDatabase.in("mysql:8.0", extraContainerFactoryMethods().toArray(String[]::new)) + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, extraContainerFactoryMethods().toArray(String[]::new)) .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java index 1211c82698940..2e453290f8293 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java @@ -12,6 +12,7 @@ import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.integrations.source.mysql.MySqlSource; import java.nio.file.Path; import org.junit.jupiter.api.Test; @@ -25,7 +26,7 @@ public Path getConfigFilePath() { @Test public void sshTimeoutExceptionMarkAsConfigErrorTest() throws Exception { - try (final var testdb = MySQLTestDatabase.in("mysql:8.0", "withNetwork")) { + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, "withNetwork")) { final SshBastionContainer bastion = new SshBastionContainer(); bastion.initAndStartBastion(testdb.getContainer().getNetwork()); final var config = testdb.integrationTestConfigBuilder() diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 6bd939fd9e3cc..0bbf9ec9c54e2 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -39,6 +39,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -77,7 +78,7 @@ public class CdcMysqlSourceTest extends CdcSourceTest void testImpl( throws SQLException { final var sqlSourceOperations = new MySqlSourceOperations(); final String cursorColumn = "cursor_column"; - try (final var testdb = MySQLTestDatabase.in("mysql:8.0") + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8) .with("CREATE TABLE cursor_table (id INTEGER PRIMARY KEY, %s %s);", cursorColumn, sqlType)) { final List expectedRecords = new ArrayList<>(); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 747a66a8dd63e..ccc96a94950c1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -21,6 +21,7 @@ import io.airbyte.commons.features.FeatureFlagsWrapper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -46,7 +47,7 @@ public MySqlSource source() { @Test public void testSettingTimezones() throws Exception { - try (final var testdb = MySQLTestDatabase.in("mysql:8.0", "withMoscowTimezone")) { + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, "withMoscowTimezone")) { final var config = testdb.testConfigBuilder() .with(JdbcUtils.JDBC_URL_PARAMS_KEY, "serverTimezone=Europe/Moscow") .withoutSsl() @@ -76,7 +77,7 @@ private JsonNode buildConfigEscapingNeeded() { @Test @Disabled("See https://github.com/airbytehq/airbyte/pull/23908#issuecomment-1463753684, enable once communication is out") public void testNullCursorValueShouldThrowException() { - try (final var testdb = MySQLTestDatabase.in("mysql:8.0") + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8) .with("CREATE TABLE null_cursor_table(id INTEGER NULL);") .with("INSERT INTO null_cursor_table(id) VALUES (1), (2), (NULL);") .with("CREATE VIEW null_cursor_view(id) AS SELECT null_cursor_table.id FROM null_cursor_table;")) { @@ -126,7 +127,7 @@ void testParseJdbcParameters() { @Test public void testJDBCSessionVariable() throws Exception { - try (final var testdb = MySQLTestDatabase.in("mysql:8.0")) { + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8)) { final var config = testdb.testConfigBuilder() .with(JdbcUtils.JDBC_URL_PARAMS_KEY, "sessionVariables=MAX_EXECUTION_TIME=28800000") .withoutSsl() diff --git a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java index 5f35def2b83f3..524d492fee57b 100644 --- a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java +++ b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java @@ -17,8 +17,21 @@ public class MySQLTestDatabase extends TestDatabase, MySQLTestDatabase, MySQLTestDatabase.MySQLConfigBuilder> { - static public MySQLTestDatabase in(String imageName, String... methods) { - final var container = new MySQLContainerFactory().shared(imageName, methods); + public static enum BaseImage { + + MYSQL_8("mysql:8.0"), + ; + + private final String reference; + + private BaseImage(String reference) { + this.reference = reference; + } + + } + + static public MySQLTestDatabase in(BaseImage baseImage, String... methods) { + final var container = new MySQLContainerFactory().shared(baseImage.reference, methods); return new MySQLTestDatabase(container).initialized(); }