From 8466684b6c405cfd9ebf2a9331748f3728990c69 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 21 Feb 2024 13:51:16 -0800 Subject: [PATCH 1/8] upgrade cdk --- .../connectors/destination-redshift/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 298b24ec4012..54f9475bbe42 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -6,7 +6,7 @@ plugins { airbyteJavaConnector { cdkVersionRequired = '0.23.2' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = false + useLocalCdk = true } java { From a5f30a4b961027a3b0c67237009557725301e927 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 23 Feb 2024 13:37:22 -0800 Subject: [PATCH 2/8] fix build --- .../destination-redshift/build.gradle | 1 + .../redshift/RedshiftInsertDestination.java | 5 +++-- .../RedshiftStagingS3Destination.java | 20 ++++++++++++------- .../RedshiftDestinationHandler.java | 17 +++++++++++----- .../redshift/typing_deduping/RedshiftState.kt | 13 ++++++++++++ .../RedshiftSqlGeneratorIntegrationTest.java | 10 +++++----- 6 files changed, 47 insertions(+), 19 deletions(-) create mode 100644 airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 54f9475bbe42..0848d1b96359 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -1,6 +1,7 @@ plugins { id 'application' id 'airbyte-java-connector' + id 'org.jetbrains.kotlin.jvm' version '1.9.22' } airbyteJavaConnector { diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index a4ba7a669557..f1bf66e6963a 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -22,6 +22,7 @@ import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftState; import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; import java.time.Duration; import java.util.HashMap; @@ -115,8 +116,8 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { - return new RedshiftDestinationHandler(databaseName, database); + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, String rawTableSchema) { + return new RedshiftDestinationHandler(databaseName, database, rawTableSchema); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 16189ce2004b..8d014d09a397 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -20,6 +20,7 @@ import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; @@ -50,6 +51,7 @@ import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftState; import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; @@ -58,6 +60,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.time.Duration; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Consumer; import javax.sql.DataSource; @@ -176,8 +179,8 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { - return new RedshiftDestinationHandler(databaseName, database); + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, String rawTableSchema) { + return new RedshiftDestinationHandler(databaseName, database, rawTableSchema); } @Override @@ -217,22 +220,25 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final TyperDeduper typerDeduper; final JdbcDatabase database = getDatabase(getDataSource(config)); final String databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); - final RedshiftDestinationHandler redshiftDestinationHandler = new RedshiftDestinationHandler(databaseName, database); final CatalogParser catalogParser; + final String rawNamespace; if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent()) { - catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get()); + rawNamespace = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get(); + catalogParser = new CatalogParser(sqlGenerator, rawNamespace); } else { + rawNamespace = JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; catalogParser = new CatalogParser(sqlGenerator); } + final RedshiftDestinationHandler redshiftDestinationHandler = new RedshiftDestinationHandler(databaseName, database, rawNamespace); parsedCatalog = catalogParser.parseCatalog(catalog); final JdbcV1V2Migrator migrator = new JdbcV1V2Migrator(getNamingResolver(), database, databaseName); final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator); + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); } else { typerDeduper = - new DefaultTyperDeduper(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator); + new DefaultTyperDeduper<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); } return StagingConsumerFactory.builder( outputRecordCollector, @@ -252,7 +258,7 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN /** * Retrieves user configured file buffer amount so as long it doesn't exceed the maximum number of * file buffers and sets the minimum number to the default - * + *

* NOTE: If Out Of Memory Exceptions (OOME) occur, this can be a likely cause as this hard limit has * not been thoroughly load tested across all instance sizes * diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java index 5a47c2436d00..ea4955b95905 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -4,8 +4,7 @@ package io.airbyte.integrations.destination.redshift.typing_deduping; -import static io.airbyte.cdk.integrations.base.JavaBaseConstants.*; - +import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; @@ -20,12 +19,13 @@ import java.util.List; import java.util.UUID; import lombok.extern.slf4j.Slf4j; +import org.jooq.SQLDialect; @Slf4j -public class RedshiftDestinationHandler extends JdbcDestinationHandler { +public class RedshiftDestinationHandler extends JdbcDestinationHandler { - public RedshiftDestinationHandler(final String databaseName, final JdbcDatabase jdbcDatabase) { - super(databaseName, jdbcDatabase); + public RedshiftDestinationHandler(final String databaseName, final JdbcDatabase jdbcDatabase, String rawNamespace) { + super(databaseName, jdbcDatabase, rawNamespace, SQLDialect.POSTGRES); } @Override @@ -69,6 +69,13 @@ protected String toJdbcTypeName(AirbyteType airbyteType) { }; } + + @Override + protected RedshiftState toDestinationState(JsonNode json) { + return new RedshiftState( + json.hasNonNull("needsSoftReset") && json.get("needsSoftReset").asBoolean()); + } + private String toJdbcTypeName(final AirbyteProtocolType airbyteProtocolType) { return switch (airbyteProtocolType) { case STRING -> "varchar"; diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt new file mode 100644 index 000000000000..c8d8f038046b --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt @@ -0,0 +1,13 @@ +package io.airbyte.integrations.destination.redshift.typing_deduping + +import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState + +data class RedshiftState(val needsSoftReset: Boolean): MinimumDestinationState { + override fun needsSoftReset(): Boolean { + return needsSoftReset + } + + override fun withSoftReset(needsSoftReset: Boolean): T { + return copy(needsSoftReset = needsSoftReset) as T + } +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java index 854fe35cfff6..be28d9dbad25 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -46,7 +46,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -public class RedshiftSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { +public class RedshiftSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { /** * Redshift's JDBC driver doesn't map certain data types onto {@link java.sql.JDBCType} usefully. @@ -151,8 +151,8 @@ protected DSLContext getDslContext() { } @Override - protected DestinationHandler getDestinationHandler() { - return new RedshiftDestinationHandler(databaseName, database); + protected DestinationHandler getDestinationHandler() { + return new RedshiftDestinationHandler(databaseName, database, namespace); } @Override @@ -180,9 +180,9 @@ protected Field toJsonValue(final String valueAsString) { public void testCreateTableIncremental() throws Exception { final Sql sql = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(sql); - List initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + List> initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); assertEquals(1, initialStates.size()); - final DestinationInitialState initialState = initialStates.getFirst(); + final DestinationInitialState initialState = initialStates.getFirst(); assertTrue(initialState.isFinalTablePresent()); assertFalse(initialState.isSchemaMismatch()); // TODO assert on table clustering, etc. From 89cd06d95edb1ef984fe6beba3b0810ad36a9fe7 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 23 Feb 2024 15:28:59 -0800 Subject: [PATCH 3/8] format --- .../destination/redshift/RedshiftInsertDestination.java | 4 +++- .../destination/redshift/RedshiftStagingS3Destination.java | 7 +++++-- .../typing_deduping/RedshiftDestinationHandler.java | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index f1bf66e6963a..81521b03b9fa 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -116,7 +116,9 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, String rawTableSchema) { + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, + final JdbcDatabase database, + String rawTableSchema) { return new RedshiftDestinationHandler(databaseName, database, rawTableSchema); } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 8d014d09a397..97e8b4393890 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -179,7 +179,9 @@ protected JdbcSqlGenerator getSqlGenerator() { } @Override - protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database, String rawTableSchema) { + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, + final JdbcDatabase database, + String rawTableSchema) { return new RedshiftDestinationHandler(databaseName, database, rawTableSchema); } @@ -235,7 +237,8 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); if (disableTypeDedupe) { - typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); + typerDeduper = + new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); } else { typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, List.of()); diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java index ea4955b95905..58258c3f26aa 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -69,7 +69,6 @@ protected String toJdbcTypeName(AirbyteType airbyteType) { }; } - @Override protected RedshiftState toDestinationState(JsonNode json) { return new RedshiftState( From 0490a8c94ba76d873e3f2e91d92c74874ca2b5da Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 23 Feb 2024 16:30:42 -0800 Subject: [PATCH 4/8] use jooq default dialect --- .../redshift/typing_deduping/RedshiftDestinationHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java index 58258c3f26aa..497d6469cd05 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -25,7 +25,8 @@ public class RedshiftDestinationHandler extends JdbcDestinationHandler { public RedshiftDestinationHandler(final String databaseName, final JdbcDatabase jdbcDatabase, String rawNamespace) { - super(databaseName, jdbcDatabase, rawNamespace, SQLDialect.POSTGRES); + // :shrug: apparently this works better than using POSTGRES + super(databaseName, jdbcDatabase, rawNamespace, SQLDialect.DEFAULT); } @Override From 245f9b4fd1d71e10143e7303cce919e75886e13c Mon Sep 17 00:00:00 2001 From: Gireesh Sreepathi Date: Thu, 29 Feb 2024 17:36:29 -0800 Subject: [PATCH 5/8] fix compilation post rebase --- .../RedshiftSqlGeneratorIntegrationTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java index be28d9dbad25..e4323aecc467 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -22,6 +22,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialStatus; import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; @@ -180,11 +181,11 @@ protected Field toJsonValue(final String valueAsString) { public void testCreateTableIncremental() throws Exception { final Sql sql = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(sql); - List> initialStates = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); - assertEquals(1, initialStates.size()); - final DestinationInitialState initialState = initialStates.getFirst(); - assertTrue(initialState.isFinalTablePresent()); - assertFalse(initialState.isSchemaMismatch()); + List> initialStatuses = destinationHandler.gatherInitialState(List.of(incrementalDedupStream)); + assertEquals(1, initialStatuses.size()); + final DestinationInitialStatus initialStatus = initialStatuses.getFirst(); + assertTrue(initialStatus.isFinalTablePresent()); + assertFalse(initialStatus.isSchemaMismatch()); // TODO assert on table clustering, etc. } From 8059a4b146ff7dd15a43a202a670bd02fdfd5689 Mon Sep 17 00:00:00 2001 From: Gireesh Sreepathi Date: Thu, 29 Feb 2024 17:39:27 -0800 Subject: [PATCH 6/8] derp --- .../typing_deduping/RedshiftSqlGeneratorIntegrationTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java index e4323aecc467..75515b5130b7 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -21,7 +21,6 @@ import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcSqlGeneratorIntegrationTest; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; -import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialState; import io.airbyte.integrations.base.destination.typing_deduping.DestinationInitialStatus; import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; From 2ff628223d51f41fd479d91a0713de5df58a7a8b Mon Sep 17 00:00:00 2001 From: Gireesh Sreepathi Date: Mon, 4 Mar 2024 11:42:48 -0800 Subject: [PATCH 7/8] version logistics Signed-off-by: Gireesh Sreepathi --- .../connectors/destination-redshift/build.gradle | 4 ++-- .../connectors/destination-redshift/metadata.yaml | 2 +- docs/integrations/destinations/redshift.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 0848d1b96359..c55b76b11eb9 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -5,9 +5,9 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.23.2' + cdkVersionRequired = '0.23.11' features = ['db-destinations', 's3-destinations', 'typing-deduping'] - useLocalCdk = true + useLocalCdk = false } java { diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 368369cfe5d2..909685693541 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 2.1.8 + dockerImageTag: 2.1.9 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index c37e01dcbb67..0df49130d9bf 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -237,6 +237,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.9 | 2024-03-04 | [\#35316](https://github.com/airbytehq/airbyte/pull/35316) | Update to CDK 0.23.11; Adopt migration framework | | 2.1.8 | 2024-02-09 | [\#35354](https://github.com/airbytehq/airbyte/pull/35354) | Update to CDK 0.23.0; Gather required initial state upfront, remove dependency on svv_table_info for table empty check | | 2.1.7 | 2024-02-09 | [\#34562](https://github.com/airbytehq/airbyte/pull/34562) | Switch back to jooq-based sql execution for standard insert | | 2.1.6 | 2024-02-08 | [\#34502](https://github.com/airbytehq/airbyte/pull/34502) | Update to CDK version 0.17.0 | From 9e0bdc6a38f7c82f52b748c0d7d821ad70c8ef9b Mon Sep 17 00:00:00 2001 From: Gireesh Sreepathi Date: Mon, 4 Mar 2024 11:59:02 -0800 Subject: [PATCH 8/8] fmt Signed-off-by: Gireesh Sreepathi --- .../redshift/typing_deduping/RedshiftState.kt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt index c8d8f038046b..d2200ea9a60c 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftState.kt @@ -1,13 +1,17 @@ +/* + * Copyright (c) 2024 Airbyte, Inc., all rights reserved. + */ + package io.airbyte.integrations.destination.redshift.typing_deduping import io.airbyte.integrations.base.destination.typing_deduping.migrators.MinimumDestinationState -data class RedshiftState(val needsSoftReset: Boolean): MinimumDestinationState { - override fun needsSoftReset(): Boolean { - return needsSoftReset - } +data class RedshiftState(val needsSoftReset: Boolean) : MinimumDestinationState { + override fun needsSoftReset(): Boolean { + return needsSoftReset + } - override fun withSoftReset(needsSoftReset: Boolean): T { - return copy(needsSoftReset = needsSoftReset) as T - } + override fun withSoftReset(needsSoftReset: Boolean): T { + return copy(needsSoftReset = needsSoftReset) as T + } }