From 985c8396739bd84826e22e3359d945722c73afd8 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 8 May 2024 11:29:14 -0700 Subject: [PATCH] Destination Redshift: Remove incremental T+D option (#37713) --- .../connectors/destination-redshift/build.gradle | 2 +- .../connectors/destination-redshift/metadata.yaml | 2 +- .../redshift/RedshiftStagingS3Destination.java | 2 -- .../destination-redshift/src/main/resources/spec.json | 8 -------- docs/integrations/destinations/redshift.md | 1 + 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index ff7a8dd38d68..035ed71d6de7 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -4,7 +4,7 @@ plugins { } airbyteJavaConnector { - cdkVersionRequired = '0.33.0' + cdkVersionRequired = '0.34.0' features = ['db-destinations', 's3-destinations', 'typing-deduping'] useLocalCdk = false } diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 45e453b2115b..f7486913dfc3 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 2.5.0 + dockerImageTag: 2.6.0 dockerRepository: airbyte/destination-redshift documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 87370e5ad7f7..981a7cfdfa94 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -46,7 +46,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; -import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.migrators.Migration; import io.airbyte.integrations.destination.redshift.operations.RedshiftS3StagingSqlOperations; @@ -267,7 +266,6 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN config, catalog, isPurgeStagingData(s3Options), - new TypeAndDedupeOperationValve(), typerDeduper, parsedCatalog, defaultNamespace, diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index 0c87a94c6ac2..6aea2788dd9d 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -246,14 +246,6 @@ "order": 9, "group": "connection" }, - "enable_incremental_final_table_updates": { - "type": "boolean", - "default": false, - "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", - "title": "Enable Loading Data Incrementally to Final Tables", - "order": 10, - "group": "connection" - }, "disable_type_dedupe": { "type": "boolean", "default": false, diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 5ccf766a15d8..a7460e4eb109 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -242,6 +242,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 2.6.0 | 2024-05-08 | [\#37713](https://github.com/airbytehq/airbyte/pull/37713) | Remove option for incremental typing and deduping | | 2.5.0 | 2024-05-06 | [\#34613](https://github.com/airbytehq/airbyte/pull/34613) | Upgrade Redshift driver to work with Cluster patch 181; Adapt to CDK 0.33.0; Minor signature changes | | 2.4.3 | 2024-04-10 | [\#36973](https://github.com/airbytehq/airbyte/pull/36973) | Limit the Standard inserts SQL statement to less than 16MB | | 2.4.2 | 2024-04-05 | [\#36365](https://github.com/airbytehq/airbyte/pull/36365) | Remove unused config option |