From 0537dde7d2d03192d1e3a507c28c34b8ba695942 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Wed, 27 Sep 2023 16:44:20 -0700 Subject: [PATCH] Bmoric/async memory bump (#30775) Co-authored-by: benmoriceau --- .../integrations/destination_async/GlobalMemoryManager.java | 4 ++-- .../connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-bigquery/metadata.yaml | 2 +- .../destination/s3/S3DestinationStrictEncrypt.java | 4 ++-- .../connectors/destination-snowflake/Dockerfile | 2 +- .../connectors/destination-snowflake/metadata.yaml | 2 +- deps.toml | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/snowflake.md | 1 + 9 files changed, 11 insertions(+), 9 deletions(-) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java index e07fd55bb806..510099ea8f54 100644 --- a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java @@ -38,7 +38,7 @@ public class GlobalMemoryManager { // In cases where a queue is rapidly expanding, a larger block size allows less allocation calls. On // the flip size, a smaller block size allows more granular memory management. Since this overhead // is minimal for now, err on a smaller block sizes. - public static final long BLOCK_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + public static final long BLOCK_SIZE_BYTES = 30 * 1024 * 1024; // 30MB private final long maxMemoryBytes; private final AtomicLong currentMemoryBytes = new AtomicLong(0); @@ -61,7 +61,7 @@ public long getCurrentMemoryBytes() { * @return the size of the allocated block, in bytes */ public synchronized long requestMemory() { - // todo(davin): what happens if the incoming record is larger than 10MB? + // todo(davin): what happens if the incoming record is larger than 30MB? if (currentMemoryBytes.get() >= maxMemoryBytes) { return 0L; } diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index d417281bf814..e83b3ff07936 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -25,5 +25,5 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.18 +LABEL io.airbyte.version=2.0.19 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index aeb2fd4c63d3..8754d50a0abe 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.0.18 + dockerImageTag: 2.0.19 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java index 0dbccd86f875..8258bbcb7d04 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; -import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index a5dcba9fadb3..9abfb8797101 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -29,5 +29,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=3.1.14 +LABEL io.airbyte.version=3.1.15 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 58a40d811fa6..fa626be84a37 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.1.14 + dockerImageTag: 3.1.15 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg diff --git a/deps.toml b/deps.toml index d6187e55b7ac..a7e67a6d8021 100644 --- a/deps.toml +++ b/deps.toml @@ -3,7 +3,7 @@ airbyte-protocol = "0.3.6" commons_io = "2.7" testcontainers = "1.19.0" datadog-version = "0.111.0" -fasterxml_version = "2.14.0" +fasterxml_version = "2.15.2" flyway = "7.14.0" glassfish_version = "2.31" hikaricp = "5.0.1" diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index db16bd853712..f349a50e6fc6 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.19 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size | | 2.0.18 | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection | | 2.0.17 | 2023-09-26 | [\#30696](https://github.com/airbytehq/airbyte/pull/30696) | Attempt unsafe typing operations with an exception clause | | 2.0.16 | 2023-09-22 | [\#30697](https://github.com/airbytehq/airbyte/pull/30697) | Improve resiliency to unclean exit during schema change | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 296b12bb6d58..0c9d1765c4a9 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -271,6 +271,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.1.15 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775 | Increase async block size | | 3.1.14 | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection | | 3.1.13 | 2023-09-19 | [\#30599](https://github.com/airbytehq/airbyte/pull/30599) | Support concurrent syncs with identical stream name but different namespace | | 3.1.12 | 2023-09-21 | [\#30671](https://github.com/airbytehq/airbyte/pull/30671) | Reduce async buffer size |