Bmoric/async memory bump (#30775)

Co-authored-by: benmoriceau <[email protected]>
airbytehq · Sep 27, 2023 · 0537dde · 0537dde
1 parent 5fd0710
commit 0537dde
Show file tree

Hide file tree

Showing 9 changed files with 11 additions and 9 deletions.
diff --git a/...core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java b/...core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java
@@ -38,7 +38,7 @@ public class GlobalMemoryManager {
   // In cases where a queue is rapidly expanding, a larger block size allows less allocation calls. On
   // the flip size, a smaller block size allows more granular memory management. Since this overhead
   // is minimal for now, err on a smaller block sizes.
-  public static final long BLOCK_SIZE_BYTES = 10 * 1024 * 1024; // 10MB
+  public static final long BLOCK_SIZE_BYTES = 30 * 1024 * 1024; // 30MB
   private final long maxMemoryBytes;
 
   private final AtomicLong currentMemoryBytes = new AtomicLong(0);
@@ -61,7 +61,7 @@ public long getCurrentMemoryBytes() {
    * @return the size of the allocated block, in bytes
    */
   public synchronized long requestMemory() {
-    // todo(davin): what happens if the incoming record is larger than 10MB?
+    // todo(davin): what happens if the incoming record is larger than 30MB?
     if (currentMemoryBytes.get() >= maxMemoryBytes) {
       return 0L;
     }

diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile
@@ -25,5 +25,5 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery
 
 COPY --from=build /airbyte /airbyte
 
-LABEL io.airbyte.version=2.0.18
+LABEL io.airbyte.version=2.0.19
 LABEL io.airbyte.name=airbyte/destination-bigquery
diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133
-  dockerImageTag: 2.0.18
+  dockerImageTag: 2.0.19
   dockerRepository: airbyte/destination-bigquery
   githubIssueLabel: destination-bigquery
   icon: bigquery.svg

diff --git a/...n-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java b/...n-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java
@@ -6,9 +6,9 @@
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.annotations.VisibleForTesting;
-import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory;
-import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig;
 import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks;
+import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig;
+import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory;
 import io.airbyte.protocol.models.v0.AirbyteConnectionStatus;
 import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status;
 

diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile
@@ -29,5 +29,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1
 
 ENV ENABLE_SENTRY true
 
-LABEL io.airbyte.version=3.1.14
+LABEL io.airbyte.version=3.1.15
 LABEL io.airbyte.name=airbyte/destination-snowflake
diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: database
   connectorType: destination
   definitionId: 424892c4-daac-4491-b35d-c6688ba547ba
-  dockerImageTag: 3.1.14
+  dockerImageTag: 3.1.15
   dockerRepository: airbyte/destination-snowflake
   githubIssueLabel: destination-snowflake
   icon: snowflake.svg

diff --git a/deps.toml b/deps.toml
@@ -3,7 +3,7 @@ airbyte-protocol = "0.3.6"
 commons_io = "2.7"
 testcontainers = "1.19.0"
 datadog-version = "0.111.0"
-fasterxml_version = "2.14.0"
+fasterxml_version = "2.15.2"
 flyway = "7.14.0"
 glassfish_version = "2.31"
 hikaricp = "5.0.1"

diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md
@@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo
 
 | Version | Date       | Pull Request                                               | Subject                                                                                                                                                         |
 |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.0.19  | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775  | Increase async block size                                                                                                                                       |
 | 2.0.18  | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection                                                                                                                             |
 | 2.0.17  | 2023-09-26 | [\#30696](https://github.com/airbytehq/airbyte/pull/30696) | Attempt unsafe typing operations with an exception clause                                                                                                       |
 | 2.0.16  | 2023-09-22 | [\#30697](https://github.com/airbytehq/airbyte/pull/30697) | Improve resiliency to unclean exit during schema change                                                                                                         |

diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md
@@ -271,6 +271,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n
 
 | Version         | Date       | Pull Request                                               | Subject                                                                                                                                                         |
 |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 3.1.15          | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775  | Increase async block size                                                                                                                                       |
 | 3.1.14          | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection                                                                                                                             |
 | 3.1.13          | 2023-09-19 | [\#30599](https://github.com/airbytehq/airbyte/pull/30599) | Support concurrent syncs with identical stream name but different namespace                                                                                     |
 | 3.1.12          | 2023-09-21 | [\#30671](https://github.com/airbytehq/airbyte/pull/30671) | Reduce async buffer size                                                                                                                                        |