Big optimal size and low rate limit

airbytehq · Oct 10, 2023 · d168a91 · d168a91
1 parent 2671973
commit d168a91
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/...rc/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncStandardFlush.java b/...rc/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncStandardFlush.java
@@ -21,7 +21,7 @@
 public class BigQueryAsyncStandardFlush implements DestinationFlushFunction {
 
   // TODO remove this once the async framework supports rate-limiting/backpressuring
-  private static final RateLimiter rateLimiter = RateLimiter.create(0.5);
+  private static final RateLimiter rateLimiter = RateLimiter.create(0.017);
 
   private final BigQuery bigQuery;
   private final ConcurrentMap<AirbyteStreamNameNamespacePair, AbstractBigQueryUploader<?>> uploaderMap;
@@ -58,7 +58,7 @@ public long getOptimalBatchSizeBytes() {
     // resource the connector will usually at most fill up around 150 MB in a single queue. By lowering
     // the batch size, the AsyncFlusher will flush in smaller batches which allows for memory to be
     // freed earlier similar to a sliding window effect
-    return 25 * 1024 * 1024;
+    return 300 * 1024 * 1024;
   }
 
 }
diff --git a/...ain/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordStandardConsumer.java b/...ain/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordStandardConsumer.java
@@ -35,7 +35,6 @@ public BigQueryRecordStandardConsumer(Consumer<AirbyteMessage> outputRecordColle
         new BigQueryAsyncStandardFlush(bigQuery, uploaderMap),
         catalog,
         new BufferManager(),
-        defaultNamespace,
-        Executors.newFixedThreadPool(1));
+        defaultNamespace);
   }
 }