From 88c90650535e7c5fcf25a3a6f9ba976f98947ae2 Mon Sep 17 00:00:00 2001 From: Alexander Tesfamichael Date: Sun, 15 Oct 2023 22:03:33 +0200 Subject: [PATCH] feat: increase buffers Sets decoded buffer to a multiple of concurrent put limit. Sets concurrent put to a reasonable runtime default. --- src/bin/migrate-from-gcs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bin/migrate-from-gcs.rs b/src/bin/migrate-from-gcs.rs index 11b312f..27267a7 100644 --- a/src/bin/migrate-from-gcs.rs +++ b/src/bin/migrate-from-gcs.rs @@ -195,7 +195,7 @@ async fn main() -> anyhow::Result<()> { let payload_stream = gcs.get(&object_meta.location).await?.into_stream(); let reader = StreamReader::new(payload_stream); - const DECODED_BUFFER_SIZE: usize = 32; + const DECODED_BUFFER_SIZE: usize = 128; let (mut decoded_tx, decoded_rx) = channel(DECODED_BUFFER_SIZE); let handle = spawn_blocking(move || { @@ -226,7 +226,7 @@ async fn main() -> anyhow::Result<()> { } }); - const CONCURRENT_PUT_LIMIT: usize = 1; + const CONCURRENT_PUT_LIMIT: usize = 32; // Skip payloads that have already been processed. decoded_rx