feat(migrate-from-gcs): batch decoding

ultrasoundmoney · Oct 16, 2023 · d6f919e · d6f919e
1 parent b19d50d
commit d6f919e
Showing 1 changed file with 11 additions and 1 deletion.
diff --git a/src/bin/migrate-from-gcs.rs b/src/bin/migrate-from-gcs.rs
@@ -195,6 +195,8 @@ async fn main() -> anyhow::Result<()> {
             let mut csv_reader = csv::Reader::from_reader(decoder);
             let mut iter = csv_reader.byte_records();
 
+            let mut batch = Vec::new();
+
             while let Some(record) = iter.next().transpose().unwrap() {
                 let execution_payload = {
                     unsafe {
@@ -213,7 +215,15 @@ async fn main() -> anyhow::Result<()> {
                     }
                 };
 
-                futures::executor::block_on(decoded_tx.send(execution_payload)).unwrap();
+                batch.push(execution_payload);
+
+                if batch.len() == 32 {
+                    for execution_payload in batch.drain(..) {
+                        futures::executor::block_on(decoded_tx.feed(execution_payload)).unwrap();
+                    }
+                    futures::executor::block_on(decoded_tx.flush()).unwrap();
+                    batch.clear();
+                }
             }
         });