From 79e9aedf4ca8b584562bf59d4303bc05d81e4397 Mon Sep 17 00:00:00 2001
From: brianreicher <reicher.b@northeastern.edu>
Date: Tue, 10 Oct 2023 20:09:43 -0400
Subject: [PATCH] Chunked reading of zip

---
 ingestion/src/downloader.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ingestion/src/downloader.rs b/ingestion/src/downloader.rs
index 5691649..6342a39 100644
--- a/ingestion/src/downloader.rs
+++ b/ingestion/src/downloader.rs
@@ -4,7 +4,6 @@ use std::fs::File;
 use std::path::Path;
 use mongo_utils::MongoDriver;
 use mongodb::bson::doc;
-use std::io::Read;
 use zip::read::ZipArchive;
 
 use crate::mongo_utils;
@@ -41,9 +40,9 @@ impl GitHubDownloader {
             let file_path: std::path::PathBuf = Path::new(zip_dirs[index]).join(filename);
             
             let mut response_body = response.bytes().await?;
-            while let Some(chunk) = response_body.concat() {
-                let chunk = chunk?;
-                file_path.write_all(&chunk)?;
+            while let chunk = response_body.chunks(8) {
+                let chunk = chunk;
+                file_path.write_all(&chunk)?; // TODO: write successfully
             }
 
             println!("Downloaded: {}", file_path.display());