feat: compress chunk parts with gzip (#47)

.gitignore

@@ -11,6 +11,7 @@ target/
 
 # Local state files.
 db/
+snapshot_db/
 StateSnapshot.json
 
 # Direnv files.

Cargo.toml

@@ -29,6 +29,7 @@ tracing = "0.1.40"
 tracing-subscriber = "0.3.17"
 zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" }
 zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" }
+deflate = { version = "1.0.0", features = ["gzip"] }
 
 [build-dependencies]
 prost-build = "0.12"

src/processor/snapshot/mod.rs

@@ -12,6 +12,7 @@ mod types;
 use async_trait::async_trait;
 use blake2::{Blake2s256, Digest};
 use bytes::BytesMut;
+use deflate::deflate_bytes_gzip;
 use ethers::types::{Address, H256, U256, U64};
 use eyre::Result;
 use prost::Message;
@@ -300,15 +301,17 @@ impl SnapshotExporter {
             .into_string()
             .expect("path to string");
 
-        // TODO: Wrap gzip compression around the outfile.
         let mut outfile = std::fs::OpenOptions::new()
             .write(true)
             .create(true)
             .open(path)?;
 
         // Serialize chunk.
         factory_deps.encode(&mut buf)?;
-        outfile.write_all(&buf)?;
+
+        // Wrap in gzip compression before writing.
+        let compressed_buf = deflate_bytes_gzip(&buf);
+        outfile.write_all(&compressed_buf)?;
         outfile.flush()?;
 
         Ok(())
@@ -359,7 +362,7 @@ impl SnapshotExporter {
             chunk_index += 1;
             let path = PathBuf::new()
                 .join(&self.basedir)
-                .join(format!("{chunk_index}.chunk"));
+                .join(format!("{chunk_index}.gz"));
 
             header
                 .storage_logs_chunks
@@ -372,15 +375,17 @@ impl SnapshotExporter {
                         .expect("path to string"),
                 });
 
-            // TODO: Wrap gzip compression around the outfile.
             let mut outfile = std::fs::OpenOptions::new()
                 .write(true)
                 .create(true)
                 .open(path)?;
 
             // Serialize chunk.
             chunk.encode(&mut buf)?;
-            outfile.write_all(&buf)?;
+
+            // Wrap in gzip compression before writing.
+            let compressed_buf = deflate_bytes_gzip(&buf);
+            outfile.write_all(&compressed_buf)?;
             outfile.flush()?;
 
             // Clear $tmp buffer.