diff --git a/.gitignore b/.gitignore index 5dd500c..7ad8c5b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ target/ # Local state files. db/ +snapshot_db/ StateSnapshot.json # Direnv files. diff --git a/Cargo.lock b/Cargo.lock index 1e94ef8..9a3865f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aes" version = "0.6.0" @@ -1002,6 +1008,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam" version = "0.7.3" @@ -1292,6 +1307,16 @@ dependencies = [ "uuid 1.6.1", ] +[[package]] +name = "deflate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f" +dependencies = [ + "adler32", + "gzip-header", +] + [[package]] name = "der" version = "0.6.1" @@ -2195,6 +2220,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "gzip-header" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95cc527b92e6029a62960ad99aa8a6660faa4555fe5f731aab13aa6a921795a2" +dependencies = [ + "crc32fast", +] + [[package]] name = "h2" version = "0.3.22" @@ -5047,6 +5081,7 @@ dependencies = [ "bytes", "chrono", "clap 4.4.8", + "deflate", "ethers", "eyre", "hex", diff --git a/Cargo.toml b/Cargo.toml index 3f0e520..ee275f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,6 +29,7 @@ tracing = "0.1.40" tracing-subscriber = "0.3.17" zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" } zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" } +deflate = { version = "1.0.0", features = ["gzip"] } [build-dependencies] prost-build = "0.12" diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs index 5d4cc35..12fbbb4 100644 --- a/src/processor/snapshot/mod.rs +++ b/src/processor/snapshot/mod.rs @@ -12,6 +12,7 @@ mod types; use async_trait::async_trait; use blake2::{Blake2s256, Digest}; use bytes::BytesMut; +use deflate::deflate_bytes_gzip; use ethers::types::{Address, H256, U256, U64}; use eyre::Result; use prost::Message; @@ -300,7 +301,6 @@ impl SnapshotExporter { .into_string() .expect("path to string"); - // TODO: Wrap gzip compression around the outfile. let mut outfile = std::fs::OpenOptions::new() .write(true) .create(true) @@ -308,7 +308,10 @@ impl SnapshotExporter { // Serialize chunk. factory_deps.encode(&mut buf)?; - outfile.write_all(&buf)?; + + // Wrap in gzip compression before writing. + let compressed_buf = deflate_bytes_gzip(&buf); + outfile.write_all(&compressed_buf)?; outfile.flush()?; Ok(()) @@ -359,7 +362,7 @@ impl SnapshotExporter { chunk_index += 1; let path = PathBuf::new() .join(&self.basedir) - .join(format!("{chunk_index}.chunk")); + .join(format!("{chunk_index}.gz")); header .storage_logs_chunks @@ -372,7 +375,6 @@ impl SnapshotExporter { .expect("path to string"), }); - // TODO: Wrap gzip compression around the outfile. let mut outfile = std::fs::OpenOptions::new() .write(true) .create(true) @@ -380,7 +382,10 @@ impl SnapshotExporter { // Serialize chunk. chunk.encode(&mut buf)?; - outfile.write_all(&buf)?; + + // Wrap in gzip compression before writing. + let compressed_buf = deflate_bytes_gzip(&buf); + outfile.write_all(&compressed_buf)?; outfile.flush()?; // Clear $tmp buffer.