From 18e725e76919200c98e6e7719cfa8589918b21b2 Mon Sep 17 00:00:00 2001 From: zeapoz Date: Wed, 20 Dec 2023 12:20:17 +0100 Subject: [PATCH] feat: inital support for Boojum format Introduces structs and calldata parsing for blocks formatted with the newer format. --- IZkSync.json => abi/IZkSync.json | 0 abi/IZkSyncV2.json | 641 ++++++++++++++++++ src/main.rs | 12 +- src/processor/json/mod.rs | 4 +- src/processor/mod.rs | 4 +- src/processor/snapshot/mod.rs | 4 +- src/processor/tree/mod.rs | 10 +- src/processor/tree/tree_wrapper.rs | 8 +- state-reconstruct-fetcher/src/constants.rs | 12 + state-reconstruct-fetcher/src/l1_fetcher.rs | 97 ++- state-reconstruct-fetcher/src/types/mod.rs | 106 +++ .../src/{types.rs => types/v1.rs} | 119 +--- state-reconstruct-fetcher/src/types/v2.rs | 319 +++++++++ 13 files changed, 1171 insertions(+), 165 deletions(-) rename IZkSync.json => abi/IZkSync.json (100%) create mode 100644 abi/IZkSyncV2.json create mode 100644 state-reconstruct-fetcher/src/types/mod.rs rename state-reconstruct-fetcher/src/{types.rs => types/v1.rs} (71%) create mode 100644 state-reconstruct-fetcher/src/types/v2.rs diff --git a/IZkSync.json b/abi/IZkSync.json similarity index 100% rename from IZkSync.json rename to abi/IZkSync.json diff --git a/abi/IZkSyncV2.json b/abi/IZkSyncV2.json new file mode 100644 index 0000000..4e3d572 --- /dev/null +++ b/abi/IZkSyncV2.json @@ -0,0 +1,641 @@ +[ + { + "inputs": [ + { + "internalType": "address", + "name": "_initialOwner", + "type": "address" + }, + { + "internalType": "address", + "name": "_zkSyncContract", + "type": "address" + }, + { + "internalType": "uint32", + "name": "_executionDelay", + "type": "uint32" + }, + { + "internalType": "address", + "name": "_validator", + "type": "address" + } + ], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "uint256", + "name": "batchNumber", + "type": "uint256" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "name": "BlockCommit", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "uint256", + "name": "batchNumber", + "type": "uint256" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "name": "BlockExecution", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "totalBatchesCommitted", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "totalBatchesVerified", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "totalBatchesExecuted", + "type": "uint256" + } + ], + "name": "BlocksRevert", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "uint256", + "name": "previousLastVerifiedBatch", + "type": "uint256" + }, + { + "indexed": true, + "internalType": "uint256", + "name": "currentLastVerifiedBatch", + "type": "uint256" + } + ], + "name": "BlocksVerification", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "_newExecutionDelay", + "type": "uint256" + } + ], + "name": "NewExecutionDelay", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "address", + "name": "_oldValidator", + "type": "address" + }, + { + "indexed": false, + "internalType": "address", + "name": "_newValidator", + "type": "address" + } + ], + "name": "NewValidator", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "previousOwner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "newOwner", + "type": "address" + } + ], + "name": "OwnershipTransferStarted", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "previousOwner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "newOwner", + "type": "address" + } + ], + "name": "OwnershipTransferred", + "type": "event" + }, + { + "inputs": [], + "name": "acceptOwnership", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "uint64", + "name": "batchNumber", + "type": "uint64" + }, + { + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "internalType": "uint64", + "name": "indexRepeatedStorageChanges", + "type": "uint64" + }, + { + "internalType": "uint256", + "name": "numberOfLayer1Txs", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "priorityOperationsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "l2LogsTreeRoot", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "internalType": "struct IExecutor.StoredBatchInfo", + "name": "", + "type": "tuple" + }, + { + "components": [ + { + "internalType": "uint64", + "name": "batchNumber", + "type": "uint64" + }, + { + "internalType": "uint64", + "name": "timestamp", + "type": "uint64" + }, + { + "internalType": "uint64", + "name": "indexRepeatedStorageChanges", + "type": "uint64" + }, + { + "internalType": "bytes32", + "name": "newStateRoot", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "numberOfLayer1Txs", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "priorityOperationsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "bootloaderHeapInitialContentsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "eventsQueueStateHash", + "type": "bytes32" + }, + { + "internalType": "bytes", + "name": "systemLogs", + "type": "bytes" + }, + { + "internalType": "bytes", + "name": "totalL2ToL1Pubdata", + "type": "bytes" + } + ], + "internalType": "struct IExecutor.CommitBatchInfo[]", + "name": "_newBatchesData", + "type": "tuple[]" + } + ], + "name": "commitBatches", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "uint64", + "name": "batchNumber", + "type": "uint64" + }, + { + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "internalType": "uint64", + "name": "indexRepeatedStorageChanges", + "type": "uint64" + }, + { + "internalType": "uint256", + "name": "numberOfLayer1Txs", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "priorityOperationsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "l2LogsTreeRoot", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "internalType": "struct IExecutor.StoredBatchInfo[]", + "name": "_newBatchesData", + "type": "tuple[]" + } + ], + "name": "executeBatches", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "executionDelay", + "outputs": [ + { + "internalType": "uint32", + "name": "", + "type": "uint32" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "_l2BatchNumber", + "type": "uint256" + } + ], + "name": "getCommittedBatchTimestamp", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getName", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "owner", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "pendingOwner", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { + "internalType": "uint64", + "name": "batchNumber", + "type": "uint64" + }, + { + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "internalType": "uint64", + "name": "indexRepeatedStorageChanges", + "type": "uint64" + }, + { + "internalType": "uint256", + "name": "numberOfLayer1Txs", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "priorityOperationsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "l2LogsTreeRoot", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "internalType": "struct IExecutor.StoredBatchInfo", + "name": "", + "type": "tuple" + }, + { + "components": [ + { + "internalType": "uint64", + "name": "batchNumber", + "type": "uint64" + }, + { + "internalType": "bytes32", + "name": "batchHash", + "type": "bytes32" + }, + { + "internalType": "uint64", + "name": "indexRepeatedStorageChanges", + "type": "uint64" + }, + { + "internalType": "uint256", + "name": "numberOfLayer1Txs", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "priorityOperationsHash", + "type": "bytes32" + }, + { + "internalType": "bytes32", + "name": "l2LogsTreeRoot", + "type": "bytes32" + }, + { + "internalType": "uint256", + "name": "timestamp", + "type": "uint256" + }, + { + "internalType": "bytes32", + "name": "commitment", + "type": "bytes32" + } + ], + "internalType": "struct IExecutor.StoredBatchInfo[]", + "name": "", + "type": "tuple[]" + }, + { + "components": [ + { + "internalType": "uint256[]", + "name": "recursiveAggregationInput", + "type": "uint256[]" + }, + { + "internalType": "uint256[]", + "name": "serializedProof", + "type": "uint256[]" + } + ], + "internalType": "struct IExecutor.ProofInput", + "name": "", + "type": "tuple" + } + ], + "name": "proveBatches", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "renounceOwnership", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "name": "revertBatches", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint32", + "name": "_executionDelay", + "type": "uint32" + } + ], + "name": "setExecutionDelay", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_newValidator", + "type": "address" + } + ], + "name": "setValidator", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "newOwner", + "type": "address" + } + ], + "name": "transferOwnership", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "validator", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "zkSyncContract", + "outputs": [ + { + "internalType": "address", + "name": "", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + } +] diff --git a/src/main.rs b/src/main.rs index 7854edd..137a80a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,7 +21,7 @@ use state_reconstruct_fetcher::{ constants::storage::{self, STATE_FILE_NAME}, l1_fetcher::{L1Fetcher, L1FetcherOptions}, snapshot::StateSnapshot, - types::CommitBlockInfoV1, + types::CommitBlock, }; use tokio::sync::{mpsc, Mutex}; use tracing_subscriber::{filter::LevelFilter, EnvFilter}; @@ -79,7 +79,7 @@ async fn main() -> Result<()> { let fetcher = L1Fetcher::new(fetcher_options, Some(snapshot.clone()))?; let processor = TreeProcessor::new(db_path.clone(), snapshot.clone()).await?; - let (tx, rx) = mpsc::channel::(5); + let (tx, rx) = mpsc::channel::(5); let processor_handle = tokio::spawn(async move { processor.run(rx).await; @@ -98,13 +98,13 @@ async fn main() -> Result<()> { let reader = BufReader::new(File::open(&file)?); let processor = TreeProcessor::new(db_path, snapshot).await?; - let (tx, rx) = mpsc::channel::(5); + let (tx, rx) = mpsc::channel::(5); tokio::spawn(async move { processor.run(rx).await; }); - let json_iter = json::iter_json_array::(reader); + let json_iter = json::iter_json_array::(reader); let mut num_objects = 0; for blk in json_iter { tx.send(blk.expect("parsing")).await?; @@ -129,7 +129,7 @@ async fn main() -> Result<()> { let fetcher = L1Fetcher::new(fetcher_options, None)?; let processor = JsonSerializationProcessor::new(Path::new(&file))?; - let (tx, rx) = mpsc::channel::(5); + let (tx, rx) = mpsc::channel::(5); let processor_handle = tokio::spawn(async move { processor.run(rx).await; @@ -172,7 +172,7 @@ async fn main() -> Result<()> { let fetcher = L1Fetcher::new(fetcher_options, None)?; let processor = SnapshotBuilder::new(db_path); - let (tx, rx) = mpsc::channel::(5); + let (tx, rx) = mpsc::channel::(5); let processor_handle = tokio::spawn(async move { processor.run(rx).await; }); diff --git a/src/processor/json/mod.rs b/src/processor/json/mod.rs index 34594c7..93d93b4 100644 --- a/src/processor/json/mod.rs +++ b/src/processor/json/mod.rs @@ -4,7 +4,7 @@ use async_trait::async_trait; use eyre::Result; use serde::ser::{SerializeSeq, Serializer}; use serde_json; -use state_reconstruct_fetcher::types::CommitBlockInfoV1; +use state_reconstruct_fetcher::types::CommitBlock; use tokio::sync::mpsc; use super::Processor; @@ -23,7 +23,7 @@ impl JsonSerializationProcessor { #[async_trait] impl Processor for JsonSerializationProcessor { - async fn run(mut self, mut rx: mpsc::Receiver) { + async fn run(mut self, mut rx: mpsc::Receiver) { let mut seq = self .serializer .serialize_seq(None) diff --git a/src/processor/mod.rs b/src/processor/mod.rs index e88ffae..326dc8f 100644 --- a/src/processor/mod.rs +++ b/src/processor/mod.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use state_reconstruct_fetcher::types::CommitBlockInfoV1; +use state_reconstruct_fetcher::types::CommitBlock; use tokio::sync::mpsc; pub mod json; @@ -8,5 +8,5 @@ pub mod tree; #[async_trait] pub trait Processor { - async fn run(self, mut rx: mpsc::Receiver); + async fn run(self, mut rx: mpsc::Receiver); } diff --git a/src/processor/snapshot/mod.rs b/src/processor/snapshot/mod.rs index 12fbbb4..ee05b5c 100644 --- a/src/processor/snapshot/mod.rs +++ b/src/processor/snapshot/mod.rs @@ -18,7 +18,7 @@ use eyre::Result; use prost::Message; use state_reconstruct_fetcher::{ constants::{ethereum, storage}, - types::CommitBlockInfoV1, + types::CommitBlock, }; use tokio::sync::mpsc; @@ -64,7 +64,7 @@ impl SnapshotBuilder { #[async_trait] impl Processor for SnapshotBuilder { - async fn run(mut self, mut rx: mpsc::Receiver) { + async fn run(mut self, mut rx: mpsc::Receiver) { while let Some(block) = rx.recv().await { // Initial calldata. for (key, value) in &block.initial_storage_changes { diff --git a/src/processor/tree/mod.rs b/src/processor/tree/mod.rs index 4128af8..4406f85 100644 --- a/src/processor/tree/mod.rs +++ b/src/processor/tree/mod.rs @@ -7,7 +7,7 @@ use async_trait::async_trait; use ethers::types::H256; use eyre::Result; use state_reconstruct_fetcher::{ - constants::storage::STATE_FILE_NAME, snapshot::StateSnapshot, types::CommitBlockInfoV1, + constants::storage::STATE_FILE_NAME, snapshot::StateSnapshot, types::CommitBlock, }; use tokio::sync::{mpsc, Mutex}; @@ -48,14 +48,14 @@ impl TreeProcessor { #[async_trait] impl Processor for TreeProcessor { - async fn run(mut self, mut rx: mpsc::Receiver) { + async fn run(mut self, mut rx: mpsc::Receiver) { while let Some(block) = rx.recv().await { let mut snapshot = self.snapshot.lock().await; // Check if we've already processed this block. - if snapshot.latest_l2_block_number >= block.block_number { + if snapshot.latest_l2_block_number >= block.l2_block_number { tracing::debug!( "Block {} has already been processed, skipping.", - block.block_number + block.l2_block_number ); continue; } @@ -63,7 +63,7 @@ impl Processor for TreeProcessor { self.tree.insert_block(&block); // Update snapshot values. - snapshot.latest_l2_block_number = block.block_number; + snapshot.latest_l2_block_number = block.l2_block_number; snapshot.index_to_key_map = self.tree.index_to_key_map.clone(); } } diff --git a/src/processor/tree/tree_wrapper.rs b/src/processor/tree/tree_wrapper.rs index 8b1a5f0..8359571 100644 --- a/src/processor/tree/tree_wrapper.rs +++ b/src/processor/tree/tree_wrapper.rs @@ -4,7 +4,7 @@ use blake2::{Blake2s256, Digest}; use ethers::types::{Address, H256, U256}; use eyre::Result; use indexmap::IndexSet; -use state_reconstruct_fetcher::{constants::storage::INITAL_STATE_PATH, types::CommitBlockInfoV1}; +use state_reconstruct_fetcher::{constants::storage::INITAL_STATE_PATH, types::CommitBlock}; use zksync_merkle_tree::{Database, MerkleTree, RocksDBWrapper}; use super::RootHash; @@ -33,7 +33,7 @@ impl TreeWrapper { } /// Inserts a block into the tree and returns the root hash of the resulting state tree. - pub fn insert_block(&mut self, block: &CommitBlockInfoV1) -> RootHash { + pub fn insert_block(&mut self, block: &CommitBlock) -> RootHash { // INITIAL CALLDATA. let mut key_value_pairs: Vec<(U256, H256)> = Vec::with_capacity(block.initial_storage_changes.len()); @@ -61,11 +61,11 @@ impl TreeWrapper { assert_eq!(root_hash.as_bytes(), block.new_state_root); tracing::debug!( "Root hash of block {} = {}", - block.block_number, + block.l2_block_number, hex::encode(root_hash) ); - tracing::debug!("Successfully processed block {}", block.block_number); + tracing::debug!("Successfully processed block {}", block.l2_block_number); root_hash } diff --git a/state-reconstruct-fetcher/src/constants.rs b/state-reconstruct-fetcher/src/constants.rs index 6ae709a..6bdb1a6 100644 --- a/state-reconstruct-fetcher/src/constants.rs +++ b/state-reconstruct-fetcher/src/constants.rs @@ -5,6 +5,9 @@ pub mod ethereum { /// Block number in Ethereum for zkSync genesis block. pub const GENESIS_BLOCK: u64 = 16_627_460; + /// Block number in Ethereum of the first Boojum-formatted block. + pub const BOOJUM_BLOCK: u64 = 18_711_784; + /// zkSync smart contract address. pub const ZK_SYNC_ADDR: &str = "0x32400084C286CF3E17e7B677ea9583e60a000324"; } @@ -19,3 +22,12 @@ pub mod storage { /// The name of the state file. pub const STATE_FILE_NAME: &str = "StateSnapshot.json"; } + +pub mod zksync { + /// Bytes in raw L2 to L1 log. + pub const L2_TO_L1_LOG_SERIALIZE_SIZE: usize = 88; + // The bitmask by applying which to the compressed state diff metadata we retrieve its operation. + pub const OPERATION_BITMASK: u8 = 7; + // The number of bits shifting the compressed state diff metadata by which we retrieve its length. + pub const LENGTH_BITS_OFFSET: u8 = 3; +} diff --git a/state-reconstruct-fetcher/src/l1_fetcher.rs b/state-reconstruct-fetcher/src/l1_fetcher.rs index a07b21c..b718440 100644 --- a/state-reconstruct-fetcher/src/l1_fetcher.rs +++ b/state-reconstruct-fetcher/src/l1_fetcher.rs @@ -1,4 +1,4 @@ -use std::{future::Future, ops::Fn, sync::Arc}; +use std::{fs::File, future::Future, ops::Fn, sync::Arc}; use ethers::{ abi::{Contract, Function}, @@ -15,9 +15,9 @@ use tokio::{ use tokio_util::sync::CancellationToken; use crate::{ - constants::ethereum::{BLOCK_STEP, GENESIS_BLOCK, ZK_SYNC_ADDR}, + constants::ethereum::{BLOCK_STEP, BOOJUM_BLOCK, GENESIS_BLOCK, ZK_SYNC_ADDR}, snapshot::StateSnapshot, - types::{CommitBlockInfoV1, ParseError}, + types::{v1, v2, CommitBlock, CommitBlockInfo, ParseError}, }; /// `MAX_RETRIES` is the maximum number of retries on failed L1 call. @@ -89,9 +89,15 @@ impl L1Metrics { } } +#[derive(Clone)] +struct Contracts { + v1: Contract, + v2: Contract, +} + pub struct L1Fetcher { provider: Provider, - contract: Contract, + contracts: Contracts, config: L1FetcherOptions, snapshot: Option>>, metrics: Arc>, @@ -105,21 +111,22 @@ impl L1Fetcher { let provider = Provider::::try_from(&config.http_url) .expect("could not instantiate HTTP Provider"); - let abi_file = std::fs::File::open("./IZkSync.json")?; - let contract = Contract::load(abi_file)?; + let v1 = Contract::load(File::open("./abi/IZkSync.json")?)?; + let v2 = Contract::load(File::open("./abi/IZkSyncV2.json")?)?; + let contracts = Contracts { v1, v2 }; let metrics = Arc::new(Mutex::new(L1Metrics::default())); Ok(L1Fetcher { provider, - contract, + contracts, config, snapshot, metrics, }) } - pub async fn run(&self, sink: mpsc::Sender) -> Result<()> { + pub async fn run(&self, sink: mpsc::Sender) -> Result<()> { // Start fetching from the `GENESIS_BLOCK` unless the `start_block` argument is supplied, // in which case, start from that instead. If no argument was supplied and a state snapshot // exists, start from the block number specified in that snapshot. @@ -237,7 +244,7 @@ impl L1Fetcher { disable_polling: bool, ) -> Result> { let metrics = self.metrics.clone(); - let event = self.contract.events_by_name("BlockCommit")?[0].clone(); + let event = self.contracts.v1.events_by_name("BlockCommit")?[0].clone(); let provider_clone = self.provider.clone(); Ok(tokio::spawn({ @@ -357,21 +364,33 @@ impl L1Fetcher { }) } + // FIXME: + #[allow(clippy::absurd_extreme_comparisons)] fn spawn_parsing_handler( &self, mut l1_tx_rx: mpsc::Receiver, - sink: mpsc::Sender, + sink: mpsc::Sender, ) -> Result>> { let metrics = self.metrics.clone(); - let function = self.contract.functions_by_name("commitBlocks")?[0].clone(); - + let contracts = self.contracts.clone(); Ok(tokio::spawn({ async move { + let mut function = + contracts.v1.functions_by_name("commitBlocks").unwrap()[0].clone(); let mut last_block_number_processed = None; while let Some(tx) = l1_tx_rx.recv().await { let block_number = tx.block_number.map(|v| v.as_u64()); - let blocks = match parse_calldata(block_number, &function, &tx.input) { + + if let Some(block_number) = block_number { + if block_number >= BOOJUM_BLOCK { + function = + contracts.v2.functions_by_name("commitBatches").unwrap()[0].clone(); + tracing::debug!("Reached `BOOJUM_BLOCK`, changing commit block format"); + } + }; + + let blocks = match parse_calldata(block_number, &function, &tx.input).await { Ok(blks) => blks, Err(e) => { tracing::error!("failed to parse calldata: {e}"); @@ -383,15 +402,14 @@ impl L1Fetcher { // NOTE: Let's see if we want to increment this in batches, instead of each block individually. let mut metrics = metrics.lock().await; metrics.l2_blocks_processed += 1; - metrics.latest_l2_block_nbr = blk.block_number; + metrics.latest_l2_block_nbr = blk.l2_block_number; sink.send(blk).await.unwrap(); } last_block_number_processed = block_number; } - // Return the last processed l1 block number, - // so we can resume from the same point later on. + // Return the last processed l1 block number, so we can resume from the same point later on. last_block_number_processed } })) @@ -414,11 +432,11 @@ impl L1Fetcher { } } -pub fn parse_calldata( +pub async fn parse_calldata( l1_block_number: Option, commit_blocks_fn: &Function, calldata: &[u8], -) -> Result> { +) -> Result> { let mut parsed_input = commit_blocks_fn .decode_input(&calldata[4..]) .map_err(|e| ParseError::InvalidCalldata(e.to_string()))?; @@ -460,16 +478,26 @@ pub fn parse_calldata( // TODO: What to do here? // assert_eq!(previous_enumeration_index, tree.next_enumeration_index()); - // Supplement every CommitBlockInfoV1 element with L1 block number information. - parse_commit_block_info(&new_blocks_data).map(|mut vec| { - vec.iter_mut() - .for_each(|e| e.l1_block_number = l1_block_number); - vec - }) + // Supplement every `CommitBlock` element with L1 block number information. + parse_commit_block_info(&new_blocks_data, l1_block_number) + .await + .map(|mut vec| { + vec.iter_mut() + .for_each(|e| e.l1_block_number = l1_block_number); + vec + }) } -fn parse_commit_block_info(data: &abi::Token) -> Result> { - let mut res = vec![]; +async fn parse_commit_block_info( + data: &abi::Token, + l1_block_number: Option, +) -> Result> { + // By default parse blocks using [`CommitBlockInfoV1`]; if we have reached [`BOOJUM_BLOCK`], use [`CommitBlockInfoV2`]. + let use_new_format = if let Some(block_number) = l1_block_number { + block_number >= BOOJUM_BLOCK + } else { + false + }; let abi::Token::Array(data) = data else { return Err(ParseError::InvalidCommitBlockInfo( @@ -478,12 +506,19 @@ fn parse_commit_block_info(data: &abi::Token) -> Result> .into()); }; + let mut result = vec![]; for d in data { - match CommitBlockInfoV1::try_from(d) { - Ok(blk) => res.push(blk), - Err(e) => tracing::error!("failed to parse commit block info: {e}"), - } + let block_info = { + if use_new_format { + CommitBlockInfo::V2(v2::CommitBlockInfo::try_from(d)?) + } else { + CommitBlockInfo::V1(v1::CommitBlockInfo::try_from(d)?) + } + }; + + let commit_block = CommitBlock::from_commit_block(block_info).await; + result.push(commit_block); } - Ok(res) + Ok(result) } diff --git a/state-reconstruct-fetcher/src/types/mod.rs b/state-reconstruct-fetcher/src/types/mod.rs new file mode 100644 index 0000000..59bbcd3 --- /dev/null +++ b/state-reconstruct-fetcher/src/types/mod.rs @@ -0,0 +1,106 @@ +use eyre::Result; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +pub mod v1; +pub mod v2; + +#[allow(clippy::enum_variant_names)] +#[derive(Error, Debug)] +pub enum ParseError { + #[error("invalid Calldata: {0}")] + InvalidCalldata(String), + + #[error("invalid StoredBlockInfo: {0}")] + InvalidStoredBlockInfo(String), + + #[error("invalid CommitBlockInfo: {0}")] + InvalidCommitBlockInfo(String), + + #[allow(dead_code)] + #[error("invalid compressed bytecode: {0}")] + InvalidCompressedByteCode(String), + + #[error("invalid compressed value: {0}")] + InvalidCompressedValue(String), +} + +/// Block with all required fields extracted from a [`CommitBlockInfo`]. +#[derive(Debug, Serialize, Deserialize)] +pub struct CommitBlock { + /// L1 block number. + pub l1_block_number: Option, + /// L2 block number. + pub l2_block_number: u64, + /// The state root of the full state tree. + pub new_state_root: Vec, + /// Storage write access as a concatenation key-value. + pub initial_storage_changes: IndexMap<[u8; 32], [u8; 32]>, + /// Storage write access as a concatenation index-value. + pub repeated_storage_changes: IndexMap, + /// (contract bytecodes) array of L2 bytecodes that were deployed. + pub factory_deps: Vec>, +} + +impl CommitBlock { + pub async fn from_commit_block(block_type: CommitBlockInfo) -> Self { + match block_type { + CommitBlockInfo::V1(block) => CommitBlock { + l1_block_number: None, + l2_block_number: block.block_number, + new_state_root: block.new_state_root, + initial_storage_changes: block.initial_storage_changes, + repeated_storage_changes: block.repeated_storage_changes, + factory_deps: block.factory_deps, + }, + CommitBlockInfo::V2(_block) => todo!(), + } + } +} + +#[derive(Debug)] +pub enum CommitBlockInfo { + V1(v1::CommitBlockInfo), + V2(v2::CommitBlockInfo), +} + +// TODO: Do we need this? +#[allow(dead_code)] +fn decompress_bytecode(data: &[u8]) -> Result> { + let dict_len = u16::from_be_bytes([data[0], data[1]]); + let end = 2 + dict_len as usize * 8; + let dict = data[2..end].to_vec(); + let encoded_data = data[end..].to_vec(); + + let dict: Vec<&[u8]> = dict.chunks(8).collect(); + + // Verify that dictionary size is below maximum. + if dict.len() > (1 << 16) + /* 2^16 */ + { + return Err(ParseError::InvalidCompressedByteCode(format!( + "too many elements in dictionary: {}", + dict.len() + )) + .into()); + } + + let mut bytecode = vec![]; + for idx in encoded_data.chunks(2) { + let idx = u16::from_be_bytes([idx[0], idx[1]]) as usize; + + if dict.len() <= idx { + return Err(ParseError::InvalidCompressedByteCode(format!( + "encoded data index ({}) exceeds dictionary size ({})", + idx, + dict.len() + )) + .into()); + } + + bytecode.append(&mut dict[idx].to_vec()); + } + + Ok(bytecode) +} diff --git a/state-reconstruct-fetcher/src/types.rs b/state-reconstruct-fetcher/src/types/v1.rs similarity index 71% rename from state-reconstruct-fetcher/src/types.rs rename to state-reconstruct-fetcher/src/types/v1.rs index fed6721..e1b865d 100644 --- a/state-reconstruct-fetcher/src/types.rs +++ b/state-reconstruct-fetcher/src/types/v1.rs @@ -1,35 +1,13 @@ -use std::vec::Vec; - use ethers::{abi, types::U256}; -use eyre::Result; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json_any_key::any_key_map; -use thiserror::Error; - -#[allow(clippy::enum_variant_names)] -#[derive(Error, Debug)] -pub enum ParseError { - #[error("invalid Calldata: {0}")] - InvalidCalldata(String), - - #[error("invalid StoredBlockInfo: {0}")] - InvalidStoredBlockInfo(String), - #[error("invalid CommitBlockInfo: {0}")] - InvalidCommitBlockInfo(String), - - #[allow(dead_code)] - #[error("invalid compressed bytecode: {0}")] - InvalidCompressedByteCode(String), -} +use super::ParseError; /// Data needed to commit new block #[derive(Debug, Serialize, Deserialize)] -pub struct CommitBlockInfoV1 { - /// L1 block number. - #[serde(skip_serializing)] - pub l1_block_number: Option, +pub struct CommitBlockInfo { /// L2 block number. pub block_number: u64, /// Unix timestamp denoting the start of the block execution. @@ -56,10 +34,10 @@ pub struct CommitBlockInfoV1 { pub factory_deps: Vec>, } -impl TryFrom<&abi::Token> for CommitBlockInfoV1 { +impl TryFrom<&abi::Token> for CommitBlockInfo { type Error = ParseError; - /// Try to parse Ethereum ABI token into [`CommitBlockInfoV1`]. + /// Try to parse Ethereum ABI token into [`CommitBlockInfo`]. /// /// * `token` - ABI token of `CommitBlockInfo` type on Ethereum. fn try_from(token: &abi::Token) -> Result { @@ -100,8 +78,7 @@ impl TryFrom<&abi::Token> for CommitBlockInfoV1 { (repeated_changes_calldata.len() - 4) / 40 ); - let mut blk = CommitBlockInfoV1 { - l1_block_number: None, + let mut blk = CommitBlockInfo { block_number: new_l2_block_number.as_u64(), timestamp: timestamp.as_u64(), index_repeated_storage_changes: new_enumeration_index, @@ -179,7 +156,7 @@ struct ExtractedToken { impl TryFrom<&abi::Token> for ExtractedToken { type Error = ParseError; - fn try_from(token: &abi::Token) -> std::result::Result { + fn try_from(token: &abi::Token) -> Result { let abi::Token::Tuple(block_elems) = token else { return Err(ParseError::InvalidCommitBlockInfo( "struct elements".to_string(), @@ -286,87 +263,3 @@ impl TryFrom<&abi::Token> for ExtractedToken { }) } } - -#[allow(dead_code)] -fn decompress_bytecode(data: &[u8]) -> Result> { - let dict_len = u16::from_be_bytes([data[0], data[1]]); - let end = 2 + dict_len as usize * 8; - let dict = data[2..end].to_vec(); - let encoded_data = data[end..].to_vec(); - - let dict: Vec<&[u8]> = dict.chunks(8).collect(); - - // Verify that dictionary size is below maximum. - if dict.len() > (1 << 16) - /* 2^16 */ - { - return Err(ParseError::InvalidCompressedByteCode(format!( - "too many elements in dictionary: {}", - dict.len() - )) - .into()); - } - - let mut bytecode = vec![]; - for idx in encoded_data.chunks(2) { - let idx = u16::from_be_bytes([idx[0], idx[1]]) as usize; - - if dict.len() <= idx { - return Err(ParseError::InvalidCompressedByteCode(format!( - "encoded data index ({}) exceeds dictionary size ({})", - idx, - dict.len() - )) - .into()); - } - - bytecode.append(&mut dict[idx].to_vec()); - } - - Ok(bytecode) -} - -#[allow(dead_code)] -pub enum L2ToL1Pubdata { - L2ToL1Log, - L2ToL2Message, - PublishedBytecode, - CompressedStateDiff, -} - -/// Data needed to commit new block -#[allow(dead_code)] -pub struct CommitBlockInfoV2 { - /// L2 block number. - pub block_number: u64, - /// Unix timestamp denoting the start of the block execution. - pub timestamp: u64, - /// The serial number of the shortcut index that's used as a unique identifier for storage keys that were used twice or more. - pub index_repeated_storage_changes: u64, - /// The state root of the full state tree. - pub new_state_root: Vec, - /// Number of priority operations to be processed. - pub number_of_l1_txs: U256, - /// Hash of all priority operations from this block. - pub priority_operations_hash: Vec, - /// Concatenation of all L2 -> L1 system logs in the block. - pub system_logs: Vec, - /// Total pubdata committed to as part of bootloader run. Contents are: l2Tol1Logs <> l2Tol1Messages <> publishedBytecodes <> stateDiffs. - pub total_l2_to_l1_pubdata: Vec, -} - -impl CommitBlockInfoV1 { - #[allow(dead_code)] - pub fn as_v2(&self) -> CommitBlockInfoV2 { - CommitBlockInfoV2 { - block_number: self.block_number, - timestamp: self.timestamp, - index_repeated_storage_changes: self.index_repeated_storage_changes, - new_state_root: self.new_state_root.clone(), - number_of_l1_txs: self.number_of_l1_txs, - priority_operations_hash: self.priority_operations_hash.clone(), - system_logs: vec![], - total_l2_to_l1_pubdata: vec![], - } - } -} diff --git a/state-reconstruct-fetcher/src/types/v2.rs b/state-reconstruct-fetcher/src/types/v2.rs new file mode 100644 index 0000000..2837d46 --- /dev/null +++ b/state-reconstruct-fetcher/src/types/v2.rs @@ -0,0 +1,319 @@ +use ethers::{abi, types::U256}; +use serde::{Deserialize, Serialize}; + +use super::ParseError; +use crate::constants::zksync::{ + L2_TO_L1_LOG_SERIALIZE_SIZE, LENGTH_BITS_OFFSET, OPERATION_BITMASK, +}; + +#[derive(Debug, Serialize, Deserialize)] +pub enum PackingType { + Add, + Sub, + Transform, + NoCompression, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum L2ToL1Pubdata { + L2ToL1Log(Vec), + L2ToL2Message(Vec), + PublishedBytecode(Vec), + CompressedStateDiff { + /// NOTE: Does this make sense? + is_reapeated_write: bool, + derived_key: U256, + compressed_value: U256, + packing_type: PackingType, + }, +} + +/// Data needed to commit new block +#[allow(dead_code)] +#[derive(Debug, Serialize, Deserialize)] +pub struct CommitBlockInfo { + /// L2 block number. + pub block_number: u64, + /// Unix timestamp denoting the start of the block execution. + pub timestamp: u64, + /// The serial number of the shortcut index that's used as a unique identifier for storage keys that were used twice or more. + pub index_repeated_storage_changes: u64, + /// The state root of the full state tree. + pub new_state_root: Vec, + /// Number of priority operations to be processed. + pub number_of_l1_txs: U256, + /// Hash of all priority operations from this block. + pub priority_operations_hash: Vec, + /// Concatenation of all L2 -> L1 system logs in the block. + pub system_logs: Vec, + /// Total pubdata committed to as part of bootloader run. Contents are: l2Tol1Logs <> l2Tol1Messages <> publishedBytecodes <> stateDiffs. + pub total_l2_to_l1_pubdata: Vec, +} + +impl TryFrom<&abi::Token> for CommitBlockInfo { + type Error = ParseError; + + /// Try to parse Ethereum ABI token into [`CommitBlockInfo`]. + /// + /// * `token` - ABI token of `CommitBlockInfo` type on Ethereum. + fn try_from(token: &abi::Token) -> Result { + let ExtractedToken { + new_l2_block_number, + timestamp, + new_enumeration_index, + state_root, + number_of_l1_txs, + priority_operations_hash, + system_logs, + total_l2_to_l1_pubdata, + } = token.try_into()?; + let new_enumeration_index = new_enumeration_index.as_u64(); + + let total_l2_to_l1_pubdata = parse_total_l2_to_l1_pubdata(total_l2_to_l1_pubdata)?; + let blk = CommitBlockInfo { + block_number: new_l2_block_number.as_u64(), + timestamp: timestamp.as_u64(), + index_repeated_storage_changes: new_enumeration_index, + new_state_root: state_root, + number_of_l1_txs, + priority_operations_hash, + system_logs, + total_l2_to_l1_pubdata, + }; + + Ok(blk) + } +} + +fn parse_total_l2_to_l1_pubdata(bytes: Vec) -> Result, ParseError> { + let mut l2_to_l1_pubdata = Vec::new(); + let mut pointer = 0; + + println!("total bytes: {}", bytes.len()); + println!("logs"); + // Skip over logs and messages. + let num_of_l1_to_l2_logs = u32::from_be_bytes(read_next_n_bytes(&bytes, &mut pointer)); + println!("num logs: {}", num_of_l1_to_l2_logs); + pointer += L2_TO_L1_LOG_SERIALIZE_SIZE * num_of_l1_to_l2_logs as usize; + + println!("pointer: {}", pointer); + println!("messages"); + let num_of_messages = u32::from_be_bytes(read_next_n_bytes(&bytes, &mut pointer)); + println!("num messages: {}", num_of_messages); + for i in 0..num_of_messages { + println!("message {}", i); + let current_message_len = u32::from_be_bytes(read_next_n_bytes(&bytes, &mut pointer)); + pointer += current_message_len as usize; + } + + println!("bytecodes"); + // Parse published bytecodes. + let num_of_bytecodes = u32::from_be_bytes(read_next_n_bytes(&bytes, &mut pointer)); + for _ in 0..num_of_bytecodes { + let current_bytecode_len = + u32::from_be_bytes(read_next_n_bytes(&bytes, &mut pointer)) as usize; + let mut bytecode = Vec::new(); + bytecode.copy_from_slice(&bytes[pointer..pointer + current_bytecode_len]); + pointer += current_bytecode_len; + l2_to_l1_pubdata.push(L2ToL1Pubdata::PublishedBytecode(bytecode)) + } + + println!("statediffs"); + // Parse compressed state diffs. + let mut state_diffs = parse_compressed_state_diffs(&bytes, &mut pointer)?; + l2_to_l1_pubdata.append(&mut state_diffs); + + Ok(l2_to_l1_pubdata) +} + +fn parse_compressed_state_diffs( + bytes: &[u8], + pointer: &mut usize, +) -> Result, ParseError> { + let mut state_diffs = Vec::new(); + println!("header"); + // Parse the header. + let _version = u8::from_be_bytes(read_next_n_bytes(bytes, pointer)); + + if *pointer >= bytes.len() { + return Ok(state_diffs); + } + + let mut buffer = [0; 4]; + buffer[..3].copy_from_slice(&bytes[*pointer..*pointer + 3]); + *pointer += 3; + let _total_compressed_len = u32::from_be_bytes(buffer); + + let enumeration_index = u8::from_be_bytes(read_next_n_bytes(bytes, pointer)); + + println!("initial writes"); + // Parse initial writes. + let num_of_initial_writes = u16::from_be_bytes(read_next_n_bytes(bytes, pointer)); + for _ in 0..num_of_initial_writes { + let derived_key = U256::from_little_endian(&read_next_n_bytes::<32>(bytes, pointer)); + + let (compressed_value, packing_type) = read_compressed_value(bytes, pointer)?; + state_diffs.push(L2ToL1Pubdata::CompressedStateDiff { + is_reapeated_write: false, + derived_key, + compressed_value, + packing_type, + }); + } + + println!("repeated writes"); + // Parse repeated writes. + while *pointer < bytes.len() { + println!("derived key"); + let derived_key = match enumeration_index { + 4 => U256::from_big_endian(&read_next_n_bytes::<4>(bytes, pointer)), + 5 => U256::from_big_endian(&read_next_n_bytes::<5>(bytes, pointer)), + _ => { + return Err(ParseError::InvalidCompressedValue(String::from( + "RepeatedDerivedKey", + ))) + } + }; + + let (compressed_value, packing_type) = read_compressed_value(bytes, pointer)?; + state_diffs.push(L2ToL1Pubdata::CompressedStateDiff { + is_reapeated_write: true, + derived_key, + compressed_value, + packing_type, + }); + } + + Ok(state_diffs) +} + +fn read_compressed_value( + bytes: &[u8], + pointer: &mut usize, +) -> Result<(U256, PackingType), ParseError> { + println!("metadata"); + let metadata = u8::from_be_bytes(read_next_n_bytes(bytes, pointer)); + let operation = metadata & OPERATION_BITMASK; + let len = if operation == 0 { + 32 + } else { + metadata >> LENGTH_BITS_OFFSET + } as usize; + + println!("packing type: {}", operation); + let packing_type = match operation { + 0 => PackingType::NoCompression, + 1 => PackingType::Add, + 2 => PackingType::Sub, + 3 => PackingType::Transform, + _ => { + return Err(ParseError::InvalidCompressedValue(String::from( + "UnknownPackingType", + ))) + } + }; + + println!("compressed value with len: {}", len); + // Read compressed value. + let mut buffer = [0; 32]; + buffer[..len].copy_from_slice(&bytes[*pointer..*pointer + len]); + *pointer += len; + let compressed_value = U256::from_big_endian(&buffer); + + Ok((compressed_value, packing_type)) +} + +fn read_next_n_bytes(bytes: &[u8], pointer: &mut usize) -> [u8; N] { + if *pointer >= bytes.len() { + return [0; N]; + } + let mut buffer = [0; N]; + buffer.copy_from_slice(&bytes[*pointer..*pointer + N]); + *pointer += N; + buffer +} + +struct ExtractedToken { + new_l2_block_number: U256, + timestamp: U256, + new_enumeration_index: U256, + state_root: Vec, + number_of_l1_txs: U256, + priority_operations_hash: Vec, + system_logs: Vec, + total_l2_to_l1_pubdata: Vec, +} + +impl TryFrom<&abi::Token> for ExtractedToken { + type Error = ParseError; + + fn try_from(token: &abi::Token) -> Result { + let abi::Token::Tuple(block_elems) = token else { + return Err(ParseError::InvalidCommitBlockInfo( + "struct elements".to_string(), + )); + }; + + let abi::Token::Uint(new_l2_block_number) = block_elems[0].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "blockNumber".to_string(), + )); + }; + + /* TODO(tuommaki): Fix the check below. + if new_l2_block_number <= latest_l2_block_number { + println!("skipping before we even get started"); + continue; + } + */ + + let abi::Token::Uint(timestamp) = block_elems[1].clone() else { + return Err(ParseError::InvalidCommitBlockInfo("timestamp".to_string())); + }; + + let abi::Token::Uint(new_enumeration_index) = block_elems[2].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "indexRepeatedStorageChanges".to_string(), + )); + }; + + let abi::Token::FixedBytes(state_root) = block_elems[3].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "newStateRoot".to_string(), + )); + }; + + let abi::Token::Uint(number_of_l1_txs) = block_elems[4].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "numberOfLayer1Txs".to_string(), + )); + }; + + let abi::Token::FixedBytes(priority_operations_hash) = block_elems[5].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "priorityOperationsHash".to_string(), + )); + }; + + let abi::Token::Bytes(system_logs) = block_elems[8].clone() else { + return Err(ParseError::InvalidCommitBlockInfo("systemLogs".to_string())); + }; + + let abi::Token::Bytes(total_l2_to_l1_pubdata) = block_elems[9].clone() else { + return Err(ParseError::InvalidCommitBlockInfo( + "totalL2ToL1Pubdata".to_string(), + )); + }; + + Ok(Self { + new_l2_block_number, + timestamp, + new_enumeration_index, + state_root, + number_of_l1_txs, + priority_operations_hash, + system_logs, + total_l2_to_l1_pubdata, + }) + } +}