From a8e42e989cc170fe91c47d4866bc35df3ca0e4c1 Mon Sep 17 00:00:00 2001 From: Trantorian Date: Tue, 27 Feb 2024 17:37:56 +0000 Subject: [PATCH 1/3] refactor(substrate_hash): :recycle: Substrate hash no longer retrieved via rpc call Substrate hash in `l2.rs` is now retrieved via use of the substrate rpc client created in `service.rs`. --- Cargo.lock | 1 + crates/client/deoxys/Cargo.toml | 1 + crates/client/deoxys/src/l2.rs | 108 ++++++++++++++--------------- crates/client/deoxys/src/lib.rs | 11 ++- crates/node/src/service.rs | 2 +- crates/pallets/starknet/src/lib.rs | 1 - 6 files changed, 65 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 034dad6c47..e6ca9b76db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5989,6 +5989,7 @@ dependencies = [ "sc-service", "serde", "serde_json", + "sp-blockchain", "sp-core", "sp-runtime", "starknet-core", diff --git a/crates/client/deoxys/Cargo.toml b/crates/client/deoxys/Cargo.toml index de23a0951c..60c17de86a 100644 --- a/crates/client/deoxys/Cargo.toml +++ b/crates/client/deoxys/Cargo.toml @@ -59,6 +59,7 @@ sc-executor = { workspace = true } sc-service = { workspace = true } sp-core = { workspace = true, features = ["std"] } sp-runtime = { workspace = true } +sp-blockchain = { workspace = true, default-features = true } mc-commitment-state-diff = { workspace = true } mc-rpc-core = { workspace = true } diff --git a/crates/client/deoxys/src/l2.rs b/crates/client/deoxys/src/l2.rs index a5a01f8a5e..6706473266 100644 --- a/crates/client/deoxys/src/l2.rs +++ b/crates/client/deoxys/src/l2.rs @@ -12,9 +12,10 @@ use mp_felt::Felt252Wrapper; use mp_storage::StarknetStorageSchemaVersion; use reqwest::Url; use serde::Deserialize; +use sp_blockchain::HeaderBackend; use sp_core::H256; use sp_runtime::generic::{Block, Header}; -use sp_runtime::traits::{BlakeTwo256, Block as BlockT}; +use sp_runtime::traits::{BlakeTwo256, Block as BlockT, UniqueSaturatedInto}; use sp_runtime::OpaqueExtrinsic; use starknet_api::api_core::ClassHash; use starknet_api::hash::StarkHash; @@ -27,7 +28,7 @@ use tokio::sync::mpsc::Sender; use tokio::task::JoinSet; use crate::commitments::lib::{build_commitment_state_diff, update_state_root}; -use crate::utility::{get_block_hash_by_number, update_highest_block_hash_and_number}; +use crate::utility::update_highest_block_hash_and_number; use crate::CommandSink; /// Contains the Starknet verified state on L2 @@ -105,45 +106,21 @@ pub struct SenderConfig { pub overrides: Arc, OpaqueExtrinsic>>>, } -// TODO: find a better place to store this -/// Stores a madara block hash and it's associated substrate hash. -pub struct BlockHashEquivalence { - pub madara: FieldElement, - pub substrate: Option, -} - -impl BlockHashEquivalence { - async fn new(state_update: &StateUpdate, block_number: u64, rpc_port: u16) -> Self { - // TODO: use an actual Substrate client to convert from Madara to Substrate block hash - let block_hash_madara = state_update.block_hash.unwrap(); - let block_hash_substrate = &get_block_hash_by_number(rpc_port, block_number).await; - - // WARNING: might causes issues related to eRFC 2497 (https://github.com/rust-lang/rust/issues/53667) - if block_number > 0 && let Some(block_hash_substrate) = block_hash_substrate { - BlockHashEquivalence { - madara: block_hash_madara, - substrate: Some(H256::from_str(&block_hash_substrate).unwrap()), - } - } else { - BlockHashEquivalence { - madara: block_hash_madara, - substrate: None, - } - } - } -} - /// Spawns workers to fetch blocks and state updates from the feeder. -pub async fn sync( +pub async fn sync( mut sender_config: SenderConfig, config: FetchConfig, start_at: u64, - rpc_port: u16, backend: Arc>, -) { + client: Arc, +) where + B: BlockT, + C: HeaderBackend, +{ update_config(&config); let SenderConfig { block_sender, state_update_sender, class_sender, command_sink, overrides } = &mut sender_config; - let client = SequencerGatewayProvider::new(config.gateway.clone(), config.feeder_gateway.clone(), config.chain_id); + let provider = + SequencerGatewayProvider::new(config.gateway.clone(), config.feeder_gateway.clone(), config.chain_id); let bonsai_dbs = BonsaiDbs { contract: Arc::clone(backend.bonsai_contract()), class: Arc::clone(backend.bonsai_class()), @@ -155,39 +132,39 @@ pub async fn sync( let mut got_state_update = false; let mut last_update_highest_block = tokio::time::Instant::now() - Duration::from_secs(20); if current_block_number == 0 { - let _ = fetch_genesis_state_update(&client, bonsai_dbs.clone()).await; + let _ = fetch_genesis_state_update(&provider, bonsai_dbs.clone()).await; } loop { if last_update_highest_block.elapsed() > Duration::from_secs(20) { last_update_highest_block = tokio::time::Instant::now(); - if let Err(e) = update_highest_block_hash_and_number(&client).await { + if let Err(e) = update_highest_block_hash_and_number(&provider).await { eprintln!("Failed to update highest block hash and number: {}", e); } } let (block, state_update) = match (got_block, got_state_update) { (false, false) => { - let block = fetch_block(&client, block_sender, current_block_number); + let block = fetch_block(&provider, block_sender, current_block_number); let state_update = fetch_state_and_class_update( - &client, + &provider, Arc::clone(&overrides), state_update_sender, class_sender, + Arc::clone(&client), current_block_number, - rpc_port, bonsai_dbs.clone(), ); tokio::join!(block, state_update) } - (false, true) => (fetch_block(&client, block_sender, current_block_number).await, Ok(())), + (false, true) => (fetch_block(&provider, block_sender, current_block_number).await, Ok(())), (true, false) => ( Ok(()), fetch_state_and_class_update( - &client, + &provider, Arc::clone(&overrides), state_update_sender, class_sender, + Arc::clone(&client), current_block_number, - rpc_port, bonsai_dbs.clone(), ) .await, @@ -243,17 +220,21 @@ pub async fn fetch_genesis_block(config: FetchConfig) -> Result( +async fn fetch_state_and_class_update( provider: &SequencerGatewayProvider, overrides: Arc, OpaqueExtrinsic>>>, state_update_sender: &Sender, class_sender: &Sender, + client: Arc, block_number: u64, - rpc_port: u16, bonsai_dbs: BonsaiDbs, -) -> Result<(), String> { +) -> Result<(), String> +where + B: BlockT, + C: HeaderBackend, +{ let state_update = fetch_state_update(&provider, block_number, bonsai_dbs).await?; - let class_update = fetch_class_update(&provider, &state_update, overrides, block_number, rpc_port).await?; + let class_update = fetch_class_update(&provider, &state_update, overrides, client, block_number).await?; // Now send state_update, which moves it. This will be received // by QueryBlockConsensusDataProvider in deoxys/crates/node/src/service.rs @@ -300,23 +281,25 @@ async fn fetch_genesis_state_update( } /// retrieves class updates from Starknet sequencer -async fn fetch_class_update( +async fn fetch_class_update( provider: &SequencerGatewayProvider, state_update: &StateUpdate, overrides: Arc, OpaqueExtrinsic>>>, + client: Arc, block_number: u64, - rpc_port: u16, -) -> Result, String> { - // defaults to downloading ALL classes if a substrate block hash could not be determined - let block_hash = BlockHashEquivalence::new(state_update, block_number - 1, rpc_port).await; - let missing_classes = match block_hash.substrate { - Some(block_hash_substrate) => fetch_missing_classes(state_update, overrides, block_hash_substrate), +) -> Result, String> +where + B: BlockT, + C: HeaderBackend, +{ + let missing_classes = match block_hash_substrate(client, block_number) { + Some(block_hash) => fetch_missing_classes(state_update, overrides, block_hash), None => aggregate_classes(state_update), }; let arc_provider = Arc::new(provider.clone()); let mut task_set = missing_classes.into_iter().fold(JoinSet::new(), |mut set, class_hash| { - set.spawn(download_class(*class_hash, block_hash.madara, Arc::clone(&arc_provider))); + set.spawn(download_class(*class_hash, block_hash_madara(state_update), Arc::clone(&arc_provider))); set }); @@ -341,6 +324,23 @@ async fn fetch_class_update( Ok(classes) } +/// Retrieves Madara block hash from state update +fn block_hash_madara(state_update: &StateUpdate) -> FieldElement { + state_update.block_hash.unwrap() +} + +/// Retrieves Substrate block hash from rpc client +fn block_hash_substrate(client: Arc, block_number: u64) -> Option +where + B: BlockT, + C: HeaderBackend, +{ + client + .hash(UniqueSaturatedInto::unique_saturated_into(block_number)) + .unwrap() + .map(|hash| H256::from_str(&hash.to_string()).unwrap()) +} + /// Downloads a class definition from the Starknet sequencer. Note that because /// of the current type hell this needs to be converted into a blockifier equivalent async fn download_class( diff --git a/crates/client/deoxys/src/lib.rs b/crates/client/deoxys/src/lib.rs index c9d5b77d1c..3086ff4979 100644 --- a/crates/client/deoxys/src/lib.rs +++ b/crates/client/deoxys/src/lib.rs @@ -20,22 +20,27 @@ pub mod starknet_sync_worker { use std::sync::Arc; use reqwest::Url; + use sp_blockchain::HeaderBackend; use sp_runtime::traits::Block as BlockT; use super::*; - pub async fn sync( + pub async fn sync( fetch_config: FetchConfig, sender_config: SenderConfig, rpc_port: u16, l1_url: Url, backend: Arc>, - ) { + client: Arc, + ) where + B: BlockT, + C: HeaderBackend, + { let first_block = utility::get_last_synced_block(rpc_port).await + 1; let _ = tokio::join!( l1::sync(l1_url.clone()), - l2::sync(sender_config, fetch_config.clone(), first_block, rpc_port, backend.clone()) + l2::sync(sender_config, fetch_config.clone(), first_block, backend, client) ); } } diff --git a/crates/node/src/service.rs b/crates/node/src/service.rs index b21a2ed0cb..b83a7093f8 100644 --- a/crates/node/src/service.rs +++ b/crates/node/src/service.rs @@ -451,7 +451,7 @@ pub fn new_full( task_manager.spawn_essential_handle().spawn( "starknet-sync-worker", Some("madara"), - starknet_sync_worker::sync(fetch_config, sender_config, rpc_port, l1_url, madara_backend), + starknet_sync_worker::sync(fetch_config, sender_config, rpc_port, l1_url, madara_backend, Arc::clone(&client)), ); task_manager.spawn_essential_handle().spawn( diff --git a/crates/pallets/starknet/src/lib.rs b/crates/pallets/starknet/src/lib.rs index 7312da2bbd..da643b4037 100644 --- a/crates/pallets/starknet/src/lib.rs +++ b/crates/pallets/starknet/src/lib.rs @@ -1063,7 +1063,6 @@ impl Pallet { let events: Vec = transaction_hashes.iter().flat_map(TxEvents::::take).collect(); let sequencer_address = Self::sequencer_address(); let block_timestamp = Self::block_timestamp(); - let chain_id = Self::chain_id(); let (transaction_commitment, event_commitment) = (Felt252Wrapper::default(), Felt252Wrapper::default()); let protocol_version = T::ProtocolVersion::get(); let extra_data = None; From 1ed101522d94106e2a1f7964f4acee25612388e7 Mon Sep 17 00:00:00 2001 From: Trantorian Date: Tue, 27 Feb 2024 17:40:00 +0000 Subject: [PATCH 2/3] chore(changelog): :green_heart: Updated `CHANGELOG.md` --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c18579816..303e810d19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ git # Madara Changelog ## Next release +- refactor(substrate_hash): Substrate hash is now retrieved via rpc client in `l2.rs` - perf(verify_l2): parallelized l2 state root update - perf(state_commitment): parallelized state commitment hash computations - fix(L1): fix l1 thread with battle tested implementation + removed l1-l2 From f165ef549b5596fc884d00fcbdfa5422154f40fe Mon Sep 17 00:00:00 2001 From: Trantorian Date: Tue, 27 Feb 2024 17:47:49 +0000 Subject: [PATCH 3/3] chore(lint): :rotating_light: Applied CI linting --- CHANGELOG.md | 3 +- Cargo.toml | 2 +- crates/client/db/Cargo.toml | 4 +- crates/client/deoxys/Cargo.toml | 18 ++++----- crates/client/rpc/Cargo.toml | 4 +- crates/node/Cargo.toml | 4 +- crates/pallets/starknet/Cargo.toml | 4 +- crates/primitives/block/Cargo.toml | 4 +- crates/primitives/contract/Cargo.toml | 58 +++++++++++++-------------- crates/primitives/felt/Cargo.toml | 2 +- crates/runtime/Cargo.toml | 4 +- 11 files changed, 54 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 303e810d19..d80b44363a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,8 @@ git # Madara Changelog ## Next release -- refactor(substrate_hash): Substrate hash is now retrieved via rpc client in `l2.rs` +- refactor(substrate_hash): Substrate hash is now retrieved via rpc client in + `l2.rs` - perf(verify_l2): parallelized l2 state root update - perf(state_commitment): parallelized state commitment hash computations - fix(L1): fix l1 thread with battle tested implementation + removed l1-l2 diff --git a/Cargo.toml b/Cargo.toml index 17bdaf323d..7b2e7e3f93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -294,7 +294,7 @@ cairo-lang-utils = { git = "https://github.com/keep-starknet-strange/cairo.git", ] } # Ethers: using the same versions as in Anvil -ethers = { git = "https://github.com/gakonst/ethers-rs"} +ethers = { git = "https://github.com/gakonst/ethers-rs" } # Zaun starknet-core-contract-client = { git = "https://github.com/keep-starknet-strange/zaun" } diff --git a/crates/client/db/Cargo.toml b/crates/client/db/Cargo.toml index 4afd997f16..259b96404d 100644 --- a/crates/client/db/Cargo.toml +++ b/crates/client/db/Cargo.toml @@ -16,7 +16,9 @@ repository = "https://github.com/keep-starknet-strange/madara" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +bonsai-trie = { workspace = true } ethers = { workspace = true } +kvdb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } log = { workspace = true, default-features = true } parity-db = { version = "0.4.12", optional = true } @@ -32,8 +34,6 @@ starknet_api = { workspace = true, default-features = true, features = [ ] } thiserror = { workspace = true } uuid = "1.4.1" -bonsai-trie = { workspace = true } -kvdb = "0.13.0" [features] default = ["kvdb-rocksdb", "parity-db"] diff --git a/crates/client/deoxys/Cargo.toml b/crates/client/deoxys/Cargo.toml index 60c17de86a..558dd7095e 100644 --- a/crates/client/deoxys/Cargo.toml +++ b/crates/client/deoxys/Cargo.toml @@ -18,16 +18,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] anyhow = "1.0.75" +ethers = { workspace = true } lazy_static = "1.4.0" reqwest = "0.11" serde_json = "1" tokio-tungstenite = "0.20.1" -ethers = { workspace = true } blockifier = { workspace = true, default-features = false, features = [ "testing", ] } +crossbeam-skiplist = { workspace = true } env_logger = "0.10.0" futures = { workspace = true, default-features = true } hex = "0.4" @@ -35,14 +36,13 @@ indexmap = { workspace = true } itertools = { workspace = true } log = { version = "0.4.14" } mockito = { workspace = true } +primitive-types = { version = "0.12.2" } rand = { version = "0.8.5" } -primitive-types = { version = "0.12.2"} rodio = { version = "0.17", optional = true } serde = { workspace = true, default-features = true } tokio = { workspace = true, features = ["macros", "parking_lot", "test-util"] } url = { workspace = true } validator = { workspace = true, features = ["derive"] } -crossbeam-skiplist ={ workspace = true } madara-runtime = { workspace = true } parity-scale-codec = { workspace = true, features = ["derive"] } @@ -57,11 +57,14 @@ starknet_api = { workspace = true, default-features = false } sc-consensus-manual-seal.workspace = true sc-executor = { workspace = true } sc-service = { workspace = true } +sp-blockchain = { workspace = true, default-features = true } sp-core = { workspace = true, features = ["std"] } sp-runtime = { workspace = true } -sp-blockchain = { workspace = true, default-features = true } +bitvec = "1.0.1" +bonsai-trie = { workspace = true } mc-commitment-state-diff = { workspace = true } +mc-db = { workspace = true } mc-rpc-core = { workspace = true } mc-storage = { workspace = true } mp-block = { workspace = true } @@ -71,13 +74,10 @@ mp-felt = { workspace = true } mp-hashers = { workspace = true } mp-storage = { workspace = true, default-features = true } mp-transactions = { workspace = true, features = ["client"] } -mc-db = { workspace = true } -bonsai-trie = { workspace = true } starknet-types-core = { workspace = true, default-features = false, features = [ - "hash", - "parity-scale-codec", + "hash", + "parity-scale-codec", ] } -bitvec = "1.0.1" [dev-dependencies] # test_utils = { path = "./test_utils" } diff --git a/crates/client/rpc/Cargo.toml b/crates/client/rpc/Cargo.toml index 64a77c80d7..b3476875d0 100644 --- a/crates/client/rpc/Cargo.toml +++ b/crates/client/rpc/Cargo.toml @@ -12,8 +12,8 @@ publish = false repository = "https://github.com/keep-starknet-strange/madara" version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] # Madara utils diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index d14692da2c..c8e00140ef 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -14,8 +14,8 @@ publish = false repository = "https://github.com/kasarlabs/deoxys" version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [[bin]] name = "deoxys" diff --git a/crates/pallets/starknet/Cargo.toml b/crates/pallets/starknet/Cargo.toml index f8a2a2ce03..19285fdc00 100644 --- a/crates/pallets/starknet/Cargo.toml +++ b/crates/pallets/starknet/Cargo.toml @@ -9,8 +9,8 @@ publish = false repository = "https://github.com/keep-starknet-strange/madara" version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] # Madara primitives diff --git a/crates/primitives/block/Cargo.toml b/crates/primitives/block/Cargo.toml index f4b3c6ef4a..c95461184c 100644 --- a/crates/primitives/block/Cargo.toml +++ b/crates/primitives/block/Cargo.toml @@ -7,8 +7,8 @@ name = "mp-block" repository = { workspace = true } version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] blockifier = { workspace = true } diff --git a/crates/primitives/contract/Cargo.toml b/crates/primitives/contract/Cargo.toml index f84763d25a..98c1d2d041 100644 --- a/crates/primitives/contract/Cargo.toml +++ b/crates/primitives/contract/Cargo.toml @@ -7,15 +7,15 @@ name = "mp-contract" repository.workspace = true version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] blockifier = { workspace = true } mp-fee = { workspace = true, features = [ - "parity-scale-codec", - "scale-info", - "serde", + "parity-scale-codec", + "scale-info", + "serde", ] } mp-felt = { workspace = true } mp-hashers = { workspace = true } @@ -30,38 +30,38 @@ starknet_api = { workspace = true } anyhow = { workspace = true, optional = true } mp-convert = { workspace = true, optional = true } parity-scale-codec = { workspace = true, features = [ - "derive", + "derive", ], optional = true } scale-info = { workspace = true, features = ["derive"], optional = true } [features] default = ["std"] parity-scale-codec = [ - "blockifier/parity-scale-codec", - "dep:parity-scale-codec", - "mp-fee/parity-scale-codec", - "mp-felt/parity-scale-codec", - "mp-transactions/parity-scale-codec", - "starknet_api/parity-scale-codec", + "blockifier/parity-scale-codec", + "dep:parity-scale-codec", + "mp-fee/parity-scale-codec", + "mp-felt/parity-scale-codec", + "mp-transactions/parity-scale-codec", + "starknet_api/parity-scale-codec", ] scale-info = [ - "blockifier/scale-info", - "dep:scale-info", - "mp-felt/scale-info", - "starknet_api/scale-info", + "blockifier/scale-info", + "dep:scale-info", + "mp-felt/scale-info", + "starknet_api/scale-info", ] std = [ - "blockifier/std", - "mp-fee/std", - "mp-felt/std", - "mp-hashers/std", - "mp-transactions/std", - "serde/std", - "sp-core/std", - "starknet_api/std", - # Optionals - "anyhow/std", - "mp-convert/std", - "parity-scale-codec?/std", - "scale-info?/std", + "blockifier/std", + "mp-fee/std", + "mp-felt/std", + "mp-hashers/std", + "mp-transactions/std", + "serde/std", + "sp-core/std", + "starknet_api/std", + # Optionals + "anyhow/std", + "mp-convert/std", + "parity-scale-codec?/std", + "scale-info?/std", ] diff --git a/crates/primitives/felt/Cargo.toml b/crates/primitives/felt/Cargo.toml index ce8e02f3c5..29048607ed 100644 --- a/crates/primitives/felt/Cargo.toml +++ b/crates/primitives/felt/Cargo.toml @@ -15,8 +15,8 @@ cairo-vm = { workspace = true } sp-core = { workspace = true } starknet-core = { workspace = true } starknet-ff = { workspace = true } -starknet_api = { workspace = true } starknet-types-core = { workspace = true } +starknet_api = { workspace = true } thiserror-no-std = { workspace = true } # Optional diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 3776bdb952..00febdcf17 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -12,8 +12,8 @@ publish = false repository = "https://github.com/keep-starknet-strange/madara" version.workspace = true - [package.metadata.docs.rs] - targets = ["x86_64-unknown-linux-gnu"] +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] [dependencies] parity-scale-codec = { workspace = true, features = [] }