From 653b3a9c053b0990b4b6c8126eb50909de989159 Mon Sep 17 00:00:00 2001 From: juan518munoz <62400508+juan518munoz@users.noreply.github.com> Date: Fri, 29 Dec 2023 14:52:31 -0300 Subject: [PATCH] feat: Add Chain Data tables (#63) * add block headers & chain mmr nodes table --------- Co-authored-by: Nacho Co-authored-by: igamigo --- src/cli/account.rs | 2 +- src/cli/input_notes.rs | 3 +- src/cli/transactions.rs | 1 + src/client/accounts.rs | 12 +- src/client/chain_data.rs | 24 ++++ src/client/mod.rs | 29 ++-- src/client/sync_state.rs | 36 ++++- src/client/transactions.rs | 13 +- src/errors.rs | 29 +++- src/mock.rs | 27 +++- src/store/accounts.rs | 33 ++--- src/store/chain_data.rs | 189 ++++++++++++++++++++++++++ src/store/mock_executor_data_store.rs | 13 +- src/store/mod.rs | 5 +- src/store/notes.rs | 89 ------------ src/store/state_sync.rs | 164 ++++++++++++++++++++++ src/store/store.sql | 26 +++- src/tests.rs | 11 +- 18 files changed, 546 insertions(+), 160 deletions(-) create mode 100644 src/client/chain_data.rs create mode 100644 src/store/chain_data.rs create mode 100644 src/store/state_sync.rs diff --git a/src/cli/account.rs b/src/cli/account.rs index 6c0dccfe9..db7354bc8 100644 --- a/src/cli/account.rs +++ b/src/cli/account.rs @@ -216,7 +216,7 @@ pub fn show_account( println!( "Storage: {}\n", - serde_json::to_string(&account_storage) + serde_json::to_string(&account_storage.slots()) .map_err(|_| "Error serializing account storage")? ); } diff --git a/src/cli/input_notes.rs b/src/cli/input_notes.rs index 385ce380c..6ea187cce 100644 --- a/src/cli/input_notes.rs +++ b/src/cli/input_notes.rs @@ -4,8 +4,9 @@ use std::path::PathBuf; use super::{Client, Parser}; use comfy_table::{presets, Attribute, Cell, ContentArrangement, Table}; -use crypto::utils::{Deserializable, Serializable}; use miden_client::store::notes::InputNoteFilter; + +use crypto::utils::{Deserializable, Serializable}; use objects::notes::RecordedNote; use objects::Digest; diff --git a/src/cli/transactions.rs b/src/cli/transactions.rs index a824c97bd..766157aa9 100644 --- a/src/cli/transactions.rs +++ b/src/cli/transactions.rs @@ -3,6 +3,7 @@ use comfy_table::Attribute; use comfy_table::Cell; use comfy_table::ContentArrangement; use comfy_table::Table; + use miden_client::client::transactions::PaymentTransactionData; use miden_client::client::transactions::TransactionStub; use miden_client::client::transactions::TransactionTemplate; diff --git a/src/client/accounts.rs b/src/client/accounts.rs index c4eff39a5..4a36b704a 100644 --- a/src/client/accounts.rs +++ b/src/client/accounts.rs @@ -1,11 +1,8 @@ use super::Client; - -use std::collections::BTreeMap; - -use crypto::{Felt, Word}; +use crypto::Felt; use miden_lib::{faucets, AuthScheme}; use objects::{ - accounts::{Account, AccountId, AccountStub, AccountType}, + accounts::{Account, AccountId, AccountStorage, AccountStub, AccountType}, assembly::ModuleAst, assets::{Asset, TokenSymbol}, Digest, @@ -188,10 +185,7 @@ impl Client { } /// Returns account storage data from a storage root. - pub fn get_account_storage( - &self, - storage_root: Digest, - ) -> Result, ClientError> { + pub fn get_account_storage(&self, storage_root: Digest) -> Result { self.store .get_account_storage(storage_root) .map_err(|err| err.into()) diff --git a/src/client/chain_data.rs b/src/client/chain_data.rs new file mode 100644 index 000000000..c3d4a8b6a --- /dev/null +++ b/src/client/chain_data.rs @@ -0,0 +1,24 @@ +use super::Client; + +#[cfg(test)] +use crate::errors::ClientError; +#[cfg(test)] +use objects::BlockHeader; + +impl Client { + #[cfg(test)] + pub fn get_block_headers( + &self, + start: u32, + finish: u32, + ) -> Result, ClientError> { + let mut headers = Vec::new(); + for block_number in start..=finish { + if let Ok(block_header) = self.store.get_block_header_by_num(block_number) { + headers.push(block_header) + } + } + + Ok(headers) + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs index 15df1cf56..60ec92fbc 100644 --- a/src/client/mod.rs +++ b/src/client/mod.rs @@ -7,15 +7,13 @@ use crate::{ store::{mock_executor_data_store::MockDataStore, Store}, }; -#[cfg(not(any(test, feature = "testing")))] -use crate::errors::RpcApiError; - use miden_tx::TransactionExecutor; -#[cfg(any(test, feature = "testing"))] +#[cfg(feature = "testing")] use crate::mock::MockRpcApi; pub mod accounts; +pub mod chain_data; pub mod notes; pub mod sync_state; pub mod transactions; @@ -37,11 +35,9 @@ pub const FILTER_ID_SHIFT: u8 = 48; pub struct Client { /// Local database containing information about the accounts managed by this client. pub(crate) store: Store, - #[cfg(not(any(test, feature = "testing")))] - /// Api client for interacting with the Miden node. - rpc_api: miden_node_proto::rpc::api_client::ApiClient, #[cfg(any(test, feature = "testing"))] pub rpc_api: MockRpcApi, + #[cfg(any(test, feature = "testing"))] pub(crate) tx_executor: TransactionExecutor, } @@ -54,17 +50,26 @@ impl Client { /// # Errors /// Returns an error if the client could not be instantiated. pub async fn new(config: ClientConfig) -> Result { - Ok(Self { + #[cfg(not(any(test, feature = "testing")))] + return Ok(Self { store: Store::new((&config).into())?, - #[cfg(not(any(test, feature = "testing")))] rpc_api: miden_node_proto::rpc::api_client::ApiClient::connect( config.node_endpoint.to_string(), ) .await - .map_err(|err| ClientError::RpcApiError(RpcApiError::ConnectionError(err)))?, - #[cfg(any(test, feature = "testing"))] + .map_err(|err| { + ClientError::RpcApiError(crate::errors::RpcApiError::ConnectionError(err)) + })?, + tx_executor: TransactionExecutor::new(crate::store::data_store::SqliteDataStore::new( + Store::new((&config).into())?, + )), + }); + + #[cfg(any(test, feature = "testing"))] + return Ok(Self { + store: Store::new((&config).into())?, rpc_api: Default::default(), tx_executor: TransactionExecutor::new(MockDataStore::new()), - }) + }); } } diff --git a/src/client/sync_state.rs b/src/client/sync_state.rs index 4322394f9..1a6d5a2c6 100644 --- a/src/client/sync_state.rs +++ b/src/client/sync_state.rs @@ -1,5 +1,4 @@ use super::Client; - use crypto::StarkField; use miden_node_proto::{ account_id::AccountId as ProtoAccountId, requests::SyncStateRequest, @@ -9,6 +8,11 @@ use objects::{accounts::AccountId, Digest}; use crate::errors::{ClientError, RpcApiError}; +pub enum SyncStatus { + SyncedToLastBlock(u32), + SyncedToBlock(u32), +} + // CONSTANTS // ================================================================================================ @@ -47,16 +51,25 @@ impl Client { /// /// Returns the block number the client has been synced to. pub async fn sync_state(&mut self) -> Result { + loop { + let response = self.single_sync_state().await?; + if let SyncStatus::SyncedToLastBlock(v) = response { + return Ok(v); + } + } + } + + async fn single_sync_state(&mut self) -> Result { let block_num = self.store.get_latest_block_number()?; let account_ids = self.store.get_account_ids()?; let note_tags = self.store.get_note_tags()?; - let nullifiers = self.store.get_unspent_input_note_nullifiers()?; // breaks - + let nullifiers = self.store.get_unspent_input_note_nullifiers()?; let response = self .sync_state_request(block_num, &account_ids, ¬e_tags, &nullifiers) .await?; + let incoming_block_header = response.block_header.unwrap(); - let new_block_num = response.chain_tip; + let new_block_num = incoming_block_header.block_num; let new_nullifiers = response .nullifiers .into_iter() @@ -71,10 +84,21 @@ impl Client { .collect::>(); self.store - .apply_state_sync(new_block_num, new_nullifiers) + .apply_state_sync( + incoming_block_header + .try_into() + .map_err(ClientError::RpcTypeConversionFailure)?, + new_nullifiers, + response.accounts, + response.mmr_delta, + ) .map_err(ClientError::StoreError)?; - Ok(new_block_num) + if response.chain_tip == new_block_num { + Ok(SyncStatus::SyncedToLastBlock(response.chain_tip)) + } else { + Ok(SyncStatus::SyncedToBlock(new_block_num)) + } } // HELPERS diff --git a/src/client/transactions.rs b/src/client/transactions.rs index 1e6e68681..fb2641fe8 100644 --- a/src/client/transactions.rs +++ b/src/client/transactions.rs @@ -1,3 +1,7 @@ +use crate::{ + errors::{self, ClientError}, + store::mock_executor_data_store::{self, MockDataStore}, +}; use crypto::utils::Serializable; use miden_lib::notes::{create_note, Script}; use miden_node_proto::{ @@ -14,11 +18,6 @@ use objects::{ }; use rand::Rng; -use crate::{ - errors::{self, ClientError}, - store::{self, mock_executor_data_store::MockDataStore}, -}; - use super::Client; pub enum TransactionTemplate { @@ -143,8 +142,8 @@ impl Client { ) -> Result<(TransactionResult, TransactionScript), ClientError> { // Create assets let (target_pub_key, target_sk_pk_felt) = - store::mock_executor_data_store::get_new_key_pair_with_advice_map(); - let target_account = store::mock_executor_data_store::get_account_with_default_account_code( + mock_executor_data_store::get_new_key_pair_with_advice_map(); + let target_account = mock_executor_data_store::get_account_with_default_account_code( target_account_id, target_pub_key, None, diff --git a/src/errors.rs b/src/errors.rs index 77f51ec34..fd0e72712 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,9 +1,10 @@ use core::fmt; -use crypto::{ - dsa::rpo_falcon512::FalconError, - utils::{DeserializationError, HexParseError}, -}; +use crypto::merkle::MmrError; +use crypto::utils::DeserializationError; +use crypto::{dsa::rpo_falcon512::FalconError, utils::HexParseError}; +use miden_node_proto::error::ParseError; use miden_tx::{TransactionExecutorError, TransactionProverError}; +use objects::AssetError; use objects::{accounts::AccountId, AccountError, Digest, NoteError, TransactionScriptError}; use tonic::{transport::Error as TransportError, Status as TonicStatus}; @@ -13,7 +14,9 @@ use tonic::{transport::Error as TransportError, Status as TonicStatus}; #[derive(Debug)] pub enum ClientError { AccountError(AccountError), + AssetError(AssetError), AuthError(FalconError), + RpcTypeConversionFailure(ParseError), NoteError(NoteError), RpcApiError(RpcApiError), StoreError(StoreError), @@ -25,7 +28,11 @@ impl fmt::Display for ClientError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { ClientError::AccountError(err) => write!(f, "account error: {err}"), + ClientError::AssetError(err) => write!(f, "asset error: {err}"), ClientError::AuthError(err) => write!(f, "account auth error: {err}"), + ClientError::RpcTypeConversionFailure(err) => { + write!(f, "failed to convert data: {err}") + } ClientError::NoteError(err) => write!(f, "note error: {err}"), ClientError::RpcApiError(err) => write!(f, "rpc api error: {err}"), ClientError::StoreError(err) => write!(f, "store error: {err}"), @@ -56,7 +63,9 @@ pub enum StoreError { AccountCodeDataNotFound(Digest), AccountDataNotFound(AccountId), AccountError(AccountError), + AccountHashMismatch(AccountId), AccountStorageNotFound(Digest), + ChainMmrNodeNotFound(u64), ColumnParsingError(rusqlite::Error), ConnectionError(rusqlite::Error), DataDeserializationError(DeserializationError), @@ -65,9 +74,11 @@ pub enum StoreError { InputSerializationError(serde_json::Error), JsonDataDeserializationError(serde_json::Error), MigrationError(rusqlite_migration::Error), + MmrError(MmrError), NoteTagAlreadyTracked(u64), QueryError(rusqlite::Error), TransactionError(rusqlite::Error), + BlockHeaderNotFound(u32), TransactionScriptError(TransactionScriptError), VaultDataNotFound(Digest), } @@ -83,12 +94,18 @@ impl fmt::Display for StoreError { write!(f, "Account data was not found for Account Id {account_id}") } AccountError(err) => write!(f, "error instantiating Account: {err}"), + AccountHashMismatch(account_id) => { + write!(f, "account hash mismatch for account {account_id}") + } AccountStorageNotFound(root) => { write!(f, "account storage data with root {} not found", root) } ColumnParsingError(err) => { write!(f, "failed to parse data retrieved from the database: {err}") } + ChainMmrNodeNotFound(node_index) => { + write!(f, "chain mmr node at index {} not found", node_index) + } ConnectionError(err) => write!(f, "failed to connect to the database: {err}"), DataDeserializationError(err) => { write!(f, "error deserializing data from the store: {err}") @@ -107,6 +124,7 @@ impl fmt::Display for StoreError { ) } MigrationError(err) => write!(f, "failed to update the database: {err}"), + MmrError(err) => write!(f, "error constructing mmr: {err}"), NoteTagAlreadyTracked(tag) => write!(f, "note tag {} is already being tracked", tag), QueryError(err) => write!(f, "failed to retrieve data from the database: {err}"), TransactionError(err) => write!(f, "failed to instantiate a new transaction: {err}"), @@ -114,6 +132,9 @@ impl fmt::Display for StoreError { write!(f, "error instantiating transaction script: {err}") } VaultDataNotFound(root) => write!(f, "account vault data for root {} not found", root), + BlockHeaderNotFound(block_num) => { + write!(f, "block header for block {} not found", block_num) + } } } } diff --git a/src/mock.rs b/src/mock.rs index a19a15ad2..c0ee5b982 100644 --- a/src/mock.rs +++ b/src/mock.rs @@ -2,7 +2,11 @@ use crate::client::transactions::{PaymentTransactionData, TransactionTemplate}; use crate::client::{Client, FILTER_ID_SHIFT}; use crate::store::mock_executor_data_store::MockDataStore; use crypto::{dsa::rpo_falcon512::KeyPair, StarkField}; +use crypto::{Felt, FieldElement}; +use miden_lib::assembler::assembler; use miden_node_proto::block_header::BlockHeader as NodeBlockHeader; +use miden_node_proto::merkle::MerklePath; +use miden_node_proto::note::NoteSyncRecord; use miden_node_proto::requests::SubmitProvenTransactionRequest; use miden_node_proto::responses::SubmitProvenTransactionResponse; use miden_node_proto::{ @@ -10,8 +14,11 @@ use miden_node_proto::{ requests::SyncStateRequest, responses::{NullifierUpdate, SyncStateResponse}, }; +use mock::mock::account::mock_account; use mock::mock::block; +use mock::mock::notes::mock_notes; use objects::utils::collections::BTreeMap; +use objects::AdviceInputs; use crate::store::accounts::AuthInfo; @@ -88,6 +95,16 @@ fn generate_sync_state_mock_requests() -> BTreeMap BTreeMap); type SerializedAccountCodeParts = (String, String, String); -type SerializedAccountStorageData = (String, String); -type SerializedAccountStorageParts = (String, String); +type SerializedAccountStorageData = (String, Vec); +type SerializedAccountStorageParts = (String, Vec); // DATABASE AUTH INFO // ================================================================================================ @@ -179,7 +179,7 @@ impl Store { } /// Retrieve account storage data by vault root - pub fn get_account_storage(&self, root: RpoDigest) -> Result, StoreError> { + pub fn get_account_storage(&self, root: RpoDigest) -> Result { let root_serialized = serde_json::to_string(&root).map_err(StoreError::InputSerializationError)?; @@ -300,7 +300,7 @@ impl Store { // ================================================================================================ /// Parse accounts colums from the provided row into native types -fn parse_accounts_columns( +pub(crate) fn parse_accounts_columns( row: &rusqlite::Row<'_>, ) -> Result { let id: i64 = row.get(0)?; @@ -312,7 +312,7 @@ fn parse_accounts_columns( } /// Parse an account from the provided parts. -fn parse_accounts( +pub(crate) fn parse_accounts( serialized_account_parts: SerializedAccountsParts, ) -> Result { let (id, nonce, vault_root, storage_root, code_root) = serialized_account_parts; @@ -422,18 +422,19 @@ fn parse_account_storage_columns( row: &rusqlite::Row<'_>, ) -> Result { let root: String = row.get(0)?; - let slots: String = row.get(1)?; - Ok((root, slots)) + let storage: Vec = row.get(1)?; + Ok((root, storage)) } /// Parse an account_storage from the provided parts. fn parse_account_storage( serialized_account_storage_parts: SerializedAccountStorageParts, -) -> Result, StoreError> { - let (_, slots) = serialized_account_storage_parts; +) -> Result { + let (_, storage) = serialized_account_storage_parts; - let slots = serde_json::from_str(&slots).map_err(StoreError::JsonDataDeserializationError)?; - Ok(slots) + let storage = + AccountStorage::read_from_bytes(&storage).map_err(StoreError::DataDeserializationError)?; + Ok(storage) } /// Serialize the provided account_storage into database compatible types. @@ -442,9 +443,9 @@ fn serialize_account_storage( ) -> Result { let root = serde_json::to_string(&account_storage.root()) .map_err(StoreError::InputSerializationError)?; - let slots: BTreeMap = account_storage.slots().leaves().collect(); - let slots = serde_json::to_string(&slots).map_err(StoreError::InputSerializationError)?; - Ok((root, slots)) + let storage = account_storage.to_bytes(); + + Ok((root, storage)) } /// Parse account_vault columns from the provided row into native types. @@ -458,7 +459,7 @@ fn parse_account_vault_columns( /// Parse a vector of assets from the provided parts. fn parse_account_vault( - serialized_account_vault_parts: SerializedAccountStorageParts, + serialized_account_vault_parts: SerializedAccountVaultParts, ) -> Result, StoreError> { let (_, assets) = serialized_account_vault_parts; diff --git a/src/store/chain_data.rs b/src/store/chain_data.rs new file mode 100644 index 000000000..5d37d4a89 --- /dev/null +++ b/src/store/chain_data.rs @@ -0,0 +1,189 @@ +use std::{collections::BTreeMap, num::NonZeroUsize}; + +use super::Store; + +use crate::errors::StoreError; + +use clap::error::Result; + +use crypto::merkle::InOrderIndex; +use objects::{BlockHeader, Digest}; +use rusqlite::{params, Transaction}; + +type SerializedBlockHeaderData = (i64, String, String, String, String, i64); +type SerializedBlockHeaderParts = (u64, String, String, String, String, u64); + +type SerializedChainMmrNodeData = (i64, String); +type SerializedChainMmrNodeParts = (u64, String); + +impl Store { + // CHAIN DATA + // -------------------------------------------------------------------------------------------- + pub fn insert_block_header( + tx: &Transaction<'_>, + block_header: BlockHeader, + chain_mmr_peaks: Vec, + forest: u64, + ) -> Result<(), StoreError> { + let (block_num, header, notes_root, sub_hash, chain_mmr, forest) = + serialize_block_header(block_header, chain_mmr_peaks, forest)?; + + const QUERY: &str = "\ + INSERT INTO block_headers + (block_num, header, notes_root, sub_hash, chain_mmr, forest) + VALUES (?, ?, ?, ?, ?, ?)"; + + tx.execute( + QUERY, + params![block_num, header, notes_root, sub_hash, chain_mmr, forest], + ) + .map_err(StoreError::QueryError) + .map(|_| ()) + } + + #[cfg(test)] + pub fn get_block_header_by_num(&self, block_number: u32) -> Result { + const QUERY: &str = "SELECT block_num, header, notes_root, sub_hash, chain_mmr FROM block_headers WHERE block_num = ?"; + self.db + .prepare(QUERY) + .map_err(StoreError::QueryError)? + .query_map(params![block_number as i64], parse_block_headers_columns) + .map_err(StoreError::QueryError)? + .map(|result| { + result + .map_err(StoreError::ColumnParsingError) + .and_then(parse_block_header) + }) + .next() + .ok_or(StoreError::BlockHeaderNotFound(block_number))? + } + + pub(crate) fn insert_chain_mmr_node( + tx: &Transaction<'_>, + id: InOrderIndex, + node: Digest, + ) -> Result<(), StoreError> { + let (id, node) = serialize_chain_mmr_node(id, node)?; + + const QUERY: &str = "INSERT INTO chain_mmr_nodes (id, node) VALUES (?, ?)"; + + tx.execute(QUERY, params![id, node]) + .map_err(StoreError::QueryError) + .map(|_| ()) + } + + pub fn insert_chain_mmr_nodes( + tx: &Transaction<'_>, + nodes: Vec<(InOrderIndex, Digest)>, + ) -> Result<(), StoreError> { + for (index, node) in nodes { + Self::insert_chain_mmr_node(tx, index, node)?; + } + + Ok(()) + } + + /// Returns all nodes in the table. + pub fn get_chain_mmr_nodes( + tx: &Transaction<'_>, + ) -> Result, StoreError> { + const QUERY: &str = "SELECT id, node FROM chain_mmr_nodes"; + tx.prepare(QUERY) + .map_err(StoreError::QueryError)? + .query_map(params![], parse_chain_mmr_nodes_columns) + .map_err(StoreError::QueryError)? + .map(|result| { + result + .map_err(StoreError::ColumnParsingError) + .and_then(parse_chain_mmr_nodes) + }) + .collect() + } +} + +// HELPERS +// ================================================================================================ + +fn serialize_block_header( + block_header: BlockHeader, + chain_mmr_peaks: Vec, + forest: u64, +) -> Result { + let block_num = block_header.block_num(); + let header = + serde_json::to_string(&block_header).map_err(StoreError::InputSerializationError)?; + let notes_root = serde_json::to_string(&block_header.note_root()) + .map_err(StoreError::InputSerializationError)?; + let sub_hash = serde_json::to_string(&block_header.sub_hash()) + .map_err(StoreError::InputSerializationError)?; + let chain_mmr_peaks = + serde_json::to_string(&chain_mmr_peaks).map_err(StoreError::InputSerializationError)?; + + Ok(( + block_num as i64, + header, + notes_root, + sub_hash, + chain_mmr_peaks, + forest as i64, + )) +} + +// Unused until we need to query the block headers table +#[allow(dead_code)] +fn parse_block_headers_columns( + row: &rusqlite::Row<'_>, +) -> Result { + let block_num: i64 = row.get(0)?; + let header: String = row.get(1)?; + let notes_root: String = row.get(2)?; + let sub_hash: String = row.get(3)?; + let chain_mmr: String = row.get(4)?; + let forest: i64 = row.get(5)?; + Ok(( + block_num as u64, + header, + notes_root, + sub_hash, + chain_mmr, + forest as u64, + )) +} + +// Unused until we need to query the block headers table +#[allow(dead_code)] +fn parse_block_header( + serialized_block_header_parts: SerializedBlockHeaderParts, +) -> Result { + let (_, header, _, _, _, _) = serialized_block_header_parts; + + serde_json::from_str(&header).map_err(StoreError::JsonDataDeserializationError) +} + +fn serialize_chain_mmr_node( + id: InOrderIndex, + node: Digest, +) -> Result { + let id: u64 = id.into(); + let node = serde_json::to_string(&node).map_err(StoreError::InputSerializationError)?; + Ok((id as i64, node)) +} + +fn parse_chain_mmr_nodes_columns( + row: &rusqlite::Row<'_>, +) -> Result { + let id: i64 = row.get(0)?; + let node = row.get(1)?; + Ok((id as u64, node)) +} + +fn parse_chain_mmr_nodes( + serialized_chain_mmr_node_parts: SerializedChainMmrNodeParts, +) -> Result<(InOrderIndex, Digest), StoreError> { + let (id, node) = serialized_chain_mmr_node_parts; + + let id = InOrderIndex::new(NonZeroUsize::new(id as usize).unwrap()); + let node: Digest = + serde_json::from_str(&node).map_err(StoreError::JsonDataDeserializationError)?; + Ok((id, node)) +} diff --git a/src/store/mock_executor_data_store.rs b/src/store/mock_executor_data_store.rs index f44d4d86f..ecd61c5eb 100644 --- a/src/store/mock_executor_data_store.rs +++ b/src/store/mock_executor_data_store.rs @@ -1,11 +1,11 @@ use miden_lib::assembler::assembler; use miden_tx::{DataStore, DataStoreError}; -use mock::{ - constants::{ACCOUNT_ID_SENDER, DEFAULT_ACCOUNT_CODE}, - mock::account::MockAccountType, - mock::notes::AssetPreservationStatus, - mock::transaction::{mock_inputs, mock_inputs_with_existing}, -}; +use mock::constants::{ACCOUNT_ID_SENDER, DEFAULT_ACCOUNT_CODE}; +use mock::mock::account::MockAccountType; +use mock::mock::notes::AssetPreservationStatus; +use mock::mock::transaction::{mock_inputs, mock_inputs_with_existing}; +use objects::transaction::ChainMmr; +use objects::AdviceInputs; use objects::{ accounts::{Account, AccountCode, AccountId, AccountStorage, AccountVault, StorageSlotType}, assembly::ModuleAst, @@ -15,7 +15,6 @@ use objects::{ notes::{Note, NoteOrigin, NoteScript, RecordedNote}, BlockHeader, Felt, Word, }; -use objects::{transaction::ChainMmr, AdviceInputs}; // MOCK DATA STORE // ================================================================================================ diff --git a/src/store/mod.rs b/src/store/mod.rs index 38436013c..88699caff 100644 --- a/src/store/mod.rs +++ b/src/store/mod.rs @@ -5,11 +5,14 @@ use clap::error::Result; use rusqlite::Connection; pub mod accounts; +pub mod chain_data; mod migrations; pub mod notes; +pub mod state_sync; pub mod transactions; -pub(crate) mod mock_executor_data_store; +#[cfg(any(test, feature = "testing"))] +pub mod mock_executor_data_store; // CLIENT STORE // ================================================================================================ diff --git a/src/store/notes.rs b/src/store/notes.rs index 6f304bc64..73b936efe 100644 --- a/src/store/notes.rs +++ b/src/store/notes.rs @@ -160,95 +160,6 @@ impl Store { }) .collect::, _>>() } - - // STATE SYNC - // -------------------------------------------------------------------------------------------- - - /// Returns the note tags that the client is interested in. - pub fn get_note_tags(&self) -> Result, StoreError> { - const QUERY: &str = "SELECT tags FROM state_sync"; - - self.db - .prepare(QUERY) - .map_err(StoreError::QueryError)? - .query_map([], |row| row.get(0)) - .expect("no binding parameters used in query") - .map(|result| { - result - .map_err(StoreError::ColumnParsingError) - .and_then(|v: String| { - serde_json::from_str(&v).map_err(StoreError::JsonDataDeserializationError) - }) - }) - .next() - .expect("state sync tags exist") - } - - /// Adds a note tag to the list of tags that the client is interested in. - pub fn add_note_tag(&mut self, tag: u64) -> Result { - let mut tags = self.get_note_tags()?; - if tags.contains(&tag) { - return Ok(false); - } - tags.push(tag); - let tags = serde_json::to_string(&tags).map_err(StoreError::InputSerializationError)?; - - const QUERY: &str = "UPDATE state_sync SET tags = ?"; - self.db - .execute(QUERY, params![tags]) - .map_err(StoreError::QueryError) - .map(|_| ())?; - - Ok(true) - } - - /// Returns the block number of the last state sync block - pub fn get_latest_block_number(&self) -> Result { - const QUERY: &str = "SELECT block_number FROM state_sync"; - - self.db - .prepare(QUERY) - .map_err(StoreError::QueryError)? - .query_map([], |row| row.get(0)) - .expect("no binding parameters used in query") - .map(|result| { - result - .map_err(StoreError::ColumnParsingError) - .map(|v: i64| v as u32) - }) - .next() - .expect("state sync block number exists") - } - - pub fn apply_state_sync( - &mut self, - block_number: u32, - nullifiers: Vec, - ) -> Result<(), StoreError> { - let tx = self - .db - .transaction() - .map_err(StoreError::TransactionError)?; - - // update state sync block number - const BLOCK_NUMBER_QUERY: &str = "UPDATE state_sync SET block_number = ?"; - tx.execute(BLOCK_NUMBER_QUERY, params![block_number]) - .map_err(StoreError::QueryError)?; - - // update spent notes - for nullifier in nullifiers { - const SPENT_QUERY: &str = - "UPDATE input_notes SET status = 'consumed' WHERE nullifier = ?"; - let nullifier = nullifier.to_string(); - tx.execute(SPENT_QUERY, params![nullifier]) - .map_err(StoreError::QueryError)?; - } - - // commit the updates - tx.commit().map_err(StoreError::QueryError)?; - - Ok(()) - } } // HELPERS diff --git a/src/store/state_sync.rs b/src/store/state_sync.rs new file mode 100644 index 000000000..326f0abfc --- /dev/null +++ b/src/store/state_sync.rs @@ -0,0 +1,164 @@ +use crypto::merkle::{Mmr, PartialMmr}; +use miden_node_proto::{mmr::MmrDelta, responses::AccountHashUpdate}; + +use objects::{BlockHeader, Digest}; +use rusqlite::params; + +use crate::{ + errors::StoreError, + store::accounts::{parse_accounts, parse_accounts_columns}, +}; + +use super::Store; + +impl Store { + // STATE SYNC + // -------------------------------------------------------------------------------------------- + + /// Returns the note tags that the client is interested in. + pub fn get_note_tags(&self) -> Result, StoreError> { + const QUERY: &str = "SELECT tags FROM state_sync"; + + self.db + .prepare(QUERY) + .map_err(StoreError::QueryError)? + .query_map([], |row| row.get(0)) + .expect("no binding parameters used in query") + .map(|result| { + result + .map_err(StoreError::ColumnParsingError) + .and_then(|v: String| { + serde_json::from_str(&v).map_err(StoreError::JsonDataDeserializationError) + }) + }) + .next() + .expect("state sync tags exist") + } + + /// Adds a note tag to the list of tags that the client is interested in. + pub fn add_note_tag(&mut self, tag: u64) -> Result { + let mut tags = self.get_note_tags()?; + if tags.contains(&tag) { + return Ok(false); + } + tags.push(tag); + let tags = serde_json::to_string(&tags).map_err(StoreError::InputSerializationError)?; + + const QUERY: &str = "UPDATE state_sync SET tags = ?"; + self.db + .execute(QUERY, params![tags]) + .map_err(StoreError::QueryError) + .map(|_| ())?; + + Ok(true) + } + + /// Returns the block number of the last state sync block + pub fn get_latest_block_number(&self) -> Result { + const QUERY: &str = "SELECT block_num FROM state_sync"; + + self.db + .prepare(QUERY) + .map_err(StoreError::QueryError)? + .query_map([], |row| row.get(0)) + .expect("no binding parameters used in query") + .map(|result| { + result + .map_err(StoreError::ColumnParsingError) + .map(|v: i64| v as u32) + }) + .next() + .expect("state sync block number exists") + } + + pub fn apply_state_sync( + &mut self, + block_header: BlockHeader, + nullifiers: Vec, + account_updates: Vec, + mmr_delta: Option, + ) -> Result<(), StoreError> { + let tx = self + .db + .transaction() + .map_err(StoreError::TransactionError)?; + + // Check if the returned account hashes match latest account hashes in the database + for account_update in account_updates { + if let (Some(account_id), Some(account_hash)) = + (account_update.account_id, account_update.account_hash) + { + let account_id_int: u64 = account_id.clone().into(); + const ACCOUNT_HASH_QUERY: &str = "SELECT hash FROM accounts WHERE id = ?"; + + if let Some(Ok(acc_stub)) = tx + .prepare(ACCOUNT_HASH_QUERY) + .map_err(StoreError::QueryError)? + .query_map(params![account_id_int as i64], parse_accounts_columns) + .map_err(StoreError::QueryError)? + .map(|result| { + result + .map_err(StoreError::ColumnParsingError) + .and_then(parse_accounts) + }) + .next() + { + if account_hash != acc_stub.hash().into() { + return Err(StoreError::AccountHashMismatch( + account_id.try_into().unwrap(), + )); + } + } + } + } + + // update state sync block number + const BLOCK_NUMBER_QUERY: &str = "UPDATE state_sync SET block_num = ?"; + tx.execute(BLOCK_NUMBER_QUERY, params![block_header.block_num()]) + .map_err(StoreError::QueryError)?; + + // update spent notes + for nullifier in nullifiers { + const SPENT_QUERY: &str = + "UPDATE input_notes SET status = 'consumed' WHERE nullifier = ?"; + let nullifier = nullifier.to_string(); + tx.execute(SPENT_QUERY, params![nullifier]) + .map_err(StoreError::QueryError)?; + } + + // update chain mmr nodes on the table + // get all elements from the chain mmr table + if let Some(mmr_delta) = mmr_delta { + // get current nodes on table + let previous_nodes = Self::get_chain_mmr_nodes(&tx)?; + + // build partial mmr from the nodes - partial_mmr should be on memory as part of our store + let leaves: Vec = previous_nodes.values().cloned().collect(); + let mmr: Mmr = leaves.into(); + let mut partial_mmr: PartialMmr = mmr + .peaks(mmr.forest()) + .map_err(StoreError::MmrError)? + .into(); + + // apply the delta + let new_authentication_nodes = partial_mmr + .apply(mmr_delta.try_into().unwrap()) + .map_err(StoreError::MmrError)?; + for (node_id, node) in new_authentication_nodes { + Store::insert_chain_mmr_node(&tx, node_id, node)?; + } + + Store::insert_block_header( + &tx, + block_header, + partial_mmr.peaks().peaks().to_vec(), + partial_mmr.forest() as u64, + )?; + } + + // commit the updates + tx.commit().map_err(StoreError::QueryError)?; + + Ok(()) + } +} diff --git a/src/store/store.sql b/src/store/store.sql index f6b03eecf..829689301 100644 --- a/src/store/store.sql +++ b/src/store/store.sql @@ -73,7 +73,7 @@ CREATE TABLE input_notes ( sender_id UNSIGNED BIG INT NOT NULL, -- the account ID of the sender tag UNSIGNED BIG INT NOT NULL, -- the note tag num_assets UNSIGNED BIG INT NOT NULL, -- the number of assets in the note - inclusion_proof BLOB NOT NULL, -- the inclusion proof of the note against a block number + inclusion_proof BLOB NULL, -- the inclusion proof of the note against a block number recipients BLOB NOT NULL, -- a list of account IDs of accounts which can consume this note status TEXT CHECK( status IN ( -- the status of the note - either pending, committed or consumed 'pending', 'committed', 'consumed' @@ -84,14 +84,32 @@ CREATE TABLE input_notes ( -- Create state sync table CREATE TABLE state_sync ( - block_number UNSIGNED BIG INT NOT NULL, -- the block number of the most recent state sync + block_num UNSIGNED BIG INT NOT NULL, -- the block number of the most recent state sync tags BLOB NOT NULL, -- the serialized list of tags - PRIMARY KEY (block_number) + PRIMARY KEY (block_num) ); -- insert initial row into state_sync table -INSERT OR IGNORE INTO state_sync (block_number, tags) +INSERT OR IGNORE INTO state_sync (block_num, tags) SELECT 0, '[]' WHERE ( SELECT COUNT(*) FROM state_sync ) = 0; + +-- Create block headers table +CREATE TABLE block_headers( + block_num UNSIGNED BIG INT NOT NULL, -- block number + header BLOB NOT NULL, -- serialized block header + notes_root BLOB NOT NULL, -- root of the notes Merkle tree in this block + sub_hash BLOB NOT NULL, -- hash of all other header fields in the block + chain_mmr BLOB NOT NULL, -- serialized peaks of the chain MMR at this block + forest UNSIGNED BIG NOT NULL, -- forest of the chain MMR at this block + PRIMARY KEY (block_num) +); + +-- Create chain mmr nodes +CREATE TABLE chain_mmr_nodes( + id UNSIGNED BIG INT NOT NULL, -- in-order index of the internal MMR node + node BLOB NOT NULL, -- internal node value (hash) + PRIMARY KEY (id) +) diff --git a/src/tests.rs b/src/tests.rs index 5c408344f..ccb172457 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -163,7 +163,7 @@ async fn test_sync_state() { // generate test data crate::mock::insert_mock_data(&mut client); - // assert that we have no consumed notes prior to syncing state + // assert that we have no consumed nor pending notes prior to syncing state assert_eq!( client .get_input_notes(InputNoteFilter::Consumed) @@ -196,6 +196,15 @@ async fn test_sync_state() { 1 ); + // verify that the pending note we had is now committed + assert_eq!( + client + .get_input_notes(InputNoteFilter::Committed) + .unwrap() + .len(), + 1 + ); + // verify that the latest block number has been updated assert_eq!( client.get_latest_block_number().unwrap(),