From cf5b9693e3660a3e07e119f217e67fd5d3b1aa04 Mon Sep 17 00:00:00 2001 From: mango-dee Date: Tue, 26 Nov 2024 16:05:41 +0800 Subject: [PATCH 01/10] Able to update and retrieve accounts --- accounts/src/account_service.rs | 68 ++-- accounts/src/account_store_interface.rs | 3 +- accounts/src/lib.rs | 2 + accounts/src/store/accounts_db.rs | 318 ++++++++++++++++++ accounts/src/store/mod.rs | 3 + .../src/grpc/grpc_accounts_streaming.rs | 4 +- cluster-endpoints/src/grpc_multiplex.rs | 6 +- core/src/structures/account_data.rs | 1 + lite-rpc/src/main.rs | 58 ++-- .../tests/quic_proxy_tpu_integrationtest.rs | 1 - quic-forward-proxy/src/outbound/tx_forward.rs | 9 +- .../src/tls_self_signed_pair_generator.rs | 4 +- 12 files changed, 407 insertions(+), 70 deletions(-) create mode 100644 accounts/src/store/accounts_db.rs create mode 100644 accounts/src/store/mod.rs diff --git a/accounts/src/account_service.rs b/accounts/src/account_service.rs index 39548f11..fb4c7566 100644 --- a/accounts/src/account_service.rs +++ b/accounts/src/account_service.rs @@ -2,17 +2,8 @@ use std::{str::FromStr, sync::Arc}; use anyhow::bail; use itertools::Itertools; -use prometheus::{opts, register_int_gauge, IntGauge}; +use prometheus::{IntGauge, opts, register_int_gauge}; use solana_account_decoder::{UiAccount, UiDataSliceConfig}; -use solana_lite_rpc_core::types::BlockInfoStream; -use solana_lite_rpc_core::{ - commitment_utils::Commitment, - structures::{ - account_data::{AccountData, AccountNotificationMessage, AccountStream}, - account_filter::AccountFilters, - }, - AnyhowJoinHandle, -}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client_api::{ config::{RpcAccountInfoConfig, RpcProgramAccountsConfig}, @@ -21,6 +12,16 @@ use solana_rpc_client_api::{ use solana_sdk::{commitment_config::CommitmentConfig, pubkey::Pubkey, slot_history::Slot}; use tokio::sync::broadcast::Sender; +use solana_lite_rpc_core::{ + AnyhowJoinHandle, + commitment_utils::Commitment, + structures::{ + account_data::{AccountData, AccountNotificationMessage, AccountStream}, + account_filter::AccountFilters, + }, +}; +use solana_lite_rpc_core::types::BlockInfoStream; + use crate::account_store_interface::{AccountLoadingError, AccountStorageInterface}; lazy_static::lazy_static! { @@ -190,27 +191,34 @@ impl AccountService { loop { match blockinfo_stream.recv().await { Ok(block_info) => { - if block_info.commitment_config.is_processed() { - // processed commitment is not processed in this loop - continue; - } - let commitment = Commitment::from(block_info.commitment_config); - let updated_accounts = this - .account_store - .process_slot_data(block_info.slot, commitment) - .await; - - if block_info.commitment_config.is_finalized() { - ACCOUNT_UPDATES_FINALIZED.add(updated_accounts.len() as i64) - } else { - ACCOUNT_UPDATES_CONFIRMED.add(updated_accounts.len() as i64); - } - - for data in updated_accounts { - let _ = this - .account_notification_sender - .send(AccountNotificationMessage { data, commitment }); + /////////////////////////////////////////////// + { + let commitment = Commitment::from(block_info.commitment_config); + this.account_store.process_slot_data(block_info.slot, commitment).await; } + /////////////////////////////////////////////// + // FIXME clarify why this is the case + // if block_info.commitment_config.is_processed() { + // // processed commitment is not processed in this loop + // continue; + // } + // let commitment = Commitment::from(block_info.commitment_config); + // let updated_accounts = this + // .account_store + // .process_slot_data(block_info.slot, commitment) + // .await; + // + // if block_info.commitment_config.is_finalized() { + // ACCOUNT_UPDATES_FINALIZED.add(updated_accounts.len() as i64) + // } else { + // ACCOUNT_UPDATES_CONFIRMED.add(updated_accounts.len() as i64); + // } + // + // for data in updated_accounts { + // let _ = this + // .account_notification_sender + // .send(AccountNotificationMessage { data, commitment }); + // } } Err(tokio::sync::broadcast::error::RecvError::Lagged(e)) => { log::error!("Block Stream Lagged to update accounts by {}", e); diff --git a/accounts/src/account_store_interface.rs b/accounts/src/account_store_interface.rs index fa64d60b..7ef4019f 100644 --- a/accounts/src/account_store_interface.rs +++ b/accounts/src/account_store_interface.rs @@ -5,11 +5,12 @@ use solana_rpc_client_api::filter::RpcFilterType; use solana_sdk::pubkey::Pubkey; use solana_sdk::slot_history::Slot; -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +#[derive(Clone, PartialEq, Eq, Debug)] pub enum AccountLoadingError { AccountNotFound, ConfigDoesnotContainRequiredFilters, OperationTimeOut, + FailedToSpawnTask(String) } #[async_trait] diff --git a/accounts/src/lib.rs b/accounts/src/lib.rs index 11d6260f..3a786c83 100644 --- a/accounts/src/lib.rs +++ b/accounts/src/lib.rs @@ -1,3 +1,5 @@ pub mod account_service; pub mod account_store_interface; pub mod inmemory_account_store; + +pub mod store; \ No newline at end of file diff --git a/accounts/src/store/accounts_db.rs b/accounts/src/store/accounts_db.rs new file mode 100644 index 00000000..d5258015 --- /dev/null +++ b/accounts/src/store/accounts_db.rs @@ -0,0 +1,318 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use solana_accounts_db::accounts::Accounts; +use solana_accounts_db::accounts_db::{AccountsDb as SolanaAccountsDb, AccountsDbConfig, AccountShrinkThreshold, CreateAncientStorage}; +use solana_accounts_db::accounts_file::StorageAccess; +use solana_accounts_db::accounts_index::{AccountSecondaryIndexes, AccountsIndexConfig, IndexLimitMb}; +use solana_accounts_db::ancestors::Ancestors; +use solana_accounts_db::partitioned_rewards::TestPartitionedEpochRewards; +use solana_rpc_client_api::filter::RpcFilterType; +use solana_sdk::account::{Account, AccountSharedData, ReadableAccount}; +use solana_sdk::clock::Slot; +use solana_sdk::genesis_config::ClusterType; +use solana_sdk::pubkey::Pubkey; +use task::spawn_blocking; +use tokio::task; + +use solana_lite_rpc_core::commitment_utils::Commitment; +use solana_lite_rpc_core::structures::account_data::AccountData; + +use crate::account_store_interface::{AccountLoadingError, AccountStorageInterface}; + +// FIXME what are all those configs +pub const BINS: usize = 8192; +pub const FLUSH_THREADS: usize = 1; + +pub const ACCOUNTS_INDEX_CONFIG: AccountsIndexConfig = AccountsIndexConfig { + bins: Some(BINS), + flush_threads: Some(FLUSH_THREADS), + drives: None, + index_limit_mb: IndexLimitMb::Unspecified, + ages_to_stay_in_cache: None, + scan_results_limit_bytes: None, + started_from_validator: false, +}; + +pub const ACCOUNTS_DB_CONFIG: AccountsDbConfig = AccountsDbConfig { + index: Some(ACCOUNTS_INDEX_CONFIG), + base_working_path: None, + accounts_hash_cache_path: None, + shrink_paths: None, + read_cache_limit_bytes: None, + write_cache_limit_bytes: None, + ancient_append_vec_offset: None, + skip_initial_hash_calc: false, + exhaustively_verify_refcounts: false, + create_ancient_storage: CreateAncientStorage::Pack, + test_partitioned_epoch_rewards: TestPartitionedEpochRewards::None, + test_skip_rewrites_but_include_in_bank_hash: false, + storage_access: StorageAccess::Mmap, +}; + +pub struct AccountsDb { + accounts: Accounts, + // FIXME probably RwLock or similar + commitments: Mutex>>, +} + +impl AccountsDb { + pub fn new() -> Self { + let db = SolanaAccountsDb::new_with_config( + vec![], + &ClusterType::MainnetBeta, + AccountSecondaryIndexes::default(), + AccountShrinkThreshold::default(), + Some(ACCOUNTS_DB_CONFIG), + None, + Arc::default(), + ); + + let accounts = Accounts::new(Arc::new(db)); + Self { + accounts, + commitments: Mutex::new(RefCell::new(HashMap::new())), + } + } + + pub fn new_for_testing() -> Self { + let db = SolanaAccountsDb::new_single_for_tests(); + let accounts = Accounts::new(Arc::new(db)); + Self { + accounts, + commitments: Mutex::new(RefCell::new(HashMap::new())), + } + } +} + +#[async_trait] +impl AccountStorageInterface for AccountsDb { + // Why is there an update_account with commitment arg? + async fn update_account(&self, account_data: AccountData, _commitment: Commitment) -> bool { + let shared_data = account_data.account.to_account_shared_data(); + let account_to_store = [(&account_data.pubkey, &shared_data)]; + self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); + + println!("update {}", account_data.pubkey); + + false + } + + // FIXME this is upsert ?! + async fn initilize_or_update_account(&self, account_data: AccountData) { + // let shared_data = account_data.account.to_account_shared_data(); + // let account_to_store = [(&account_data.pubkey, &shared_data)]; + // self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); + + println!("init or update {}", account_data.pubkey); + + let shared_data = account_data.account.to_account_shared_data(); + let account_to_store = [(&account_data.pubkey, &shared_data)]; + self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); + + // FIXME do we need this + } + + async fn get_account(&self, account_pk: Pubkey, commitment: Commitment) -> Result, AccountLoadingError> { + let ancestors = self.get_ancestors_from_commitment(commitment); + + let accounts_db = self.accounts.accounts_db.clone(); + Ok( + spawn_blocking(move || { + accounts_db + .load_with_fixed_root(&ancestors, &account_pk) + .map(|(shared_data, slot)| Self::to_account_data(account_pk, slot, shared_data)) + }) + .await + .map_err(|e| AccountLoadingError::FailedToSpawnTask(format!("Failed to spawn task: {:?}", e)))? + ) + } + + async fn get_program_accounts(&self, program_pubkey: Pubkey, account_filter: Option>, commitment: Commitment) -> Option> { + let ancestors = self.get_ancestors_from_commitment(commitment); + + // self.accounts.load_by_program() + + todo!() + } + + async fn process_slot_data(&self, slot: Slot, commitment: Commitment) -> Vec { + if commitment == Commitment::Finalized { + self.accounts.add_root(slot); + } + println!("{} - {:?}", slot, commitment); + + //FIXME ensure Finalized <= Confirmed <= Processed + // update all if this assumption is broken + // make sure is monotonic increasing + // self.commitments.insert(commitment, slot); + + // FIXME do we need to return data from here? - why + self.commitments.lock().unwrap().borrow_mut().insert(commitment, slot); + vec![] + } +} + +impl AccountsDb { + fn get_ancestors_from_commitment(&self, commitment: Commitment) -> Ancestors { + let lock = self.commitments.lock().unwrap(); + let slot = lock.borrow().get(&commitment).unwrap().clone(); + Ancestors::from(vec![slot]) + } + + fn to_account_data(pk: Pubkey, slot: Slot, shared_data: AccountSharedData) -> AccountData { + AccountData { + pubkey: pk, + account: Arc::new(Account { + lamports: shared_data.lamports(), + data: Vec::from(shared_data.data()), + owner: shared_data.owner().clone(), + executable: shared_data.executable(), + rent_epoch: shared_data.rent_epoch(), + }), + updated_slot: slot, + } + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use solana_sdk::pubkey::Pubkey; + + use solana_lite_rpc_core::commitment_utils::Commitment; + + use crate::account_store_interface::AccountStorageInterface; + use crate::store::accounts_db::create_account_data; + use crate::store::AccountsDb; + + #[tokio::test] + async fn store_new_account() { + let test_instance = AccountsDb::new_for_testing(); + + let program_key = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let account_1_key = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + let account_1_data = create_account_data(1, account_1_key, program_key, &[0u8; 23]); + test_instance.initilize_or_update_account(account_1_data).await; + + let result = test_instance.get_account(account_1_key, Commitment::Confirmed).await; + } +} + +pub fn create_account_data( + updated_slot: Slot, + pubkey: Pubkey, + program: Pubkey, + data: &[u8], +) -> AccountData { + AccountData { + pubkey, + account: Arc::new(Account { + lamports: 42, + data: Vec::from(data), + owner: program, + executable: false, + rent_epoch: 0, + }), + updated_slot, + } + + + // #[test] + // fn test() { + // let db = AccountsDb::new_single_for_tests(); + // let accounts = Accounts::new(Arc::new(db)); + // + // // let num_slots = 4; + // // let num_accounts = 10_000; + // // println!("Creating {num_accounts} accounts"); + // + // // let pubkeys: Vec<_> = (0..num_slots) + // // .into_iter() + // // .map(|slot| { + // // let mut pubkeys: Vec = vec![]; + // // create_test_accounts( + // // &accounts, + // // &mut pubkeys, + // // num_accounts / num_slots, + // // slot as u64, + // // ); + // // pubkeys + // // }) + // // .collect(); + // // + // // let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect(); + // + // // println!("{:?}", pubkeys); + // let pubkey = solana_sdk::pubkey::new_rand(); + // let mut rng = rand::thread_rng(); + // let program = Pubkey::new_unique(); + // // let acc = create_random_account( + // // &mut rng, + // // 1, + // // pubkey, + // // program, + // // ); + // // println!("{acc:?}"); + // + // let account = AccountSharedData::new( + // 1 as u64, + // 0, + // AccountSharedData::default().owner(), + // ); + // + // println!("{program:?}"); + // + // let account_for_storage = [(&pubkey, &account)]; + // let to_store = (1u64, account_for_storage.as_slice()); + // + // accounts.store_accounts_cached(to_store) + // } + // + // fn create_random_account( + // rng: &mut ThreadRng, + // updated_slot: Slot, + // pubkey: Pubkey, + // program: Pubkey, + // ) -> AccountData { + // let length: usize = rng.gen_range(100..1000); + // AccountData { + // pubkey, + // account: Arc::new(Account { + // lamports: rng.gen(), + // data: (0..length).map(|_| rng.gen::()).collect_vec(), + // owner: program, + // executable: false, + // rent_epoch: 0, + // }), + // updated_slot, + // } + // } +} +// +// pub fn create_test_accounts( +// accounts: &Accounts, +// pubkeys: &mut Vec, +// num: usize, +// slot: Slot, +// ) { +// let data_size = 0; +// +// for t in 0..num { +// let pubkey = solana_sdk::pubkey::new_rand(); +// let account = AccountSharedData::new( +// (t + 1) as u64, +// data_size, +// AccountSharedData::default().owner(), +// ); +// // accounts.store_slow_uncached(slot, &pubkey, &account); +// let random_account = self +// +// accounts.store_accounts_cached() +// pubkeys.push(pubkey); +// } +// } diff --git a/accounts/src/store/mod.rs b/accounts/src/store/mod.rs new file mode 100644 index 00000000..8c921f69 --- /dev/null +++ b/accounts/src/store/mod.rs @@ -0,0 +1,3 @@ +pub use accounts_db::AccountsDb; + +mod accounts_db; \ No newline at end of file diff --git a/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs b/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs index dc4a7f5b..4f52721a 100644 --- a/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs +++ b/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs @@ -7,10 +7,8 @@ use std::{ time::Duration, }; -use geyser_grpc_connector::yellowstone_grpc_util::{ - connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig, -}; use geyser_grpc_connector::{GeyserGrpcClient, GrpcSourceConfig}; +use geyser_grpc_connector::yellowstone_grpc_util::{connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig}; use itertools::Itertools; use solana_lite_rpc_core::{ commitment_utils::Commitment, diff --git a/cluster-endpoints/src/grpc_multiplex.rs b/cluster-endpoints/src/grpc_multiplex.rs index 9922c4ec..e3197ebe 100644 --- a/cluster-endpoints/src/grpc_multiplex.rs +++ b/cluster-endpoints/src/grpc_multiplex.rs @@ -332,8 +332,12 @@ pub fn create_grpc_multiplex_blocks_subscription( let mut startup_completed = false; const MAX_ALLOWED_CLEANUP_WITHOUT_RECV: u8 = 12; // 12*5 = 60s without recving data 'recv_loop: loop { - debug!("channel capacities: processed_block_sender={}, block_info_sender_confirmed={}, block_info_sender_finalized={}", + debug!("channel capacities:\nprocessed_block_sender={}, \ + \nblock_info_sender_processed={}, \ + \nblock_info_sender_confirmed={}, \ + \nblock_info_sender_finalized={}", processed_block_sender.capacity(), + block_info_sender_processed.capacity(), block_info_sender_confirmed.capacity(), block_info_sender_finalized.capacity() ); diff --git a/core/src/structures/account_data.rs b/core/src/structures/account_data.rs index e479e1f0..362819ba 100644 --- a/core/src/structures/account_data.rs +++ b/core/src/structures/account_data.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use solana_rpc_client_api::filter::RpcFilterType; use solana_sdk::{account::Account, pubkey::Pubkey, slot_history::Slot}; +use solana_sdk::account::AccountSharedData; use tokio::sync::broadcast::Receiver; use crate::commitment_utils::Commitment; diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index e294b53b..91a9dfec 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -69,6 +69,7 @@ use tokio::sync::mpsc; use tokio::sync::RwLock; use tracing_subscriber::fmt::format::FmtSpan; use tracing_subscriber::EnvFilter; +use solana_lite_rpc_accounts::store::AccountsDb; // jemalloc seems to be better at keeping the memory footprint reasonable over // longer periods of time @@ -188,10 +189,26 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: info!("Disabled grpc stream inspection"); } + info!("Waiting for first finalized block info..."); + let finalized_block_info = wait_till_block_of_commitment_is_recieved( + blockinfo_notifier.resubscribe(), + CommitmentConfig::finalized(), + ) + .await; + info!("Got finalized block info: {:?}", finalized_block_info.slot); + + let (epoch_data, _current_epoch_info) = EpochCache::bootstrap_epoch(&rpc_client).await?; + + let block_information_store = + BlockInformationStore::new(BlockInformation::from_block_info(&finalized_block_info)); + let accounts_service = if let Some(account_stream) = processed_account_stream { // lets use inmemory storage for now - let inmemory_account_storage: Arc = - Arc::new(InmemoryAccountStore::new()); + // let inmemory_account_storage: Arc = + // Arc::new(InmemoryAccountStore::new()); + + let accounts_storage: Arc = Arc::new(AccountsDb::new()); + const MAX_CONNECTIONS_IN_PARALLEL: usize = 10; // Accounts notifications will be spurious when slots change // 256 seems very reasonable so that there are no account notification is missed and memory usage @@ -201,45 +218,36 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: Arc::new(AccountsOnDemand::new( rpc_client.clone(), gprc_sources, - inmemory_account_storage, + accounts_storage, account_notification_sender.clone(), )) } else { - inmemory_account_storage + accounts_storage }; - let account_service = AccountService::new(account_storage, account_notification_sender); + let account_service = AccountService::new( + account_storage, + account_notification_sender + ); account_service.process_account_stream( account_stream.resubscribe(), blockinfo_notifier.resubscribe(), ); - account_service - .populate_from_rpc( - rpc_client.clone(), - &account_filters, - MAX_CONNECTIONS_IN_PARALLEL, - ) - .await?; + // FIXME there should be a flag so that development can happen faster + // account_service + // .populate_from_rpc( + // rpc_client.clone(), + // &account_filters, + // MAX_CONNECTIONS_IN_PARALLEL, + // ) + // .await?; Some(account_service) } else { None }; - info!("Waiting for first finalized block info..."); - let finalized_block_info = wait_till_block_of_commitment_is_recieved( - blockinfo_notifier.resubscribe(), - CommitmentConfig::finalized(), - ) - .await; - info!("Got finalized block info: {:?}", finalized_block_info.slot); - - let (epoch_data, _current_epoch_info) = EpochCache::bootstrap_epoch(&rpc_client).await?; - - let block_information_store = - BlockInformationStore::new(BlockInformation::from_block_info(&finalized_block_info)); - let data_cache = DataCache { block_information_store, cluster_info: ClusterInfo::default(), diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 5cdc6319..a5a36c15 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -19,7 +19,6 @@ use solana_streamer::nonblocking::quic::{ConnectionPeerType, SpawnNonBlockingSer use solana_streamer::packet::PacketBatch; use solana_streamer::quic::StreamStats; use solana_streamer::streamer::StakedNodes; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 46b225f8..23859470 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -14,12 +14,12 @@ use quinn::{ use solana_lite_rpc_core::network_utils::apply_gso_workaround; use solana_sdk::quic::QUIC_MAX_TIMEOUT; use solana_streamer::nonblocking::quic::ALPN_TPU_PROTOCOL_ID; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::collections::HashMap; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +use solana_streamer::tls_certificates::new_dummy_x509_certificate; use tokio::sync::mpsc::Receiver; use tokio::sync::RwLock; @@ -267,12 +267,7 @@ async fn new_endpoint_with_validator_identity(validator_identity: ValidatorIdent validator_identity ); // the counterpart of this function is get_remote_pubkey+get_pubkey_from_tls_certificate - let (certificate, key) = new_self_signed_tls_certificate( - &validator_identity.get_keypair_for_tls(), - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC connection certificates"); - + let (certificate, key) = new_dummy_x509_certificate(&validator_identity.get_keypair_for_tls() ); create_tpu_client_endpoint(certificate, key) } diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index 79052b38..e6af2d45 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -35,8 +35,8 @@ impl SelfSignedTlsConfigProvider { fn gen_tls_certificate_and_key(hostnames: Vec) -> (Certificate, PrivateKey) { let cert = generate_simple_self_signed(hostnames).unwrap(); - let key = cert.serialize_private_key_der(); - (Certificate(cert.serialize_der().unwrap()), PrivateKey(key)) + let key = cert.key_pair.serialize_der(); + (Certificate(cert.key_pair.serialize_der()), PrivateKey(key)) } fn build_client_crypto_insecure() -> ClientConfig { From 852a21ef561e9c8b7058812603f7b2c81ac97776 Mon Sep 17 00:00:00 2001 From: mango-dee Date: Wed, 27 Nov 2024 12:58:23 +0800 Subject: [PATCH 02/10] WIP to seek feedback --- accounts/src/store/accounts_db.rs | 175 ++++++++++++++++++++++-------- 1 file changed, 131 insertions(+), 44 deletions(-) diff --git a/accounts/src/store/accounts_db.rs b/accounts/src/store/accounts_db.rs index d5258015..a25a0db8 100644 --- a/accounts/src/store/accounts_db.rs +++ b/accounts/src/store/accounts_db.rs @@ -1,23 +1,27 @@ -use std::cell::RefCell; use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use async_trait::async_trait; +use itertools::Itertools; use solana_accounts_db::accounts::Accounts; use solana_accounts_db::accounts_db::{AccountsDb as SolanaAccountsDb, AccountsDbConfig, AccountShrinkThreshold, CreateAncientStorage}; use solana_accounts_db::accounts_file::StorageAccess; -use solana_accounts_db::accounts_index::{AccountSecondaryIndexes, AccountsIndexConfig, IndexLimitMb}; +use solana_accounts_db::accounts_index::{AccountSecondaryIndexes, AccountsIndexConfig, IndexLimitMb, ScanConfig, ScanResult}; use solana_accounts_db::ancestors::Ancestors; use solana_accounts_db::partitioned_rewards::TestPartitionedEpochRewards; use solana_rpc_client_api::filter::RpcFilterType; use solana_sdk::account::{Account, AccountSharedData, ReadableAccount}; -use solana_sdk::clock::Slot; +use solana_sdk::clock::{BankId, Slot}; use solana_sdk::genesis_config::ClusterType; use solana_sdk::pubkey::Pubkey; +use solana_sdk::transaction_context::TransactionAccount; use task::spawn_blocking; +use tokio::sync::RwLock; use tokio::task; +use Commitment::{Confirmed, Finalized}; use solana_lite_rpc_core::commitment_utils::Commitment; +use solana_lite_rpc_core::commitment_utils::Commitment::Processed; use solana_lite_rpc_core::structures::account_data::AccountData; use crate::account_store_interface::{AccountLoadingError, AccountStorageInterface}; @@ -55,7 +59,7 @@ pub const ACCOUNTS_DB_CONFIG: AccountsDbConfig = AccountsDbConfig { pub struct AccountsDb { accounts: Accounts, // FIXME probably RwLock or similar - commitments: Mutex>>, + commitments: RwLock>, } impl AccountsDb { @@ -73,7 +77,7 @@ impl AccountsDb { let accounts = Accounts::new(Arc::new(db)); Self { accounts, - commitments: Mutex::new(RefCell::new(HashMap::new())), + commitments: RwLock::new(HashMap::new()), } } @@ -82,48 +86,33 @@ impl AccountsDb { let accounts = Accounts::new(Arc::new(db)); Self { accounts, - commitments: Mutex::new(RefCell::new(HashMap::new())), + commitments: RwLock::new(HashMap::new()), } } } #[async_trait] impl AccountStorageInterface for AccountsDb { - // Why is there an update_account with commitment arg? async fn update_account(&self, account_data: AccountData, _commitment: Commitment) -> bool { - let shared_data = account_data.account.to_account_shared_data(); - let account_to_store = [(&account_data.pubkey, &shared_data)]; - self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); - - println!("update {}", account_data.pubkey); - - false + self.initilize_or_update_account(account_data).await; + true } - // FIXME this is upsert ?! async fn initilize_or_update_account(&self, account_data: AccountData) { - // let shared_data = account_data.account.to_account_shared_data(); - // let account_to_store = [(&account_data.pubkey, &shared_data)]; - // self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); - - println!("init or update {}", account_data.pubkey); - let shared_data = account_data.account.to_account_shared_data(); let account_to_store = [(&account_data.pubkey, &shared_data)]; self.accounts.store_accounts_cached((account_data.updated_slot, account_to_store.as_slice())); - - // FIXME do we need this } async fn get_account(&self, account_pk: Pubkey, commitment: Commitment) -> Result, AccountLoadingError> { - let ancestors = self.get_ancestors_from_commitment(commitment); + let ancestors = self.get_ancestors_from_commitment(commitment).await; let accounts_db = self.accounts.accounts_db.clone(); Ok( spawn_blocking(move || { accounts_db .load_with_fixed_root(&ancestors, &account_pk) - .map(|(shared_data, slot)| Self::to_account_data(account_pk, slot, shared_data)) + .map(|(shared_data, slot)| Self::convert_to_account_data(account_pk, slot, shared_data)) }) .await .map_err(|e| AccountLoadingError::FailedToSpawnTask(format!("Failed to spawn task: {:?}", e)))? @@ -131,38 +120,136 @@ impl AccountStorageInterface for AccountsDb { } async fn get_program_accounts(&self, program_pubkey: Pubkey, account_filter: Option>, commitment: Commitment) -> Option> { - let ancestors = self.get_ancestors_from_commitment(commitment); - - // self.accounts.load_by_program() - - todo!() + let slot = self.get_slot_from_commitment(commitment).await; + let ancestors = Ancestors::from(vec![slot]); + let accounts_db = self.accounts.accounts_db.clone(); + let scan_config = ScanConfig::new(true); + + let transaction_accounts: ScanResult> = spawn_blocking(move || { + let filter = move |data: &AccountSharedData| { + if data.owner() != &program_pubkey { + return false; + } + match &account_filter { + Some(filters) => { + filters.iter().all(|filter| { + match filter { + RpcFilterType::DataSize(size) => data.data().len() == *size as usize, + RpcFilterType::Memcmp(cmp) => cmp.bytes_match(data.data()), + RpcFilterType::TokenAccountState => unimplemented!() // FIXME + } + }) + } + None => true + } + }; + + // implementation from solana accounts.rs with slight adjustments - load_by_program_with_filter + let mut collector = Vec::new(); + accounts_db + .scan_accounts( + &ancestors, + BankId::from(slot), + |some_account_tuple| { + Self::load_while_filtering(&mut collector, some_account_tuple, |account| { + filter(account) + }) + }, + &scan_config, + ) + .map(|_| collector) + // TODO The interface should be adjusted and return Rest> instead + }).await.unwrap_or(Ok(vec![])); + + Some( + transaction_accounts.unwrap() + .into_iter() + .map(|ta| { Self::convert_to_account_data(ta.0, slot, ta.1) }) + .collect_vec() + ) } + async fn process_slot_data(&self, slot: Slot, commitment: Commitment) -> Vec { - if commitment == Commitment::Finalized { + if commitment == Finalized { self.accounts.add_root(slot); } - println!("{} - {:?}", slot, commitment); - //FIXME ensure Finalized <= Confirmed <= Processed - // update all if this assumption is broken - // make sure is monotonic increasing - // self.commitments.insert(commitment, slot); + let commitments = self.commitments.read().await.clone(); + + let processed = commitments.get(&Processed).cloned().unwrap_or(Slot::from(0u64)); + let confirmed = commitments.get(&Confirmed).cloned().unwrap_or(Slot::from(0u64)); + let finalized = commitments.get(&Finalized).cloned().unwrap_or(Slot::from(0u64)); + + match commitment { + Processed => { + if slot > processed { + self.commitments.write().await.insert(Processed, slot); + } + } + Confirmed => { + if slot > processed { + self.commitments.write().await.insert(Processed, slot); + } + if slot > confirmed { + self.commitments.write().await.insert(Confirmed, slot); + } + } + Finalized => { + if slot > processed { + self.commitments.write().await.insert(Processed, slot); + } + + if slot > confirmed { + self.commitments.write().await.insert(Confirmed, slot); + } + + if slot > finalized { + self.commitments.write().await.insert(Finalized, slot); + } + } + } // FIXME do we need to return data from here? - why - self.commitments.lock().unwrap().borrow_mut().insert(commitment, slot); + // self.commitments.write().await.insert(commitment, slot); vec![] } } impl AccountsDb { - fn get_ancestors_from_commitment(&self, commitment: Commitment) -> Ancestors { - let lock = self.commitments.lock().unwrap(); - let slot = lock.borrow().get(&commitment).unwrap().clone(); + // borrowed from solana accounts.rs + fn load_while_filtering bool>( + collector: &mut Vec, + some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>, + filter: F, + ) { + if let Some(mapped_account_tuple) = some_account_tuple + .filter(|(_, account, _)| Self::is_loadable(account.lamports()) && filter(account)) + .map(|(pubkey, account, _slot)| (*pubkey, account)) + { + collector.push(mapped_account_tuple) + } + } + + // borrowed from solana accounts.rs + fn is_loadable(lamports: u64) -> bool { + // Don't ever load zero lamport accounts into runtime because + // the existence of zero-lamport accounts are never deterministic!! + lamports > 0 + } + + async fn get_slot_from_commitment(&self, commitment: Commitment) -> Slot { + let lock = self.commitments.read().await; + let result = lock.get(&commitment).unwrap().clone(); + result + } + + async fn get_ancestors_from_commitment(&self, commitment: Commitment) -> Ancestors { + let slot = self.get_slot_from_commitment(commitment).await; Ancestors::from(vec![slot]) } - fn to_account_data(pk: Pubkey, slot: Slot, shared_data: AccountSharedData) -> AccountData { + fn convert_to_account_data(pk: Pubkey, slot: Slot, shared_data: AccountSharedData) -> AccountData { AccountData { pubkey: pk, account: Arc::new(Account { @@ -183,7 +270,7 @@ mod tests { use solana_sdk::pubkey::Pubkey; - use solana_lite_rpc_core::commitment_utils::Commitment; + use solana_lite_rpc_core::commitment_utils::Commitment::Confirmed; use crate::account_store_interface::AccountStorageInterface; use crate::store::accounts_db::create_account_data; @@ -199,7 +286,7 @@ mod tests { let account_1_data = create_account_data(1, account_1_key, program_key, &[0u8; 23]); test_instance.initilize_or_update_account(account_1_data).await; - let result = test_instance.get_account(account_1_key, Commitment::Confirmed).await; + let result = test_instance.get_account(account_1_key, Confirmed).await; } } From 7b6b15acf30da7bd846d2f14beb38add2b4791fa Mon Sep 17 00:00:00 2001 From: mango-dee Date: Fri, 29 Nov 2024 14:45:33 +0800 Subject: [PATCH 03/10] Wraps up accounts db impl --- accounts/src/account_service.rs | 44 +- accounts/src/store/accounts_db.rs | 689 +++++++++++++++++++--------- core/src/structures/account_data.rs | 1 - 3 files changed, 497 insertions(+), 237 deletions(-) diff --git a/accounts/src/account_service.rs b/accounts/src/account_service.rs index fb4c7566..ce5cba91 100644 --- a/accounts/src/account_service.rs +++ b/accounts/src/account_service.rs @@ -191,34 +191,24 @@ impl AccountService { loop { match blockinfo_stream.recv().await { Ok(block_info) => { - /////////////////////////////////////////////// - { - let commitment = Commitment::from(block_info.commitment_config); - this.account_store.process_slot_data(block_info.slot, commitment).await; + + let commitment = Commitment::from(block_info.commitment_config); + let updated_accounts = this.account_store.process_slot_data(block_info.slot, commitment).await; + match commitment { + Commitment::Processed => {} + Commitment::Confirmed => { + ACCOUNT_UPDATES_CONFIRMED.add(updated_accounts.len() as i64); + } + Commitment::Finalized => { + ACCOUNT_UPDATES_FINALIZED.add(updated_accounts.len() as i64) + } + } + + for data in updated_accounts { + let _ = this + .account_notification_sender + .send(AccountNotificationMessage { data, commitment }); } - /////////////////////////////////////////////// - // FIXME clarify why this is the case - // if block_info.commitment_config.is_processed() { - // // processed commitment is not processed in this loop - // continue; - // } - // let commitment = Commitment::from(block_info.commitment_config); - // let updated_accounts = this - // .account_store - // .process_slot_data(block_info.slot, commitment) - // .await; - // - // if block_info.commitment_config.is_finalized() { - // ACCOUNT_UPDATES_FINALIZED.add(updated_accounts.len() as i64) - // } else { - // ACCOUNT_UPDATES_CONFIRMED.add(updated_accounts.len() as i64); - // } - // - // for data in updated_accounts { - // let _ = this - // .account_notification_sender - // .send(AccountNotificationMessage { data, commitment }); - // } } Err(tokio::sync::broadcast::error::RecvError::Lagged(e)) => { log::error!("Block Stream Lagged to update accounts by {}", e); diff --git a/accounts/src/store/accounts_db.rs b/accounts/src/store/accounts_db.rs index a25a0db8..142ca011 100644 --- a/accounts/src/store/accounts_db.rs +++ b/accounts/src/store/accounts_db.rs @@ -1,12 +1,12 @@ -use std::collections::HashMap; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use async_trait::async_trait; use itertools::Itertools; use solana_accounts_db::accounts::Accounts; use solana_accounts_db::accounts_db::{AccountsDb as SolanaAccountsDb, AccountsDbConfig, AccountShrinkThreshold, CreateAncientStorage}; use solana_accounts_db::accounts_file::StorageAccess; -use solana_accounts_db::accounts_index::{AccountSecondaryIndexes, AccountsIndexConfig, IndexLimitMb, ScanConfig, ScanResult}; +use solana_accounts_db::accounts_index::{AccountSecondaryIndexes, AccountsIndexConfig, IndexLimitMb}; use solana_accounts_db::ancestors::Ancestors; use solana_accounts_db::partitioned_rewards::TestPartitionedEpochRewards; use solana_rpc_client_api::filter::RpcFilterType; @@ -15,9 +15,6 @@ use solana_sdk::clock::{BankId, Slot}; use solana_sdk::genesis_config::ClusterType; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction_context::TransactionAccount; -use task::spawn_blocking; -use tokio::sync::RwLock; -use tokio::task; use Commitment::{Confirmed, Finalized}; use solana_lite_rpc_core::commitment_utils::Commitment; @@ -58,8 +55,9 @@ pub const ACCOUNTS_DB_CONFIG: AccountsDbConfig = AccountsDbConfig { pub struct AccountsDb { accounts: Accounts, - // FIXME probably RwLock or similar - commitments: RwLock>, + processed_slot: AtomicU64, + confirmed_slot: AtomicU64, + finalised_slot: AtomicU64, } impl AccountsDb { @@ -77,7 +75,9 @@ impl AccountsDb { let accounts = Accounts::new(Arc::new(db)); Self { accounts, - commitments: RwLock::new(HashMap::new()), + processed_slot: AtomicU64::new(0), + confirmed_slot: AtomicU64::new(0), + finalised_slot: AtomicU64::new(0), } } @@ -86,7 +86,9 @@ impl AccountsDb { let accounts = Accounts::new(Arc::new(db)); Self { accounts, - commitments: RwLock::new(HashMap::new()), + processed_slot: AtomicU64::new(0), + confirmed_slot: AtomicU64::new(0), + finalised_slot: AtomicU64::new(0), } } } @@ -105,68 +107,48 @@ impl AccountStorageInterface for AccountsDb { } async fn get_account(&self, account_pk: Pubkey, commitment: Commitment) -> Result, AccountLoadingError> { - let ancestors = self.get_ancestors_from_commitment(commitment).await; - - let accounts_db = self.accounts.accounts_db.clone(); + let ancestors = self.get_ancestors_from_commitment(commitment); Ok( - spawn_blocking(move || { - accounts_db - .load_with_fixed_root(&ancestors, &account_pk) - .map(|(shared_data, slot)| Self::convert_to_account_data(account_pk, slot, shared_data)) - }) - .await - .map_err(|e| AccountLoadingError::FailedToSpawnTask(format!("Failed to spawn task: {:?}", e)))? + self.accounts + .load_with_fixed_root(&ancestors, &account_pk) + .map(|(shared_data, slot)| Self::convert_to_account_data(account_pk, slot, shared_data)) ) } async fn get_program_accounts(&self, program_pubkey: Pubkey, account_filter: Option>, commitment: Commitment) -> Option> { - let slot = self.get_slot_from_commitment(commitment).await; - let ancestors = Ancestors::from(vec![slot]); - let accounts_db = self.accounts.accounts_db.clone(); - let scan_config = ScanConfig::new(true); - - let transaction_accounts: ScanResult> = spawn_blocking(move || { - let filter = move |data: &AccountSharedData| { - if data.owner() != &program_pubkey { - return false; - } - match &account_filter { - Some(filters) => { - filters.iter().all(|filter| { - match filter { - RpcFilterType::DataSize(size) => data.data().len() == *size as usize, - RpcFilterType::Memcmp(cmp) => cmp.bytes_match(data.data()), - RpcFilterType::TokenAccountState => unimplemented!() // FIXME - } - }) - } - None => true + let slot = self.get_slot_from_commitment(commitment); + + let filter = |data: &AccountSharedData| { + match &account_filter { + Some(filters) => { + filters.iter().all(|filter| { + match filter { + RpcFilterType::DataSize(size) => data.data().len() == *size as usize, + RpcFilterType::Memcmp(cmp) => cmp.bytes_match(data.data()), + RpcFilterType::TokenAccountState => unimplemented!() // FIXME + } + }) } - }; - - // implementation from solana accounts.rs with slight adjustments - load_by_program_with_filter - let mut collector = Vec::new(); - accounts_db - .scan_accounts( - &ancestors, - BankId::from(slot), - |some_account_tuple| { - Self::load_while_filtering(&mut collector, some_account_tuple, |account| { - filter(account) - }) - }, - &scan_config, - ) - .map(|_| collector) - // TODO The interface should be adjusted and return Rest> instead - }).await.unwrap_or(Ok(vec![])); - - Some( - transaction_accounts.unwrap() - .into_iter() - .map(|ta| { Self::convert_to_account_data(ta.0, slot, ta.1) }) - .collect_vec() - ) + None => true + } + }; + + let transaction_accounts: Vec = self.accounts.load_by_program_slot(slot, Some(&program_pubkey)) + .into_iter() + .filter(|ta| filter(&ta.1)) + .collect(); + + + let result = transaction_accounts + .into_iter() + .map(|ta| { Self::convert_to_account_data(ta.0, slot, ta.1) }) + .collect_vec(); + + if result.is_empty() { + None + } else { + Some(result) + } } @@ -175,77 +157,62 @@ impl AccountStorageInterface for AccountsDb { self.accounts.add_root(slot); } - let commitments = self.commitments.read().await.clone(); - - let processed = commitments.get(&Processed).cloned().unwrap_or(Slot::from(0u64)); - let confirmed = commitments.get(&Confirmed).cloned().unwrap_or(Slot::from(0u64)); - let finalized = commitments.get(&Finalized).cloned().unwrap_or(Slot::from(0u64)); + let processed = self.processed_slot.load(Ordering::Relaxed); + let confirmed = self.confirmed_slot.load(Ordering::Relaxed); + let finalized = self.finalised_slot.load(Ordering::Relaxed); match commitment { Processed => { if slot > processed { - self.commitments.write().await.insert(Processed, slot); + self.processed_slot.store(slot, Ordering::Relaxed); } } Confirmed => { if slot > processed { - self.commitments.write().await.insert(Processed, slot); + self.processed_slot.store(slot, Ordering::Relaxed); } if slot > confirmed { - self.commitments.write().await.insert(Confirmed, slot); + self.confirmed_slot.store(slot, Ordering::Relaxed); } } Finalized => { if slot > processed { - self.commitments.write().await.insert(Processed, slot); + self.processed_slot.store(slot, Ordering::Relaxed); } if slot > confirmed { - self.commitments.write().await.insert(Confirmed, slot); + self.confirmed_slot.store(slot, Ordering::Relaxed); } if slot > finalized { - self.commitments.write().await.insert(Finalized, slot); + self.finalised_slot.store(slot, Ordering::Relaxed); } } } - // FIXME do we need to return data from here? - why - // self.commitments.write().await.insert(commitment, slot); - vec![] + assert!(self.processed_slot.load(Ordering::Relaxed) >= self.confirmed_slot.load(Ordering::Relaxed)); + assert!(self.confirmed_slot.load(Ordering::Relaxed) >= self.finalised_slot.load(Ordering::Relaxed)); + + self.accounts.load_all(&Ancestors::from(vec![slot]), slot, false) + .unwrap() + .into_iter() + .filter(|(_, _, updated_slot)| *updated_slot == slot) + .map(|(key, data, slot)| Self::convert_to_account_data(key, slot, data)) + .collect_vec() } } impl AccountsDb { - // borrowed from solana accounts.rs - fn load_while_filtering bool>( - collector: &mut Vec, - some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>, - filter: F, - ) { - if let Some(mapped_account_tuple) = some_account_tuple - .filter(|(_, account, _)| Self::is_loadable(account.lamports()) && filter(account)) - .map(|(pubkey, account, _slot)| (*pubkey, account)) - { - collector.push(mapped_account_tuple) + fn get_slot_from_commitment(&self, commitment: Commitment) -> Slot { + match commitment { + Processed => self.processed_slot.load(Ordering::Relaxed), + Confirmed => self.confirmed_slot.load(Ordering::Relaxed), + Finalized => self.finalised_slot.load(Ordering::Relaxed) } } - // borrowed from solana accounts.rs - fn is_loadable(lamports: u64) -> bool { - // Don't ever load zero lamport accounts into runtime because - // the existence of zero-lamport accounts are never deterministic!! - lamports > 0 - } - - async fn get_slot_from_commitment(&self, commitment: Commitment) -> Slot { - let lock = self.commitments.read().await; - let result = lock.get(&commitment).unwrap().clone(); - result - } - - async fn get_ancestors_from_commitment(&self, commitment: Commitment) -> Ancestors { - let slot = self.get_slot_from_commitment(commitment).await; + fn get_ancestors_from_commitment(&self, commitment: Commitment) -> Ancestors { + let slot = self.get_slot_from_commitment(commitment); Ancestors::from(vec![slot]) } @@ -270,7 +237,7 @@ mod tests { use solana_sdk::pubkey::Pubkey; - use solana_lite_rpc_core::commitment_utils::Commitment::Confirmed; + use solana_lite_rpc_core::commitment_utils::Commitment::Processed; use crate::account_store_interface::AccountStorageInterface; use crate::store::accounts_db::create_account_data; @@ -278,15 +245,394 @@ mod tests { #[tokio::test] async fn store_new_account() { - let test_instance = AccountsDb::new_for_testing(); + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + let ad = create_account_data(2, ak, pk, 1); + ti.initilize_or_update_account(ad).await; + + ti.process_slot_data(2, Processed).await; + + let result = ti.get_account(ak, Processed).await; + assert!(result.is_ok()); + let data = result.unwrap().unwrap(); + assert_eq!(data.updated_slot, 2); + assert_eq!(data.account.lamports, 1); + } + + mod get_account { + use std::str::FromStr; + + use solana_sdk::pubkey::Pubkey; + + use solana_lite_rpc_core::commitment_utils::Commitment::{Confirmed, Finalized, Processed}; + + use crate::account_store_interface::AccountStorageInterface; + use crate::store::accounts_db::create_account_data; + use crate::store::AccountsDb; + + #[tokio::test] + async fn different_commitments() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.process_slot_data(4, Confirmed).await; + ti.process_slot_data(3, Finalized).await; + + ti.initilize_or_update_account(create_account_data(5, ak, pk, 10)).await; + ti.initilize_or_update_account(create_account_data(4, ak, pk, 20)).await; + ti.initilize_or_update_account(create_account_data(3, ak, pk, 30)).await; + + let processed = ti.get_account(ak, Processed).await.unwrap().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_account(ak, Confirmed).await.unwrap().unwrap(); + assert_eq!(confirmed.updated_slot, 4); + assert_eq!(confirmed.account.lamports, 20); + + let finalized = ti.get_account(ak, Finalized).await.unwrap().unwrap(); + assert_eq!(finalized.updated_slot, 3); + assert_eq!(finalized.account.lamports, 30); + } + + #[tokio::test] + async fn becoming_available_after_slot_update() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + ti.initilize_or_update_account(create_account_data(5, ak, pk, 10)).await; + +// Slot = Processed + ti.process_slot_data(5, Processed).await; + + let processed = ti.get_account(ak, Processed).await.unwrap().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_account(ak, Confirmed).await.unwrap(); + assert_eq!(confirmed, None); + + let finalized = ti.get_account(ak, Finalized).await.unwrap(); + assert_eq!(finalized, None); + +// Slot = Confirmed + ti.process_slot_data(5, Confirmed).await; + + let processed = ti.get_account(ak, Processed).await.unwrap().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_account(ak, Confirmed).await.unwrap().unwrap(); + assert_eq!(confirmed.updated_slot, 5); + assert_eq!(confirmed.account.lamports, 10); + + let finalized = ti.get_account(ak, Finalized).await.unwrap(); + assert_eq!(finalized, None); + +// Slot = Finalized + ti.process_slot_data(5, Finalized).await; + + let processed = ti.get_account(ak, Processed).await.unwrap().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_account(ak, Confirmed).await.unwrap().unwrap(); + assert_eq!(confirmed.updated_slot, 5); + assert_eq!(confirmed.account.lamports, 10); + + let finalized = ti.get_account(ak, Finalized).await.unwrap().unwrap(); + assert_eq!(finalized.updated_slot, 5); + assert_eq!(finalized.account.lamports, 10); + } + } + + mod get_program_accounts { + use std::str::FromStr; + + use solana_rpc_client_api::filter::{Memcmp, RpcFilterType}; + use solana_sdk::pubkey::Pubkey; + + use solana_lite_rpc_core::commitment_utils::Commitment::{Confirmed, Finalized, Processed}; + + use crate::account_store_interface::AccountStorageInterface; + use crate::store::accounts_db::{create_account_data, create_account_data_with_data}; + use crate::store::AccountsDb; + + #[tokio::test] + async fn different_commitments() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.process_slot_data(4, Confirmed).await; + ti.process_slot_data(3, Finalized).await; + + ti.initilize_or_update_account(create_account_data(5, ak, pk, 10)).await; + ti.initilize_or_update_account(create_account_data(4, ak, pk, 20)).await; + ti.initilize_or_update_account(create_account_data(3, ak, pk, 30)).await; + + let processed = ti.get_program_accounts(pk, None, Processed).await.unwrap().pop().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_program_accounts(pk, None, Confirmed).await.unwrap().pop().unwrap(); + assert_eq!(confirmed.updated_slot, 4); + assert_eq!(confirmed.account.lamports, 20); + + let finalized = ti.get_program_accounts(pk, None, Finalized).await.unwrap().pop().unwrap(); + assert_eq!(finalized.updated_slot, 3); + assert_eq!(finalized.account.lamports, 30); + } + + #[tokio::test] + async fn becoming_available_after_slot_update() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + ti.initilize_or_update_account(create_account_data(5, ak, pk, 10)).await; + +// Slot = Processed + ti.process_slot_data(5, Processed).await; + + let processed = ti.get_program_accounts(pk, None, Processed).await.unwrap().pop().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_program_accounts(pk, None, Confirmed).await; + assert_eq!(confirmed, None); + + let finalized = ti.get_program_accounts(pk, None, Finalized).await; + assert_eq!(finalized, None); + +// Slot = Confirmed + ti.process_slot_data(5, Confirmed).await; + + let processed = ti.get_program_accounts(pk, None, Processed).await.unwrap().pop().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); + + let confirmed = ti.get_program_accounts(pk, None, Confirmed).await.unwrap().pop().unwrap(); + assert_eq!(confirmed.updated_slot, 5); + assert_eq!(confirmed.account.lamports, 10); + + let finalized = ti.get_program_accounts(pk, None, Finalized).await; + assert_eq!(finalized, None); - let program_key = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); - let account_1_key = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); +// Slot = Finalized + ti.process_slot_data(5, Finalized).await; - let account_1_data = create_account_data(1, account_1_key, program_key, &[0u8; 23]); - test_instance.initilize_or_update_account(account_1_data).await; + let processed = ti.get_program_accounts(pk, None, Processed).await.unwrap().pop().unwrap(); + assert_eq!(processed.updated_slot, 5); + assert_eq!(processed.account.lamports, 10); - let result = test_instance.get_account(account_1_key, Confirmed).await; + let confirmed = ti.get_program_accounts(pk, None, Confirmed).await.unwrap().pop().unwrap(); + assert_eq!(confirmed.updated_slot, 5); + assert_eq!(confirmed.account.lamports, 10); + + let finalized = ti.get_program_accounts(pk, None, Finalized).await.unwrap().pop().unwrap(); + assert_eq!(finalized.updated_slot, 5); + assert_eq!(finalized.account.lamports, 10); + } + + #[tokio::test] + async fn filter_by_data_size() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak1 = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + let ak2 = Pubkey::from_str("5VsPdDtqyFw6BmxrTZXKfnTLZy3TgzVA2MA1vZKAfddw").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak1, pk, Vec::from("abc"))).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak2, pk, Vec::from("abcdef"))).await; + + let mut result = ti.get_program_accounts(pk, Some(vec![RpcFilterType::DataSize(3)]), Processed).await.unwrap(); + assert_eq!(result.len(), 1); + let result = result.pop().unwrap(); + assert_eq!(result.pubkey, ak1); + assert_eq!(result.updated_slot, 5); + assert_eq!(result.account.data, Vec::from("abc")); + } + + #[tokio::test] + async fn filter_by_mem_cmp() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak1 = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + let ak2 = Pubkey::from_str("5VsPdDtqyFw6BmxrTZXKfnTLZy3TgzVA2MA1vZKAfddw").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak1, pk, Vec::from("abc"))).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak2, pk, Vec::from("abcdef"))).await; + + let mut result = ti.get_program_accounts(pk, Some(vec![RpcFilterType::Memcmp(Memcmp::new_raw_bytes(1, Vec::from("bcdef")))]), Processed).await.unwrap(); + assert_eq!(result.len(), 1); + let result = result.pop().unwrap(); + assert_eq!(result.pubkey, ak2); + assert_eq!(result.updated_slot, 5); + assert_eq!(result.account.data, Vec::from("abcdef")); + } + + #[tokio::test] + async fn multiple_filter() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak1 = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + let ak2 = Pubkey::from_str("5VsPdDtqyFw6BmxrTZXKfnTLZy3TgzVA2MA1vZKAfddw").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak1, pk, Vec::from("abc"))).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak2, pk, Vec::from("abcdef"))).await; + + let mut result = ti.get_program_accounts(pk, Some(vec![ + RpcFilterType::DataSize(6), + RpcFilterType::Memcmp(Memcmp::new_raw_bytes(1, Vec::from("bcdef"))), + ]), Processed).await.unwrap(); + + assert_eq!(result.len(), 1); + let result = result.pop().unwrap(); + assert_eq!(result.pubkey, ak2); + assert_eq!(result.updated_slot, 5); + assert_eq!(result.account.data, Vec::from("abcdef")); + } + + + #[tokio::test] + async fn contradicting_filter() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak1 = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + let ak2 = Pubkey::from_str("5VsPdDtqyFw6BmxrTZXKfnTLZy3TgzVA2MA1vZKAfddw").unwrap(); + + ti.process_slot_data(5, Processed).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak1, pk, Vec::from("abc"))).await; + ti.initilize_or_update_account(create_account_data_with_data(5, ak2, pk, Vec::from("abcdef"))).await; + + let result = ti.get_program_accounts(pk, Some(vec![ + RpcFilterType::Memcmp(Memcmp::new_raw_bytes(0, Vec::from("a"))), + RpcFilterType::Memcmp(Memcmp::new_raw_bytes(0, Vec::from("b"))), + ]), Processed).await; + assert_eq!(result, None); + } + } + + mod process_slot_data { + use std::str::FromStr; + use std::sync::atomic::Ordering::Relaxed; + + use solana_sdk::pubkey::Pubkey; + + use solana_lite_rpc_core::commitment_utils::Commitment::{Confirmed, Finalized, Processed}; + + use crate::account_store_interface::AccountStorageInterface; + use crate::store::accounts_db::create_account_data; + use crate::store::AccountsDb; + + #[tokio::test] + async fn first_time_invocation() { + let ti = AccountsDb::new_for_testing(); + + ti.process_slot_data(3, Processed).await; + ti.process_slot_data(2, Confirmed).await; + ti.process_slot_data(1, Finalized).await; + + assert_eq!(ti.processed_slot.load(Relaxed), 3); + assert_eq!(ti.confirmed_slot.load(Relaxed), 2); + assert_eq!(ti.finalised_slot.load(Relaxed), 1); + } + + #[tokio::test] + async fn only_updates_processed_slot() { + let ti = AccountsDb::new_for_testing(); + + ti.process_slot_data(1, Processed).await; + ti.process_slot_data(1, Confirmed).await; + ti.process_slot_data(1, Finalized).await; + + ti.process_slot_data(2, Processed).await; + + assert_eq!(ti.processed_slot.load(Relaxed), 2); + assert_eq!(ti.confirmed_slot.load(Relaxed), 1); + assert_eq!(ti.finalised_slot.load(Relaxed), 1); + } + + #[tokio::test] + async fn update_processed_slot_when_confirmed_slot_is_ahead() { + let ti = AccountsDb::new_for_testing(); + + ti.process_slot_data(1, Processed).await; + ti.process_slot_data(1, Confirmed).await; + ti.process_slot_data(1, Finalized).await; + + ti.process_slot_data(2, Confirmed).await; + + assert_eq!(ti.processed_slot.load(Relaxed), 2); + assert_eq!(ti.confirmed_slot.load(Relaxed), 2); + assert_eq!(ti.finalised_slot.load(Relaxed), 1); + } + + #[tokio::test] + async fn update_processed_and_confirmed_slot_when_finalized_slot_is_ahead() { + let ti = AccountsDb::new_for_testing(); + + ti.process_slot_data(1, Processed).await; + ti.process_slot_data(1, Confirmed).await; + ti.process_slot_data(1, Finalized).await; + + ti.process_slot_data(2, Finalized).await; + + assert_eq!(ti.processed_slot.load(Relaxed), 2); + assert_eq!(ti.confirmed_slot.load(Relaxed), 2); + assert_eq!(ti.finalised_slot.load(Relaxed), 2); + } + + #[tokio::test] + async fn returns_updated_account_if_commitment_changes_to_finalized() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + let result = ti.process_slot_data(3, Processed).await; + assert_eq!(result.len(), 0); + + ti.initilize_or_update_account(create_account_data(3, ak, pk, 10)).await; + + let result = ti.process_slot_data(3, Confirmed).await; + assert_eq!(result.len(), 0); + + let result = ti.process_slot_data(3, Finalized).await; + assert_eq!(result.len(), 1) + } + + #[tokio::test] + async fn does_not_return_updated_account_if_different_slot_gets_finalized() { + let ti = AccountsDb::new_for_testing(); + + let pk = Pubkey::from_str("HZGMUF6kdCUK6nuc3TdNR6X5HNdGtg5HmVQ8cV2pRiHE").unwrap(); + let ak = Pubkey::from_str("6rRiMihF7UdJz25t5QvS7PgP9yzfubN7TBRv26ZBVAhE").unwrap(); + + ti.initilize_or_update_account(create_account_data(3, ak, pk, 10)).await; + ti.process_slot_data(3, Finalized).await; + + let result = ti.process_slot_data(4, Finalized).await; + assert_eq!(result.len(), 0) + } } } @@ -294,112 +640,37 @@ pub fn create_account_data( updated_slot: Slot, pubkey: Pubkey, program: Pubkey, - data: &[u8], + lamports: u64, ) -> AccountData { AccountData { pubkey, account: Arc::new(Account { - lamports: 42, - data: Vec::from(data), + lamports, + data: Vec::from([]), owner: program, executable: false, rent_epoch: 0, }), updated_slot, } +} - // #[test] - // fn test() { - // let db = AccountsDb::new_single_for_tests(); - // let accounts = Accounts::new(Arc::new(db)); - // - // // let num_slots = 4; - // // let num_accounts = 10_000; - // // println!("Creating {num_accounts} accounts"); - // - // // let pubkeys: Vec<_> = (0..num_slots) - // // .into_iter() - // // .map(|slot| { - // // let mut pubkeys: Vec = vec![]; - // // create_test_accounts( - // // &accounts, - // // &mut pubkeys, - // // num_accounts / num_slots, - // // slot as u64, - // // ); - // // pubkeys - // // }) - // // .collect(); - // // - // // let pubkeys: Vec<_> = pubkeys.into_iter().flatten().collect(); - // - // // println!("{:?}", pubkeys); - // let pubkey = solana_sdk::pubkey::new_rand(); - // let mut rng = rand::thread_rng(); - // let program = Pubkey::new_unique(); - // // let acc = create_random_account( - // // &mut rng, - // // 1, - // // pubkey, - // // program, - // // ); - // // println!("{acc:?}"); - // - // let account = AccountSharedData::new( - // 1 as u64, - // 0, - // AccountSharedData::default().owner(), - // ); - // - // println!("{program:?}"); - // - // let account_for_storage = [(&pubkey, &account)]; - // let to_store = (1u64, account_for_storage.as_slice()); - // - // accounts.store_accounts_cached(to_store) - // } - // - // fn create_random_account( - // rng: &mut ThreadRng, - // updated_slot: Slot, - // pubkey: Pubkey, - // program: Pubkey, - // ) -> AccountData { - // let length: usize = rng.gen_range(100..1000); - // AccountData { - // pubkey, - // account: Arc::new(Account { - // lamports: rng.gen(), - // data: (0..length).map(|_| rng.gen::()).collect_vec(), - // owner: program, - // executable: false, - // rent_epoch: 0, - // }), - // updated_slot, - // } - // } -} -// -// pub fn create_test_accounts( -// accounts: &Accounts, -// pubkeys: &mut Vec, -// num: usize, -// slot: Slot, -// ) { -// let data_size = 0; -// -// for t in 0..num { -// let pubkey = solana_sdk::pubkey::new_rand(); -// let account = AccountSharedData::new( -// (t + 1) as u64, -// data_size, -// AccountSharedData::default().owner(), -// ); -// // accounts.store_slow_uncached(slot, &pubkey, &account); -// let random_account = self -// -// accounts.store_accounts_cached() -// pubkeys.push(pubkey); -// } -// } +pub fn create_account_data_with_data( + updated_slot: Slot, + ak: Pubkey, + program: Pubkey, + data: Vec, +) -> AccountData { + AccountData { + pubkey: ak, + account: Arc::new(Account { + lamports: 1, + data, + owner: program, + executable: false, + rent_epoch: 0, + }), + updated_slot, + } +} \ No newline at end of file diff --git a/core/src/structures/account_data.rs b/core/src/structures/account_data.rs index 362819ba..e479e1f0 100644 --- a/core/src/structures/account_data.rs +++ b/core/src/structures/account_data.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use solana_rpc_client_api::filter::RpcFilterType; use solana_sdk::{account::Account, pubkey::Pubkey, slot_history::Slot}; -use solana_sdk::account::AccountSharedData; use tokio::sync::broadcast::Receiver; use crate::commitment_utils::Commitment; From c0e83b158580494cdde86b3e76ad39a66fe62bad Mon Sep 17 00:00:00 2001 From: mango-dee Date: Fri, 29 Nov 2024 14:51:55 +0800 Subject: [PATCH 04/10] Prepare PR --- accounts/src/store/accounts_db.rs | 2 +- lite-rpc/src/cli.rs | 24 +++++++--- lite-rpc/src/main.rs | 79 ++++++++++++++++--------------- 3 files changed, 58 insertions(+), 47 deletions(-) diff --git a/accounts/src/store/accounts_db.rs b/accounts/src/store/accounts_db.rs index 142ca011..233418ec 100644 --- a/accounts/src/store/accounts_db.rs +++ b/accounts/src/store/accounts_db.rs @@ -11,7 +11,7 @@ use solana_accounts_db::ancestors::Ancestors; use solana_accounts_db::partitioned_rewards::TestPartitionedEpochRewards; use solana_rpc_client_api::filter::RpcFilterType; use solana_sdk::account::{Account, AccountSharedData, ReadableAccount}; -use solana_sdk::clock::{BankId, Slot}; +use solana_sdk::clock::Slot; use solana_sdk::genesis_config::ClusterType; use solana_sdk::pubkey::Pubkey; use solana_sdk::transaction_context::TransactionAccount; diff --git a/lite-rpc/src/cli.rs b/lite-rpc/src/cli.rs index ed20ea35..aabec40a 100644 --- a/lite-rpc/src/cli.rs +++ b/lite-rpc/src/cli.rs @@ -1,20 +1,22 @@ +use std::{env, time::Duration}; use std::borrow::Cow; use std::fmt::{Debug, Display, Formatter}; use std::net::SocketAddr; use std::str::FromStr; -use std::{env, time::Duration}; -use crate::postgres_logger::{self, PostgresSessionConfig}; -use crate::{ - DEFAULT_FANOUT_SIZE, DEFAULT_GRPC_ADDR, DEFAULT_RETRY_TIMEOUT, DEFAULT_RPC_ADDR, - DEFAULT_WS_ADDR, MAX_RETRIES, -}; use anyhow::Context; use clap::Parser; use dotenv::dotenv; -use solana_lite_rpc_services::quic_connection_utils::QuicConnectionParameters; use solana_rpc_client_api::client_error::reqwest::Url; +use solana_lite_rpc_services::quic_connection_utils::QuicConnectionParameters; + +use crate::{ + DEFAULT_FANOUT_SIZE, DEFAULT_GRPC_ADDR, DEFAULT_RETRY_TIMEOUT, DEFAULT_RPC_ADDR, + DEFAULT_WS_ADDR, MAX_RETRIES, +}; +use crate::postgres_logger::{self, PostgresSessionConfig}; + #[derive(Parser, Debug, Clone)] #[command(author, version, about, long_about = None)] pub struct Args { @@ -89,6 +91,9 @@ pub struct Config { #[serde(default)] pub account_filters: Option, + #[serde(default)] + pub use_accounts_db: Option, + #[serde(default)] pub enable_accounts_on_demand_accounts_service: Option, @@ -219,6 +224,11 @@ impl Config { config.account_filters = env::var("ACCOUNT_FILTERS").ok().or(config.account_filters); + config.use_accounts_db = env::var("USE_ACCOUNTS_DB") + .map(|value| value.parse::().unwrap()) + .ok() + .or(config.use_accounts_db); + config.enable_accounts_on_demand_accounts_service = env::var("ENABLE_ACCOUNT_ON_DEMAND") .map(|value| value.parse::().unwrap()) .ok() diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 91a9dfec..2c908245 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -1,25 +1,37 @@ -pub mod rpc_tester; +use std::net::{SocketAddr, ToSocketAddrs}; +use std::sync::Arc; +use std::time::Duration; -use crate::rpc_tester::RpcTester; use anyhow::bail; use dashmap::DashMap; use itertools::Itertools; +use log::info; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_sdk::commitment_config::CommitmentConfig; +use solana_sdk::signature::Keypair; +use solana_sdk::signer::Signer; +use tokio::io::AsyncReadExt; +use tokio::sync::mpsc; +use tokio::sync::RwLock; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::fmt::format::FmtSpan; + use lite_rpc::bridge::LiteBridge; use lite_rpc::bridge_pubsub::LitePubSubBridge; use lite_rpc::cli::Config; +use lite_rpc::DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE; +use lite_rpc::postgres_logger; use lite_rpc::postgres_logger::PostgresLogger; use lite_rpc::service_spawner::ServiceSpawner; use lite_rpc::start_server::start_servers; -use lite_rpc::DEFAULT_MAX_NUMBER_OF_TXS_IN_QUEUE; -use log::info; use solana_lite_rpc_accounts::account_service::AccountService; use solana_lite_rpc_accounts::account_store_interface::AccountStorageInterface; use solana_lite_rpc_accounts::inmemory_account_store::InmemoryAccountStore; +use solana_lite_rpc_accounts::store::AccountsDb; use solana_lite_rpc_accounts_on_demand::accounts_on_demand::AccountsOnDemand; use solana_lite_rpc_address_lookup_tables::address_lookup_table_store::AddressLookupTableStore; use solana_lite_rpc_blockstore::history::History; use solana_lite_rpc_cluster_endpoints::endpoint_stremers::EndpointStreaming; - use solana_lite_rpc_cluster_endpoints::geyser_grpc_connector::{ GrpcConnectionTimeouts, GrpcSourceConfig, }; @@ -30,6 +42,7 @@ use solana_lite_rpc_cluster_endpoints::grpc_subscription::create_grpc_subscripti use solana_lite_rpc_cluster_endpoints::json_rpc_leaders_getter::JsonRpcLeaderGetter; use solana_lite_rpc_cluster_endpoints::json_rpc_subscription::create_json_rpc_polling_subscription; use solana_lite_rpc_cluster_endpoints::rpc_polling::poll_blocks::NUM_PARALLEL_TASKS_DEFAULT; +use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_core::keypair_loader::load_identity_keypair; use solana_lite_rpc_core::stores::{ block_information_store::{BlockInformation, BlockInformationStore}, @@ -38,38 +51,26 @@ use solana_lite_rpc_core::stores::{ subscription_store::SubscriptionStore, tx_store::TxStore, }; -use solana_lite_rpc_core::structures::account_filter::AccountFilters; -use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use solana_lite_rpc_core::structures::{ epoch::EpochCache, identity_stakes::IdentityStakes, notifications::NotificationSender, }; +use solana_lite_rpc_core::structures::account_filter::AccountFilters; +use solana_lite_rpc_core::structures::leaderschedule::CalculatedSchedule; use solana_lite_rpc_core::traits::address_lookup_table_interface::AddressLookupTableInterface; use solana_lite_rpc_core::types::BlockStream; use solana_lite_rpc_core::utils::wait_till_block_of_commitment_is_recieved; -use solana_lite_rpc_core::AnyhowJoinHandle; use solana_lite_rpc_prioritization_fees::account_prio_service::AccountPrioService; +use solana_lite_rpc_prioritization_fees::start_block_priofees_task; use solana_lite_rpc_services::data_caching_service::DataCachingService; use solana_lite_rpc_services::tpu_utils::tpu_connection_path::TpuConnectionPath; use solana_lite_rpc_services::tpu_utils::tpu_service::{TpuService, TpuServiceConfig}; use solana_lite_rpc_services::transaction_replayer::TransactionReplayer; use solana_lite_rpc_services::tx_sender::TxSender; - -use lite_rpc::postgres_logger; -use solana_lite_rpc_prioritization_fees::start_block_priofees_task; use solana_lite_rpc_util::obfuscate_rpcurl; -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_sdk::commitment_config::CommitmentConfig; -use solana_sdk::signature::Keypair; -use solana_sdk::signer::Signer; -use std::net::{SocketAddr, ToSocketAddrs}; -use std::sync::Arc; -use std::time::Duration; -use tokio::io::AsyncReadExt; -use tokio::sync::mpsc; -use tokio::sync::RwLock; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::EnvFilter; -use solana_lite_rpc_accounts::store::AccountsDb; + +use crate::rpc_tester::RpcTester; + +pub mod rpc_tester; // jemalloc seems to be better at keeping the memory footprint reasonable over // longer periods of time @@ -115,6 +116,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: enable_address_lookup_tables, address_lookup_tables_binary, account_filters, + use_accounts_db, enable_accounts_on_demand_accounts_service, quic_connection_parameters, .. @@ -203,11 +205,11 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: BlockInformationStore::new(BlockInformation::from_block_info(&finalized_block_info)); let accounts_service = if let Some(account_stream) = processed_account_stream { - // lets use inmemory storage for now - // let inmemory_account_storage: Arc = - // Arc::new(InmemoryAccountStore::new()); - - let accounts_storage: Arc = Arc::new(AccountsDb::new()); + let accounts_storage: Arc = if use_accounts_db.unwrap_or(false) { + Arc::new(AccountsDb::new()) + } else { + Arc::new(InmemoryAccountStore::new()) + }; const MAX_CONNECTIONS_IN_PARALLEL: usize = 10; // Accounts notifications will be spurious when slots change @@ -227,7 +229,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: let account_service = AccountService::new( account_storage, - account_notification_sender + account_notification_sender, ); account_service.process_account_stream( @@ -235,14 +237,13 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: blockinfo_notifier.resubscribe(), ); - // FIXME there should be a flag so that development can happen faster - // account_service - // .populate_from_rpc( - // rpc_client.clone(), - // &account_filters, - // MAX_CONNECTIONS_IN_PARALLEL, - // ) - // .await?; + account_service + .populate_from_rpc( + rpc_client.clone(), + &account_filters, + MAX_CONNECTIONS_IN_PARALLEL, + ) + .await?; Some(account_service) } else { None @@ -330,7 +331,7 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: leader_schedule, data_cache.clone(), ) - .await?; + .await?; let tx_sender = TxSender::new(data_cache.clone(), tpu_service.clone()); let tx_replayer = TransactionReplayer::new(tpu_service.clone(), data_cache.clone(), retry_after); From 057c343a14da1a79e265f4c40d4ae37d74ee0343 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 9 Dec 2024 12:37:39 +0100 Subject: [PATCH 05/10] bump version --- Cargo.lock | 97 +++++++------------------------- Cargo.toml | 44 ++++++++------- accounts-on-demand/Cargo.toml | 2 +- accounts/Cargo.toml | 2 +- address-lookup-tables/Cargo.toml | 2 +- bench/Cargo.toml | 2 +- benchrunner-service/Cargo.toml | 2 +- blockstore/Cargo.toml | 2 +- cluster-endpoints/Cargo.toml | 2 +- core/Cargo.toml | 2 +- lite-rpc/Cargo.toml | 2 +- prioritization_fees/Cargo.toml | 2 +- services/Cargo.toml | 2 +- stake_vote/Cargo.toml | 2 +- util/Cargo.toml | 2 +- 15 files changed, 56 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46d29ea2..6af8902f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -577,7 +577,7 @@ dependencies = [ [[package]] name = "bench" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "bincode", @@ -1300,7 +1300,7 @@ dependencies = [ [[package]] name = "custom-tpu-send-transactions" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "bincode", @@ -1881,8 +1881,8 @@ dependencies = [ "regex", "solana-sdk", "tokio", - "tonic 0.12.3", - "tonic-health 0.12.3", + "tonic", + "tonic-health", "tracing", "url", "yellowstone-grpc-client", @@ -2907,7 +2907,7 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lite-rpc" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-channel", @@ -3765,16 +3765,6 @@ dependencies = [ "prost-derive 0.11.9", ] -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive 0.12.6", -] - [[package]] name = "prost" version = "0.13.3" @@ -3841,19 +3831,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools 0.12.1", - "proc-macro2", - "quote", - "syn 2.0.89", -] - [[package]] name = "prost-derive" version = "0.13.3" @@ -5067,7 +5044,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-accounts" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5108,7 +5085,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-accounts-on-demand" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5151,7 +5128,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-address-lookup-tables" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5189,7 +5166,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-benchrunner-service" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5220,7 +5197,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-blockstore" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5254,7 +5231,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-cluster-endpoints" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-channel", @@ -5290,14 +5267,14 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tonic-health 0.10.2", + "tonic-health", "tracing", "yellowstone-grpc-proto", ] [[package]] name = "solana-lite-rpc-core" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-trait", @@ -5332,7 +5309,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-prioritization-fees" -version = "0.2.4" +version = "0.3.0" dependencies = [ "dashmap", "itertools 0.10.5", @@ -5431,7 +5408,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-services" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-channel", @@ -5471,7 +5448,7 @@ dependencies = [ [[package]] name = "solana-lite-rpc-util" -version = "0.2.4" +version = "0.3.0" dependencies = [ "anyhow", "async-channel", @@ -6956,27 +6933,6 @@ dependencies = [ "winnow 0.6.20", ] -[[package]] -name = "tonic" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" -dependencies = [ - "async-trait", - "base64 0.21.7", - "bytes", - "http 0.2.12", - "http-body 0.4.6", - "percent-encoding", - "pin-project", - "prost 0.12.6", - "tokio", - "tokio-stream", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.12.3" @@ -7025,19 +6981,6 @@ dependencies = [ "syn 2.0.89", ] -[[package]] -name = "tonic-health" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80db390246dfb46553481f6024f0082ba00178ea495dbb99e70ba9a4fafb5e1" -dependencies = [ - "async-stream", - "prost 0.12.6", - "tokio", - "tokio-stream", - "tonic 0.10.2", -] - [[package]] name = "tonic-health" version = "0.12.3" @@ -7048,7 +6991,7 @@ dependencies = [ "prost 0.13.3", "tokio", "tokio-stream", - "tonic 0.12.3", + "tonic", ] [[package]] @@ -7861,8 +7804,8 @@ dependencies = [ "bytes", "futures", "thiserror", - "tonic 0.12.3", - "tonic-health 0.12.3", + "tonic", + "tonic-health", "yellowstone-grpc-proto", ] @@ -7878,7 +7821,7 @@ dependencies = [ "solana-account-decoder", "solana-sdk", "solana-transaction-status", - "tonic 0.12.3", + "tonic", "tonic-build", ] diff --git a/Cargo.toml b/Cargo.toml index cf669411..9a7bcf1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,13 +21,26 @@ members = [ ] [workspace.package] -version = "0.2.4" +version = "0.3.0" authors = ["gmgalactus ", "GroovieGermanikus "] repository = "https://github.com/blockworks-foundation/lite-rpc" license = "AGPL" edition = "2021" [workspace.dependencies] + +solana-lite-rpc-services = { path = "services", version="0.3.0" } +solana-lite-rpc-core = { path = "core", version="0.3.0" } +solana-lite-rpc-util = { path = "util", version="0.3.0" } +solana-lite-rpc-cluster-endpoints = { path = "cluster-endpoints", version="0.3.0" } +solana-lite-rpc-blockstore = { path = "blockstore", version="0.3.0" } +solana-lite-rpc-stakevote = { path = "stake_vote", version="0.3.0" } +solana-lite-rpc-prioritization-fees = { path = "prioritization_fees", version="0.3.0" } +solana-lite-rpc-address-lookup-tables = { path = "address-lookup-tables", version="0.3.0" } +solana-lite-rpc-accounts = { path = "accounts", version = "0.3.0"} +solana-lite-rpc-accounts-on-demand = { path = "accounts-on-demand", version = "0.3.0"} +bench = { path = "bench", version="0.3.0" } + solana-sdk = "~2.0.16" solana-rpc-client = "~2.0.16" solana-rpc-client-api = "~2.0.16" @@ -42,6 +55,11 @@ solana-ledger = "~2.0.16" solana-program = "~2.0.16" solana-accounts-db = "~2.0.16" solana-address-lookup-table-program = "~2.0.16" + +yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v2.0.0+solana.2.0.16" } +geyser-grpc-connector = { git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git", branch = "2.0.16" } # FIXME use tag instead of branch + + itertools = "0.10.5" rangetools = "0.1.4" serde = { version = "1.0.160", features = ["derive"] } @@ -51,6 +69,7 @@ bs58 = "0.4.0" base64 = "0.21.0" borsh = "0.10.3" thiserror = "1.0.40" +async-trait = "0.1.68" futures = "0.3.28" futures-util = "0.3.28" bytes = "1.4.0" @@ -71,31 +90,14 @@ dotenv = "0.15.0" async-channel = "1.8.0" merge-streams = "0.1.2" jemallocator = "0.5" - quinn = "0.10.2" quinn-proto = "0.10.5" rustls = { version = "0.21.7", default-features = false, features = ["quic"] } rcgen = "0.9.3" -solana-lite-rpc-services = {path = "services", version="0.2.4"} -solana-lite-rpc-core = {path = "core", version="0.2.4"} -solana-lite-rpc-util = {path = "util", version="0.2.4"} -solana-lite-rpc-cluster-endpoints = {path = "cluster-endpoints", version="0.2.4"} -solana-lite-rpc-blockstore = {path = "blockstore", version="0.2.4"} -solana-lite-rpc-stakevote = {path = "stake_vote", version="0.2.4"} -solana-lite-rpc-prioritization-fees = {path = "prioritization_fees", version="0.2.4"} -solana-lite-rpc-address-lookup-tables = {path = "address-lookup-tables", version="0.2.4"} -solana-lite-rpc-accounts = {path = "accounts", version = "0.2.4"} -solana-lite-rpc-accounts-on-demand = {path = "accounts-on-demand", version = "0.2.4"} -bench = { path = "bench", version="0.2.4" } - -yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v2.0.0+solana.2.0.16" } -geyser-grpc-connector = { git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git", branch = "2.0.16" } # FIXME use tag instead of branch - - -async-trait = "0.1.68" -tonic-health = "0.10" +tonic = "0.12" +tonic-health = "0.12" [patch.crates-io.curve25519-dalek] git = "https://github.com/anza-xyz/curve25519-dalek.git" -rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" \ No newline at end of file +rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" diff --git a/accounts-on-demand/Cargo.toml b/accounts-on-demand/Cargo.toml index 21219332..d16e4591 100644 --- a/accounts-on-demand/Cargo.toml +++ b/accounts-on-demand/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-accounts-on-demand" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Library which implements accounts on demand service in lite-rpc. Whenever account is not available it will fetch from rpc and create a geyser stream to update it automatically." rust-version = "1.73.0" diff --git a/accounts/Cargo.toml b/accounts/Cargo.toml index a0635736..aa1b8b3b 100644 --- a/accounts/Cargo.toml +++ b/accounts/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-accounts" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Library which implements accounts in lite-rpc" rust-version = "1.73.0" diff --git a/address-lookup-tables/Cargo.toml b/address-lookup-tables/Cargo.toml index 1d34b6f2..0d57c2bf 100644 --- a/address-lookup-tables/Cargo.toml +++ b/address-lookup-tables/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-address-lookup-tables" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Library to save and update address lookup tables in lite-rpc" rust-version = "1.73.0" diff --git a/bench/Cargo.toml b/bench/Cargo.toml index ebd586a7..3a501f3d 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bench" -version = "0.2.4" +version = "0.3.0" edition = "2021" [[bin]] diff --git a/benchrunner-service/Cargo.toml b/benchrunner-service/Cargo.toml index 6949df82..0c5c6668 100644 --- a/benchrunner-service/Cargo.toml +++ b/benchrunner-service/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "solana-lite-rpc-benchrunner-service" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Service for running recurring benchmarks" rust-version = "1.73.0" diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml index 2e5c6846..a6cf8f33 100644 --- a/blockstore/Cargo.toml +++ b/blockstore/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-blockstore" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "History implementations used by solana lite rpc" rust-version = "1.75.0" diff --git a/cluster-endpoints/Cargo.toml b/cluster-endpoints/Cargo.toml index 2c0619e3..a7a8b914 100644 --- a/cluster-endpoints/Cargo.toml +++ b/cluster-endpoints/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-cluster-endpoints" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Core classes and methods used by solana lite rpc" rust-version = "1.75.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index dd62a55f..92aa5ef1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-core" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Core classes and methods used by solana lite rpc" rust-version = "1.75.0" diff --git a/lite-rpc/Cargo.toml b/lite-rpc/Cargo.toml index 0ad19bb2..5a0b391e 100644 --- a/lite-rpc/Cargo.toml +++ b/lite-rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lite-rpc" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "A lite version of solana rpc to send and confirm transactions" rust-version = "1.73.0" diff --git a/prioritization_fees/Cargo.toml b/prioritization_fees/Cargo.toml index b65f6334..84859b9d 100644 --- a/prioritization_fees/Cargo.toml +++ b/prioritization_fees/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-prioritization-fees" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Expose priority fees stats per block via RPC and WebSocket" diff --git a/services/Cargo.toml b/services/Cargo.toml index 47b09909..2f6e21b3 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-services" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Services used by solana lite rpc" rust-version = "1.73.0" diff --git a/stake_vote/Cargo.toml b/stake_vote/Cargo.toml index 84811433..68407faa 100644 --- a/stake_vote/Cargo.toml +++ b/stake_vote/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-stakevote" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "History implementations used by solana lite rpc" rust-version = "1.75.0" diff --git a/util/Cargo.toml b/util/Cargo.toml index b00e5ea7..901f676a 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solana-lite-rpc-util" -version = "0.2.4" +version = "0.3.0" edition = "2021" description = "Utility for non-domain functions" rust-version = "1.73.0" From 715a56dc2625dfe8e9b1af422565c29ed7a4a679 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 9 Dec 2024 13:01:32 +0100 Subject: [PATCH 06/10] cargo lock --- Cargo.lock | 145 +++++++++++++++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6af8902f..d926ab0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,7 +490,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "itoa", @@ -516,7 +516,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "mime", @@ -583,7 +583,7 @@ dependencies = [ "bincode", "clap 4.5.21", "csv", - "dashmap", + "dashmap 5.5.3", "dirs", "futures", "itertools 0.10.5", @@ -600,7 +600,7 @@ dependencies = [ "solana-rpc-client-api", "solana-sdk", "solana-transaction-status", - "spl-memo 4.0.4", + "spl-memo 4.0.0", "tokio", "tracing", "tracing-subscriber", @@ -1305,7 +1305,7 @@ dependencies = [ "anyhow", "bincode", "clap 4.5.21", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "log", @@ -1369,6 +1369,20 @@ dependencies = [ "rayon", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.6.0" @@ -1862,8 +1876,8 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "2.0.0" -source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?branch=2.0.16#214886a27da719807fbc3cbdbf7fb717bc280e7e" +version = "0.11.0+yellowstone.2.0.0" +source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?branch=2.0.16#9ce723fb06cdcd1981f7da7acffda59f5b4a613d" dependencies = [ "anyhow", "async-stream", @@ -1871,7 +1885,7 @@ dependencies = [ "bincode", "clap 4.5.21", "csv", - "dashmap", + "dashmap 6.1.0", "derive_more", "futures", "itertools 0.10.5", @@ -1982,7 +1996,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http 1.2.0", "indexmap 2.6.0", "slab", "tokio", @@ -2120,9 +2134,9 @@ dependencies = [ [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -2147,7 +2161,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http 1.2.0", ] [[package]] @@ -2158,7 +2172,7 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "pin-project-lite", ] @@ -2221,7 +2235,7 @@ dependencies = [ "futures-channel", "futures-util", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "httparse", "httpdate", @@ -2283,7 +2297,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "hyper 1.5.1", "pin-project-lite", @@ -2479,9 +2493,9 @@ dependencies = [ [[package]] name = "index_list" -version = "0.2.13" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e6ba961c14e98151cd6416dd3685efe786a94c38bc1a535c06ceff0a1600813" +checksum = "fa38453685e5fe724fd23ff6c1a158c1e2ca21ce0c2718fa11e96e70e99fd4de" [[package]] name = "indexmap" @@ -2565,15 +2579,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itoa" version = "1.0.13" @@ -2921,7 +2926,7 @@ dependencies = [ "chrono", "clap 4.5.21", "const_env", - "dashmap", + "dashmap 5.5.3", "dotenv", "futures", "futures-util", @@ -3767,12 +3772,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", - "prost-derive 0.13.3", + "prost-derive 0.13.4", ] [[package]] @@ -3799,20 +3804,19 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "bytes", "heck 0.5.0", - "itertools 0.13.0", + "itertools 0.12.1", "log", "multimap 0.10.0", "once_cell", "petgraph", "prettyplease 0.2.25", - "prost 0.13.3", - "prost-types 0.13.3", + "prost 0.13.4", + "prost-types 0.13.4", "regex", "syn 2.0.89", "tempfile", @@ -3833,12 +3837,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.89", @@ -3855,11 +3859,11 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ - "prost 0.13.3", + "prost 0.13.4", ] [[package]] @@ -4312,9 +4316,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.18" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "log", "once_cell", @@ -4795,7 +4799,7 @@ dependencies = [ "bytemuck_derive", "bzip2", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "index_list", "indexmap 2.6.0", "itertools 0.12.1", @@ -4911,7 +4915,7 @@ checksum = "e4042e0b2c2b44db89d59eab832b4a66c6fc03e2a7e9383e7b75faf76d162c61" dependencies = [ "async-trait", "bincode", - "dashmap", + "dashmap 5.5.3", "futures", "futures-util", "indexmap 2.6.0", @@ -5053,7 +5057,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "lazy_static", @@ -5094,7 +5098,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "lazy_static", @@ -5137,7 +5141,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "lazy_static", @@ -5205,7 +5209,7 @@ dependencies = [ "bincode", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", "futures-util", "itertools 0.10.5", @@ -5241,7 +5245,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "derive_more", "futures", "geyser-grpc-connector", @@ -5283,7 +5287,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "log", @@ -5311,7 +5315,7 @@ dependencies = [ name = "solana-lite-rpc-prioritization-fees" version = "0.3.0" dependencies = [ - "dashmap", + "dashmap 5.5.3", "itertools 0.10.5", "jsonrpsee", "lazy_static", @@ -5338,7 +5342,7 @@ dependencies = [ "bytes", "chrono", "clap 4.5.21", - "dashmap", + "dashmap 5.5.3", "dotenv", "futures", "itertools 0.10.5", @@ -5379,7 +5383,7 @@ dependencies = [ "clap 4.5.21", "countmap", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "dotenv", "futures", "itertools 0.10.5", @@ -5420,7 +5424,7 @@ dependencies = [ "chrono", "countmap", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "futures", "itertools 0.10.5", "lazy_static", @@ -5458,7 +5462,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap", + "dashmap 5.5.3", "derive_more", "futures", "itertools 0.10.5", @@ -5958,7 +5962,7 @@ dependencies = [ "async-channel", "bytes", "crossbeam-channel", - "dashmap", + "dashmap 5.5.3", "futures-util", "histogram", "indexmap 2.6.0", @@ -6304,9 +6308,9 @@ dependencies = [ [[package]] name = "spl-memo" -version = "4.0.4" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a49f49f95f2d02111ded31696ab38a081fab623d4c76bd4cb074286db4560836" +checksum = "f0f180b03318c3dbab3ef4e1e4d46d5211ae3c780940dd0a28695aba4b59a75a" dependencies = [ "solana-program 1.18.26", ] @@ -6832,12 +6836,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.18", - "rustls-pki-types", + "rustls 0.23.19", "tokio", ] @@ -6946,7 +6949,7 @@ dependencies = [ "bytes", "flate2", "h2 0.4.7", - "http 1.1.0", + "http 1.2.0", "http-body 1.0.1", "http-body-util", "hyper 1.5.1", @@ -6954,12 +6957,12 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost 0.13.3", + "prost 0.13.4", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "socket2", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -6975,8 +6978,8 @@ checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" dependencies = [ "prettyplease 0.2.25", "proc-macro2", - "prost-build 0.13.3", - "prost-types 0.13.3", + "prost-build 0.13.4", + "prost-types 0.13.4", "quote", "syn 2.0.89", ] @@ -6988,7 +6991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eaf34ddb812120f5c601162d5429933c9b527d901ab0e7f930d3147e33a09b2" dependencies = [ "async-stream", - "prost 0.13.3", + "prost 0.13.4", "tokio", "tokio-stream", "tonic", @@ -7816,7 +7819,7 @@ source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v2.0.0+solana. dependencies = [ "anyhow", "bincode", - "prost 0.13.3", + "prost 0.13.4", "protobuf-src", "solana-account-decoder", "solana-sdk", From 56c5fd852c83007db1e9c4d293165265bd099c8a Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 9 Dec 2024 13:04:30 +0100 Subject: [PATCH 07/10] use connector tag --- Cargo.lock | 78 +++++++++++++----------------------------------------- Cargo.toml | 5 ++-- 2 files changed, 20 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d926ab0b..2732f4e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,7 +583,7 @@ dependencies = [ "bincode", "clap 4.5.21", "csv", - "dashmap 5.5.3", + "dashmap", "dirs", "futures", "itertools 0.10.5", @@ -1305,7 +1305,7 @@ dependencies = [ "anyhow", "bincode", "clap 4.5.21", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "log", @@ -1369,20 +1369,6 @@ dependencies = [ "rayon", ] -[[package]] -name = "dashmap" -version = "6.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" -dependencies = [ - "cfg-if", - "crossbeam-utils", - "hashbrown 0.14.5", - "lock_api", - "once_cell", - "parking_lot_core", -] - [[package]] name = "data-encoding" version = "2.6.0" @@ -1877,22 +1863,14 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" version = "0.11.0+yellowstone.2.0.0" -source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?branch=2.0.16#9ce723fb06cdcd1981f7da7acffda59f5b4a613d" +source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?tag=v0.11.0+yellowstone.2.0.0+solana.2.0#78d33f3d3c55d0af6d0c02e4e0de8317f93c5c54" dependencies = [ "anyhow", "async-stream", - "base64 0.21.7", - "bincode", - "clap 4.5.21", - "csv", - "dashmap 6.1.0", - "derive_more", "futures", "itertools 0.10.5", "log", - "lz4_flex", "merge-streams", - "regex", "solana-sdk", "tokio", "tonic", @@ -2926,7 +2904,7 @@ dependencies = [ "chrono", "clap 4.5.21", "const_env", - "dashmap 5.5.3", + "dashmap", "dotenv", "futures", "futures-util", @@ -3007,15 +2985,6 @@ dependencies = [ "libc", ] -[[package]] -name = "lz4_flex" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" -dependencies = [ - "twox-hash", -] - [[package]] name = "matchers" version = "0.1.0" @@ -4799,7 +4768,7 @@ dependencies = [ "bytemuck_derive", "bzip2", "crossbeam-channel", - "dashmap 5.5.3", + "dashmap", "index_list", "indexmap 2.6.0", "itertools 0.12.1", @@ -4915,7 +4884,7 @@ checksum = "e4042e0b2c2b44db89d59eab832b4a66c6fc03e2a7e9383e7b75faf76d162c61" dependencies = [ "async-trait", "bincode", - "dashmap 5.5.3", + "dashmap", "futures", "futures-util", "indexmap 2.6.0", @@ -5057,7 +5026,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "lazy_static", @@ -5098,7 +5067,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "lazy_static", @@ -5141,7 +5110,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "lazy_static", @@ -5209,7 +5178,7 @@ dependencies = [ "bincode", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "futures", "futures-util", "itertools 0.10.5", @@ -5245,7 +5214,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "derive_more", "futures", "geyser-grpc-connector", @@ -5287,7 +5256,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "log", @@ -5315,7 +5284,7 @@ dependencies = [ name = "solana-lite-rpc-prioritization-fees" version = "0.3.0" dependencies = [ - "dashmap 5.5.3", + "dashmap", "itertools 0.10.5", "jsonrpsee", "lazy_static", @@ -5342,7 +5311,7 @@ dependencies = [ "bytes", "chrono", "clap 4.5.21", - "dashmap 5.5.3", + "dashmap", "dotenv", "futures", "itertools 0.10.5", @@ -5383,7 +5352,7 @@ dependencies = [ "clap 4.5.21", "countmap", "crossbeam-channel", - "dashmap 5.5.3", + "dashmap", "dotenv", "futures", "itertools 0.10.5", @@ -5424,7 +5393,7 @@ dependencies = [ "chrono", "countmap", "crossbeam-channel", - "dashmap 5.5.3", + "dashmap", "futures", "itertools 0.10.5", "lazy_static", @@ -5462,7 +5431,7 @@ dependencies = [ "bs58 0.4.0", "bytes", "chrono", - "dashmap 5.5.3", + "dashmap", "derive_more", "futures", "itertools 0.10.5", @@ -5962,7 +5931,7 @@ dependencies = [ "async-channel", "bytes", "crossbeam-channel", - "dashmap 5.5.3", + "dashmap", "futures-util", "histogram", "indexmap 2.6.0", @@ -6947,7 +6916,6 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "flate2", "h2 0.4.7", "http 1.2.0", "http-body 1.0.1", @@ -7182,16 +7150,6 @@ dependencies = [ "webpki-roots 0.24.0", ] -[[package]] -name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - [[package]] name = "typenum" version = "1.17.0" diff --git a/Cargo.toml b/Cargo.toml index 9a7bcf1e..dbf72a3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ solana-accounts-db = "~2.0.16" solana-address-lookup-table-program = "~2.0.16" yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v2.0.0+solana.2.0.16" } -geyser-grpc-connector = { git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git", branch = "2.0.16" } # FIXME use tag instead of branch +geyser-grpc-connector = { git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git", tag = "v0.11.0+yellowstone.2.0.0+solana.2.0" } itertools = "0.10.5" @@ -93,10 +93,9 @@ jemallocator = "0.5" quinn = "0.10.2" quinn-proto = "0.10.5" rustls = { version = "0.21.7", default-features = false, features = ["quic"] } -rcgen = "0.9.3" - tonic = "0.12" tonic-health = "0.12" +rcgen = "0.9.3" [patch.crates-io.curve25519-dalek] git = "https://github.com/anza-xyz/curve25519-dalek.git" From 4b68f6b52cf55219046c87d5341777f5066d55b7 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 9 Dec 2024 13:17:23 +0100 Subject: [PATCH 08/10] clippy+fmt --- accounts/src/inmemory_account_store.rs | 8 +++++--- .../src/address_lookup_table_store.rs | 14 ++++---------- .../examples/bench_postgres_simple_select.rs | 2 +- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/accounts/src/inmemory_account_store.rs b/accounts/src/inmemory_account_store.rs index 065366b6..d2dbfb14 100644 --- a/accounts/src/inmemory_account_store.rs +++ b/accounts/src/inmemory_account_store.rs @@ -92,10 +92,12 @@ impl AccountDataByCommitment { .unwrap_or(true); let mut updated = false; - if !self.processed_accounts.contains_key(&data.updated_slot) { + + if let std::collections::btree_map::Entry::Vacant(e) = + self.processed_accounts.entry(data.updated_slot) + { // processed not present for the slot - self.processed_accounts - .insert(data.updated_slot, data.clone()); + e.insert(data.clone()); updated = true; } diff --git a/address-lookup-tables/src/address_lookup_table_store.rs b/address-lookup-tables/src/address_lookup_table_store.rs index e0e28d33..4f359a73 100644 --- a/address-lookup-tables/src/address_lookup_table_store.rs +++ b/address-lookup-tables/src/address_lookup_table_store.rs @@ -170,17 +170,11 @@ impl AddressLookupTableStore { } pub async fn get_accounts(&self, alt: &Pubkey, accounts: &[u8]) -> Vec { - match self - .get_accounts_in_address_lookup_table(alt, accounts) + self.get_accounts_in_address_lookup_table(alt, accounts) .await - { - Some(x) => x, - None => { - // forget alt for now, start loading it for next blocks - // loading should be on its way - vec![] - } - } + // fallback to empty vec; forget alt for now, start loading it for next blocks + // loading should be on its way + .unwrap_or_default() } pub fn serialize_binary(&self) -> Vec { diff --git a/blockstore/examples/bench_postgres_simple_select.rs b/blockstore/examples/bench_postgres_simple_select.rs index 268c0696..ee070aad 100644 --- a/blockstore/examples/bench_postgres_simple_select.rs +++ b/blockstore/examples/bench_postgres_simple_select.rs @@ -27,7 +27,7 @@ pub async fn main() -> anyhow::Result<()> { } async fn parallel_queries(pg_session_config: PostgresSessionConfig) { - let many_sessions = vec![ + let many_sessions = [ PostgresSession::new(pg_session_config.clone()) .await .unwrap(), From 3b91ec7b0b800aeec56cea3b08eab1a782adec7e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 9 Dec 2024 13:25:01 +0100 Subject: [PATCH 09/10] docker: use 1.81.0 for building --- Dockerfile | 2 +- Dockerfile-benchrunner | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e9e42c5..e949214a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.2 -FROM rust:1.75.0 as base +FROM rust:1.81.0 as base RUN cargo install cargo-chef@0.1.62 --locked RUN rustup component add rustfmt RUN apt-get update && apt-get install -y clang cmake ssh diff --git a/Dockerfile-benchrunner b/Dockerfile-benchrunner index 866eb7d0..64035b8f 100644 --- a/Dockerfile-benchrunner +++ b/Dockerfile-benchrunner @@ -1,5 +1,5 @@ # syntax = docker/dockerfile:1.2 -FROM rust:1.75.0 as base +FROM rust:1.81.0 as base RUN cargo install cargo-chef@0.1.62 --locked RUN rustup component add rustfmt RUN apt-get update && apt-get install -y clang cmake ssh From 7cf9d8a9d4dccc6606707f145b24fa6f4b6d4361 Mon Sep 17 00:00:00 2001 From: mango-dee Date: Thu, 12 Dec 2024 12:46:54 +0800 Subject: [PATCH 10/10] Fixes build --- quic-forward-proxy/src/tls_self_signed_pair_generator.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs index e6af2d45..79052b38 100644 --- a/quic-forward-proxy/src/tls_self_signed_pair_generator.rs +++ b/quic-forward-proxy/src/tls_self_signed_pair_generator.rs @@ -35,8 +35,8 @@ impl SelfSignedTlsConfigProvider { fn gen_tls_certificate_and_key(hostnames: Vec) -> (Certificate, PrivateKey) { let cert = generate_simple_self_signed(hostnames).unwrap(); - let key = cert.key_pair.serialize_der(); - (Certificate(cert.key_pair.serialize_der()), PrivateKey(key)) + let key = cert.serialize_private_key_der(); + (Certificate(cert.serialize_der().unwrap()), PrivateKey(key)) } fn build_client_crypto_insecure() -> ClientConfig {