From 393932bf3ffc10bc51cd0b15923e12a3c9b2a36f Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Fri, 22 Nov 2024 15:43:59 +0200 Subject: [PATCH 1/3] update grpc --- Cargo.lock | 33 ++++++++++++++++++++++++++------- Cargo.toml | 4 ++-- solfees-be/Cargo.toml | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80975d0..c9ab5f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3899,7 +3899,7 @@ dependencies = [ "spl-token-group-interface", "spl-token-metadata-interface", "thiserror", - "zstd", + "zstd 0.11.2+zstd.1.5.2", ] [[package]] @@ -4636,7 +4636,7 @@ dependencies = [ [[package]] name = "solfees-be" -version = "2.0.0" +version = "2.1.0" dependencies = [ "anyhow", "bincode", @@ -5352,6 +5352,7 @@ dependencies = [ "tower-layer", "tower-service", "tracing", + "zstd 0.13.2", ] [[package]] @@ -6092,9 +6093,9 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c73838262cb899d64e2c94ac6ca2fb5bc5d565aa4f04a68b969645a40a06c2c" +checksum = "6adea92d06ab083f1c5845cf0aa9fb00320470573501f59d58c5c0c4e64720ae" dependencies = [ "bytes", "futures", @@ -6106,9 +6107,9 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b740e0601335c649640732e4cb4061a4ef4bb27017e8e76e4caa3c03566dd0e" +checksum = "91ffb8d470e9876b946d1ac103a022ae8522ec1ecd41e7883d609d31f035326d" dependencies = [ "anyhow", "bincode", @@ -6235,7 +6236,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe 7.2.1", ] [[package]] @@ -6248,6 +6258,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.13+zstd.1.5.6" diff --git a/Cargo.toml b/Cargo.toml index 44058d4..8c17a5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,8 +49,8 @@ tower = "0.5.1" tracing = "0.1.40" tracing-subscriber = "0.3.18" vergen = "9.0.0" -yellowstone-grpc-client = "2.0.0" -yellowstone-grpc-proto = "2.0.0" +yellowstone-grpc-client = "3.0.0" +yellowstone-grpc-proto = "3.0.0" [workspace.lints.clippy] clone_on_ref_ptr = "deny" diff --git a/solfees-be/Cargo.toml b/solfees-be/Cargo.toml index 37fb331..912d060 100644 --- a/solfees-be/Cargo.toml +++ b/solfees-be/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solfees-be" -version = "2.0.0" +version = "2.1.0" authors = { workspace = true } edition = { workspace = true } description = "Backend of solfees.io" From d10997363814d71b17729466184d871775f9db4d Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Fri, 22 Nov 2024 16:47:27 +0200 Subject: [PATCH 2/3] ignore txs with zero unit limit --- CHANGELOG.md | 6 ++++++ solfees-be/src/rpc_solana.rs | 6 +++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b868d3..d6362a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,12 @@ The minor version will be incremented upon a breaking change and the patch versi ### Breaking +## [2.1.0] - 2024-11-22 + +### Breaking + +- api: ??? ([#22](https://github.com/solana-stream-solutions/solfees/pull/22)) + ## [2.0.0] - 2024-11-18 - api: add getLeaderSchedule ([#1](https://github.com/solana-stream-solutions/solfees/pull/1)) diff --git a/solfees-be/src/rpc_solana.rs b/solfees-be/src/rpc_solana.rs index e9af61f..5c807f9 100644 --- a/solfees-be/src/rpc_solana.rs +++ b/solfees-be/src/rpc_solana.rs @@ -1325,6 +1325,7 @@ impl StreamsSlotInfo { let mut fees = Vec::with_capacity(self.transactions.len()); for transaction in self.transactions.iter().filter(|tx| { !tx.vote + && tx.unit_limit > 0 && filter .read_write .iter() @@ -1387,7 +1388,10 @@ impl RecentPrioritizationFeesSlot { let mut writable_account_fees = HashMap::>::with_capacity(transactions.len()); - for transaction in transactions.iter().filter(|tx| !tx.vote) { + for transaction in transactions + .iter() + .filter(|tx| !tx.vote && tx.unit_limit > 0) + { transaction_fees.push(transaction.unit_price); for writable_account in transaction.accounts.writable.iter().copied() { writable_account_fees From 76bd01cd8d575675eedc9b2508345049c9a071eb Mon Sep 17 00:00:00 2001 From: Kirill Fomichev Date: Sun, 24 Nov 2024 12:29:32 +0200 Subject: [PATCH 3/3] api: change solfees fee calculation --- CHANGELOG.md | 4 +- Cargo.lock | 2 +- solfees-be/Cargo.toml | 2 +- solfees-be/src/rpc_solana.rs | 227 +++++++++++++++++++++++++---------- 4 files changed, 165 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6362a7..c90b623 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,11 @@ The minor version will be incremented upon a breaking change and the patch versi ### Breaking -## [2.1.0] - 2024-11-22 +## [3.0.0] - 2024-11-24 ### Breaking -- api: ??? ([#22](https://github.com/solana-stream-solutions/solfees/pull/22)) +- api: change solfees fee calculation ([#22](https://github.com/solana-stream-solutions/solfees/pull/22)) ## [2.0.0] - 2024-11-18 diff --git a/Cargo.lock b/Cargo.lock index c9ab5f0..47de501 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4636,7 +4636,7 @@ dependencies = [ [[package]] name = "solfees-be" -version = "2.1.0" +version = "3.0.0" dependencies = [ "anyhow", "bincode", diff --git a/solfees-be/Cargo.toml b/solfees-be/Cargo.toml index 912d060..2c31471 100644 --- a/solfees-be/Cargo.toml +++ b/solfees-be/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "solfees-be" -version = "2.1.0" +version = "3.0.0" authors = { workspace = true } edition = { workspace = true } description = "Backend of solfees.io" diff --git a/solfees-be/src/rpc_solana.rs b/solfees-be/src/rpc_solana.rs index 5c807f9..1cf26f7 100644 --- a/solfees-be/src/rpc_solana.rs +++ b/solfees-be/src/rpc_solana.rs @@ -999,7 +999,7 @@ impl SolanaRpc { .iter() .map(|(slot, value)| RpcPrioritizationFee { slot: *slot, - prioritization_fee: value.fees.get_fee(&pubkeys, percentile), + prioritization_fee: value.fees.get_fee(&pubkeys, &[], false, percentile).1, }) .collect::>(); @@ -1322,40 +1322,47 @@ impl StreamsSlotInfo { } fn get_filtered(&self, filter: &SlotSubscribeFilter) -> SlotsSubscribeOutput { - let mut fees = Vec::with_capacity(self.transactions.len()); - for transaction in self.transactions.iter().filter(|tx| { - !tx.vote - && tx.unit_limit > 0 - && filter - .read_write - .iter() - .all(|pubkey| tx.accounts.writable.contains(pubkey)) - && filter - .read_only - .iter() - .all(|pubkey| tx.accounts.readable.contains(pubkey)) - }) { - if transaction.unit_price > 0 || !filter.skip_zeros { - fees.push(transaction.unit_price); - } - } - let total_transactions_filtered = fees.len(); - - let fee_average = if total_transactions_filtered == 0 { - 0f64 - } else { - fees.iter().copied().sum::() as f64 / total_transactions_filtered as f64 - }; - let fee_levels = if filter.levels.is_empty() { - vec![] - } else { - fees.sort_unstable(); - filter - .levels - .iter() - .map(|percentile| RecentPrioritizationFeesSlot::get_percentile(&fees, *percentile)) - .collect() - }; + let total_transactions_filtered = self + .transactions + .iter() + .filter(|tx| { + !tx.vote + && tx.unit_limit > 0 + && filter + .read_write + .iter() + .all(|pubkey| tx.accounts.writable.contains(pubkey)) + && filter + .read_only + .iter() + .all(|pubkey| tx.accounts.readable.contains(pubkey)) + }) + .count(); + + let fee_average = self + .fees + .get_fee( + &filter.read_write, + &filter.read_only, + filter.skip_zeros, + None, + ) + .0; + + let fee_levels = filter + .levels + .iter() + .map(|level| { + self.fees + .get_fee( + &filter.read_write, + &filter.read_only, + filter.skip_zeros, + Some(*level), + ) + .1 + }) + .collect(); SlotsSubscribeOutput::Slot { leader: self.leader.map(|pk| pk.to_string()).unwrap_or_default(), @@ -1378,65 +1385,153 @@ impl StreamsSlotInfo { #[derive(Debug)] struct RecentPrioritizationFeesSlot { - transaction_fees: Vec, - writable_account_fees: HashMap>, + transaction_fees: CollectedFees, + transaction_fees_nz: CollectedFees, + writable_account_fees: HashMap, + writable_account_fees_nz: HashMap, + readable_account_fees: HashMap, + readable_account_fees_nz: HashMap, } impl RecentPrioritizationFeesSlot { fn create(transactions: &[GeyserTransaction]) -> Self { let mut transaction_fees = Vec::with_capacity(transactions.len()); + let mut transaction_fees_nz = Vec::with_capacity(transactions.len()); let mut writable_account_fees = HashMap::>::with_capacity(transactions.len()); + let mut writable_account_fees_nz = + HashMap::>::with_capacity(transactions.len()); + let mut readable_account_fees = + HashMap::>::with_capacity(transactions.len()); + let mut readable_account_fees_nz = + HashMap::>::with_capacity(transactions.len()); for transaction in transactions .iter() .filter(|tx| !tx.vote && tx.unit_limit > 0) { transaction_fees.push(transaction.unit_price); - for writable_account in transaction.accounts.writable.iter().copied() { + for account in transaction.accounts.writable.iter().copied() { writable_account_fees - .entry(writable_account) + .entry(account) + .or_default() + .push(transaction.unit_price); + } + for account in transaction.accounts.readable.iter().copied() { + readable_account_fees + .entry(account) .or_default() .push(transaction.unit_price); } - } - transaction_fees.sort_unstable(); - for (_account, fees) in writable_account_fees.iter_mut() { - fees.sort_unstable(); + if transaction.unit_price > 0 { + transaction_fees_nz.push(transaction.unit_price); + for account in transaction.accounts.writable.iter().copied() { + writable_account_fees_nz + .entry(account) + .or_default() + .push(transaction.unit_price); + } + for account in transaction.accounts.readable.iter().copied() { + readable_account_fees_nz + .entry(account) + .or_default() + .push(transaction.unit_price); + } + } } + let conv = |map: HashMap>| { + map.into_iter() + .map(|(account, fees)| (account, CollectedFees::new(fees))) + .collect() + }; + Self { - transaction_fees, - writable_account_fees, + transaction_fees: CollectedFees::new(transaction_fees), + transaction_fees_nz: CollectedFees::new(transaction_fees_nz), + writable_account_fees: conv(writable_account_fees), + writable_account_fees_nz: conv(writable_account_fees_nz), + readable_account_fees: conv(readable_account_fees), + readable_account_fees_nz: conv(readable_account_fees_nz), } } - fn get_fee(&self, account_keys: &[Pubkey], percentile: Option) -> u64 { - let mut fee = - Self::get_with_percentile(&self.transaction_fees, percentile).unwrap_or_default(); + fn get_fee( + &self, + writeable_account_keys: &[Pubkey], + readable_account_keys: &[Pubkey], + skip_zeros: bool, + percentile: Option, + ) -> (f64, u64) { + let (txs, write_map, read_map) = if skip_zeros { + ( + &self.transaction_fees_nz, + &self.writable_account_fees_nz, + &self.readable_account_fees_nz, + ) + } else { + ( + &self.transaction_fees, + &self.writable_account_fees, + &self.readable_account_fees, + ) + }; - for account_key in account_keys { - if let Some(fees) = self.writable_account_fees.get(account_key) { - if let Some(account_fee) = Self::get_with_percentile(fees, percentile) { - fee = std::cmp::max(fee, account_fee); - } - } + let (mut avg, mut fee) = txs.get_with_percentile(percentile); + + if let Some((aavg, afee)) = writeable_account_keys + .iter() + .zip(std::iter::repeat(write_map)) + .chain( + readable_account_keys + .iter() + .zip(std::iter::repeat(read_map)), + ) + .filter_map(|(account, map)| { + map.get(account) + .map(|fees| fees.get_with_percentile(percentile)) + }) + .reduce(|(avg1, fee1), (avg2, fee2)| (avg1.max(avg2), fee1.max(fee2))) + { + avg = avg.max(aavg); + fee = fee.max(afee); } - fee + (avg, fee) } +} - fn get_with_percentile(fees: &[u64], percentile: Option) -> Option { - match percentile { - Some(percentile) => Self::get_percentile(fees, percentile), - None => fees.first().copied(), - } +#[derive(Debug)] +struct CollectedFees { + fees: Vec, + average: f64, +} + +impl From> for CollectedFees { + fn from(fees: Vec) -> Self { + Self::new(fees) + } +} + +impl CollectedFees { + fn new(mut fees: Vec) -> Self { + fees.sort_unstable(); + let average = fees.iter().map(|fee| *fee as f64).sum::() / fees.len() as f64; + Self { fees, average } + } + + fn get_with_percentile(&self, percentile: Option) -> (f64, u64) { + let fee = match percentile { + Some(percentile) => self.get_percentile(percentile), + None => self.fees.first().copied(), + }; + (self.average, fee.unwrap_or_default()) } - fn get_percentile(fees: &[u64], percentile: u16) -> Option { - let index = (percentile as usize).min(9_999) * fees.len() / 10_000; - fees.get(index).copied() + fn get_percentile(&self, percentile: u16) -> Option { + let index = (percentile as usize).min(9_999) * self.fees.len() / 10_000; + self.fees.get(index).copied() } } @@ -1548,7 +1643,7 @@ enum SlotsSubscribeOutput { total_transactions_vote: usize, total_transactions: usize, fee_average: f64, - fee_levels: Vec>, + fee_levels: Vec, total_fee: u64, total_units_consumed: u64, }, @@ -1571,7 +1666,7 @@ enum SlotsSubscribeOutputSolana { total_transactions_vote: usize, total_transactions: usize, fee_average: f64, - fee_levels: Vec>, + fee_levels: Vec, }, } @@ -1615,7 +1710,7 @@ struct SolfeesPrioritizationFee { total_transactions_vote: usize, total_transactions: usize, fee_average: f64, - fee_levels: Vec>, + fee_levels: Vec, } impl TryFrom for SolfeesPrioritizationFee {