diff --git a/Cargo.lock b/Cargo.lock index 0716fa34d..e1b88fa6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3049,6 +3049,7 @@ dependencies = [ "secp256k1", "serde", "smallvec", + "sweep-bptree", "thiserror", "tokio", ] @@ -3408,6 +3409,7 @@ dependencies = [ name = "kaspa-utils" version = "0.14.2" dependencies = [ + "arc-swap", "async-channel 2.2.1", "async-trait", "bincode", @@ -3420,6 +3422,7 @@ dependencies = [ "ipnet", "itertools 0.11.0", "log", + "once_cell", "parking_lot", "rand 0.8.5", "rlimit", @@ -3803,6 +3806,7 @@ name = "kaspad" version = "0.14.2" dependencies = [ "async-channel 2.2.1", + "cfg-if 1.0.0", "clap 4.5.4", "dhat", "dirs", @@ -3899,9 +3903,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.37" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81eb4061c0582dedea1cbc7aff2240300dd6982e0239d1c99e65c1dbf4a30ba7" +checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" dependencies = [ "cc", "libc", @@ -4174,9 +4178,9 @@ checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "mimalloc" -version = "0.1.41" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f41a2280ded0da56c8cf898babb86e8f10651a34adcfff190ae9a1159c6908d" +checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633" dependencies = [ "libmimalloc-sys", ] @@ -5639,6 +5643,7 @@ name = "simpa" version = "0.14.2" dependencies = [ "async-channel 2.2.1", + "cfg-if 1.0.0", "clap 4.5.4", "dhat", "futures", @@ -5783,6 +5788,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "sweep-bptree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea7b1b7c5eaabc40bab84ec98b2f12523d97e91c9bfc430fe5d2a1ea15c9960" + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 2409c7354..0c6b06d8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,7 +124,7 @@ kaspa-utxoindex = { version = "0.14.2", path = "indexes/utxoindex" } kaspa-wallet = { version = "0.14.2", path = "wallet/native" } kaspa-wallet-cli-wasm = { version = "0.14.2", path = "wallet/wasm" } kaspa-wallet-keys = { version = "0.14.2", path = "wallet/keys" } -kaspa-wallet-pskt = { version = "0.14.1", path = "wallet/pskt" } +kaspa-wallet-pskt = { version = "0.14.2", path = "wallet/pskt" } kaspa-wallet-core = { version = "0.14.2", path = "wallet/core" } kaspa-wallet-macros = { version = "0.14.2", path = "wallet/macros" } kaspa-wasm = { version = "0.14.2", path = "wasm" } diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index c84915480..53478c174 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -229,6 +229,15 @@ impl Rpc { } } } + RpcApiOps::GetFeeEstimate => { + let result = rpc.get_fee_estimate_call(GetFeeEstimateRequest {}).await?; + self.println(&ctx, result); + } + RpcApiOps::GetFeeEstimateExperimental => { + let verbose = if argv.is_empty() { false } else { argv.remove(0).parse().unwrap_or(false) }; + let result = rpc.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await?; + self.println(&ctx, result); + } _ => { tprintln!(ctx, "rpc method exists but is not supported by the cli: '{op_str}'\r\n"); return Ok(()); diff --git a/components/addressmanager/src/stores/address_store.rs b/components/addressmanager/src/stores/address_store.rs index accfcfda4..fe4ddb244 100644 --- a/components/addressmanager/src/stores/address_store.rs +++ b/components/addressmanager/src/stores/address_store.rs @@ -21,6 +21,7 @@ pub struct Entry { impl MemSizeEstimator for Entry {} pub trait AddressesStoreReader { + #[allow(dead_code)] fn get(&self, key: AddressKey) -> Result; } diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml index 7082ed7b1..b9a183ea8 100644 --- a/consensus/Cargo.toml +++ b/consensus/Cargo.toml @@ -30,6 +30,7 @@ kaspa-muhash.workspace = true kaspa-notify.workspace = true kaspa-pow.workspace = true kaspa-txscript.workspace = true +kaspa-txscript-errors.workspace = true kaspa-utils.workspace = true log.workspace = true once_cell.workspace = true diff --git a/consensus/core/src/api/args.rs b/consensus/core/src/api/args.rs new file mode 100644 index 000000000..ebc76d97d --- /dev/null +++ b/consensus/core/src/api/args.rs @@ -0,0 +1,47 @@ +use std::collections::HashMap; + +use crate::tx::TransactionId; + +/// A struct provided to consensus for transaction validation processing calls +#[derive(Clone, Debug, Default)] +pub struct TransactionValidationArgs { + /// Optional fee/mass threshold above which a bound transaction in not rejected + pub feerate_threshold: Option, +} + +impl TransactionValidationArgs { + pub fn new(feerate_threshold: Option) -> Self { + Self { feerate_threshold } + } +} + +/// A struct provided to consensus for transactions validation batch processing calls +pub struct TransactionValidationBatchArgs { + tx_args: HashMap, +} + +impl TransactionValidationBatchArgs { + const DEFAULT_ARGS: TransactionValidationArgs = TransactionValidationArgs { feerate_threshold: None }; + + pub fn new() -> Self { + Self { tx_args: HashMap::new() } + } + + /// Set some fee/mass threshold for transaction `transaction_id`. + pub fn set_feerate_threshold(&mut self, transaction_id: TransactionId, feerate_threshold: f64) { + self.tx_args + .entry(transaction_id) + .and_modify(|x| x.feerate_threshold = Some(feerate_threshold)) + .or_insert(TransactionValidationArgs::new(Some(feerate_threshold))); + } + + pub fn get(&self, transaction_id: &TransactionId) -> &TransactionValidationArgs { + self.tx_args.get(transaction_id).unwrap_or(&Self::DEFAULT_ARGS) + } +} + +impl Default for TransactionValidationBatchArgs { + fn default() -> Self { + Self::new() + } +} diff --git a/consensus/core/src/api/mod.rs b/consensus/core/src/api/mod.rs index 23e7abb53..6df33579c 100644 --- a/consensus/core/src/api/mod.rs +++ b/consensus/core/src/api/mod.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::{ acceptance_data::AcceptanceData, + api::args::{TransactionValidationArgs, TransactionValidationBatchArgs}, block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, blockstatus::BlockStatus, coinbase::MinerData, @@ -25,6 +26,7 @@ use kaspa_hashes::Hash; pub use self::stats::{BlockCount, ConsensusStats}; +pub mod args; pub mod counters; pub mod stats; @@ -62,14 +64,18 @@ pub trait ConsensusApi: Send + Sync { } /// Populates the mempool transaction with maximally found UTXO entry data and proceeds to full transaction - /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated. - fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { + /// validation if all are found. If validation is successful, also `transaction.calculated_fee` is expected to be populated. + fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { unimplemented!() } /// Populates the mempool transactions with maximally found UTXO entry data and proceeds to full transactions - /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated. - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { + /// validation if all are found. If validation is successful, also `transaction.calculated_fee` is expected to be populated. + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { unimplemented!() } @@ -184,6 +190,10 @@ pub trait ConsensusApi: Send + Sync { unimplemented!() } + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], pov_daa_score: u64) -> Hash { + unimplemented!() + } + fn validate_pruning_proof(&self, proof: &PruningPointProof) -> PruningImportResult<()> { unimplemented!() } diff --git a/consensus/core/src/block.rs b/consensus/core/src/block.rs index dde6fd5e7..cbd76b42d 100644 --- a/consensus/core/src/block.rs +++ b/consensus/core/src/block.rs @@ -5,6 +5,7 @@ use crate::{ BlueWorkType, }; use kaspa_hashes::Hash; +use kaspa_utils::mem_size::MemSizeEstimator; use std::sync::Arc; /// A mutable block structure where header and transactions within can still be mutated. @@ -66,6 +67,20 @@ impl Block { pub fn from_precomputed_hash(hash: Hash, parents: Vec) -> Block { Block::from_header(Header::from_precomputed_hash(hash, parents)) } + + pub fn asses_for_cache(&self) -> Option<()> { + (self.estimate_mem_bytes() < 1_000_000).then_some(()) + } +} + +impl MemSizeEstimator for Block { + fn estimate_mem_bytes(&self) -> usize { + // Calculates mem bytes of the block (for cache tracking purposes) + size_of::() + + self.header.estimate_mem_bytes() + + size_of::>() + + self.transactions.iter().map(Transaction::estimate_mem_bytes).sum::() + } } /// An abstraction for a recallable transaction selector with persistent state @@ -105,6 +120,8 @@ pub struct BlockTemplate { pub selected_parent_timestamp: u64, pub selected_parent_daa_score: u64, pub selected_parent_hash: Hash, + /// Expected length is one less than txs length due to lack of coinbase transaction + pub calculated_fees: Vec, } impl BlockTemplate { @@ -115,8 +132,17 @@ impl BlockTemplate { selected_parent_timestamp: u64, selected_parent_daa_score: u64, selected_parent_hash: Hash, + calculated_fees: Vec, ) -> Self { - Self { block, miner_data, coinbase_has_red_reward, selected_parent_timestamp, selected_parent_daa_score, selected_parent_hash } + Self { + block, + miner_data, + coinbase_has_red_reward, + selected_parent_timestamp, + selected_parent_daa_score, + selected_parent_hash, + calculated_fees, + } } pub fn to_virtual_state_approx_id(&self) -> VirtualStateApproxId { diff --git a/consensus/core/src/config/genesis.rs b/consensus/core/src/config/genesis.rs index 204098ad2..9f9ea21e5 100644 --- a/consensus/core/src/config/genesis.rs +++ b/consensus/core/src/config/genesis.rs @@ -231,7 +231,7 @@ mod tests { fn test_genesis_hashes() { [GENESIS, TESTNET_GENESIS, TESTNET11_GENESIS, SIMNET_GENESIS, DEVNET_GENESIS].into_iter().for_each(|genesis| { let block: Block = (&genesis).into(); - assert_hashes_eq(calc_hash_merkle_root(block.transactions.iter()), block.header.hash_merkle_root); + assert_hashes_eq(calc_hash_merkle_root(block.transactions.iter(), false), block.header.hash_merkle_root); assert_hashes_eq(block.hash(), genesis.hash); }); } diff --git a/consensus/core/src/config/params.rs b/consensus/core/src/config/params.rs index e2f2639a1..f512cf1e1 100644 --- a/consensus/core/src/config/params.rs +++ b/consensus/core/src/config/params.rs @@ -501,7 +501,8 @@ pub const SIMNET_PARAMS: Params = Params { target_time_per_block: Testnet11Bps::target_time_per_block(), past_median_time_sample_rate: Testnet11Bps::past_median_time_sample_rate(), difficulty_sample_rate: Testnet11Bps::difficulty_adjustment_sample_rate(), - max_block_parents: Testnet11Bps::max_block_parents(), + // For simnet, we deviate from TN11 configuration and allow at least 64 parents in order to support mempool benchmarks out of the box + max_block_parents: if Testnet11Bps::max_block_parents() > 64 { Testnet11Bps::max_block_parents() } else { 64 }, mergeset_size_limit: Testnet11Bps::mergeset_size_limit(), merge_depth: Testnet11Bps::merge_depth_bound(), finality_depth: Testnet11Bps::finality_depth(), diff --git a/consensus/core/src/errors/tx.rs b/consensus/core/src/errors/tx.rs index e1936d37a..f21409857 100644 --- a/consensus/core/src/errors/tx.rs +++ b/consensus/core/src/errors/tx.rs @@ -1,4 +1,5 @@ use crate::constants::MAX_SOMPI; +use crate::subnets::SubnetworkId; use crate::tx::TransactionOutpoint; use kaspa_txscript_errors::TxScriptError; use thiserror::Error; @@ -80,6 +81,9 @@ pub enum TxRuleError { #[error("failed to verify the signature script: {0}")] SignatureInvalid(TxScriptError), + #[error("failed to verify empty signature script. Inner error: {0}")] + SignatureEmpty(TxScriptError), + #[error("input {0} sig op count is {1}, but the calculated value is {2}")] WrongSigOpCount(usize, u64, u64), @@ -88,6 +92,14 @@ pub enum TxRuleError { #[error("calculated contextual mass (including storage mass) {0} is not equal to the committed mass field {1}")] WrongMass(u64, u64), + + #[error("transaction subnetwork id {0} is neither native nor coinbase")] + SubnetworksDisabled(SubnetworkId), + + /// [`TxRuleError::FeerateTooLow`] is not a consensus error but a mempool error triggered by the + /// fee/mass RBF validation rule + #[error("fee rate per contextual mass gram is not greater than the fee rate of the replaced transaction")] + FeerateTooLow, } pub type TxResult = std::result::Result; diff --git a/consensus/core/src/header.rs b/consensus/core/src/header.rs index b6c2b9bc7..b57337afc 100644 --- a/consensus/core/src/header.rs +++ b/consensus/core/src/header.rs @@ -1,6 +1,7 @@ use crate::{hashing, BlueWorkType}; use borsh::{BorshDeserialize, BorshSerialize}; use kaspa_hashes::Hash; +use kaspa_utils::mem_size::MemSizeEstimator; use serde::{Deserialize, Serialize}; /// @category Consensus @@ -92,6 +93,12 @@ impl Header { } } +impl MemSizeEstimator for Header { + fn estimate_mem_bytes(&self) -> usize { + size_of::() + self.parents_by_level.iter().map(|l| l.len()).sum::() * size_of::() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/core/src/merkle.rs b/consensus/core/src/merkle.rs index cfd5c9045..59c6ca7c4 100644 --- a/consensus/core/src/merkle.rs +++ b/consensus/core/src/merkle.rs @@ -2,14 +2,10 @@ use crate::{hashing, tx::Transaction}; use kaspa_hashes::Hash; use kaspa_merkle::calc_merkle_root; -pub fn calc_hash_merkle_root_with_options<'a>(txs: impl ExactSizeIterator, include_mass_field: bool) -> Hash { +pub fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator, include_mass_field: bool) -> Hash { calc_merkle_root(txs.map(|tx| hashing::tx::hash(tx, include_mass_field))) } -pub fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { - calc_merkle_root(txs.map(|tx| hashing::tx::hash(tx, false))) -} - #[cfg(test)] mod tests { use crate::merkle::calc_hash_merkle_root; @@ -242,7 +238,7 @@ mod tests { ), ]; assert_eq!( - calc_hash_merkle_root(txs.iter()), + calc_hash_merkle_root(txs.iter(), false), Hash::from_slice(&[ 0x46, 0xec, 0xf4, 0x5b, 0xe3, 0xba, 0xca, 0x34, 0x9d, 0xfe, 0x8a, 0x78, 0xde, 0xaf, 0x05, 0x3b, 0x0a, 0xa6, 0xd5, 0x38, 0x97, 0x4d, 0xa5, 0x0f, 0xd6, 0xef, 0xb4, 0xd2, 0x66, 0xbc, 0x8d, 0x21, diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index 2456f8444..756c4d40a 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -4,7 +4,7 @@ use std::str::{self, FromStr}; use borsh::{BorshDeserialize, BorshSerialize}; use kaspa_utils::hex::{FromHex, ToHex}; use kaspa_utils::{serde_impl_deser_fixed_bytes_ref, serde_impl_ser_fixed_bytes_ref}; -use thiserror::Error; +use thiserror::Error; /// The size of the array used to store subnetwork IDs. pub const SUBNETWORK_ID_SIZE: usize = 20; @@ -59,35 +59,34 @@ impl SubnetworkId { *self == SUBNETWORK_ID_COINBASE || *self == SUBNETWORK_ID_REGISTRY } + /// Returns true if the subnetwork is the native subnetwork + #[inline] + pub fn is_native(&self) -> bool { + *self == SUBNETWORK_ID_NATIVE + } + /// Returns true if the subnetwork is the native or a built-in subnetwork #[inline] pub fn is_builtin_or_native(&self) -> bool { - *self == SUBNETWORK_ID_NATIVE || self.is_builtin() + self.is_native() || self.is_builtin() } } -#[derive(Error, Debug, Clone)] -pub enum SubnetworkConversionError { - #[error("Invalid bytes")] - InvalidBytes, - - #[error(transparent)] - SliceError(#[from] std::array::TryFromSliceError), - - #[error(transparent)] - HexError(#[from] faster_hex::Error), -} - +#[derive(Error, Debug, Clone)] +pub enum SubnetworkConversionError { + #[error(transparent)] + SliceError(#[from] std::array::TryFromSliceError), + + #[error(transparent)] + HexError(#[from] faster_hex::Error), +} + impl TryFrom<&[u8]> for SubnetworkId { - type Error = SubnetworkConversionError; + type Error = SubnetworkConversionError; fn try_from(value: &[u8]) -> Result { let bytes = <[u8; SUBNETWORK_ID_SIZE]>::try_from(value)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Error::InvalidBytes) - } else { - Ok(Self(bytes)) - } + Ok(Self(bytes)) } } @@ -109,30 +108,22 @@ impl ToHex for SubnetworkId { } impl FromStr for SubnetworkId { - type Err = SubnetworkConversionError; + type Err = SubnetworkConversionError; #[inline] fn from_str(hex_str: &str) -> Result { let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Err::InvalidBytes) - } else { - Ok(Self(bytes)) - } + Ok(Self(bytes)) } } impl FromHex for SubnetworkId { - type Error = SubnetworkConversionError; + type Error = SubnetworkConversionError; fn from_hex(hex_str: &str) -> Result { let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Error::InvalidBytes) - } else { - Ok(Self(bytes)) - } + Ok(Self(bytes)) } } diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index c2d3ba2e0..3595dcb8b 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -230,6 +230,23 @@ impl Transaction { } } +impl MemSizeEstimator for Transaction { + fn estimate_mem_bytes(&self) -> usize { + // Calculates mem bytes of the transaction (for cache tracking purposes) + size_of::() + + self.payload.len() + + self + .inputs + .iter() + .map(|i| i.signature_script.len() + size_of::()) + .chain(self.outputs.iter().map(|o| { + // size_of::() already counts SCRIPT_VECTOR_SIZE bytes within, so we only add the delta + o.script_public_key.script().len().saturating_sub(SCRIPT_VECTOR_SIZE) + size_of::() + })) + .sum::() + } +} + /// Represents any kind of transaction which has populated UTXO entry data and can be verified/signed etc pub trait VerifiableTransaction { fn tx(&self) -> &Transaction; @@ -406,6 +423,19 @@ impl> MutableTransaction { *entry = None; } } + + /// Returns the calculated feerate. The feerate is calculated as the amount of fee + /// this transactions pays per gram of the full contextual (compute & storage) mass. The + /// function returns a value when calculated fee exists and the contextual mass is greater + /// than zero, otherwise `None` is returned. + pub fn calculated_feerate(&self) -> Option { + let contextual_mass = self.tx.as_ref().mass(); + if contextual_mass > 0 { + self.calculated_fee.map(|fee| fee as f64 / contextual_mass as f64) + } else { + None + } + } } impl> AsRef for MutableTransaction { diff --git a/consensus/src/consensus/mod.rs b/consensus/src/consensus/mod.rs index 7e1690b2a..c37f9bd0b 100644 --- a/consensus/src/consensus/mod.rs +++ b/consensus/src/consensus/mod.rs @@ -40,7 +40,11 @@ use crate::{ }; use kaspa_consensus_core::{ acceptance_data::AcceptanceData, - api::{stats::BlockCount, BlockValidationFutures, ConsensusApi, ConsensusStats}, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + stats::BlockCount, + BlockValidationFutures, ConsensusApi, ConsensusStats, + }, block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, blockhash::BlockHashExtensions, blockstatus::BlockStatus, @@ -49,10 +53,12 @@ use kaspa_consensus_core::{ errors::{ coinbase::CoinbaseResult, consensus::{ConsensusError, ConsensusResult}, + difficulty::DifficultyError, + pruning::PruningImportError, tx::TxResult, }, - errors::{difficulty::DifficultyError, pruning::PruningImportError}, header::Header, + merkle::calc_hash_merkle_root, muhash::MuHashExtensions, network::NetworkType, pruning::{PruningPointProof, PruningPointTrustedData, PruningPointsList}, @@ -418,13 +424,17 @@ impl ConsensusApi for Consensus { BlockValidationFutures { block_task: Box::pin(block_task), virtual_state_task: Box::pin(virtual_state_task) } } - fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { - self.virtual_processor.validate_mempool_transaction(transaction)?; + fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { + self.virtual_processor.validate_mempool_transaction(transaction, args)?; Ok(()) } - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - self.virtual_processor.validate_mempool_transactions_in_parallel(transactions) + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { + self.virtual_processor.validate_mempool_transactions_in_parallel(transactions, args) } fn populate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { @@ -666,6 +676,11 @@ impl ConsensusApi for Consensus { self.services.coinbase_manager.modify_coinbase_payload(payload, miner_data) } + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], pov_daa_score: u64) -> Hash { + let storage_mass_activated = pov_daa_score > self.config.storage_mass_activation_daa_score; + calc_hash_merkle_root(txs.iter(), storage_mass_activated) + } + fn validate_pruning_proof(&self, proof: &PruningPointProof) -> Result<(), PruningImportError> { self.services.pruning_proof_manager.validate_pruning_point_proof(proof) } diff --git a/consensus/src/consensus/test_consensus.rs b/consensus/src/consensus/test_consensus.rs index a937388ba..472bdbd83 100644 --- a/consensus/src/consensus/test_consensus.rs +++ b/consensus/src/consensus/test_consensus.rs @@ -176,7 +176,7 @@ impl TestConsensus { let cb = Transaction::new(TX_VERSION, vec![], vec![], 0, SUBNETWORK_ID_COINBASE, 0, cb_payload); txs.insert(0, cb); - header.hash_merkle_root = calc_hash_merkle_root(txs.iter()); + header.hash_merkle_root = calc_hash_merkle_root(txs.iter(), false); MutableBlock::new(header, txs) } diff --git a/consensus/src/model/services/reachability.rs b/consensus/src/model/services/reachability.rs index d80efd760..39f5ceba2 100644 --- a/consensus/src/model/services/reachability.rs +++ b/consensus/src/model/services/reachability.rs @@ -17,6 +17,7 @@ pub trait ReachabilityService { fn is_any_dag_ancestor_result(&self, list: &mut impl Iterator, queried: Hash) -> Result; fn get_next_chain_ancestor(&self, descendant: Hash, ancestor: Hash) -> Hash; fn get_chain_parent(&self, this: Hash) -> Hash; + fn has_reachability_data(&self, this: Hash) -> bool; } impl ReachabilityService for T { @@ -56,6 +57,10 @@ impl ReachabilityService for T { fn get_chain_parent(&self, this: Hash) -> Hash { self.get_parent(this).unwrap() } + + fn has_reachability_data(&self, this: Hash) -> bool { + self.has(this).unwrap() + } } /// Multi-threaded reachability service imp @@ -108,6 +113,10 @@ impl ReachabilityService for MTReachability fn get_chain_parent(&self, this: Hash) -> Hash { self.store.read().get_parent(this).unwrap() } + + fn has_reachability_data(&self, this: Hash) -> bool { + self.store.read().has(this).unwrap() + } } impl MTReachabilityService { diff --git a/consensus/src/model/stores/headers.rs b/consensus/src/model/stores/headers.rs index b0c25b596..64e10a90b 100644 --- a/consensus/src/model/stores/headers.rs +++ b/consensus/src/model/stores/headers.rs @@ -29,9 +29,7 @@ pub struct HeaderWithBlockLevel { impl MemSizeEstimator for HeaderWithBlockLevel { fn estimate_mem_bytes(&self) -> usize { - size_of::
() - + self.header.parents_by_level.iter().map(|l| l.len()).sum::() * size_of::() - + size_of::() + self.header.as_ref().estimate_mem_bytes() + size_of::() } } diff --git a/consensus/src/pipeline/body_processor/body_validation_in_context.rs b/consensus/src/pipeline/body_processor/body_validation_in_context.rs index 2425556d0..042410fa8 100644 --- a/consensus/src/pipeline/body_processor/body_validation_in_context.rs +++ b/consensus/src/pipeline/body_processor/body_validation_in_context.rs @@ -94,13 +94,17 @@ mod tests { }; use kaspa_consensus_core::{ api::ConsensusApi, - merkle::calc_hash_merkle_root, + merkle::calc_hash_merkle_root as calc_hash_merkle_root_with_options, subnets::SUBNETWORK_ID_NATIVE, tx::{Transaction, TransactionInput, TransactionOutpoint}, }; use kaspa_core::assert_match; use kaspa_hashes::Hash; + fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { + calc_hash_merkle_root_with_options(txs, false) + } + #[tokio::test] async fn validate_body_in_context_test() { let config = ConfigBuilder::new(DEVNET_PARAMS) diff --git a/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs b/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs index e5a51d815..c413552b9 100644 --- a/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs +++ b/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, sync::Arc}; use super::BlockBodyProcessor; use crate::errors::{BlockProcessResult, RuleError}; -use kaspa_consensus_core::{block::Block, merkle::calc_hash_merkle_root_with_options, tx::TransactionOutpoint}; +use kaspa_consensus_core::{block::Block, merkle::calc_hash_merkle_root, tx::TransactionOutpoint}; impl BlockBodyProcessor { pub fn validate_body_in_isolation(self: &Arc, block: &Block) -> BlockProcessResult { @@ -29,7 +29,7 @@ impl BlockBodyProcessor { } fn check_hash_merkle_root(block: &Block, storage_mass_activated: bool) -> BlockProcessResult<()> { - let calculated = calc_hash_merkle_root_with_options(block.transactions.iter(), storage_mass_activated); + let calculated = calc_hash_merkle_root(block.transactions.iter(), storage_mass_activated); if calculated != block.header.hash_merkle_root { return Err(RuleError::BadMerkleRoot(block.header.hash_merkle_root, calculated)); } @@ -137,13 +137,17 @@ mod tests { api::{BlockValidationFutures, ConsensusApi}, block::MutableBlock, header::Header, - merkle::calc_hash_merkle_root, + merkle::calc_hash_merkle_root as calc_hash_merkle_root_with_options, subnets::{SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, tx::{scriptvec, ScriptPublicKey, Transaction, TransactionId, TransactionInput, TransactionOutpoint, TransactionOutput}, }; use kaspa_core::assert_match; use kaspa_hashes::Hash; + fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { + calc_hash_merkle_root_with_options(txs, false) + } + #[test] fn validate_body_in_isolation_test() { let consensus = TestConsensus::new(&Config::new(MAINNET_PARAMS)); diff --git a/consensus/src/pipeline/pruning_processor/processor.rs b/consensus/src/pipeline/pruning_processor/processor.rs index d0d4dbbc1..13cbc0b01 100644 --- a/consensus/src/pipeline/pruning_processor/processor.rs +++ b/consensus/src/pipeline/pruning_processor/processor.rs @@ -268,6 +268,12 @@ impl PruningProcessor { .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) .chain(proof.iter().flatten().map(|h| h.hash)) .collect(); + let keep_level_zero_relations: BlockHashSet = std::iter::empty() + .chain(data.anticone.iter().copied()) + .chain(data.daa_window_blocks.iter().map(|th| th.header.hash)) + .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) + .chain(proof[0].iter().map(|h| h.hash)) + .collect(); let keep_headers: BlockHashSet = self.past_pruning_points(); info!("Header and Block pruning: waiting for consensus write permissions..."); @@ -281,16 +287,16 @@ impl PruningProcessor { { let mut counter = 0; let mut batch = WriteBatch::default(); - for kept in keep_relations.iter().copied() { + for kept in keep_level_zero_relations.iter().copied() { let Some(ghostdag) = self.ghostdag_store.get_data(kept).unwrap_option() else { continue; }; - if ghostdag.unordered_mergeset().any(|h| !keep_relations.contains(&h)) { + if ghostdag.unordered_mergeset().any(|h| !keep_level_zero_relations.contains(&h)) { let mut mutable_ghostdag: ExternalGhostdagData = ghostdag.as_ref().into(); - mutable_ghostdag.mergeset_blues.retain(|h| keep_relations.contains(h)); - mutable_ghostdag.mergeset_reds.retain(|h| keep_relations.contains(h)); - mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_relations.contains(k)); - if !keep_relations.contains(&mutable_ghostdag.selected_parent) { + mutable_ghostdag.mergeset_blues.retain(|h| keep_level_zero_relations.contains(h)); + mutable_ghostdag.mergeset_reds.retain(|h| keep_level_zero_relations.contains(h)); + mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_level_zero_relations.contains(k)); + if !keep_level_zero_relations.contains(&mutable_ghostdag.selected_parent) { mutable_ghostdag.selected_parent = ORIGIN; } counter += 1; @@ -396,6 +402,19 @@ impl PruningProcessor { // other parts of the code assume the existence of GD data etc.) statuses_write.set_batch(&mut batch, current, StatusHeaderOnly).unwrap(); } + + // Delete level-0 relations for blocks which only belong to higher proof levels. + // Note: it is also possible to delete level relations for level x > 0 for any block that only belongs + // to proof levels higher than x, but this requires maintaining such per level usage mapping. + // Since the main motivation of this deletion step is to reduce the + // number of origin's children in level 0, and this is not a bottleneck in any other + // level, we currently chose to only delete level-0 redundant relations. + if !keep_level_zero_relations.contains(¤t) { + let mut staging_level_relations = StagingRelationsStore::new(&mut level_relations_write[0]); + relations::delete_level_relations(MemoryWriter, &mut staging_level_relations, current).unwrap_option(); + staging_level_relations.commit(&mut batch).unwrap(); + self.ghostdag_store.delete_batch(&mut batch, current).unwrap_option(); + } } else { // Count only blocks which get fully pruned including DAG relations counter += 1; diff --git a/consensus/src/pipeline/virtual_processor/processor.rs b/consensus/src/pipeline/virtual_processor/processor.rs index 3bfd3a465..6596f624c 100644 --- a/consensus/src/pipeline/virtual_processor/processor.rs +++ b/consensus/src/pipeline/virtual_processor/processor.rs @@ -48,12 +48,13 @@ use crate::{ }; use kaspa_consensus_core::{ acceptance_data::AcceptanceData, + api::args::{TransactionValidationArgs, TransactionValidationBatchArgs}, block::{BlockTemplate, MutableBlock, TemplateBuildMode, TemplateTransactionSelector}, blockstatus::BlockStatus::{StatusDisqualifiedFromChain, StatusUTXOValid}, coinbase::MinerData, config::genesis::GenesisBlock, header::Header, - merkle::calc_hash_merkle_root_with_options, + merkle::calc_hash_merkle_root, pruning::PruningPointsList, tx::{MutableTransaction, Transaction}, utxo::{ @@ -76,8 +77,10 @@ use kaspa_hashes::Hash; use kaspa_muhash::MuHash; use kaspa_notify::{events::EventType, notifier::Notify}; +use super::errors::{PruningImportError, PruningImportResult}; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use itertools::Itertools; +use kaspa_consensus_core::tx::ValidatedTransaction; use kaspa_utils::binary_heap::BinaryHeapExtensions; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; use rand::{seq::SliceRandom, Rng}; @@ -93,8 +96,6 @@ use std::{ sync::{atomic::Ordering, Arc}, }; -use super::errors::{PruningImportError, PruningImportResult}; - pub struct VirtualStateProcessor { // Channels receiver: CrossbeamReceiver, @@ -757,30 +758,28 @@ impl VirtualStateProcessor { virtual_utxo_view: &impl UtxoView, virtual_daa_score: u64, virtual_past_median_time: u64, + args: &TransactionValidationArgs, ) -> TxResult<()> { self.transaction_validator.validate_tx_in_isolation(&mutable_tx.tx)?; self.transaction_validator.utxo_free_tx_validation(&mutable_tx.tx, virtual_daa_score, virtual_past_median_time)?; - self.validate_mempool_transaction_in_utxo_context(mutable_tx, virtual_utxo_view, virtual_daa_score)?; + self.validate_mempool_transaction_in_utxo_context(mutable_tx, virtual_utxo_view, virtual_daa_score, args)?; Ok(()) } - pub fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction) -> TxResult<()> { + pub fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; let virtual_daa_score = virtual_state.daa_score; let virtual_past_median_time = virtual_state.past_median_time; - if mutable_tx.tx.inputs.len() > 1 { - // use pool to apply par_iter to inputs - self.thread_pool.install(|| { - self.validate_mempool_transaction_impl(mutable_tx, virtual_utxo_view, virtual_daa_score, virtual_past_median_time) - }) - } else { - self.validate_mempool_transaction_impl(mutable_tx, virtual_utxo_view, virtual_daa_score, virtual_past_median_time) - } + self.validate_mempool_transaction_impl(mutable_tx, virtual_utxo_view, virtual_daa_score, virtual_past_median_time, args) } - pub fn validate_mempool_transactions_in_parallel(&self, mutable_txs: &mut [MutableTransaction]) -> Vec> { + pub fn validate_mempool_transactions_in_parallel( + &self, + mutable_txs: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; @@ -791,7 +790,13 @@ impl VirtualStateProcessor { mutable_txs .par_iter_mut() .map(|mtx| { - self.validate_mempool_transaction_impl(mtx, &virtual_utxo_view, virtual_daa_score, virtual_past_median_time) + self.validate_mempool_transaction_impl( + mtx, + &virtual_utxo_view, + virtual_daa_score, + virtual_past_median_time, + args.get(&mtx.id()), + ) }) .collect::>>() }) @@ -828,12 +833,9 @@ impl VirtualStateProcessor { txs: &[Transaction], virtual_state: &VirtualState, utxo_view: &V, - ) -> Vec> { - self.thread_pool.install(|| { - txs.par_iter() - .map(|tx| self.validate_block_template_transaction(tx, virtual_state, &utxo_view)) - .collect::>>() - }) + ) -> Vec> { + self.thread_pool + .install(|| txs.par_iter().map(|tx| self.validate_block_template_transaction(tx, virtual_state, &utxo_view)).collect()) } fn validate_block_template_transaction( @@ -841,13 +843,14 @@ impl VirtualStateProcessor { tx: &Transaction, virtual_state: &VirtualState, utxo_view: &impl UtxoView, - ) -> TxResult<()> { + ) -> TxResult { // No need to validate the transaction in isolation since we rely on the mining manager to submit transactions // which were previously validated through `validate_mempool_transaction_and_populate`, hence we only perform // in-context validations self.transaction_validator.utxo_free_tx_validation(tx, virtual_state.daa_score, virtual_state.past_median_time)?; - self.validate_transaction_in_utxo_context(tx, utxo_view, virtual_state.daa_score, TxValidationFlags::Full)?; - Ok(()) + let ValidatedTransaction { calculated_fee, .. } = + self.validate_transaction_in_utxo_context(tx, utxo_view, virtual_state.daa_score, TxValidationFlags::Full)?; + Ok(calculated_fee) } pub fn build_block_template( @@ -864,7 +867,7 @@ impl VirtualStateProcessor { // optimizing for the common case where all txs are valid. Following selection calls // are called within the lock in order to preserve validness of already validated txs let mut txs = tx_selector.select_transactions(); - + let mut calculated_fees = Vec::with_capacity(txs.len()); let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; @@ -872,9 +875,14 @@ impl VirtualStateProcessor { let mut invalid_transactions = HashMap::new(); let results = self.validate_block_template_transactions_in_parallel(&txs, &virtual_state, &virtual_utxo_view); for (tx, res) in txs.iter().zip(results) { - if let Err(e) = res { - invalid_transactions.insert(tx.id(), e); - tx_selector.reject_selection(tx.id()); + match res { + Err(e) => { + invalid_transactions.insert(tx.id(), e); + tx_selector.reject_selection(tx.id()); + } + Ok(fee) => { + calculated_fees.push(fee); + } } } @@ -889,12 +897,16 @@ impl VirtualStateProcessor { let next_batch_results = self.validate_block_template_transactions_in_parallel(&next_batch, &virtual_state, &virtual_utxo_view); for (tx, res) in next_batch.into_iter().zip(next_batch_results) { - if let Err(e) = res { - invalid_transactions.insert(tx.id(), e); - tx_selector.reject_selection(tx.id()); - has_rejections = true; - } else { - txs.push(tx); + match res { + Err(e) => { + invalid_transactions.insert(tx.id(), e); + tx_selector.reject_selection(tx.id()); + has_rejections = true; + } + Ok(fee) => { + txs.push(tx); + calculated_fees.push(fee); + } } } } @@ -911,7 +923,7 @@ impl VirtualStateProcessor { drop(virtual_read); // Build the template - self.build_block_template_from_virtual_state(virtual_state, miner_data, txs) + self.build_block_template_from_virtual_state(virtual_state, miner_data, txs, calculated_fees) } pub(crate) fn validate_block_template_transactions( @@ -939,6 +951,7 @@ impl VirtualStateProcessor { virtual_state: Arc, miner_data: MinerData, mut txs: Vec, + calculated_fees: Vec, ) -> Result { // [`calc_block_parents`] can use deep blocks below the pruning point for this calculation, so we // need to hold the pruning lock. @@ -962,7 +975,7 @@ impl VirtualStateProcessor { // Hash according to hardfork activation let storage_mass_activated = virtual_state.daa_score > self.storage_mass_activation_daa_score; - let hash_merkle_root = calc_hash_merkle_root_with_options(txs.iter(), storage_mass_activated); + let hash_merkle_root = calc_hash_merkle_root(txs.iter(), storage_mass_activated); let accepted_id_merkle_root = kaspa_merkle::calc_merkle_root(virtual_state.accepted_tx_ids.iter().copied()); let utxo_commitment = virtual_state.multiset.clone().finalize(); @@ -992,6 +1005,7 @@ impl VirtualStateProcessor { selected_parent_timestamp, selected_parent_daa_score, selected_parent_hash, + calculated_fees, )) } diff --git a/consensus/src/pipeline/virtual_processor/test_block_builder.rs b/consensus/src/pipeline/virtual_processor/test_block_builder.rs index 872bf15b4..2654a6a5f 100644 --- a/consensus/src/pipeline/virtual_processor/test_block_builder.rs +++ b/consensus/src/pipeline/virtual_processor/test_block_builder.rs @@ -61,6 +61,6 @@ impl TestBlockBuilder { let pov_virtual_utxo_view = (&virtual_read.utxo_set).compose(accumulated_diff); self.validate_block_template_transactions(&txs, &pov_virtual_state, &pov_virtual_utxo_view)?; drop(virtual_read); - self.build_block_template_from_virtual_state(pov_virtual_state, miner_data, txs) + self.build_block_template_from_virtual_state(pov_virtual_state, miner_data, txs, vec![]) } } diff --git a/consensus/src/pipeline/virtual_processor/utxo_validation.rs b/consensus/src/pipeline/virtual_processor/utxo_validation.rs index 0e3ca7533..f6dc3a470 100644 --- a/consensus/src/pipeline/virtual_processor/utxo_validation.rs +++ b/consensus/src/pipeline/virtual_processor/utxo_validation.rs @@ -15,6 +15,7 @@ use crate::{ }; use kaspa_consensus_core::{ acceptance_data::{AcceptedTxEntry, MergesetBlockAcceptanceData}, + api::args::TransactionValidationArgs, coinbase::*, hashing, header::Header, @@ -248,7 +249,7 @@ impl VirtualStateProcessor { } } let populated_tx = PopulatedTransaction::new(transaction, entries); - let res = self.transaction_validator.validate_populated_transaction_and_get_fee(&populated_tx, pov_daa_score, flags); + let res = self.transaction_validator.validate_populated_transaction_and_get_fee(&populated_tx, pov_daa_score, flags, None); match res { Ok(calculated_fee) => Ok(ValidatedTransaction::new(populated_tx, calculated_fee)), Err(tx_rule_error) => { @@ -290,6 +291,7 @@ impl VirtualStateProcessor { mutable_tx: &mut MutableTransaction, utxo_view: &impl UtxoView, pov_daa_score: u64, + args: &TransactionValidationArgs, ) -> TxResult<()> { self.populate_mempool_transaction_in_utxo_context(mutable_tx, utxo_view)?; @@ -308,10 +310,12 @@ impl VirtualStateProcessor { mutable_tx.tx.set_mass(contextual_mass); // At this point we know all UTXO entries are populated, so we can safely pass the tx as verifiable + let mass_and_feerate_threshold = args.feerate_threshold.map(|threshold| (contextual_mass, threshold)); let calculated_fee = self.transaction_validator.validate_populated_transaction_and_get_fee( &mutable_tx.as_verifiable(), pov_daa_score, TxValidationFlags::SkipMassCheck, // we can skip the mass check since we just set it + mass_and_feerate_threshold, )?; mutable_tx.calculated_fee = Some(calculated_fee); Ok(()) diff --git a/consensus/src/processes/parents_builder.rs b/consensus/src/processes/parents_builder.rs index 2e6d5ae42..4069c3dca 100644 --- a/consensus/src/processes/parents_builder.rs +++ b/consensus/src/processes/parents_builder.rs @@ -10,8 +10,6 @@ use crate::model::{ stores::{headers::HeaderStoreReader, reachability::ReachabilityStoreReader, relations::RelationsStoreReader}, }; -use super::reachability::ReachabilityResultExtensions; - #[derive(Clone)] pub struct ParentsManager { max_block_level: BlockLevel, @@ -52,17 +50,8 @@ impl .expect("at least one of the parents is expected to be in the future of the pruning point"); direct_parent_headers.swap(0, first_parent_in_future_of_pruning_point); - let origin_children = self.relations_service.get_children(ORIGIN).unwrap().read().iter().copied().collect_vec(); - let origin_children_headers = - origin_children.iter().copied().map(|parent| self.headers_store.get_header(parent).unwrap()).collect_vec(); - - // First, handle the genesis parent case. This avoids the need to check this possibility within the loop below. - if direct_parents == [self.genesis_hash] { - return vec![vec![self.genesis_hash]]; - } - - // Full capacity of max levels is unexpected - let mut parents = Vec::with_capacity(self.max_block_level as usize / 2); + let mut origin_children_headers = None; + let mut parents = Vec::with_capacity(self.max_block_level as usize); for block_level in 0..self.max_block_level { if direct_parent_headers.iter().all(|h| block_level <= h.block_level) { @@ -108,11 +97,7 @@ impl }; for (i, parent) in grandparents.into_iter().enumerate() { - let is_in_origin_children_future = self - .reachability_service - .is_any_dag_ancestor_result(&mut origin_children.iter().copied(), parent) - .unwrap_option() - .is_some_and(|r| r); + let has_reachability_data = self.reachability_service.has_reachability_data(parent); // Reference blocks are the blocks that are used in reachability queries to check if // a candidate is in the future of another candidate. In most cases this is just the @@ -121,13 +106,24 @@ impl // If we make sure to add a parent in the future of the pruning point first, we can // know that any pruned candidate that is in the past of some blocks in the pruning // point anticone should be a parent (in the relevant level) of one of - // the virtual genesis children in the pruning point anticone. So we can check which - // virtual genesis children have this block as parent and use those block as + // the origin children in the pruning point anticone. So we can check which + // origin children have this block as parent and use those block as // reference blocks. - let reference_blocks = if is_in_origin_children_future { + let reference_blocks = if has_reachability_data { smallvec![parent] } else { - let mut reference_blocks = SmallVec::with_capacity(origin_children.len()); + // Here we explicitly declare the type because otherwise Rust would make it mutable. + let origin_children_headers: &Vec<_> = origin_children_headers.get_or_insert_with(|| { + self.relations_service + .get_children(ORIGIN) + .unwrap() + .read() + .iter() + .copied() + .map(|parent| self.headers_store.get_header(parent).unwrap()) + .collect_vec() + }); + let mut reference_blocks = SmallVec::with_capacity(origin_children_headers.len()); for child_header in origin_children_headers.iter() { if self.parents_at_level(child_header, block_level).contains(&parent) { reference_blocks.push(child_header.hash); @@ -144,7 +140,7 @@ impl continue; } - if !is_in_origin_children_future { + if !has_reachability_data { continue; } diff --git a/consensus/src/processes/pruning_proof/mod.rs b/consensus/src/processes/pruning_proof/mod.rs index c04ac26fb..2449f3fcb 100644 --- a/consensus/src/processes/pruning_proof/mod.rs +++ b/consensus/src/processes/pruning_proof/mod.rs @@ -12,6 +12,7 @@ use std::{ }; use itertools::Itertools; +use kaspa_math::int::SignedInteger; // use kaspa_math::int::SignedInteger; use parking_lot::{Mutex, RwLock}; use rocksdb::WriteBatch; @@ -49,7 +50,7 @@ use crate::{ }, stores::{ depth::DbDepthStore, - ghostdag::{DbGhostdagStore, GhostdagData, GhostdagStore, GhostdagStoreReader}, + ghostdag::{CompactGhostdagData, DbGhostdagStore, GhostdagData, GhostdagStore, GhostdagStoreReader}, headers::{DbHeadersStore, HeaderStore, HeaderStoreReader, HeaderWithBlockLevel}, headers_selected_tip::DbHeadersSelectedTipStore, past_pruning_points::{DbPastPruningPointsStore, PastPruningPointsStore}, @@ -225,18 +226,30 @@ impl PruningProofManager { let pruning_point_header = proof[0].last().unwrap().clone(); let pruning_point = pruning_point_header.hash; - let proof_zero_set = BlockHashSet::from_iter(proof[0].iter().map(|header| header.hash)); + // Create a copy of the proof, since we're going to be mutating the proof passed to us + let proof_sets: Vec> = (0..=self.max_block_level) + .map(|level| BlockHashSet::from_iter(proof[level as usize].iter().map(|header| header.hash))) + .collect(); + let mut trusted_gd_map: BlockHashMap = BlockHashMap::new(); for tb in trusted_set.iter() { trusted_gd_map.insert(tb.block.hash(), tb.ghostdag.clone().into()); - if proof_zero_set.contains(&tb.block.hash()) { - continue; - } + let tb_block_level = calc_block_level(&tb.block.header, self.max_block_level); + + (0..=tb_block_level).for_each(|current_proof_level| { + // If this block was in the original proof, ignore it + if proof_sets[current_proof_level as usize].contains(&tb.block.hash()) { + return; + } - proof[0].push(tb.block.header.clone()); + proof[current_proof_level as usize].push(tb.block.header.clone()); + }); } - proof[0].sort_by(|a, b| a.blue_work.cmp(&b.blue_work)); + proof.iter_mut().for_each(|level_proof| { + level_proof.sort_by(|a, b| a.blue_work.cmp(&b.blue_work)); + }); + self.populate_reachability_and_headers(&proof); { @@ -473,6 +486,7 @@ impl PruningProofManager { &self, proof: &PruningPointProof, ctx: &mut TempProofContext, + log_validating: bool, ) -> PruningImportResult> { let headers_store = &ctx.headers_store; let ghostdag_stores = &ctx.ghostdag_stores; @@ -490,7 +504,9 @@ impl PruningProofManager { return Err(PruningImportError::PruningValidationInterrupted); } - info!("Validating level {level} from the pruning point proof ({} headers)", proof[level as usize].len()); + if log_validating { + info!("Validating level {level} from the pruning point proof ({} headers)", proof[level as usize].len()); + } let level_idx = level as usize; let mut selected_tip = None; for (i, header) in proof[level as usize].iter().enumerate() { @@ -597,32 +613,33 @@ impl PruningProofManager { Ok(()) } - /// Returns the common ancestor of the proof and the current consensus if there is one. - /// - /// ghostdag_stores currently contain only entries for blocks in the proof. - /// While iterating through the selected parent chain of the current consensus, if we find any - /// that is already in ghostdag_stores that must mean it's a common ancestor of the proof - /// and current consensus - fn find_proof_and_consensus_common_ancestor( + // find_proof_and_consensus_common_chain_ancestor_ghostdag_data returns an option of a tuple + // that contains the ghostdag data of the proof and current consensus common ancestor. If no + // such ancestor exists, it returns None. + fn find_proof_and_consensus_common_ancestor_ghostdag_data( &self, - ghostdag_store: &Arc, - current_consensus_selected_tip_header: Arc
, + proof_ghostdag_stores: &[Arc], + current_consensus_ghostdag_stores: &[Arc], + proof_selected_tip: Hash, level: BlockLevel, - relations_service: &MTRelationsService, - ) -> Option { - let mut chain_block = current_consensus_selected_tip_header.clone(); - - for _ in 0..(2 * self.pruning_proof_m as usize) { - if chain_block.direct_parents().is_empty() || chain_block.hash.is_origin() { - break; - } - if ghostdag_store.has(chain_block.hash).unwrap() { - return Some(chain_block.hash); - } - chain_block = self.find_selected_parent_header_at_level(&chain_block, level, relations_service).unwrap(); + proof_selected_tip_gd: CompactGhostdagData, + ) -> Option<(CompactGhostdagData, CompactGhostdagData)> { + let mut proof_current = proof_selected_tip; + let mut proof_current_gd = proof_selected_tip_gd; + loop { + match current_consensus_ghostdag_stores[level as usize].get_compact_data(proof_current).unwrap_option() { + Some(current_gd) => { + break Some((proof_current_gd, current_gd)); + } + None => { + proof_current = proof_current_gd.selected_parent; + if proof_current.is_origin() { + break None; + } + proof_current_gd = proof_ghostdag_stores[level as usize].get_compact_data(proof_current).unwrap(); + } + }; } - - None } pub fn validate_pruning_point_proof(&self, proof: &PruningPointProof) -> PruningImportResult<()> { @@ -630,23 +647,43 @@ impl PruningProofManager { return Err(PruningImportError::ProofNotEnoughLevels(self.max_block_level as usize + 1)); } + // Initialize the stores for the proof + let mut proof_stores_and_processes = self.init_validate_pruning_point_proof_stores_and_processes(proof)?; let proof_pp_header = proof[0].last().expect("checked if empty"); let proof_pp = proof_pp_header.hash; let proof_pp_level = calc_block_level(proof_pp_header, self.max_block_level); - let mut stores_and_processes = self.init_validate_pruning_point_proof_stores_and_processes(proof)?; - let selected_tip_by_level = self.populate_stores_for_validate_pruning_point_proof(proof, &mut stores_and_processes)?; - let ghostdag_stores = stores_and_processes.ghostdag_stores; + let proof_selected_tip_by_level = + self.populate_stores_for_validate_pruning_point_proof(proof, &mut proof_stores_and_processes, true)?; + let proof_ghostdag_stores = proof_stores_and_processes.ghostdag_stores; + + // Get the proof for the current consensus and recreate the stores for it + // This is expected to be fast because if a proof exists, it will be cached. + // If no proof exists, this is empty + let mut current_consensus_proof = self.get_pruning_point_proof(); + if current_consensus_proof.is_empty() { + // An empty proof can only happen if we're at genesis. We're going to create a proof for this case that contains the genesis header only + let genesis_header = self.headers_store.get_header(self.genesis_hash).unwrap(); + current_consensus_proof = Arc::new((0..=self.max_block_level).map(|_| vec![genesis_header.clone()]).collect_vec()); + } + let mut current_consensus_stores_and_processes = + self.init_validate_pruning_point_proof_stores_and_processes(¤t_consensus_proof)?; + let _ = self.populate_stores_for_validate_pruning_point_proof( + ¤t_consensus_proof, + &mut current_consensus_stores_and_processes, + false, + )?; + let current_consensus_ghostdag_stores = current_consensus_stores_and_processes.ghostdag_stores; let pruning_read = self.pruning_point_store.read(); - // let relations_read = self.relations_stores.read(); + let relations_read = self.relations_stores.read(); let current_pp = pruning_read.get().unwrap().pruning_point; - let current_pp_header = self.headers_store.get_header_with_block_level(current_pp).unwrap(); + let current_pp_header = self.headers_store.get_header(current_pp).unwrap(); - for (level_idx, selected_tip) in selected_tip_by_level.iter().copied().enumerate() { + for (level_idx, selected_tip) in proof_selected_tip_by_level.iter().copied().enumerate() { let level = level_idx as BlockLevel; self.validate_proof_selected_tip(selected_tip, level, proof_pp_level, proof_pp, proof_pp_header)?; - let proof_selected_tip_gd = ghostdag_stores[level_idx].get_compact_data(selected_tip).unwrap(); + let proof_selected_tip_gd = proof_ghostdag_stores[level_idx].get_compact_data(selected_tip).unwrap(); // Next check is to see if this proof is "better" than what's in the current consensus // Step 1 - look at only levels that have a full proof (least 2m blocks in the proof) @@ -655,43 +692,26 @@ impl PruningProofManager { } // Step 2 - if we can find a common ancestor between the proof and current consensus - // we can determine if the proof is better. The proof is better if the score difference between the - // old current consensus's tips and the common ancestor is less than the score difference between the + // we can determine if the proof is better. The proof is better if the blue work difference between the + // old current consensus's tips and the common ancestor is less than the blue work difference between the // proof's tip and the common ancestor - let relations_service = MTRelationsService::new(self.relations_stores.clone(), level); - let current_consensus_selected_tip_header = if current_pp_header.block_level >= level { - current_pp_header.header.clone() - } else { - self.find_selected_parent_header_at_level(¤t_pp_header.header, level, &relations_service).unwrap() - }; - if let Some(common_ancestor) = self.find_proof_and_consensus_common_ancestor( - &ghostdag_stores[level_idx], - current_consensus_selected_tip_header.clone(), + if let Some((proof_common_ancestor_gd, common_ancestor_gd)) = self.find_proof_and_consensus_common_ancestor_ghostdag_data( + &proof_ghostdag_stores, + ¤t_consensus_ghostdag_stores, + selected_tip, level, - &relations_service, + proof_selected_tip_gd, ) { - // Fill the GD store with data from current consensus, - // starting from the common ancestor until the current level selected tip - let _ = self.fill_proof_ghostdag_data( - proof[level_idx].first().unwrap().hash, - common_ancestor, - current_consensus_selected_tip_header.hash, - &ghostdag_stores[level_idx], - &relations_service, - level != 0, - None, - false, - ); - // let common_ancestor_blue_work = ghostdag_stores[level_idx].get_blue_work(common_ancestor).unwrap(); - // let selected_tip_blue_work_diff = - // SignedInteger::from(proof_selected_tip_gd.blue_work) - SignedInteger::from(common_ancestor_blue_work); - // for parent in self.parents_manager.parents_at_level(¤t_pp_header.header, level).iter().copied() { - // let parent_blue_work = ghostdag_stores[level_idx].get_blue_work(parent).unwrap(); - // let parent_blue_work_diff = SignedInteger::from(parent_blue_work) - SignedInteger::from(common_ancestor_blue_work); - // if parent_blue_work_diff >= selected_tip_blue_work_diff { - // return Err(PruningImportError::PruningProofInsufficientBlueWork); - // } - // } + let selected_tip_blue_work_diff = + SignedInteger::from(proof_selected_tip_gd.blue_work) - SignedInteger::from(proof_common_ancestor_gd.blue_work); + for parent in self.parents_manager.parents_at_level(¤t_pp_header, level).iter().copied() { + let parent_blue_work = current_consensus_ghostdag_stores[level_idx].get_blue_work(parent).unwrap(); + let parent_blue_work_diff = + SignedInteger::from(parent_blue_work) - SignedInteger::from(common_ancestor_gd.blue_work); + if parent_blue_work_diff >= selected_tip_blue_work_diff { + return Err(PruningImportError::PruningProofInsufficientBlueWork); + } + } return Ok(()); } @@ -710,34 +730,30 @@ impl PruningProofManager { for level in (0..=self.max_block_level).rev() { let level_idx = level as usize; - let proof_selected_tip = selected_tip_by_level[level_idx]; - let proof_selected_tip_gd = ghostdag_stores[level_idx].get_compact_data(proof_selected_tip).unwrap(); + let proof_selected_tip = proof_selected_tip_by_level[level_idx]; + let proof_selected_tip_gd = proof_ghostdag_stores[level_idx].get_compact_data(proof_selected_tip).unwrap(); if proof_selected_tip_gd.blue_score < 2 * self.pruning_proof_m { continue; } - return Ok(()); - - // match relations_read[level_idx].get_parents(current_pp).unwrap_option() { - // Some(parents) => { - // if parents - // .iter() - // .copied() - // .any(|parent| ghostdag_stores[level_idx].get_blue_score(parent).unwrap() < 2 * self.pruning_proof_m) - // { - // return Ok(()); - // } - // } - // None => { - // // If the current pruning point doesn't have a parent at this level, we consider the proof state to be better. - // return Ok(()); - // } - // } + match relations_read[level_idx].get_parents(current_pp).unwrap_option() { + Some(parents) => { + if parents.iter().copied().any(|parent| { + current_consensus_ghostdag_stores[level_idx].get_blue_score(parent).unwrap() < 2 * self.pruning_proof_m + }) { + return Ok(()); + } + } + None => { + // If the current pruning point doesn't have a parent at this level, we consider the proof state to be better. + return Ok(()); + } + } } drop(pruning_read); - // drop(relations_read); - drop(stores_and_processes.db_lifetime); + drop(proof_stores_and_processes.db_lifetime); + drop(current_consensus_stores_and_processes.db_lifetime); Err(PruningImportError::PruningProofNotEnoughHeaders) } @@ -927,8 +943,16 @@ impl PruningProofManager { tries += 1; if finished_headers { - warn!("Failed to find sufficient root for level {level} after {tries} tries. Headers below the current depth of {required_level_0_depth} are already pruned. Trying anyway."); - break Ok((ghostdag_store, selected_tip, root)); + if has_required_block { + // Normally this scenario doesn't occur when syncing with nodes that already have the safety margin change in place. + // However, when syncing with an older node version that doesn't have a safety margin for the proof, it's possible to + // try to find 2500 depth worth of headers at a level, but the proof only contains about 2000 headers. To be able to sync + // with such an older node. As long as we found the required block, we can still proceed. + warn!("Failed to find sufficient root for level {level} after {tries} tries. Headers below the current depth of {required_level_0_depth} are already pruned. Required block found so trying anyway."); + break Ok((ghostdag_store, selected_tip, root)); + } else { + panic!("Failed to find sufficient root for level {level} after {tries} tries. Headers below the current depth of {required_level_0_depth} are already pruned"); + } } required_level_0_depth <<= 1; warn!("Failed to find sufficient root for level {level} after {tries} tries. Retrying again to find with depth {required_level_0_depth}"); @@ -989,8 +1013,12 @@ impl PruningProofManager { .map_err(|err| format!("level: {}, err: {}", level, err)) .unwrap(); + // (New Logic) This is the root we calculated by going through block relations let root = roots_by_level[level]; - let old_root = if level != self.max_block_level as usize { + // (Old Logic) This is the root we can calculate given that the GD records are already filled + // The root calc logic below is the original logic before the on-demand higher level GD calculation + // We only need depth_based_root to sanity check the new logic + let depth_based_root = if level != self.max_block_level as usize { let block_at_depth_m_at_next_level = self .block_at_depth(&*ghostdag_stores[level + 1], selected_tip_by_level[level + 1], self.pruning_proof_m) .map_err(|err| format!("level + 1: {}, err: {}", level + 1, err)) @@ -1012,8 +1040,8 @@ impl PruningProofManager { block_at_depth_2m }; - // new root is expected to be always an ancestor of old root because new root takes a safety margin - assert!(self.reachability_service.is_dag_ancestor_of(root, old_root)); + // new root is expected to be always an ancestor of depth_based_root because new root takes a safety margin + assert!(self.reachability_service.is_dag_ancestor_of(root, depth_based_root)); let mut headers = Vec::with_capacity(2 * self.pruning_proof_m as usize); let mut queue = BinaryHeap::>::new(); diff --git a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs index e53200fec..830fd36fb 100644 --- a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs +++ b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs @@ -1,12 +1,11 @@ use crate::constants::{MAX_SOMPI, SEQUENCE_LOCK_TIME_DISABLED, SEQUENCE_LOCK_TIME_MASK}; -use kaspa_consensus_core::hashing::sighash::{SigHashReusedValues, SigHashReusedValuesSync}; -use kaspa_consensus_core::{hashing::sighash::SigHashReusedValuesUnsync, tx::VerifiableTransaction}; +use kaspa_consensus_core::{ + hashing::sighash::{SigHashReusedValues, SigHashReusedValuesUnsync}, + tx::{TransactionInput, VerifiableTransaction}, +}; use kaspa_core::warn; -use kaspa_txscript::caches::Cache; -use kaspa_txscript::{get_sig_op_count, SigCacheKey, TxScriptEngine}; -use rayon::iter::IntoParallelIterator; -use rayon::ThreadPool; -use std::sync::Arc; +use kaspa_txscript::{get_sig_op_count, TxScriptEngine}; +use kaspa_txscript_errors::TxScriptError; use super::{ errors::{TxResult, TxRuleError}, @@ -32,10 +31,12 @@ impl TransactionValidator { tx: &(impl VerifiableTransaction + std::marker::Sync), pov_daa_score: u64, flags: TxValidationFlags, + mass_and_feerate_threshold: Option<(u64, f64)>, ) -> TxResult { self.check_transaction_coinbase_maturity(tx, pov_daa_score)?; let total_in = self.check_transaction_input_amounts(tx)?; let total_out = Self::check_transaction_output_values(tx, total_in)?; + let fee = total_in - total_out; if flags != TxValidationFlags::SkipMassCheck && pov_daa_score > self.storage_mass_activation_daa_score { // Storage mass hardfork was activated self.check_mass_commitment(tx)?; @@ -45,6 +46,11 @@ impl TransactionValidator { } } Self::check_sequence_lock(tx, pov_daa_score)?; + + // The following call is not a consensus check (it could not be one in the first place since it uses floating number) + // but rather a mempool Replace by Fee validation rule. It was placed here purposely for avoiding unneeded script checks. + Self::check_feerate_threshold(fee, mass_and_feerate_threshold)?; + match flags { TxValidationFlags::Full | TxValidationFlags::SkipMassCheck => { Self::check_sig_op_counts::<_, SigHashReusedValuesUnsync>(tx)?; @@ -52,7 +58,19 @@ impl TransactionValidator { } TxValidationFlags::SkipScriptChecks => {} } - Ok(total_in - total_out) + Ok(fee) + } + + fn check_feerate_threshold(fee: u64, mass_and_feerate_threshold: Option<(u64, f64)>) -> TxResult<()> { + // An actual check can only occur if some mass and threshold are provided, + // otherwise, the check does not verify anything and exits successfully. + if let Some((contextual_mass, feerate_threshold)) = mass_and_feerate_threshold { + assert!(contextual_mass > 0); + if fee as f64 / contextual_mass as f64 <= feerate_threshold { + return Err(TxRuleError::FeerateTooLow); + } + } + Ok(()) } fn check_transaction_coinbase_maturity(&self, tx: &impl VerifiableTransaction, pov_daa_score: u64) -> TxResult<()> { @@ -149,64 +167,24 @@ impl TransactionValidator { Ok(()) } - pub fn check_scripts(&self, tx: &(impl VerifiableTransaction + std::marker::Sync)) -> TxResult<()> { - check_scripts(&self.sig_cache, tx) - } -} + pub fn check_scripts(&self, tx: &impl VerifiableTransaction) -> TxResult<()> { + let mut reused_values = SigHashReusedValuesUnsync::new(); + for (i, (input, entry)) in tx.populated_inputs().enumerate() { + let mut engine = TxScriptEngine::from_transaction_input(tx, input, i, entry, &mut reused_values, &self.sig_cache) + .map_err(|err| map_script_err(err, input))?; + engine.execute().map_err(|err| map_script_err(err, input))?; + } -pub fn check_scripts(sig_cache: &Cache, tx: &(impl VerifiableTransaction + Sync)) -> TxResult<()> { - if tx.inputs().len() > 1 { - check_scripts_par_iter(sig_cache, tx) - } else { - check_scripts_single_threaded(sig_cache, tx) + Ok(()) } } -pub fn check_scripts_single_threaded(sig_cache: &Cache, tx: &impl VerifiableTransaction) -> TxResult<()> { - let reused_values = SigHashReusedValuesUnsync::new(); - for (i, (input, entry)) in tx.populated_inputs().enumerate() { - let mut engine = TxScriptEngine::from_transaction_input(tx, input, i, entry, &reused_values, sig_cache) - .map_err(TxRuleError::SignatureInvalid)?; - engine.execute().map_err(TxRuleError::SignatureInvalid)?; +fn map_script_err(script_err: TxScriptError, input: &TransactionInput) -> TxRuleError { + if input.signature_script.is_empty() { + TxRuleError::SignatureEmpty(script_err) + } else { + TxRuleError::SignatureInvalid(script_err) } - Ok(()) -} - -pub fn check_scripts_par_iter( - sig_cache: &Cache, - tx: &(impl VerifiableTransaction + std::marker::Sync), -) -> TxResult<()> { - use rayon::iter::ParallelIterator; - let reused_values = std::sync::Arc::new(SigHashReusedValuesSync::new()); - (0..tx.inputs().len()) - .into_par_iter() - .try_for_each(|idx| { - let reused_values = reused_values.clone(); // Clone the Arc to share ownership - let (input, utxo) = tx.populated_input(idx); - let mut engine = TxScriptEngine::from_transaction_input(tx, input, idx, utxo, &reused_values, sig_cache)?; - engine.execute() - }) - .map_err(TxRuleError::SignatureInvalid) -} - -pub fn check_scripts_par_iter_thread( - sig_cache: &Cache, - tx: &(impl VerifiableTransaction + std::marker::Sync), - pool: &ThreadPool, -) -> TxResult<()> { - use rayon::iter::ParallelIterator; - pool.install(|| { - let reused_values = Arc::new(SigHashReusedValuesSync::new()); - (0..tx.inputs().len()) - .into_par_iter() - .try_for_each(|idx| { - let reused_values = reused_values.clone(); // Clone the Arc to share ownership - let (input, utxo) = tx.populated_input(idx); - let mut engine = TxScriptEngine::from_transaction_input(tx, input, idx, utxo, &reused_values, sig_cache)?; - engine.execute() - }) - .map_err(TxRuleError::SignatureInvalid) - }) } #[cfg(test)] diff --git a/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs b/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs index 67901612d..914624f94 100644 --- a/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs +++ b/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs @@ -17,6 +17,7 @@ impl TransactionValidator { check_duplicate_transaction_inputs(tx)?; check_gas(tx)?; check_transaction_payload(tx)?; + check_transaction_subnetwork(tx)?; check_transaction_version(tx) } @@ -146,10 +147,18 @@ fn check_transaction_output_value_ranges(tx: &Transaction) -> TxResult<()> { Ok(()) } +fn check_transaction_subnetwork(tx: &Transaction) -> TxResult<()> { + if tx.is_coinbase() || tx.subnetwork_id.is_native() { + Ok(()) + } else { + Err(TxRuleError::SubnetworksDisabled(tx.subnetwork_id.clone())) + } +} + #[cfg(test)] mod tests { use kaspa_consensus_core::{ - subnets::{SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, + subnets::{SubnetworkId, SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, tx::{scriptvec, ScriptPublicKey, Transaction, TransactionId, TransactionInput, TransactionOutpoint, TransactionOutput}, }; use kaspa_core::assert_match; @@ -261,6 +270,10 @@ mod tests { tv.validate_tx_in_isolation(&valid_tx).unwrap(); + let mut tx: Transaction = valid_tx.clone(); + tx.subnetwork_id = SubnetworkId::from_byte(3); + assert_match!(tv.validate_tx_in_isolation(&tx), Err(TxRuleError::SubnetworksDisabled(_))); + let mut tx = valid_tx.clone(); tx.inputs = vec![]; assert_match!(tv.validate_tx_in_isolation(&tx), Err(TxRuleError::NoTxInputs)); diff --git a/crypto/hashes/src/hashers.rs b/crypto/hashes/src/hashers.rs index 7f6775aaa..b45026bd2 100644 --- a/crypto/hashes/src/hashers.rs +++ b/crypto/hashes/src/hashers.rs @@ -51,7 +51,7 @@ macro_rules! sha256_hasher { // SHA256 doesn't natively support domain separation, so we hash it to make it constant size. let mut tmp_state = Sha256::new(); tmp_state.update($domain_sep); - let mut out = Self(Sha256::new()); + let mut out = $name(Sha256::new()); out.write(tmp_state.finalize()); out diff --git a/crypto/muhash/Cargo.toml b/crypto/muhash/Cargo.toml index b5badb664..cef8ec5bf 100644 --- a/crypto/muhash/Cargo.toml +++ b/crypto/muhash/Cargo.toml @@ -26,3 +26,5 @@ rand.workspace = true name = "bench" harness = false +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] } diff --git a/crypto/txscript/src/standard.rs b/crypto/txscript/src/standard.rs index fb7eb455a..3c0f12a18 100644 --- a/crypto/txscript/src/standard.rs +++ b/crypto/txscript/src/standard.rs @@ -100,9 +100,9 @@ pub mod test_helpers { (script_public_key, redeem_script) } - // Creates a transaction that spends the first output of provided transaction. - // Assumes that the output being spent has opTrueScript as it's scriptPublicKey. - // Creates the value of the spent output minus provided `fee` (in sompi). + /// Creates a transaction that spends the first output of provided transaction. + /// Assumes that the output being spent has opTrueScript as its scriptPublicKey. + /// Creates the value of the spent output minus provided `fee` (in sompi). pub fn create_transaction(tx_to_spend: &Transaction, fee: u64) -> Transaction { let (script_public_key, redeem_script) = op_true_script(); let signature_script = pay_to_script_hash_signature_script(redeem_script, vec![]).expect("the script is canonical"); @@ -111,6 +111,42 @@ pub mod test_helpers { let output = TransactionOutput::new(tx_to_spend.outputs[0].value - fee, script_public_key); Transaction::new(TX_VERSION, vec![input], vec![output], 0, SUBNETWORK_ID_NATIVE, 0, vec![]) } + + /// Creates a transaction that spends the outputs of specified indexes (if they exist) of every provided transaction and returns an optional change. + /// Assumes that the outputs being spent have opTrueScript as their scriptPublicKey. + /// + /// If some change is provided, creates two outputs, first one with the value of the spent outputs minus `change` + /// and `fee` (in sompi) and second one of `change` amount. + /// + /// If no change is provided, creates only one output with the value of the spent outputs minus and `fee` (in sompi) + pub fn create_transaction_with_change<'a>( + txs_to_spend: impl Iterator, + output_indexes: Vec, + change: Option, + fee: u64, + ) -> Transaction { + let (script_public_key, redeem_script) = op_true_script(); + let signature_script = pay_to_script_hash_signature_script(redeem_script, vec![]).expect("the script is canonical"); + let mut inputs_value: u64 = 0; + let mut inputs = vec![]; + for tx_to_spend in txs_to_spend { + for i in output_indexes.iter().copied() { + if i < tx_to_spend.outputs.len() { + let previous_outpoint = TransactionOutpoint::new(tx_to_spend.id(), i as u32); + inputs.push(TransactionInput::new(previous_outpoint, signature_script.clone(), MAX_TX_IN_SEQUENCE_NUM, 1)); + inputs_value += tx_to_spend.outputs[i].value; + } + } + } + let outputs = match change { + Some(change) => vec![ + TransactionOutput::new(inputs_value - fee - change, script_public_key.clone()), + TransactionOutput::new(change, script_public_key), + ], + None => vec![TransactionOutput::new(inputs_value - fee, script_public_key.clone())], + }; + Transaction::new(TX_VERSION, inputs, outputs, 0, SUBNETWORK_ID_NATIVE, 0, vec![]) + } } #[cfg(test)] diff --git a/kaspad/Cargo.toml b/kaspad/Cargo.toml index 0decbc9cc..3507339f2 100644 --- a/kaspad/Cargo.toml +++ b/kaspad/Cargo.toml @@ -41,9 +41,9 @@ kaspa-utxoindex.workspace = true kaspa-wrpc-server.workspace = true async-channel.workspace = true +cfg-if.workspace = true clap.workspace = true dhat = { workspace = true, optional = true } -serde.workspace = true dirs.workspace = true futures-util.workspace = true itertools.workspace = true @@ -52,13 +52,16 @@ num_cpus.workspace = true rand.workspace = true rayon.workspace = true rocksdb.workspace = true +serde.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } workflow-log.workspace = true + toml = "0.8.10" serde_with = "3.7.0" [features] heap = ["dhat", "kaspa-alloc/heap"] devnet-prealloc = ["kaspa-consensus/devnet-prealloc"] +semaphore-trace = ["kaspa-utils/semaphore-trace"] diff --git a/kaspad/src/args.rs b/kaspad/src/args.rs index 2774269d3..56dd7c1de 100644 --- a/kaspad/src/args.rs +++ b/kaspad/src/args.rs @@ -134,7 +134,7 @@ impl Default for Args { #[cfg(feature = "devnet-prealloc")] prealloc_address: None, #[cfg(feature = "devnet-prealloc")] - prealloc_amount: 1_000_000, + prealloc_amount: 10_000_000_000, disable_upnp: false, disable_dns_seeding: false, diff --git a/kaspad/src/daemon.rs b/kaspad/src/daemon.rs index 15be08f91..9cee7566a 100644 --- a/kaspad/src/daemon.rs +++ b/kaspad/src/daemon.rs @@ -166,7 +166,13 @@ impl Runtime { let log_dir = get_log_dir(args); // Initialize the logger - kaspa_core::log::init_logger(log_dir.as_deref(), &args.log_level); + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + kaspa_core::log::init_logger(log_dir.as_deref(), &format!("{},{}=debug", args.log_level, kaspa_utils::sync::semaphore_module_path())); + } else { + kaspa_core::log::init_logger(log_dir.as_deref(), &args.log_level); + } + }; // Configure the panic behavior // As we log the panic, we want to set it up after the logger @@ -205,7 +211,7 @@ pub fn create_core(args: Args, fd_total_budget: i32) -> (Arc, Arc (Arc, Arc) { let network = args.network(); - assert_ne!(network.network_type(), NetworkType::Mainnet, "Experimental version; Mainnet is disallowed"); + // assert_ne!(network.network_type(), NetworkType::Mainnet, "Experimental version; Mainnet is disallowed"); let mut fd_remaining = fd_total_budget; let utxo_files_limit = if args.utxoindex { let utxo_files_limit = fd_remaining * 10 / 100; @@ -518,16 +524,17 @@ do you confirm? (answer y/n or pass --yes to the Kaspad command line to confirm let (address_manager, port_mapping_extender_svc) = AddressManager::new(config.clone(), meta_db, tick_service.clone()); - let mining_monitor = Arc::new(MiningMonitor::new(mining_counters.clone(), tx_script_cache_counters.clone(), tick_service.clone())); let mining_manager = MiningManagerProxy::new(Arc::new(MiningManager::new_with_extended_config( config.target_time_per_block, false, config.max_block_mass, config.ram_scale, config.block_template_cache_lifetime, - mining_counters, + mining_counters.clone(), config.storage_mass_activation_daa_score, ))); + let mining_monitor = + Arc::new(MiningMonitor::new(mining_manager.clone(), mining_counters, tx_script_cache_counters.clone(), tick_service.clone())); let flow_context = Arc::new(FlowContext::new( consensus_manager.clone(), diff --git a/mining/Cargo.toml b/mining/Cargo.toml index facd45d6a..0c7eb2525 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -27,8 +27,9 @@ parking_lot.workspace = true rand.workspace = true serde.workspace = true smallvec.workspace = true +sweep-bptree = "0.4.1" thiserror.workspace = true -tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "signal" ] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } [dev-dependencies] kaspa-txscript.workspace = true diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index 59ff685dd..16cfcc234 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -1,6 +1,16 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use kaspa_mining::model::topological_index::TopologicalIndex; -use std::collections::{hash_set::Iter, HashMap, HashSet}; +use itertools::Itertools; +use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, +}; +use kaspa_hashes::{HasherBase, TransactionID}; +use kaspa_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier, Policy}; +use rand::{thread_rng, Rng}; +use std::{ + collections::{hash_set::Iter, HashMap, HashSet}, + sync::Arc, +}; #[derive(Default)] pub struct Dag @@ -68,5 +78,211 @@ pub fn bench_compare_topological_index_fns(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_compare_topological_index_fns); +fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) +} + +fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) +} + +pub fn bench_mempool_sampling(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool sampling"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + group.bench_function("mempool one-shot sample", |b| { + b.iter(|| { + black_box({ + let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + selected.iter().map(|k| k.mass).sum::() + }) + }) + }); + + // Benchmark frontier insertions and removals (see comparisons below) + let remove = map.values().take(map.len() / 10).cloned().collect_vec(); + group.bench_function("frontier remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + frontier.remove(r).then_some(()).unwrap(); + } + for r in remove.iter().cloned() { + frontier.insert(r).then_some(()).unwrap(); + } + 0 + }) + }) + }); + + // Benchmark hashmap insertions and removals for comparison + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("map remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + map.remove(&r.0).unwrap(); + } + for r in remove.iter().cloned() { + map.insert(r.0, r.1.clone()); + } + 0 + }) + }) + }); + + // Benchmark std btree set insertions and removals for comparison + // Results show that frontier (sweep bptree) and std btree set are roughly the same. + // The slightly higher cost for sweep bptree should be attributed to subtree weight + // maintenance (see FeerateWeight) + #[allow(clippy::mutable_key_type)] + let mut std_btree = std::collections::BTreeSet::from_iter(map.values().cloned()); + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("std btree remove/add", |b| { + b.iter(|| { + black_box({ + for (_, key) in remove.iter() { + std_btree.remove(key).then_some(()).unwrap(); + } + for (_, key) in remove.iter() { + std_btree.insert(key.clone()); + } + 0 + }) + }) + }); + group.finish(); +} + +pub fn bench_mempool_selectors(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool selectors"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000].into_iter().rev() { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + group.bench_function(format!("rebalancing selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("sample inplace selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + + if frontier.total_mass() <= 500_000 { + group.bench_function(format!("take all selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.bench_function(format!("dynamic selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.finish(); +} + +pub fn bench_inplace_sampling_worst_case(c: &mut Criterion) { + let mut group = c.benchmark_group("mempool inplace sampling"); + let max_fee = u64::MAX; + let fee_steps = (0..10).map(|i| max_fee / 100u64.pow(i)).collect_vec(); + for subgroup_size in [300, 200, 100, 80, 50, 30] { + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i < 300 { fee_steps[i as usize / subgroup_size] } else { 1 }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("inplace sampling worst case (subgroup size: {})", subgroup_size), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + } + + group.finish(); +} + +criterion_group!( + benches, + bench_mempool_sampling, + bench_mempool_selectors, + bench_inplace_sampling_worst_case, + bench_compare_topological_index_fns +); criterion_main!(benches); diff --git a/mining/errors/src/mempool.rs b/mining/errors/src/mempool.rs index e33737f9d..be8ff389a 100644 --- a/mining/errors/src/mempool.rs +++ b/mining/errors/src/mempool.rs @@ -4,7 +4,7 @@ use kaspa_consensus_core::{ }; use thiserror::Error; -#[derive(Error, Debug, Clone)] +#[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum RuleError { /// A consensus transaction rule error /// @@ -24,9 +24,15 @@ pub enum RuleError { #[error("transaction {0} is already in the mempool")] RejectDuplicate(TransactionId), - #[error("output {0} already spent by transaction {1} in the memory pool")] + #[error("output {0} already spent by transaction {1} in the mempool")] RejectDoubleSpendInMempool(TransactionOutpoint, TransactionId), + #[error("replace by fee found no double spending transaction in the mempool")] + RejectRbfNoDoubleSpend, + + #[error("replace by fee found more than one double spending transaction in the mempool")] + RejectRbfTooManyDoubleSpendingTransactions, + /// New behavior: a transaction is rejected if the mempool is full #[error("number of high-priority transactions in mempool ({0}) has reached the maximum allowed ({1})")] RejectMempoolIsFull(usize, u64), @@ -95,7 +101,7 @@ impl From for RuleError { pub type RuleResult = std::result::Result; -#[derive(Error, Debug, Clone)] +#[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum NonStandardError { #[error("transaction version {1} is not in the valid range of {2}-{3}")] RejectVersion(TransactionId, u16, u16, u16), diff --git a/mining/src/block_template/builder.rs b/mining/src/block_template/builder.rs index 9645ca171..6f0dbe674 100644 --- a/mining/src/block_template/builder.rs +++ b/mining/src/block_template/builder.rs @@ -1,25 +1,17 @@ -use super::{errors::BuilderResult, policy::Policy}; -use crate::{block_template::selector::TransactionsSelector, model::candidate_tx::CandidateTransaction}; +use super::errors::BuilderResult; use kaspa_consensus_core::{ api::ConsensusApi, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, - merkle::calc_hash_merkle_root_with_options, tx::COINBASE_TRANSACTION_INDEX, }; -use kaspa_core::{ - debug, - time::{unix_now, Stopwatch}, -}; +use kaspa_core::time::{unix_now, Stopwatch}; -pub(crate) struct BlockTemplateBuilder { - policy: Policy, -} +pub(crate) struct BlockTemplateBuilder {} impl BlockTemplateBuilder { - pub(crate) fn new(max_block_mass: u64) -> Self { - let policy = Policy::new(max_block_mass); - Self { policy } + pub(crate) fn new() -> Self { + Self {} } /// BuildBlockTemplate creates a block template for a miner to consume @@ -89,12 +81,10 @@ impl BlockTemplateBuilder { &self, consensus: &dyn ConsensusApi, miner_data: &MinerData, - transactions: Vec, + selector: Box, build_mode: TemplateBuildMode, ) -> BuilderResult { let _sw = Stopwatch::<20>::with_threshold("build_block_template op"); - debug!("Considering {} transactions for a new block template", transactions.len()); - let selector = Box::new(TransactionsSelector::new(self.policy.clone(), transactions)); Ok(consensus.build_block_template(miner_data.clone(), selector, build_mode)?) } @@ -103,7 +93,6 @@ impl BlockTemplateBuilder { consensus: &dyn ConsensusApi, new_miner_data: &MinerData, block_template_to_modify: &BlockTemplate, - storage_mass_activation_daa_score: u64, ) -> BuilderResult { let mut block_template = block_template_to_modify.clone(); @@ -116,9 +105,8 @@ impl BlockTemplateBuilder { coinbase_tx.outputs.last_mut().unwrap().script_public_key = new_miner_data.script_public_key.clone(); } // Update the hash merkle root according to the modified transactions - let storage_mass_activated = block_template.block.header.daa_score > storage_mass_activation_daa_score; block_template.block.header.hash_merkle_root = - calc_hash_merkle_root_with_options(block_template.block.transactions.iter(), storage_mass_activated); + consensus.calc_transaction_hash_merkle_root(&block_template.block.transactions, block_template.block.header.daa_score); let new_timestamp = unix_now(); if new_timestamp > block_template.block.header.timestamp { // Only if new time stamp is later than current, update the header. Otherwise, diff --git a/mining/src/block_template/model/tx.rs b/mining/src/block_template/model/tx.rs index b0c7e3f56..65493e63b 100644 --- a/mining/src/block_template/model/tx.rs +++ b/mining/src/block_template/model/tx.rs @@ -73,7 +73,8 @@ impl CandidateList { /// * tx1: start 0, end 100 /// * tx2: start 100, end 105 /// * tx3: start 105, end 2000 - /// And r=102, then find will return tx2. + /// + /// And r=102, then [`CandidateList::find`] will return tx2. pub(crate) fn find(&self, r: f64) -> usize { let mut min = 0; let mut max = self.candidates.len() - 1; diff --git a/mining/src/block_template/policy.rs b/mining/src/block_template/policy.rs index ff5197255..12ee98e28 100644 --- a/mining/src/block_template/policy.rs +++ b/mining/src/block_template/policy.rs @@ -1,14 +1,14 @@ /// Policy houses the policy (configuration parameters) which is used to control /// the generation of block templates. See the documentation for -/// NewBlockTemplate for more details on each of these parameters are used. +/// NewBlockTemplate for more details on how each of these parameters are used. #[derive(Clone)] -pub(crate) struct Policy { +pub struct Policy { /// max_block_mass is the maximum block mass to be used when generating a block template. pub(crate) max_block_mass: u64, } impl Policy { - pub(crate) fn new(max_block_mass: u64) -> Self { + pub fn new(max_block_mass: u64) -> Self { Self { max_block_mass } } } diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index a55ecb93d..6acacb22d 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -18,7 +18,7 @@ use kaspa_consensus_core::{ /// candidate transactions should be. A smaller alpha makes the distribution /// more uniform. ALPHA is used when determining a candidate transaction's /// initial p value. -const ALPHA: i32 = 3; +pub(crate) const ALPHA: i32 = 3; /// REBALANCE_THRESHOLD is the percentage of candidate transactions under which /// we don't rebalance. Rebalancing is a heavy operation so we prefer to avoid @@ -28,7 +28,7 @@ const ALPHA: i32 = 3; /// if REBALANCE_THRESHOLD is 0.95, there's a 1-in-20 chance of collision. const REBALANCE_THRESHOLD: f64 = 0.95; -pub(crate) struct TransactionsSelector { +pub struct RebalancingWeightedTransactionSelector { policy: Policy, /// Transaction store transactions: Vec, @@ -52,8 +52,8 @@ pub(crate) struct TransactionsSelector { gas_usage_map: HashMap, } -impl TransactionsSelector { - pub(crate) fn new(policy: Policy, mut transactions: Vec) -> Self { +impl RebalancingWeightedTransactionSelector { + pub fn new(policy: Policy, mut transactions: Vec) -> Self { let _sw = Stopwatch::<100>::with_threshold("TransactionsSelector::new op"); // Sort the transactions by subnetwork_id. transactions.sort_by(|a, b| a.tx.subnetwork_id.cmp(&b.tx.subnetwork_id)); @@ -103,7 +103,7 @@ impl TransactionsSelector { /// select_transactions loops over the candidate transactions /// and appends the ones that will be included in the next block into /// selected_txs. - pub(crate) fn select_transactions(&mut self) -> Vec { + pub fn select_transactions(&mut self) -> Vec { let _sw = Stopwatch::<15>::with_threshold("select_transaction op"); let mut rng = rand::thread_rng(); @@ -225,7 +225,7 @@ impl TransactionsSelector { } } -impl TemplateTransactionSelector for TransactionsSelector { +impl TemplateTransactionSelector for RebalancingWeightedTransactionSelector { fn select_transactions(&mut self) -> Vec { self.select_transactions() } @@ -269,7 +269,13 @@ mod tests { use kaspa_txscript::{pay_to_script_hash_signature_script, test_helpers::op_true_script}; use std::{collections::HashSet, sync::Arc}; - use crate::{mempool::config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, model::candidate_tx::CandidateTransaction}; + use crate::{ + mempool::{ + config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + model::frontier::selectors::{SequenceSelector, SequenceSelectorInput, SequenceSelectorTransaction}, + }, + model::candidate_tx::CandidateTransaction, + }; #[test] fn test_reject_transaction() { @@ -277,29 +283,43 @@ mod tests { // Create a vector of transactions differing by output value so they have unique ids let transactions = (0..TX_INITIAL_COUNT).map(|i| create_transaction(SOMPI_PER_KASPA * (i + 1) as u64)).collect_vec(); + let masses: HashMap<_, _> = transactions.iter().map(|tx| (tx.tx.id(), tx.calculated_mass)).collect(); + let sequence: SequenceSelectorInput = + transactions.iter().map(|tx| SequenceSelectorTransaction::new(tx.tx.clone(), tx.calculated_mass)).collect(); + let policy = Policy::new(100_000); - let mut selector = TransactionsSelector::new(policy, transactions); - let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); - let mut reject_count = 32; - for i in 0..10 { - let selected_txs = selector.select_transactions(); - if i > 0 { - assert_eq!( - selected_txs.len(), - reject_count, - "subsequent select calls are expected to only refill the previous rejections" - ); - reject_count /= 2; - } - for tx in selected_txs.iter() { - kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); - assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + let selectors: [Box; 2] = [ + Box::new(RebalancingWeightedTransactionSelector::new(policy.clone(), transactions)), + Box::new(SequenceSelector::new(sequence, policy.clone())), + ]; + + for mut selector in selectors { + let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); + let mut reject_count = 32; + let mut total_mass = 0; + for i in 0..10 { + let selected_txs = selector.select_transactions(); + if i > 0 { + assert_eq!( + selected_txs.len(), + reject_count, + "subsequent select calls are expected to only refill the previous rejections" + ); + reject_count /= 2; + } + for tx in selected_txs.iter() { + total_mass += masses[&tx.id()]; + kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); + assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + } + assert!(total_mass <= policy.max_block_mass); + selected_txs.iter().take(reject_count).for_each(|x| { + total_mass -= masses[&x.id()]; + selector.reject_selection(x.id()); + kept.remove(&x.id()).then_some(()).expect("was just inserted"); + rejected.insert(x.id()).then_some(()).expect("was just verified"); + }); } - selected_txs.iter().take(reject_count).for_each(|x| { - selector.reject_selection(x.id()); - kept.remove(&x.id()).then_some(()).expect("was just inserted"); - rejected.insert(x.id()).then_some(()).expect("was just verified"); - }); } } diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb new file mode 100644 index 000000000..694f47450 --- /dev/null +++ b/mining/src/feerate/fee_estimation.ipynb @@ -0,0 +1,496 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerates\n", + "\n", + "The feerate value represents the fee/mass ratio of a transaction in `sompi/gram` units.\n", + "Given a feerate value recommendation, one should calculate the required fee by taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)`. \n", + "\n", + "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehensive and addresses some additional edge cases, but the code in this notebook highly reflects it." + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", + "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We compute the probability weight of each transaction by raising `feerate` to the power of `alpha` (currently set to `3`). Essentially, alpha represents the amount of bias we want towards higher feerate transactions. " + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "ALPHA = 3.0" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total mempool weight: 64108.589999995806\n" + ] + } + ], + "source": [ + "total_weight = sum(np.array(feerates)**ALPHA)\n", + "print('Total mempool weight: ', total_weight)" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "avg_mass = 2000\n", + "bps = 1\n", + "block_mass_limit = 500_000\n", + "network_mass_rate = bps * block_mass_limit" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inclusion interval: 0.004\n" + ] + } + ], + "source": [ + "print('Inclusion interval: ', avg_mass/network_mass_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "class FeerateBucket:\n", + " def __init__(self, feerate, estimated_seconds):\n", + " self.feerate = feerate\n", + " self.estimated_seconds = estimated_seconds\n", + " \n", + "\n", + "class FeerateEstimations:\n", + " def __init__(self, low_bucket, mid_bucket, normal_bucket, priority_bucket):\n", + " self.low_bucket = low_bucket \n", + " self.mid_bucket = mid_bucket \n", + " self.normal_bucket = normal_bucket\n", + " self.priority_bucket = priority_bucket\n", + " \n", + " def __repr__(self):\n", + " return 'Feerates:\\t{}, {}, {}, {} \\nTimes:\\t\\t{}, {}, {}, {}'.format(\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate, \n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds, \n", + " self.priority_bucket.estimated_seconds)\n", + " def feerates(self):\n", + " return np.array([\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate\n", + " ])\n", + " \n", + " def times(self):\n", + " return np.array([\n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds,\n", + " self.priority_bucket.estimated_seconds\n", + " ])\n", + " \n", + "class FeerateEstimator:\n", + " \"\"\"\n", + " `total_weight`: The total probability weight of all current mempool ready \n", + " transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^ALPHA\n", + " \n", + " 'inclusion_interval': The amortized time between transactions given the current \n", + " transaction masses present in the mempool, i.e., the inverse \n", + " of the transaction inclusion rate. For instance, if the average \n", + " transaction mass is 2500 grams, the block mass limit is 500,000\n", + " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", + " \"\"\"\n", + " def __init__(self, total_weight, inclusion_interval):\n", + " self.total_weight = total_weight\n", + " self.inclusion_interval = inclusion_interval\n", + "\n", + " \"\"\"\n", + " Feerate to time function: f(feerate) = inclusion_interval * (1/p(feerate))\n", + " where p(feerate) = feerate^ALPHA/(total_weight + feerate^ALPHA) represents \n", + " the probability function for drawing `feerate` from the mempool\n", + " in a single trial. The inverse 1/p is the expected number of trials until\n", + " success (with repetition), thus multiplied by inclusion_interval it provides an\n", + " approximation to the overall expected waiting time\n", + " \"\"\"\n", + " def feerate_to_time(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / feerate**ALPHA + c1\n", + "\n", + " \"\"\"\n", + " The inverse function of `feerate_to_time`\n", + " \"\"\"\n", + " def time_to_feerate(self, time):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return ((c1 * c2 / time) / (1 - c1 / time))**(1 / ALPHA)\n", + " \n", + " \"\"\"\n", + " The antiderivative function of \n", + " feerate_to_time excluding the constant shift `+ c1`\n", + " \"\"\"\n", + " def feerate_to_time_antiderivative(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / (-2.0 * feerate**(ALPHA - 1))\n", + " \n", + " \"\"\"\n", + " Returns the feerate value for which the integral area is `frac` of the total area.\n", + " See figures below for illustration\n", + " \"\"\"\n", + " def quantile(self, lower, upper, frac):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " z1 = self.feerate_to_time_antiderivative(lower)\n", + " z2 = self.feerate_to_time_antiderivative(upper)\n", + " z = frac * z2 + (1.0 - frac) * z1\n", + " return ((c1 * c2) / (-2 * z))**(1.0 / (ALPHA - 1.0))\n", + " \n", + " def calc_estimations(self):\n", + " # Choose `high` such that it provides sub-second waiting time\n", + " high = self.time_to_feerate(1.0)\n", + " \n", + " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", + " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", + " \n", + " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66\n", + " # quantile between low and high\n", + " normal = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.66))\n", + " \n", + " # Choose an additional point between normal and low\n", + " mid = max(self.time_to_feerate(1800.0), self.quantile(1.0, high, 0.5))\n", + " \n", + " return FeerateEstimations(\n", + " FeerateBucket(low, self.feerate_to_time(low)),\n", + " FeerateBucket(mid, self.feerate_to_time(mid)),\n", + " FeerateBucket(normal, self.feerate_to_time(normal)),\n", + " FeerateBucket(high, self.feerate_to_time(high)))" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 104, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=0, inclusion_interval=1/100)\n", + "# estimator.quantile(2, 3, 0.5)\n", + "estimator.time_to_feerate(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerate estimation\n", + "\n", + "The figure below illustrates the estimator selection. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHVlJREFUeJzt3XuQnHW95/H3t+8990lmMplkQhIwoIAJlxj14PFwRNR4OcA5LoVb60HLLU4VeNTaU7Wl7JaHtWTX2rPKrq5yFgTFEnVTikcUvLABBY5ASBASyIUk5DKT20xuc0vm0tPf/aOfCZNkkpnMdOeZfvrzqup6nufXTz/9bS6f3zO//j1Pm7sjIiLRFQu7ABERKS0FvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYm4RNgFADQ1NfmiRYvCLkNEpKysW7fuoLs3T7TfjAj6RYsWsXbt2rDLEBEpK2a2azL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgr66Dfsr+Xf/rtZo4eGwq7FBGRGausg37XoX6+/dR22g8fD7sUEZEZa8KgN7MFZvaUmW0ys9fM7PNB+11mtsfMXg4eHx7zmi+Z2TYz22JmHyxV8S11GQAO9AyU6i1ERMreZG6BkAP+wd1fMrNaYJ2ZPRE8d4+7/4+xO5vZpcAtwGXAPOD/mdnF7j5SzMJhTND3KuhFRM5kwjN6d9/n7i8F673AJmD+WV5yA/ATdx909x3ANmBFMYo9VVNNCgMO9AyW4vAiIpFwTmP0ZrYIuBJ4IWj6rJmtN7MHzawxaJsPtI95WQdn7ximLBGP0VCVpFNDNyIiZzTpoDezGuBnwBfcvQe4F7gIuALYB3x9dNdxXu7jHO82M1trZmu7urrOufBR9dmkxuhFRM5iUkFvZkkKIf+wuz8C4O4H3H3E3fPA/bw5PNMBLBjz8jZg76nHdPf73H25uy9vbp7wdspnVJ9Nsl9BLyJyRpOZdWPAA8Amd//GmPbWMbvdBLwarD8K3GJmaTNbDCwB1hSv5JM1VCU1Ri8ichaTmXVzDfBJYIOZvRy03Ql8wsyuoDAssxP4OwB3f83MVgEbKczYuaMUM25GNVSlONw/xFAuTypR1pcFiIiUxIRB7+7PMv64++Nnec3dwN3TqGvSGqqSAHT1DTK/IXs+3lJEpKyU/SlwYxD0+kJWRGR8ZR/0DdkUgKZYioicQfkH/Ykzen0hKyIynrIP+ppMgphp6EZE5EzKPuhjZtSkEzqjFxE5g7IPeoDqdIJO3dhMRGRckQj6qlScfUcV9CIi44lE0FenE7oNgojIGUQi6GvSCfoGcxwbyoVdiojIjBOJoK/NFC7w3avhGxGR00Qj6NOFufR7j+q3Y0VEThWNoD9xRq+gFxE5VSSCvjqdwFDQi4iMJxJBH48ZtZkEe7s1Ri8icqpIBD0UboWgM3oRkdNFJ+hTCfYcUdCLiJwqOkGfSbCvewD3036HXESkokUm6GszSYZG8hzqHwq7FBGRGSVCQa8pliIi44lO0Kd1dayIyHiiE/QZXR0rIjKeyAR9JhkjGTcFvYjIKSIT9GZGXSbJPl00JSJyksgEPUB1Ok7HkWNhlyEiMqNEKuhrM0k6dNGUiMhJIhX0ddkkh/qH9AMkIiJjRCro64OZNzqrFxF5U7SCPlsI+t2HNE4vIjIqUkFfly1cNNWuL2RFRE6IVNBnk3FSiRi7DyvoRURGTRj0ZrbAzJ4ys01m9pqZfT5on2VmT5jZ1mDZGLSbmX3TzLaZ2Xozu6rUH2JMrdRnk7Qr6EVETpjMGX0O+Ad3fxvwLuAOM7sU+CKw2t2XAKuDbYCVwJLgcRtwb9GrPovadEJn9CIiY0wY9O6+z91fCtZ7gU3AfOAG4KFgt4eAG4P1G4AfeMHzQIOZtRa98jOoyybZffiY7ksvIhI4pzF6M1sEXAm8ALS4+z4odAbAnGC3+UD7mJd1BG2nHus2M1trZmu7urrOvfIzqM8mGRjWfelFREZNOujNrAb4GfAFd+85267jtJ12eu3u97n7cndf3tzcPNkyJjQ680bDNyIiBZMKejNLUgj5h939kaD5wOiQTLDsDNo7gAVjXt4G7C1OuRMbvWhKX8iKiBRMZtaNAQ8Am9z9G2OeehS4NVi/FfjFmPa/DWbfvAvoHh3iOR/qsgp6EZGxEpPY5xrgk8AGM3s5aLsT+Bqwysw+A+wG/k3w3OPAh4FtwDHg00WteALJeIwazbwRETlhwqB392cZf9wd4Lpx9nfgjmnWNS11mQS7dBsEEREgYlfGjqqvSvLGwf6wyxARmREiGfSNVSm6egfpH9TtikVEIhn0DVWFL2R36KxeRCSaQd9YlQIU9CIiENGgb8jqjF5EZFQkgz4Rj1GfTSroRUSIaNBD4Z43b3T1hV2GiEjoIhv0DdnCFEvdxVJEKl10g74qSe9AjsO6i6WIVLjIBr1m3oiIFEQ26Efn0usKWRGpdJEN+rpMkriZzuhFpOJFNuhjMaOhKsm2Ts28EZHKFtmgB2isTrFlf2/YZYiIhCrSQT+7OkX74WMMDI+EXYqISGgiHfQfHHmaZ1KfI333bLjncli/KuySRETOu8n8wlRZuqTz17y/6+ukYoOFhu52+OXnCutLbw6vMBGR8yyyZ/Tv2f0dUj54cuPwcVj9lXAKEhEJSWSDvnbwwPhPdHec30JEREIW2aDvTbeM/0R92/ktREQkZJEN+mcvuJ3hWObkxmQWrvtyOAWJiIQksl/GbpmzEoB37vjfNA53kaudR+oDd+mLWBGpOJENeiiE/R+r3scPX9jNPTcu46alGrYRkcoT2aGbUQ1VKeJmbNmvWyGISGWKfNDHY8bsmhSb9/eEXYqISCgiH/QAs2tSvLqnO+wyRERCURFB31yT5mDfEJ29A2GXIiJy3lVG0NemAdi0T3eyFJHKUxlBX1MI+tf2avhGRCrPhEFvZg+aWaeZvTqm7S4z22NmLwePD4957ktmts3MtpjZB0tV+LlIJ+M0ZJNs3KsvZEWk8kzmjP77wIfGab/H3a8IHo8DmNmlwC3AZcFrvmNm8WIVOx2za1K8pqAXkQo0YdC7+9PA4Uke7wbgJ+4+6O47gG3AimnUVzTNNWl2HuynfzAXdikiIufVdMboP2tm64OhncagbT7QPmafjqDtNGZ2m5mtNbO1XV1d0yhjcppr0ziwWT8tKCIVZqpBfy9wEXAFsA/4etBu4+zr4x3A3e9z9+Xuvry5uXmKZUze6Mybjfs0fCMilWVKQe/uB9x9xN3zwP28OTzTASwYs2sbsHd6JRZHTTpBVSrOho6jYZciInJeTSnozax1zOZNwOiMnEeBW8wsbWaLgSXAmumVWBxmxpzaNC+3K+hFpLJMePdKM/sxcC3QZGYdwD8C15rZFRSGZXYCfwfg7q+Z2SpgI5AD7nD3kdKUfu5a6jKs2XGYvsEcNelI37hTROSECdPO3T8xTvMDZ9n/buDu6RRVKnPrMjjw6p5u3nXh7LDLERE5LyriythRLXWFX5x6RcM3IlJBKiros6k4jVVJjdOLSEWpqKCHwjTLPynoRaSCVFzQz63LsL97gM4e3bJYRCpDxQX96Di9hm9EpFJUXNDPqU0TMwW9iFSOigv6RDzGnNoML+6c7H3aRETKW8UFPcC8hgwvtx9lYHjGXMslIlIyFRr0WYZHnPUd+sUpEYm+ig16QMM3IlIRKjLos8k4TTUp1uxQ0ItI9FVk0APMrc+wbtcRRvLj3i5fRCQyKjbo5zdk6RvMsUk/RCIiEVexQa9xehGpFBUb9HWZJPXZJM9tPxR2KSIiJVWxQQ/Q1pjlj9sPkRvJh12KiEjJVHTQXzCrir7BHBv2aD69iERXRQd9W2NhnP7ZrQdDrkREpHQqOuirUgnm1KV5dpuCXkSiq6KDHmBBQxXrdh3h2FAu7FJEREpCQT8rSy7vvKCrZEUkoio+6Oc3ZEnETOP0IhJZFR/0iXiM+Q1ZntzcGXYpIiIlUfFBD7CoqZodB/vZcbA/7FJERIpOQQ9c2FQNwOpNB0KuRESk+BT0QF02SXNNmtWbNHwjItGjoA8snF3Fmp2H6T4+HHYpIiJFpaAPLG6qZiTvPP16V9iliIgUlYI+MLc+Q1UqrnF6EYmcCYPezB40s04ze3VM2ywze8LMtgbLxqDdzOybZrbNzNab2VWlLL6YYmYsml3NE5sOMJgbCbscEZGimcwZ/feBD53S9kVgtbsvAVYH2wArgSXB4zbg3uKUeX4saamhf3CEZ17XxVMiEh0TBr27Pw2cen+AG4CHgvWHgBvHtP/AC54HGsystVjFltqCxiqyyTiPbdgXdikiIkUz1TH6FnffBxAs5wTt84H2Mft1BG2nMbPbzGytma3t6poZX4DGY8bipmqe2HiAgWEN34hINBT7y1gbp83H29Hd73P35e6+vLm5uchlTN3FLTX0DeZ4Rve+EZGImGrQHxgdkgmWo1cadQALxuzXBuydennnX1tjFR9P/ZGrH/lzuKsB7rkc1q8KuywRkSmbatA/CtwarN8K/GJM+98Gs2/eBXSPDvGUi0sP/oavxu5nVu4A4NDdDr/8nMJeRMrWZKZX/hh4DrjEzDrM7DPA14DrzWwrcH2wDfA48AawDbgfuL0kVZfQe3Z/hwyDJzcOH4fVXwmnIBGRaUpMtIO7f+IMT103zr4O3DHdosJUO3iGC6a6O85vISIiRaIrY0/Rm24Z/4n6tvNbiIhIkSjoT/HsBbczHMuc3JjMwnVfDqcgEZFpmnDoptJsmbMSKIzV1wweoNOaaPnYf8WW3hxyZSIiU6OgH8eWOSvZMmclm/b18LuNB3i46p1cE3ZRIiJTpKGbs1gyp4aqVJwfPLcz7FJERKZMQX8WiXiMt7XW8cTGA+w9ejzsckREpkRBP4Gl8+txhx+9sDvsUkREpkRBP4G6bJLFTdX8aM1u3adeRMqSgn4SlrbVc7h/iF+9UlZ3cxARART0k3LBrCqaalL88x+2k8+PezNOEZEZS0E/CWbG1Rc0srWzj6e2dE78AhGRGURBP0lLWmqpzya59/fbwy5FROScKOgnKR4zrljQwNpdR3hx56m/rCgiMnMp6M/BZfPqqErF+dbqrWGXIiIyaQr6c5CMx7jqgkae3nqQNTt0Vi8i5UFBf46WttVTk07wT7/dTOH2+yIiM5uC/hwl4zGWL2rkxZ1HeFo/IC4iZUBBPwWXz6unPpvkv/9ms+bVi8iMp6CfgnjMeOfiWby2t4dH/rQn7HJERM5KQT9Fb51bS2t9hq/9ehN9g7mwyxEROSMF/RSZGe9d0szBviG+9aSmW4rIzKWgn4a59Rkuba3lgWd2sONgf9jliIiMS0E/TX92URPxmPGln63XdEsRmZEU9NNUnU7wnrc08fyOw/zkxfawyxEROY2Cvggum1fHgsYsdz+2if3dA2GXIyJyEgV9EZgZ73vrHAZzI9z58w0awhGRGUVBXyQNVSnefeFsntzcyQ/1+7IiMoMo6IvoigUNLJpdxVd/tZEt+3vDLkdEBFDQF5WZ8f63tZCIG3//45cYGNaPiYtI+KYV9Ga208w2mNnLZrY2aJtlZk+Y2dZg2VicUstDdTrB9W9r4fUDffznf3lV4/UiErpinNH/pbtf4e7Lg+0vAqvdfQmwOtiuKAtnV7Ni8Sx+uq6DHzy3K+xyRKTClWLo5gbgoWD9IeDGErzHjPeuxbO4sKmar/xyI89tPxR2OSJSwaYb9A78zszWmdltQVuLu+8DCJZzpvkeZcnM+MBlLTRUJbn94XW6RYKIhGa6QX+Nu18FrATuMLP3TvaFZnabma01s7VdXV3TLGNmSififGRpK4O5PJ984AU6e3UxlYicf9MKenffGyw7gZ8DK4ADZtYKECw7z/Da+9x9ubsvb25unk4ZM1pjVYqPLZtHZ88gtz64ht6B4bBLEpEKM+WgN7NqM6sdXQc+ALwKPArcGux2K/CL6RZZ7ubWZfjw2+eyZX8v//6htRwb0v3rReT8mc4ZfQvwrJm9AqwBHnP33wBfA643s63A9cF2xVs4u5oPXDqXNTsP86kHX6RfP1YiIudJYqovdPc3gGXjtB8CrptOUVF1ydxaAH67cT+f+t4avv/pFVSnp/yvQERkUnRl7Hl2ydxaPnTZXNbtOsK/vf95DvUNhl2SiEScgj4EF7fU8uG3t/La3h7++jt/ZNchTb0UkdJR0IfkouYa/vqq+XT1DXLjt/+VHU99D+65HO5qKCzXrwq7RBGJCA0Qh6i1PsvHr2pj+OX/S8vv/xlsqPBEdzv88nOF9aU3h1egiESCzuhD1lid4oupVVSNhvyo4eOw+ivhFCUikaKgnwHqhg6M/0R3x/ktREQiSUE/A/SmW8Zt78vM1W2ORWTaFPQzwLMX3M5wLHNS2wBp7uy5iU9970XaDx8LqTIRiQIF/QywZc5KnrjoTnrSc3GMnvRcVi/5T/S85Sae236I6+/5A//nD9sZHsmHXaqIlCHNupkhtsxZyZY5K09qWwZc2FzNH17v4r/9ejOP/GkPd33sMt590exwihSRsqQz+hmuNpPko0vn8dGlrew7epxP3P88n3noRbZ16sfHRWRydEZfJi5qrmHhrCpebj/Ks1sP8sHNz3DzOxZwx19eRFtjVdjlicgMpqAvI4l4jOWLZnHpvDrW7DjMqhfbWbW2nb+5aj63X/sWFjVVh12iiMxACvoyVJVKcO0lc7h6YSPrdh3hkZf28NN1HXx06Tw+fc0irrygMewSRWQGUdCXsdpMkmsvmcM7Fs3ipd1H+O1r+3n0lb0snV/Pp65ZxEeWtpJOxMMuU0RCZjPhgpzly5f72rVrp/TaJzcf4JX27iJXVJ6Gcnk27eth/Z5uDvcP0ViV5KYr2/j41W1cOq8u7PJEpMjMbJ27L59oP53RR0gqEWPZggaWttWz+/AxXt3bw0PP7eTBf93BW+fW8vGr2/irZfOYU5eZ8FgiEh0K+ggyMxbOrmbh7GqOD4/w+v5eNu/v5auPbeLuxzZx1cJGVl4+lw9eNpcFszRjRyTqFPQRl03GWbaggWULGjjcP8S2zj62d/Xx1cc28dXHNnH5vDre97YW/uLiZpa11ZOI69IKkahR0FeQWdUpViyexYrFszh6bIjtXf1s7+rjW09u5Zurt1KbSfDnS5r4i4ub+bOLmmhrzGJmYZctItOkoK9QDVUprl6Y4uqFjQwMj7D78DF2HTrGM68f5PEN+wGYW5dhxeJZvGPxLFYsmsWSOTXEYgp+kXKjoBcyyTgXt9RycUst7s6h/iH2HDnOnqPHeWpzJ4++sheA+mySZW31LG1r4O1t9Sxtq2duXUZn/SIznIJeTmJmNNWkaapJs2xBA+5Oz0COPUePs/focTbv7+XZbQfJB7NyZ1WnWNZWz+Xz61nSUsslLbUsbqomldBYv8hMoaCXszIz6rNJ6rNJLm0tzMXPjeTp6huks2eQA70DbNjTzR9e7zoR/vGYsbipmkuCvxLeMqeGRU1VLJxdTU1a/8mJnG/6v07OWSIeo7U+S2t99kRbbiTPkWPDHOof5HD/EIf6hvjj9oM8vmEfYy/Jm12dYnFTYernotlVLGyqZuGsKlobMjRVp/UdgEgJKOilKBLxGM21aZpr0ye1D4/kOXpsmKPHhjh6fJju48Ps7xng9QO99AzkTto3GTda67PMa8gwryHLvPos8xqytDZkaK3P0FyTprEqpc5A5Bwp6KWkkmfoAKDQCXQH4d83kKN3MEfvwDDth4+xcW8PfYO5E8NBo+JmzK5JnThmc01h2TRm2VidpLEqRUNVUvf6EUFBLyFKxmMnvvgdTz7v9A/l6B3I0TeY49jQCMeGcvQPjtA/mONQ3xAvDR0Zt0MYlU3GaagqBH9jdZKGqhSNwXZ9trBdm0kUHukktZkENcG2OgmJCgW9zFixmFGbSVKbSZ51P3dnYDhf6ASGRhgYHn3kT6wfG8px5NgQg7leBoZHOD40wkS380vFY1Sn49RmktRlT+4IatIJsqk41akEVak42VS8sEwWtt9sG/N8Mq4rjyUUJQt6M/sQ8L+AOPBdd/9aqd5LKpuZkQ3CdLK/puvuDOYKHcFQLs9gLs/QSJ6hXP7N7VyewZHC830DOQ73DzGc8zf3G8kzcqY/Jc4gGTeyyUKt6UScdCJGJhknm4yTTsZIJ2In2gvbwXoiRjo5Zj1x+v7JRIxEzEjGY6TGW0/ESMUL6/GY6fqHMKxfBau/At0dUN8G130Zlt5c8rctSdCbWRz4NnA90AG8aGaPuvvGUryfyLkyMzLJOJnk9IZnRvJOLp9neMQZHsmTC5aFh5MLlsP509ty+UJH0TMwzJFjQ+TdGcl7cExnZKSwHD1+MRmFobNEvNAZJONGIhYjmbBCZxAvdAqjncToMh6LEY9BIhYjHjMSMSMWLE/ffrNTiY95/uT12ATHKNQVixW+n4nFjJgZMYOYjXZYhSm9Y9tPPGKMux43w4LtuBWOMXq8mFGaTnD9Kvjl52D4eGG7u72wDSUP+1Kd0a8Atrn7GwBm9hPgBkBBL5FSCK04pb48wN3JOyc6h9xohzDiJzqbvBc6nrw7+bwz4k4+T7D0U5Zvtp/oYIL3yOed4VyeweGRE/vmvbCfO4UHwb7BMdwhz+n75oNjlqOxHcBJHcaJtjc7hbi9uR4zTnRIxpttP+y9kxY/fvKbDB8vnOGXadDPB9rHbHcA7yzFG82uTrNYv5UqMmONdlLuflrHMdqxjAQdw9jOZ+x+I2M6mRMdDn7K9pj1Me95cnuwzuT2OdPxTnpfd/LB8cZ2hKM1jb6m2Q+O/w+ou6Pk/w5KFfTj/d1zUr9uZrcBtwFccMEFU36j0VvwiojMaPe0FYZrTlXfVvK3LtUUgA5gwZjtNmDv2B3c/T53X+7uy5ubm0tUhojIDHHdlyGZPbktmS20l1ipgv5FYImZLTazFHAL8GiJ3ktEZOZbejN87JtQvwCwwvJj3yzfWTfunjOzzwK/pTC98kF3f60U7yUiUjaW3nxegv1UJZsr4O6PA4+X6vgiIjI5ukxPRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgz9/BvLWdmXcCuKb68CTjD3YIiI+qfUZ+vvOnzhWehu094D5kZEfTTYWZr3X152HWUUtQ/oz5fedPnm/k0dCMiEnEKehGRiItC0N8XdgHnQdQ/oz5fedPnm+HKfoxeRETOLgpn9CIichZlG/Rm9qCZdZrZq2HXUgpmtsDMnjKzTWb2mpl9PuyaisnMMma2xsxeCT7ffwm7plIws7iZ/cnMfhV2LaVgZjvNbIOZvWxma8Oup9jMrMHMfmpmm4P/F98ddk1TUbZDN2b2XqAP+IG7Xx52PcVmZq1Aq7u/ZGa1wDrgRnffGHJpRWFmBlS7e5+ZJYFngc+7+/Mhl1ZUZvYfgOVAnbt/NOx6is3MdgLL3c/0y9flzcweAp5x9+8Gv5ZX5e5Hw67rXJXtGb27Pw0cDruOUnH3fe7+UrDeC2wC5odbVfF4QV+wmQwe5XnWcQZm1gZ8BPhu2LXIuTOzOuC9wAMA7j5UjiEPZRz0lcTMFgFXAi+EW0lxBcMaLwOdwBPuHqnPB/xP4D8C+bALKSEHfmdm68zstrCLKbILgS7ge8Hw23fNrDrsoqZCQT/DmVkN8DPgC+7eE3Y9xeTuI+5+BdAGrDCzyAzBmdlHgU53Xxd2LSV2jbtfBawE7giGVKMiAVwF3OvuVwL9wBfDLWlqFPQzWDB2/TPgYXd/JOx6SiX4c/j3wIdCLqWYrgH+KhjD/gnwPjP7YbglFZ+77w2WncDPgRXhVlRUHUDHmL80f0oh+MuOgn6GCr6sfADY5O7fCLueYjOzZjNrCNazwPuBzeFWVTzu/iV3b3P3RcAtwJPu/u9CLquozKw6mChAMKTxASAys+DcfT/QbmaXBE3XAWU5GSIRdgFTZWY/Bq4FmsysA/hHd38g3KqK6hrgk8CGYBwb4E53fzzEmoqpFXjIzOIUTjhWuXskpyBGWAvw88I5CQngR+7+m3BLKrq/Bx4OZty8AXw65HqmpGynV4qIyORo6EZEJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiFPQi4hE3P8H1DStq24uP4EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1499744606513134, 1.3970589103224236, 1.9124681884207781, 6.361686926992798 \n", + "Times:\t\t168.62498827393395, 94.04820895845543, 36.664092522353194, 1.0000000000000004" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interpolating the original function using two of the points\n", + "\n", + "The code below reverse engineers the original curve using only 2 of the estimated points" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHW9JREFUeJzt3Xl0XGeZ5/HvU6WSVLKszVpiSbblJMZZyGLjOAHTaUgAk8CQhKUn0ECGgTZNBw6cYTJDaOYA5wwzORMCPX2gM52QNMlAJxMghEAHTAiBEAix5Wze4tiJN8mbbEebtZbqmT/qypZt2ZalKt/Srd/nHJ1776t7q57K8ruv3nrvvebuiIhIdMXCLkBERHJLQS8iEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQirijsAgBqa2u9paUl7DJERKaVNWvW7Hf3ulPtlxdB39LSQmtra9hliIhMK2a2fSL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYib1kG/aU8Pt698mc6+obBLERHJW9M66LcfOMR3n3yVnQf7wy5FRCRvTeugb6goBWBv90DIlYiI5K9TBr2ZzTGzJ81so5mtN7PPB+1fM7N2M3sh+Ll2zDG3mtkWM9tkZstzVfzhoO9R0IuInMhE7nWTAr7o7s+Z2UxgjZk9Hvzu2+7+zbE7m9kFwI3AhUAj8Bsze4O7j2SzcIDa8mLMYG/3YLZfWkQkMk7Zo3f33e7+XLDeA2wEmk5yyHXAg+4+6O5bgS3A0mwUe6yieIza8hL2aehGROSETmuM3sxagEXAs0HTZ83sJTO718yqg7YmYOeYw9oY58RgZivMrNXMWjs6Ok678FENFSUaoxcROYkJB72ZlQM/Ab7g7t3AncA5wKXAbuCO0V3HOdyPa3C/y92XuPuSurpT3k75hBpmlmroRkTkJCYU9GaWIBPyP3T3hwHcfa+7j7h7GribI8MzbcCcMYc3A7uyV/LR6itK2acvY0VETmgis24MuAfY6O7fGtM+e8xuNwDrgvVHgRvNrMTM5gMLgFXZK/loDRUl7O8dYiiVztVbiIhMaxOZdbMM+Biw1sxeCNq+DHzYzC4lMyyzDfg0gLuvN7OHgA1kZuzcnIsZN6NGp1h29A7SVJXM1duIiExbpwx6d3+a8cfdHzvJMd8AvjGFuibsrDEXTSnoRUSON62vjAWorygB0BRLEZETmPZBf+Q2CJp5IyIynmkf9DVlxRTFTHPpRUROYNoHfSxm1M8sUY9eROQEpn3Qg+bSi4icTCSCXrdBEBE5sYgEfSl7uhT0IiLjiUzQdw+k6BtKhV2KiEjeiUTQj14otatTvXoRkWNFIugbDwe9nh0rInKsiAR95qIpBb2IyPEiEfQNFaWYwS59ISsicpxIBH0iHqNhZql69CIi44hE0ENm+EZBLyJyvMgE/eyqJLs1dCMicpzIBH1TVZL2zn7cj3s8rYhIQYtM0DdWljKUSnPg0FDYpYiI5JXoBH0wl363LpoSETlK5IK+XV/IiogcJXJBr5k3IiJHi0zQV5clKE3E2N2loBcRGSsyQW9mNFYmdWMzEZFjRCboITN806ahGxGRo0Qq6JuqkrS/rqAXERkrUkE/d1YZ+3sH6R8aCbsUEZG8Eamgn1NTBsDO1/tCrkREJH9EK+irM1MsdxxQ0IuIjIpU0M9Vj15E5DiRCvqaGcWUFcfZcVBBLyIy6pRBb2ZzzOxJM9toZuvN7PNBe42ZPW5mm4NlddBuZvaPZrbFzF4ys8W5/hBjamVuTRk7D2rmjYjIqIn06FPAF939fOAK4GYzuwD4EvCEuy8Angi2Aa4BFgQ/K4A7s171STRXl7FTPXoRkcNOGfTuvtvdnwvWe4CNQBNwHXBfsNt9wPXB+nXA/Z7xZ6DKzGZnvfITmFtTxo6DfbovvYhI4LTG6M2sBVgEPAs0uPtuyJwMgPpgtyZg55jD2oK2Y19rhZm1mllrR0fH6Vd+AnNrkvQPj+i+9CIigQkHvZmVAz8BvuDu3SfbdZy247rX7n6Xuy9x9yV1dXUTLeOURufS6wtZEZGMCQW9mSXIhPwP3f3hoHnv6JBMsNwXtLcBc8Yc3gzsyk65p3Z4iqWCXkQEmNisGwPuATa6+7fG/OpR4KZg/SbgZ2PaPx7MvrkC6Bod4jkTmqsV9CIiYxVNYJ9lwMeAtWb2QtD2ZeA24CEz+ySwA/hQ8LvHgGuBLUAf8ImsVnwKyeI4dTNLNMVSRCRwyqB396cZf9wd4Opx9nfg5inWNSVza8rYduBQmCWIiOSNSF0ZO2p+7QwFvYhIILJBv7d7kEODqbBLEREJXSSD/uzaGQBs3a9evYhIJIN+fp2CXkRkVCSDvmWWgl5EZFQkg740EaepKqmgFxEhokEPmS9kX1PQi4hEO+i3dvTqLpYiUvAiHfTdAykO6i6WIlLgohv0mnkjIgJEOOhH59JrnF5ECl1kg76pKkkiburRi0jBi2zQF8VjzJs1gy37esMuRUQkVJENeoAF9eUKehEpeNEO+oaZbD9wiIHhkbBLEREJTaSDvqtviLTD+f/tVyy77bc88nx72CWJiJxxkQ36R55v58HVO4HMk8nbO/u59eG1CnsRKTiRDfrbV25iMJU+qq1/eITbV24KqSIRkXBENuh3dY7/zNgTtYuIRFVkg76xKnla7SIiURXZoL9l+UKSifhRbclEnFuWLwypIhGRcBSFXUCuXL+oCYCv/3w9r/cNUzezhL+/9vzD7SIihSKyPXrIhP1Dn34zAF++9jyFvIgUpEgHPUBL7QwSceOVvbpCVkQKU+SDPhGPcU5dORt3d4ddiohIKCIf9AAXNFawYZeCXkQKU0EE/YWNlezrGaSjZzDsUkREzriCCPoLZlcAsEHDNyJSgAor6DV8IyIF6JRBb2b3mtk+M1s3pu1rZtZuZi8EP9eO+d2tZrbFzDaZ2fJcFX46KssSNFcn1aMXkYI0kR7994F3j9P+bXe/NPh5DMDMLgBuBC4MjvknM4uPc+wZd8HsCtbv6gq7DBGRM+6UQe/uTwEHJ/h61wEPuvugu28FtgBLp1Bf1lzQWMHW/YfoG0qFXYqIyBk1lTH6z5rZS8HQTnXQ1gTsHLNPW9B2HDNbYWatZtba0dExhTIm5sLGStzh5T09OX8vEZF8MtmgvxM4B7gU2A3cEbTbOPv6eC/g7ne5+xJ3X1JXVzfJMibugsbMF7Lr9YWsiBSYSQW9u+919xF3TwN3c2R4pg2YM2bXZmDX1ErMjsbKUqrLEqxr0zi9iBSWSQW9mc0es3kDMDoj51HgRjMrMbP5wAJg1dRKzA4z45I5VbzY1hl2KSIiZ9Qpb1NsZg8AbwNqzawN+CrwNjO7lMywzDbg0wDuvt7MHgI2ACngZncfyU3pp++S5iqeemUzhwZTzCiJ7B2aRUSOcsq0c/cPj9N8z0n2/wbwjakUlSuXzqki7bC2vYsrzp4VdjkiImdEQVwZO+ri5koAXtyp4RsRKRwFFfSzykuYW1OmcXoRKSgFFfRA5gvZnZp5IyKFo/CCvrmS9s5+9vUMhF2KiMgZUXBBv2huFYB69SJSMAou6C9srKQoZjy/4/WwSxEROSMKLuhLE3EubKygdZuCXkQKQ8EFPcBlLTW80NbJYCpvruUSEcmZwgz6+TUMpdK8pPveiEgBKMygb6kBYNXWid5mX0Rk+irIoK+ZUcy59eWs3qagF5HoK8igh0yvfs221xlJj3u7fBGRyCjYoF86v5qewRQv79GDSEQk2go26EfH6VdrnF5EIq5gg765uoymqiTPvHYg7FJERHKqYIMe4K3n1vKnVw9onF5EIq2gg37Zglp6BlKsbdd8ehGJrsIO+nMyT5l6enNHyJWIiOROQQf9rPISLphdwdNb9oddiohIzhR00AP8xYJantveSd9QKuxSRERyouCDftm5tQyNpHU7BBGJrIIP+staaiiOx3h6s4ZvRCSaCj7ok8VxLj+7hic37Qu7FBGRnCj4oAe46rx6Xu04xLb9h8IuRUQk6xT0wNXnNQDwxMvq1YtI9CjogbmzylhQX84TG/eGXYqISNYp6ANXn9/Aqq0H6R4YDrsUEZGsUtAH3nF+Pam089QrukpWRKJFQR9YNLea6rIET2zUOL2IRMspg97M7jWzfWa2bkxbjZk9bmabg2V10G5m9o9mtsXMXjKzxbksPpviMeOq8xr4zca9DKZGwi5HRCRrJtKj/z7w7mPavgQ84e4LgCeCbYBrgAXBzwrgzuyUeWa89+LZ9Ayk+KPufSMiEXLKoHf3p4Bj7w9wHXBfsH4fcP2Y9vs9489AlZnNzlaxubbs3FoqSov4xUu7wy5FRCRrJjtG3+DuuwGCZX3Q3gTsHLNfW9B2HDNbYWatZtba0ZEfX4AWF8VYfuFZPL5ewzciEh3Z/jLWxmkb9/FN7n6Xuy9x9yV1dXVZLmPyrr14Nj2DKf7wioZvRCQaiiZ53F4zm+3uu4OhmdGpKm3AnDH7NQO7plLgmbbsnFqSiRife+B5BoZHaKxKcsvyhVy/aNw/TERE8t5ke/SPAjcF6zcBPxvT/vFg9s0VQNfoEM908dja3QyNOP3DIzjQ3tnPrQ+v5ZHn28MuTURkUiYyvfIB4BlgoZm1mdkngduAd5rZZuCdwTbAY8BrwBbgbuDvclJ1Dt2+ctNxDwvvHx7h9pWbQqpIRGRqTjl04+4fPsGvrh5nXwdunmpRYdrV2X9a7SIi+U5Xxh6jsSp5Wu0iIvlOQX+MW5YvJJmIH9WWTMS5ZfnCkCoSEZmayc66iazR2TW3r9xEe2c/8ZjxP254o2bdiMi0pR79OK5f1MQfv3QVd3zoEkbSTkNladgliYhMmoL+JN5z8WyqyhL832e2h12KiMikKehPojQR598vmcOvN+xld5dm3YjI9KSgP4WPXjGPtDsPPLsj7FJERCZFQX8Kc2rKuGphPf+6aidDqXTY5YiInDYF/QR87M3z2N87yL+tnVa37RERART0E3LlgjoW1Jfzz79/jczFvyIi04eCfgJiMeNv//IcXt7Tw+825ce980VEJkpBP0Hvu7SRxspS7vz9q2GXIiJyWhT0E5SIx/jUX5zNqq0HWbP99bDLERGZMAX9abhx6RyqyhJ857ebwy5FRGTCFPSnoay4iBVXns2TmzpYs/3Y56WLiOQnBf1p+g9vaaG2vIT/9atNmoEjItOCgv40lRUX8dm3n8OzWw/y9BY9QFxE8p+CfhI+fPlcmqqSfHPlJtJp9epFJL8p6CehpCjOF96xgBfbuvjZi3pouIjkNwX9JH1gcTMXN1dy2y9f5tBgKuxyREROSEE/SbGY8dV/dyF7uwf5p99tCbscEZETUtBPwZvmVXPDoibufmor2w8cCrscEZFxKein6EvXnEdxUYy//+k6TbcUkbykoJ+ihopS/us15/H0lv38aE1b2OWIiBxHQZ8Ff710Lktbavjvv9jAvu6BsMsRETmKgj4LYjHjtg9cxEAqzVce0RCOiOQXBX2WnF1Xzn9+1xv49Ya9PLh6Z9jliIgcpqDPok+99Wzeem4tX//5erbs6wm7HBERQEGfVbGY8a2/uoSy4iI+98ALDAyPhF2SiMjUgt7MtpnZWjN7wcxag7YaM3vczDYHy+rslDo91FeU8s0PXczG3d18/efrwy5HRCQrPfq3u/ul7r4k2P4S8IS7LwCeCLYLylXnNfB3bzuHB1bt5IfPbg+7HBEpcLkYurkOuC9Yvw+4Pgfvkfe++K6FvH1hHV97dD2rt+khJSISnqkGvQO/NrM1ZrYiaGtw990AwbJ+iu8xLcVjxj/cuIjm6jI+84M17DjQF3ZJIlKgphr0y9x9MXANcLOZXTnRA81shZm1mllrR0fHFMvIT5XJBHd/fAmptPPxe59lf+9g2CWJSAGaUtC7+65guQ/4KbAU2GtmswGC5b4THHuXuy9x9yV1dXVTKSOvnVtfzj03Xcae7gH+4/dX65bGInLGTTrozWyGmc0cXQfeBawDHgVuCna7CfjZVIuc7t40r5rvfmQx63d18zf3t9I/pGmXInLmTKVH3wA8bWYvAquAf3P3XwG3Ae80s83AO4Ptgnf1+Q3c/sGLeea1A3zyvtUKexE5Y4ome6C7vwZcMk77AeDqqRQVVe9f3AzAF3/0Ip+8bzX33HQZyeJ4yFWJSNTpytgz7P2Lm/nWX13Cn187wEfveZbXDw2FXZKIRJyCPgQ3LGrmOx9ZzNr2Lj7wf/7EzoOaeikiuTPpoRuZmmsvmk1teQmfum8177/zT3z8ink8uHonuzr7aaxKcsvyhVy/qCnsMkUkAtSjD9HS+TX85DNvYTiV5o7HX6G9sx8H2jv7ufXhtTzyfHvYJYpIBCjoQ7agYSal43wh2z88wu0rN4VQkYhEjYI+D+ztGv/xg7s6+89wJSISRQr6PNBYlRy3vTKZ0GMJRWTKFPR54JblC0kmjh6+iRl09g/zN/e3srtLPXsRmTwFfR64flET//P9F9FUlcSApqokd3zwEr7ynvN5est+3nHH7/mXP25lJK3evYicPsuHoYElS5Z4a2tr2GXkpZ0H+/jKI+v4/SsdXNRUydfedwFvmlcTdlkikgfMbM2Yhz6dkHr0eW5OTRnf/8RlfOcji9jbPcAH7nyGz/xgDdv2Hwq7NBGZJnTB1DRgZrz34kauOq+eu5/ayj8/9Sq/2biXv758Hn/7l+dwVmVp2CWKSB7T0M00tK97gG//5hUeam0jbsaHljTzmbedQ3N1WdilicgZNNGhGwX9NLbjQB93/v5VfrxmJ+5w3aVNfGJZC29sqgy7NBE5AxT0BWRXZz93PfUa/2/1TvqHR1gyr5qb3tLCu994Fom4voYRiSoFfQHq6h/mR607uf+Z7ew42Ef9zBJuWNzEBxc3s6BhZtjliUiWKegL2Eja+d2mfTywagdPbupgJO1cMqeKDy5u4j0XN1IzozjsEkUkCxT0AkBHzyA/e6GdH69p4+U9PcQMLp8/i2suOovlF55FQ4Vm7IhMVwp6OYq7s2F3N79cu4dfrtvNqx2ZefhvmlfNVefVc+WCOi5srCAWs5ArFZGJUtDLSW3e28Ov1u1h5YY9rGvvBmDWjGL+YkEtV76hjmXn1qq3L5LnFPQyYR09g/xhcwdPvdLBU5v3czB4ju3cmjKWzq9haUsNl82voWVWGWbq8YvkCwW9TEo67azf1c2zWw+wautBWre/fjj4a8tLuHROFRc3V3JRcyUXNVVSW14ScsUihUtBL1nh7rza0cuqra/Tuu0gL7Z18tr+Q4z+Z9NYWcpFzZVc2FjJGxpmsvCsmcytKSOusX6RnJto0OteN3JSZsa59TM5t34mH7l8LgA9A8Os39XNuvYuXmrr4qW2Tlau33v4mJKiGOfUlbPwrJksaChnQf1M5teW0VxdRmni+McmikhuKejltM0sTXDF2bO44uxZh9sODabYsq+XTXt72Ly3h017e3nm1QP8dMwDzs2gsTLJvFllzJs1g5ZgObemjKaqJBXJIn0HIJIDCnrJihklRVwyp4pL5lQd1d7VP8yrHb3sONDHtgOH2B4sV67fc3js//BrFMeZXZWksSpJY2UpjVVJZleW0lSV5KzKUupmllBeopOByOlS0EtOVSYTLJ5bzeK51cf9rqt/mB0H+thxsI/dXf20d/azu3OAXV39bNjVzf7eweOOKU3EqC0voW5mCXXB8vB2sF5dlqC6rJiKZELfFYigoJcQVSYTmdk7zePfbXNgeIQ9XZng39M1wP7eQTp6BtnfO0RHzyDbD/QdNSvoWGaZ96guKw6WmfWqsmKqyxJUzSimKpmgvLSIitIiZpYmKC8pYmZpETOKi3TxmESGgl7yVmkiTkvtDFpqZ5x0v+GRNAcPZcK/o3eQzr4hXj80nFn2DfN63xBd/cN09A7yyt5eOvuGODQ0ctLXNIPy4iLKSzPBP/YkMHoiKCuOkzy8jFMW/CQTRUfWi+OUBfuUFMU07CShyFnQm9m7gf8NxIHvufttuXovKWyJeIyGitLTupJ3MDVCV98wXf3DdA+k6B1M0TMwTO9Aip6BzHrPYGa9dyBFz2DmhLHzYB/dAykODaboHz75yeJYMYNk4sjJoTQRo6QocwIoGbteFKM0Mdp+pK2kKB7sd2Tf0f2Ki2Ik4jGK4kZxfPz1RLCu4azwPPJ8O7ev3MSuzn4aq5Lcsnwh1y9qyvn75iTozSwOfBd4J9AGrDazR919Qy7eT+R0lRTFqa+IUz+F2zyk085AaoS+oRH6hzLLvqHUkfXhEfqHUkH7kX36hzNtQ6k0A8MjDKbSDAyn6eofZnA4zWAqzWAq0z44nGYgNUI2L3eJGYdDPzHmBDC6XhSPURw3io75fVHMKIob8VhmPWZGUcyIx4NlzIjb2O3YkfbYkX2KYkYsNv4+R+8XIxaDoljm5BQzgqVhY9ZjRrDMHGfB9lHrwe8txpH1Y14j139tPfJ8O7c+vPZwB6G9s59bH14LkPOwz1WPfimwxd1fAzCzB4HrAAW9REYsZsGwTG5HQN2dVNoPnxQyJ4Dj11PpNEMpZ3gksz6ccoZG0qRG0gyPOMNB2/BI+qj1Y48bSnnm+JHMPr2pFEOpNCNpP/yTOrxMM5KGkXT6cNvYfaYTO3zCOHLiiFnm3/PoiWP0hBAfc3IYbY+ZBfuCceQkE4tltl/e083wyNH/TPqHR7h95aZpG/RNwM4x223A5Tl6L5FIM7PDPevp9PgYdyftkEqnSacJTgo+7gnh2BPF4RPJiJP2zEnIHdLBa46kHXdnJNh2z+yfHt1n7Pp422Ne47j18Y457vUy6yPuh+vKvP+R7cxy9J+DHxfyo3Z19uf830Wugn68v4GO+pRmtgJYATB37twclSEiYcn0fCEeG70aurCvil52229pHyfUG6uSOX/vXD1QtA2YM2a7Gdg1dgd3v8vdl7j7krq6uhyVISKSH25ZvpDkMbcASSbi3LJ8Yc7fO1c9+tXAAjObD7QDNwIfydF7iYjkvdFx+MjMunH3lJl9FlhJ5u+1e919fS7eS0Rkurh+UdMZCfZj5Wy6gLs/BjyWq9cXEZGJydUYvYiI5AkFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4syzef/TyRZh1gFsn+ThtcD+LJaTj6L+GfX5pjd9vvDMc/dT3kMmL4J+Ksys1d2XhF1HLkX9M+rzTW/6fPlPQzciIhGnoBcRibgoBP1dYRdwBkT9M+rzTW/6fHlu2o/Ri4jIyUWhRy8iIicxbYPezO41s31mti7sWnLBzOaY2ZNmttHM1pvZ58OuKZvMrNTMVpnZi8Hn+3rYNeWCmcXN7Hkz+0XYteSCmW0zs7Vm9oKZtYZdT7aZWZWZ/djMXg7+X3xz2DVNxrQdujGzK4Fe4H53f2PY9WSbmc0GZrv7c2Y2E1gDXO/uG0IuLSvMzIAZ7t5rZgngaeDz7v7nkEvLKjP7T8ASoMLd3xt2PdlmZtuAJe6er/PMp8TM7gP+4O7fM7NioMzdO8Ou63RN2x69uz8FHAy7jlxx993u/lyw3gNsBM78o2lyxDN6g81E8DM9ex0nYGbNwHuA74Vdi5w+M6sArgTuAXD3oekY8jCNg76QmFkLsAh4NtxKsisY1ngB2Ac87u6R+nzAPwD/BUiHXUgOOfBrM1tjZivCLibLzgY6gH8Jht++Z2Yzwi5qMhT0ec7MyoGfAF9w9+6w68kmdx9x90uBZmCpmUVmCM7M3gvsc/c1YdeSY8vcfTFwDXBzMKQaFUXAYuBOd18EHAK+FG5Jk6Ogz2PB2PVPgB+6+8Nh15MrwZ/DvwPeHXIp2bQMeF8whv0gcJWZ/SDckrLP3XcFy33AT4Gl4VaUVW1A25i/NH9MJvinHQV9ngq+rLwH2Oju3wq7nmwzszozqwrWk8A7gJfDrSp73P1Wd2929xbgRuC37v7RkMvKKjObEUwUIBjSeBcQmVlw7r4H2GlmC4Omq4FpORmiKOwCJsvMHgDeBtSaWRvwVXe/J9yqsmoZ8DFgbTCODfBld38sxJqyaTZwn5nFyXQ4HnL3SE5BjLAG4KeZPglFwL+6+6/CLSnrPgf8MJhx8xrwiZDrmZRpO71SREQmRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P7IeihVMYkBBAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(array([168.62498827, 94.03830275, 36.64656385, 0.97773399]),\n", + " array([168.62498827, 94.04820896, 36.66409252, 1. ]))" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x1, x2 = pred.low_bucket.feerate**ALPHA, pred.normal_bucket.feerate**ALPHA\n", + "y1, y2 = pred.low_bucket.estimated_seconds, pred.normal_bucket.estimated_seconds\n", + "b2 = (y1 - y2*x2/x1) / (1 - x1/x2)\n", + "b1 = (y1 - b2) * x1\n", + "def p(ff):\n", + " return b1/ff**ALPHA + b2\n", + "\n", + "plt.figure()\n", + "plt.plot(x, p(x))\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "p(pred.feerates()), pred.times()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Challenge: outliers\n", + "\n", + "The segment below illustrates a challenge in the current approach. It is sufficient to add a single outlier \n", + "to the total weight (with `feerate=100`), and the `feerate_to_time` function is notably influenced. In truth, this tx should not affect our prediction because it only captures the first slot of each block, however because we sample with repetition it has a significant impact on the function. The following figure shows the `feerate_to_time` function with such an outlier " + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG1xJREFUeJzt3X1wHPWd5/H3t+dJD7bkJ/lRfiIxEEwIJF7wHrm7HA6PSWGqNslxt0tce9RRdWGX7N3ebUJSBXck2SN3uZBNEXJFYgdnNxXiImQhVLjgNbAbskAwT+bBGBubYPlRfpJlW5Ilzff+6JY8lmakkZGmx9OfV5Vqun/9m57vyJY++vWve9rcHRERSZ4g7gJERCQeCgARkYRSAIiIJJQCQEQkoRQAIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUOm4CxjJjBkzfNGiRXGXISJyVnnxxRcPuHvLaP2qOgAWLVrExo0b4y5DROSsYma/L6efDgGJiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklA1GQB7Orr49hNb2N5+LO5SRESqVk0GwIHOk3z3yW2803487lJERKpWTQZAXSZ8W929/TFXIiJSvWo0AFIAdCkARERKqskAqM+GAaARgIhIaTUZAAMjAAWAiEhptRkA6fBtdZ3Mx1yJiEj1qskASKcCMinTHICIyAhqMgAAcumUDgGJiIyg7AAws5SZvWxmj0Xri83seTPbamY/M7Ns1J6L1rdF2xcV7OP2qH2LmV093m+mUC4dKABEREYwlhHAF4HNBevfBO5x9yXAYeDmqP1m4LC7fxC4J+qHmV0A3AgsBa4B7jOz1PsrvzQFgIjIyMoKADNrBT4F/DBaN+AK4KGoy1rghmh5ZbROtH1F1H8l8KC797j7DmAbcOl4vIlisulAcwAiIiModwTwHeCvgIHTaqYDR9y9L1pvA+ZFy/OAnQDR9o6o/2B7keeMuzAAdBaQiEgpowaAmX0a2O/uLxY2F+nqo2wb6TmFr3eLmW00s43t7e2jlVdSNqVDQCIiIylnBHA5cL2ZvQs8SHjo5zvAFDNLR31agd3RchswHyDa3gwcKmwv8pxB7n6/uy9z92UtLS1jfkMDMumArpMKABGRUkYNAHe/3d1b3X0R4STuk+7+x8BTwGeibquAR6LlR6N1ou1PurtH7TdGZwktBpYAvxu3dzJETnMAIiIjSo/epaQvAQ+a2deBl4HVUftq4G/NbBvhX/43Arj7G2a2DngT6ANudfcJ+w2tQ0AiIiMbUwC4+9PA09HydoqcxePu3cBnSzz/G8A3xlrkmcjpEJCIyIhq9krgrK4EFhEZUQ0HQEB3n04DFREppWYDIJcO6M87vf0KARGRYmo6AEB3BRMRKaVmAyCb1n2BRURGUrMBMDAC6NZNYUREiqrZAMimdWN4EZGR1HAA6BCQiMhIajYANAksIjIyBYCISELVbABkU+Fb61EAiIgUVbMBoBGAiMjIajYABiaBT+gD4UREiqrZAKjLRKeBKgBERIqq+QA43qMAEBEppmYDIBUY6cA40ds3emcRkQSq2QCAcB7ghEYAIiJF1XYApAKOn9QIQESkmJoOgExKIwARkVJqOgDSKdMIQESkhJoPAF0HICJSXE0HQCYION6jEYCISDG1HQBpBYCISCm1HQAp47gOAYmIFFXTAZBNBZzQJLCISFE1HQCZVEB3b5583uMuRUSk6tR8AIA+ElpEpJgaDwAD0LUAIiJF1HQADNwVTFcDi4gMV9MBkIluCqMRgIjIcLUdACndFUxEpJQaD4BoDkAXg4mIDFPjAaARgIhIKTUdAAOTwBoBiIgMV9MBoBGAiEhpNR4Aug5ARKSUmg6AVGAEpkNAIiLFjBoAZlZnZr8zs1fN7A0z+x9R+2Ize97MtprZz8wsG7XnovVt0fZFBfu6PWrfYmZXT9SbKng9cukUx7oVACIiQ5UzAugBrnD3jwAXA9eY2XLgm8A97r4EOAzcHPW/GTjs7h8E7on6YWYXADcCS4FrgPvMLDWeb6aYXDqgUwEgIjLMqAHgoWPRaib6cuAK4KGofS1wQ7S8Mlon2r7CzCxqf9Dde9x9B7ANuHRc3sUIMumATh0CEhEZpqw5ADNLmdkrwH5gPfAOcMTdB36ztgHzouV5wE6AaHsHML2wvchzJkw2FdDZ3TvRLyMictYpKwDcvd/dLwZaCf9q/1CxbtGjldhWqv00ZnaLmW00s43t7e3llDeibDrgaJdGACIiQ43pLCB3PwI8DSwHpphZOtrUCuyOltuA+QDR9mbgUGF7kecUvsb97r7M3Ze1tLSMpbyisqmAzh6NAEREhirnLKAWM5sSLdcDnwQ2A08Bn4m6rQIeiZYfjdaJtj/p7h613xidJbQYWAL8brzeSCnZdKCzgEREikiP3oU5wNrojJ0AWOfuj5nZm8CDZvZ14GVgddR/NfC3ZraN8C//GwHc/Q0zWwe8CfQBt7r7hF+iO3AWkLsTzkWLiAiUEQDuvgm4pEj7doqcxePu3cBnS+zrG8A3xl7mmcumA/ryTk9fnrrMhJ91KiJy1qjpK4EhDABA1wKIiAxR8wGQGwwATQSLiBSq+QD4w+NP8kz2NhZ/rxXuuRA2rYu7JBGRqlDOJPBZ67z9j/PJfd8iG/SEDR074Ze3hcsXfS6+wkREqkBNjwA+/t59ZL3n9MbeLthwVzwFiYhUkZoOgMk9+4pv6GirbCEiIlWopgOgMzer+Ibm1soWIiJShWo6AJ5Z8AV6g7rTGzP1sOKOeAoSEakiNT0JvGXmtQB85O3vMtcOYs2t4S9/TQCLiNR2AEAYAl/aej7XXTiHb37morjLERGpGjV9CGiAPhFURGS4RARALh3Q0aUAEBEplIwAyKQ4fFwBICJSKBEBUJcOOHLiZNxliIhUlUQEQC6T0iEgEZEhEhEAdZmA4yf76e3Px12KiEjVSEYApMMbwWgUICJySjICILoT2JETCgARkQEJCYDwbXZ0aSJYRGRAIgIgpxGAiMgwiQiAuui2kAoAEZFTkhEAAyMATQKLiAxKRADk0gEGdOhiMBGRQYkIADOjLpPSCEBEpEAiAgCgPpPSHICISIHEBEAuE2gEICJSIDEBkE0FHDmuOQARkQGJCYBcJuCQJoFFRAYlJgAasmkOawQgIjIoMQFQn01x/GQ/3b39cZciIlIVEhMADdHFYAc1ChARAZIUANkoAI71xFyJiEh1SEwA1Gc1AhARKZScABg4BHRMASAiAgkKgIZsGtAhIBGRAYkJgEzKSAemQ0AiIpHEBICZ0ZhLc0AjABERoIwAMLP5ZvaUmW02szfM7ItR+zQzW29mW6PHqVG7mdl3zWybmW0ys48W7GtV1H+rma2auLdVXH0mpTkAEZFIOSOAPuAv3f1DwHLgVjO7APgysMHdlwAbonWAa4El0dctwPchDAzgTuAy4FLgzoHQqJS6TKARgIhIZNQAcPc97v5StNwJbAbmASuBtVG3tcAN0fJK4Mceeg6YYmZzgKuB9e5+yN0PA+uBa8b13YyiPptSAIiIRMY0B2Bmi4BLgOeBWe6+B8KQAGZG3eYBOwue1ha1lWof+hq3mNlGM9vY3t4+lvJG1ZBJc+j4Sdx9XPcrInI2KjsAzGwS8HPgL9z96Ehdi7T5CO2nN7jf7+7L3H1ZS0tLueWVpT6borff6ezpG9f9ioicjcoKADPLEP7y/4m7Pxw174sO7RA97o/a24D5BU9vBXaP0F4xjbnwYrD9R7sr+bIiIlWpnLOADFgNbHb3bxdsehQYOJNnFfBIQfvno7OBlgMd0SGiXwNXmdnUaPL3qqitYiblwovB9h3VPICISLqMPpcDNwGvmdkrUdtXgLuBdWZ2M/Ae8Nlo26+A64BtwAngTwHc/ZCZfQ14Iep3l7sfGpd3UabGKAD2dmgEICIyagC4+zMUP34PsKJIfwduLbGvNcCasRQ4ngZHAJ0KABGRxFwJDJBJBdRlAvZpBCAikqwAgHAUoDkAEZEEBkBDNs1enQUkIpK8AGjMpRQAIiIkMQCyado7e8jndTWwiCRb4gJgUi5Nf951XwARSbzkBUDdwMVgOgwkIsmWuABozOpiMBERSGAATI5GALuOdMVciYhIvBIXAA3ZFOnAaDt8Iu5SRERilbgAMDOa6zO0HdYIQESSLXEBAOFE8E6NAEQk4RIZAE11GdoOaQQgIsmW0ABIc6Srl2O6M5iIJFgiA2BF7z/yTPY2Gv/nDLjnQti0Lu6SREQqrpwbwtSU8/Y/zifb/w/ZIPpE0I6d8MvbwuWLPhdfYSIiFZa4EcDH37uPrA/5OOjeLthwVzwFiYjEJHEBMLlnX/ENHW2VLUREJGaJC4DO3KziG5pbK1uIiEjMEhcAzyz4Ar1B3emNmXpYcUc8BYmIxCRxk8BbZl4LwLJ37mVGfzs2ZR624k5NAItI4iQuACAMgV/0Xc76zft46qZPsHhGY9wliYhUXOIOAQ2Y0pABYMeBYzFXIiISj8QGwNTGLADb24/HXImISDwSGwD1mRT1mRTbDygARCSZEhsAEB4G2qERgIgkVOIDYOv+zrjLEBGJRaIDYMakHAeOneTgsZ7RO4uI1JjEBwDAlr0aBYhI8iQ8AMIzgTYrAEQkgRIdAA3ZNJNyad7aczTuUkREKi7RAQAwrTHLZgWAiCRQ4gNgxqQsW/cfo68/H3cpIiIVpQCYlKOnL8+7B3U9gIgkS+IDoGVyeCbQpraOmCsREamsxAfAtMYs2XTAKzuPxF2KiEhFJT4AAjNmTc7x8nsKABFJllEDwMzWmNl+M3u9oG2ama03s63R49So3czsu2a2zcw2mdlHC56zKuq/1cxWTczbOTMzm+rYvOco3b39cZciIlIx5YwAHgCuGdL2ZWCDuy8BNkTrANcCS6KvW4DvQxgYwJ3AZcClwJ0DoVENZjfV0Zd33tit00FFJDlGDQB3/yfg0JDmlcDaaHktcENB+4899BwwxczmAFcD6939kLsfBtYzPFRiM7s5vEfwq5oHEJEEOdM5gFnuvgcgepwZtc8Ddhb0a4vaSrUPY2a3mNlGM9vY3t5+huWNzaRcmqa6NC/+/nBFXk9EpBqM9ySwFWnzEdqHN7rf7+7L3H1ZS0vLuBY3krlT6vnndw7gXrQsEZGac6YBsC86tEP0uD9qbwPmF/RrBXaP0F41WqfWc/hEL2/v0z2CRSQZzjQAHgUGzuRZBTxS0P756Gyg5UBHdIjo18BVZjY1mvy9KmqrGvOnNgDw7DsHYq5ERKQyyjkN9KfAs8B5ZtZmZjcDdwNXmtlW4MpoHeBXwHZgG/AD4AsA7n4I+BrwQvR1V9RWNZrqMzTXZ3h2+8G4SxERqYj0aB3c/d+V2LSiSF8Hbi2xnzXAmjFVV2HzptTz7DsH6c87qaDYtIWISO1I/JXAhRZOb+Bodx8vvaezgUSk9ikACiyc3kBg8A+b98VdiojIhFMAFMilU7RObeAf3lQAiEjtUwAMsXhGI++0H+fdA7o/gIjUNgXAEItnNALw6zf2xlyJiMjEUgAM0VyfYXZTHX//yq64SxERmVAKgCLOmz2ZzXs62bqvM+5SREQmjAKgiCUzJxEYGgWISE1TABTRmEszf1oDD7+0i/68PhxORGqTAqCEpXOa2NPRzVNv7R+9s4jIWUgBUMI5LZOYXJdm7bPvxl2KiMiEUACUkAqMC+c285utB9jero+IFpHaowAYwdK5TaQC4we/2R53KSIi404BMILGXJqlc5pYt7GNtsMn4i5HRGRcKQBGsWzRVADue/qdmCsRERlfCoBRTK7LcMGcJta9sJMd+nwgEakhCoAyXLZ4GqnA+Npjb8ZdiojIuFEAlKExl+YPFk3jybf28/QWXRcgIrVBAVCmi+dPYVpjlq/+4nU6u3vjLkdE5H1TAJQpFRgrzp/J7o4uvv7Y5rjLERF53xQAYzB3Sj0fWzCVn23cyeOv7Ym7HBGR90UBMEbLz5nOnOY6/su6V9myVx8XLSJnLwXAGKUC47oL55AKjP/4440cONYTd0kiImdEAXAGJtWlue7Ds9nT0cVNq5+no0uTwiJy9lEAnKE5zfV86sNzeHvfMT6/5nmOnDgZd0kiImOiAHgfFk5v5NoLZ/P6rqP80ff/mV1HuuIuSUSkbAqA9+kDLZO44eK57Drcxcp7n+G57QfjLklEpCwKgHHQOrWBz3yslbzDv//Bc9z75Fb6+vNxlyUiMiIFwDiZPinHv102nyUzJ/GtJ97mhu/9ltd3dcRdlohISQqAcZRNB1y9dDbXXTibHQePc/29z/CVX7zG3o7uuEsTERkmHXcBtcbMWDJrMvOnNfDc9oP87IWd/PzFNv5k+UL+9PJFtE5tiLtEERFAATBh6jIpPnHeTC5ZMJXntx/kR7/dwY9+u4Orl87mpuULWX7OdILA4i5TRBJMATDBmuszXLV0Nss/MJ1NbR08vaWdx1/fy6ymHNd/ZC6fvmguH57XrDAQkYpTAFRIU12Gj39wBpctnsaOA8fZsreTNb99lx/8ZgfTGrN84rwWPnHeTJYvnsbMprq4yxWRBFAAVFgmFXDurMmcO2syXb39/P7Acd49eILHX9vLwy/tAmDelHqWLZrKxxZOZencJs6dNZnJdZnRd75pHWy4CzraoLkVVtwBF31ugt+RiJytFAAxqs+kOH9OE+fPaSLvzv6jPezu6GJPRzcbNu/nkVd2D/adO6WOC+Y0sWTWZBZOa2DB9AYWTGtgTnM9qcDCX/6/vA16o6uRO3aG66AQEJGiKh4AZnYN8DdACvihu99d6RqqUWDG7OY6ZjeHh3/cnc7uPg4c6+HA8ZMc7OzhlZ1HePKt/eT91PPSgTFvaj0PdX+Vlv4hH0XR20XvE/+dYx+4gab6TBgUIiKRigaAmaWA7wFXAm3AC2b2qLvrbutDmBlN9Rma6jOc03KqPZ93Onv66OjqHfw62tXL9P72ovtJde7mkq+tB2ByXZop9RmmNmaZ0pBlSn2G5voMjbk0jdkUDbk0k3IpGrJpGqPHSbk0DdkUjbk0uXRALp0ilw40aS0ynmI6fFvpEcClwDZ33w5gZg8CKwEFQJmCwGiOfnEXOrZxFk09e4f1P5xp4V8vbKG7tz/86svT0dVLe2cP3b399PTlOdmXp69wWFGGdGDk0gHZgVDIBNSlU2TTAXWZU0GRSQWkU0Y6MNKpgEzKSAVGOggG28JHI5MKom02ZFv4PDMjZUZg4fchKLKcsqhfEK4XLod9jCA41S8wou0W7SfsZwbGwGO4n/AxbMcouc2ibBxYH6hjsI8pPKVAjIdvKx0A84CdBettwGUVrqEmPbPgC1z5zl+TyZ+66rg3qOP5xX/GxTOnjPr8/rzT25+ntz8MhN7+gvX+PL19Tl8+T3/e6cv7aY/hcp6+fJ7u7n4OnzjVnncn7+Ehrf684w79HrXno+W8M7b4qQ3DAoXSITIQNhQJplP9TgXN4GsMyZrTVodsLPW80/c4dNuphbL6FWkofN5I9RYG59D9lVvvSO2l9j+8pvLqLedFDVh9+CvMzA8/fMuGu2ouAIp9f0772TezW4BbABYsWHDGL7Rs0TQ+tnDqGT//7LMEXpuDFwwj0yvu4NoPf5Zr4y6tDPkoUMIgcfr6nb7+aDkKJ3fIFwTJwPJgwERhkx8MnjBk8gXbfOhzouXB/gXPdwdn4BHcgdPWfbB9YJ3B9VP98vni7YXrlNjf4OuOsM0JGwtrCCs9XcGmItuKR/DQ5sKoPm1/w/qV3reXWBn6Z0Cpesutafhrjf17M1Ltw75lJb6HpV5zQMvBA8U3dLSNuL/xUOkAaAPmF6y3ArsLO7j7/cD9AMuWLXtffxgmbqh90efO2jN+UikjlYIcqbhLEamse1rDwz5DNbdO+EtX+sPgXgCWmNliM8sCNwKPVrgGEZHqseIOyNSf3papD9snWEVHAO7eZ2Z/Bvya8DTQNe7+RiVrEBGpKgOj9gScBYS7/wr4VaVfV0SkasV0+Fb3AxARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJZaU+BKoamFk78Pu464jMAEp8alNVUH1nrpprg+qur5prg+qubyJrW+juLaN1quoAqCZmttHdl8VdRymq78xVc21Q3fVVc21Q3fVVQ206BCQiklAKABGRhFIAlO/+uAsYheo7c9VcG1R3fdVcG1R3fbHXpjkAEZGE0ghARCShFACjMLP5ZvaUmW02szfM7Itx1zSUmaXM7GUzeyzuWoYysylm9pCZvRV9D/8w7poKmdl/jv5dXzezn5pZXYy1rDGz/Wb2ekHbNDNbb2Zbo8fYbnRdor7/Hf3bbjKzX5jZlGqqr2DbfzUzN7MZ1VSbmf25mW2J/g/+r0rXpQAYXR/wl+7+IWA5cKuZXRBzTUN9EdgcdxEl/A3w/9z9fOAjVFGdZjYPuA1Y5u4XEt6l7sYYS3oAuGZI25eBDe6+BNgQrcflAYbXtx640N0vAt4Gbq90UQUeYHh9mNl84ErgvUoXVOABhtRmZv8GWAlc5O5LgW9VuigFwCjcfY+7vxQtdxL+ApsXb1WnmFkr8Cngh3HXMpSZNQH/ClgN4O4n3f1IvFUNkwbqzSwNNAC74yrE3f8JODSkeSWwNlpeC9xQ0aIKFKvP3Z9w975o9Tlg4u9kXkKJ7x/APcBfAbFNeJao7T8Bd7t7T9Rnf6XrUgCMgZktAi4Bno+3ktN8h/A/dz7uQoo4B2gHfhQdovqhmTXGXdQAd99F+FfXe8AeoMPdn4i3qmFmufseCP8YAWbGXM9I/gPweNxFFDKz64Fd7v5q3LUUcS7wL83seTP7RzP7g0oXoAAok5lNAn4O/IW7H427HgAz+zSw391fjLuWEtLAR4Hvu/slwHHiPYRxmuh4+kpgMTAXaDSzP4m3qrOTmX2V8HDpT+KuZYCZNQBfBe6Iu5YS0sBUwkPL/w1YZ2ZWyQIUAGUwswzhL/+fuPvDcddT4HLgejN7F3gQuMLM/i7ekk7TBrS5+8CI6SHCQKgWnwR2uHu7u/cCDwP/IuaahtpnZnMAoseKHyYYjZmtAj4N/LFX13nlHyAM91ejn5FW4CUzmx1rVae0AQ976HeEo/iKTlIrAEYRJfJqYLO7fzvuegq5++3u3uruiwgnL59096r5C9bd9wI7zey8qGkF8GaMJQ31HrDczBqif+cVVNEkdeRRYFW0vAp4JMZahjGza4AvAde7+4m46ynk7q+5+0x3XxT9jLQBH43+X1aDvweuADCzc4EsFf7gOgXA6C4HbiL86/qV6Ou6uIs6i/w58BMz2wRcDPx1zPUMikYmDwEvAa8R/jzEdnWmmf0UeBY4z8zazOxm4G7gSjPbSngmy91VVt+9wGRgffSz8X+rrL6qUKK2NcA50amhDwKrKj2C0pXAIiIJpRGAiEhCKQBERBJKASAiklAKABGRhFIAiIgklAJARCShFAAiIgmlABARSaj/D6Gz7+yGqKSPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1539704395225572, 1.4115360845240776, 4.139754128892224, 16.2278954349457 \n", + "Times:\t\t2769.889957638353, 1513.4606486459202, 60.00000000000002, 1.0000000000000007" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight + 100**ALPHA, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Outliers: solution\n", + "\n", + "Compute the estimator conditioned on the event the the top most transaction captures the first slot. This decreases `total_weight` on the one hand (thus increasing `p`), while increasing `inclusion_interval` on the other, by capturing a block slot. If this estimator gives lower prediction times we switch to it, and then repeat the process with the next highest transaction. The process convegres when the estimator is no longer improving or if all block slots are captured. " + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHmVJREFUeJzt3X2QXXWd5/H39z72U/op6TzQnZggEVREwR7Ah3IdIw5RxzCzYuG4Gl1qU7MwPoyzpehUyYxTM+PUuAO6A9REQeMui7DImjgLg5mAhSgBwlMMhJAQIOkkJB066Tx2p/ve7/5xTie307e7k7597+m+5/OqunXP+Z3fOed7fehPfufR3B0REYmfRNQFiIhINBQAIiIxpQAQEYkpBYCISEwpAEREYkoBICISUwoAEZGYGjcAzOwOM9tnZpuKLPtvZuZmNiucNzP7vpltM7ONZnZJQd/lZrY1/Cyf3J8hIiJn60xGAD8Grjy90czmA1cAOwqalwKLw88K4LawbytwI3AZcClwo5m1lFK4iIiUJjVeB3d/xMwWFll0E/A1YHVB2zLgJx7cXrzezJrNbB7wQWCtu/cAmNlaglC5a6x9z5o1yxcuLLZrEREZzVNPPbXf3dvG6zduABRjZp8Adrn7c2ZWuKgd2Fkw3xW2jdY+poULF7Jhw4aJlCgiEltm9tqZ9DvrADCzOuAvgY8UW1ykzcdoL7b9FQSHj1iwYMHZliciImdoIlcBvRlYBDxnZq8CHcDTZjaX4F/28wv6dgC7x2gfwd1Xununu3e2tY07ghERkQk66wBw99+5+2x3X+juCwn+uF/i7q8Da4DPhVcDXQ70uvse4EHgI2bWEp78/UjYJiIiETmTy0DvAh4DzjezLjO7dozu9wPbgW3AD4DrAMKTv38DPBl+vj10QlhERKJhU/l9AJ2dna6TwCIiZ8fMnnL3zvH66U5gEZGYUgCIiMRUVQbA4b4Bblr7Es/uPBh1KSIiU1ZVBkAu73xv3Vaefu1A1KWIiExZVRkA9dng/rYj/YMRVyIiMnVVZQCkkwlq0gkFgIjIGKoyACAYBRzuUwCIiIymegMgk+Jw30DUZYiITFlVGwC1maQOAYmIjKFqA6Auk+SIDgGJiIyqqgPgkA4BiYiMqooDQCeBRUTGUsUBoHMAIiJjqeoAONo/yFR+2qmISJSqNgDqsynyDsdO5KIuRURkSqraAKjLJAE9DkJEZDRVGwD1meB5QDoRLCJSXNUGgEYAIiJjq9oAGHoiqB4HISJSXNUGwMkRgA4BiYgUNW4AmNkdZrbPzDYVtP2jmb1oZhvN7P+aWXPBsm+Y2TYz22Jmf1DQfmXYts3Mbpj8nzJc3dA5AB0CEhEp6kxGAD8GrjytbS1wobtfBLwEfAPAzN4GXAO8PVznVjNLmlkSuAVYCrwN+HTYt2zqssEIQCeBRUSKGzcA3P0RoOe0tl+6+9Bf1vVARzi9DPipu/e7+yvANuDS8LPN3be7+wngp2Hfshm6CkiHgEREipuMcwD/GXggnG4HdhYs6wrbRmsvm2TCSCeNI/06CSwiUkxJAWBmfwkMAncONRXp5mO0F9vmCjPbYGYburu7SymPbErPAxIRGc2EA8DMlgMfBz7jpx640wXML+jWAeweo30Ed1/p7p3u3tnW1jbR8gDIpBIc0iEgEZGiJhQAZnYl8HXgE+5+rGDRGuAaM8ua2SJgMfAE8CSw2MwWmVmG4ETxmtJKH186aToHICIyitR4HczsLuCDwCwz6wJuJLjqJwusNTOA9e7+p+7+vJndA7xAcGjoenfPhdv5M+BBIAnc4e7Pl+H3DJNJJjh0XOcARESKGTcA3P3TRZpvH6P/3wJ/W6T9fuD+s6quRNl0kl4FgIhIUVV7JzBATSqhABARGUVVB0A2HbwXWC+FEREZqboDIJVgIOf0DeSjLkVEZMqp6gCoSQWPg9BhIBGRkao6ALLp4OcpAERERqruAEgpAERERlPVAVCT1iEgEZHRVHUAaAQgIjK6qg4AjQBEREZX1QGQ0QhARGRUVR0ACTNq0noekIhIMVUdABC8E0ABICIyUgwCQM8DEhEpRgEgIhJTVR8AmVSCg8cUACIip6v6AKjROwFERIqq+gDIphIc6lMAiIicrvoDIJ2kfzBP30Au6lJERKaUqg+AmvBmMF0KKiIyXNUHQG34OIgDOhEsIjLMuAFgZneY2T4z21TQ1mpma81sa/jdErabmX3fzLaZ2UYzu6RgneVh/61mtrw8P2ekoecB9Rw9UaldiohMC2cyAvgxcOVpbTcA69x9MbAunAdYCiwOPyuA2yAIDOBG4DLgUuDGodAot9rM0AhAASAiUmjcAHD3R4Ce05qXAavC6VXAVQXtP/HAeqDZzOYBfwCsdfcedz8ArGVkqJRFrUYAIiJFTfQcwBx33wMQfs8O29uBnQX9usK20drLbugQ0AEFgIjIMJN9EtiKtPkY7SM3YLbCzDaY2Ybu7u6SC0omjGwqQY8OAYmIDDPRANgbHtoh/N4XtncB8wv6dQC7x2gfwd1Xununu3e2tbVNsLzh6jJJjQBERE4z0QBYAwxdybMcWF3Q/rnwaqDLgd7wENGDwEfMrCU8+fuRsK0iatJJenQZqIjIMKnxOpjZXcAHgVlm1kVwNc93gHvM7FpgB3B12P1+4KPANuAY8AUAd+8xs78Bngz7fdvdTz+xXDbZVIKeo/2V2p2IyLQwbgC4+6dHWbSkSF8Hrh9lO3cAd5xVdZOkNp3UVUAiIqep+juBIbgX4MBRHQISESkUiwCoSSc5PpDTA+FERArEIgB0M5iIyEjxCICMAkBE5HSxCICTdwPrZjARkZNiEQA6BCQiMlKsAkB3A4uInBKLAMimExgaAYiIFIpFACTMqMsm6T6iABARGRKLAACoy6TYf0SPgxARGRKLADh/3wP8YvBP+ZftH4abLoSN90RdkohI5MZ9FtB0d/6+B7ji5b8j7X1BQ+9O+MWXgumLPhVdYSIiEav6EcD7d9xKOt83vHHgOKz7djQFiYhMEVUfADP69xZf0NtV2UJERKaYqg+Aw9k5xRc0dVS2EBGRKabqA+DRBdcxkKgZ3piuhSXfiqYgEZEpoupPAm+ZvRSA97x6C00n9tFXN4+6pX+tE8AiEntVPwKAIARue9dqzu2/k7vff7/++IuIEJMAAKhJJ0gYdB/WzWAiIhCjADAzGrK6G1hEZEhJAWBmf25mz5vZJjO7y8xqzGyRmT1uZlvN7G4zy4R9s+H8tnD5wsn4AWejNpPUCEBEJDThADCzduBLQKe7XwgkgWuAfwBucvfFwAHg2nCVa4ED7n4ecFPYr6Jq00n2KQBERIDSDwGlgFozSwF1wB7gQ8C94fJVwFXh9LJwnnD5EjOzEvd/VuoyKQWAiEhowgHg7ruA7wI7CP7w9wJPAQfdfTDs1gW0h9PtwM5w3cGw/8yJ7n8iGrIpeo6cYDCXr+RuRUSmpFIOAbUQ/Kt+EXAOUA8sLdLVh1YZY1nhdleY2QYz29Dd3T3R8oqqzybJubNf7wUQESnpENCHgVfcvdvdB4D7gPcCzeEhIYAOYHc43QXMBwiXNwE9p2/U3Ve6e6e7d7a1tZVQ3kgNNUFZe3qPT+p2RUSmo1ICYAdwuZnVhcfylwAvAA8Dnwz7LAdWh9NrwnnC5Q+5+4gRQDnNyKYBeL23b5yeIiLVr5RzAI8TnMx9GvhduK2VwNeBr5rZNoJj/LeHq9wOzAzbvwrcUELdE9KQHRoBKABEREp6FpC73wjceFrzduDSIn37gKtL2V+patIJUglj7yEFgIhIbO4EhuBu4Bk1KY0ARESIWQAA1GdTOgksIkJsA0AjABGR2AVAQzbF3kN95PMVvQBJRGTKiWUADOScnmO6GUxE4i2WAQC6F0BEJH4BUKN7AUREIIYBMCMcAew+qCuBRCTeYhcAdZkkqYTRdeBY1KWIiEQqdgFgZjTVptnZoxGAiMRb7AIAgvMAO3o0AhCReItlADTWpNmpQ0AiEnOxDICm2jSH+wbpPT4QdSkiIpGJZQA0hpeC6kSwiMRZPAOgNngxjE4Ei0icxToANAIQkTiLZQDUpBJkUwm6DmgEICLxFcsAMLPgSiBdCioiMRbLAACYoXsBRCTmYhsATXVpdvQc03sBRCS2SgoAM2s2s3vN7EUz22xm7zGzVjNba2Zbw++WsK+Z2ffNbJuZbTSzSybnJ0xMS22G/sE8e/SCeBGJqVJHAN8D/s3dLwDeCWwGbgDWuftiYF04D7AUWBx+VgC3lbjvkjTXBVcCvdJ9NMoyREQiM+EAMLNG4APA7QDufsLdDwLLgFVht1XAVeH0MuAnHlgPNJvZvAlXXqKW+gwAr+w/ElUJIiKRKmUEcC7QDfzIzJ4xsx+aWT0wx933AITfs8P+7cDOgvW7wrZI1GeSZJIJXtYIQERiqpQASAGXALe5+8XAUU4d7inGirSNOANrZivMbIOZbeju7i6hvLGZGS31aV7ZrwAQkXgqJQC6gC53fzycv5cgEPYOHdoJv/cV9J9fsH4HsPv0jbr7SnfvdPfOtra2EsobX1NNmu3dOgQkIvE04QBw99eBnWZ2fti0BHgBWAMsD9uWA6vD6TXA58KrgS4HeocOFUWluT7DroPH6R/MRVmGiEgkUiWu/0XgTjPLANuBLxCEyj1mdi2wA7g67Hs/8FFgG3As7Buplro0eYedPcc4b/aMqMsREamokgLA3Z8FOossWlKkrwPXl7K/ydZcF1wJ9HL3UQWAiMRObO8EBmgNA2DbPp0HEJH4iXUAZFIJmmrTvPj64ahLERGpuFgHAEBrfYYX9xyKugwRkYqLfQDMrM+wff9RTgzmoy5FRKSiFAANGXJ5Z7seCSEiMRP7AJjVkAVgi84DiEjMxD4AWuoyJEwBICLxE/sASCaM1vqMAkBEYif2AQDBlUAv6EogEYkZBQCwzH7D/zm+Av+rZrjpQth4T9QliYiUXanPApr2zt/3AB/uvZlMoj9o6N0Jv/hSMH3Rp6IrTESkzGI/Anj/jlvJeP/wxoHjsO7b0RQkIlIhsQ+AGf17iy/o7apsISIiFRb7ADicnVN8QVNHZQsREamw2AfAowuuYyBRM7wxXQtLvhVNQSIiFRL7k8BbZi8F4PJXbqF5YB/99fOovfKvdQJYRKpe7EcAEITAynev5s39d/Ivl6zWH38RiQUFQCibSjKrIcNTrx2IuhQRkYpQABSY11TLU68dYDCnR0OLSPVTABQ4p7mWYydyekOYiMRCyQFgZkkze8bM/jWcX2Rmj5vZVjO728wyYXs2nN8WLl9Y6r4n2znNwdVAT77aE3ElIiLlNxkjgC8Dmwvm/wG4yd0XAweAa8P2a4ED7n4ecFPYb0qZUZOmqTbNhld1HkBEql9JAWBmHcDHgB+G8wZ8CLg37LIKuCqcXhbOEy5fEvafUuY21vDEKz24e9SliIiUVakjgJuBrwFDZ01nAgfdfTCc7wLaw+l2YCdAuLw37D+lnNNcQ/eRfl5941jUpYiIlNWEA8DMPg7sc/enCpuLdPUzWFa43RVmtsHMNnR3d0+0vAmb31oHwKNbK79vEZFKKmUE8D7gE2b2KvBTgkM/NwPNZjZ0h3EHsDuc7gLmA4TLm4ARZ1vdfaW7d7p7Z1tbWwnlTUxzbXAe4JGt+yu+bxGRSppwALj7N9y9w90XAtcAD7n7Z4CHgU+G3ZYDq8PpNeE84fKHfAoeaDcz5rfU8tjLbzCg+wFEpIqV4z6ArwNfNbNtBMf4bw/bbwdmhu1fBW4ow74nxYLWOo70D/LczoNRlyIiUjaT8jA4d/8V8KtwejtwaZE+fcDVk7G/cpvfWocZPLJ1P50LW6MuR0SkLHQncBE16SRzG2v41ZZ9UZciIlI2CoBRLJxZz8auXl7v7Yu6FBGRslAAjOLNbfUArN08yisjRUSmOQXAKFrrM7TWpXlw0+tRlyIiUhYKgFGYGYvaGnhs+xv0Hh+IuhwRkUmnABjDm9vqyeWdh17UYSARqT4KgDHMbayhsSbF6md2j99ZRGSaUQCMwcx4y5wZ/HrrfroP90ddjojIpFIAjOOCuTPIufOL5zQKEJHqogAYx8yGLHMas9z3dFfUpYiITCoFwBl4y5wZbNp9iC16V7CIVBEFwBl469xGUgnjf65/NepSREQmjQLgDNRmkiye3cB9T+/icJ/uCRCR6qAAOEMXdTRz7ESO+57eFXUpIiKTQgFwhuY21TC3sYZVv32VfH7KvcdGROSsKQDOwjvnN7F9/1E9IE5EqoIC4Cy8ZfYMmuvS/PND25iCb7MUETkrCoCzkEgY717Qwu929fJrvTReRKY5BcBZumDeDGbUpLj531/SKEBEpjUFwFlKJRL83sJWnt5xkAef17kAEZm+JhwAZjbfzB42s81m9ryZfTlsbzWztWa2NfxuCdvNzL5vZtvMbKOZXTJZP6LS3j6vkZn1Gf7+gc0M5PJRlyMiMiGljAAGgb9w97cClwPXm9nbgBuAde6+GFgXzgMsBRaHnxXAbSXsO1KJhPHe82by2hvHuHP9a1GXIyIyIRMOAHff4+5Ph9OHgc1AO7AMWBV2WwVcFU4vA37igfVAs5nNm3DlEVs0s54FrXV895cvsfeQXhwvItPPpJwDMLOFwMXA48Acd98DQUgAs8Nu7cDOgtW6wrZpycz4/fPb6BvI8Vdrno+6HBGRs1ZyAJhZA/Az4CvufmisrkXaRlxGY2YrzGyDmW3o7u4utbyyaq7LcOmiVh7Y9DoPPq+Xx4vI9FJSAJhZmuCP/53ufl/YvHfo0E74vS9s7wLmF6zeAYx4y4q7r3T3TnfvbGtrK6W8irhkQQttM7Lc8LON7NOhIBGZRkq5CsiA24HN7v5PBYvWAMvD6eXA6oL2z4VXA10O9A4dKprOkgnjyrfP5Uj/IF+5+1k9J0hEpo1SRgDvAz4LfMjMng0/HwW+A1xhZluBK8J5gPuB7cA24AfAdSXse0pprc/wgcVt/PblN7jl4W1RlyMickZSE13R3R+l+HF9gCVF+jtw/UT3N9W9/ZxGdh08zn9f+xKL5zRw5YXT9gInEYkJ3Qk8ScyMJRfMZl5TDV+5+1k27eqNuiQRkTEpACZRKpngY++YRyaV4HN3PMHL3UeiLklEZFQKgElWn01x1Tvb6R/I8Sc/WM+ON45FXZKISFEKgDJoqc9w1cXtHDo+yKdWPsa2fYejLklEZAQFQJnMasjyRxe3c7hvgP9422M89dqBqEsSERlGAVBGbTOyXP3u+SQM/uQH6/n5M3qhvIhMHQqAMmuqTfPJd3cwqyHLV+5+lhtXb+LEoB4hLSLRUwBUQF0mxR9d3M7FC5pZ9dhr/PGtv2HL6zovICLRUgBUSDJhfGBxGx97xzy27z/Kx//Hr7nl4W16oYyIREYBUGHnzW7gM5ctYOHMev7xwS1cefMj/GrLvvFXFBGZZAqACNRlUnz0HfP4w4vm0XP0BJ//0ZN8/o4n2Nh1MOrSRCRGJvwsICnduW0NLJhZx3M7e1n/yht84p9/w++f38YXlyzmkgUtUZcnIlVOARCxVCLBu9/UwoXtjTzX1cvjr/Tw8K2/5Z0dTXz2PQv5+EXzqEkng84b74F134beLmjqgCXfgos+Fe0PEJFpy4KHdE5NnZ2dvmHDhgmt+9Lew/y/jdPvdQMnBvO8sOcQm3b18sbREzTVpln2rnP4/IwnWPTYN7GB46c6p2vhD7+vEBCRYczsKXfvHK+fzgFMMZlUgnfNb+Yzly3gjy9uZ3Zjlv/9+A4yv/rb4X/8AQaOByMCEZEJ0CGgKcrMmN9ax/zWOvoHc7Q//kbRft7bxe6Dx2lvrq1whSIy3SkApoFsKsnh7Bwa+0e+eH5Xfibv/85DzGuq4bJFrfzeolYuWdDCebMbSCc1wBOR0SkApolHF1zHFS//Hen8qRfPDyRqeHT+f+U/WBu7Dh7n3zfv4+fP7gYgk0xw/twZXNjexNvPaeSt8xo5r62Bprp0VD9BRKYYBcA0sWX2UgDev+NWZvTv5XB2Do8uuI49s5fyLuBd85txdw4eH2DvoT66D/fTfaSfnz+zi7ue2HFyOy11ac6b3cCb24LPm2bW0d5SS0dzHY21KcxGe8uniFQbBcA0smX20pNBUIyZ0VKXoaUuwwVzgzZ353DfIPuP9HPg2AAHjp1gz8E+Xth9iKMncsPWr8skaW+upaOllnOaa5nXVMOshixtM4LPrIbgk0np0JJINah4AJjZlcD3gCTwQ3f/TqVriBMzo7E2TWPtyEM/fQM5eo8PcKhvgMN9gxw+Psjh/gFe2HOI9dt7OD6QK7JFaKxJ0TYjy8yGLE21aZpr0zSFn+a6YF/NdZmTbTNqUtRnUtSkExphiEwhFQ0AM0sCtwBXAF3Ak2a2xt1fqGQdEqhJJ6lJJ5nTWFN0+WAuz7ETufAzyLETOY6G38f6c+w6cJxX9h+lfyDH8YEcA7mx7ylJGNRmktRnUtRnU9RnkzRkg3Coy6ZoyCapy6SoTSfJphLUpJNk0wlqUsF3NjV8/uR3Qf900kgnEiQSChqZRiK6ybPSI4BLgW3uvh3AzH4KLAMUAFNQKpmgsTZRdPRQTC7v9A3k6B/MD/s+kcszMJhnIOfBdC7PicE8R/oGOXB0gMF8MD+Q8/A7T6m3JybNSCWDTzqRIJ1MBNPJRBASyUTR6VRBWyqRIJmA5NC3GYmEkUoE30kzkonwEy5LDi0Plw31P7U83J4F04mwzoSd+piBWbDMgETCCPIs+B7qc7IvRiJxqr9Z0G/oOxGOuoa2Y5xaPrSdoXYr2E6xWoZGcMF08J+1RnUl2ngP/OJLwX09AL07g3koewhUOgDagZ0F813AZRWuQcokmbDwX/albcfdyTsM5vPk8s5gzoPvvDOYz4+czzu5XDCfcyefd/Lu5PMMm88VtJ0YzHN8IHdyX0GfsP/JdSDvjo/yPTQ9de+lr6yhGBgKpaEGK2yjMDiGlg8F2akNDbWdWt+Gbf9k15PBdGpbBbs+uV87vZaCjZxe2+lBR5HlheueKttGto23HLjj4DeZnR/lJs8qC4Bi/1QY9v8fM1sBrABYsGDBhHd07qx6/ssHzp3w+iJnKl8QPCdDJp8n56eW5fLBJ18wXRhIubzjhSHDqSA8FTxBe9BWGEQAwwPr5LrDtuMj1g0C7PT2YBmc2v7QsmBPhdOE00GDe2Hb8L6EtQxfHsz4yeUjt0/YVrj9wr4Mq+X0bZ3amY9Y3yko/eR+C9dnxG899WMK/3AVe6JOsb7D/rMIv9t69o9cGYLDQWVW6QDoAuYXzHcAuws7uPtKYCUEzwKa6I5SyQQNuhFKRKa6mzqCwz6na+oo+64r/RfySWCxmS0yswxwDbCmwjWIiEwdS74VPNixULo2aC+zio4A3H3QzP4MeJDgMtA73P35StYgIjKlDB3nj8FVQLj7/cD9ld6viMiUddGnInmsuw6Si4jElAJARCSmFAAiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZgyL/YUoynCzLqB16KuYwJmAaM84anqxOm3gn5vNaum3/omd28br9OUDoDpysw2uHtn1HVUQpx+K+j3VrM4/dYhOgQkIhJTCgARkZhSAJTHyqgLqKA4/VbQ761mcfqtgM4BiIjElkYAIiIxpQCYJGY238weNrPNZva8mX056prKzcySZvaMmf1r1LWUm5k1m9m9ZvZi+N/xe6KuqZzM7M/D/x1vMrO7zKwm6pomk5ndYWb7zGxTQVurma01s63hd0uUNVaCAmDyDAJ/4e5vBS4Hrjezt0VcU7l9GdgcdREV8j3g39z9AuCdVPHvNrN24EtAp7tfSPD2vmuirWrS/Ri48rS2G4B17r4YWBfOVzUFwCRx9z3u/nQ4fZjgD0R7tFWVj5l1AB8Dfhh1LeVmZo3AB4DbAdz9hLsfjLaqsksBtWaWAuqA3RHXM6nc/RGg57TmZcCqcHoVcFVFi4qAAqAMzGwhcDHweLSVlNXNwNeAfNSFVMC5QDfwo/CQ1w/NrD7qosrF3XcB3wV2AHuAXnf/ZbRVVcQcd98DwT/ogNkR11N2CoBJZmYNwM+Ar7j7oajrKQcz+ziwz92firqWCkkBlwC3ufvFwFGq+PBAeOx7GbAIOAeoN7P/FG1VUg4KgElkZmmCP/53uvt9UddTRu8DPmFmrwI/BT5kZv8r2pLKqgvocvehEd29BIFQrT4MvOLu3e4+ANwHvDfimiphr5nNAwi/90VcT9kpACaJmRnBMeLN7v5PUddTTu7+DXfvcPeFBCcHH3L3qv0Xoru/Duw0s/PDpiXACxGWVG47gMvNrC783/USqvikd4E1wPJwejmwOsJaKiIVdQFV5H3AZ4HfmdmzYds33f3+CGuSyfNF4E4zywDbgS9EXE/ZuPvjZnYv8DTB1W3PUGV3yZrZXcAHgVlm1gXcCHwHuMfMriUIwaujq7AydCewiEhM6RCQiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmFAAiIjGlABARian/D+hfAG5Faoo0AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1531420689155165, 1.4085104512204296, 2.816548045571761, 11.10120050773006 \n", + "Times:\t\t874.010579873836, 479.615551452334, 60.00000000000001, 1.0000000000000004" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def build_estimator():\n", + " _feerates = [1.0]*10 + [1.1]*10 + [1.2]*10 + [1.5]*3000 + [2]*3000\\\n", + "+ [2.1]*3000 + [3]*10 + [4]*10 + [5]*10 + [6] + [7] + [10] + [100] + [200]*200\n", + " _total_weight = sum(np.array(_feerates)**ALPHA)\n", + " _network_mass_rate = bps * block_mass_limit\n", + " estimator = FeerateEstimator(total_weight=_total_weight, \n", + " inclusion_interval=avg_mass/_network_mass_rate)\n", + " \n", + " nr = _network_mass_rate\n", + " for i in range(len(_feerates)-1, -1, -1):\n", + " tw = sum(np.array(_feerates[:i])**ALPHA)\n", + " nr -= avg_mass\n", + " if nr <= 0:\n", + " print(\"net mass rate {}\", nr)\n", + " break\n", + " e = FeerateEstimator(total_weight=tw, \n", + " inclusion_interval=avg_mass/nr)\n", + " if e.feerate_to_time(1.0) < estimator.feerate_to_time(1.0):\n", + " # print(\"removing {}\".format(_feerates[i]))\n", + " estimator = e\n", + " else:\n", + " break\n", + " \n", + " return estimator\n", + "\n", + "estimator = build_estimator()\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:gr]", + "language": "python", + "name": "conda-env-gr-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs new file mode 100644 index 000000000..5ef3579a5 --- /dev/null +++ b/mining/src/feerate/mod.rs @@ -0,0 +1,231 @@ +//! See the accompanying fee_estimation.ipynb Jupyter Notebook which details the reasoning +//! behind this fee estimator. + +use crate::block_template::selector::ALPHA; +use itertools::Itertools; +use std::fmt::Display; + +/// A type representing fee/mass of a transaction in `sompi/gram` units. +/// Given a feerate value recommendation, calculate the required fee by +/// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +pub type Feerate = f64; + +#[derive(Clone, Copy, Debug)] +pub struct FeerateBucket { + pub feerate: f64, + pub estimated_seconds: f64, +} + +impl Display for FeerateBucket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:.4}, {:.4}s)", self.feerate, self.estimated_seconds) + } +} + +#[derive(Clone, Debug)] +pub struct FeerateEstimations { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl FeerateEstimations { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +impl Display for FeerateEstimations { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "(fee/mass, secs) priority: {}, ", self.priority_bucket)?; + write!(f, "normal: {}, ", self.normal_buckets.iter().format(", "))?; + write!(f, "low: {}", self.low_buckets.iter().format(", ")) + } +} + +pub struct FeerateEstimatorArgs { + pub network_blocks_per_second: u64, + pub maximum_mass_per_block: u64, +} + +impl FeerateEstimatorArgs { + pub fn new(network_blocks_per_second: u64, maximum_mass_per_block: u64) -> Self { + Self { network_blocks_per_second, maximum_mass_per_block } + } + + pub fn network_mass_per_second(&self) -> u64 { + self.network_blocks_per_second * self.maximum_mass_per_block + } +} + +#[derive(Debug, Clone)] +pub struct FeerateEstimator { + /// The total probability weight of current mempool ready transactions, i.e., `Σ_{tx in mempool}(tx.fee/tx.mass)^alpha`. + /// Note that some estimators might consider a reduced weight which excludes outliers. See [`Frontier::build_feerate_estimator`] + total_weight: f64, + + /// The amortized time **in seconds** between transactions, given the current transaction masses present in the mempool. Or in + /// other words, the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, + /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. + inclusion_interval: f64, +} + +impl FeerateEstimator { + pub fn new(total_weight: f64, inclusion_interval: f64) -> Self { + assert!(total_weight >= 0.0); + assert!((0f64..1f64).contains(&inclusion_interval)); + Self { total_weight, inclusion_interval } + } + + pub(crate) fn feerate_to_time(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / feerate.powi(ALPHA) + c1 + } + + fn time_to_feerate(&self, time: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + assert!(c1 < time, "{c1}, {time}"); + ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) + } + + /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` + #[inline] + fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) + } + + /// Returns the feerate value for which the integral area is `frac` of the total area between `lower` and `upper`. + fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { + assert!((0f64..=1f64).contains(&frac)); + assert!(0.0 < lower && lower <= upper, "{lower}, {upper}"); + let (c1, c2) = (self.inclusion_interval, self.total_weight); + if c1 == 0.0 || c2 == 0.0 { + // if c1 · c2 == 0.0, the integral area is empty, so we simply return `lower` + return lower; + } + let z1 = self.feerate_to_time_antiderivative(lower); + let z2 = self.feerate_to_time_antiderivative(upper); + // Get the total area corresponding to `frac` of the integral area between `lower` and `upper` + // which can be expressed as z1 + frac * (z2 - z1) + let z = frac * z2 + (1f64 - frac) * z1; + // Calc the x value (feerate) corresponding to said area + ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) + } + + pub fn calc_estimations(&self, minimum_standard_feerate: f64) -> FeerateEstimations { + let min = minimum_standard_feerate; + // Choose `high` such that it provides sub-second waiting time + let high = self.time_to_feerate(1f64).max(min); + // Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile + let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); + // Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66 quantile between low and high. + let normal = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.66)); + // Choose an additional point between normal and low + let mid = self.time_to_feerate(1800f64).max(self.quantile(min, high, 0.5)); + /* Intuition for the above: + 1. The quantile calculations make sure that we return interesting points on the `feerate_to_time` curve. + 2. They also ensure that the times don't diminish too high if small increments to feerate would suffice + to cover large fractions of the integral area (reflecting the position within the waiting-time distribution) + */ + FeerateEstimations { + priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, + normal_buckets: vec![ + FeerateBucket { feerate: normal, estimated_seconds: self.feerate_to_time(normal) }, + FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + ], + low_buckets: vec![FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }], + } + } +} + +#[derive(Clone, Debug)] +pub struct FeeEstimateVerbose { + pub estimations: FeerateEstimations, + + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + use itertools::Itertools; + + #[test] + fn test_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; + let estimations = estimator.calc_estimations(1.0); + let buckets = estimations.ordered_buckets(); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + } + dbg!(estimations); + } + + #[test] + fn test_min_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + println!("{estimations}"); + let buckets = estimations.ordered_buckets(); + assert!(buckets.last().unwrap().feerate >= minimum_feerate); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + assert!(i.estimated_seconds <= j.estimated_seconds); + } + } + + #[test] + fn test_zero_values() { + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.1 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(estimator.inclusion_interval, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.1, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + } +} diff --git a/mining/src/lib.rs b/mining/src/lib.rs index 2986577ef..745fb63f9 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -8,12 +8,17 @@ use mempool::tx::Priority; mod block_template; pub(crate) mod cache; pub mod errors; +pub mod feerate; pub mod manager; mod manager_tests; pub mod mempool; pub mod model; pub mod monitor; +// Exposed for benchmarks +pub use block_template::{policy::Policy, selector::RebalancingWeightedTransactionSelector}; +pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, search_tree::SearchTree, Frontier}; + #[cfg(test)] pub mod testutils; diff --git a/mining/src/manager.rs b/mining/src/manager.rs index 4350401e6..5603d8d4d 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -2,27 +2,31 @@ use crate::{ block_template::{builder::BlockTemplateBuilder, errors::BuilderError}, cache::BlockTemplateCache, errors::MiningManagerResult, + feerate::{FeeEstimateVerbose, FeerateEstimations, FeerateEstimatorArgs}, mempool::{ config::Config, - model::tx::{MempoolTransaction, TxRemovalReason}, + model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, populate_entries_and_try_validate::{ populate_mempool_transactions_in_parallel, validate_mempool_transaction, validate_mempool_transactions_in_parallel, }, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, Mempool, }, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, topological_sort::IntoIterTopologically, + tx_insert::TransactionInsertion, tx_query::TransactionQuery, }, MempoolCountersSnapshot, MiningCounters, P2pTxCountSample, }; use itertools::Itertools; use kaspa_consensus_core::{ - api::ConsensusApi, - block::{BlockTemplate, TemplateBuildMode}, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, errors::{block::RuleError as BlockRuleError, tx::TxRuleError}, tx::{MutableTransaction, Transaction, TransactionId, TransactionOutput}, @@ -94,12 +98,7 @@ impl MiningManager { } // Miner data is new -- make the minimum changes required // Note the call returns a modified clone of the cached block template - let block_template = BlockTemplateBuilder::modify_block_template( - consensus, - miner_data, - &immutable_template, - self.storage_mass_activation_daa_score, - )?; + let block_template = BlockTemplateBuilder::modify_block_template(consensus, miner_data, &immutable_template)?; // No point in updating cache since we have no reason to believe this coinbase will be used more // than the previous one, and we want to maintain the original template caching time @@ -116,14 +115,14 @@ impl MiningManager { loop { attempts += 1; - let transactions = self.block_candidate_transactions(); - let block_template_builder = BlockTemplateBuilder::new(self.config.maximum_mass_per_block); + let selector = self.build_selector(); + let block_template_builder = BlockTemplateBuilder::new(); let build_mode = if attempts < self.config.maximum_build_block_template_attempts { TemplateBuildMode::Standard } else { TemplateBuildMode::Infallible }; - match block_template_builder.build_block_template(consensus, miner_data, transactions, build_mode) { + match block_template_builder.build_block_template(consensus, miner_data, selector, build_mode) { Ok(block_template) => { let block_template = cache_lock.set_immutable_cached_template(block_template); match attempts { @@ -206,8 +205,62 @@ impl MiningManager { } } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - self.mempool.read().block_candidate_transactions() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.mempool.read().build_selector() + } + + /// Returns realtime feerate estimations based on internal mempool state + pub(crate) fn get_realtime_feerate_estimations(&self) -> FeerateEstimations { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let estimator = self.mempool.read().build_feerate_estimator(args); + estimator.calc_estimations(self.config.minimum_feerate()) + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub(crate) fn get_realtime_feerate_estimations_verbose( + &self, + consensus: &dyn ConsensusApi, + prefix: kaspa_addresses::Prefix, + ) -> MiningManagerResult { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let network_mass_per_second = args.network_mass_per_second(); + let mempool_read = self.mempool.read(); + let estimator = mempool_read.build_feerate_estimator(args); + let ready_transactions_count = mempool_read.ready_transaction_count(); + let ready_transaction_total_mass = mempool_read.ready_transaction_total_mass(); + drop(mempool_read); + let mut resp = FeeEstimateVerbose { + estimations: estimator.calc_estimations(self.config.minimum_feerate()), + network_mass_per_second, + mempool_ready_transactions_count: ready_transactions_count as u64, + mempool_ready_transactions_total_mass: ready_transaction_total_mass, + + next_block_template_feerate_min: -1.0, + next_block_template_feerate_median: -1.0, + next_block_template_feerate_max: -1.0, + }; + // calculate next_block_template_feerate_xxx + { + let script_public_key = kaspa_txscript::pay_to_address_script(&kaspa_addresses::Address::new( + prefix, + kaspa_addresses::Version::PubKey, + &[0u8; 32], + )); + let miner_data: MinerData = MinerData::new(script_public_key, vec![]); + + let BlockTemplate { block: kaspa_consensus_core::block::MutableBlock { transactions, .. }, calculated_fees, .. } = + self.get_block_template(consensus, &miner_data)?; + + let Some(Stats { max, median, min }) = feerate_stats(transactions, calculated_fees) else { + return Ok(resp); + }; + + resp.next_block_template_feerate_max = max; + resp.next_block_template_feerate_min = min; + resp.next_block_template_feerate_median = median; + } + Ok(resp) } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. @@ -218,54 +271,65 @@ impl MiningManager { #[cfg(test)] pub(crate) fn block_template_builder(&self) -> BlockTemplateBuilder { - BlockTemplateBuilder::new(self.config.maximum_mass_per_block) + BlockTemplateBuilder::new() } /// validate_and_insert_transaction validates the given transaction, and /// adds it to the set of known transactions that have not yet been /// added to any block. /// - /// The returned transactions are clones of objects owned by the mempool. + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// + /// On success, returns transactions that where unorphaned following the insertion + /// of the provided transaction. + /// + /// The returned transactions are references of objects owned by the mempool. pub fn validate_and_insert_transaction( &self, consensus: &dyn ConsensusApi, transaction: Transaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { - self.validate_and_insert_mutable_transaction(consensus, MutableTransaction::from_tx(transaction), priority, orphan) + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { + self.validate_and_insert_mutable_transaction(consensus, MutableTransaction::from_tx(transaction), priority, orphan, rbf_policy) } - /// Exposed only for tests. Ordinary users should call `validate_and_insert_transaction` instead - pub fn validate_and_insert_mutable_transaction( + /// Exposed for tests only + /// + /// See `validate_and_insert_transaction` + pub(crate) fn validate_and_insert_mutable_transaction( &self, consensus: &dyn ConsensusApi, transaction: MutableTransaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { // read lock on mempool - let mut transaction = self.mempool.read().pre_validate_and_populate_transaction(consensus, transaction)?; + let TransactionPreValidation { mut transaction, feerate_threshold } = + self.mempool.read().pre_validate_and_populate_transaction(consensus, transaction, rbf_policy)?; + let args = TransactionValidationArgs::new(feerate_threshold); // no lock on mempool - let validation_result = validate_mempool_transaction(consensus, &mut transaction); + let validation_result = validate_mempool_transaction(consensus, &mut transaction, &args); // write lock on mempool let mut mempool = self.mempool.write(); - if let Some(accepted_transaction) = - mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan)? - { - let unorphaned_transactions = mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction); - drop(mempool); - - // The capacity used here may be exceeded since accepted unorphaned transaction may themselves unorphan other transactions. - let mut accepted_transactions = Vec::with_capacity(unorphaned_transactions.len() + 1); - // We include the original accepted transaction as well - accepted_transactions.push(accepted_transaction); - accepted_transactions.extend(self.validate_and_insert_unorphaned_transactions(consensus, unorphaned_transactions)); - self.counters.increase_tx_counts(1, priority); - - Ok(accepted_transactions) - } else { - Ok(vec![]) + match mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan, rbf_policy)? { + TransactionPostValidation { removed, accepted: Some(accepted_transaction) } => { + let unorphaned_transactions = mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction); + drop(mempool); + + // The capacity used here may be exceeded since accepted unorphaned transaction may themselves unorphan other transactions. + let mut accepted_transactions = Vec::with_capacity(unorphaned_transactions.len() + 1); + // We include the original accepted transaction as well + accepted_transactions.push(accepted_transaction); + accepted_transactions.extend(self.validate_and_insert_unorphaned_transactions(consensus, unorphaned_transactions)); + self.counters.increase_tx_counts(1, priority); + + Ok(TransactionInsertion::new(removed, accepted_transactions)) + } + TransactionPostValidation { removed, accepted: None } => Ok(TransactionInsertion::new(removed, vec![])), } } @@ -276,6 +340,9 @@ impl MiningManager { ) -> Vec> { // The capacity used here may be exceeded (see next comment). let mut accepted_transactions = Vec::with_capacity(incoming_transactions.len()); + // The validation args map is immutably empty since unorphaned transactions do not require pre processing so there + // are no feerate thresholds to use. Instead, we rely on this being checked during post processing. + let args = TransactionValidationBatchArgs::new(); // We loop as long as incoming unorphaned transactions do unorphan other transactions when they // get validated and inserted into the mempool. while !incoming_transactions.is_empty() { @@ -290,8 +357,11 @@ impl MiningManager { let mut validation_results = Vec::with_capacity(transactions.len()); while let Some(upper_bound) = self.next_transaction_chunk_upper_bound(&transactions, lower_bound) { assert!(lower_bound < upper_bound, "the chunk is never empty"); - validation_results - .extend(validate_mempool_transactions_in_parallel(consensus, &mut transactions[lower_bound..upper_bound])); + validation_results.extend(validate_mempool_transactions_in_parallel( + consensus, + &mut transactions[lower_bound..upper_bound], + &args, + )); lower_bound = upper_bound; } assert_eq!(transactions.len(), validation_results.len(), "every transaction should have a matching validation result"); @@ -304,19 +374,21 @@ impl MiningManager { .zip(validation_results) .flat_map(|((transaction, priority), validation_result)| { let orphan_id = transaction.id(); + let rbf_policy = Mempool::get_orphan_transaction_rbf_policy(priority); match mempool.post_validate_and_insert_transaction( consensus, validation_result, transaction, priority, Orphan::Forbidden, + rbf_policy, ) { - Ok(Some(accepted_transaction)) => { + Ok(TransactionPostValidation { removed: _, accepted: Some(accepted_transaction) }) => { accepted_transactions.push(accepted_transaction.clone()); self.counters.increase_tx_counts(1, priority); mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction) } - Ok(None) => vec![], + Ok(TransactionPostValidation { removed: _, accepted: None }) => vec![], Err(err) => { debug!("Failed to unorphan transaction {0} due to rule error: {1}", orphan_id, err); vec![] @@ -332,14 +404,18 @@ impl MiningManager { /// Validates a batch of transactions, handling iteratively only the independent ones, and /// adds those to the set of known transactions that have not yet been added to any block. /// + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// /// Returns transactions that where unorphaned following the insertion of the provided - /// transactions. The returned transactions are clones of objects owned by the mempool. + /// transactions. The returned transactions are references of objects owned by the mempool. pub fn validate_and_insert_transaction_batch( &self, consensus: &dyn ConsensusApi, transactions: Vec, priority: Priority, orphan: Orphan, + rbf_policy: RbfPolicy, ) -> Vec>> { const TRANSACTION_CHUNK_SIZE: usize = 250; @@ -353,12 +429,18 @@ impl MiningManager { // read lock on mempool // Here, we simply log and drop all erroneous transactions since the caller doesn't care about those anyway let mut transactions = Vec::with_capacity(sorted_transactions.len()); + let mut args = TransactionValidationBatchArgs::new(); for chunk in &sorted_transactions.chunks(TRANSACTION_CHUNK_SIZE) { let mempool = self.mempool.read(); let txs = chunk.filter_map(|tx| { let transaction_id = tx.id(); - match mempool.pre_validate_and_populate_transaction(consensus, tx) { - Ok(tx) => Some(tx), + match mempool.pre_validate_and_populate_transaction(consensus, tx, rbf_policy) { + Ok(TransactionPreValidation { transaction, feerate_threshold }) => { + if let Some(threshold) = feerate_threshold { + args.set_feerate_threshold(transaction.id(), threshold); + } + Some(transaction) + } Err(RuleError::RejectAlreadyAccepted(transaction_id)) => { debug!("Ignoring already accepted transaction {}", transaction_id); None @@ -387,8 +469,11 @@ impl MiningManager { let mut validation_results = Vec::with_capacity(transactions.len()); while let Some(upper_bound) = self.next_transaction_chunk_upper_bound(&transactions, lower_bound) { assert!(lower_bound < upper_bound, "the chunk is never empty"); - validation_results - .extend(validate_mempool_transactions_in_parallel(consensus, &mut transactions[lower_bound..upper_bound])); + validation_results.extend(validate_mempool_transactions_in_parallel( + consensus, + &mut transactions[lower_bound..upper_bound], + &args, + )); lower_bound = upper_bound; } assert_eq!(transactions.len(), validation_results.len(), "every transaction should have a matching validation result"); @@ -399,13 +484,20 @@ impl MiningManager { let mut mempool = self.mempool.write(); let txs = chunk.flat_map(|(transaction, validation_result)| { let transaction_id = transaction.id(); - match mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan) { - Ok(Some(accepted_transaction)) => { + match mempool.post_validate_and_insert_transaction( + consensus, + validation_result, + transaction, + priority, + orphan, + rbf_policy, + ) { + Ok(TransactionPostValidation { removed: _, accepted: Some(accepted_transaction) }) => { insert_results.push(Ok(accepted_transaction.clone())); self.counters.increase_tx_counts(1, priority); mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction) } - Ok(None) => { + Ok(TransactionPostValidation { removed: _, accepted: None }) | Err(RuleError::RejectDuplicate(_)) => { // Either orphaned or already existing in the mempool vec![] } @@ -769,35 +861,60 @@ impl MiningManagerProxy { consensus.clone().spawn_blocking(move |c| self.inner.get_block_template(c, &miner_data)).await } + /// Returns realtime feerate estimations based on internal mempool state + pub async fn get_realtime_feerate_estimations(self) -> FeerateEstimations { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations()).await.unwrap() + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub async fn get_realtime_feerate_estimations_verbose( + self, + consensus: &ConsensusProxy, + prefix: kaspa_addresses::Prefix, + ) -> MiningManagerResult { + consensus.clone().spawn_blocking(move |c| self.inner.get_realtime_feerate_estimations_verbose(c, prefix)).await + } + /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// - /// The returned transactions are clones of objects owned by the mempool. + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// + /// The returned transactions are references of objects owned by the mempool. pub async fn validate_and_insert_transaction( self, consensus: &ConsensusProxy, transaction: Transaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { - consensus.clone().spawn_blocking(move |c| self.inner.validate_and_insert_transaction(c, transaction, priority, orphan)).await + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { + consensus + .clone() + .spawn_blocking(move |c| self.inner.validate_and_insert_transaction(c, transaction, priority, orphan, rbf_policy)) + .await } /// Validates a batch of transactions, handling iteratively only the independent ones, and /// adds those to the set of known transactions that have not yet been added to any block. /// + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// /// Returns transactions that where unorphaned following the insertion of the provided - /// transactions. The returned transactions are clones of objects owned by the mempool. + /// transactions. The returned transactions are references of objects owned by the mempool. pub async fn validate_and_insert_transaction_batch( self, consensus: &ConsensusProxy, transactions: Vec, priority: Priority, orphan: Orphan, + rbf_policy: RbfPolicy, ) -> Vec>> { consensus .clone() - .spawn_blocking(move |c| self.inner.validate_and_insert_transaction_batch(c, transactions, priority, orphan)) + .spawn_blocking(move |c| self.inner.validate_and_insert_transaction_batch(c, transactions, priority, orphan, rbf_policy)) .await } @@ -902,3 +1019,103 @@ impl MiningManagerProxy { count } } + +/// Represents statistical information about fee rates of transactions. +struct Stats { + /// The maximum fee rate observed. + max: f64, + /// The median fee rate observed. + median: f64, + /// The minimum fee rate observed. + min: f64, +} +/// Calculates the maximum, median, and minimum fee rates (fee per unit mass) +/// for a set of transactions, excluding the first transaction which is assumed +/// to be the coinbase transaction. +/// +/// # Arguments +/// +/// * `transactions` - A vector of `Transaction` objects. The first transaction +/// is assumed to be the coinbase transaction and is excluded from fee rate +/// calculations. +/// * `calculated_fees` - A vector of fees associated with the transactions. +/// This vector should have one less element than the `transactions` vector +/// since the first transaction (coinbase) does not have a fee. +/// +/// # Returns +/// +/// Returns an `Option` containing the maximum, median, and minimum fee +/// rates if the input vectors are valid. Returns `None` if the vectors are +/// empty or if the lengths are inconsistent. +fn feerate_stats(transactions: Vec, calculated_fees: Vec) -> Option { + if calculated_fees.is_empty() { + return None; + } + if transactions.len() != calculated_fees.len() + 1 { + error!( + "[feerate_stats] block template transactions length ({}) is expected to be one more than `calculated_fees` length ({})", + transactions.len(), + calculated_fees.len() + ); + return None; + } + debug_assert!(transactions[0].is_coinbase()); + let mut feerates = calculated_fees + .into_iter() + .zip(transactions + .iter() + // skip coinbase tx + .skip(1) + .map(Transaction::mass)) + .map(|(fee, mass)| fee as f64 / mass as f64) + .collect_vec(); + feerates.sort_unstable_by(f64::total_cmp); + + let max = feerates[feerates.len() - 1]; + let min = feerates[0]; + let median = feerates[feerates.len() / 2]; + + Some(Stats { max, median, min }) +} + +#[cfg(test)] +mod tests { + use super::*; + use kaspa_consensus_core::subnets; + use std::iter::repeat; + + fn transactions(length: usize) -> Vec { + let tx = || { + let tx = Transaction::new(0, vec![], vec![], 0, Default::default(), 0, vec![]); + tx.set_mass(2); + tx + }; + let mut txs = repeat(tx()).take(length).collect_vec(); + txs[0].subnetwork_id = subnets::SUBNETWORK_ID_COINBASE; + txs + } + + #[test] + fn feerate_stats_test() { + let calculated_fees = vec![100u64, 200, 300, 400]; + let txs = transactions(calculated_fees.len() + 1); + let Stats { max, median, min } = feerate_stats(txs, calculated_fees).unwrap(); + assert_eq!(max, 200.0); + assert_eq!(median, 150.0); + assert_eq!(min, 50.0); + } + + #[test] + fn feerate_stats_empty_test() { + let calculated_fees = vec![]; + let txs = transactions(calculated_fees.len() + 1); + assert!(feerate_stats(txs, calculated_fees).is_none()); + } + + #[test] + fn feerate_stats_inconsistent_test() { + let calculated_fees = vec![100u64, 200, 300, 400]; + let txs = transactions(calculated_fees.len()); + assert!(feerate_stats(txs, calculated_fees).is_none()); + } +} diff --git a/mining/src/manager_tests.rs b/mining/src/manager_tests.rs index f919a3654..97161d920 100644 --- a/mining/src/manager_tests.rs +++ b/mining/src/manager_tests.rs @@ -7,19 +7,21 @@ mod tests { mempool::{ config::{Config, DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE}, errors::RuleError, - tx::{Orphan, Priority}, + model::frontier::selectors::TakeAllSelector, + tx::{Orphan, Priority, RbfPolicy}, }, - model::{candidate_tx::CandidateTransaction, tx_query::TransactionQuery}, + model::tx_query::TransactionQuery, testutils::consensus_mock::ConsensusMock, MiningCounters, }; + use itertools::Itertools; use kaspa_addresses::{Address, Prefix, Version}; use kaspa_consensus_core::{ api::ConsensusApi, block::TemplateBuildMode, coinbase::MinerData, constants::{MAX_TX_IN_SEQUENCE_NUM, SOMPI_PER_KASPA, TX_VERSION}, - errors::tx::{TxResult, TxRuleError}, + errors::tx::TxRuleError, mass::transaction_estimated_serialized_size, subnets::SUBNETWORK_ID_NATIVE, tx::{ @@ -28,11 +30,12 @@ mod tests { }, }; use kaspa_hashes::Hash; + use kaspa_mining_errors::mempool::RuleResult; use kaspa_txscript::{ pay_to_address_script, pay_to_script_hash_signature_script, - test_helpers::{create_transaction, op_true_script}, + test_helpers::{create_transaction, create_transaction_with_change, op_true_script}, }; - use std::sync::Arc; + use std::{iter::once, sync::Arc}; use tokio::sync::mpsc::{error::TryRecvError, unbounded_channel}; const TARGET_TIME_PER_BLOCK: u64 = 1_000; @@ -42,72 +45,106 @@ mod tests { #[test] fn test_validate_and_insert_transaction() { const TX_COUNT: u32 = 10; - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transactions_to_insert = (0..TX_COUNT).map(|i| create_transaction_with_utxo_entry(i, 0)).collect::>(); - for transaction in transactions_to_insert.iter() { - let result = mining_manager.validate_and_insert_mutable_transaction( - consensus.as_ref(), - transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "inserting a valid transaction failed"); - } - // The UtxoEntry was filled manually for those transactions, so the transactions won't be considered orphans. - // Therefore, all the transactions expected to be contained in the mempool. - let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); - assert_eq!( - transactions_to_insert.len(), - transactions_from_pool.len(), - "wrong number of transactions in mempool: expected: {}, got: {}", - transactions_to_insert.len(), - transactions_from_pool.len() - ); - transactions_to_insert.iter().for_each(|tx_to_insert| { - let found_exact_match = transactions_from_pool.contains(tx_to_insert); - let tx_from_pool = transactions_from_pool.iter().find(|tx_from_pool| tx_from_pool.id() == tx_to_insert.id()); - let found_transaction_id = tx_from_pool.is_some(); - if found_transaction_id && !found_exact_match { - let tx = tx_from_pool.unwrap(); - assert_eq!( - tx_to_insert.calculated_fee.unwrap(), - tx.calculated_fee.unwrap(), - "wrong fee in transaction {}: expected: {}, got: {}", - tx.id(), - tx_to_insert.calculated_fee.unwrap(), - tx.calculated_fee.unwrap() - ); - assert_eq!( - tx_to_insert.calculated_compute_mass.unwrap(), - tx.calculated_compute_mass.unwrap(), - "wrong mass in transaction {}: expected: {}, got: {}", - tx.id(), - tx_to_insert.calculated_compute_mass.unwrap(), - tx.calculated_compute_mass.unwrap() - ); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + let transactions_to_insert = (0..TX_COUNT).map(|i| create_transaction_with_utxo_entry(i, 0)).collect::>(); + for transaction in transactions_to_insert.iter() { + let result = into_mempool_result(mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy, + )); + match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => { + assert!(result.is_ok(), "({priority:?}, {orphan:?}, {rbf_policy:?}) inserting a valid transaction failed"); + } + RbfPolicy::Mandatory => { + assert!(result.is_err(), "({priority:?}, {orphan:?}, {rbf_policy:?}) replacing a valid transaction without replacement in mempool should fail"); + let err = result.unwrap_err(); + assert_eq!( + RuleError::RejectRbfNoDoubleSpend, + err, + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong error: expected {} got: {}", + RuleError::RejectRbfNoDoubleSpend, + err, + ); + } + } } - assert!(found_exact_match, "missing transaction {} in the mempool, no exact match", tx_to_insert.id()); - }); - // The parent's transaction was inserted into the consensus, so we want to verify that - // the child transaction is not considered an orphan and inserted into the mempool. - let transaction_not_an_orphan = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction_not_an_orphan.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "inserting the child transaction {} into the mempool failed", transaction_not_an_orphan.id()); - let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); - assert!( - contained_by(transaction_not_an_orphan.id(), &transactions_from_pool), - "missing transaction {} in the mempool", - transaction_not_an_orphan.id() - ); + // The UtxoEntry was filled manually for those transactions, so the transactions won't be considered orphans. + // Therefore, all the transactions expected to be contained in the mempool if replace by fee policy allowed it. + let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); + let transactions_inserted = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => transactions_to_insert.clone(), + RbfPolicy::Mandatory => { + vec![] + } + }; + assert_eq!( + transactions_inserted.len(), + transactions_from_pool.len(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong number of transactions in mempool: expected: {}, got: {}", + transactions_inserted.len(), + transactions_from_pool.len() + ); + transactions_inserted.iter().for_each(|tx_to_insert| { + let found_exact_match = transactions_from_pool.contains(tx_to_insert); + let tx_from_pool = transactions_from_pool.iter().find(|tx_from_pool| tx_from_pool.id() == tx_to_insert.id()); + let found_transaction_id = tx_from_pool.is_some(); + if found_transaction_id && !found_exact_match { + let tx = tx_from_pool.unwrap(); + assert_eq!( + tx_to_insert.calculated_fee.unwrap(), + tx.calculated_fee.unwrap(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong fee in transaction {}: expected: {}, got: {}", + tx.id(), + tx_to_insert.calculated_fee.unwrap(), + tx.calculated_fee.unwrap() + ); + assert_eq!( + tx_to_insert.calculated_compute_mass.unwrap(), + tx.calculated_compute_mass.unwrap(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong mass in transaction {}: expected: {}, got: {}", + tx.id(), + tx_to_insert.calculated_compute_mass.unwrap(), + tx.calculated_compute_mass.unwrap() + ); + } + assert!( + found_exact_match, + "({priority:?}, {orphan:?}, {rbf_policy:?}) missing transaction {} in the mempool, no exact match", + tx_to_insert.id() + ); + }); + + // The parent's transaction was inserted into the consensus, so we want to verify that + // the child transaction is not considered an orphan and inserted into the mempool. + let transaction_not_an_orphan = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction_not_an_orphan.clone(), + priority, + orphan, + RbfPolicy::Forbidden, + ); + assert!( + result.is_ok(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) inserting the child transaction {} into the mempool failed", + transaction_not_an_orphan.id() + ); + let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); + assert!( + contained_by(transaction_not_an_orphan.id(), &transactions_from_pool), + "({priority:?}, {orphan:?}, {rbf_policy:?}) missing transaction {} in the mempool", + transaction_not_an_orphan.id() + ); + } } /// test_simulated_error_in_consensus verifies that a predefined result is actually @@ -115,127 +152,397 @@ mod tests { /// insert a transaction. #[test] fn test_simulated_error_in_consensus() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - - // Build an invalid transaction with some gas and inform the consensus mock about the result it should return - // when the mempool will submit this transaction for validation. - let mut transaction = create_transaction_with_utxo_entry(0, 1); - Arc::make_mut(&mut transaction.tx).gas = 1000; - let status = Err(TxRuleError::TxHasGas); - consensus.set_status(transaction.id(), status.clone()); - - // Try validate and insert the transaction into the mempool - let result = into_status(mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction.tx.as_ref().clone(), - Priority::Low, - Orphan::Allowed, - )); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + + // Build an invalid transaction with some gas and inform the consensus mock about the result it should return + // when the mempool will submit this transaction for validation. + let mut transaction = create_transaction_with_utxo_entry(0, 1); + Arc::make_mut(&mut transaction.tx).gas = 1000; + let tx_err = TxRuleError::TxHasGas; + let expected = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => Err(RuleError::from(tx_err.clone())), + RbfPolicy::Mandatory => Err(RuleError::RejectRbfNoDoubleSpend), + }; + consensus.set_status(transaction.id(), Err(tx_err)); + + // Try validate and insert the transaction into the mempool + let result = into_mempool_result(mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy, + )); - assert_eq!( - status, result, - "Unexpected result when trying to insert an invalid transaction: expected: {status:?}, got: {result:?}", - ); - let pool_tx = mining_manager.get_transaction(&transaction.id(), TransactionQuery::All); - assert!(pool_tx.is_none(), "Mempool contains a transaction that should have been rejected"); + assert_eq!( + expected, result, + "({priority:?}, {orphan:?}, {rbf_policy:?}) unexpected result when trying to insert an invalid transaction: expected: {expected:?}, got: {result:?}", + ); + let pool_tx = mining_manager.get_transaction(&transaction.id(), TransactionQuery::All); + assert!( + pool_tx.is_none(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) mempool contains a transaction that should have been rejected" + ); + } } /// test_insert_double_transactions_to_mempool verifies that an attempt to insert a transaction /// more than once into the mempool will result in raising an appropriate error. #[test] fn test_insert_double_transactions_to_mempool() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transaction = create_transaction_with_utxo_entry(0, 0); + let transaction = create_transaction_with_utxo_entry(0, 0); - // submit the transaction to the mempool - let result = mining_manager.validate_and_insert_mutable_transaction( - consensus.as_ref(), - transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "mempool should have accepted a valid transaction but did not"); - - // submit the same transaction again to the mempool - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction.tx.as_ref().clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_err(), "mempool should refuse a double submit of the same transaction but accepts it"); - if let Err(MiningManagerError::MempoolError(RuleError::RejectDuplicate(transaction_id))) = result { - assert_eq!( - transaction.id(), - transaction_id, - "the error returned by the mempool should include id {} but provides {}", - transaction.id(), - transaction_id + // submit the transaction to the mempool + let result = mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy.for_insert(), ); - } else { - panic!( - "the nested error returned by the mempool should be variant RuleError::RejectDuplicate but is {:?}", - result.err().unwrap() + assert!( + result.is_ok(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should have accepted a valid transaction but did not" ); + + // submit the same transaction again to the mempool + let result = into_mempool_result(mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.tx.as_ref().clone(), + priority, + orphan, + rbf_policy, + )); + match result { + Err(RuleError::RejectDuplicate(transaction_id)) => { + assert_eq!( + transaction.id(), + transaction_id, + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should include transaction id {} but provides {}", + transaction.id(), + transaction_id + ); + } + Err(err) => { + panic!( + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should be {:?} but is {err:?}", + RuleError::RejectDuplicate(transaction.id()) + ); + } + Ok(()) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should refuse a double submit of the same transaction but accepts it"); + } + } } } - // test_double_spend_in_mempool verifies that an attempt to insert a transaction double-spending - // another transaction already in the mempool will result in raising an appropriate error. + /// test_double_spend_in_mempool verifies that an attempt to insert a transaction double-spending + /// another transaction already in the mempool will result in raising an appropriate error. #[test] fn test_double_spend_in_mempool() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transaction = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); - assert!( - consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), - "the consensus mock should have spendable UTXOs for the newly created transaction {}", - transaction.id() - ); + let transaction = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), + "({priority:?}, {orphan:?}, {rbf_policy:?}) the consensus mock should have spendable UTXOs for the newly created transaction {}", + transaction.id() + ); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), transaction.clone(), Priority::Low, Orphan::Allowed); - assert!(result.is_ok(), "the mempool should accept a valid transaction when it is able to populate its UTXO entries"); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + RbfPolicy::Forbidden, + ); + assert!(result.is_ok(), "({priority:?}, {orphan:?}, {rbf_policy:?}) the mempool should accept a valid transaction when it is able to populate its UTXO entries"); - let mut double_spending_transaction = transaction.clone(); - double_spending_transaction.outputs[0].value -= 1; // do some minor change so that txID is different - double_spending_transaction.finalize(); - assert_ne!( - transaction.id(), - double_spending_transaction.id(), - "two transactions differing by only one output value should have different ids" - ); - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - double_spending_transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_err(), "mempool should refuse a double spend transaction but accepts it"); - if let Err(MiningManagerError::MempoolError(RuleError::RejectDoubleSpendInMempool(_, transaction_id))) = result { - assert_eq!( - transaction.id(), - transaction_id, - "the error returned by the mempool should include id {} but provides {}", + let mut double_spending_transaction = transaction.clone(); + double_spending_transaction.outputs[0].value += 1; // do some minor change so that txID is different while not increasing fee + double_spending_transaction.finalize(); + assert_ne!( transaction.id(), - transaction_id - ); - } else { - panic!( - "the nested error returned by the mempool should be variant RuleError::RejectDoubleSpendInMempool but is {:?}", - result.err().unwrap() + double_spending_transaction.id(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) two transactions differing by only one output value should have different ids" ); + let result = into_mempool_result(mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + double_spending_transaction.clone(), + priority, + orphan, + rbf_policy, + )); + match result { + Err(RuleError::RejectDoubleSpendInMempool(_, transaction_id)) => { + assert_eq!( + transaction.id(), + transaction_id, + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should include id {} but provides {}", + transaction.id(), + transaction_id + ); + } + Err(err) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should be RuleError::RejectDoubleSpendInMempool but is {err:?}"); + } + Ok(()) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should refuse a double spend transaction ineligible to RBF but accepts it"); + } + } + } + } + + /// test_replace_by_fee_in_mempool verifies that an attempt to insert a double-spending transaction + /// will cause or not the transaction(s) double spending in the mempool to be replaced/removed, + /// depending on varying factors. + #[test] + fn test_replace_by_fee_in_mempool() { + const BASE_FEE: u64 = DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE; + + struct TxOp { + /// Funding transaction indexes + tx: Vec, + /// Funding transaction output indexes + output: Vec, + /// Add a change output to the transaction + change: bool, + /// Transaction fee + fee: u64, + /// Children binary tree depth + depth: usize, + } + + impl TxOp { + fn change(&self) -> Option { + self.change.then_some(900 * SOMPI_PER_KASPA) + } + } + + struct Test { + name: &'static str, + /// Initial transactions in the mempool + starts: Vec, + /// Replacement transaction submitted to the mempool + replacement: TxOp, + /// Expected RBF result for the 3 policies [Forbidden, Allowed, Mandatory] + expected: [bool; 3], + } + + impl Test { + fn run_rbf(&self, rbf_policy: RbfPolicy, expected: bool) { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + let funding_transactions = create_and_add_funding_transactions(&consensus, 10); + + // RPC submit the initial transactions + let (transactions, children): (Vec<_>, Vec<_>) = + self.starts + .iter() + .map(|tx_op| { + let transaction = create_funded_transaction( + select_transactions(&funding_transactions, &tx_op.tx), + tx_op.output.clone(), + tx_op.change(), + tx_op.fee, + ); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), + "[{}, {:?}] the consensus should have spendable UTXOs for the newly created transaction {}", + self.name, rbf_policy, transaction.id() + ); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.clone(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); + assert!( + result.is_ok(), + "[{}, {:?}] the mempool should accept a valid transaction when it is able to populate its UTXO entries", + self.name, rbf_policy, + ); + let children = create_children_tree(&transaction, tx_op.depth); + let children_count = (2_usize.pow(tx_op.depth as u32) - 1) * transaction.outputs.len(); + assert_eq!( + children.len(), children_count, + "[{}, {:?}] a parent transaction with {} output(s) should generate a binary children tree of depth {} with {} children but got {}", + self.name, rbf_policy, transaction.outputs.len(), tx_op.depth, children_count, children.len(), + ); + validate_and_insert_transactions( + &mining_manager, + consensus.as_ref(), + children.iter(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); + (transaction, children) + }) + .unzip(); + + // RPC submit transaction replacement + let transaction_replacement = create_funded_transaction( + select_transactions(&funding_transactions, &self.replacement.tx), + self.replacement.output.clone(), + self.replacement.change(), + self.replacement.fee, + ); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction_replacement.clone())), + "[{}, {:?}] the consensus should have spendable UTXOs for the newly created transaction {}", + self.name, + rbf_policy, + transaction_replacement.id() + ); + let tx_count = mining_manager.transaction_count(TransactionQuery::TransactionsOnly); + let expected_tx_count = match expected { + true => tx_count + 1 - transactions.len() - children.iter().map(|x| x.len()).sum::(), + false => tx_count, + }; + let priority = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Mandatory => Priority::High, + RbfPolicy::Allowed => Priority::Low, + }; + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction_replacement.clone(), + priority, + Orphan::Forbidden, + rbf_policy, + ); + if expected { + assert!(result.is_ok(), "[{}, {:?}] mempool should accept a RBF transaction", self.name, rbf_policy,); + let tx_insertion = result.unwrap(); + assert_eq!( + tx_insertion.removed.as_ref().unwrap().id(), + transactions[0].id(), + "[{}, {:?}] RBF should return the removed transaction", + self.name, + rbf_policy, + ); + transactions.iter().for_each(|x| { + assert!( + !mining_manager.has_transaction(&x.id(), TransactionQuery::All), + "[{}, {:?}] RBF replaced transaction should no longer be in the mempool", + self.name, + rbf_policy, + ); + }); + assert_transaction_count( + &mining_manager, + expected_tx_count, + &format!( + "[{}, {:?}] RBF should remove all chained transactions of the removed mempool transaction(s)", + self.name, rbf_policy + ), + ); + } else { + assert!(result.is_err(), "[{}, {:?}] mempool should reject the RBF transaction", self.name, rbf_policy); + transactions.iter().for_each(|x| { + assert!( + mining_manager.has_transaction(&x.id(), TransactionQuery::All), + "[{}, {:?}] RBF transaction target is no longer in the mempool", + self.name, + rbf_policy + ); + }); + assert_transaction_count( + &mining_manager, + expected_tx_count, + &format!("[{}, {:?}] a failing RBF should leave the mempool unchanged", self.name, rbf_policy), + ); + } + } + + fn run(&self) { + [RbfPolicy::Forbidden, RbfPolicy::Allowed, RbfPolicy::Mandatory].iter().copied().enumerate().for_each( + |(i, rbf_policy)| { + self.run_rbf(rbf_policy, self.expected[i]); + }, + ) + } + } + + let tests = vec![ + Test { + name: "1 input, 1 output <=> 1 input, 1 output, constant fee", + starts: vec![TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }], + replacement: TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "1 input, 1 output <=> 1 input, 1 output, increased fee", + starts: vec![TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }], + replacement: TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 2 outputs, increased fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "4 inputs, 2 outputs <=> 2 inputs, 2 outputs, constant fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 1 output, constant fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 4 inputs, 2 output, constant fee (MUST FAIL on fee/mass)", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "2 inputs, 1 output <=> 4 inputs, 2 output, increased fee (MUST FAIL on fee/mass)", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: false, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE + 10, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 1 output, constant fee, partial double spend overlap", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 2], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "(2 inputs, 2 outputs) * 2 <=> 4 inputs, 2 outputs, increased fee, 2 double spending mempool transactions (MUST FAIL on Mandatory)", + starts: vec![ + TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }, + TxOp { tx: vec![0, 1], output: vec![1], change: true, fee: BASE_FEE, depth: 2 }, + ], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, false], + }, + ]; + + for test in tests { + test.run(); } } - // test_handle_new_block_transactions verifies that all the transactions in the block were successfully removed from the mempool. + /// test_handle_new_block_transactions verifies that all the transactions in the block were successfully removed from the mempool. #[test] fn test_handle_new_block_transactions() { let consensus = Arc::new(ConsensusMock::new()); @@ -250,6 +557,7 @@ mod tests { transaction.tx.as_ref().clone(), Priority::Low, Orphan::Allowed, + RbfPolicy::Forbidden, ); assert!(result.is_ok(), "the insertion of a new valid transaction in the mempool failed"); } @@ -295,8 +603,8 @@ mod tests { } #[test] - // test_double_spend_with_block verifies that any transactions which are now double spends as a result of the block's new transactions - // will be removed from the mempool. + /// test_double_spend_with_block verifies that any transactions which are now double spends as a result of the block's new transactions + /// will be removed from the mempool. fn test_double_spend_with_block() { let consensus = Arc::new(ConsensusMock::new()); let counters = Arc::new(MiningCounters::default()); @@ -308,6 +616,7 @@ mod tests { transaction_in_the_mempool.tx.as_ref().clone(), Priority::Low, Orphan::Allowed, + RbfPolicy::Forbidden, ); assert!(result.is_ok()); @@ -326,7 +635,7 @@ mod tests { ); } - // test_orphan_transactions verifies that a transaction could be a part of a new block template only if it's not an orphan. + /// test_orphan_transactions verifies that a transaction could be a part of a new block template only if it's not an orphan. #[test] fn test_orphan_transactions() { let consensus = Arc::new(ConsensusMock::new()); @@ -340,8 +649,13 @@ mod tests { assert_eq!(parent_txs.len(), TX_PAIRS_COUNT); assert_eq!(child_txs.len(), TX_PAIRS_COUNT); for orphan in child_txs.iter() { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), orphan.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + orphan.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid orphan transaction {}", orphan.id()); } let (populated_txs, orphans) = mining_manager.get_all_transactions(TransactionQuery::All); @@ -485,10 +799,15 @@ mod tests { ); // Add the remaining parent transaction into the mempool - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), parent_txs[0].clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + parent_txs[0].clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the insertion of the remaining parent transaction in the mempool failed"); - let unorphaned_txs = result.unwrap(); + let unorphaned_txs = result.unwrap().accepted; let (populated_txs, orphans) = mining_manager.get_all_transactions(TransactionQuery::All); assert_eq!( unorphaned_txs.len(), SKIPPED_TXS + 1, @@ -592,8 +911,13 @@ mod tests { // Try submit children while rejecting orphans for (tx, test) in child_txs.iter().zip(tests.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Forbidden); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Forbidden, + RbfPolicy::Forbidden, + ); assert!(result.is_err(), "mempool should reject an orphan transaction with {:?} when asked to do so", test.priority); if let Err(MiningManagerError::MempoolError(RuleError::RejectDisallowedOrphan(transaction_id))) = result { assert_eq!( @@ -613,8 +937,13 @@ mod tests { // Try submit children while accepting orphans for (tx, test) in child_txs.iter().zip(tests.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert_eq!( test.should_enter_orphan_pool, result.is_ok(), @@ -623,7 +952,7 @@ mod tests { test.insert_result() ); if let Ok(unorphaned_txs) = result { - assert!(unorphaned_txs.is_empty(), "mempool should unorphan no transaction since it only contains orphans"); + assert!(unorphaned_txs.accepted.is_empty(), "mempool should unorphan no transaction since it only contains orphans"); } else if let Err(MiningManagerError::MempoolError(RuleError::RejectOrphanPoolIsFull(pool_len, config_len))) = result { assert_eq!( (config.maximum_orphan_transaction_count as usize, config.maximum_orphan_transaction_count), @@ -642,10 +971,15 @@ mod tests { // Submit all the parents for (i, (tx, test)) in parent_txs.iter().zip(tests.iter()).enumerate() { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "mempool should accept a valid transaction with {:?} when asked to do so", test.priority,); - let unorphaned_txs = result.as_ref().unwrap(); + let unorphaned_txs = &result.as_ref().unwrap().accepted; assert_eq!( test.should_unorphan, unorphaned_txs.len() > 1, @@ -682,8 +1016,13 @@ mod tests { // Add to mempool a transaction that spends child_tx_2 (as high priority) let spending_tx = create_transaction(&child_tx_2, 1_000); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), spending_tx.clone(), Priority::High, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + spending_tx.clone(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the insertion in the mempool of the spending transaction failed"); // Revalidate, to make sure spending_tx is still valid @@ -725,7 +1064,7 @@ mod tests { assert!(orphan_txs.is_empty(), "orphan pool should be empty"); } - // test_modify_block_template verifies that modifying a block template changes coinbase data correctly. + /// test_modify_block_template verifies that modifying a block template changes coinbase data correctly. #[test] fn test_modify_block_template() { let consensus = Arc::new(ConsensusMock::new()); @@ -737,17 +1076,27 @@ mod tests { let (parent_txs, child_txs) = create_arrays_of_parent_and_children_transactions(&consensus, TX_PAIRS_COUNT); for (parent_tx, child_tx) in parent_txs.iter().zip(child_txs.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), parent_tx.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + parent_tx.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid parent transaction {}", parent_tx.id()); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), child_tx.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + child_tx.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid child transaction {}", parent_tx.id()); } // Collect all parent transactions for the next block template. // They are ready since they have no parents in the mempool. - let transactions = mining_manager.block_candidate_transactions(); + let transactions = mining_manager.build_selector().select_transactions(); assert_eq!( TX_PAIRS_COUNT, transactions.len(), @@ -755,7 +1104,7 @@ mod tests { ); parent_txs.iter().for_each(|x| { assert!( - transactions.iter().any(|tx| tx.tx.id() == x.id()), + transactions.iter().any(|tx| tx.id() == x.id()), "the parent transaction {} should be candidate for the next block template", x.id() ); @@ -771,8 +1120,9 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec, ) { + let transactions = transactions.into_iter().map(Arc::new).collect::>(); for _ in 0..4 { // Run a few times to get more randomness compare_modified_template_to_built( @@ -839,7 +1189,7 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec>, first_op: OpType, second_op: OpType, ) { @@ -848,7 +1198,12 @@ mod tests { // Build a fresh template for coinbase2 as a reference let builder = mining_manager.block_template_builder(); - let result = builder.build_block_template(consensus, &miner_data_2, transactions, TemplateBuildMode::Standard); + let result = builder.build_block_template( + consensus, + &miner_data_2, + Box::new(TakeAllSelector::new(transactions)), + TemplateBuildMode::Standard, + ); assert!(result.is_ok(), "build block template failed for miner data 2"); let expected_template = result.unwrap(); @@ -933,6 +1288,68 @@ mod tests { mutable_tx } + fn create_and_add_funding_transactions(consensus: &Arc, count: usize) -> Vec { + // Make the funding amounts always different so that funding txs have different ids + (0..count) + .map(|i| { + let funding_tx = create_transaction_without_input(vec![1_000 * SOMPI_PER_KASPA, 2_500 * SOMPI_PER_KASPA + i as u64]); + consensus.add_transaction(funding_tx.clone(), 1); + funding_tx + }) + .collect_vec() + } + + fn select_transactions<'a>(transactions: &'a [Transaction], indexes: &'a [usize]) -> impl Iterator { + indexes.iter().map(|i| &transactions[*i]) + } + + fn create_funded_transaction<'a>( + txs_to_spend: impl Iterator, + output_indexes: Vec, + change: Option, + fee: u64, + ) -> Transaction { + create_transaction_with_change(txs_to_spend, output_indexes, change, fee) + } + + fn create_children_tree(parent: &Transaction, depth: usize) -> Vec { + let mut tree = vec![]; + let root = [parent.clone()]; + let mut parents = &root[..]; + let mut first_child = 0; + for _ in 0..depth { + let mut children = vec![]; + for parent in parents { + children.extend(parent.outputs.iter().enumerate().map(|(i, output)| { + create_transaction_with_change( + once(parent), + vec![i], + Some(output.value / 2), + DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + ) + })); + } + tree.extend(children); + parents = &tree[first_child..]; + first_child = tree.len() + } + tree + } + + fn validate_and_insert_transactions<'a>( + mining_manager: &MiningManager, + consensus: &dyn ConsensusApi, + transactions: impl Iterator, + priority: Priority, + orphan: Orphan, + rbf_policy: RbfPolicy, + ) { + transactions.for_each(|transaction| { + let result = mining_manager.validate_and_insert_transaction(consensus, transaction.clone(), priority, orphan, rbf_policy); + assert!(result.is_ok(), "the mempool should accept a valid transaction when it is able to populate its UTXO entries"); + }); + } + fn create_arrays_of_parent_and_children_transactions( consensus: &Arc, count: usize, @@ -974,11 +1391,13 @@ mod tests { transactions.iter().any(|x| x.as_ref().id() == transaction_id) } - fn into_status(result: MiningManagerResult) -> TxResult<()> { + fn into_mempool_result(result: MiningManagerResult) -> RuleResult<()> { match result { Ok(_) => Ok(()), - Err(MiningManagerError::MempoolError(RuleError::RejectTxRule(err))) => Err(err), - _ => Ok(()), + Err(MiningManagerError::MempoolError(err)) => Err(err), + _ => { + panic!("result is an unsupported error"); + } } } @@ -1000,4 +1419,26 @@ mod tests { let script = pay_to_address_script(&address); MinerData::new(script, vec![]) } + + #[allow(dead_code)] + fn all_priority_orphan_combinations() -> impl Iterator { + [Priority::Low, Priority::High] + .iter() + .flat_map(|priority| [Orphan::Allowed, Orphan::Forbidden].iter().map(|orphan| (*priority, *orphan))) + } + + fn all_priority_orphan_rbf_policy_combinations() -> impl Iterator { + [Priority::Low, Priority::High].iter().flat_map(|priority| { + [Orphan::Allowed, Orphan::Forbidden].iter().flat_map(|orphan| { + [RbfPolicy::Forbidden, RbfPolicy::Allowed, RbfPolicy::Mandatory] + .iter() + .map(|rbf_policy| (*priority, *orphan, *rbf_policy)) + }) + }) + } + + fn assert_transaction_count(mining_manager: &MiningManager, expected_count: usize, message: &str) { + let count = mining_manager.transaction_count(TransactionQuery::TransactionsOnly); + assert_eq!(expected_count, count, "{message} mempool transaction count: expected {}, got {}", expected_count, count); + } } diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index aecbc0711..419a4362a 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -1,7 +1,6 @@ use kaspa_consensus_core::constants::TX_VERSION; -pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u64 = 1_000_000; -pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u64 = 50_000; +pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u32 = 1_000_000; pub(crate) const DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS: u64 = 5; pub(crate) const DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS: u64 = 60; @@ -29,8 +28,7 @@ pub(crate) const DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION: u16 = TX_VERSION; #[derive(Clone, Debug)] pub struct Config { - pub maximum_transaction_count: u64, - pub maximum_ready_transaction_count: u64, + pub maximum_transaction_count: u32, pub maximum_build_block_template_attempts: u64, pub transaction_expire_interval_daa_score: u64, pub transaction_expire_scan_interval_daa_score: u64, @@ -47,13 +45,13 @@ pub struct Config { pub minimum_relay_transaction_fee: u64, pub minimum_standard_transaction_version: u16, pub maximum_standard_transaction_version: u16, + pub network_blocks_per_second: u64, } impl Config { #[allow(clippy::too_many_arguments)] pub fn new( - maximum_transaction_count: u64, - maximum_ready_transaction_count: u64, + maximum_transaction_count: u32, maximum_build_block_template_attempts: u64, transaction_expire_interval_daa_score: u64, transaction_expire_scan_interval_daa_score: u64, @@ -70,10 +68,10 @@ impl Config { minimum_relay_transaction_fee: u64, minimum_standard_transaction_version: u16, maximum_standard_transaction_version: u16, + network_blocks_per_second: u64, ) -> Self { Self { maximum_transaction_count, - maximum_ready_transaction_count, maximum_build_block_template_attempts, transaction_expire_interval_daa_score, transaction_expire_scan_interval_daa_score, @@ -90,6 +88,7 @@ impl Config { minimum_relay_transaction_fee, minimum_standard_transaction_version, maximum_standard_transaction_version, + network_blocks_per_second, } } @@ -98,7 +97,6 @@ impl Config { pub const fn build_default(target_milliseconds_per_block: u64, relay_non_std_transactions: bool, max_block_mass: u64) -> Self { Self { maximum_transaction_count: DEFAULT_MAXIMUM_TRANSACTION_COUNT, - maximum_ready_transaction_count: DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT, maximum_build_block_template_attempts: DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS, transaction_expire_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS * 1000 / target_milliseconds_per_block, transaction_expire_scan_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_SCAN_INTERVAL_SECONDS * 1000 @@ -118,11 +116,18 @@ impl Config { minimum_relay_transaction_fee: DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, minimum_standard_transaction_version: DEFAULT_MINIMUM_STANDARD_TRANSACTION_VERSION, maximum_standard_transaction_version: DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION, + network_blocks_per_second: 1000 / target_milliseconds_per_block, } } pub fn apply_ram_scale(mut self, ram_scale: f64) -> Self { - self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u64; // Allow only scaling down + self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u32; // Allow only scaling down self } + + /// Returns the minimum standard fee/mass ratio currently required by the mempool + pub(crate) fn minimum_feerate(&self) -> f64 { + // The parameter minimum_relay_transaction_fee is in sompi/kg units so divide by 1000 to get sompi/gram + self.minimum_relay_transaction_fee as f64 / 1000.0 + } } diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index a3e26e7c9..e5cd7dbeb 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -1,6 +1,6 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, tx_query::TransactionQuery, }, @@ -12,7 +12,10 @@ use self::{ model::{accepted_transactions::AcceptedTransactions, orphan_pool::OrphanPool, pool::Pool, transactions_pool::TransactionsPool}, tx::Priority, }; -use kaspa_consensus_core::tx::{MutableTransaction, TransactionId}; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId}, +}; use kaspa_core::time::Stopwatch; use std::sync::Arc; @@ -23,6 +26,7 @@ pub(crate) mod handle_new_block_transactions; pub(crate) mod model; pub(crate) mod populate_entries_and_try_validate; pub(crate) mod remove_transaction; +pub(crate) mod replace_by_fee; pub(crate) mod validate_and_insert_transaction; /// Mempool contains transactions intended to be inserted into a block and mined. @@ -111,9 +115,23 @@ impl Mempool { count } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - let _sw = Stopwatch::<10>::with_threshold("block_candidate_transactions op"); - self.transaction_pool.all_ready_transactions() + pub(crate) fn ready_transaction_count(&self) -> usize { + self.transaction_pool.ready_transaction_count() + } + + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.transaction_pool.ready_transaction_total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + let _sw = Stopwatch::<10>::with_threshold("build_selector op"); + self.transaction_pool.build_selector() + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.transaction_pool.build_feerate_estimator(args) } pub(crate) fn all_transaction_ids_with_priority(&self, priority: Priority) -> Vec { @@ -158,4 +176,51 @@ pub mod tx { Forbidden, Allowed, } + + /// Replace by Fee (RBF) policy + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum RbfPolicy { + /// ### RBF is forbidden + /// + /// Inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - no double spend + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend error. + Forbidden, + + /// ### RBF may occur + /// + /// Identifies double spends in mempool and their owning transactions checking in order every input of the incoming + /// transaction. + /// + /// Removes all mempool transactions owning double spends and inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - on absence of double spends, always succeeds + /// - on double spends, the incoming transaction has a higher fee/mass ratio than the mempool transaction owning + /// the first double spend + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend or a tx fee/mass too low error. + Allowed, + + /// ### RBF must occur + /// + /// Identifies double spends in mempool and their owning transactions checking in order every input of the incoming + /// transaction. + /// + /// Removes the mempool transaction owning the double spends and inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - at least one double spend + /// - all double spends belong to the same mempool transaction + /// - the incoming transaction has a higher fee/mass ratio than the mempool double spending transaction. + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend or a tx fee/mass too low error. + Mandatory, + } } diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs new file mode 100644 index 000000000..e8d2b54ab --- /dev/null +++ b/mining/src/mempool/model/frontier.rs @@ -0,0 +1,454 @@ +use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, + model::candidate_tx::CandidateTransaction, + Policy, RebalancingWeightedTransactionSelector, +}; + +use feerate_key::FeerateTransactionKey; +use kaspa_consensus_core::block::TemplateTransactionSelector; +use kaspa_core::trace; +use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use search_tree::SearchTree; +use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; +use std::collections::HashSet; + +pub(crate) mod feerate_key; +pub(crate) mod search_tree; +pub(crate) mod selectors; + +/// If the frontier contains less than 4x the block mass limit, we consider +/// inplace sampling to be less efficient (due to collisions) and thus use +/// the rebalancing selector +const COLLISION_FACTOR: u64 = 4; + +/// Multiplication factor for in-place sampling. We sample 20% more than the +/// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. +const MASS_LIMIT_FACTOR: f64 = 1.2; + +/// A rough estimation for the average transaction mass. The usage is a non-important edge case +/// hence we just throw this here (as oppose to performing an accurate estimation) +const TYPICAL_TX_MASS: f64 = 2000.0; + +/// Management of the transaction pool frontier, that is, the set of transactions in +/// the transaction pool which have no mempool ancestors and are essentially ready +/// to enter the next block template. +#[derive(Default)] +pub struct Frontier { + /// Frontier transactions sorted by feerate order and searchable for weight sampling + search_tree: SearchTree, + + /// Total masses: Σ_{tx in frontier} tx.mass + total_mass: u64, +} + +impl Frontier { + pub fn total_weight(&self) -> f64 { + self.search_tree.total_weight() + } + + pub fn total_mass(&self) -> u64 { + self.total_mass + } + + pub fn len(&self) -> usize { + self.search_tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.insert(key) { + self.total_mass += mass; + true + } else { + false + } + } + + pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.remove(key) { + self.total_mass -= mass; + true + } else { + false + } + } + + /// Samples the frontier in-place based on the provided policy and returns a SequenceSelector. + /// + /// This sampling algorithm should be used when frontier total mass is high enough compared to + /// policy mass limit so that the probability of sampling collisions remains low. + /// + /// Convergence analysis: + /// 1. Based on the above we can safely assume that `k << n`, where `n` is the total number of frontier items + /// and `k` is the number of actual samples (since `desired_mass << total_mass` and mass per item is bounded) + /// 2. Indeed, if the weight distribution is not too spread (i.e., `max(weights) = O(min(weights))`), `k << n` means + /// that the probability of collisions is low enough and the sampling process will converge in `O(k log(n))` w.h.p. + /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below + /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled with + /// sufficient probability (in constant time). Following each sampling collision we search for a consecutive range of + /// top elements which were already sampled and narrow the sampling space to exclude them all. We do this by computing + /// the prefix weight up to the top most item which wasn't sampled yet (inclusive) and then continue the sampling process + /// over the narrowed space. This process is repeated until acquiring the desired mass. + /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling + /// space. However, if `top.weight` is much larger than the remaining weight, the above f64 subtraction will yield a number + /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. + /// 5. Q. Why not just use u64 weights? + /// A. The current weight calculation is `feerate^alpha` with `alpha=3`. Using u64 would mean that the feerate space + /// is limited to a range of size `(2^64)^(1/3) = ~2^21 = ~2M`. Already with current usages, the feerate can vary + /// from `~1/50` (2000 sompi for a transaction with 100K storage mass), to `5M` (100 KAS fee for a transaction with + /// 2000 mass = 100·100_000_000/2000), resulting in a range of size 250M (`5M/(1/50)`). + /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for + /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation + /// will readjust it. + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy, _collisions: &mut u64) -> SequenceSelectorInput + where + R: Rng + ?Sized, + { + debug_assert!(!self.search_tree.is_empty(), "expected to be called only if not empty"); + + // Sample 20% more than the hard limit in order to allow the SequenceSelector to + // compensate for consensus rejections. + // Note: this is a soft limit which is why the loop below might pass it if the + // next sampled transaction happens to cross the bound + let desired_mass = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; + + let mut distr = Uniform::new(0f64, self.total_weight()); + let mut down_iter = self.search_tree.descending_iter(); + let mut top = down_iter.next().unwrap(); + let mut cache = HashSet::new(); + let mut sequence = SequenceSelectorInput::default(); + let mut total_selected_mass: u64 = 0; + let mut collisions = 0; + + // The sampling process is converging so the cache will eventually hold all entries, which guarantees loop exit + 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= desired_mass { + let query = distr.sample(rng); + let item = { + let mut item = self.search_tree.search(query); + while !cache.insert(item.tx.id()) { + collisions += 1; + // Try to narrow the sampling space in order to reduce further sampling collisions + if cache.contains(&top.tx.id()) { + loop { + match down_iter.next() { + Some(next) => top = next, + None => break 'outer, + } + // Loop until finding a top item which was not sampled yet + if !cache.contains(&top.tx.id()) { + break; + } + } + let remaining_weight = self.search_tree.prefix_weight(top); + distr = Uniform::new(0f64, remaining_weight); + } + let query = distr.sample(rng); + item = self.search_tree.search(query); + } + item + }; + sequence.push(item.tx.clone(), item.mass); + total_selected_mass += item.mass; // Max standard mass + Mempool capacity bound imply this will not overflow + } + trace!("[mempool frontier sample inplace] collisions: {collisions}, cache: {}", cache.len()); + *_collisions += collisions; + sequence + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier. + /// + /// The logic is divided into three cases: + /// 1. The frontier is small and can fit entirely into a block: perform no sampling and return + /// a TakeAllSelector + /// 2. The frontier has at least ~4x the capacity of a block: expected collision rate is low, perform + /// in-place k*log(n) sampling and return a SequenceSelector + /// 3. The frontier has 1-4x capacity of a block. In this case we expect a high collision rate while + /// the number of overall transactions is still low, so we take all of the transactions and use the + /// rebalancing weighted selector (performing the actual sampling out of the mempool lock) + /// + /// The above thresholds were selected based on benchmarks. Overall, this dynamic selection provides + /// full transaction selection in less than 150 µs even if the frontier has 1M entries (!!). See mining/benches + /// for more details. + pub fn build_selector(&self, policy: &Policy) -> Box { + if self.total_mass <= policy.max_block_mass { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { + let mut rng = rand::thread_rng(); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy, &mut 0), policy.clone())) + } else { + Box::new(RebalancingWeightedTransactionSelector::new( + policy.clone(), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + } + + /// Exposed for benchmarking purposes + pub fn build_selector_sample_inplace(&self, _collisions: &mut u64) -> Box { + let mut rng = rand::thread_rng(); + let policy = Policy::new(500_000); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy, _collisions), policy)) + } + + /// Exposed for benchmarking purposes + pub fn build_selector_take_all(&self) -> Box { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } + + /// Exposed for benchmarking purposes + pub fn build_rebalancing_selector(&self) -> Box { + Box::new(RebalancingWeightedTransactionSelector::new( + Policy::new(500_000), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + let average_transaction_mass = match self.len() { + 0 => TYPICAL_TX_MASS, + n => self.total_mass() as f64 / n as f64, + }; + let bps = args.network_blocks_per_second as f64; + let mut mass_per_block = args.maximum_mass_per_block as f64; + let mut inclusion_interval = average_transaction_mass / (mass_per_block * bps); + let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); + + // Search for better estimators by possibly removing extremely high outliers + let mut down_iter = self.search_tree.descending_iter().skip(1); + loop { + // Update values for the next iteration. In order to remove the outlier from the + // total weight, we must compensate by capturing a block slot. + mass_per_block -= average_transaction_mass; + if mass_per_block <= average_transaction_mass { + // Out of block slots, break + break; + } + + // Re-calc the inclusion interval based on the new block "capacity". + // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 + inclusion_interval = average_transaction_mass / (mass_per_block * bps); + + // Compute the weight up to, and including, current key (or use zero weight if next is none) + let next = down_iter.next(); + let prefix_weight = next.map(|key| self.search_tree.prefix_weight(key)).unwrap_or_default(); + let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); + + // Test the pending estimator vs. the current one + if pending_estimator.feerate_to_time(1.0) < estimator.feerate_to_time(1.0) { + estimator = pending_estimator; + } else { + // The pending estimator is no better, break. Indicates that the reduction in + // network mass per second is more significant than the removed weight + break; + } + + if next.is_none() { + break; + } + } + estimator + } +} + +#[cfg(test)] +mod tests { + use super::*; + use feerate_key::tests::build_feerate_key; + use itertools::Itertools; + use rand::thread_rng; + use std::collections::HashMap; + + #[test] + pub fn test_highly_irregular_sampling() { + let mut rng = thread_rng(); + let cap = 1000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + if i == 0 { + // Add an extremely large fee in order to create extremely high variance + fee = 100_000_000 * 1_000_000; // 1M KAS + } + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + } + + #[test] + pub fn test_mempool_sampling_small() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_sample_inplace(&mut 0); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + } + + #[test] + pub fn test_total_mass_tracking() { + let mut rng = thread_rng(); + let cap = 10000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = rng.gen_range(1..100000); // Use distinct mass values to challenge the test + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap / 2; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let prev_total_mass = frontier.total_mass(); + // Assert the total mass + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Add a bunch of duplicates and make sure the total mass remains the same + let mut dup_items = frontier.search_tree.ascending_iter().take(len / 2).cloned().collect_vec(); + for dup in dup_items.iter().cloned() { + (!frontier.insert(dup)).then_some(()).unwrap(); + } + assert_eq!(prev_total_mass, frontier.total_mass()); + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Remove a few elements from the map in order to randomize the iterator + dup_items.iter().take(10).for_each(|k| { + map.remove(&k.tx.id()); + }); + + // Add and remove random elements some of which will be duplicate insertions and some missing removals + for item in map.values().step_by(2) { + frontier.remove(item); + if let Some(item2) = dup_items.pop() { + frontier.insert(item2); + } + } + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + } + + #[test] + fn test_feerate_estimator() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + // 304 (~500,000/1650) extreme outliers is an edge case where the build estimator logic should be tested at + if i <= 303 { + // Add an extremely large fee in order to create extremely high variance + fee = i * 10_000_000 * 1_000_000; + } + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(1.0); + + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } + + #[test] + fn test_constant_feerate_estimator() { + const MIN_FEERATE: f64 = 1.0; + let cap = 20_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mass: u64 = 1650; + let fee = (mass as f64 * MIN_FEERATE) as u64; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + println!(); + println!("Testing a frontier with {} txs...", len.min(cap)); + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(MIN_FEERATE); + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } +} diff --git a/mining/src/mempool/model/frontier/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs new file mode 100644 index 000000000..843ef0ff1 --- /dev/null +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -0,0 +1,108 @@ +use crate::{block_template::selector::ALPHA, mempool::model::tx::MempoolTransaction}; +use kaspa_consensus_core::tx::Transaction; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct FeerateTransactionKey { + pub fee: u64, + pub mass: u64, + weight: f64, + pub tx: Arc, +} + +impl Eq for FeerateTransactionKey {} + +impl PartialEq for FeerateTransactionKey { + fn eq(&self, other: &Self) -> bool { + self.tx.id() == other.tx.id() + } +} + +impl FeerateTransactionKey { + pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + // NOTE: any change to the way this weight is calculated (such as scaling by some factor) + // requires a reversed update to total_weight in `Frontier::build_feerate_estimator`. This + // is because the math methods in FeeEstimator assume this specific weight function. + Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } + } + + pub fn feerate(&self) -> f64 { + self.fee as f64 / self.mass as f64 + } + + pub fn weight(&self) -> f64 { + self.weight + } +} + +impl std::hash::Hash for FeerateTransactionKey { + fn hash(&self, state: &mut H) { + // Transaction id is a sufficient identifier for this key + self.tx.id().hash(state); + } +} + +impl PartialOrd for FeerateTransactionKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FeerateTransactionKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Our first priority is the feerate. + // The weight function is monotonic in feerate so we prefer using it + // since it is cached + match self.weight().total_cmp(&other.weight()) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // If feerates (and thus weights) are equal, prefer the higher fee in absolute value + match self.fee.cmp(&other.fee) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // + // At this point we don't compare the mass fields since if both feerate + // and fee are equal, mass must be equal as well + // + + // Finally, we compare transaction ids in order to allow multiple transactions with + // the same fee and mass to exist within the same sorted container + self.tx.id().cmp(&other.tx.id()) + } +} + +impl From<&MempoolTransaction> for FeerateTransactionKey { + fn from(tx: &MempoolTransaction) -> Self { + let mass = tx.mtx.tx.mass(); + let fee = tx.mtx.calculated_fee.expect("fee is expected to be populated"); + assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); + Self::new(fee, mass, tx.mtx.tx.clone()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use kaspa_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, + }; + use kaspa_hashes::{HasherBase, TransactionID}; + use std::sync::Arc; + + fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) + } + + /// Test helper for generating a feerate key with a unique tx (per u64 id) + pub(crate) fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) + } +} diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs new file mode 100644 index 000000000..fc18b2118 --- /dev/null +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -0,0 +1,335 @@ +use super::feerate_key::FeerateTransactionKey; +use std::iter::FusedIterator; +use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; +use sweep_bptree::tree::{Argument, SearchArgument}; +use sweep_bptree::{BPlusTree, NodeStoreVec}; + +type FeerateKey = FeerateTransactionKey; + +/// A struct for implementing "weight space" search using the SearchArgument customization. +/// The weight space is the range `[0, total_weight)` and each key has a "logical" interval allocation +/// within this space according to its tree position and weight. +/// +/// We implement the search efficiently by maintaining subtree weights which are updated with each +/// element insertion/removal. Given a search query `p ∈ [0, total_weight)` we then find the corresponding +/// element in log time by walking down from the root and adjusting the query according to subtree weights. +/// For instance if the query point is `123.56` and the top 3 subtrees have weights `120, 10.5 ,100` then we +/// recursively query the middle subtree with the point `123.56 - 120 = 3.56`. +/// +/// See SearchArgument implementation below for more details. +#[derive(Clone, Copy, Debug, Default)] +struct FeerateWeight(f64); + +impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } +} + +impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.weight()).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } +} + +impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.weight(); + sum += w; + if query < sum { + return Some(i); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + // Search algorithm: Locate the next subtree to visit by iterating through `arguments` + // and subtracting the query until the correct range is found + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } + } +} + +/// Visitor struct which accumulates the prefix weight up to a provided key (inclusive) in log time. +/// +/// The basic idea is to use the subtree weights stored in the tree for walking down from the root +/// to the leaf (corresponding to the searched key), and accumulating all weights proceeding the walk-down path +struct PrefixWeightVisitor<'a> { + /// The key to search up to + key: &'a FeerateKey, + /// This field accumulates the prefix weight during the visit process + accumulated_weight: f64, +} + +impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + /// Returns the index of the first `key ∈ keys` such that `key > self.key`. If no such key + /// exists, the returned index will be the length of `keys`. + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // self.key is not in keys, idx is the index of the following key + idx + } + Ok(idx) => { + // Exact match, return the following index + idx + 1 + } + } + } +} + +impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // Invariants: + // a. arguments.len() == keys.len() + 1 (n inner node keys are the separators between n+1 subtrees) + // b. idx <= keys.len() (hence idx < arguments.len()) + + // Based on the invariants, we first accumulate all the subtree weights up to idx + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + + // ..and then go down to the idx'th subtree + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + // idx is the index of the key following self.key + let idx = self.search_in_keys(keys); + // Accumulate all key weights up to idx (which is inclusive if self.key ∈ tree) + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + // ..and return the final result + Some(self.accumulated_weight) + } +} + +type InnerTree = BPlusTree>; + +/// A transaction search tree sorted by feerate order and searchable for probabilistic weighted sampling. +/// +/// All `log(n)` expressions below are in base 64 (based on constants chosen within the sweep_bptree crate). +/// +/// The tree has the following properties: +/// 1. Linear time ordered access (ascending / descending) +/// 2. Insertions/removals in log(n) time +/// 3. Search for a weight point `p ∈ [0, total_weight)` in log(n) time +/// 4. Compute the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) +/// according to key order, in log(n) time +/// 5. Access the total weight in O(1) time. The total weight has numerical stability since it +/// is recomputed from subtree weights for each item insertion/removal +/// +/// Computing the prefix weight is a crucial operation if the tree is used for random sampling and +/// the tree is highly imbalanced in terms of weight variance. See [`Frontier::sample_inplace`] for +/// more details. +pub struct SearchTree { + tree: InnerTree, +} + +impl Default for SearchTree { + fn default() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } +} + +impl SearchTree { + pub fn new() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Inserts a key into the tree in log(n) time. Returns `false` if the key was already in the tree. + pub fn insert(&mut self, key: FeerateKey) -> bool { + self.tree.insert(key, ()).is_none() + } + + /// Remove a key from the tree in log(n) time. Returns `false` if the key was not in the tree. + pub fn remove(&mut self, key: &FeerateKey) -> bool { + self.tree.remove(key).is_some() + } + + /// Search for a weight point `query ∈ [0, total_weight)` in log(n) time + pub fn search(&self, query: f64) -> &FeerateKey { + self.tree.get_by_argument(query).expect("clamped").0 + } + + /// Access the total weight in O(1) time + pub fn total_weight(&self) -> f64 { + self.tree.root_argument().weight() + } + + /// Computes the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) + /// according to key order, in log(n) time + pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { + self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() + } + + /// Iterate the tree in descending key order (going down from the + /// highest key). Linear in the number of keys *actually* iterated. + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().rev().map(|(key, ())| key) + } + + /// Iterate the tree in ascending key order (going up from the + /// lowest key). Linear in the number of keys *actually* iterated. + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().map(|(key, ())| key) + } + + /// The lowest key in the tree (by key order) + pub fn first(&self) -> Option<&FeerateKey> { + self.tree.first().map(|(k, ())| k) + } + + /// The highest key in the tree (by key order) + pub fn last(&self) -> Option<&FeerateKey> { + self.tree.last().map(|(k, ())| k) + } +} + +#[cfg(test)] +mod tests { + use super::super::feerate_key::tests::build_feerate_key; + use super::*; + use itertools::Itertools; + use std::collections::HashSet; + use std::ops::Sub; + + #[test] + fn test_feerate_weight_queries() { + let mut tree = SearchTree::new(); + let mass = 2000; + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + s.insert(key.clone()); + tree.insert(key); + } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + tree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); + v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, &item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly + let eps: f64 = 0.001; + let mut sum = 0.0; + for expected in v.iter() { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = tree.search(sample); + assert_eq!(expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well + } + sum += weight; + } + + println!("{}, {}", sum, tree.total_weight()); + + // Test clamped search bounds + assert_eq!(tree.first(), Some(tree.search(f64::NEG_INFINITY))); + assert_eq!(tree.first(), Some(tree.search(-1.0))); + assert_eq!(tree.first(), Some(tree.search(-eps))); + assert_eq!(tree.first(), Some(tree.search(0.0))); + assert_eq!(tree.last(), Some(tree.search(sum))); + assert_eq!(tree.last(), Some(tree.search(sum + eps))); + assert_eq!(tree.last(), Some(tree.search(sum + 1.0))); + assert_eq!(tree.last(), Some(tree.search(1.0 / 0.0))); + assert_eq!(tree.last(), Some(tree.search(f64::INFINITY))); + let _ = tree.search(f64::NAN); + + // Assert prefix weights + let mut prefix = Vec::with_capacity(v.len()); + prefix.push(v[0].weight()); + for i in 1..v.len() { + prefix.push(prefix[i - 1] + v[i].weight()); + } + let eps = v.iter().map(|k| k.weight()).min_by(f64::total_cmp).unwrap() * 1e-4; + for (expected_prefix, key) in prefix.into_iter().zip(v) { + let prefix = tree.prefix_weight(&key); + assert!(expected_prefix.sub(prefix).abs() < eps); + } + } + + #[test] + fn test_tree_rev_iter() { + let mut tree = SearchTree::new(); + let mass = 2000; + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + v.push(key.clone()); + tree.insert(key); + } + v.sort(); + + for (expected, item) in v.into_iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + } +} diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs new file mode 100644 index 000000000..a30ecc145 --- /dev/null +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -0,0 +1,162 @@ +use crate::Policy; +use kaspa_consensus_core::{ + block::TemplateTransactionSelector, + tx::{Transaction, TransactionId}, +}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +pub struct SequenceSelectorTransaction { + pub tx: Arc, + pub mass: u64, +} + +impl SequenceSelectorTransaction { + pub fn new(tx: Arc, mass: u64) -> Self { + Self { tx, mass } + } +} + +type SequencePriorityIndex = u32; + +/// The input sequence for the [`SequenceSelector`] transaction selector +#[derive(Default)] +pub struct SequenceSelectorInput { + /// We use the btree map ordered by insertion order in order to follow + /// the initial sequence order while allowing for efficient removal of previous selections + inner: BTreeMap, +} + +impl FromIterator for SequenceSelectorInput { + fn from_iter>(iter: T) -> Self { + Self { inner: BTreeMap::from_iter(iter.into_iter().enumerate().map(|(i, v)| (i as SequencePriorityIndex, v))) } + } +} + +impl SequenceSelectorInput { + pub fn push(&mut self, tx: Arc, mass: u64) { + let idx = self.inner.len() as SequencePriorityIndex; + self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); + } + + pub fn iter(&self) -> impl Iterator { + self.inner.values() + } +} + +/// Helper struct for storing data related to previous selections +struct SequenceSelectorSelection { + tx_id: TransactionId, + mass: u64, + priority_index: SequencePriorityIndex, +} + +/// A selector which selects transactions in the order they are provided. The selector assumes +/// that the transactions were already selected via weighted sampling and simply tries them one +/// after the other until the block mass limit is reached. +pub struct SequenceSelector { + input_sequence: SequenceSelectorInput, + selected_vec: Vec, + /// Maps from selected tx ids to tx mass so that the total used mass can be subtracted on tx reject + selected_map: Option>, + total_selected_mass: u64, + overall_candidates: usize, + overall_rejections: usize, + policy: Policy, +} + +impl SequenceSelector { + pub fn new(input_sequence: SequenceSelectorInput, policy: Policy) -> Self { + Self { + overall_candidates: input_sequence.inner.len(), + selected_vec: Vec::with_capacity(input_sequence.inner.len()), + input_sequence, + selected_map: Default::default(), + total_selected_mass: Default::default(), + overall_rejections: Default::default(), + policy, + } + } + + #[inline] + fn reset_selection(&mut self) { + self.selected_vec.clear(); + self.selected_map = None; + } +} + +impl TemplateTransactionSelector for SequenceSelector { + fn select_transactions(&mut self) -> Vec { + // Remove selections from the previous round if any + for selection in self.selected_vec.drain(..) { + self.input_sequence.inner.remove(&selection.priority_index); + } + // Reset selection data structures + self.reset_selection(); + let mut transactions = Vec::with_capacity(self.input_sequence.inner.len()); + + // Iterate the input sequence in order + for (&priority_index, tx) in self.input_sequence.inner.iter() { + if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { + // We assume the sequence is relatively small, hence we keep on searching + // for transactions with lower mass which might fit into the remaining gap + continue; + } + self.total_selected_mass += tx.mass; + self.selected_vec.push(SequenceSelectorSelection { tx_id: tx.tx.id(), mass: tx.mass, priority_index }); + transactions.push(tx.tx.as_ref().clone()) + } + transactions + } + + fn reject_selection(&mut self, tx_id: TransactionId) { + // Lazy-create the map only when there are actual rejections + let selected_map = self.selected_map.get_or_insert_with(|| self.selected_vec.iter().map(|tx| (tx.tx_id, tx.mass)).collect()); + let mass = selected_map.remove(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + // Selections must be counted in total selected mass, so this subtraction cannot underflow + self.total_selected_mass -= mass; + self.overall_rejections += 1; + } + + fn is_successful(&self) -> bool { + const SUFFICIENT_MASS_THRESHOLD: f64 = 0.8; + const LOW_REJECTION_FRACTION: f64 = 0.2; + + // We consider the operation successful if either mass occupation is above 80% or rejection rate is below 20% + self.overall_rejections == 0 + || (self.total_selected_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD + || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION + } +} + +/// A selector that selects all the transactions it holds and is always considered successful. +/// If all mempool transactions have combined mass which is <= block mass limit, this selector +/// should be called and provided with all the transactions. +pub struct TakeAllSelector { + txs: Vec>, +} + +impl TakeAllSelector { + pub fn new(txs: Vec>) -> Self { + Self { txs } + } +} + +impl TemplateTransactionSelector for TakeAllSelector { + fn select_transactions(&mut self) -> Vec { + // Drain on the first call so that subsequent calls return nothing + self.txs.drain(..).map(|tx| tx.as_ref().clone()).collect() + } + + fn reject_selection(&mut self, _tx_id: TransactionId) { + // No need to track rejections (for reduced mass), since there's nothing else to select + } + + fn is_successful(&self) -> bool { + // Considered successful because we provided all mempool transactions to this + // selector, so there's no point in retries + true + } +} diff --git a/mining/src/mempool/model/mod.rs b/mining/src/mempool/model/mod.rs index 88997e46f..bfe622293 100644 --- a/mining/src/mempool/model/mod.rs +++ b/mining/src/mempool/model/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod accepted_transactions; +pub(crate) mod frontier; pub(crate) mod map; pub(crate) mod orphan_pool; pub(crate) mod pool; diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index cf70150df..bc3469409 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -1,27 +1,31 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, mempool::{ config::Config, errors::{RuleError, RuleResult}, model::{ map::MempoolTransactionCollection, pool::{Pool, TransactionsEdges}, - tx::MempoolTransaction, + tx::{DoubleSpend, MempoolTransaction}, utxo_set::MempoolUtxoSet, }, tx::Priority, }, - model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + model::topological_index::TopologicalIndex, + Policy, }; use kaspa_consensus_core::{ - tx::TransactionId, - tx::{MutableTransaction, TransactionOutpoint}, + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId, TransactionOutpoint}, }; use kaspa_core::{time::unix_now, trace, warn}; use std::{ - collections::{hash_map::Keys, hash_set::Iter, HashSet}, + collections::{hash_map::Keys, hash_set::Iter}, sync::Arc, }; +use super::frontier::Frontier; + /// Pool of transactions to be included in a block template /// /// ### Rust rewrite notes @@ -54,7 +58,7 @@ pub(crate) struct TransactionsPool { /// Transactions dependencies formed by outputs present in pool - successor relations. chained_transactions: TransactionsEdges, /// Transactions with no parents in the mempool -- ready to be inserted into a block template - ready_transactions: HashSet, + ready_transactions: Frontier, last_expire_scan_daa_score: u64, /// last expire scan time in milliseconds @@ -105,7 +109,7 @@ impl TransactionsPool { let parents = self.get_parent_transaction_ids_in_pool(&transaction.mtx); self.parent_transactions.insert(id, parents.clone()); if parents.is_empty() { - self.ready_transactions.insert(id); + self.ready_transactions.insert((&transaction).into()); } for parent_id in parents { let entry = self.chained_transactions.entry(parent_id).or_default(); @@ -133,18 +137,20 @@ impl TransactionsPool { if let Some(parents) = self.parent_transactions.get_mut(chain) { parents.remove(transaction_id); if parents.is_empty() { - self.ready_transactions.insert(*chain); + let tx = self.all_transactions.get(chain).unwrap(); + self.ready_transactions.insert(tx.into()); } } } } self.parent_transactions.remove(transaction_id); self.chained_transactions.remove(transaction_id); - self.ready_transactions.remove(transaction_id); // Remove the transaction itself let removed_tx = self.all_transactions.remove(transaction_id).ok_or(RuleError::RejectMissingTransaction(*transaction_id))?; + self.ready_transactions.remove(&(&removed_tx).into()); + // TODO: consider using `self.parent_transactions.get(transaction_id)` // The tradeoff to consider is whether it might be possible that a parent tx exists in the pool // however its relation as parent is not registered. This can supposedly happen in rare cases where @@ -161,15 +167,18 @@ impl TransactionsPool { self.ready_transactions.len() } - /// all_ready_transactions returns all fully populated mempool transactions having no parents in the mempool. - /// These transactions are ready for being inserted in a block template. - pub(crate) fn all_ready_transactions(&self) -> Vec { - // The returned transactions are leaving the mempool so they are cloned - self.ready_transactions - .iter() - .take(self.config.maximum_ready_transaction_count as usize) - .map(|id| CandidateTransaction::from_mutable(&self.all_transactions.get(id).unwrap().mtx)) - .collect() + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.ready_transactions.total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.ready_transactions.build_feerate_estimator(args) } /// Is the mempool transaction identified by `transaction_id` unchained, thus having no successor? @@ -182,6 +191,7 @@ impl TransactionsPool { } false } + /// Returns the exceeding low-priority transactions having the lowest fee rates in order /// to have room for at least `free_slots` new transactions. The returned transactions /// are guaranteed to be unchained (no successor in mempool) and to not be parent of @@ -228,8 +238,8 @@ impl TransactionsPool { // An error is returned if the mempool is filled with high priority and other unremovable transactions. let tx_count = self.len() + free_slots - transactions_to_remove.len(); - if tx_count as u64 > self.config.maximum_transaction_count { - let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count); + if tx_count as u64 > self.config.maximum_transaction_count as u64 { + let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count as u64); warn!("{}", err.to_string()); return Err(err); } @@ -245,10 +255,29 @@ impl TransactionsPool { self.utxo_set.get_outpoint_owner_id(outpoint) } + /// Make sure no other transaction in the mempool is already spending an output which one of this transaction inputs spends pub(crate) fn check_double_spends(&self, transaction: &MutableTransaction) -> RuleResult<()> { self.utxo_set.check_double_spends(transaction) } + /// Returns the first double spend of every transaction in the mempool double spending on `transaction` + pub(crate) fn get_double_spend_transaction_ids(&self, transaction: &MutableTransaction) -> Vec { + self.utxo_set.get_double_spend_transaction_ids(transaction) + } + + pub(crate) fn get_double_spend_owner<'a>(&'a self, double_spend: &DoubleSpend) -> RuleResult<&'a MempoolTransaction> { + match self.get(&double_spend.owner_id) { + Some(transaction) => Ok(transaction), + None => { + // This case should never arise in the first place. + // Anyway, in case it does, if a double spent transaction id is found but the matching + // transaction cannot be located in the mempool a replacement is no longer possible + // so a double spend error is returned. + Err(double_spend.into()) + } + } + } + pub(crate) fn collect_expired_low_priority_transactions(&mut self, virtual_daa_score: u64) -> Vec { let now = unix_now(); if virtual_daa_score < self.last_expire_scan_daa_score + self.config.transaction_expire_scan_interval_daa_score diff --git a/mining/src/mempool/model/tx.rs b/mining/src/mempool/model/tx.rs index 1e549c997..9b65faeb2 100644 --- a/mining/src/mempool/model/tx.rs +++ b/mining/src/mempool/model/tx.rs @@ -1,8 +1,9 @@ -use crate::mempool::tx::Priority; -use kaspa_consensus_core::{tx::MutableTransaction, tx::TransactionId}; +use crate::mempool::tx::{Priority, RbfPolicy}; +use kaspa_consensus_core::tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint}; +use kaspa_mining_errors::mempool::RuleError; use std::{ - cmp::Ordering, fmt::{Display, Formatter}, + sync::Arc, }; pub(crate) struct MempoolTransaction { @@ -33,26 +34,51 @@ impl MempoolTransaction { } } -impl Ord for MempoolTransaction { - fn cmp(&self, other: &Self) -> Ordering { - self.fee_rate().total_cmp(&other.fee_rate()).then(self.id().cmp(&other.id())) +impl RbfPolicy { + #[cfg(test)] + /// Returns an alternate policy accepting a transaction insertion in case the policy requires a replacement + pub(crate) fn for_insert(&self) -> RbfPolicy { + match self { + RbfPolicy::Forbidden | RbfPolicy::Allowed => *self, + RbfPolicy::Mandatory => RbfPolicy::Allowed, + } } } -impl Eq for MempoolTransaction {} +pub(crate) struct DoubleSpend { + pub outpoint: TransactionOutpoint, + pub owner_id: TransactionId, +} + +impl DoubleSpend { + pub fn new(outpoint: TransactionOutpoint, owner_id: TransactionId) -> Self { + Self { outpoint, owner_id } + } +} -impl PartialOrd for MempoolTransaction { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) +impl From for RuleError { + fn from(value: DoubleSpend) -> Self { + RuleError::RejectDoubleSpendInMempool(value.outpoint, value.owner_id) } } -impl PartialEq for MempoolTransaction { - fn eq(&self, other: &Self) -> bool { - self.fee_rate() == other.fee_rate() +impl From<&DoubleSpend> for RuleError { + fn from(value: &DoubleSpend) -> Self { + RuleError::RejectDoubleSpendInMempool(value.outpoint, value.owner_id) } } +pub(crate) struct TransactionPreValidation { + pub transaction: MutableTransaction, + pub feerate_threshold: Option, +} + +#[derive(Default)] +pub(crate) struct TransactionPostValidation { + pub removed: Option>, + pub accepted: Option>, +} + #[derive(PartialEq, Eq)] pub(crate) enum TxRemovalReason { Muted, @@ -63,6 +89,7 @@ pub(crate) enum TxRemovalReason { DoubleSpend, InvalidInBlockTemplate, RevalidationWithMissingOutpoints, + ReplacedByFee, } impl TxRemovalReason { @@ -76,6 +103,7 @@ impl TxRemovalReason { TxRemovalReason::DoubleSpend => "double spend", TxRemovalReason::InvalidInBlockTemplate => "invalid in block template", TxRemovalReason::RevalidationWithMissingOutpoints => "revalidation with missing outpoints", + TxRemovalReason::ReplacedByFee => "replaced by fee", } } diff --git a/mining/src/mempool/model/utxo_set.rs b/mining/src/mempool/model/utxo_set.rs index 38c2bcb4e..808d67488 100644 --- a/mining/src/mempool/model/utxo_set.rs +++ b/mining/src/mempool/model/utxo_set.rs @@ -1,7 +1,9 @@ +use std::collections::HashSet; + use crate::{ mempool::{ - errors::{RuleError, RuleResult}, - model::map::OutpointIndex, + errors::RuleResult, + model::{map::OutpointIndex, tx::DoubleSpend}, }, model::TransactionIdSet, }; @@ -70,14 +72,36 @@ impl MempoolUtxoSet { /// Make sure no other transaction in the mempool is already spending an output which one of this transaction inputs spends pub(crate) fn check_double_spends(&self, transaction: &MutableTransaction) -> RuleResult<()> { + match self.get_first_double_spend(transaction) { + Some(double_spend) => Err(double_spend.into()), + None => Ok(()), + } + } + + pub(crate) fn get_first_double_spend(&self, transaction: &MutableTransaction) -> Option { let transaction_id = transaction.id(); for input in transaction.tx.inputs.iter() { if let Some(existing_transaction_id) = self.get_outpoint_owner_id(&input.previous_outpoint) { if *existing_transaction_id != transaction_id { - return Err(RuleError::RejectDoubleSpendInMempool(input.previous_outpoint, *existing_transaction_id)); + return Some(DoubleSpend::new(input.previous_outpoint, *existing_transaction_id)); + } + } + } + None + } + + /// Returns the first double spend of every transaction in the mempool double spending on `transaction` + pub(crate) fn get_double_spend_transaction_ids(&self, transaction: &MutableTransaction) -> Vec { + let transaction_id = transaction.id(); + let mut double_spends = vec![]; + let mut visited = HashSet::new(); + for input in transaction.tx.inputs.iter() { + if let Some(existing_transaction_id) = self.get_outpoint_owner_id(&input.previous_outpoint) { + if *existing_transaction_id != transaction_id && visited.insert(*existing_transaction_id) { + double_spends.push(DoubleSpend::new(input.previous_outpoint, *existing_transaction_id)); } } } - Ok(()) + double_spends } } diff --git a/mining/src/mempool/populate_entries_and_try_validate.rs b/mining/src/mempool/populate_entries_and_try_validate.rs index 0c0dcf9a1..a5c125280 100644 --- a/mining/src/mempool/populate_entries_and_try_validate.rs +++ b/mining/src/mempool/populate_entries_and_try_validate.rs @@ -1,5 +1,12 @@ use crate::mempool::{errors::RuleResult, model::pool::Pool, Mempool}; -use kaspa_consensus_core::{api::ConsensusApi, constants::UNACCEPTED_DAA_SCORE, tx::MutableTransaction, tx::UtxoEntry}; +use kaspa_consensus_core::{ + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, + constants::UNACCEPTED_DAA_SCORE, + tx::{MutableTransaction, UtxoEntry}, +}; use kaspa_mining_errors::mempool::RuleError; impl Mempool { @@ -14,15 +21,20 @@ impl Mempool { } } -pub(crate) fn validate_mempool_transaction(consensus: &dyn ConsensusApi, transaction: &mut MutableTransaction) -> RuleResult<()> { - Ok(consensus.validate_mempool_transaction(transaction)?) +pub(crate) fn validate_mempool_transaction( + consensus: &dyn ConsensusApi, + transaction: &mut MutableTransaction, + args: &TransactionValidationArgs, +) -> RuleResult<()> { + Ok(consensus.validate_mempool_transaction(transaction, args)?) } pub(crate) fn validate_mempool_transactions_in_parallel( consensus: &dyn ConsensusApi, transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, ) -> Vec> { - consensus.validate_mempool_transactions_in_parallel(transactions).into_iter().map(|x| x.map_err(RuleError::from)).collect() + consensus.validate_mempool_transactions_in_parallel(transactions, args).into_iter().map(|x| x.map_err(RuleError::from)).collect() } pub(crate) fn populate_mempool_transactions_in_parallel( diff --git a/mining/src/mempool/replace_by_fee.rs b/mining/src/mempool/replace_by_fee.rs new file mode 100644 index 000000000..6acd1a618 --- /dev/null +++ b/mining/src/mempool/replace_by_fee.rs @@ -0,0 +1,149 @@ +use crate::mempool::{ + errors::{RuleError, RuleResult}, + model::tx::{DoubleSpend, MempoolTransaction, TxRemovalReason}, + tx::RbfPolicy, + Mempool, +}; +use kaspa_consensus_core::tx::{MutableTransaction, Transaction}; +use std::sync::Arc; + +impl Mempool { + /// Returns the replace by fee (RBF) constraint fee/mass threshold for an incoming transaction and a policy. + /// + /// Fails if the transaction does not meet some condition of the RBF policy, excluding the fee/mass condition. + /// + /// See [`RbfPolicy`] variants for details of each policy process and success conditions. + pub(super) fn get_replace_by_fee_constraint( + &self, + transaction: &MutableTransaction, + rbf_policy: RbfPolicy, + ) -> RuleResult> { + match rbf_policy { + RbfPolicy::Forbidden => { + // When RBF is forbidden, fails early on any double spend + self.transaction_pool.check_double_spends(transaction)?; + Ok(None) + } + + RbfPolicy::Allowed => { + // When RBF is allowed, never fails since both insertion and replacement are possible + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + if double_spends.is_empty() { + Ok(None) + } else { + let mut feerate_threshold = 0f64; + for double_spend in double_spends { + // We take the max over all double spends as the required threshold + feerate_threshold = feerate_threshold.max(self.get_double_spend_feerate(&double_spend)?); + } + Ok(Some(feerate_threshold)) + } + } + + RbfPolicy::Mandatory => { + // When RBF is mandatory, fails early if we do not have exactly one double spending transaction + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.len() { + 0 => Err(RuleError::RejectRbfNoDoubleSpend), + 1 => { + let feerate_threshold = self.get_double_spend_feerate(&double_spends[0])?; + Ok(Some(feerate_threshold)) + } + _ => Err(RuleError::RejectRbfTooManyDoubleSpendingTransactions), + } + } + } + } + + /// Executes replace by fee (RBF) for an incoming transaction and a policy. + /// + /// See [`RbfPolicy`] variants for details of each policy process and success conditions. + /// + /// On success, `transaction` is guaranteed to embed no double spend with the mempool. + /// + /// On success with the [`RbfPolicy::Mandatory`] policy, some removed transaction is always returned. + pub(super) fn execute_replace_by_fee( + &mut self, + transaction: &MutableTransaction, + rbf_policy: RbfPolicy, + ) -> RuleResult>> { + match rbf_policy { + RbfPolicy::Forbidden => { + self.transaction_pool.check_double_spends(transaction)?; + Ok(None) + } + + RbfPolicy::Allowed => { + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.is_empty() { + true => Ok(None), + false => { + let removed = self.validate_double_spending_transaction(transaction, &double_spends[0])?.mtx.tx.clone(); + for double_spend in double_spends.iter().skip(1) { + // Validate the feerate threshold is passed for all double spends + self.validate_double_spending_transaction(transaction, double_spend)?; + } + // We apply consequences such as removal only after we fully validate against all double spends + for double_spend in double_spends { + self.remove_transaction( + &double_spend.owner_id, + true, + TxRemovalReason::ReplacedByFee, + format!("by {}", transaction.id()).as_str(), + )?; + } + Ok(Some(removed)) + } + } + } + + RbfPolicy::Mandatory => { + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.len() { + 0 => Err(RuleError::RejectRbfNoDoubleSpend), + 1 => { + let removed = self.validate_double_spending_transaction(transaction, &double_spends[0])?.mtx.tx.clone(); + self.remove_transaction( + &double_spends[0].owner_id, + true, + TxRemovalReason::ReplacedByFee, + format!("by {}", transaction.id()).as_str(), + )?; + Ok(Some(removed)) + } + _ => Err(RuleError::RejectRbfTooManyDoubleSpendingTransactions), + } + } + } + } + + fn get_double_spend_feerate(&self, double_spend: &DoubleSpend) -> RuleResult { + let owner = self.transaction_pool.get_double_spend_owner(double_spend)?; + match owner.mtx.calculated_feerate() { + Some(double_spend_feerate) => Ok(double_spend_feerate), + // Getting here is unexpected since a mempool owned tx should be populated with fee + // and mass at this stage but nonetheless we fail gracefully + None => Err(double_spend.into()), + } + } + + fn validate_double_spending_transaction<'a>( + &'a self, + transaction: &MutableTransaction, + double_spend: &DoubleSpend, + ) -> RuleResult<&'a MempoolTransaction> { + let owner = self.transaction_pool.get_double_spend_owner(double_spend)?; + if let (Some(transaction_feerate), Some(double_spend_feerate)) = + (transaction.calculated_feerate(), owner.mtx.calculated_feerate()) + { + if transaction_feerate > double_spend_feerate { + return Ok(owner); + } else { + return Err(double_spend.into()); + } + } + // Getting here is unexpected since both txs should be populated with + // fee and mass at this stage but nonetheless we fail gracefully + Err(double_spend.into()) + } +} diff --git a/mining/src/mempool/validate_and_insert_transaction.rs b/mining/src/mempool/validate_and_insert_transaction.rs index 591fa5c4a..bcfedc2db 100644 --- a/mining/src/mempool/validate_and_insert_transaction.rs +++ b/mining/src/mempool/validate_and_insert_transaction.rs @@ -2,9 +2,9 @@ use crate::mempool::{ errors::{RuleError, RuleResult}, model::{ pool::Pool, - tx::{MempoolTransaction, TxRemovalReason}, + tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, }, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, Mempool, }; use kaspa_consensus_core::{ @@ -13,21 +13,21 @@ use kaspa_consensus_core::{ tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint, UtxoEntry}, }; use kaspa_core::{debug, info}; -use std::sync::Arc; impl Mempool { pub(crate) fn pre_validate_and_populate_transaction( &self, consensus: &dyn ConsensusApi, mut transaction: MutableTransaction, - ) -> RuleResult { + rbf_policy: RbfPolicy, + ) -> RuleResult { self.validate_transaction_unacceptance(&transaction)?; // Populate mass in the beginning, it will be used in multiple places throughout the validation and insertion. transaction.calculated_compute_mass = Some(consensus.calculate_transaction_compute_mass(&transaction.tx)); self.validate_transaction_in_isolation(&transaction)?; - self.transaction_pool.check_double_spends(&transaction)?; + let feerate_threshold = self.get_replace_by_fee_constraint(&transaction, rbf_policy)?; self.populate_mempool_entries(&mut transaction); - Ok(transaction) + Ok(TransactionPreValidation { transaction, feerate_threshold }) } pub(crate) fn post_validate_and_insert_transaction( @@ -37,7 +37,8 @@ impl Mempool { transaction: MutableTransaction, priority: Priority, orphan: Orphan, - ) -> RuleResult>> { + rbf_policy: RbfPolicy, + ) -> RuleResult { let transaction_id = transaction.id(); // First check if the transaction was not already added to the mempool. @@ -46,28 +47,29 @@ impl Mempool { // concurrently. if self.transaction_pool.has(&transaction_id) { debug!("Transaction {0} is not post validated since already in the mempool", transaction_id); - return Ok(None); + return Err(RuleError::RejectDuplicate(transaction_id)); } self.validate_transaction_unacceptance(&transaction)?; - // Re-check double spends since validate_and_insert_transaction is no longer atomic - self.transaction_pool.check_double_spends(&transaction)?; - match validation_result { Ok(_) => {} Err(RuleError::RejectMissingOutpoint) => { if orphan == Orphan::Forbidden { return Err(RuleError::RejectDisallowedOrphan(transaction_id)); } + let _ = self.get_replace_by_fee_constraint(&transaction, rbf_policy)?; self.orphan_pool.try_add_orphan(consensus.get_virtual_daa_score(), transaction, priority)?; - return Ok(None); + return Ok(TransactionPostValidation::default()); } Err(err) => { return Err(err); } } + // Check double spends and try to remove them if the RBF policy requires it + let removed_transaction = self.execute_replace_by_fee(&transaction, rbf_policy)?; + self.validate_transaction_in_context(&transaction)?; // Before adding the transaction, check if there is room in the pool @@ -78,7 +80,7 @@ impl Mempool { // Add the transaction to the mempool as a MempoolTransaction and return a clone of the embedded Arc let accepted_transaction = self.transaction_pool.add_transaction(transaction, consensus.get_virtual_daa_score(), priority)?.mtx.tx.clone(); - Ok(Some(accepted_transaction)) + Ok(TransactionPostValidation { removed: removed_transaction, accepted: Some(accepted_transaction) }) } /// Validates that the transaction wasn't already accepted into the DAG @@ -184,9 +186,26 @@ impl Mempool { // The one we just removed from the orphan pool. assert_eq!(transactions.len(), 1, "the list returned by remove_orphan is expected to contain exactly one transaction"); let transaction = transactions.pop().unwrap(); + let rbf_policy = Self::get_orphan_transaction_rbf_policy(transaction.priority); self.validate_transaction_unacceptance(&transaction.mtx)?; - self.transaction_pool.check_double_spends(&transaction.mtx)?; + let _ = self.get_replace_by_fee_constraint(&transaction.mtx, rbf_policy)?; Ok(transaction) } + + /// Returns the RBF policy to apply to an orphan/unorphaned transaction by inferring it from the transaction priority. + pub(crate) fn get_orphan_transaction_rbf_policy(priority: Priority) -> RbfPolicy { + // The RBF policy applied to an orphaned transaction is not recorded in the orphan pool + // but we can infer it from the priority: + // + // - high means a submitted tx via RPC which forbids RBF + // - low means a tx arrived via P2P which allows RBF + // + // Note that the RPC submit transaction replacement case, implying a mandatory RBF, forbids orphans + // so is excluded here. + match priority { + Priority::High => RbfPolicy::Forbidden, + Priority::Low => RbfPolicy::Allowed, + } + } } diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index f1fdf7c71..b8cc34cc4 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -1,10 +1,11 @@ -use kaspa_consensus_core::tx::{MutableTransaction, Transaction}; +use crate::FeerateTransactionKey; +use kaspa_consensus_core::tx::Transaction; use std::sync::Arc; /// Transaction with additional metadata needed in order to be a candidate /// in the transaction selection algorithm #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct CandidateTransaction { +pub struct CandidateTransaction { /// The actual transaction pub tx: Arc, /// Populated fee @@ -14,9 +15,7 @@ pub(crate) struct CandidateTransaction { } impl CandidateTransaction { - pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { - let mass = tx.tx.mass(); - assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } + pub fn from_key(key: FeerateTransactionKey) -> Self { + Self { tx: key.tx, calculated_fee: key.fee, calculated_mass: key.mass } } } diff --git a/mining/src/model/mod.rs b/mining/src/model/mod.rs index 3f17a50c8..dcec6f17f 100644 --- a/mining/src/model/mod.rs +++ b/mining/src/model/mod.rs @@ -1,10 +1,11 @@ use kaspa_consensus_core::tx::TransactionId; use std::collections::HashSet; -pub(crate) mod candidate_tx; +pub mod candidate_tx; pub mod owner_txs; pub mod topological_index; pub mod topological_sort; +pub mod tx_insert; pub mod tx_query; /// A set of unique transaction ids diff --git a/mining/src/model/tx_insert.rs b/mining/src/model/tx_insert.rs new file mode 100644 index 000000000..4c006fb99 --- /dev/null +++ b/mining/src/model/tx_insert.rs @@ -0,0 +1,14 @@ +use kaspa_consensus_core::tx::Transaction; +use std::sync::Arc; + +#[derive(Debug)] +pub struct TransactionInsertion { + pub removed: Option>, + pub accepted: Vec>, +} + +impl TransactionInsertion { + pub fn new(removed: Option>, accepted: Vec>) -> Self { + Self { removed, accepted } + } +} diff --git a/mining/src/monitor.rs b/mining/src/monitor.rs index 517bd8276..876ce9b7a 100644 --- a/mining/src/monitor.rs +++ b/mining/src/monitor.rs @@ -1,4 +1,5 @@ use super::MiningCounters; +use crate::manager::MiningManagerProxy; use kaspa_core::{ debug, info, task::{ @@ -13,6 +14,8 @@ use std::{sync::Arc, time::Duration}; const MONITOR: &str = "mempool-monitor"; pub struct MiningMonitor { + mining_manager: MiningManagerProxy, + // Counters counters: Arc, @@ -24,11 +27,12 @@ pub struct MiningMonitor { impl MiningMonitor { pub fn new( + mining_manager: MiningManagerProxy, counters: Arc, tx_script_cache_counters: Arc, tick_service: Arc, ) -> MiningMonitor { - MiningMonitor { counters, tx_script_cache_counters, tick_service } + MiningMonitor { mining_manager, counters, tx_script_cache_counters, tick_service } } pub async fn worker(self: &Arc) { @@ -62,6 +66,8 @@ impl MiningMonitor { delta.low_priority_tx_counts, delta.tx_accepted_counts, ); + let feerate_estimations = self.mining_manager.clone().get_realtime_feerate_estimations().await; + debug!("Realtime feerate estimations: {}", feerate_estimations); } if tx_script_cache_snapshot != last_tx_script_cache_snapshot { debug!( diff --git a/mining/src/testutils/consensus_mock.rs b/mining/src/testutils/consensus_mock.rs index 94d774c42..d68e062a0 100644 --- a/mining/src/testutils/consensus_mock.rs +++ b/mining/src/testutils/consensus_mock.rs @@ -1,6 +1,9 @@ use super::coinbase_mock::CoinbaseManagerMock; use kaspa_consensus_core::{ - api::ConsensusApi, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, block::{BlockTemplate, MutableBlock, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, coinbase::MinerData, constants::BLOCK_VERSION, @@ -16,7 +19,7 @@ use kaspa_consensus_core::{ utxo::utxo_collection::UtxoCollection, }; use kaspa_core::time::unix_now; -use kaspa_hashes::ZERO_HASH; +use kaspa_hashes::{Hash, ZERO_HASH}; use parking_lot::RwLock; use std::{collections::HashMap, sync::Arc}; @@ -83,7 +86,7 @@ impl ConsensusApi for ConsensusMock { let coinbase = coinbase_manager.expected_coinbase_transaction(miner_data.clone()); txs.insert(0, coinbase.tx); let now = unix_now(); - let hash_merkle_root = calc_hash_merkle_root(txs.iter()); + let hash_merkle_root = self.calc_transaction_hash_merkle_root(&txs, 0); let header = Header::new_finalized( BLOCK_VERSION, vec![], @@ -100,10 +103,10 @@ impl ConsensusApi for ConsensusMock { ); let mutable_block = MutableBlock::new(header, txs); - Ok(BlockTemplate::new(mutable_block, miner_data, coinbase.has_red_reward, now, 0, ZERO_HASH)) + Ok(BlockTemplate::new(mutable_block, miner_data, coinbase.has_red_reward, now, 0, ZERO_HASH, vec![])) } - fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction) -> TxResult<()> { + fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction, _: &TransactionValidationArgs) -> TxResult<()> { // If a predefined status was registered to simulate an error, return it right away if let Some(status) = self.statuses.read().get(&mutable_tx.id()) { if status.is_err() { @@ -138,12 +141,16 @@ impl ConsensusApi for ConsensusMock { Ok(()) } - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - transactions.iter_mut().map(|x| self.validate_mempool_transaction(x)).collect() + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + _: &TransactionValidationBatchArgs, + ) -> Vec> { + transactions.iter_mut().map(|x| self.validate_mempool_transaction(x, &Default::default())).collect() } fn populate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - transactions.iter_mut().map(|x| self.validate_mempool_transaction(x)).collect() + transactions.iter_mut().map(|x| self.validate_mempool_transaction(x, &Default::default())).collect() } fn calculate_transaction_compute_mass(&self, transaction: &Transaction) -> u64 { @@ -170,4 +177,8 @@ impl ConsensusApi for ConsensusMock { let coinbase_manager = CoinbaseManagerMock::new(); Ok(coinbase_manager.modify_coinbase_payload(payload, miner_data)) } + + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], _pov_daa_score: u64) -> Hash { + calc_hash_merkle_root(txs.iter(), false) + } } diff --git a/protocol/flows/src/flow_context.rs b/protocol/flows/src/flow_context.rs index 3d365c54f..14d4168ac 100644 --- a/protocol/flows/src/flow_context.rs +++ b/protocol/flows/src/flow_context.rs @@ -25,8 +25,8 @@ use kaspa_core::{ }; use kaspa_core::{time::unix_now, warn}; use kaspa_hashes::Hash; -use kaspa_mining::manager::MiningManagerProxy; use kaspa_mining::mempool::tx::{Orphan, Priority}; +use kaspa_mining::{manager::MiningManagerProxy, mempool::tx::RbfPolicy}; use kaspa_notify::notifier::Notify; use kaspa_p2p_lib::{ common::ProtocolError, @@ -618,16 +618,48 @@ impl FlowContext { transaction: Transaction, orphan: Orphan, ) -> Result<(), ProtocolError> { - let accepted_transactions = - self.mining_manager().clone().validate_and_insert_transaction(consensus, transaction, Priority::High, orphan).await?; + let transaction_insertion = self + .mining_manager() + .clone() + .validate_and_insert_transaction(consensus, transaction, Priority::High, orphan, RbfPolicy::Forbidden) + .await?; self.broadcast_transactions( - accepted_transactions.iter().map(|x| x.id()), + transaction_insertion.accepted.iter().map(|x| x.id()), false, // RPC transactions are considered high priority, so we don't want to throttle them ) .await; Ok(()) } + /// Replaces the rpc-submitted transaction into the mempool and propagates it to peers. + /// + /// Returns the removed mempool transaction on successful replace by fee. + /// + /// Transactions submitted through rpc are considered high priority. This definition does not affect the tx selection algorithm + /// but only changes how we manage the lifetime of the tx. A high-priority tx does not expire and is repeatedly rebroadcasted to + /// peers + pub async fn submit_rpc_transaction_replacement( + &self, + consensus: &ConsensusProxy, + transaction: Transaction, + ) -> Result, ProtocolError> { + let transaction_insertion = self + .mining_manager() + .clone() + .validate_and_insert_transaction(consensus, transaction, Priority::High, Orphan::Forbidden, RbfPolicy::Mandatory) + .await?; + self.broadcast_transactions( + transaction_insertion.accepted.iter().map(|x| x.id()), + false, // RPC transactions are considered high priority, so we don't want to throttle them + ) + .await; + // The combination of args above of Orphan::Forbidden and RbfPolicy::Mandatory should always result + // in a removed transaction returned, however we prefer failing gracefully in case of future internal mempool changes + transaction_insertion.removed.ok_or(ProtocolError::Other( + "Replacement transaction was actually accepted but the *replaced* transaction was not returned from the mempool", + )) + } + /// Returns true if the time has come for running the task cleaning mempool transactions. async fn should_run_mempool_scanning_task(&self) -> bool { self.transactions_spread.write().await.should_run_mempool_scanning_task() diff --git a/protocol/flows/src/flowcontext/orphans.rs b/protocol/flows/src/flowcontext/orphans.rs index 75c223caa..d1122b0e0 100644 --- a/protocol/flows/src/flowcontext/orphans.rs +++ b/protocol/flows/src/flowcontext/orphans.rs @@ -78,7 +78,7 @@ impl OrphanBlocksPool { if self.orphans.contains_key(&orphan_hash) { return None; } - + orphan_block.asses_for_cache()?; let (roots, orphan_ancestors) = match self.get_orphan_roots(consensus, orphan_block.header.direct_parents().iter().copied().collect()).await { FindRootsOutput::Roots(roots, orphan_ancestors) => (roots, orphan_ancestors), diff --git a/protocol/flows/src/v5/txrelay/flow.rs b/protocol/flows/src/v5/txrelay/flow.rs index 6a177e57f..af7e2b6c7 100644 --- a/protocol/flows/src/v5/txrelay/flow.rs +++ b/protocol/flows/src/v5/txrelay/flow.rs @@ -10,7 +10,7 @@ use kaspa_mining::{ errors::MiningManagerError, mempool::{ errors::RuleError, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, }, model::tx_query::TransactionQuery, P2pTxCountSample, @@ -219,7 +219,7 @@ impl RelayTransactionsFlow { .ctx .mining_manager() .clone() - .validate_and_insert_transaction_batch(&consensus, transactions, Priority::Low, Orphan::Allowed) + .validate_and_insert_transaction_batch(&consensus, transactions, Priority::Low, Orphan::Allowed, RbfPolicy::Allowed) .await; for res in insert_results.iter() { diff --git a/protocol/p2p/src/common.rs b/protocol/p2p/src/common.rs index d32552cd8..a0b314fb6 100644 --- a/protocol/p2p/src/common.rs +++ b/protocol/p2p/src/common.rs @@ -100,6 +100,10 @@ impl ProtocolError { pub fn from_reject_message(reason: String) -> Self { if reason == LOOPBACK_CONNECTION_MESSAGE || reason == DUPLICATE_CONNECTION_MESSAGE { ProtocolError::IgnorableReject(reason) + } else if reason.contains("cannot find full block") { + let hint = "Hint: If this error persists, it might be due to the other peer having pruned block data after syncing headers and UTXOs. In such a case, you may need to reset the database."; + let detailed_reason = format!("{}. {}", reason, hint); + ProtocolError::Rejected(detailed_reason) } else { ProtocolError::Rejected(reason) } diff --git a/rothschild/src/main.rs b/rothschild/src/main.rs index bbabca71b..ad212b5e4 100644 --- a/rothschild/src/main.rs +++ b/rothschild/src/main.rs @@ -21,7 +21,7 @@ use secp256k1::{rand::thread_rng, Keypair}; use tokio::time::{interval, MissedTickBehavior}; const DEFAULT_SEND_AMOUNT: u64 = 10 * SOMPI_PER_KASPA; -const FEE_PER_MASS: u64 = 10; +const FEE_RATE: u64 = 10; const MILLIS_PER_TICK: u64 = 10; const ADDRESS_PREFIX: Prefix = Prefix::Testnet; const ADDRESS_VERSION: Version = Version::PubKey; @@ -438,7 +438,7 @@ fn clean_old_pending_outpoints(pending: &mut HashMap) } fn required_fee(num_utxos: usize, num_outs: u64) -> u64 { - FEE_PER_MASS * estimated_mass(num_utxos, num_outs) + FEE_RATE * estimated_mass(num_utxos, num_outs) } fn estimated_mass(num_utxos: usize, num_outs: u64) -> u64 { diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index a02fbf746..6d5357406 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -114,6 +114,13 @@ pub enum RpcApiOps { VirtualDaaScoreChangedNotification, PruningPointUtxoSetOverrideNotification, NewBlockTemplateNotification, + + /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy + SubmitTransactionReplacement, + + // Fee estimation related commands + GetFeeEstimate, + GetFeeEstimateExperimental, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index 36f8ef308..8fd26b058 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -132,6 +132,17 @@ pub trait RpcApi: Sync + Send + AnySync { } async fn submit_transaction_call(&self, request: SubmitTransactionRequest) -> RpcResult; + /// Submits a transaction replacement to the mempool, applying a mandatory Replace by Fee policy. + /// + /// Returns the ID of the inserted transaction and the transaction the submission replaced in the mempool. + async fn submit_transaction_replacement(&self, transaction: RpcTransaction) -> RpcResult { + self.submit_transaction_replacement_call(SubmitTransactionReplacementRequest { transaction }).await + } + async fn submit_transaction_replacement_call( + &self, + request: SubmitTransactionReplacementRequest, + ) -> RpcResult; + /// Requests information about a specific block. async fn get_block(&self, hash: RpcHash, include_transactions: bool) -> RpcResult { Ok(self.get_block_call(GetBlockRequest::new(hash, include_transactions)).await?.block) @@ -304,6 +315,22 @@ pub trait RpcApi: Sync + Send + AnySync { request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // Fee estimation API + + async fn get_fee_estimate(&self) -> RpcResult { + Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) + } + async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; + + async fn get_fee_estimate_experimental(&self, verbose: bool) -> RpcResult { + self.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await + } + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/core/src/error.rs b/rpc/core/src/error.rs index 59f6b910e..6e1083ab7 100644 --- a/rpc/core/src/error.rs +++ b/rpc/core/src/error.rs @@ -1,4 +1,4 @@ -use kaspa_consensus_core::{subnets::SubnetworkConversionError, tx::TransactionId}; +use kaspa_consensus_core::{subnets::SubnetworkConversionError, tx::TransactionId}; use kaspa_utils::networking::IpAddress; use std::{net::AddrParseError, num::TryFromIntError}; use thiserror::Error; @@ -116,9 +116,9 @@ pub enum RpcError { #[error("transaction query must either not filter transactions or include orphans")] InconsistentMempoolTxQuery, - #[error(transparent)] - SubnetParsingError(#[from] SubnetworkConversionError), - + #[error(transparent)] + SubnetParsingError(#[from] SubnetworkConversionError), + #[error(transparent)] WasmError(#[from] workflow_wasm::error::Error), diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs new file mode 100644 index 000000000..f9de9de90 --- /dev/null +++ b/rpc/core/src/model/feerate_estimate.rs @@ -0,0 +1,55 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeerateBucket { + /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds + pub feerate: f64, + + /// The estimated inclusion time for a transaction with fee/mass = feerate + pub estimated_seconds: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimate { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: RpcFeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl RpcFeeEstimate { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimateVerboseExperimentalData { + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index 7366bf3cc..5c003546c 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -299,6 +299,31 @@ impl SubmitTransactionResponse { } } +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmitTransactionReplacementRequest { + pub transaction: RpcTransaction, +} + +impl SubmitTransactionReplacementRequest { + pub fn new(transaction: RpcTransaction) -> Self { + Self { transaction } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmitTransactionReplacementResponse { + pub transaction_id: RpcTransactionId, + pub replaced_transaction: RpcTransaction, +} + +impl SubmitTransactionReplacementResponse { + pub fn new(transaction_id: RpcTransactionId, replaced_transaction: RpcTransaction) -> Self { + Self { transaction_id, replaced_transaction } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct GetSubnetworkRequest { @@ -825,6 +850,35 @@ impl GetDaaScoreTimestampEstimateResponse { } } +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Fee rate estimations + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateRequest {} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateResponse { + pub estimate: RpcFeeEstimate, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalRequest { + pub verbose: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalResponse { + /// The usual feerate estimate response + pub estimate: RpcFeeEstimate, + + /// Experimental verbose data + pub verbose: Option, +} + // ---------------------------------------------------------------------------- // Subscriptions & notifications // ---------------------------------------------------------------------------- diff --git a/rpc/core/src/model/mod.rs b/rpc/core/src/model/mod.rs index fd07a109e..8950bd1cb 100644 --- a/rpc/core/src/model/mod.rs +++ b/rpc/core/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod blue_work; +pub mod feerate_estimate; pub mod hash; pub mod header; pub mod hex_cnv; @@ -15,6 +16,7 @@ pub mod tx; pub use address::*; pub use block::*; pub use blue_work::*; +pub use feerate_estimate::*; pub use hash::*; pub use header::*; pub use hex_cnv::*; diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index c7eebd8d1..c19a28eb5 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -253,6 +253,7 @@ impl RpcApi for GrpcClient { route!(get_connected_peer_info_call, GetConnectedPeerInfo); route!(add_peer_call, AddPeer); route!(submit_transaction_call, SubmitTransaction); + route!(submit_transaction_replacement_call, SubmitTransactionReplacement); route!(get_subnetwork_call, GetSubnetwork); route!(get_virtual_chain_from_block_call, GetVirtualChainFromBlock); route!(get_blocks_call, GetBlocks); @@ -271,6 +272,8 @@ impl RpcApi for GrpcClient { route!(get_mempool_entries_by_addresses_call, GetMempoolEntriesByAddresses); route!(get_coin_supply_call, GetCoinSupply); route!(get_daa_score_timestamp_estimate_call, GetDaaScoreTimestampEstimate); + route!(get_fee_estimate_call, GetFeeEstimate); + route!(get_fee_estimate_experimental_call, GetFeeEstimateExperimental); // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index ec7242635..01075c183 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -58,7 +58,10 @@ message KaspadRequest { GetMetricsRequestMessage getMetricsRequest = 1090; GetServerInfoRequestMessage getServerInfoRequest = 1092; GetSyncStatusRequestMessage getSyncStatusRequest = 1094; - GetDaaScoreTimestampEstimateRequestMessage GetDaaScoreTimestampEstimateRequest = 1096; + GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; + SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; + GetFeeEstimateRequestMessage getFeeEstimateRequest = 1106; + GetFeeEstimateExperimentalRequestMessage getFeeEstimateExperimentalRequest = 1108; } } @@ -117,7 +120,10 @@ message KaspadResponse { GetMetricsResponseMessage getMetricsResponse= 1091; GetServerInfoResponseMessage getServerInfoResponse = 1093; GetSyncStatusResponseMessage getSyncStatusResponse = 1095; - GetDaaScoreTimestampEstimateResponseMessage GetDaaScoreTimestampEstimateResponse = 1097; + GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; + SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; + GetFeeEstimateResponseMessage getFeeEstimateResponse = 1107; + GetFeeEstimateExperimentalResponseMessage getFeeEstimateExperimentalResponse = 1109; } } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index e558c6548..a0bed8977 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -307,6 +307,21 @@ message SubmitTransactionResponseMessage{ RPCError error = 1000; } +// SubmitTransactionReplacementRequestMessage submits a transaction to the mempool, applying a mandatory Replace by Fee policy +message SubmitTransactionReplacementRequestMessage{ + RpcTransaction transaction = 1; +} + +message SubmitTransactionReplacementResponseMessage{ + // The transaction ID of the submitted transaction + string transactionId = 1; + + // The previous transaction replaced in the mempool by the newly submitted one + RpcTransaction replacedTransaction = 2; + + RPCError error = 1000; +} + // NotifyVirtualChainChangedRequestMessage registers this connection for virtualChainChanged notifications. // // See: VirtualChainChangedNotificationMessage @@ -844,10 +859,66 @@ message GetSyncStatusResponseMessage{ } message GetDaaScoreTimestampEstimateRequestMessage { - repeated uint64 daa_scores = 1; + repeated uint64 daa_scores = 1; } message GetDaaScoreTimestampEstimateResponseMessage{ - repeated uint64 timestamps = 1; - RPCError error = 1000; + repeated uint64 timestamps = 1; + RPCError error = 1000; +} + +message RpcFeerateBucket { + // Fee/mass of a transaction in `sompi/gram` units + double feerate = 1; + double estimated_seconds = 2; +} + +// Data required for making fee estimates. +// +// Feerate values represent fee/mass of a transaction in `sompi/gram` units. +// Given a feerate value recommendation, calculate the required fee by +// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +message RpcFeeEstimate { + // Top-priority feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + RpcFeerateBucket priority_bucket = 1; + + // A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + // provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + // times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + // between them, one can compose a complete feerate function on the client side. The API makes an effort + // to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + repeated RpcFeerateBucket normal_buckets = 2; + + // A vector of *low* priority feerate values. The first value of this vector is guaranteed to + // exist and provide an estimation for sub-*hour* DAG inclusion. + repeated RpcFeerateBucket low_buckets = 3; +} + +message RpcFeeEstimateVerboseExperimentalData { + uint64 mempool_ready_transactions_count = 1; + uint64 mempool_ready_transactions_total_mass = 2; + uint64 network_mass_per_second = 3; + + double next_block_template_feerate_min = 11; + double next_block_template_feerate_median = 12; + double next_block_template_feerate_max = 13; +} + +message GetFeeEstimateRequestMessage { +} + +message GetFeeEstimateResponseMessage { + RpcFeeEstimate estimate = 1; + RPCError error = 1000; +} + +message GetFeeEstimateExperimentalRequestMessage { + bool verbose = 1; +} + +message GetFeeEstimateExperimentalResponseMessage { + RpcFeeEstimate estimate = 1; + RpcFeeEstimateVerboseExperimentalData verbose = 2; + + RPCError error = 1000; } diff --git a/rpc/grpc/core/src/convert/feerate_estimate.rs b/rpc/grpc/core/src/convert/feerate_estimate.rs new file mode 100644 index 000000000..d1bff8f45 --- /dev/null +++ b/rpc/grpc/core/src/convert/feerate_estimate.rs @@ -0,0 +1,66 @@ +use crate::protowire; +use crate::{from, try_from}; +use kaspa_rpc_core::RpcError; + +// ---------------------------------------------------------------------------- +// rpc_core to protowire +// ---------------------------------------------------------------------------- + +from!(item: &kaspa_rpc_core::RpcFeerateBucket, protowire::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +from!(item: &kaspa_rpc_core::RpcFeeEstimate, protowire::RpcFeeEstimate, { + Self { + priority_bucket: Some((&item.priority_bucket).into()), + normal_buckets: item.normal_buckets.iter().map(|b| b.into()).collect(), + low_buckets: item.low_buckets.iter().map(|b| b.into()).collect(), + } +}); + +from!(item: &kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); + +// ---------------------------------------------------------------------------- +// protowire to rpc_core +// ---------------------------------------------------------------------------- + +try_from!(item: &protowire::RpcFeerateBucket, kaspa_rpc_core::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +try_from!(item: &protowire::RpcFeeEstimate, kaspa_rpc_core::RpcFeeEstimate, { + Self { + priority_bucket: item.priority_bucket + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("RpcFeeEstimate".to_string(), "priority_bucket".to_string()))? + .try_into()?, + normal_buckets: item.normal_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + low_buckets: item.low_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + } +}); + +try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, kaspa_rpc_core::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); diff --git a/rpc/grpc/core/src/convert/kaspad.rs b/rpc/grpc/core/src/convert/kaspad.rs index 0fef61523..d43442fe3 100644 --- a/rpc/grpc/core/src/convert/kaspad.rs +++ b/rpc/grpc/core/src/convert/kaspad.rs @@ -36,6 +36,7 @@ pub mod kaspad_request_convert { impl_into_kaspad_request!(GetConnectedPeerInfo); impl_into_kaspad_request!(AddPeer); impl_into_kaspad_request!(SubmitTransaction); + impl_into_kaspad_request!(SubmitTransactionReplacement); impl_into_kaspad_request!(GetSubnetwork); impl_into_kaspad_request!(GetVirtualChainFromBlock); impl_into_kaspad_request!(GetBlocks); @@ -57,6 +58,8 @@ pub mod kaspad_request_convert { impl_into_kaspad_request!(GetServerInfo); impl_into_kaspad_request!(GetSyncStatus); impl_into_kaspad_request!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_request!(GetFeeEstimate); + impl_into_kaspad_request!(GetFeeEstimateExperimental); impl_into_kaspad_request!(NotifyBlockAdded); impl_into_kaspad_request!(NotifyNewBlockTemplate); @@ -167,6 +170,7 @@ pub mod kaspad_response_convert { impl_into_kaspad_response!(GetConnectedPeerInfo); impl_into_kaspad_response!(AddPeer); impl_into_kaspad_response!(SubmitTransaction); + impl_into_kaspad_response!(SubmitTransactionReplacement); impl_into_kaspad_response!(GetSubnetwork); impl_into_kaspad_response!(GetVirtualChainFromBlock); impl_into_kaspad_response!(GetBlocks); @@ -188,6 +192,8 @@ pub mod kaspad_response_convert { impl_into_kaspad_response!(GetServerInfo); impl_into_kaspad_response!(GetSyncStatus); impl_into_kaspad_response!(GetDaaScoreTimestampEstimate); + impl_into_kaspad_response!(GetFeeEstimate); + impl_into_kaspad_response!(GetFeeEstimateExperimental); impl_into_kaspad_notify_response!(NotifyBlockAdded); impl_into_kaspad_notify_response!(NotifyNewBlockTemplate); diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index 9babf29c8..75606dc9a 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -248,6 +248,13 @@ from!(item: RpcResult<&kaspa_rpc_core::SubmitTransactionResponse>, protowire::Su Self { transaction_id: item.transaction_id.to_string(), error: None } }); +from!(item: &kaspa_rpc_core::SubmitTransactionReplacementRequest, protowire::SubmitTransactionReplacementRequestMessage, { + Self { transaction: Some((&item.transaction).into()) } +}); +from!(item: RpcResult<&kaspa_rpc_core::SubmitTransactionReplacementResponse>, protowire::SubmitTransactionReplacementResponseMessage, { + Self { transaction_id: item.transaction_id.to_string(), replaced_transaction: Some((&item.replaced_transaction).into()), error: None } +}); + from!(item: &kaspa_rpc_core::GetSubnetworkRequest, protowire::GetSubnetworkRequestMessage, { Self { subnetwork_id: item.subnetwork_id.to_string() } }); @@ -394,6 +401,25 @@ from!(item: RpcResult<&kaspa_rpc_core::GetDaaScoreTimestampEstimateResponse>, pr Self { timestamps: item.timestamps.clone(), error: None } }); +// Fee estimate API + +from!(&kaspa_rpc_core::GetFeeEstimateRequest, protowire::GetFeeEstimateRequestMessage); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateResponse>, protowire::GetFeeEstimateResponseMessage, { + Self { estimate: Some((&item.estimate).into()), error: None } +}); +from!(item: &kaspa_rpc_core::GetFeeEstimateExperimentalRequest, protowire::GetFeeEstimateExperimentalRequestMessage, { + Self { + verbose: item.verbose + } +}); +from!(item: RpcResult<&kaspa_rpc_core::GetFeeEstimateExperimentalResponse>, protowire::GetFeeEstimateExperimentalResponseMessage, { + Self { + estimate: Some((&item.estimate).into()), + verbose: item.verbose.as_ref().map(|x| x.into()), + error: None + } +}); + from!(&kaspa_rpc_core::PingRequest, protowire::PingRequestMessage); from!(RpcResult<&kaspa_rpc_core::PingResponse>, protowire::PingResponseMessage); @@ -647,6 +673,26 @@ try_from!(item: &protowire::SubmitTransactionResponseMessage, RpcResult, { + Self { + transaction_id: RpcHash::from_str(&item.transaction_id)?, + replaced_transaction: item + .replaced_transaction + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("SubmitTransactionReplacementRequestMessage".to_string(), "replaced_transaction".to_string()))? + .try_into()?, + } +}); + try_from!(item: &protowire::GetSubnetworkRequestMessage, kaspa_rpc_core::GetSubnetworkRequest, { Self { subnetwork_id: kaspa_rpc_core::RpcSubnetworkId::from_str(&item.subnetwork_id)? } }); @@ -791,6 +837,30 @@ try_from!(item: &protowire::GetDaaScoreTimestampEstimateResponseMessage, RpcResu Self { timestamps: item.timestamps.clone() } }); +try_from!(&protowire::GetFeeEstimateRequestMessage, kaspa_rpc_core::GetFeeEstimateRequest); +try_from!(item: &protowire::GetFeeEstimateResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateResponseMessage".to_string(), "estimate".to_string()))? + .try_into()? + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalRequestMessage, kaspa_rpc_core::GetFeeEstimateExperimentalRequest, { + Self { + verbose: item.verbose + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateExperimentalResponseMessage".to_string(), "estimate".to_string()))? + .try_into()?, + verbose: item.verbose.as_ref().map(|x| x.try_into()).transpose()? + } +}); + try_from!(&protowire::PingRequestMessage, kaspa_rpc_core::PingRequest); try_from!(&protowire::PingResponseMessage, RpcResult); diff --git a/rpc/grpc/core/src/convert/mod.rs b/rpc/grpc/core/src/convert/mod.rs index 2f3252d22..d4948f57d 100644 --- a/rpc/grpc/core/src/convert/mod.rs +++ b/rpc/grpc/core/src/convert/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod error; +pub mod feerate_estimate; pub mod header; pub mod kaspad; pub mod mempool; diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index 7cc23f160..605d27efd 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -61,6 +61,7 @@ pub enum KaspadPayloadOps { GetConnectedPeerInfo, AddPeer, SubmitTransaction, + SubmitTransactionReplacement, GetSubnetwork, GetVirtualChainFromBlock, GetBlockCount, @@ -81,6 +82,8 @@ pub enum KaspadPayloadOps { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, // Subscription commands for starting/stopping notifications NotifyBlockAdded, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index 802cb6cd6..a70fb629f 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -55,6 +55,7 @@ impl Factory { GetConnectedPeerInfo, AddPeer, SubmitTransaction, + SubmitTransactionReplacement, GetSubnetwork, GetVirtualChainFromBlock, GetBlockCount, @@ -75,6 +76,8 @@ impl Factory { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, NotifyBlockAdded, NotifyNewBlockTemplate, NotifyFinalityConflict, diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index ddf78ccbd..2f4afa9c9 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -134,6 +134,13 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn submit_transaction_replacement_call( + &self, + _request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { Err(RpcError::NotImplemented) } @@ -228,6 +235,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs new file mode 100644 index 000000000..8df695c0c --- /dev/null +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -0,0 +1,49 @@ +use kaspa_mining::feerate::{FeeEstimateVerbose, FeerateBucket, FeerateEstimations}; +use kaspa_rpc_core::{ + message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, RpcFeeEstimate, + RpcFeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, RpcFeerateBucket, +}; + +pub trait FeerateBucketConverter { + fn into_rpc(self) -> RpcFeerateBucket; +} + +impl FeerateBucketConverter for FeerateBucket { + fn into_rpc(self) -> RpcFeerateBucket { + RpcFeerateBucket { feerate: self.feerate, estimated_seconds: self.estimated_seconds } + } +} + +pub trait FeeEstimateConverter { + fn into_rpc(self) -> RpcFeeEstimate; +} + +impl FeeEstimateConverter for FeerateEstimations { + fn into_rpc(self) -> RpcFeeEstimate { + RpcFeeEstimate { + priority_bucket: self.priority_bucket.into_rpc(), + normal_buckets: self.normal_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + low_buckets: self.low_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + } + } +} + +pub trait FeeEstimateVerboseConverter { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse; +} + +impl FeeEstimateVerboseConverter for FeeEstimateVerbose { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse { + RpcFeeEstimateVerboseResponse { + estimate: self.estimations.into_rpc(), + verbose: Some(RpcFeeEstimateVerbose { + network_mass_per_second: self.network_mass_per_second, + mempool_ready_transactions_count: self.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: self.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: self.next_block_template_feerate_min, + next_block_template_feerate_median: self.next_block_template_feerate_median, + next_block_template_feerate_max: self.next_block_template_feerate_max, + }), + } + } +} diff --git a/rpc/service/src/converter/mod.rs b/rpc/service/src/converter/mod.rs index 2e1460385..fd167d349 100644 --- a/rpc/service/src/converter/mod.rs +++ b/rpc/service/src/converter/mod.rs @@ -1,3 +1,4 @@ pub mod consensus; +pub mod feerate_estimate; pub mod index; pub mod protocol; diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index 00cc0d082..fc6f266fb 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -1,6 +1,7 @@ //! Core server implementation for ClientAPI use super::collector::{CollectorFromConsensus, CollectorFromIndex}; +use crate::converter::feerate_estimate::{FeeEstimateConverter, FeeEstimateVerboseConverter}; use crate::converter::{consensus::ConsensusConverter, index::IndexConverter, protocol::ProtocolConverter}; use crate::service::NetworkType::{Mainnet, Testnet}; use async_trait::async_trait; @@ -34,6 +35,7 @@ use kaspa_index_core::{ connection::IndexChannelConnection, indexed_utxos::UtxoSetByScriptPublicKey, notification::Notification as IndexNotification, notifier::IndexNotifier, }; +use kaspa_mining::feerate::FeeEstimateVerbose; use kaspa_mining::model::tx_query::TransactionQuery; use kaspa_mining::{manager::MiningManagerProxy, mempool::tx::Orphan}; use kaspa_notify::listener::ListenerLifespan; @@ -61,9 +63,11 @@ use kaspa_rpc_core::{ Notification, RpcError, RpcResult, }; use kaspa_txscript::{extract_script_pub_key_address, pay_to_address_script}; +use kaspa_utils::expiring_cache::ExpiringCache; use kaspa_utils::{channel::Channel, triggers::SingleTrigger}; use kaspa_utils_tower::counters::TowerConnectionCounters; use kaspa_utxoindex::api::UtxoIndexProxy; +use std::time::Duration; use std::{ collections::HashMap, iter::once, @@ -109,6 +113,8 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + fee_estimate_cache: ExpiringCache, + fee_estimate_verbose_cache: ExpiringCache>, } const RPC_CORE: &str = "rpc-core"; @@ -208,6 +214,8 @@ impl RpcCoreService { perf_monitor, p2p_tower_counters, grpc_tower_counters, + fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), + fee_estimate_verbose_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } } @@ -506,6 +514,22 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(SubmitTransactionResponse::new(transaction_id)) } + async fn submit_transaction_replacement_call( + &self, + request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + let transaction: Transaction = (&request.transaction).try_into()?; + let transaction_id = transaction.id(); + let session = self.consensus_manager.consensus().unguarded_session(); + let replaced_transaction = + self.flow_context.submit_rpc_transaction_replacement(&session, transaction).await.map_err(|err| { + let err = RpcError::RejectedTransaction(transaction_id, err.to_string()); + debug!("{err}"); + err + })?; + Ok(SubmitTransactionReplacementResponse::new(transaction_id, (&*replaced_transaction).into())) + } + async fn get_current_network_call(&self, _: GetCurrentNetworkRequest) -> RpcResult { Ok(GetCurrentNetworkResponse::new(*self.config.net)) } @@ -647,6 +671,36 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetDaaScoreTimestampEstimateResponse::new(timestamps)) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + let mining_manager = self.mining_manager.clone(); + let estimate = + self.fee_estimate_cache.get(async move { mining_manager.get_realtime_feerate_estimations().await.into_rpc() }).await; + Ok(GetFeeEstimateResponse { estimate }) + } + + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + if request.verbose { + let mining_manager = self.mining_manager.clone(); + let consensus_manager = self.consensus_manager.clone(); + let prefix = self.config.prefix(); + + let response = self + .fee_estimate_verbose_cache + .get(async move { + let session = consensus_manager.consensus().unguarded_session(); + mining_manager.get_realtime_feerate_estimations_verbose(&session, prefix).await.map(FeeEstimateVerbose::into_rpc) + }) + .await?; + Ok(response) + } else { + let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; + Ok(GetFeeEstimateExperimentalResponse { estimate, verbose: None }) + } + } + async fn ping_call(&self, _: PingRequest) -> RpcResult { Ok(PingResponse {}) } diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 4e9fffc0c..e57024c4b 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -596,27 +596,30 @@ impl RpcApi for KaspaRpcClient { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, - GetSyncStatus, + GetSinkBlueScore, GetSubnetwork, + GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, Shutdown, SubmitBlock, SubmitTransaction, + SubmitTransactionReplacement, Unban, ] ); diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index af4626681..09330eb49 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -44,28 +44,31 @@ impl Router { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, + GetSinkBlueScore, GetSubnetwork, GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, Shutdown, SubmitBlock, SubmitTransaction, + SubmitTransactionReplacement, Unban, ] ); diff --git a/simpa/Cargo.toml b/simpa/Cargo.toml index b52aa6fd9..30162ba4f 100644 --- a/simpa/Cargo.toml +++ b/simpa/Cargo.toml @@ -22,6 +22,7 @@ kaspa-perf-monitor.workspace = true kaspa-utils.workspace = true async-channel.workspace = true +cfg-if.workspace = true clap.workspace = true dhat = { workspace = true, optional = true } futures-util.workspace = true @@ -38,3 +39,4 @@ tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } [features] heap = ["dhat", "kaspa-alloc/heap"] +semaphore-trace = ["kaspa-utils/semaphore-trace"] diff --git a/simpa/src/main.rs b/simpa/src/main.rs index c586fd786..c0b3937c3 100644 --- a/simpa/src/main.rs +++ b/simpa/src/main.rs @@ -20,7 +20,12 @@ use kaspa_consensus_core::{ BlockHashSet, BlockLevel, HashMapCustomHasher, }; use kaspa_consensus_notify::root::ConsensusNotificationRoot; -use kaspa_core::{info, task::service::AsyncService, task::tick::TickService, time::unix_now, trace, warn}; +use kaspa_core::{ + info, + task::{service::AsyncService, tick::TickService}, + time::unix_now, + trace, warn, +}; use kaspa_database::prelude::ConnBuilder; use kaspa_database::{create_temp_db, load_existing_db}; use kaspa_hashes::Hash; @@ -133,7 +138,13 @@ fn main() { let args = Args::parse(); // Initialize the logger - kaspa_core::log::init_logger(None, &args.log_level); + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + kaspa_core::log::init_logger(None, &format!("{},{}=debug", args.log_level, kaspa_utils::sync::semaphore_module_path())); + } else { + kaspa_core::log::init_logger(None, &args.log_level); + } + }; // Configure the panic behavior // As we log the panic, we want to set it up after the logger diff --git a/testing/integration/src/common/utils.rs b/testing/integration/src/common/utils.rs index 824bda388..cb87a98c8 100644 --- a/testing/integration/src/common/utils.rs +++ b/testing/integration/src/common/utils.rs @@ -36,8 +36,8 @@ const fn estimated_mass(num_inputs: usize, num_outputs: u64) -> u64 { } pub const fn required_fee(num_inputs: usize, num_outputs: u64) -> u64 { - const FEE_PER_MASS: u64 = 10; - FEE_PER_MASS * estimated_mass(num_inputs, num_outputs) + const FEE_RATE: u64 = 10; + FEE_RATE * estimated_mass(num_inputs, num_outputs) } /// Builds a TX DAG based on the initial UTXO set and on constant params diff --git a/testing/integration/src/mempool_benchmarks.rs b/testing/integration/src/mempool_benchmarks.rs index 3df716594..00d9b7803 100644 --- a/testing/integration/src/mempool_benchmarks.rs +++ b/testing/integration/src/mempool_benchmarks.rs @@ -295,8 +295,8 @@ async fn bench_bbt_latency_2() { const BLOCK_COUNT: usize = usize::MAX; const MEMPOOL_TARGET: u64 = 600_000; - const TX_COUNT: usize = 1_400_000; - const TX_LEVEL_WIDTH: usize = 20_000; + const TX_COUNT: usize = 1_000_000; + const TX_LEVEL_WIDTH: usize = 300_000; const TPS_PRESSURE: u64 = u64::MAX; const SUBMIT_BLOCK_CLIENTS: usize = 20; diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index 3224cefee..dc26b539f 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -301,6 +301,17 @@ async fn sanity_test() { }) } + KaspadPayloadOps::SubmitTransactionReplacement => { + let rpc_client = client.clone(); + tst!(op, { + // Build an erroneous transaction... + let transaction = Transaction::new(0, vec![], vec![], 0, SubnetworkId::default(), 0, vec![]); + let result = rpc_client.submit_transaction_replacement((&transaction).into()).await; + // ...that gets rejected by the consensus + assert!(result.is_err()); + }) + } + KaspadPayloadOps::GetSubnetwork => { let rpc_client = client.clone(); tst!(op, { @@ -546,6 +557,33 @@ async fn sanity_test() { }) } + KaspadPayloadOps::GetFeeEstimate => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate().await.unwrap(); + info!("{:?}", response.priority_bucket); + assert!(!response.normal_buckets.is_empty()); + assert!(!response.low_buckets.is_empty()); + for bucket in response.ordered_buckets() { + info!("{:?}", bucket); + } + }) + } + + KaspadPayloadOps::GetFeeEstimateExperimental => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate_experimental(true).await.unwrap(); + assert!(!response.estimate.normal_buckets.is_empty()); + assert!(!response.estimate.low_buckets.is_empty()); + for bucket in response.estimate.ordered_buckets() { + info!("{:?}", bucket); + } + assert!(response.verbose.is_some()); + info!("{:?}", response.verbose); + }) + } + KaspadPayloadOps::NotifyBlockAdded => { let rpc_client = client.clone(); let id = listener_id; diff --git a/testing/integration/src/tasks/tx/sender.rs b/testing/integration/src/tasks/tx/sender.rs index 26a334a76..d29e74373 100644 --- a/testing/integration/src/tasks/tx/sender.rs +++ b/testing/integration/src/tasks/tx/sender.rs @@ -114,7 +114,7 @@ impl Task for TransactionSenderTask { break; } prev_mempool_size = mempool_size; - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; } if stopper == Stopper::Signal { warn!("Tx sender task signaling to stop"); diff --git a/utils/Cargo.toml b/utils/Cargo.toml index a3002afab..641ecb61a 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -10,7 +10,7 @@ license.workspace = true repository.workspace = true [dependencies] -parking_lot.workspace = true +arc-swap.workspace = true async-channel.workspace = true borsh.workspace = true cfg-if.workspace = true @@ -18,12 +18,14 @@ event-listener.workspace = true faster-hex.workspace = true ipnet.workspace = true itertools.workspace = true +log.workspace = true +once_cell.workspace = true +parking_lot.workspace = true serde.workspace = true smallvec.workspace = true thiserror.workspace = true triggered.workspace = true uuid.workspace = true -log.workspace = true wasm-bindgen.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] @@ -41,3 +43,6 @@ rand.workspace = true [[bench]] name = "bench" harness = false + +[features] +semaphore-trace = [] diff --git a/utils/alloc/Cargo.toml b/utils/alloc/Cargo.toml index 4a3068f25..be07fd988 100644 --- a/utils/alloc/Cargo.toml +++ b/utils/alloc/Cargo.toml @@ -10,13 +10,13 @@ include.workspace = true repository.workspace = true [target.'cfg(not(target_os = "macos"))'.dependencies] -mimalloc = { version = "0.1.39", default-features = false, features = [ +mimalloc = { version = "0.1.43", default-features = false, features = [ 'override', ] } [target.'cfg(target_os = "macos")'.dependencies] # override is unstable in MacOS and is thus excluded -mimalloc = { version = "0.1.39", default-features = false } +mimalloc = { version = "0.1.43", default-features = false } [features] heap = [] diff --git a/utils/src/expiring_cache.rs b/utils/src/expiring_cache.rs new file mode 100644 index 000000000..175bea548 --- /dev/null +++ b/utils/src/expiring_cache.rs @@ -0,0 +1,152 @@ +use arc_swap::ArcSwapOption; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +struct Entry { + item: T, + timestamp: Instant, +} + +/// An expiring cache for a single object +pub struct ExpiringCache { + store: ArcSwapOption>, + refetch: Duration, + expire: Duration, + fetching: AtomicBool, +} + +impl ExpiringCache { + /// Constructs a new expiring cache where `fetch` is the amount of time required to trigger a data + /// refetch and `expire` is the time duration after which the stored item is guaranteed not to be returned. + /// + /// Panics if `refetch > expire`. + pub fn new(refetch: Duration, expire: Duration) -> Self { + assert!(refetch <= expire); + Self { store: Default::default(), refetch, expire, fetching: Default::default() } + } + + /// Returns the cached item or possibly fetches a new one using the `refetch_future` task. The + /// decision whether to refetch depends on the configured expiration and refetch times for this cache. + pub async fn get(&self, refetch_future: F) -> T + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let mut fetching = false; + + { + let guard = self.store.load(); + if let Some(entry) = guard.as_ref() { + if let Some(elapsed) = Instant::now().checked_duration_since(entry.timestamp) { + if elapsed < self.refetch { + return entry.item.clone(); + } + // Refetch is triggered, attempt to capture the task + fetching = self.fetching.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok(); + // If the fetch task is not captured and expire time is not over yet, return with prev value. Another + // thread is refetching the data but we can return with the not-too-old value + if !fetching && elapsed < self.expire { + return entry.item.clone(); + } + } + // else -- In rare cases where now < timestamp, fall through to re-update the cache + } + } + + // We reach here if either we are the refetching thread or the current data has fully expired + let new_item = refetch_future.await; + let timestamp = Instant::now(); + // Update the store even if we were not in charge of refetching - let the last thread make the final update + self.store.store(Some(Arc::new(Entry { item: new_item.clone(), timestamp }))); + + if fetching { + let result = self.fetching.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst); + assert!(result.is_ok(), "refetching was captured") + } + + new_item + } +} + +#[cfg(test)] +mod tests { + use super::ExpiringCache; + use std::time::Duration; + use tokio::join; + + #[tokio::test] + #[ignore] + // Tested during development but can be sensitive to runtime machine times so there's no point + // in keeping it part of CI. The test should be activated if the ExpiringCache struct changes. + async fn test_expiring_cache() { + let fetch = Duration::from_millis(500); + let expire = Duration::from_millis(1000); + let mid_point = Duration::from_millis(700); + let expire_point = Duration::from_millis(1200); + let cache: ExpiringCache = ExpiringCache::new(fetch, expire); + + // Test two consecutive calls + let item1 = cache + .get(async move { + println!("first call"); + 1 + }) + .await; + assert_eq!(1, item1); + let item2 = cache + .get(async move { + // cache was just updated with item1, refetch should not be triggered + panic!("should not be called"); + }) + .await; + assert_eq!(1, item2); + + // Test two calls after refetch point + // Sleep until after the refetch point but before expire + tokio::time::sleep(mid_point).await; + let call3 = cache.get(async move { + println!("third call before sleep"); + // keep this refetch busy so that call4 still gets the first item + tokio::time::sleep(Duration::from_millis(100)).await; + println!("third call after sleep"); + 3 + }); + let call4 = cache.get(async move { + // refetch is captured by call3 and we should be before expire + panic!("should not be called"); + }); + let (item3, item4) = join!(call3, call4); + println!("item 3: {}, item 4: {}", item3, item4); + assert_eq!(3, item3); + assert_eq!(1, item4); + + // Test 2 calls after expire + tokio::time::sleep(expire_point).await; + let call5 = cache.get(async move { + println!("5th call before sleep"); + tokio::time::sleep(Duration::from_millis(100)).await; + println!("5th call after sleep"); + 5 + }); + let call6 = cache.get(async move { 6 }); + let (item5, item6) = join!(call5, call6); + println!("item 5: {}, item 6: {}", item5, item6); + assert_eq!(5, item5); + assert_eq!(6, item6); + + let item7 = cache + .get(async move { + // cache was just updated with item5, refetch should not be triggered + panic!("should not be called"); + }) + .await; + // call 5 finished after call 6 + assert_eq!(5, item7); + } +} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index bd3143719..79e96c44f 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -2,6 +2,7 @@ pub mod any; pub mod arc; pub mod binary_heap; pub mod channel; +pub mod expiring_cache; pub mod hashmap; pub mod hex; pub mod iter; diff --git a/utils/src/sync/mod.rs b/utils/src/sync/mod.rs index 40fb147cb..14afe7977 100644 --- a/utils/src/sync/mod.rs +++ b/utils/src/sync/mod.rs @@ -1,2 +1,7 @@ pub mod rwlock; pub(crate) mod semaphore; + +#[cfg(feature = "semaphore-trace")] +pub fn semaphore_module_path() -> &'static str { + semaphore::get_module_path() +} diff --git a/utils/src/sync/semaphore.rs b/utils/src/sync/semaphore.rs index 4b94e8f2f..2ea6dcc03 100644 --- a/utils/src/sync/semaphore.rs +++ b/utils/src/sync/semaphore.rs @@ -4,6 +4,64 @@ use std::{ time::Duration, }; +#[cfg(feature = "semaphore-trace")] +mod trace { + use super::*; + use log::debug; + use once_cell::sync::Lazy; + use std::sync::atomic::AtomicU64; + use std::time::SystemTime; + + static SYS_START: Lazy = Lazy::new(SystemTime::now); + + #[inline] + pub(super) fn sys_now() -> u64 { + SystemTime::now().duration_since(*SYS_START).unwrap_or_default().as_micros() as u64 + } + + #[derive(Debug, Default)] + pub struct TraceInner { + readers_start: AtomicU64, + readers_time: AtomicU64, + log_time: AtomicU64, + log_value: AtomicU64, + } + + impl TraceInner { + pub(super) fn mark_readers_start(&self) { + self.readers_start.store(sys_now(), Ordering::Relaxed); + } + + pub(super) fn mark_readers_end(&self) { + let start = self.readers_start.load(Ordering::Relaxed); + let now = sys_now(); + if start < now { + let readers_time = self.readers_time.fetch_add(now - start, Ordering::Relaxed) + now - start; + let log_time = self.log_time.load(Ordering::Relaxed); + if log_time + (Duration::from_secs(10).as_micros() as u64) < now { + let log_value = self.log_value.load(Ordering::Relaxed); + debug!( + "Semaphore: log interval: {:?}, readers time: {:?}, fraction: {:.2}", + Duration::from_micros(now - log_time), + Duration::from_micros(readers_time - log_value), + (readers_time - log_value) as f64 / (now - log_time) as f64 + ); + self.log_value.store(readers_time, Ordering::Relaxed); + self.log_time.store(now, Ordering::Relaxed); + } + } + } + } +} + +#[cfg(feature = "semaphore-trace")] +use trace::*; + +#[cfg(feature = "semaphore-trace")] +pub(crate) fn get_module_path() -> &'static str { + module_path!() +} + /// A low-level non-fair semaphore. The semaphore is non-fair in the sense that clients acquiring /// a lower number of permits might get their allocation before earlier clients which requested more /// permits -- if the semaphore can provide the lower allocation but not the larger. This non-fairness @@ -15,13 +73,28 @@ use std::{ pub(crate) struct Semaphore { counter: AtomicUsize, signal: Event, + #[cfg(feature = "semaphore-trace")] + trace_inner: TraceInner, } impl Semaphore { pub const MAX_PERMITS: usize = usize::MAX; - pub const fn new(available_permits: usize) -> Semaphore { - Semaphore { counter: AtomicUsize::new(available_permits), signal: Event::new() } + pub fn new(available_permits: usize) -> Semaphore { + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + Semaphore { + counter: AtomicUsize::new(available_permits), + signal: Event::new(), + trace_inner: Default::default(), + } + } else { + Semaphore { + counter: AtomicUsize::new(available_permits), + signal: Event::new(), + } + } + } } /// Tries to acquire `permits` slots from the semaphore. Upon success, returns the acquired slot @@ -33,7 +106,14 @@ impl Semaphore { } match self.counter.compare_exchange_weak(count, count - permits, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => return Some(count), + Ok(_) => { + #[cfg(feature = "semaphore-trace")] + if permits == 1 && count == Self::MAX_PERMITS { + // permits == 1 indicates a reader, count == Self::MAX_PERMITS indicates it is the first reader + self.trace_inner.mark_readers_start(); + } + return Some(count); + } Err(c) => count = c, } } @@ -75,6 +155,12 @@ impl Semaphore { /// Returns the released slot pub fn release(&self, permits: usize) -> usize { let slot = self.counter.fetch_add(permits, Ordering::AcqRel) + permits; + + #[cfg(feature = "semaphore-trace")] + if permits == 1 && slot == Self::MAX_PERMITS { + // permits == 1 indicates a reader, slot == Self::MAX_PERMITS indicates it is the last reader + self.trace_inner.mark_readers_end(); + } self.signal.notify(permits); slot } diff --git a/utils/src/vec.rs b/utils/src/vec.rs index 01bd59b9e..fa1d67a27 100644 --- a/utils/src/vec.rs +++ b/utils/src/vec.rs @@ -4,6 +4,10 @@ pub trait VecExtensions { /// Inserts the provided `value` at `index` while swapping the item at index to the end of the container fn swap_insert(&mut self, index: usize, value: T); + + /// Merges two containers one into the other and returns the result. The method is identical + /// to [`Vec::append`] but can be used more ergonomically in a fluent calling fashion + fn merge(self, other: Self) -> Self; } impl VecExtensions for Vec { @@ -19,4 +23,9 @@ impl VecExtensions for Vec { let loc = self.len() - 1; self.swap(index, loc); } + + fn merge(mut self, mut other: Self) -> Self { + self.append(&mut other); + self + } } diff --git a/wallet/core/src/storage/local/transaction/fsio.rs b/wallet/core/src/storage/local/transaction/fsio.rs index ac44a136d..b4a74334c 100644 --- a/wallet/core/src/storage/local/transaction/fsio.rs +++ b/wallet/core/src/storage/local/transaction/fsio.rs @@ -115,7 +115,7 @@ impl TransactionRecordStore for TransactionStore { let mut transactions = vec![]; for id in ids { - let path = folder.join(&id.to_hex()); + let path = folder.join(id.to_hex()); match read(&path, None).await { Ok(tx) => { transactions.push(Arc::new(tx)); @@ -144,7 +144,7 @@ impl TransactionRecordStore for TransactionStore { let mut located = 0; for id in ids { - let path = folder.join(&id.to_hex()); + let path = folder.join(id.to_hex()); match read(&path, None).await { Ok(tx) => { @@ -167,7 +167,7 @@ impl TransactionRecordStore for TransactionStore { let iter = ids.iter().skip(range.start).take(range.len()); for id in iter { - let path = folder.join(&id.to_hex()); + let path = folder.join(id.to_hex()); match read(&path, None).await { Ok(tx) => { transactions.push(Arc::new(tx)); diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index 6c335d59a..1e6d70c2b 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -151,6 +151,13 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn submit_transaction_replacement_call( + &self, + _request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { Err(RpcError::NotImplemented) } @@ -245,6 +252,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/wallet/core/src/tx/generator/generator.rs b/wallet/core/src/tx/generator/generator.rs index 533f2e046..7040276b1 100644 --- a/wallet/core/src/tx/generator/generator.rs +++ b/wallet/core/src/tx/generator/generator.rs @@ -39,21 +39,22 @@ //! interface or via an async Stream interface. //! //! Q: Why is this not implemented as a single loop? +//! //! A: There are a number of requirements that need to be handled: //! -//! 1. UTXO entry consumption while creating inputs may results in -//! additional fees, requiring additional UTXO entries to cover -//! the fees. Goto 1. (this is a classic issue, can be solved using padding) +//! 1. UTXO entry consumption while creating inputs may result in +//! additional fees, requiring additional UTXO entries to cover +//! the fees. Goto 1. (this is a classic issue, can be solved using padding) //! -//! 2. The overall design strategy for this processor is to allow -//! concurrent processing of a large number of transactions and UTXOs. -//! This implementation avoids in-memory aggregation of all -//! transactions that may result in OOM conditions. +//! 2. The overall design strategy for this processor is to allow +//! concurrent processing of a large number of transactions and UTXOs. +//! This implementation avoids in-memory aggregation of all +//! transactions that may result in OOM conditions. //! -//! 3. If used with a large UTXO set, the transaction generation process -//! needs to be asynchronous to avoid blocking the main thread. In the -//! context of WASM32 SDK, not doing that while working with large -//! UTXO sets will result in a browser UI freezing. +//! 3. If used with a large UTXO set, the transaction generation process +//! needs to be asynchronous to avoid blocking the main thread. In the +//! context of WASM32 SDK, not doing that while working with large +//! UTXO sets will result in a browser UI freezing. //! use crate::imports::*; @@ -553,16 +554,18 @@ impl Generator { /// /// The general processing pattern can be described as follows: /// - /// loop { - /// 1. Obtain UTXO entry from [`Generator::get_utxo_entry()`] - /// 2. Check if UTXO entries have been depleted, if so, handle sweep processing. - /// 3. Create a new Input for the transaction from the UTXO entry. - /// 4. Check if the transaction mass threshold has been reached, if so, yield the transaction. - /// 5. Register input with the [`Data`] structures. - /// 6. Check if the final transaction amount has been reached, if so, yield the transaction. - /// } - /// - /// + /** + loop { + 1. Obtain UTXO entry from [`Generator::get_utxo_entry()`] + 2. Check if UTXO entries have been depleted, if so, handle sweep processing. + 3. Create a new Input for the transaction from the UTXO entry. + 4. Check if the transaction mass threshold has been reached, if so, yield the transaction. + 5. Register input with the [`Data`] structures. + 6. Check if the final transaction amount has been reached, if so, yield the transaction. + + } + */ + fn generate_transaction_data(&self, context: &mut Context, stage: &mut Stage) -> Result<(DataKind, Data)> { let calc = &self.inner.mass_calculator; let mut data = Data::new(calc); diff --git a/wallet/core/src/utxo/context.rs b/wallet/core/src/utxo/context.rs index 51ef0e5ea..26e19bc55 100644 --- a/wallet/core/src/utxo/context.rs +++ b/wallet/core/src/utxo/context.rs @@ -568,6 +568,7 @@ impl UtxoContext { // remove UTXOs from account set let outgoing_transactions = self.processor().outgoing(); + #[allow(clippy::mutable_key_type)] let mut accepted_outgoing_transactions = HashSet::::new(); utxos.retain(|utxo| { diff --git a/wallet/core/src/utxo/processor.rs b/wallet/core/src/utxo/processor.rs index e788272f2..49bebbd7a 100644 --- a/wallet/core/src/utxo/processor.rs +++ b/wallet/core/src/utxo/processor.rs @@ -264,6 +264,7 @@ impl UtxoProcessor { Ok(()) } + #[allow(clippy::mutable_key_type)] pub async fn handle_pending(&self, current_daa_score: u64) -> Result<()> { let params = self.network_params()?; @@ -388,6 +389,7 @@ impl UtxoProcessor { pub async fn handle_utxo_changed(&self, utxos: UtxosChangedNotification) -> Result<()> { let current_daa_score = self.current_daa_score().expect("DAA score expected when handling UTXO Changed notifications"); + #[allow(clippy::mutable_key_type)] let mut updated_contexts: HashSet = HashSet::default(); let removed = (*utxos.removed).clone().into_iter().filter_map(|entry| entry.address.clone().map(|address| (address, entry))); diff --git a/wallet/pskt/examples/multisig.rs b/wallet/pskt/examples/multisig.rs index 96659d294..a34bef9b5 100644 --- a/wallet/pskt/examples/multisig.rs +++ b/wallet/pskt/examples/multisig.rs @@ -1,5 +1,5 @@ use kaspa_consensus_core::{ - hashing::sighash::{calc_schnorr_signature_hash, SigHashReusedValuesUnsync}, + hashing::sighash::{calc_schnorr_signature_hash, SigHashReusedValues}, tx::{TransactionId, TransactionOutpoint, UtxoEntry}, }; use kaspa_txscript::{multisig_redeem_script, opcodes::codes::OpData65, pay_to_script_hash_script, script_builder::ScriptBuilder}; @@ -49,8 +49,8 @@ fn main() { println!("Serialized after setting sequence: {}", ser_updated); let signer_pskt: PSKT = serde_json::from_str(&ser_updated).expect("Failed to deserialize"); - let reused_values = SigHashReusedValuesUnsync::new(); - let sign = |signer_pskt: PSKT, kp: &Keypair| { + let mut reused_values = SigHashReusedValues::new(); + let mut sign = |signer_pskt: PSKT, kp: &Keypair| { signer_pskt .pass_signature_sync(|tx, sighash| -> Result, String> { let tx = dbg!(tx); @@ -59,7 +59,7 @@ fn main() { .iter() .enumerate() .map(|(idx, _input)| { - let hash = calc_schnorr_signature_hash(&tx.as_verifiable(), idx, sighash[idx], &reused_values); + let hash = calc_schnorr_signature_hash(&tx.as_verifiable(), idx, sighash[idx], &mut reused_values); let msg = secp256k1::Message::from_digest_slice(hash.as_bytes().as_slice()).unwrap(); Ok(SignInputOk { signature: Signature::Schnorr(kp.sign_schnorr(msg)), diff --git a/wallet/pskt/src/lib.rs b/wallet/pskt/src/lib.rs index 060aded57..e26d5c9ea 100644 --- a/wallet/pskt/src/lib.rs +++ b/wallet/pskt/src/lib.rs @@ -1,5 +1,4 @@ use kaspa_bip32::{secp256k1, DerivationPath, KeyFingerprint}; -use kaspa_consensus_core::hashing::sighash::SigHashReusedValuesUnsync; use serde::{Deserialize, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; use std::{collections::BTreeMap, fmt::Display, fmt::Formatter, future::Future, marker::PhantomData, ops::Deref}; @@ -18,7 +17,7 @@ pub use global::{Global, GlobalBuilder}; pub use input::{Input, InputBuilder}; use kaspa_consensus_core::tx::UtxoEntry; use kaspa_consensus_core::{ - hashing::sighash_type::SigHashType, + hashing::{sighash::SigHashReusedValues, sighash_type::SigHashType}, subnets::SUBNETWORK_ID_NATIVE, tx::{MutableTransaction, SignableTransaction, Transaction, TransactionId, TransactionInput, TransactionOutput}, }; @@ -398,10 +397,10 @@ impl PSKT { { let tx = tx.as_verifiable(); let cache = Cache::new(10_000); - let reused_values = SigHashReusedValuesUnsync::new(); + let mut reused_values = SigHashReusedValues::new(); tx.populated_inputs().enumerate().try_for_each(|(idx, (input, entry))| { - TxScriptEngine::from_transaction_input(&tx, input, idx, entry, &reused_values, &cache)?.execute()?; + TxScriptEngine::from_transaction_input(&tx, input, idx, entry, &mut reused_values, &cache)?.execute()?; >::Ok(()) })?; }