From 18cb3c3dc522528c7432780f3bf65eccd5e40098 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Fri, 1 Jun 2018 01:34:57 +0300 Subject: [PATCH 01/18] Aux classes --- .../Fee/EstimationResult.cs | 17 +++++++++++++ .../Fee/EstimatorBucket.cs | 25 +++++++++++++++++++ .../Fee/FeeCalculation.cs | 14 +++++++++++ .../Fee/FeeEstimateHorizon.cs | 17 +++++++++++++ .../Fee/FeeEstimateMode.cs | 16 ++++++++++++ .../Fee/FeeReason.cs | 23 +++++++++++++++++ 6 files changed, 112 insertions(+) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs new file mode 100644 index 00000000000..b8e67befe26 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Used to return detailed information about a fee estimate calculation + /// + public class EstimationResult + { + public EstimatorBucket Pass { get; set; } + public EstimatorBucket Fail { get; set; } + public double Decay { get; set; } + public int Scale { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs new file mode 100644 index 00000000000..a10541fb275 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Used to return detailed information about a feerate bucket + /// + public class EstimatorBucket + { + public double Start { get; set; } + public double End { get; set; } + public double WithinTarget { get; set; } + public double TotalConfirmed { get; set; } + public double InMempool { get; set; } + public double LeftMempool { get; set; } + + public EstimatorBucket() + { + this.Start = -1; + this.End = -1; + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs new file mode 100644 index 00000000000..9d7e9c86b62 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public class FeeCalculation + { + public EstimationResult Estimation { get; set; } + public FeeReason Reason { get; set; } + public int DesiredTarget { get; set; } + public int ReturnedTarget { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs new file mode 100644 index 00000000000..283b8f73134 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs @@ -0,0 +1,17 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Identifier for each of the 3 different TxConfirmStats which will track + /// history over different time horizons + /// + public enum FeeEstimateHorizon + { + ShortHalfLife, + MedHalfLife, + LongHalfLife + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs new file mode 100644 index 00000000000..dc50a38bb35 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Used to determine type of fee estimation requested + /// + public enum FeeEstimateMode + { + Unset, + Economical, + Conservative + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs new file mode 100644 index 00000000000..c53feef1a3d --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Enumeration of reason for returned fee estimate + /// + public enum FeeReason + { + None, + HalfEstimate, + FullEstimate, + DoubleEstimate, + Coservative, + MemPoolMin, + PayTxFee, + Fallback, + Required, + MaxTxFee + } +} From a50d777df80bf220893769ed83a440df3ab5612b Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Fri, 1 Jun 2018 18:59:58 +0300 Subject: [PATCH 02/18] TxConfirmStats start rewriting --- .../Fee/Algorithm014/BlockPolicyEstimator.cs | 405 ++++++++++++++++++ .../Fee/Algorithm014/TxConfirmStats.cs | 358 ++++++++++++++++ .../Fee/TxConfirmStats.cs | 122 +++--- 3 files changed, 829 insertions(+), 56 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs new file mode 100644 index 00000000000..b23d4254390 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs @@ -0,0 +1,405 @@ +using System.Collections.Generic; +using System.IO; +using Microsoft.Extensions.Logging; +using NBitcoin; +using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Features.MemoryPool.Interfaces; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.Algorithm014 +{ + /// + /// The BlockPolicyEstimator is used for estimating the feerate needed + /// for a transaction to be included in a block within a certain number of + /// blocks. + /// + /// + /// At a high level the algorithm works by grouping transactions into buckets + /// based on having similar feerates and then tracking how long it + /// takes transactions in the various buckets to be mined. It operates under + /// the assumption that in general transactions of higher feerate will be + /// included in blocks before transactions of lower feerate. So for + /// example if you wanted to know what feerate you should put on a transaction to + /// be included in a block within the next 5 blocks, you would start by looking + /// at the bucket with the highest feerate transactions and verifying that a + /// sufficiently high percentage of them were confirmed within 5 blocks and + /// then you would look at the next highest feerate bucket, and so on, stopping at + /// the last bucket to pass the test. The average feerate of transactions in this + /// bucket will give you an indication of the lowest feerate you can put on a + /// transaction and still have a sufficiently high chance of being confirmed + /// within your desired 5 blocks. + /// + /// Here is a brief description of the implementation: + /// When a transaction enters the mempool, we + /// track the height of the block chain at entry. Whenever a block comes in, + /// we count the number of transactions in each bucket and the total amount of feerate + /// paid in each bucket. Then we calculate how many blocks Y it took each + /// transaction to be mined and we track an array of counters in each bucket + /// for how long it to took transactions to get confirmed from 1 to a max of 25 + /// and we increment all the counters from Y up to 25. This is because for any + /// number Z>=Y the transaction was successfully mined within Z blocks. We + /// want to save a history of this information, so at any time we have a + /// counter of the total number of transactions that happened in a given feerate + /// bucket and the total number that were confirmed in each number 1-25 blocks + /// or less for any bucket. We save this history by keeping an exponentially + /// decaying moving average of each one of these stats. Furthermore we also + /// keep track of the number unmined (in mempool) transactions in each bucket + /// and for how many blocks they have been outstanding and use that to increase + /// the number of transactions we've seen in that feerate bucket when calculating + /// an estimate for any number of confirmations below the number of blocks + /// they've been outstanding. + /// + /// We will instantiate an instance of this class to track transactions that were + /// included in a block. We will lump transactions into a bucket according to their + /// approximate feerate and then track how long it took for those txs to be included in a block + /// + /// The tracking of unconfirmed (mempool) transactions is completely independent of the + /// historical tracking of transactions that have been confirmed in a block. + /// + /// We want to be able to estimate feerates that are needed on tx's to be included in + /// a certain number of blocks.Every time a block is added to the best chain, this class records + /// stats on the transactions included in that block + /// + public class BlockPolicyEstimator + { + /// Require an avg of 1 tx in the combined feerate bucket per block to have stat significance. + private const double SufficientFeeTxs = 1; + + /// Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough. + private const double MinSuccessPct = .95; + + /// Minimum value for tracking feerates. + private const long MinFeeRate = 10; + + /// Maximum value for tracking feerates. + private const double MaxFeeRate = 1e7; + + /// + /// Spacing of FeeRate buckets. + /// + /// + /// We have to lump transactions into buckets based on feerate, but we want to be able + /// to give accurate estimates over a large range of potential feerates. + /// Therefore it makes sense to exponentially space the buckets. + /// + private const double FeeSpacing = 1.1; + + /// Track confirm delays up to 25 blocks, can't estimate beyond that. + private const int MaxBlockConfirms = 25; + + /// Decay of .998 is a half-life of 346 blocks or about 2.4 days. + private const double DefaultDecay = .998; + + /// Value for infinite priority. + public const double InfPriority = 1e9 * 21000000ul * Money.COIN; + + /// Maximum money value. + private static readonly Money MaxMoney = new Money(21000000 * Money.COIN); + + /// Value for infinite fee rate. + private static readonly double InfFeeRate = MaxMoney.Satoshi; + + /// Classes to track historical data on transaction confirmations. + private readonly TxConfirmStats feeStats; + + /// Map of txids to information about that transaction. + private readonly Dictionary mapMemPoolTxs; + + /// Minimum tracked Fee. Passed to constructor to avoid dependency on main./// + private readonly FeeRate minTrackedFee; + + /// Best seen block height. + private int nBestSeenHeight; + + /// Setting for the node. + private readonly MempoolSettings mempoolSettings; + + /// Logger for logging on this object. + private readonly ILogger logger; + + /// Count of tracked transactions. + private int trackedTxs; + + /// Count of untracked transactions. + private int untrackedTxs; + + /// + /// Constructs an instance of the block policy estimator object. + /// + /// Mempool settings. + /// Factory for creating loggers. + /// Full node settings. + public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory loggerFactory, NodeSettings nodeSettings) + { + this.mapMemPoolTxs = new Dictionary(); + this.mempoolSettings = mempoolSettings; + this.nBestSeenHeight = 0; + this.trackedTxs = 0; + this.untrackedTxs = 0; + this.logger = loggerFactory.CreateLogger(this.GetType().FullName); + + this.minTrackedFee = nodeSettings.MinRelayTxFeeRate < new FeeRate(new Money(MinFeeRate)) + ? new FeeRate(new Money(MinFeeRate)) + : nodeSettings.MinRelayTxFeeRate; + var vfeelist = new List(); + for (double bucketBoundary = this.minTrackedFee.FeePerK.Satoshi; + bucketBoundary <= MaxFeeRate; + bucketBoundary *= FeeSpacing) + vfeelist.Add(bucketBoundary); + vfeelist.Add(InfFeeRate); + this.feeStats = new TxConfirmStats(this.logger); + this.feeStats.Initialize(vfeelist, MaxBlockConfirms, DefaultDecay); + } + + /// + /// Process all the transactions that have been included in a block. + /// + /// The block height for the block. + /// Collection of memory pool entries. + public void ProcessBlock(int nBlockHeight, List entries) + { + if (nBlockHeight <= this.nBestSeenHeight) + return; + + // Must update nBestSeenHeight in sync with ClearCurrent so that + // calls to removeTx (via processBlockTx) correctly calculate age + // of unconfirmed txs to remove from tracking. + this.nBestSeenHeight = nBlockHeight; + + // Clear the current block state and update unconfirmed circular buffer + this.feeStats.ClearCurrent(nBlockHeight); + + int countedTxs = 0; + // Repopulate the current block states + for (int i = 0; i < entries.Count; i++) + if (this.ProcessBlockTx(nBlockHeight, entries[i])) + countedTxs++; + + // Update all exponential averages with the current block state + this.feeStats.UpdateMovingAverages(); + + // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) + // Logging.Logs.EstimateFee.LogInformation( + // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); + + this.trackedTxs = 0; + this.untrackedTxs = 0; + } + + /// + /// Process a transaction confirmed in a block. + /// + /// Height of the block. + /// The memory pool entry. + /// Whether it was able to successfully process the transaction. + private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) + { + if (!this.RemoveTx(entry.TransactionHash)) + return false; + + // How many blocks did it take for miners to include this transaction? + // blocksToConfirm is 1-based, so a transaction included in the earliest + // possible block has confirmation count of 1 + int blocksToConfirm = nBlockHeight - entry.EntryHeight; + if (blocksToConfirm <= 0) + { + // This can't happen because we don't process transactions from a block with a height + // lower than our greatest seen height + this.logger.LogInformation($"Blockpolicy error Transaction had negative blocksToConfirm"); + return false; + } + + // Feerates are stored and reported as BTC-per-kb: + FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + return true; + } + + /// + /// Process a transaction accepted to the mempool. + /// + /// Memory pool entry. + /// Whether to update fee estimate. + public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) + { + int txHeight = entry.EntryHeight; + uint256 hash = entry.TransactionHash; + if (this.mapMemPoolTxs.ContainsKey(hash)) + { + this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); + return; + } + + if (txHeight != this.nBestSeenHeight) + return; + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!validFeeEstimate) + { + this.untrackedTxs++; + return; + } + this.trackedTxs++; + + // Feerates are stored and reported as BTC-per-kb: + FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); + this.mapMemPoolTxs[hash].blockHeight = txHeight; + this.mapMemPoolTxs[hash].bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + } + + /// + /// Remove a transaction from the mempool tracking stats. + /// + /// Transaction hash. + /// Whether the transaction was successfully removed. + /// + /// This function is called from TxMemPool.RemoveUnchecked to ensure + /// txs removed from the mempool for any reason are no longer + /// tracked. Txs that were part of a block have already been removed in + /// ProcessBlockTx to ensure they are never double tracked, but it is + /// of no harm to try to remove them again. + /// + public bool RemoveTx(uint256 hash) + { + TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); + if (pos != null) + { + this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex); + this.mapMemPoolTxs.Remove(hash); + return true; + } + return false; + } + + /// + /// Return a feerate estimate + /// + /// The desired number of confirmations to be included in a block. + public FeeRate EstimateFee(int confTarget) + { + // Return failure if trying to analyze a target we're not tracking + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget <= 1 || confTarget > this.feeStats.GetMaxConfirms()) + return new FeeRate(0); + + double median = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, MinSuccessPct, true, + this.nBestSeenHeight); + + if (median < 0) + return new FeeRate(0); + + return new FeeRate(new Money((int)median)); + } + + /// + /// Estimate feerate needed to be included in a block within + /// confTarget blocks. If no answer can be given at confTarget, return an + /// estimate at the lowest target where one can be given. + /// + public FeeRate EstimateSmartFee(int confTarget, ITxMempool pool, out int answerFoundAtTarget) + { + answerFoundAtTarget = confTarget; + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > this.feeStats.GetMaxConfirms()) + return new FeeRate(0); + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + double median = -1; + while (median < 0 && confTarget <= this.feeStats.GetMaxConfirms()) + median = this.feeStats.EstimateMedianVal(confTarget++, SufficientFeeTxs, MinSuccessPct, true, + this.nBestSeenHeight); + + answerFoundAtTarget = confTarget - 1; + + // If mempool is limiting txs , return at least the min feerate from the mempool + if (pool != null) + { + Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; + if (minPoolFee > 0 && minPoolFee.Satoshi > median) + return new FeeRate(minPoolFee); + } + + if (median < 0) + return new FeeRate(0); + + return new FeeRate((int)median); + } + + /// + /// Write estimation data to a file. + /// + /// Stream to write to. + /// TODO: Implement write estimation + public void Write(Stream fileout) + { + } + + /// + /// Read estimation data from a file. + /// + /// Stream to read data from. + /// Version number of the file. + /// TODO: Implement read estimation + public void Read(Stream filein, int nFileVersion) + { + } + + /// + /// Return an estimate of the priority. + /// + /// The desired number of confirmations to be included in a block. + /// Estimate of the priority. + /// TODO: Implement priority estimation + public double EstimatePriority(int confTarget) + { + return -1; + } + + /// + /// Return an estimated smart priority. + /// + /// The desired number of confirmations to be included in a block. + /// Memory pool transactions. + /// Block height where answer was found. + /// The smart priority. + public double EstimateSmartPriority(int confTarget, ITxMempool pool, out int answerFoundAtTarget) + { + answerFoundAtTarget = confTarget; + + // If mempool is limiting txs, no priority txs are allowed + Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; + if (minPoolFee > 0) + return InfPriority; + + return -1; + } + + /// + /// Transaction statistics information. + /// + public class TxStatsInfo + { + /// The block height. + public int blockHeight; + + /// The index into the confirmed transactions bucket map. + public int bucketIndex; + + /// + /// Constructs a instance of a transaction stats info object. + /// + public TxStatsInfo() + { + this.blockHeight = 0; + this.bucketIndex = 0; + } + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs new file mode 100644 index 00000000000..375f123612c --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs @@ -0,0 +1,358 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Extensions.Logging; +using NBitcoin; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.Algorithm014 +{ + /// + /// Transation confirmation statistics. + /// + public class TxConfirmStats + { + /// Instance logger for logging messages. + private readonly ILogger logger; + + /// + /// Moving average of total fee rate of all transactions in each bucket. + /// + /// + /// Track the historical moving average of this total over blocks. + /// + private List avg; + + /// Map of bucket upper-bound to index into all vectors by bucket. + private Dictionary bucketMap; + + //Define the buckets we will group transactions into. + + /// The upper-bound of the range for the bucket (inclusive). + private List buckets; + + // Count the total # of txs confirmed within Y blocks in each bucket. + // Track the historical moving average of theses totals over blocks. + + /// Confirmation average. confAvg[Y][X]. + private List> confAvg; + + /// Current block confirmations. curBlockConf[Y][X]. + private List> curBlockConf; + + /// Current block transaction count. + private List curBlockTxCt; + + /// Current block fee rate. + private List curBlockVal; + + // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X + // Combine the total value with the tx counts to calculate the avg feerate per bucket + + /// Decay value to use. + private double decay; + + /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket + private List oldUnconfTxs; + + /// + /// Historical moving average of transaction counts. + /// + /// + /// For each bucket X: + /// Count the total # of txs in each bucket. + /// Track the historical moving average of this total over blocks + /// + private List txCtAvg; + + /// + /// Mempool counts of outstanding transactions. + /// + /// + /// For each bucket X, track the number of transactions in the mempool + /// that are unconfirmed for each possible confirmation value Y + /// unconfTxs[Y][X] + /// + private List> unconfTxs; + + /// + /// Constructs an instance of the transaction confirmation stats object. + /// + /// Instance logger to use for message logging. + public TxConfirmStats(ILogger logger) + { + this.logger = logger; + } + + /// + /// Initialize the data structures. This is called by BlockPolicyEstimator's + /// constructor with default values. + /// + /// Contains the upper limits for the bucket boundaries. + /// Max number of confirms to track. + /// How much to decay the historical moving average per block. + public void Initialize(List defaultBuckets, int maxConfirms, double decay) + { + this.buckets = new List(); + this.bucketMap = new Dictionary(); + + this.decay = decay; + for (int i = 0; i < defaultBuckets.Count; i++) + { + this.buckets.Add(defaultBuckets[i]); + this.bucketMap[defaultBuckets[i]] = i; + } + this.confAvg = new List>(); + this.curBlockConf = new List>(); + this.unconfTxs = new List>(); + + for (int i = 0; i < maxConfirms; i++) + { + this.confAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); + this.curBlockConf.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); + this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); + } + + this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); + this.curBlockTxCt = new List(Enumerable.Repeat(default(int), this.buckets.Count)); + this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + this.curBlockVal = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + } + + /// + /// Clear the state of the curBlock variables to start counting for the new block. + /// + /// Block height. + public void ClearCurrent(int nBlockHeight) + { + for (var j = 0; j < this.buckets.Count; j++) + { + this.oldUnconfTxs[j] += this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j]; + this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j] = 0; + for (int i = 0; i < this.curBlockConf.Count; i++) + this.curBlockConf[i][j] = 0; + this.curBlockTxCt[j] = 0; + this.curBlockVal[j] = 0; + } + } + + /// + /// Record a new transaction data point in the current block stats. + /// + /// The number of blocks it took this transaction to confirm. blocksToConfirm is 1-based and has to be >= 1. + /// The feerate of the transaction. + public void Record(int blocksToConfirm, double val) + { + // blocksToConfirm is 1-based + if (blocksToConfirm < 1) + return; + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; + for (int i = blocksToConfirm; i <= this.curBlockConf.Count; i++) + this.curBlockConf[i - 1][bucketindex]++; + this.curBlockTxCt[bucketindex]++; + this.curBlockVal[bucketindex] += val; + } + + /// + /// Record a new transaction entering the mempool. + /// + /// The block height. + /// + /// The feerate of the transaction. + public int NewTx(int nBlockHeight, double val) + { + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; + int blockIndex = nBlockHeight % this.unconfTxs.Count; + this.unconfTxs[blockIndex][bucketindex]++; + return bucketindex; + } + + /// + /// Remove a transaction from mempool tracking stats. + /// + /// The height of the mempool entry. + /// The best sceen height. + /// The bucket index. + public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex) + { + //nBestSeenHeight is not updated yet for the new block + int blocksAgo = nBestSeenHeight - entryHeight; + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet + blocksAgo = 0; + if (blocksAgo < 0) + { + this.logger.LogInformation($"Blockpolicy error, blocks ago is negative for mempool tx"); + return; //This can't happen because we call this with our best seen height, no entries can have higher + } + + if (blocksAgo >= this.unconfTxs.Count) + { + if (this.oldUnconfTxs[bucketIndex] > 0) + this.oldUnconfTxs[bucketIndex]--; + else + this.logger.LogInformation( + $"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); + } + else + { + int blockIndex = entryHeight % this.unconfTxs.Count; + if (this.unconfTxs[blockIndex][bucketIndex] > 0) + this.unconfTxs[blockIndex][bucketIndex]--; + else + this.logger.LogInformation( + $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); + } + } + + /// + /// Update our estimates by decaying our historical moving average and updating + /// with the data gathered from the current block. + /// + public void UpdateMovingAverages() + { + for (var j = 0; j < this.buckets.Count; j++) + { + for (var i = 0; i < this.confAvg.Count; i++) + this.confAvg[i][j] = this.confAvg[i][j] * this.decay + this.curBlockConf[i][j]; + this.avg[j] = this.avg[j] * this.decay + this.curBlockVal[j]; + this.txCtAvg[j] = this.txCtAvg[j] * this.decay + this.curBlockTxCt[j]; + } + } + + /// + /// Calculate a feerate estimate. Find the lowest value bucket (or range of buckets + /// to make sure we have enough data points) whose transactions still have sufficient likelihood + /// of being confirmed within the target number of confirmations. + /// + /// Target number of confirmations. + /// Required average number of transactions per block in a bucket range. + /// The success probability we require. + /// Return the lowest feerate such that all higher values pass minSuccess OR return the highest feerate such that all lower values fail minSuccess. + /// The current block height. + /// + public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, + bool requireGreater, + int nBlockHeight) + { + // Counters for a bucket (or range of buckets) + double nConf = 0; // Number of tx's confirmed within the confTarget + double totalNum = 0; // Total number of tx's that were ever confirmed + int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + + int maxbucketindex = this.buckets.Count - 1; + + // requireGreater means we are looking for the lowest feerate such that all higher + // values pass, so we start at maxbucketindex (highest feerate) and look at successively + // smaller buckets until we reach failure. Otherwise, we are looking for the highest + // feerate such that all lower values fail, and we go in the opposite direction. + int startbucket = requireGreater ? maxbucketindex : 0; + int step = requireGreater ? -1 : 1; + + // We'll combine buckets until we have enough samples. + // The near and far variables will define the range we've combined + // The best variables are the last range we saw which still had a high + // enough confirmation rate to count as success. + // The cur variables are the current range we're counting. + int curNearBucket = startbucket; + int bestNearBucket = startbucket; + int curFarBucket = startbucket; + int bestFarBucket = startbucket; + + bool foundAnswer = false; + int bins = this.unconfTxs.Count; + + // Start counting from highest(default) or lowest feerate transactions + for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) + { + curFarBucket = bucket; + nConf += this.confAvg[confTarget - 1][bucket]; + totalNum += this.txCtAvg[bucket]; + for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) + extraNum += this.unconfTxs[(nBlockHeight - confct) % bins][bucket]; + extraNum += this.oldUnconfTxs[bucket]; + // If we have enough transaction data points in this range of buckets, + // we can test for success + // (Only count the confirmed data points, so that each confirmation count + // will be looking at the same amount of data and same bucket breaks) + if (totalNum >= sufficientTxVal / (1 - this.decay)) + { + double curPct = nConf / (totalNum + extraNum); + + // Check to see if we are no longer getting confirmed at the success rate + if (requireGreater && curPct < successBreakPoint) + break; + if (!requireGreater && curPct > successBreakPoint) + break; + + // Otherwise update the cumulative stats, and the bucket variables + // and reset the counters + foundAnswer = true; + nConf = 0; + totalNum = 0; + extraNum = 0; + bestNearBucket = curNearBucket; + bestFarBucket = curFarBucket; + curNearBucket = bucket + step; + } + } + + double median = -1; + double txSum = 0; + + // Calculate the "average" feerate of the best bucket range that met success conditions + // Find the bucket with the median transaction and then report the average feerate from that bucket + // This is a compromise between finding the median which we can't since we don't save all tx's + // and reporting the average which is less accurate + int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; + int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + for (int j = minBucket; j <= maxBucket; j++) + txSum += this.txCtAvg[j]; + if (foundAnswer && txSum != 0) + { + txSum = txSum / 2; + for (int j = minBucket; j <= maxBucket; j++) + if (this.txCtAvg[j] < txSum) + { + txSum -= this.txCtAvg[j]; + } + else + { + // we're in the right bucket + median = this.avg[j] / this.txCtAvg[j]; + break; + } + } + + this.logger.LogInformation( + $"{confTarget}: For conf success {(requireGreater ? $">" : $"<")} {successBreakPoint} need feerate {(requireGreater ? $">" : $"<")}: {median} from buckets {this.buckets[minBucket]} -{this.buckets[maxBucket]} Cur Bucket stats {100 * nConf / (totalNum + extraNum)} {nConf}/({totalNum}+{extraNum} mempool)"); + + return median; + } + + /// + /// Return the max number of confirms we're tracking. + /// + /// The max number of confirms. + public int GetMaxConfirms() + { + return this.confAvg.Count; + } + + /// + /// Write state of estimation data to a file. + /// + /// Stream to write to. + public void Write(BitcoinStream stream) + { + } + + /// + /// Read saved state of estimation data from a file and replace all internal data structures and + /// variables with this state. + /// + /// Stream to read from. + public void Read(Stream filein) + { + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index b66eadcefa2..1636936573b 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -3,6 +3,7 @@ using System.Linq; using Microsoft.Extensions.Logging; using NBitcoin; +using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { @@ -15,35 +16,51 @@ public class TxConfirmStats private readonly ILogger logger; /// - /// Moving average of total fee rate of all transactions in each bucket. + /// The upper-bound of the range for the bucket (inclusive). /// /// - /// Track the historical moving average of this total over blocks. + /// Define the buckets we will group transactions into. /// - private List avg; + private List buckets; /// Map of bucket upper-bound to index into all vectors by bucket. - private Dictionary bucketMap; - - //Define the buckets we will group transactions into. - - /// The upper-bound of the range for the bucket (inclusive). - private List buckets; + private SortedDictionary bucketMap; - // Count the total # of txs confirmed within Y blocks in each bucket. - // Track the historical moving average of theses totals over blocks. + /// + /// Historical moving average of transaction counts. + /// + /// + /// For each bucket X: + /// Count the total # of txs in each bucket. + /// Track the historical moving average of this total over blocks + /// + private List txCtAvg; - /// Confirmation average. confAvg[Y][X]. + /// + /// Confirmation average. confAvg[Y][X] + /// + /// + /// Count the total # of txs confirmed within Y blocks in each bucket. + /// Track the historical moving average of theses totals over blocks. + /// private List> confAvg; - /// Current block confirmations. curBlockConf[Y][X]. - private List> curBlockConf; - - /// Current block transaction count. - private List curBlockTxCt; + /// + /// Failed average. failAvg[Y][X] + /// + /// + /// Track moving avg of txs which have been evicted from the mempool + /// after failing to be confirmed within Y blocks + /// + private List> failAvg; - /// Current block fee rate. - private List curBlockVal; + /// + /// Moving average of total fee rate of all transactions in each bucket. + /// + /// + /// Track the historical moving average of this total over blocks. + /// + private List avg; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X // Combine the total value with the tx counts to calculate the avg feerate per bucket @@ -51,18 +68,10 @@ public class TxConfirmStats /// Decay value to use. private double decay; - /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket - private List oldUnconfTxs; - /// - /// Historical moving average of transaction counts. + /// Resolution (# of blocks) with which confirmations are tracked /// - /// - /// For each bucket X: - /// Count the total # of txs in each bucket. - /// Track the historical moving average of this total over blocks - /// - private List txCtAvg; + private int scale; /// /// Mempool counts of outstanding transactions. @@ -74,6 +83,10 @@ public class TxConfirmStats /// private List> unconfTxs; + /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket + private List oldUnconfTxs; + + /// /// Constructs an instance of the transaction confirmation stats object. /// @@ -88,35 +101,33 @@ public TxConfirmStats(ILogger logger) /// constructor with default values. /// /// Contains the upper limits for the bucket boundaries. - /// Max number of confirms to track. + /// Max number of periods to track. /// How much to decay the historical moving average per block. - public void Initialize(List defaultBuckets, int maxConfirms, double decay) + public void Initialize(List defaultBuckets, IDictionary defaultBucketMap, int maxPeriods, double decay, int scale) { - this.buckets = new List(); - this.bucketMap = new Dictionary(); - + Guard.Assert(scale != 0); this.decay = decay; - for (int i = 0; i < defaultBuckets.Count; i++) - { - this.buckets.Add(defaultBuckets[i]); - this.bucketMap[defaultBuckets[i]] = i; - } + this.scale = scale; this.confAvg = new List>(); - this.curBlockConf = new List>(); + this.failAvg = new List>(); + this.buckets = new List(defaultBuckets); + this.bucketMap = new SortedDictionary(defaultBucketMap); this.unconfTxs = new List>(); - for (int i = 0; i < maxConfirms; i++) + for (int i = 0; i < maxPeriods; i++) { this.confAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); - this.curBlockConf.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); + this.failAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); + } + + for (int i = 0; i < GetMaxConfirms(); i++) + { this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); } - this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); - this.curBlockTxCt = new List(Enumerable.Repeat(default(int), this.buckets.Count)); this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - this.curBlockVal = new List(Enumerable.Repeat(default(double), this.buckets.Count)); this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); } /// @@ -129,10 +140,6 @@ public void ClearCurrent(int nBlockHeight) { this.oldUnconfTxs[j] += this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j]; this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j] = 0; - for (int i = 0; i < this.curBlockConf.Count; i++) - this.curBlockConf[i][j] = 0; - this.curBlockTxCt[j] = 0; - this.curBlockVal[j] = 0; } } @@ -146,11 +153,14 @@ public void Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; - for (int i = blocksToConfirm; i <= this.curBlockConf.Count; i++) - this.curBlockConf[i - 1][bucketindex]++; - this.curBlockTxCt[bucketindex]++; - this.curBlockVal[bucketindex] += val; + int periodsToConfirm = (blocksToConfirm + this.scale - 1) / this.scale; + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; + for (int i = periodsToConfirm; i <= this.confAvg.Count; i++) + { + this.confAvg[i - 1][bucketindex]++; + } + this.txCtAvg[bucketindex]++; + this.avg[bucketindex] += val; } /// @@ -161,7 +171,7 @@ public void Record(int blocksToConfirm, double val) /// The feerate of the transaction. public int NewTx(int nBlockHeight, double val) { - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; int blockIndex = nBlockHeight % this.unconfTxs.Count; this.unconfTxs[blockIndex][bucketindex]++; return bucketindex; @@ -335,7 +345,7 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s /// The max number of confirms. public int GetMaxConfirms() { - return this.confAvg.Count; + return this.scale * this.confAvg.Count; } /// From fb4db2d7b4621f145abdfffbb495ebd2bc66feb6 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Mon, 4 Jun 2018 01:17:21 +0300 Subject: [PATCH 03/18] BlockPolicyEstimator algo finished --- .../Fee/BlockPolicyEstimator.cs | 709 +++++++++++++----- .../Fee/FeeFilterRounder.cs | 10 + .../Fee/TxConfirmStats.cs | 144 +++- 3 files changed, 643 insertions(+), 220 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index 35a1f3846c0..1b6932ff15d 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -1,9 +1,12 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Features.MemoryPool.Interfaces; +using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { @@ -23,55 +26,84 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee /// at the bucket with the highest feerate transactions and verifying that a /// sufficiently high percentage of them were confirmed within 5 blocks and /// then you would look at the next highest feerate bucket, and so on, stopping at - /// the last bucket to pass the test. The average feerate of transactions in this + /// the last bucket to pass the test.The average feerate of transactions in this /// bucket will give you an indication of the lowest feerate you can put on a /// transaction and still have a sufficiently high chance of being confirmed /// within your desired 5 blocks. /// /// Here is a brief description of the implementation: - /// When a transaction enters the mempool, we - /// track the height of the block chain at entry. Whenever a block comes in, - /// we count the number of transactions in each bucket and the total amount of feerate - /// paid in each bucket. Then we calculate how many blocks Y it took each - /// transaction to be mined and we track an array of counters in each bucket - /// for how long it to took transactions to get confirmed from 1 to a max of 25 - /// and we increment all the counters from Y up to 25. This is because for any - /// number Z>=Y the transaction was successfully mined within Z blocks. We - /// want to save a history of this information, so at any time we have a - /// counter of the total number of transactions that happened in a given feerate - /// bucket and the total number that were confirmed in each number 1-25 blocks - /// or less for any bucket. We save this history by keeping an exponentially - /// decaying moving average of each one of these stats. Furthermore we also - /// keep track of the number unmined (in mempool) transactions in each bucket - /// and for how many blocks they have been outstanding and use that to increase - /// the number of transactions we've seen in that feerate bucket when calculating - /// an estimate for any number of confirmations below the number of blocks - /// they've been outstanding. - /// - /// We will instantiate an instance of this class to track transactions that were - /// included in a block. We will lump transactions into a bucket according to their - /// approximate feerate and then track how long it took for those txs to be included in a block - /// - /// The tracking of unconfirmed (mempool) transactions is completely independent of the - /// historical tracking of transactions that have been confirmed in a block. - /// - /// We want to be able to estimate feerates that are needed on tx's to be included in - /// a certain number of blocks.Every time a block is added to the best chain, this class records - /// stats on the transactions included in that block + /// When a transaction enters the mempool, we track the height of the block chain + /// at entry. All further calculations are conducted only on this set of "seen" + /// transactions.Whenever a block comes in, we count the number of transactions + /// in each bucket and the total amount of feerate paid in each bucket. Then we + /// calculate how many blocks Y it took each transaction to be mined. We convert + /// from a number of blocks to a number of periods Y' each encompassing "scale" + /// blocks.This is tracked in 3 different data sets each up to a maximum + /// number of periods.Within each data set we have an array of counters in each + /// feerate bucket and we increment all the counters from Y' up to max periods + /// representing that a tx was successfully confirmed in less than or equal to + /// that many periods. We want to save a history of this information, so at any + /// time we have a counter of the total number of transactions that happened in a + /// given feerate bucket and the total number that were confirmed in each of the + /// periods or less for any bucket. We save this history by keeping an + /// exponentially decaying moving average of each one of these stats. This is + /// done for a different decay in each of the 3 data sets to keep relevant data + /// from different time horizons. Furthermore we also keep track of the number + /// unmined (in mempool or left mempool without being included in a block) + /// transactions in each bucket and for how many blocks they have been + /// outstanding and use both of these numbers to increase the number of transactions + /// we've seen in that feerate bucket when calculating an estimate for any number + /// of confirmations below the number of blocks they've been outstanding. /// public class BlockPolicyEstimator { - /// Require an avg of 1 tx in the combined feerate bucket per block to have stat significance. - private const double SufficientFeeTxs = 1; + ///Track confirm delays up to 12 blocks for short horizon + private const int ShortBlockPeriods = 12; + private const int ShortScale = 1; + + ///Track confirm delays up to 48 blocks for medium horizon + private const int MedBlockPeriods = 24; + private const int MedScale = 2; + + ///Track confirm delays up to 1008 blocks for long horizon + private const int LongBlockPeriods = 42; + private const int LongScale = 24; + + ///Historical estimates that are older than this aren't valid + private const int OldestEstimateHistory = 6 * 1008; + + ///Decay of .962 is a half-life of 18 blocks or about 3 hours + private const double ShortDecay = .962; + + ///Decay of .998 is a half-life of 144 blocks or about 1 day + private const double MedDecay = .9952; + + ///Decay of .9995 is a half-life of 1008 blocks or about 1 week + private const double LongDecay = .99931; - /// Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough. - private const double MinSuccessPct = .95; + ///Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks + private const double HalfSuccessPct = .6; - /// Minimum value for tracking feerates. - private const long MinFeeRate = 10; + ///Require greater than 85% of X feerate transactions to be confirmed within Y blocks + private const double SuccessPct = .85; - /// Maximum value for tracking feerates. - private const double MaxFeeRate = 1e7; + ///Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks + private const double DoubleSuccessPct = .95; + + /// Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance. + private const double SufficientFeeTxs = 0.1; + + /// Require an avg of 0.5 tx when using short decay since there are fewer blocks considered + private const double SufficientTxsShort = 0.5; + + /// Minimum and Maximum values for tracking feerates + /// The MinBucketFeeRate should just be set to the lowest reasonable feerate we + /// might ever want to track. Historically this has been 1000 since it was + /// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it + /// invalidates old estimates files. So leave it at 1000 unless it becomes + /// necessary to lower it, and then lower it substantially. + private const double MinBucketFeeRate = 1000; + private const double MaxBucketFeeRate = 1e7; /// /// Spacing of FeeRate buckets. @@ -81,47 +113,50 @@ public class BlockPolicyEstimator /// to give accurate estimates over a large range of potential feerates. /// Therefore it makes sense to exponentially space the buckets. /// - private const double FeeSpacing = 1.1; - - /// Track confirm delays up to 25 blocks, can't estimate beyond that. - private const int MaxBlockConfirms = 25; + private const double FeeSpacing = 1.05; - /// Decay of .998 is a half-life of 346 blocks or about 2.4 days. - private const double DefaultDecay = .998; + private const double InfFeeRate = 1e99; - /// Value for infinite priority. - public const double InfPriority = 1e9 * 21000000ul * Money.COIN; + /// Best seen block height. + private int nBestSeenHeight; - /// Maximum money value. - private static readonly Money MaxMoney = new Money(21000000 * Money.COIN); + private int firstRecordedHeight; + private int historicalFirst; + private int historicalBest; - /// Value for infinite fee rate. - private static readonly double InfFeeRate = MaxMoney.Satoshi; + /// Logger for logging on this object. + private readonly ILogger logger; /// Classes to track historical data on transaction confirmations. private readonly TxConfirmStats feeStats; + private readonly TxConfirmStats shortStats; + private readonly TxConfirmStats longStats; /// Map of txids to information about that transaction. private readonly Dictionary mapMemPoolTxs; - /// Minimum tracked Fee. Passed to constructor to avoid dependency on main./// - private readonly FeeRate minTrackedFee; - - /// Best seen block height. - private int nBestSeenHeight; - /// Setting for the node. private readonly MempoolSettings mempoolSettings; - /// Logger for logging on this object. - private readonly ILogger logger; - /// Count of tracked transactions. private int trackedTxs; /// Count of untracked transactions. private int untrackedTxs; + /// + /// The upper-bound of the range for the bucket (inclusive). + /// + /// + /// Define the buckets we will group transactions into. + /// + private List buckets; + + /// Map of bucket upper-bound to index into all vectors by bucket. + private SortedDictionary bucketMap; + + private object lockObject; + /// /// Constructs an instance of the block policy estimator object. /// @@ -130,24 +165,35 @@ public class BlockPolicyEstimator /// Full node settings. public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory loggerFactory, NodeSettings nodeSettings) { - this.mapMemPoolTxs = new Dictionary(); + Guard.Assert(MinBucketFeeRate > 0); + this.lockObject = new object(); + this.logger = loggerFactory.CreateLogger(this.GetType().FullName); this.mempoolSettings = mempoolSettings; + this.mapMemPoolTxs = new Dictionary(); + this.buckets = new List(); + this.bucketMap = new SortedDictionary(); this.nBestSeenHeight = 0; + this.firstRecordedHeight = 0; + this.historicalFirst = 0; + this.historicalBest = 0; this.trackedTxs = 0; this.untrackedTxs = 0; - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); + int bucketIndex = 0; + for(double bucketBoundary = MinBucketFeeRate; bucketBoundary <= MaxBucketFeeRate; bucketBoundary *= FeeSpacing, bucketIndex++) + { + this.buckets.Add(bucketBoundary); + this.bucketMap.Add(bucketBoundary, bucketIndex); + } + this.buckets.Add(InfFeeRate); + this.bucketMap.Add(InfFeeRate, bucketIndex); + Guard.Assert(this.bucketMap.Count == this.buckets.Count); - this.minTrackedFee = nodeSettings.MinRelayTxFeeRate < new FeeRate(new Money(MinFeeRate)) - ? new FeeRate(new Money(MinFeeRate)) - : nodeSettings.MinRelayTxFeeRate; - var vfeelist = new List(); - for (double bucketBoundary = this.minTrackedFee.FeePerK.Satoshi; - bucketBoundary <= MaxFeeRate; - bucketBoundary *= FeeSpacing) - vfeelist.Add(bucketBoundary); - vfeelist.Add(InfFeeRate); this.feeStats = new TxConfirmStats(this.logger); - this.feeStats.Initialize(vfeelist, MaxBlockConfirms, DefaultDecay); + this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); + this.shortStats = new TxConfirmStats(this.logger); + this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); + this.longStats = new TxConfirmStats(this.logger); + this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); } /// @@ -157,32 +203,52 @@ public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory logg /// Collection of memory pool entries. public void ProcessBlock(int nBlockHeight, List entries) { - if (nBlockHeight <= this.nBestSeenHeight) - return; - - // Must update nBestSeenHeight in sync with ClearCurrent so that - // calls to removeTx (via processBlockTx) correctly calculate age - // of unconfirmed txs to remove from tracking. - this.nBestSeenHeight = nBlockHeight; - - // Clear the current block state and update unconfirmed circular buffer - this.feeStats.ClearCurrent(nBlockHeight); - - int countedTxs = 0; - // Repopulate the current block states - for (int i = 0; i < entries.Count; i++) - if (this.ProcessBlockTx(nBlockHeight, entries[i])) - countedTxs++; - - // Update all exponential averages with the current block state - this.feeStats.UpdateMovingAverages(); - - // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) - // Logging.Logs.EstimateFee.LogInformation( - // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); - - this.trackedTxs = 0; - this.untrackedTxs = 0; + lock (this.lockObject) + { + if (nBlockHeight <= this.nBestSeenHeight) + // Ignore side chains and re-orgs; assuming they are random + // they don't affect the estimate. + // And if an attacker can re-org the chain at will, then + // you've got much bigger problems than "attacker can influence + // transaction fees." + return; + + // Must update nBestSeenHeight in sync with ClearCurrent so that + // calls to removeTx (via processBlockTx) correctly calculate age + // of unconfirmed txs to remove from tracking. + this.nBestSeenHeight = nBlockHeight; + + // Update unconfirmed circular buffer + this.feeStats.ClearCurrent(nBlockHeight); + this.shortStats.ClearCurrent(nBlockHeight); + this.longStats.ClearCurrent(nBlockHeight); + + // Decay all exponential averages + this.feeStats.UpdateMovingAverages(); + this.shortStats.UpdateMovingAverages(); + this.longStats.UpdateMovingAverages(); + + int countedTxs = 0; + // Repopulate the current block states + foreach (var entry in entries) + { + if (this.ProcessBlockTx(nBlockHeight, entry)) + countedTxs++; + } + + if (this.firstRecordedHeight == 0 && countedTxs > 0) + { + this.firstRecordedHeight = this.nBestSeenHeight; + this.logger.LogInformation("Blockpolicy first recorded height {0}", this.firstRecordedHeight); + } + + // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) + // Logging.Logs.EstimateFee.LogInformation( + // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); + + this.trackedTxs = 0; + this.untrackedTxs = 0; + } } /// @@ -193,7 +259,7 @@ public void ProcessBlock(int nBlockHeight, List entries) /// Whether it was able to successfully process the transaction. private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) { - if (!this.RemoveTx(entry.TransactionHash)) + if (!this.RemoveTx(entry.TransactionHash, true)) return false; // How many blocks did it take for miners to include this transaction? @@ -212,6 +278,8 @@ private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.shortStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.longStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); return true; } @@ -222,32 +290,40 @@ private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) /// Whether to update fee estimate. public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) { - int txHeight = entry.EntryHeight; - uint256 hash = entry.TransactionHash; - if (this.mapMemPoolTxs.ContainsKey(hash)) + lock (this.lockObject) { - this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); - return; + int txHeight = entry.EntryHeight; + uint256 hash = entry.TransactionHash; + if (this.mapMemPoolTxs.ContainsKey(hash)) + { + this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); + return; + } + + if (txHeight != this.nBestSeenHeight) + return; + + // Only want to be updating estimates when our blockchain is synced, + // otherwise we'll miscalculate how many blocks its taking to get included. + if (!validFeeEstimate) + { + this.untrackedTxs++; + return; + } + this.trackedTxs++; + + // Feerates are stored and reported as BTC-per-kb: + FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); + this.mapMemPoolTxs[hash].blockHeight = txHeight; + int bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + this.mapMemPoolTxs[hash].bucketIndex = bucketIndex; + int bucketIndex2 = this.shortStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + Guard.Assert(bucketIndex == bucketIndex2); + int bucketIndex3 = this.longStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + Guard.Assert(bucketIndex == bucketIndex3); } - - if (txHeight != this.nBestSeenHeight) - return; - - // Only want to be updating estimates when our blockchain is synced, - // otherwise we'll miscalculate how many blocks its taking to get included. - if (!validFeeEstimate) - { - this.untrackedTxs++; - return; - } - this.trackedTxs++; - - // Feerates are stored and reported as BTC-per-kb: - FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); - this.mapMemPoolTxs[hash].blockHeight = txHeight; - this.mapMemPoolTxs[hash].bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); } /// @@ -262,16 +338,74 @@ public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) /// ProcessBlockTx to ensure they are never double tracked, but it is /// of no harm to try to remove them again. /// - public bool RemoveTx(uint256 hash) + public bool RemoveTx(uint256 hash, bool inBlock) + { + lock (this.lockObject) + { + TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); + if (pos != null) + { + this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.shortStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.longStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.mapMemPoolTxs.Remove(hash); + return true; + } + return false; + } + } + + /// + /// Return an estimate fee according to horizon + /// + /// The desired number of confirmations to be included in a block + public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) { - TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); - if (pos != null) + TxConfirmStats stats; + double sufficientTxs = SufficientFeeTxs; + switch (horizon) { - this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex); - this.mapMemPoolTxs.Remove(hash); - return true; + case FeeEstimateHorizon.ShortHalfLife: + { + stats = this.shortStats; + sufficientTxs = SufficientTxsShort; + break; + } + case FeeEstimateHorizon.MedHalfLife: + { + stats = this.feeStats; + break; + } + case FeeEstimateHorizon.LongHalfLife: + { + stats = this.longStats; + break; + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + lock(this.lockObject) + { + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > stats.GetMaxConfirms()) + { + return new FeeRate(0); + } + if (successThreshold > 1) + { + return new FeeRate(0); + } + + double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, + true, this.nBestSeenHeight, result); + + if (median < 0) + return new FeeRate(0); + + return new FeeRate(Convert.ToInt64(median)); } - return false; } /// @@ -280,58 +414,264 @@ public bool RemoveTx(uint256 hash) /// The desired number of confirmations to be included in a block. public FeeRate EstimateFee(int confTarget) { - // Return failure if trying to analyze a target we're not tracking // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || confTarget > this.feeStats.GetMaxConfirms()) + if (confTarget <= 1) + { return new FeeRate(0); + } - double median = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); + return EstimateRawFee(confTarget, DoubleSuccessPct, FeeEstimateHorizon.MedHalfLife, null); + } - if (median < 0) - return new FeeRate(0); + public int HighestTargetTracked(FeeEstimateHorizon horizon) + { + switch (horizon) + { + case FeeEstimateHorizon.ShortHalfLife: + { + return this.shortStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.MedHalfLife: + { + return this.feeStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.LongHalfLife: + { + return this.longStats.GetMaxConfirms(); + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + } - return new FeeRate(new Money((int)median)); + private int BlockSpan() + { + if (this.firstRecordedHeight == 0) return 0; + Guard.Assert(this.nBestSeenHeight >= this.firstRecordedHeight); + + return this.nBestSeenHeight - this.firstRecordedHeight; } - /// - /// Estimate feerate needed to be included in a block within - /// confTarget blocks. If no answer can be given at confTarget, return an - /// estimate at the lowest target where one can be given. - /// - public FeeRate EstimateSmartFee(int confTarget, ITxMempool pool, out int answerFoundAtTarget) + private int HistoricalBlockSpan() { - answerFoundAtTarget = confTarget; + if (this.historicalFirst == 0) return 0; + Guard.Assert(this.historicalBest >= this.historicalFirst); - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || confTarget > this.feeStats.GetMaxConfirms()) - return new FeeRate(0); + if (this.nBestSeenHeight - this.historicalBest > OldestEstimateHistory) + { + return 0; + } - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; + return this.historicalBest - this.historicalFirst; + } - double median = -1; - while (median < 0 && confTarget <= this.feeStats.GetMaxConfirms()) - median = this.feeStats.EstimateMedianVal(confTarget++, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); + private int MaxUsableEstimate() + { + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return Math.Min(this.longStats.GetMaxConfirms(), Math.Max(BlockSpan(), HistoricalBlockSpan()) / 2); + } - answerFoundAtTarget = confTarget - 1; + /// + /// Return a fee estimate at the required successThreshold from the shortest + /// time horizon which tracks confirmations up to the desired target.If + /// checkShorterHorizon is requested, also allow short time horizon estimates + /// for a lower target to reduce the given answer + /// + private double EstimateCombinedFee(int confTarget, double successThreshold, + bool checkShorterHorizon, EstimationResult result) + { + double estimate = -1; + if (confTarget >= 1 && confTarget <= this.longStats.GetMaxConfirms()) + { + // Find estimate from shortest time horizon possible + if (confTarget <= this.shortStats.GetMaxConfirms()) + { // short horizon + estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, + successThreshold, true, this.nBestSeenHeight, result); + } + else if (confTarget <= this.feeStats.GetMaxConfirms()) + { // medium horizon + estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, + successThreshold, true, this.nBestSeenHeight, result); + } + else + { // long horizon + estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, + successThreshold, true, this.nBestSeenHeight, result); + } + if (checkShorterHorizon) + { + EstimationResult tempResult = new EstimationResult(); + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > this.feeStats.GetMaxConfirms()) + { + double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), + SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + { + estimate = medMax; + if (result != null) + { + result = tempResult; + } + } + } + if (confTarget > this.shortStats.GetMaxConfirms()) + { + double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), + SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + { + estimate = shortMax; + if (result != null) + { + result = tempResult; + } + } + } + } + } + return estimate; + } - // If mempool is limiting txs , return at least the min feerate from the mempool - if (pool != null) + /// + /// Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + /// at 2 * target for any longer time horizons. + /// + private double EstimateConservativeFee(int doubleTarget, EstimationResult result) + { + double estimate = -1; + EstimationResult tempResult = new EstimationResult(); + if (doubleTarget <= this.shortStats.GetMaxConfirms()) + { + estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, + DoubleSuccessPct, true, this.nBestSeenHeight, result); + } + if (doubleTarget <= this.feeStats.GetMaxConfirms()) { - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0 && minPoolFee.Satoshi > median) - return new FeeRate(minPoolFee); + double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, + DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); + if (longEstimate > estimate) + { + estimate = longEstimate; + if (result != null) + { + result = tempResult; + } + } } + return estimate; + } - if (median < 0) - return new FeeRate(0); + /// + /// Returns the max of the feerates calculated with a 60% + /// threshold required at target / 2, an 85% threshold required at target and a + /// 95% threshold required at 2 * target.Each calculation is performed at the + /// shortest time horizon which tracks the required target.Conservative + /// estimates, however, required the 95% threshold at 2 * target be met for any + /// longer time horizons also. + /// + public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) + { + lock (this.lockObject) + { - return new FeeRate((int)median); + if (feeCalc != null) + { + feeCalc.DesiredTarget = confTarget; + feeCalc.ReturnedTarget = confTarget; + } + + double median = -1; + EstimationResult tempResult = new EstimationResult(); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > this.longStats.GetMaxConfirms()) + { + return new FeeRate(0); // error condition + } + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + { + confTarget = 2; + } + + int maxUsableEstimate = MaxUsableEstimate(); + if (confTarget > maxUsableEstimate) + { + confTarget = maxUsableEstimate; + } + if (feeCalc != null) + { + feeCalc.ReturnedTarget = confTarget; + } + + if (confTarget <= 1) + { + return new FeeRate(0); // error condition + } + + Guard.Assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + // true is passed to estimateCombined fee for target/2 and target so + // that we check the max confirms for shorter time horizons as well. + // This is necessary to preserve monotonically increasing estimates. + // For non-conservative estimates we do the same thing for 2*target, but + // for conservative estimates we want to skip these shorter horizons + // checks for 2*target because we are taking the max over all time + // horizons so we already have monotonically increasing estimates and + // the purpose of conservative estimates is not to let short term + // fluctuations lower our estimates by too much. + double halfEst = EstimateCombinedFee(confTarget / 2, HalfSuccessPct, true, tempResult); + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.HalfEstimate; + } + median = halfEst; + double actualEst = EstimateCombinedFee(confTarget, SuccessPct, true, tempResult); + if (actualEst > median) + { + median = actualEst; + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.FullEstimate; + } + } + double doubleEst = EstimateCombinedFee(2 * confTarget, DoubleSuccessPct, !conservative, tempResult); + if (doubleEst > median) + { + median = doubleEst; + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.DoubleEstimate; + } + } + + if (conservative || median == -1) + { + double consEst = EstimateConservativeFee(2 * confTarget, tempResult); + if (consEst > median) + { + median = consEst; + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.Coservative; + } + } + } + + if (median < 0) return new FeeRate(0); // error condition + + return new FeeRate(Convert.ToInt64(median)); + } } - /// /// Write estimation data to a file. /// @@ -351,36 +691,21 @@ public void Read(Stream filein, int nFileVersion) { } - /// - /// Return an estimate of the priority. - /// - /// The desired number of confirmations to be included in a block. - /// Estimate of the priority. - /// TODO: Implement priority estimation - public double EstimatePriority(int confTarget) - { - return -1; - } - - /// - /// Return an estimated smart priority. - /// - /// The desired number of confirmations to be included in a block. - /// Memory pool transactions. - /// Block height where answer was found. - /// The smart priority. - public double EstimateSmartPriority(int confTarget, ITxMempool pool, out int answerFoundAtTarget) + public void FlushUncomfirmed() { - answerFoundAtTarget = confTarget; - - // If mempool is limiting txs, no priority txs are allowed - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0) - return InfPriority; - - return -1; + lock (this.lockObject) + { + int numEntries = this.mapMemPoolTxs.Count; + // Remove every entry in mapMemPoolTxs + while (this.mapMemPoolTxs.Count > 0) + { + var mi = this.mapMemPoolTxs.First(); ; + RemoveTx(mi.Key, false); // this calls erase() on mapMemPoolTxs + } + this.logger.LogInformation($"Recorded {numEntries} unconfirmed txs from mempool"); + } } - + /// /// Transaction statistics information. /// diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs new file mode 100644 index 00000000000..3e8344b9266 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs @@ -0,0 +1,10 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public class FeeFilterRounder + { + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 1636936573b..2c52664ab63 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Utilities; +using System; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { @@ -120,14 +121,19 @@ public void Initialize(List defaultBuckets, IDictionary def this.failAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); } - for (int i = 0; i < GetMaxConfirms(); i++) - { - this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); - } this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); + ClearInMemoryCounters(this.buckets.Count); + } + + private void ClearInMemoryCounters(int bucketsCount) + { + for (int i = 0; i < GetMaxConfirms(); i++) + { + this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), bucketsCount).ToList()); + } + this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), bucketsCount)); } /// @@ -183,7 +189,7 @@ public int NewTx(int nBlockHeight, double val) /// The height of the mempool entry. /// The best sceen height. /// The bucket index. - public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex) + public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool inBlock) { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; @@ -212,6 +218,15 @@ public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex) this.logger.LogInformation( $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); } + if(!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period + { + Guard.Assert(this.scale != 0); + int periodsAgo = blocksAgo / this.scale; + for (int i = 0; i < periodsAgo && i < this.failAvg.Count; i++) + { + this.failAvg[i][bucketIndex]++; + } + } } /// @@ -223,9 +238,11 @@ public void UpdateMovingAverages() for (var j = 0; j < this.buckets.Count; j++) { for (var i = 0; i < this.confAvg.Count; i++) - this.confAvg[i][j] = this.confAvg[i][j] * this.decay + this.curBlockConf[i][j]; - this.avg[j] = this.avg[j] * this.decay + this.curBlockVal[j]; - this.txCtAvg[j] = this.txCtAvg[j] * this.decay + this.curBlockTxCt[j]; + this.confAvg[i][j] = this.confAvg[i][j] * this.decay; + for (var i = 0; i < this.failAvg.Count; i++) + this.failAvg[i][j] = this.failAvg[i][j] * this.decay; + this.avg[j] = this.avg[j] * this.decay; + this.txCtAvg[j] = this.txCtAvg[j] * this.decay; } } @@ -241,13 +258,14 @@ public void UpdateMovingAverages() /// The current block height. /// public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, - bool requireGreater, - int nBlockHeight) + bool requireGreater, int nBlockHeight, EstimationResult result) { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + this.scale - 1) / this.scale; int maxbucketindex = this.buckets.Count - 1; @@ -270,13 +288,23 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s bool foundAnswer = false; int bins = this.unconfTxs.Count; + bool newBucketRange = true; + bool passing = true; + EstimatorBucket passBucket = new EstimatorBucket(); ; + EstimatorBucket failBucket = new EstimatorBucket(); // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) + { + curNearBucket = bucket; + newBucketRange = false; + } curFarBucket = bucket; - nConf += this.confAvg[confTarget - 1][bucket]; + nConf += this.confAvg[periodTarget - 1][bucket]; totalNum += this.txCtAvg[bucket]; + failNum += this.failAvg[periodTarget - 1][bucket]; for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) extraNum += this.unconfTxs[(nBlockHeight - confct) % bins][bucket]; extraNum += this.oldUnconfTxs[bucket]; @@ -286,23 +314,47 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - this.decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; + if ((requireGreater && curPct < successBreakPoint)|| + (!requireGreater && curPct > successBreakPoint)) + { + if (passing == true) + { + // First time we hit a failure record the failed bucket + int failMinBucket = Math.Min(curNearBucket, curFarBucket); + int failMaxBucket = Math.Max(curNearBucket, curFarBucket); + failBucket.Start = failMinBucket > 0 ? this.buckets[failMinBucket - 1] : 0; + failBucket.End = this.buckets[failMaxBucket]; + failBucket.WithinTarget = nConf; + failBucket.TotalConfirmed = totalNum; + failBucket.InMempool = extraNum; + failBucket.LeftMempool = failNum; + passing = false; + } + continue; + } // Otherwise update the cumulative stats, and the bucket variables // and reset the counters - foundAnswer = true; - nConf = 0; - totalNum = 0; - extraNum = 0; - bestNearBucket = curNearBucket; - bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + else + { + failBucket = new EstimatorBucket(); // Reset any failed bucket, currently passing + foundAnswer = true; + passing = true; + passBucket.WithinTarget = nConf; + nConf = 0; + passBucket.TotalConfirmed = totalNum; + totalNum = 0; + passBucket.InMempool = extraNum; + passBucket.LeftMempool = failNum; + failNum = 0; + extraNum = 0; + bestNearBucket = curNearBucket; + bestFarBucket = curFarBucket; + newBucketRange = true; + } } } @@ -313,14 +365,17 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + int minBucket = Math.Min(bestNearBucket, bestFarBucket); + int maxBucket = Math.Max(bestNearBucket, bestFarBucket); for (int j = minBucket; j <= maxBucket; j++) + { txSum += this.txCtAvg[j]; + } if (foundAnswer && txSum != 0) { txSum = txSum / 2; for (int j = minBucket; j <= maxBucket; j++) + { if (this.txCtAvg[j] < txSum) { txSum -= this.txCtAvg[j]; @@ -331,11 +386,44 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s median = this.avg[j] / this.txCtAvg[j]; break; } + } + + passBucket.Start = minBucket > 0 ? this.buckets[minBucket - 1] : 0; + passBucket.End = this.buckets[maxBucket]; + } + + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) + { + int failMinBucket = Math.Min(curNearBucket, curFarBucket); + int failMaxBucket = Math.Max(curNearBucket, curFarBucket); + failBucket.Start = failMinBucket > 0 ? this.buckets[failMinBucket - 1] : 0; + failBucket.End = this.buckets[failMaxBucket]; + failBucket.WithinTarget = nConf; + failBucket.TotalConfirmed = totalNum; + failBucket.InMempool = extraNum; + failBucket.LeftMempool = failNum; } this.logger.LogInformation( - $"{confTarget}: For conf success {(requireGreater ? $">" : $"<")} {successBreakPoint} need feerate {(requireGreater ? $">" : $"<")}: {median} from buckets {this.buckets[minBucket]} -{this.buckets[maxBucket]} Cur Bucket stats {100 * nConf / (totalNum + extraNum)} {nConf}/({totalNum}+{extraNum} mempool)"); - + $"FeeEst: {confTarget} {(requireGreater ? $">" : $"<")} " + + $"{successBreakPoint} decay {this.decay} feerate: {median}" + + $" from ({passBucket.Start} - {passBucket.End}" + + $" {100 * passBucket.WithinTarget / (passBucket.TotalConfirmed + passBucket.InMempool + passBucket.LeftMempool)}" + + $" {passBucket.WithinTarget}/({passBucket.TotalConfirmed}" + + $" {passBucket.InMempool} mem {passBucket.LeftMempool} out) " + + $"Fail: ({failBucket.Start} - {failBucket.End} " + + $"{100 * failBucket.WithinTarget / (failBucket.TotalConfirmed + failBucket.InMempool + failBucket.LeftMempool)}" + + $" {failBucket.WithinTarget}/({failBucket.TotalConfirmed}" + + $" {failBucket.InMempool} mem {failBucket.LeftMempool} out)"); + + if (result != null) + { + result.Pass = passBucket; + result.Fail = failBucket; + result.Decay = this.decay; + result.Scale = this.scale; + } return median; } From f698ea5e02ba1f2084a709acc450e70d339e9218 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Mon, 4 Jun 2018 22:54:35 +0300 Subject: [PATCH 04/18] Small changes --- .../Fee/BlockPolicyEstimator.cs | 3 +-- .../Fee/FeeFilterRounder.cs | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index 1b6932ff15d..19eecae9c9d 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -163,12 +163,11 @@ public class BlockPolicyEstimator /// Mempool settings. /// Factory for creating loggers. /// Full node settings. - public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory loggerFactory, NodeSettings nodeSettings) + public BlockPolicyEstimator(ILoggerFactory loggerFactory) { Guard.Assert(MinBucketFeeRate > 0); this.lockObject = new object(); this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - this.mempoolSettings = mempoolSettings; this.mapMemPoolTxs = new Dictionary(); this.buckets = new List(); this.bucketMap = new SortedDictionary(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs index 3e8344b9266..086fa60c0c6 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs @@ -1,10 +1,37 @@ using System; using System.Collections.Generic; using System.Text; +using NBitcoin; +using System.Linq; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { public class FeeFilterRounder { + private const decimal MaxFilterFeeRate = 1e7M; + private const decimal FeeFilterSpacing = 1.1M; + + private SortedSet feeSet; + + public FeeFilterRounder(FeeRate minIncrementalFee) + { + Money minFeeLimit = Math.Max(new Money(1), minIncrementalFee.FeePerK / 2); + this.feeSet = new SortedSet(); + this.feeSet.Add(0); + for (decimal bucketBoundary = minFeeLimit.ToDecimal(MoneyUnit.BTC); bucketBoundary <= MaxFilterFeeRate; bucketBoundary *= FeeFilterSpacing) + { + this.feeSet.Add(bucketBoundary); + } + } + + public Money Round(Money currentMinFee) + { + var it = this.feeSet.FirstOrDefault(f => f >= currentMinFee.ToDecimal(MoneyUnit.BTC)); + if ((it != this.feeSet.FirstOrDefault() && RandomUtils.GetInt32() % 3 != 0) || it == this.feeSet.LastOrDefault()) + { + it--; + } + return new Money(it, MoneyUnit.BTC); + } } } From 10465eb3829bdc39bab41753edf554e5308fb9c0 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Wed, 6 Jun 2018 01:54:19 +0300 Subject: [PATCH 05/18] Test, fixes, start Read/Write implementation --- .../TestChainFactory.cs | 2 +- .../FeeTests.cs | 68 +++++++------------ .../MemoryPoolTransactionTests.cs | 10 +-- .../MempoolManagerTest.cs | 2 +- .../MempoolPersistenceTest.cs | 2 +- .../TestChainFactory.cs | 2 +- .../Fee/BlockPolicyEstimator.cs | 28 +++++--- .../SerializationEntity/BlockPolicyData.cs | 11 +++ .../Fee/TxConfirmStats.cs | 4 +- .../Interfaces/ITxMempool.cs | 25 +------ .../TxMemPool.cs | 21 +----- .../MinerTests.cs | 2 +- 12 files changed, 72 insertions(+), 105 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index 95ddc3a9f89..e8617bf12b2 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -128,7 +128,7 @@ public static async Task> MineBlocksAsync(TestChainContext testChain /// private static async Task> MineBlocksAsync(TestChainContext testChainContext, int count, Script receiver, bool mutateLastBlock) { - var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(testChainContext.NodeSettings), testChainContext.LoggerFactory, testChainContext.NodeSettings); + var blockPolicyEstimator = new BlockPolicyEstimator(testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempool = new TxMempool(testChainContext.DateTimeProvider, blockPolicyEstimator, testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs index 8b1058616e2..1079727892f 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs @@ -17,7 +17,7 @@ public void BlockPolicyEstimates() var dateTimeSet = new DateTimeProviderSet(); var settings = NodeSettings.Default(); TxMempool mpool = new TxMempool(DateTimeProvider.Default, - new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); TestMemPoolEntryHelper entry = new TestMemPoolEntryHelper(); Money basefee = new Money(2000); Money deltaFee = new Money(100); @@ -50,8 +50,8 @@ public void BlockPolicyEstimates() int answerFound; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) @@ -81,21 +81,15 @@ public void BlockPolicyEstimates() } mpool.RemoveForBlock(block, ++blocknum); block.Clear(); - if (blocknum == 30) + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(2) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(3) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(4).FeePerK < 8 * baseRate.FeePerK + deltaFee); - Assert.True(mpool.EstimateFee(4).FeePerK > 8 * baseRate.FeePerK - deltaFee); - - Assert.True(mpool.EstimateSmartFee(1, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(3, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(4, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(8, out answerFound) == mpool.EstimateFee(8) && answerFound == 8); + Assert.True(mpool.EstimateFee(2).FeePerK < 9 * baseRate.FeePerK + deltaFee); + Assert.True(mpool.EstimateFee(2).FeePerK > 9 * baseRate.FeePerK - deltaFee); } } @@ -114,15 +108,16 @@ public void BlockPolicyEstimates() Assert.True(origFeeEst[i - 1] <= origFeeEst[i - 2]); } int mult = 11 - i; - if (i > 1) + if (i % 2 == 0) //At scale 2, test logic is only correct for even targets { Assert.True(origFeeEst[i - 1] < mult * baseRate.FeePerK + deltaFee); Assert.True(origFeeEst[i - 1] > mult * baseRate.FeePerK - deltaFee); } - else - { - Assert.True(origFeeEst[i - 1] == new FeeRate(0).FeePerK); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) + { + origFeeEst.Add(mpool.EstimateFee(i).FeePerK); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -131,7 +126,7 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, ++blocknum); Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] + deltaFee); Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); @@ -155,10 +150,9 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, ++blocknum); } - for (int i = 1; i < 10; i++) + for (int i = 1; i < 9; i++) { Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK > origFeeEst[answerFound - 1] - deltaFee); } // Mine all those transactions @@ -176,14 +170,15 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, 265); block.Clear(); Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { - Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); + Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || + mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 600 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) + while (blocknum < 865) { for (int j = 0; j < 10; j++) { // For each fee multiple @@ -202,23 +197,10 @@ public void BlockPolicyEstimates() block.Clear(); } Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] - deltaFee); - } - - // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee - // and that estimateSmartPriority returns essentially an infinite value - mpool.AddUnchecked(txf.GetHash(), entry.Fee(feeV[5]).Time(dateTimeSet.GetTime()).Priority(0).Height(blocknum).FromTx(txf, mpool)); - // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5] - mpool.TrimToSize(1); - Assert.True(mpool.GetMinFee(1).FeePerK > feeV[5]); - for (int i = 1; i < 10; i++) - { - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.EstimateFee(i).FeePerK); - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.GetMinFee(1).FeePerK); - Assert.True(mpool.EstimateSmartPriority(i, out answerFound) == BlockPolicyEstimator.InfPriority); - } + } } public class DateTimeProviderSet : DateTimeProvider diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs index a54c51c3df0..92141d95127 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs @@ -47,7 +47,7 @@ public void MempoolRemoveTest() } var settings = NodeSettings.Default(); - TxMempool testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + TxMempool testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); // Nothing in pool, remove should do nothing: var poolSize = testPool.Size; @@ -113,7 +113,7 @@ private void CheckSort(TxMempool pool, List sortedSource, List mockTxMempool = new Mock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs index 53fe29682e8..f56a381df62 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs @@ -273,7 +273,7 @@ private static MempoolManager CreateTestMempool(NodeSettings settings, out TxMem NodeSettings nodeSettings = NodeSettings.Default(); var loggerFactory = nodeSettings.LoggerFactory; ConsensusSettings consensusSettings = new ConsensusSettings().Load(nodeSettings); - txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings); + txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); var coins = new InMemoryCoinView(settings.Network.GenesisHash); var chain = new ConcurrentChain(Network.Main.GetGenesis().Header); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs index addec059feb..bebba360835 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs @@ -91,7 +91,7 @@ public static async Task CreateAsync(Network network, Script ConsensusLoop consensusLoop = new ConsensusLoop(new AsyncLoopFactory(loggerFactory), new NodeLifetime(), chain, cachedCoinView, blockPuller, deployments, loggerFactory, new ChainState(new InvalidBlockHashStore(dateTimeProvider)), connectionManager, dateTimeProvider, new Signals.Signals(), consensusSettings, nodeSettings, peerBanning, consensusRules); await consensusLoop.StartAsync(); - BlockPolicyEstimator blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings); + BlockPolicyEstimator blockPolicyEstimator = new BlockPolicyEstimator(loggerFactory, nodeSettings); TxMempool mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); MempoolSchedulerLock mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index 19eecae9c9d..09655cc9bbc 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; using Stratis.Bitcoin.Features.MemoryPool.Interfaces; using Stratis.Bitcoin.Utilities; @@ -117,6 +118,8 @@ public class BlockPolicyEstimator private const double InfFeeRate = 1e99; + private const string FileName = "fee.json"; + /// Best seen block height. private int nBestSeenHeight; @@ -134,10 +137,7 @@ public class BlockPolicyEstimator /// Map of txids to information about that transaction. private readonly Dictionary mapMemPoolTxs; - - /// Setting for the node. - private readonly MempoolSettings mempoolSettings; - + /// Count of tracked transactions. private int trackedTxs; @@ -157,13 +157,15 @@ public class BlockPolicyEstimator private object lockObject; + private FileStorage fileStorage; + /// /// Constructs an instance of the block policy estimator object. /// /// Mempool settings. /// Factory for creating loggers. /// Full node settings. - public BlockPolicyEstimator(ILoggerFactory loggerFactory) + public BlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSettings) { Guard.Assert(MinBucketFeeRate > 0); this.lockObject = new object(); @@ -193,6 +195,7 @@ public BlockPolicyEstimator(ILoggerFactory loggerFactory) this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); this.longStats = new TxConfirmStats(this.logger); this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); + this.fileStorage = new FileStorage(nodeSettings.DataFolder.WalletPath); } /// @@ -675,9 +678,11 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con /// Write estimation data to a file. /// /// Stream to write to. - /// TODO: Implement write estimation public void Write(Stream fileout) { + var data = new BlockPolicyData(); + data.BestSeenHeight = this.nBestSeenHeight; + this.fileStorage.SaveToFile(data, FileName); } /// @@ -685,9 +690,16 @@ public void Write(Stream fileout) /// /// Stream to read data from. /// Version number of the file. - /// TODO: Implement read estimation - public void Read(Stream filein, int nFileVersion) + public void Read() { + lock(this.lockObject) + { + var data = this.fileStorage.LoadByFileName(FileName); + if (data != null) + { + this.nBestSeenHeight = data.BestSeenHeight; + } + } } public void FlushUncomfirmed() diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs new file mode 100644 index 00000000000..8637522ea73 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity +{ + public class BlockPolicyData + { + public int BestSeenHeight { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 2c52664ab63..8f94981c413 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -306,7 +306,9 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s totalNum += this.txCtAvg[bucket]; failNum += this.failAvg[periodTarget - 1][bucket]; for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) - extraNum += this.unconfTxs[(nBlockHeight - confct) % bins][bucket]; + { + extraNum += this.unconfTxs[Math.Abs(nBlockHeight - confct) % bins][bucket]; + } extraNum += this.oldUnconfTxs[bucket]; // If we have enough transaction data points in this range of buckets, // we can test for success diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs index d096ab233ee..58446fd8fd8 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs @@ -120,30 +120,7 @@ public interface ITxMempool /// The confirmation target blocks. /// The fee rate estimate. FeeRate EstimateFee(int nBlocks); - - /// - /// Estimates the priority using . - /// - /// The confirmation target blocks. - /// The estimated priority. - double EstimatePriority(int nBlocks); - - /// - /// Estimates the smart fee using . - /// - /// The confirmation target blocks. - /// The block where the fee was found. - /// The fee rate estimate. - FeeRate EstimateSmartFee(int nBlocks, out int answerFoundAtBlocks); - - /// - /// Estimates the smart priority using . - /// - /// The confirmation target blocks. - /// The block where the priority was found. - /// The estimated priority. - double EstimateSmartPriority(int nBlocks, out int answerFoundAtBlocks); - + /// /// Whether the transaction hash exists in the memory pool. /// diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs index 04dd19c1170..26bbb970646 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs @@ -280,24 +280,7 @@ public FeeRate EstimateFee(int nBlocks) { return this.MinerPolicyEstimator.EstimateFee(nBlocks); } - - /// - public FeeRate EstimateSmartFee(int nBlocks, out int answerFoundAtBlocks) - { - return this.MinerPolicyEstimator.EstimateSmartFee(nBlocks, this, out answerFoundAtBlocks); - } - - /// - public double EstimatePriority(int nBlocks) - { - return this.MinerPolicyEstimator.EstimatePriority(nBlocks); - } - - /// - public double EstimateSmartPriority(int nBlocks, out int answerFoundAtBlocks) - { - return this.MinerPolicyEstimator.EstimateSmartPriority(nBlocks, this, out answerFoundAtBlocks); - } + /// public void SetSanityCheck(double dFrequency = 1.0) @@ -683,7 +666,7 @@ private void RemoveUnchecked(TxMempoolEntry it) this.mapLinks.Remove(it); this.MapTx.Remove(it); this.nTransactionsUpdated++; - this.MinerPolicyEstimator.RemoveTx(hash); + this.MinerPolicyEstimator.RemoveTx(hash, false); } /// diff --git a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs index 832741035fb..40f8455d52e 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs @@ -173,7 +173,7 @@ public async Task InitializeAsync() date1.time = dateTimeProvider.GetTime(); date1.timeutc = dateTimeProvider.GetUtcNow(); this.DateTimeProvider = date1; - this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), new LoggerFactory(), nodeSettings), new LoggerFactory(), nodeSettings); + this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new LoggerFactory(), nodeSettings), new LoggerFactory(), nodeSettings); this.mempoolLock = new MempoolSchedulerLock(); // Simple block creation, nothing special yet: From 02470b3ce14731cdd4738a0be72de659c571c4a5 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Thu, 7 Jun 2018 01:46:00 +0300 Subject: [PATCH 06/18] Read ad Write for estimation --- .../Fee/BlockPolicyEstimator.cs | 63 ++++++++++++++-- .../SerializationEntity/BlockPolicyData.cs | 6 ++ .../Fee/SerializationEntity/TxConfirmData.cs | 16 +++++ .../Fee/TxConfirmStats.cs | 72 ++++++++++++++++++- 4 files changed, 148 insertions(+), 9 deletions(-) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index 09655cc9bbc..fb9a65b3917 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -131,9 +131,9 @@ public class BlockPolicyEstimator private readonly ILogger logger; /// Classes to track historical data on transaction confirmations. - private readonly TxConfirmStats feeStats; - private readonly TxConfirmStats shortStats; - private readonly TxConfirmStats longStats; + private TxConfirmStats feeStats; + private TxConfirmStats shortStats; + private TxConfirmStats longStats; /// Map of txids to information about that transaction. private readonly Dictionary mapMemPoolTxs; @@ -682,6 +682,20 @@ public void Write(Stream fileout) { var data = new BlockPolicyData(); data.BestSeenHeight = this.nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()) + { + data.HistoricalFirst = this.firstRecordedHeight; + data.HistoricalBest = this.nBestSeenHeight; + } + else + { + data.HistoricalFirst = this.historicalFirst; + data.HistoricalBest = this.historicalBest; + } + data.Buckets = this.buckets; + data.ShortStats = this.shortStats.Write(); + data.MedStats = this.feeStats.Write(); + data.LongStats = this.longStats.Write(); this.fileStorage.SaveToFile(data, FileName); } @@ -690,16 +704,51 @@ public void Write(Stream fileout) /// /// Stream to read data from. /// Version number of the file. - public void Read() + public bool Read() { - lock(this.lockObject) + try { - var data = this.fileStorage.LoadByFileName(FileName); - if (data != null) + lock (this.lockObject) { + var data = this.fileStorage.LoadByFileName(FileName); + if (data != null) + { + throw new ApplicationException("Corrupt estimates file or file not found"); + } + if (data.HistoricalFirst > data.HistoricalBest || data.HistoricalBest > data.BestSeenHeight) + { + throw new ApplicationException("Corrupt estimates file. Historical block range for estimates is invalid"); + } + if (data.Buckets.Count <= 1 || data.Buckets.Count > 1000) + { + throw new ApplicationException("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + } this.nBestSeenHeight = data.BestSeenHeight; + this.historicalFirst = data.HistoricalFirst; + this.historicalBest = data.HistoricalBest; + this.buckets = data.Buckets; + this.bucketMap = new SortedDictionary(); + for(int i = 0; i< this.buckets.Count; i++) + { + this.bucketMap.Add(this.buckets[i], i); + } + this.feeStats = new TxConfirmStats(this.logger); + this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); + this.feeStats.Read(data.MedStats); + this.shortStats = new TxConfirmStats(this.logger); + this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); + this.shortStats.Read(data.ShortStats); + this.longStats = new TxConfirmStats(this.logger); + this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); + this.longStats.Read(data.LongStats); } } + catch (Exception e) + { + this.logger.LogError("Error while reading policy estimation data from file", e); + return false; + } + return true; } public void FlushUncomfirmed() diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs index 8637522ea73..10dbb54076d 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs @@ -7,5 +7,11 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity public class BlockPolicyData { public int BestSeenHeight { get; set; } + public int HistoricalFirst { get; set; } + public int HistoricalBest { get; set; } + public List Buckets { get; set; } + public TxConfirmData ShortStats { get; set; } + public TxConfirmData MedStats { get; set; } + public TxConfirmData LongStats { get; set; } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs new file mode 100644 index 00000000000..472861cd742 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity +{ + public class TxConfirmData + { + public double Decay { get; set; } + public int Scale { get; set; } + public List Avg { get; set; } + public List TxCtAvg { get; set; } + public List> ConfAvg { get; set; } + public List> FailAvg { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 8f94981c413..fd07a895e31 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -5,6 +5,7 @@ using NBitcoin; using Stratis.Bitcoin.Utilities; using System; +using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { @@ -442,8 +443,17 @@ public int GetMaxConfirms() /// Write state of estimation data to a file. /// /// Stream to write to. - public void Write(BitcoinStream stream) + public TxConfirmData Write() { + return new TxConfirmData + { + Decay = this.decay, + Scale = this.scale, + Avg = this.avg, + TxCtAvg = this.txCtAvg, + ConfAvg = this.confAvg, + FailAvg = this.failAvg + }; } /// @@ -451,8 +461,66 @@ public void Write(BitcoinStream stream) /// variables with this state. /// /// Stream to read from. - public void Read(Stream filein) + public void Read(TxConfirmData data) { + try + { + if (data == null) + { + throw new ArgumentNullException(nameof(data)); + } + if(data.Decay <= 0 || data.Decay >= 1) + { + throw new ApplicationException("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + } + if(data.Scale == 0) + { + throw new ApplicationException("Corrupt estimates file. Scale must be non-zero"); + } + if(data.Avg.Count != this.buckets.Count) + { + throw new ApplicationException("Corrupt estimates file. Mismatch in feerate average bucket count"); + } + if (data.TxCtAvg.Count != this.buckets.Count) + { + throw new ApplicationException("Corrupt estimates file. Mismatch in tx count bucket count"); + } + int maxPeriods = data.ConfAvg.Count; + double maxConfirms = data.Scale * maxPeriods; + + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) + { + throw new ApplicationException("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + } + for (int i = 0; i < maxPeriods; i++) + { + if (data.ConfAvg[i].Count != this.buckets.Count) + { + throw new ApplicationException("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } + } + if (maxPeriods != data.FailAvg.Count) + { + throw new ApplicationException("Corrupt estimates file. Mismatch in confirms tracked for failures"); + } + for (int i = 0; i < maxPeriods; i++) + { + if (data.FailAvg[i].Count != this.buckets.Count) + { + throw new ApplicationException("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + } + this.decay = data.Decay; + this.scale = data.Scale; + this.avg = data.Avg; + this.txCtAvg = data.TxCtAvg; + this.confAvg = data.ConfAvg; + this.failAvg = data.FailAvg; + } + catch (Exception e) + { + this.logger.LogError("Error while reading tx confirm data from file", e); + } } } } From 46ebc17f738491cfd377a6aca81524aa1d4024b8 Mon Sep 17 00:00:00 2001 From: andrewzvvv Date: Thu, 7 Jun 2018 17:16:56 +0300 Subject: [PATCH 07/18] Using of Read and Write methods --- .../Fee/BlockPolicyEstimator.cs | 3 +-- .../Interfaces/ITxMempool.cs | 5 ++--- src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs | 1 + .../MempoolPersistence.cs | 1 + src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs | 6 ++++-- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index fb9a65b3917..35ba37b1270 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -677,8 +677,7 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con /// /// Write estimation data to a file. /// - /// Stream to write to. - public void Write(Stream fileout) + public void Write() { var data = new BlockPolicyData(); data.BestSeenHeight = this.nBestSeenHeight; diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs index 58446fd8fd8..dbb48740769 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs @@ -172,7 +172,7 @@ public interface ITxMempool /// Read fee estimates from a stream. /// /// Stream to read from. - void ReadFeeEstimates(BitcoinStream stream); + void ReadFeeEstimates(); /// /// Called when a block is connected. Removes transactions from mempool and updates the miner fee estimator. @@ -217,7 +217,6 @@ public interface ITxMempool /// /// Write fee estimates to a stream. /// - /// Stream to write to. - void WriteFeeEstimates(BitcoinStream stream); + void WriteFeeEstimates(); } } \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs index e79a8971f38..488e0b45633 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs @@ -292,6 +292,7 @@ internal async Task AddMempoolEntriesToMempoolAsync(IEnumerable toSave = memPool.MapTx.Values.ToArray().Select(tx => MempoolPersistenceEntry.FromTxMempoolEntry(tx)); return this.Save(network, toSave, fileName); } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs index 26bbb970646..12f73e9a7ac 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs @@ -979,13 +979,15 @@ public static bool AllowFree(double dPriority) } /// - public void WriteFeeEstimates(BitcoinStream stream) + public void WriteFeeEstimates() { + this.MinerPolicyEstimator.Write(); } /// - public void ReadFeeEstimates(BitcoinStream stream) + public void ReadFeeEstimates() { + this.MinerPolicyEstimator.Read(); } /// From a22c8ccb96931021e24730fae0b3224fb3f8e40f Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sat, 26 Oct 2019 21:03:05 +0200 Subject: [PATCH 08/18] Missing brace --- src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index a1aeecfe376..7d855c2b8d0 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -221,6 +221,7 @@ public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool { this.logger.LogInformation( $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); + } } if(!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period From 2b0b28b89c6fa2042ff4c900cb22ae721149afd5 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sat, 26 Oct 2019 21:21:28 +0200 Subject: [PATCH 09/18] Fix usages of block policy estimator --- .../ColdStakingControllerTest.cs | 3 +- .../FeeTests.cs | 2 +- .../MemoryPoolTransactionTests.cs | 10 ++-- .../MempoolManagerTest.cs | 2 +- .../MempoolPersistenceTest.cs | 2 +- .../TestChainFactory.cs | 4 +- .../Fee/BlockPolicyEstimator.cs | 2 +- .../MinerTests.cs | 2 +- ...gnedMultisigTransactionBroadcasterTests.cs | 50 +++---------------- .../PoW/SmartContractMinerTests.cs | 2 +- 10 files changed, 20 insertions(+), 59 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs index f0ba2ab0001..229ed759963 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs +++ b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs @@ -138,8 +138,7 @@ private void CreateMempoolManager() { this.mempoolSettings = new MempoolSettings(this.nodeSettings); this.consensusSettings = new ConsensusSettings(this.nodeSettings); - this.txMemPool = new TxMempool(this.dateTimeProvider, new BlockPolicyEstimator( - new MempoolSettings(this.nodeSettings), this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); + this.txMemPool = new TxMempool(this.dateTimeProvider, new BlockPolicyEstimator(this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); this.chainIndexer = new ChainIndexer(this.Network); this.nodeDeployments = new NodeDeployments(this.Network, this.chainIndexer); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs index 4612cc852bd..26305019d03 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs @@ -17,7 +17,7 @@ public void BlockPolicyEstimates() { var dateTimeSet = new DateTimeProviderSet(); NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var mpool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var mpool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); var entry = new TestMemPoolEntryHelper(); var basefee = new Money(2000); var deltaFee = new Money(100); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs index 198d3f3210b..30b501b3e58 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs @@ -48,7 +48,7 @@ public void MempoolRemoveTest() } NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); // Nothing in pool, remove should do nothing: long poolSize = testPool.Size; @@ -116,7 +116,7 @@ private void CheckSort(TxMempool pool, List sortedSource, List(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs index 74433c7b664..ead753fa367 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs @@ -282,7 +282,7 @@ private MempoolManager CreateTestMempool(NodeSettings settings, out TxMempool tx NodeSettings nodeSettings = NodeSettings.Default(settings.Network); ILoggerFactory loggerFactory = nodeSettings.LoggerFactory; var consensusSettings = new ConsensusSettings(nodeSettings); - txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings); + txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); var coins = new InMemoryCoinView(settings.Network.GenesisHash); var chain = new ChainIndexer(settings.Network); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs index 509f9b41dce..d5f617b69d5 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs @@ -108,7 +108,7 @@ public static async Task CreatePosAsync(Network network, Scri await consensus.InitializeAsync(genesis).ConfigureAwait(false); var mempoolSettings = new MempoolSettings(nodeSettings); - var blockPolicyEstimator = new BlockPolicyEstimator(mempoolSettings, loggerFactory, nodeSettings); + var blockPolicyEstimator = new BlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); @@ -188,7 +188,7 @@ public static async Task CreateAsync(Network network, Script chainState.BlockStoreTip = genesis; await consensus.InitializeAsync(genesis).ConfigureAwait(false); - var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings); + var blockPolicyEstimator = new BlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index 2aede3eaed3..d46f8534060 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -587,7 +587,7 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con } double median = -1; - EstimationResult tempResult = new EstimationResult(); + var tempResult = new EstimationResult(); // Return failure if trying to analyze a target we're not tracking if (confTarget <= 0 || confTarget > this.longStats.GetMaxConfirms()) diff --git a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs index 8c408d8334f..68e4338d7b0 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs @@ -205,7 +205,7 @@ public async Task InitializeAsync() }; this.DateTimeProvider = dateTimeProviderSet; - this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings); + this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); this.mempoolLock = new MempoolSchedulerLock(); // We can't make transactions until we have inputs diff --git a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs index d900569d776..40dab1a1ee1 100644 --- a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs +++ b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs @@ -29,8 +29,6 @@ public class SignedMultisigTransactionBroadcasterTests : IDisposable private readonly IFederatedPegSettings federatedPegSettings; private readonly IBroadcasterManager broadcasterManager; - private readonly IAsyncProvider asyncProvider; - private readonly INodeLifetime nodeLifetime; private readonly MempoolManager mempoolManager; private readonly IDateTimeProvider dateTimeProvider; private readonly MempoolSettings mempoolSettings; @@ -55,8 +53,6 @@ public SignedMultisigTransactionBroadcasterTests() this.loggerFactory.CreateLogger(null).ReturnsForAnyArgs(this.logger); this.leaderReceiverSubscription = Substitute.For(); this.broadcasterManager = Substitute.For(); - this.asyncProvider = Substitute.For(); - this.nodeLifetime = Substitute.For(); this.ibdState = Substitute.For(); this.federationWalletManager = Substitute.For(); @@ -72,31 +68,15 @@ public SignedMultisigTransactionBroadcasterTests() MempoolExpiry = MempoolValidator.DefaultMempoolExpiry }; - this.blockPolicyEstimator = new BlockPolicyEstimator( - this.mempoolSettings, - this.loggerFactory, - this.nodeSettings); + this.blockPolicyEstimator = new BlockPolicyEstimator(this.loggerFactory, this.nodeSettings); - this.txMempool = new TxMempool( - this.dateTimeProvider, - this.blockPolicyEstimator, - this.loggerFactory, - this.nodeSettings); + this.txMempool = new TxMempool(this.dateTimeProvider, this.blockPolicyEstimator, this.loggerFactory, this.nodeSettings); this.mempoolValidator = Substitute.For(); this.mempoolPersistence = Substitute.For(); this.coinView = Substitute.For(); - this.mempoolManager = new MempoolManager( - new MempoolSchedulerLock(), - this.txMempool, - this.mempoolValidator, - this.dateTimeProvider, - this.mempoolSettings, - this.mempoolPersistence, - this.coinView, - this.loggerFactory, - this.nodeSettings.Network); + this.mempoolManager = new MempoolManager(new MempoolSchedulerLock(), this.txMempool, this.mempoolValidator, this.dateTimeProvider, this.mempoolSettings, this.mempoolPersistence, this.coinView, this.loggerFactory, this.nodeSettings.Network); } [Fact] @@ -104,13 +84,7 @@ public async Task Call_GetSignedTransactionsAsync_Signed_Transactions_Broadcasts { this.federatedPegSettings.PublicKey.Returns(PublicKey); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); @@ -130,13 +104,7 @@ public async Task Dont_Do_Work_In_IBD() { this.ibdState.IsInitialBlockDownload().Returns(true); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); @@ -158,13 +126,7 @@ public async Task Dont_Do_Work_Inactive_Federation() this.ibdState.IsInitialBlockDownload().Returns(true); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); diff --git a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs index b88f1e7a2ad..85437dd1369 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs @@ -269,7 +269,7 @@ public async Task InitializeAsync([CallerMemberName] string callingMethod = "") timeutc = DateTimeProvider.Default.GetUtcNow() }; - this.mempool = new TxMempool(dateTimeProviderSet, new BlockPolicyEstimator(new MempoolSettings(this.NodeSettings), this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); + this.mempool = new TxMempool(dateTimeProviderSet, new BlockPolicyEstimator(this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); this.mempoolLock = new MempoolSchedulerLock(); var blocks = new List(); From 43945d33f2eca3f25dc40ca190ec7f1919bc0fed Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sat, 26 Oct 2019 22:06:08 +0200 Subject: [PATCH 10/18] Initial code style fixes --- .../Fee/BlockPolicyEstimator.cs | 43 +++++--- .../Fee/TxConfirmStats.cs | 97 +++++++++---------- 2 files changed, 75 insertions(+), 65 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs index d46f8534060..dc72f46a453 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs @@ -168,6 +168,7 @@ public class BlockPolicyEstimator public BlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSettings) { Guard.Assert(MinBucketFeeRate > 0); + this.lockObject = new object(); this.logger = loggerFactory.CreateLogger(this.GetType().FullName); this.mapMemPoolTxs = new Dictionary(); @@ -180,21 +181,27 @@ public BlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSetti this.trackedTxs = 0; this.untrackedTxs = 0; int bucketIndex = 0; - for(double bucketBoundary = MinBucketFeeRate; bucketBoundary <= MaxBucketFeeRate; bucketBoundary *= FeeSpacing, bucketIndex++) + + for (double bucketBoundary = MinBucketFeeRate; bucketBoundary <= MaxBucketFeeRate; bucketBoundary *= FeeSpacing, bucketIndex++) { this.buckets.Add(bucketBoundary); this.bucketMap.Add(bucketBoundary, bucketIndex); } + this.buckets.Add(InfFeeRate); this.bucketMap.Add(InfFeeRate, bucketIndex); + Guard.Assert(this.bucketMap.Count == this.buckets.Count); this.feeStats = new TxConfirmStats(this.logger); this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); + this.shortStats = new TxConfirmStats(this.logger); this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); + this.longStats = new TxConfirmStats(this.logger); this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); + this.fileStorage = new FileStorage(nodeSettings.DataFolder.WalletPath); } @@ -208,15 +215,13 @@ public void ProcessBlock(int nBlockHeight, List entries) lock (this.lockObject) { if (nBlockHeight <= this.nBestSeenHeight) - // Ignore side chains and re-orgs; assuming they are random - // they don't affect the estimate. - // And if an attacker can re-org the chain at will, then - // you've got much bigger problems than "attacker can influence - // transaction fees." + { + // Ignore side chains and re-orgs; assuming they are random they don't affect the estimate. + // And if an attacker can re-org the chain at will, then you've got much bigger problems than "attacker can influence transaction fees." return; + } - // Must update nBestSeenHeight in sync with ClearCurrent so that - // calls to removeTx (via processBlockTx) correctly calculate age + // Must update nBestSeenHeight in sync with ClearCurrent so that calls to removeTx (via processBlockTx) correctly calculate age // of unconfirmed txs to remove from tracking. this.nBestSeenHeight = nBlockHeight; @@ -231,8 +236,9 @@ public void ProcessBlock(int nBlockHeight, List entries) this.longStats.UpdateMovingAverages(); int countedTxs = 0; + // Repopulate the current block states - foreach (var entry in entries) + foreach (TxMempoolEntry entry in entries) { if (this.ProcessBlockTx(nBlockHeight, entry)) countedTxs++; @@ -244,7 +250,7 @@ public void ProcessBlock(int nBlockHeight, List entries) this.logger.LogInformation("Blockpolicy first recorded height {0}", this.firstRecordedHeight); } - // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) + // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) // Logging.Logs.EstimateFee.LogInformation( // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); @@ -268,10 +274,10 @@ private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) // blocksToConfirm is 1-based, so a transaction included in the earliest // possible block has confirmation count of 1 int blocksToConfirm = nBlockHeight - entry.EntryHeight; + if (blocksToConfirm <= 0) { - // This can't happen because we don't process transactions from a block with a height - // lower than our greatest seen height + // This can't happen because we don't process transactions from a block with a height lower than our greatest seen height. this.logger.LogInformation($"Blockpolicy error Transaction had negative blocksToConfirm"); return false; } @@ -282,6 +288,7 @@ private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); this.shortStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); this.longStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + return true; } @@ -296,6 +303,7 @@ public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) { int txHeight = entry.EntryHeight; uint256 hash = entry.TransactionHash; + if (this.mapMemPoolTxs.ContainsKey(hash)) { this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); @@ -305,24 +313,27 @@ public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) if (txHeight != this.nBestSeenHeight) return; - // Only want to be updating estimates when our blockchain is synced, - // otherwise we'll miscalculate how many blocks its taking to get included. + // Only want to be updating estimates when our blockchain is synced, otherwise we'll miscalculate how many blocks its taking to get included. if (!validFeeEstimate) { this.untrackedTxs++; return; } + this.trackedTxs++; - // Feerates are stored and reported as BTC-per-kb: - var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + // Feerates are stored and reported as BTC-per-kb: + var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); this.mapMemPoolTxs[hash].blockHeight = txHeight; + int bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); this.mapMemPoolTxs[hash].bucketIndex = bucketIndex; + int bucketIndex2 = this.shortStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); Guard.Assert(bucketIndex == bucketIndex2); + int bucketIndex3 = this.longStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); Guard.Assert(bucketIndex == bucketIndex3); } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 7d855c2b8d0..e679a7ed30f 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -10,11 +10,17 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// - /// Transation confirmation statistics. + /// Transaction confirmation statistics. /// + /// + /// We will instantiate an instance of this class to track transactions that were + /// included in a block. We will lump transactions into a bucket according to their + /// approximate feerate and then track how long it took for those txs to be included in a block. + /// The tracking of unconfirmed (mempool) transactions is completely independent of the + /// historical tracking of transactions that have been confirmed in a block. + /// public class TxConfirmStats { - /// Instance logger for logging messages. private readonly ILogger logger; /// @@ -87,8 +93,7 @@ public class TxConfirmStats /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket private List oldUnconfTxs; - - + /// /// Constructs an instance of the transaction confirmation stats object. /// @@ -108,6 +113,7 @@ public TxConfirmStats(ILogger logger) public void Initialize(List defaultBuckets, IDictionary defaultBucketMap, int maxPeriods, double decay, int scale) { Guard.Assert(scale != 0); + this.decay = decay; this.scale = scale; this.confAvg = new List>(); @@ -121,19 +127,18 @@ public void Initialize(List defaultBuckets, IDictionary def this.confAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); this.failAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); } - - + this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + ClearInMemoryCounters(this.buckets.Count); } private void ClearInMemoryCounters(int bucketsCount) { for (int i = 0; i < GetMaxConfirms(); i++) - { this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), bucketsCount).ToList()); - } + this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), bucketsCount)); } @@ -160,12 +165,13 @@ public void Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; + int periodsToConfirm = (blocksToConfirm + this.scale - 1) / this.scale; int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; + for (int i = periodsToConfirm; i <= this.confAvg.Count; i++) - { this.confAvg[i - 1][bucketindex]++; - } + this.txCtAvg[bucketindex]++; this.avg[bucketindex] += val; } @@ -181,6 +187,7 @@ public int NewTx(int nBlockHeight, double val) int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; int blockIndex = nBlockHeight % this.unconfTxs.Count; this.unconfTxs[blockIndex][bucketindex]++; + return bucketindex; } @@ -194,8 +201,10 @@ public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool { //nBestSeenHeight is not updated yet for the new block int blocksAgo = nBestSeenHeight - entryHeight; + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet blocksAgo = 0; + if (blocksAgo < 0) { this.logger.LogInformation($"Blockpolicy error, blocks ago is negative for mempool tx"); @@ -207,31 +216,25 @@ public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool if (this.oldUnconfTxs[bucketIndex] > 0) this.oldUnconfTxs[bucketIndex]--; else - { - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); - } + this.logger.LogInformation($"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); } else { int blockIndex = entryHeight % this.unconfTxs.Count; + if (this.unconfTxs[blockIndex][bucketIndex] > 0) this.unconfTxs[blockIndex][bucketIndex]--; else - { - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); - } + this.logger.LogInformation($"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); } if(!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period { Guard.Assert(this.scale != 0); int periodsAgo = blocksAgo / this.scale; + for (int i = 0; i < periodsAgo && i < this.failAvg.Count; i++) - { this.failAvg[i][bucketIndex]++; - } } } @@ -245,8 +248,10 @@ public void UpdateMovingAverages() { for (var i = 0; i < this.confAvg.Count; i++) this.confAvg[i][j] = this.confAvg[i][j] * this.decay; + for (var i = 0; i < this.failAvg.Count; i++) this.failAvg[i][j] = this.failAvg[i][j] * this.decay; + this.avg[j] = this.avg[j] * this.decay; this.txCtAvg[j] = this.txCtAvg[j] * this.decay; } @@ -263,8 +268,7 @@ public void UpdateMovingAverages() /// Return the lowest feerate such that all higher values pass minSuccess OR return the highest feerate such that all lower values fail minSuccess. /// The current block height. /// - public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, - bool requireGreater, int nBlockHeight, EstimationResult result) + public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, int nBlockHeight, EstimationResult result) { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget @@ -307,15 +311,17 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s curNearBucket = bucket; newBucketRange = false; } + curFarBucket = bucket; nConf += this.confAvg[periodTarget - 1][bucket]; totalNum += this.txCtAvg[bucket]; failNum += this.failAvg[periodTarget - 1][bucket]; + for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) - { extraNum += this.unconfTxs[Math.Abs(nBlockHeight - confct) % bins][bucket]; - } + extraNum += this.oldUnconfTxs[bucket]; + // If we have enough transaction data points in this range of buckets, // we can test for success // (Only count the confirmed data points, so that each confirmation count @@ -325,10 +331,9 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if ((requireGreater && curPct < successBreakPoint)|| - (!requireGreater && curPct > successBreakPoint)) + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) { - if (passing == true) + if (passing) { // First time we hit a failure record the failed bucket int failMinBucket = Math.Min(curNearBucket, curFarBucket); @@ -341,13 +346,12 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s failBucket.LeftMempool = failNum; passing = false; } + continue; } - - // Otherwise update the cumulative stats, and the bucket variables - // and reset the counters else { + // Otherwise update the cumulative stats, and the bucket variables and reset the counters failBucket = new EstimatorBucket(); // Reset any failed bucket, currently passing foundAnswer = true; passing = true; @@ -375,13 +379,16 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s // and reporting the average which is less accurate int minBucket = Math.Min(bestNearBucket, bestFarBucket); int maxBucket = Math.Max(bestNearBucket, bestFarBucket); + for (int j = minBucket; j <= maxBucket; j++) { txSum += this.txCtAvg[j]; } + if (foundAnswer && txSum != 0) { txSum = txSum / 2; + for (int j = minBucket; j <= maxBucket; j++) { if (this.txCtAvg[j] < txSum) @@ -413,7 +420,7 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s failBucket.LeftMempool = failNum; } - this.logger.LogInformation( + this.logger.LogDebug( $"FeeEst: {confTarget} {(requireGreater ? $">" : $"<")} " + $"{successBreakPoint} decay {this.decay} feerate: {median}" + $" from ({passBucket.Start} - {passBucket.End}" + @@ -432,6 +439,7 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s result.Decay = this.decay; result.Scale = this.scale; } + return median; } @@ -471,50 +479,41 @@ public void Read(TxConfirmData data) try { if (data == null) - { throw new ArgumentNullException(nameof(data)); - } + if(data.Decay <= 0 || data.Decay >= 1) - { throw new ApplicationException("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - } + if(data.Scale == 0) - { throw new ApplicationException("Corrupt estimates file. Scale must be non-zero"); - } + if(data.Avg.Count != this.buckets.Count) - { throw new ApplicationException("Corrupt estimates file. Mismatch in feerate average bucket count"); - } + if (data.TxCtAvg.Count != this.buckets.Count) - { throw new ApplicationException("Corrupt estimates file. Mismatch in tx count bucket count"); - } + int maxPeriods = data.ConfAvg.Count; double maxConfirms = data.Scale * maxPeriods; if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) - { throw new ApplicationException("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); - } + for (int i = 0; i < maxPeriods; i++) { if (data.ConfAvg[i].Count != this.buckets.Count) - { throw new ApplicationException("Corrupt estimates file. Mismatch in feerate conf average bucket count"); - } } + if (maxPeriods != data.FailAvg.Count) - { throw new ApplicationException("Corrupt estimates file. Mismatch in confirms tracked for failures"); - } + for (int i = 0; i < maxPeriods; i++) { if (data.FailAvg[i].Count != this.buckets.Count) - { throw new ApplicationException("Corrupt estimates file. Mismatch in one of failure average bucket counts"); - } } + this.decay = data.Decay; this.scale = data.Scale; this.avg = data.Avg; From d2be61d57ef66170fa00a7d237c49cf740dca0ab Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sat, 26 Oct 2019 22:47:10 +0200 Subject: [PATCH 11/18] Remove unused files --- .../Fee/FeeEstimateMode.cs | 16 -------- .../Fee/FeeFilterRounder.cs | 37 ------------------- 2 files changed, 53 deletions(-) delete mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs delete mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs deleted file mode 100644 index dc50a38bb35..00000000000 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateMode.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee -{ - /// - /// Used to determine type of fee estimation requested - /// - public enum FeeEstimateMode - { - Unset, - Economical, - Conservative - } -} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs deleted file mode 100644 index 086fa60c0c6..00000000000 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeFilterRounder.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using NBitcoin; -using System.Linq; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee -{ - public class FeeFilterRounder - { - private const decimal MaxFilterFeeRate = 1e7M; - private const decimal FeeFilterSpacing = 1.1M; - - private SortedSet feeSet; - - public FeeFilterRounder(FeeRate minIncrementalFee) - { - Money minFeeLimit = Math.Max(new Money(1), minIncrementalFee.FeePerK / 2); - this.feeSet = new SortedSet(); - this.feeSet.Add(0); - for (decimal bucketBoundary = minFeeLimit.ToDecimal(MoneyUnit.BTC); bucketBoundary <= MaxFilterFeeRate; bucketBoundary *= FeeFilterSpacing) - { - this.feeSet.Add(bucketBoundary); - } - } - - public Money Round(Money currentMinFee) - { - var it = this.feeSet.FirstOrDefault(f => f >= currentMinFee.ToDecimal(MoneyUnit.BTC)); - if ((it != this.feeSet.FirstOrDefault() && RandomUtils.GetInt32() % 3 != 0) || it == this.feeSet.LastOrDefault()) - { - it--; - } - return new Money(it, MoneyUnit.BTC); - } - } -} From 3cfe2a5e614f303d4ed8b27a406dd93863305c41 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sun, 27 Oct 2019 00:06:46 +0200 Subject: [PATCH 12/18] Start creating interface --- ...ator.cs => BitcoinBlockPolicyEstimator.cs} | 298 +++++++----------- .../Fee/IBlockPolicyEstimator.cs | 82 +++++ .../SerializationEntity/BlockPolicyData.cs | 10 +- .../Fee/StratisBlockPolicyEstimator.cs | 60 ++++ .../Fee/TxStatsInfo.cs | 23 ++ .../Interfaces/ITxMempool.cs | 2 +- .../MempoolFeature.cs | 6 +- .../TxMemPool.cs | 4 +- 8 files changed, 298 insertions(+), 187 deletions(-) rename src/Stratis.Bitcoin.Features.MemoryPool/Fee/{BlockPolicyEstimator.cs => BitcoinBlockPolicyEstimator.cs} (77%) create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs create mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs similarity index 77% rename from src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs rename to src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs index dc72f46a453..b3d54c1f910 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs @@ -1,27 +1,27 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Extensions.Logging; using NBitcoin; using Stratis.Bitcoin.Configuration; using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; -using Stratis.Bitcoin.Features.MemoryPool.Interfaces; using Stratis.Bitcoin.Utilities; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// /// The BlockPolicyEstimator is used for estimating the feerate needed - /// for a transaction to be included in a block within a certain number of - /// blocks. + /// for a transaction to be included in a block within a certain number of blocks. /// /// + /// This is based on the fee estimator used in Bitcoin Core 0.15 onwards; it operates + /// differently from the 0.14 version. + /// /// At a high level the algorithm works by grouping transactions into buckets /// based on having similar feerates and then tracking how long it - /// takes transactions in the various buckets to be mined. It operates under + /// takes transactions in the various buckets to be mined. It operates under /// the assumption that in general transactions of higher feerate will be - /// included in blocks before transactions of lower feerate. So for + /// included in blocks before transactions of lower feerate. So for /// example if you wanted to know what feerate you should put on a transaction to /// be included in a block within the next 5 blocks, you would start by looking /// at the bucket with the highest feerate transactions and verifying that a @@ -34,10 +34,10 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee /// /// Here is a brief description of the implementation: /// When a transaction enters the mempool, we track the height of the block chain - /// at entry. All further calculations are conducted only on this set of "seen" + /// at entry. All further calculations are conducted only on this set of "seen" /// transactions.Whenever a block comes in, we count the number of transactions /// in each bucket and the total amount of feerate paid in each bucket. Then we - /// calculate how many blocks Y it took each transaction to be mined. We convert + /// calculate how many blocks Y it took each transaction to be mined. We convert /// from a number of blocks to a number of periods Y' each encompassing "scale" /// blocks.This is tracked in 3 different data sets each up to a maximum /// number of periods.Within each data set we have an array of counters in each @@ -46,49 +46,49 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee /// that many periods. We want to save a history of this information, so at any /// time we have a counter of the total number of transactions that happened in a /// given feerate bucket and the total number that were confirmed in each of the - /// periods or less for any bucket. We save this history by keeping an - /// exponentially decaying moving average of each one of these stats. This is + /// periods or less for any bucket. We save this history by keeping an + /// exponentially decaying moving average of each one of these stats. This is /// done for a different decay in each of the 3 data sets to keep relevant data - /// from different time horizons. Furthermore we also keep track of the number + /// from different time horizons. Furthermore we also keep track of the number /// unmined (in mempool or left mempool without being included in a block) /// transactions in each bucket and for how many blocks they have been /// outstanding and use both of these numbers to increase the number of transactions /// we've seen in that feerate bucket when calculating an estimate for any number /// of confirmations below the number of blocks they've been outstanding. /// - public class BlockPolicyEstimator + public class BitcoinBlockPolicyEstimator : IBlockPolicyEstimator { - ///Track confirm delays up to 12 blocks for short horizon + /// Track confirm delays up to 12 blocks for short horizon private const int ShortBlockPeriods = 12; private const int ShortScale = 1; - ///Track confirm delays up to 48 blocks for medium horizon + /// Track confirm delays up to 48 blocks for medium horizon private const int MedBlockPeriods = 24; private const int MedScale = 2; - ///Track confirm delays up to 1008 blocks for long horizon + /// Track confirm delays up to 1008 blocks for long horizon private const int LongBlockPeriods = 42; private const int LongScale = 24; - ///Historical estimates that are older than this aren't valid + /// Historical estimates that are older than this aren't valid private const int OldestEstimateHistory = 6 * 1008; - ///Decay of .962 is a half-life of 18 blocks or about 3 hours + /// Decay of .962 is a half-life of 18 blocks or about 3 hours private const double ShortDecay = .962; - ///Decay of .998 is a half-life of 144 blocks or about 1 day + /// Decay of .998 is a half-life of 144 blocks or about 1 day private const double MedDecay = .9952; - ///Decay of .9995 is a half-life of 1008 blocks or about 1 week + /// Decay of .9995 is a half-life of 1008 blocks or about 1 week private const double LongDecay = .99931; - ///Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks + /// Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks private const double HalfSuccessPct = .6; - ///Require greater than 85% of X feerate transactions to be confirmed within Y blocks + /// Require greater than 85% of X feerate transactions to be confirmed within Y blocks private const double SuccessPct = .85; - ///Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks + /// Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks private const double DoubleSuccessPct = .95; /// Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance. @@ -97,12 +97,13 @@ public class BlockPolicyEstimator /// Require an avg of 0.5 tx when using short decay since there are fewer blocks considered private const double SufficientTxsShort = 0.5; - /// Minimum and Maximum values for tracking feerates - /// The MinBucketFeeRate should just be set to the lowest reasonable feerate we - /// might ever want to track. Historically this has been 1000 since it was - /// inheriting DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it - /// invalidates old estimates files. So leave it at 1000 unless it becomes - /// necessary to lower it, and then lower it substantially. + /// + /// Minimum and maximum values for tracking feerates. + /// The should just be set to the lowest reasonable feerate we + /// might ever want to track. Historically this has been 1000 since it was inheriting + /// DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it invalidates old estimates + /// files. So leave it at 1000 unless it becomes necessary to lower it, and then lower it substantially. + /// private const double MinBucketFeeRate = 1000; private const double MaxBucketFeeRate = 1e7; @@ -120,6 +121,7 @@ public class BlockPolicyEstimator private const string FileName = "fee.json"; + // TODO: How is this used? Should probably use ChainIndexer instead /// Best seen block height. private int nBestSeenHeight; @@ -162,10 +164,9 @@ public class BlockPolicyEstimator /// /// Constructs an instance of the block policy estimator object. /// - /// Mempool settings. /// Factory for creating loggers. /// Full node settings. - public BlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSettings) + public BitcoinBlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSettings) { Guard.Assert(MinBucketFeeRate > 0); @@ -205,11 +206,7 @@ public BlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSetti this.fileStorage = new FileStorage(nodeSettings.DataFolder.WalletPath); } - /// - /// Process all the transactions that have been included in a block. - /// - /// The block height for the block. - /// Collection of memory pool entries. + /// public void ProcessBlock(int nBlockHeight, List entries) { lock (this.lockObject) @@ -292,11 +289,7 @@ private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) return true; } - /// - /// Process a transaction accepted to the mempool. - /// - /// Memory pool entry. - /// Whether to update fee estimate. + /// public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) { lock (this.lockObject) @@ -339,39 +332,26 @@ public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) } } - /// - /// Remove a transaction from the mempool tracking stats. - /// - /// Transaction hash. - /// Whether the transaction was successfully removed. - /// - /// This function is called from TxMemPool.RemoveUnchecked to ensure - /// txs removed from the mempool for any reason are no longer - /// tracked. Txs that were part of a block have already been removed in - /// ProcessBlockTx to ensure they are never double tracked, but it is - /// of no harm to try to remove them again. - /// + /// public bool RemoveTx(uint256 hash, bool inBlock) { lock (this.lockObject) { TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); - if (pos != null) - { - this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); - this.shortStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); - this.longStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); - this.mapMemPoolTxs.Remove(hash); - return true; - } - return false; + + if (pos == null) + return false; + + this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.shortStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.longStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.mapMemPoolTxs.Remove(hash); + + return true; } } - /// - /// Return an estimate fee according to horizon - /// - /// The desired number of confirmations to be included in a block + /// public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) { TxConfirmStats stats; @@ -399,43 +379,30 @@ public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstima throw new ArgumentException(nameof(horizon)); } } + lock(this.lockObject) { // Return failure if trying to analyze a target we're not tracking if (confTarget <= 0 || confTarget > stats.GetMaxConfirms()) - { return new FeeRate(0); - } + if (successThreshold > 1) - { return new FeeRate(0); - } - double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, - true, this.nBestSeenHeight, result); - - if (median < 0) - return new FeeRate(0); + double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, this.nBestSeenHeight, result); - return new FeeRate(Convert.ToInt64(median)); + return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); } } - /// - /// Return a feerate estimate - /// - /// The desired number of confirmations to be included in a block. + /// public FeeRate EstimateFee(int confTarget) { // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1) - { - return new FeeRate(0); - } - - return EstimateRawFee(confTarget, DoubleSuccessPct, FeeEstimateHorizon.MedHalfLife, null); + return confTarget <= 1 ? new FeeRate(0) : EstimateRawFee(confTarget, DoubleSuccessPct, FeeEstimateHorizon.MedHalfLife, null); } + /// public int HighestTargetTracked(FeeEstimateHorizon horizon) { switch (horizon) @@ -461,7 +428,9 @@ public int HighestTargetTracked(FeeEstimateHorizon horizon) private int BlockSpan() { - if (this.firstRecordedHeight == 0) return 0; + if (this.firstRecordedHeight == 0) + return 0; + Guard.Assert(this.nBestSeenHeight >= this.firstRecordedHeight); return this.nBestSeenHeight - this.firstRecordedHeight; @@ -469,13 +438,13 @@ private int BlockSpan() private int HistoricalBlockSpan() { - if (this.historicalFirst == 0) return 0; + if (this.historicalFirst == 0) + return 0; + Guard.Assert(this.historicalBest >= this.historicalFirst); if (this.nBestSeenHeight - this.historicalBest > OldestEstimateHistory) - { return 0; - } return this.historicalBest - this.historicalFirst; } @@ -492,60 +461,62 @@ private int MaxUsableEstimate() /// checkShorterHorizon is requested, also allow short time horizon estimates /// for a lower target to reduce the given answer /// - private double EstimateCombinedFee(int confTarget, double successThreshold, - bool checkShorterHorizon, EstimationResult result) + private double EstimateCombinedFee(int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult result) { double estimate = -1; + if (confTarget >= 1 && confTarget <= this.longStats.GetMaxConfirms()) { // Find estimate from shortest time horizon possible if (confTarget <= this.shortStats.GetMaxConfirms()) - { // short horizon - estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, - successThreshold, true, this.nBestSeenHeight, result); + { + // short horizon + estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, result); } else if (confTarget <= this.feeStats.GetMaxConfirms()) - { // medium horizon - estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, - successThreshold, true, this.nBestSeenHeight, result); + { + // medium horizon + estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); } else - { // long horizon - estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, - successThreshold, true, this.nBestSeenHeight, result); + { + // long horizon + estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); } + if (checkShorterHorizon) { - EstimationResult tempResult = new EstimationResult(); + var tempResult = new EstimationResult(); + // If a lower confTarget from a more recent horizon returns a lower answer use it. if (confTarget > this.feeStats.GetMaxConfirms()) { - double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), - SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); + double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); + if (medMax > 0 && (estimate == -1 || medMax < estimate)) { estimate = medMax; + if (result != null) - { result = tempResult; - } } } + if (confTarget > this.shortStats.GetMaxConfirms()) { - double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), - SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); + double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) { estimate = shortMax; + if (result != null) - { result = tempResult; - } } } } } + return estimate; } @@ -556,41 +527,32 @@ private double EstimateCombinedFee(int confTarget, double successThreshold, private double EstimateConservativeFee(int doubleTarget, EstimationResult result) { double estimate = -1; - EstimationResult tempResult = new EstimationResult(); + var tempResult = new EstimationResult(); + if (doubleTarget <= this.shortStats.GetMaxConfirms()) - { - estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, - DoubleSuccessPct, true, this.nBestSeenHeight, result); - } + estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, result); + if (doubleTarget <= this.feeStats.GetMaxConfirms()) { - double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, - DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); + double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); + if (longEstimate > estimate) { estimate = longEstimate; + if (result != null) - { result = tempResult; - } } } + return estimate; } - /// - /// Returns the max of the feerates calculated with a 60% - /// threshold required at target / 2, an 85% threshold required at target and a - /// 95% threshold required at 2 * target.Each calculation is performed at the - /// shortest time horizon which tracks the required target.Conservative - /// estimates, however, required the 95% threshold at 2 * target be met for any - /// longer time horizons also. - /// + /// public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) { lock (this.lockObject) { - if (feeCalc != null) { feeCalc.DesiredTarget = confTarget; @@ -602,30 +564,22 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con // Return failure if trying to analyze a target we're not tracking if (confTarget <= 0 || confTarget > this.longStats.GetMaxConfirms()) - { return new FeeRate(0); // error condition - } // It's not possible to get reasonable estimates for confTarget of 1 if (confTarget == 1) - { confTarget = 2; - } int maxUsableEstimate = MaxUsableEstimate(); + if (confTarget > maxUsableEstimate) - { confTarget = maxUsableEstimate; - } + if (feeCalc != null) - { feeCalc.ReturnedTarget = confTarget; - } if (confTarget <= 1) - { return new FeeRate(0); // error condition - } Guard.Assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints @@ -639,26 +593,33 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con // the purpose of conservative estimates is not to let short term // fluctuations lower our estimates by too much. double halfEst = EstimateCombinedFee(confTarget / 2, HalfSuccessPct, true, tempResult); + if (feeCalc != null) { feeCalc.Estimation = tempResult; feeCalc.Reason = FeeReason.HalfEstimate; } + median = halfEst; double actualEst = EstimateCombinedFee(confTarget, SuccessPct, true, tempResult); + if (actualEst > median) { median = actualEst; + if (feeCalc != null) { feeCalc.Estimation = tempResult; feeCalc.Reason = FeeReason.FullEstimate; } } + double doubleEst = EstimateCombinedFee(2 * confTarget, DoubleSuccessPct, !conservative, tempResult); + if (doubleEst > median) { median = doubleEst; + if (feeCalc != null) { feeCalc.Estimation = tempResult; @@ -669,9 +630,11 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con if (conservative || median == -1) { double consEst = EstimateConservativeFee(2 * confTarget, tempResult); + if (consEst > median) { median = consEst; + if (feeCalc != null) { feeCalc.Estimation = tempResult; @@ -680,18 +643,17 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con } } - if (median < 0) return new FeeRate(0); // error condition - - return new FeeRate(Convert.ToInt64(median)); + return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); } } - /// - /// Write estimation data to a file. - /// + + /// public void Write() { var data = new BlockPolicyData(); + data.BestSeenHeight = this.nBestSeenHeight; + if (BlockSpan() > HistoricalBlockSpan()) { data.HistoricalFirst = this.firstRecordedHeight; @@ -702,18 +664,16 @@ public void Write() data.HistoricalFirst = this.historicalFirst; data.HistoricalBest = this.historicalBest; } + data.Buckets = this.buckets; data.ShortStats = this.shortStats.Write(); data.MedStats = this.feeStats.Write(); data.LongStats = this.longStats.Write(); + this.fileStorage.SaveToFile(data, FileName); } - /// - /// Read estimation data from a file. - /// - /// Stream to read data from. - /// Version number of the file. + /// public bool Read() { try @@ -722,32 +682,31 @@ public bool Read() { var data = this.fileStorage.LoadByFileName(FileName); if (data != null) - { throw new ApplicationException("Corrupt estimates file or file not found"); - } + if (data.HistoricalFirst > data.HistoricalBest || data.HistoricalBest > data.BestSeenHeight) - { throw new ApplicationException("Corrupt estimates file. Historical block range for estimates is invalid"); - } + if (data.Buckets.Count <= 1 || data.Buckets.Count > 1000) - { throw new ApplicationException("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); - } + this.nBestSeenHeight = data.BestSeenHeight; this.historicalFirst = data.HistoricalFirst; this.historicalBest = data.HistoricalBest; this.buckets = data.Buckets; this.bucketMap = new SortedDictionary(); + for(int i = 0; i< this.buckets.Count; i++) - { this.bucketMap.Add(this.buckets[i], i); - } + this.feeStats = new TxConfirmStats(this.logger); this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); this.feeStats.Read(data.MedStats); + this.shortStats = new TxConfirmStats(this.logger); this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); this.shortStats.Read(data.ShortStats); + this.longStats = new TxConfirmStats(this.logger); this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); this.longStats.Read(data.LongStats); @@ -758,42 +717,25 @@ public bool Read() this.logger.LogError("Error while reading policy estimation data from file", e); return false; } + return true; } - public void FlushUncomfirmed() + /// + public void FlushUnconfirmed() { lock (this.lockObject) { int numEntries = this.mapMemPoolTxs.Count; + // Remove every entry in mapMemPoolTxs while (this.mapMemPoolTxs.Count > 0) { var mi = this.mapMemPoolTxs.First(); ; RemoveTx(mi.Key, false); // this calls erase() on mapMemPoolTxs } - this.logger.LogInformation($"Recorded {numEntries} unconfirmed txs from mempool"); - } - } - - /// - /// Transaction statistics information. - /// - public class TxStatsInfo - { - /// The block height. - public int blockHeight; - /// The index into the confirmed transactions bucket map. - public int bucketIndex; - - /// - /// Constructs a instance of a transaction stats info object. - /// - public TxStatsInfo() - { - this.blockHeight = 0; - this.bucketIndex = 0; + this.logger.LogInformation($"Recorded {numEntries} unconfirmed txs from mempool"); } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs new file mode 100644 index 00000000000..fe43ee2f74d --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using NBitcoin; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public interface IBlockPolicyEstimator + { + /// + /// Return a feerate estimate (deprecated, per Bitcoin Core source). + /// + /// The desired number of confirmations to be included in a block. + FeeRate EstimateFee(int confTarget); + + /// + /// Remove a transaction from the mempool tracking stats. + /// + /// Transaction hash. + /// Whether the transaction was successfully removed. + /// + /// This function is called from TxMemPool.RemoveUnchecked to ensure + /// txs removed from the mempool for any reason are no longer + /// tracked. Txs that were part of a block have already been removed in + /// ProcessBlockTx to ensure they are never double tracked, but it is + /// of no harm to try to remove them again. + /// + bool RemoveTx(uint256 hash, bool inBlock); + + /// + /// Return a specific fee estimate calculation with a given success threshold and time horizon, + /// and optionally return detailed data about calculation. + /// + /// The desired number of confirmations to be included in a block + FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result); + + /// + /// Returns the max of the feerates calculated with a 60% + /// threshold required at target / 2, an 85% threshold required at target and a + /// 95% threshold required at 2 * target.Each calculation is performed at the + /// shortest time horizon which tracks the required target.Conservative + /// estimates, however, required the 95% threshold at 2 * target be met for any + /// longer time horizons also. + /// + FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative); + + /// + /// Process all the transactions that have been included in a block. + /// + /// The block height for the block. + /// Collection of memory pool entries. + void ProcessBlock(int nBlockHeight, List entries); + + /// + /// Process a transaction accepted to the mempool. + /// + /// Memory pool entry. + /// Whether to update fee estimate. + void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate); + + /// + /// Calculation of highest target that estimates are tracked for. + /// + /// + int HighestTargetTracked(FeeEstimateHorizon horizon); + + /// + /// Write estimation data to a file. + /// + void Write(); + + /// + /// Read estimation data from a file. + /// + /// Stream to read data from. + /// Version number of the file. + bool Read(); + + /// + /// Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool. + /// + void FlushUnconfirmed(); + } +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs index 10dbb54076d..04846c8ee58 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs @@ -1,17 +1,21 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections.Generic; namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity { public class BlockPolicyData { public int BestSeenHeight { get; set; } + public int HistoricalFirst { get; set; } + public int HistoricalBest { get; set; } + public List Buckets { get; set; } + public TxConfirmData ShortStats { get; set; } + public TxConfirmData MedStats { get; set; } + public TxConfirmData LongStats { get; set; } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs new file mode 100644 index 00000000000..002fbd43550 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using NBitcoin; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public class StratisBlockPolicyEstimator : IBlockPolicyEstimator + { + // TODO: Need to complete the Bitcoin version of the algorithm, then restructure it to compute its constants in a non-network-specific way + + public FeeRate EstimateFee(int confTarget) + { + throw new System.NotImplementedException(); + } + + public bool RemoveTx(uint256 hash, bool inBlock) + { + throw new System.NotImplementedException(); + } + + public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) + { + throw new System.NotImplementedException(); + } + + public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) + { + throw new System.NotImplementedException(); + } + + public void ProcessBlock(int nBlockHeight, List entries) + { + throw new System.NotImplementedException(); + } + + public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) + { + throw new System.NotImplementedException(); + } + + public int HighestTargetTracked(FeeEstimateHorizon horizon) + { + throw new System.NotImplementedException(); + } + + public void Write() + { + throw new System.NotImplementedException(); + } + + public bool Read() + { + throw new System.NotImplementedException(); + } + + public void FlushUnconfirmed() + { + throw new System.NotImplementedException(); + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs new file mode 100644 index 00000000000..56dc1909425 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs @@ -0,0 +1,23 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Transaction statistics information. + /// + public class TxStatsInfo + { + /// The block height. + public int blockHeight; + + /// The index into the confirmed transactions bucket map. + public int bucketIndex; + + /// + /// Constructs an instance of a transaction stats info object. + /// + public TxStatsInfo() + { + this.blockHeight = 0; + this.bucketIndex = 0; + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs index 9cb5182a301..bfd49ceecea 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs @@ -11,7 +11,7 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Interfaces public interface ITxMempool { /// Gets the miner policy estimator. - BlockPolicyEstimator MinerPolicyEstimator { get; } + IBlockPolicyEstimator MinerPolicyEstimator { get; } /// Get the number of transactions in the memory pool. long Size { get; } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs index ab27096055f..25f89f395c4 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs @@ -142,12 +142,12 @@ public static class FullNodeBuilderMempoolExtension /// Include the memory pool feature and related services in the full node. /// /// Full node builder. - /// Whether or not to inject the mempool rules now, or defer it to another feature. /// Full node builder. public static IFullNodeBuilder UseMempool(this IFullNodeBuilder fullNodeBuilder) { + // TODO: Need to inject appropriate version of IBlockPolicyEstimator. Is it worth splitting this into two different extension methods? LoggingConfiguration.RegisterFeatureNamespace("mempool"); - LoggingConfiguration.RegisterFeatureNamespace("estimatefee"); + LoggingConfiguration.RegisterFeatureNamespace("estimatefee"); fullNodeBuilder.ConfigureFeature(features => { @@ -158,7 +158,7 @@ public static IFullNodeBuilder UseMempool(this IFullNodeBuilder fullNodeBuilder) { services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton() diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs index 154705afd8a..f80cf9a6ef7 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs @@ -182,7 +182,7 @@ public class TxMempool : ITxMempool /// The block policy estimator object. /// Factory for creating instance logger. /// Full node settings. - public TxMempool(IDateTimeProvider dateTimeProvider, BlockPolicyEstimator blockPolicyEstimator, ILoggerFactory loggerFactory, NodeSettings nodeSettings) + public TxMempool(IDateTimeProvider dateTimeProvider, IBlockPolicyEstimator blockPolicyEstimator, ILoggerFactory loggerFactory, NodeSettings nodeSettings) { this.MapTx = new IndexedTransactionSet(); this.mapLinks = new TxlinksMap(); @@ -203,7 +203,7 @@ public TxMempool(IDateTimeProvider dateTimeProvider, BlockPolicyEstimator blockP } /// Gets the miner policy estimator. - public BlockPolicyEstimator MinerPolicyEstimator { get; } + public IBlockPolicyEstimator MinerPolicyEstimator { get; } /// Get the number of transactions in the memory pool. public long Size From 7fc793eb1411da08e240c9923a8a4f65d9681c34 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Sun, 27 Oct 2019 20:28:49 +0200 Subject: [PATCH 13/18] Cleanup --- .../ColdStakingControllerTest.cs | 2 +- .../TestChainFactory.cs | 2 +- .../FeeTests.cs | 2 +- .../MemoryPoolTransactionTests.cs | 10 +- .../MempoolManagerTest.cs | 2 +- .../MempoolPersistenceTest.cs | 2 +- .../TestChainFactory.cs | 4 +- .../Fee/BitcoinBlockPolicyEstimator.cs | 452 +++++++++--------- .../Fee/FeeReason.cs | 8 +- .../Fee/IBlockPolicyEstimator.cs | 48 +- .../Fee/TxConfirmStats.cs | 156 +++--- .../MinerTests.cs | 2 +- ...gnedMultisigTransactionBroadcasterTests.cs | 4 +- .../PoW/SmartContractMinerTests.cs | 2 +- 14 files changed, 343 insertions(+), 353 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs index 229ed759963..f68649125be 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs +++ b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs @@ -138,7 +138,7 @@ private void CreateMempoolManager() { this.mempoolSettings = new MempoolSettings(this.nodeSettings); this.consensusSettings = new ConsensusSettings(this.nodeSettings); - this.txMemPool = new TxMempool(this.dateTimeProvider, new BlockPolicyEstimator(this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); + this.txMemPool = new TxMempool(this.dateTimeProvider, new BitcoinBlockPolicyEstimator(this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); this.chainIndexer = new ChainIndexer(this.Network); this.nodeDeployments = new NodeDeployments(this.Network, this.chainIndexer); diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index 8cc732af14b..e6ab4841448 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -181,7 +181,7 @@ public static async Task> MineBlocksAsync(TestChainContext testChain /// private static async Task> MineBlocksAsync(TestChainContext testChainContext, int count, Script receiver, bool mutateLastBlock) { - var blockPolicyEstimator = new BlockPolicyEstimator(testChainContext.LoggerFactory, testChainContext.NodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempool = new TxMempool(testChainContext.DateTimeProvider, blockPolicyEstimator, testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs index 26305019d03..72fcf7fbb27 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs @@ -17,7 +17,7 @@ public void BlockPolicyEstimates() { var dateTimeSet = new DateTimeProviderSet(); NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var mpool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var mpool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); var entry = new TestMemPoolEntryHelper(); var basefee = new Money(2000); var deltaFee = new Money(100); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs index 30b501b3e58..2bb377c1400 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs @@ -48,7 +48,7 @@ public void MempoolRemoveTest() } NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var testPool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); // Nothing in pool, remove should do nothing: long poolSize = testPool.Size; @@ -116,7 +116,7 @@ private void CheckSort(TxMempool pool, List sortedSource, List(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs index ead753fa367..0c68db908d9 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs @@ -282,7 +282,7 @@ private MempoolManager CreateTestMempool(NodeSettings settings, out TxMempool tx NodeSettings nodeSettings = NodeSettings.Default(settings.Network); ILoggerFactory loggerFactory = nodeSettings.LoggerFactory; var consensusSettings = new ConsensusSettings(nodeSettings); - txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); + txMemPool = new TxMempool(dateTimeProvider, new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); var coins = new InMemoryCoinView(settings.Network.GenesisHash); var chain = new ChainIndexer(settings.Network); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs index d5f617b69d5..1ec049f9580 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs @@ -108,7 +108,7 @@ public static async Task CreatePosAsync(Network network, Scri await consensus.InitializeAsync(genesis).ConfigureAwait(false); var mempoolSettings = new MempoolSettings(nodeSettings); - var blockPolicyEstimator = new BlockPolicyEstimator(loggerFactory, nodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); @@ -188,7 +188,7 @@ public static async Task CreateAsync(Network network, Script chainState.BlockStoreTip = genesis; await consensus.InitializeAsync(genesis).ConfigureAwait(false); - var blockPolicyEstimator = new BlockPolicyEstimator(loggerFactory, nodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs index b3d54c1f910..517aa516311 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs @@ -247,48 +247,13 @@ public void ProcessBlock(int nBlockHeight, List entries) this.logger.LogInformation("Blockpolicy first recorded height {0}", this.firstRecordedHeight); } - // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) - // Logging.Logs.EstimateFee.LogInformation( - // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); + this.logger.LogDebug($"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); this.trackedTxs = 0; this.untrackedTxs = 0; } } - /// - /// Process a transaction confirmed in a block. - /// - /// Height of the block. - /// The memory pool entry. - /// Whether it was able to successfully process the transaction. - private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) - { - if (!this.RemoveTx(entry.TransactionHash, true)) - return false; - - // How many blocks did it take for miners to include this transaction? - // blocksToConfirm is 1-based, so a transaction included in the earliest - // possible block has confirmation count of 1 - int blocksToConfirm = nBlockHeight - entry.EntryHeight; - - if (blocksToConfirm <= 0) - { - // This can't happen because we don't process transactions from a block with a height lower than our greatest seen height. - this.logger.LogInformation($"Blockpolicy error Transaction had negative blocksToConfirm"); - return false; - } - - // Feerates are stored and reported as BTC-per-kb: - var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); - this.shortStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); - this.longStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); - - return true; - } - /// public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) { @@ -351,50 +316,6 @@ public bool RemoveTx(uint256 hash, bool inBlock) } } - /// - public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) - { - TxConfirmStats stats; - double sufficientTxs = SufficientFeeTxs; - switch (horizon) - { - case FeeEstimateHorizon.ShortHalfLife: - { - stats = this.shortStats; - sufficientTxs = SufficientTxsShort; - break; - } - case FeeEstimateHorizon.MedHalfLife: - { - stats = this.feeStats; - break; - } - case FeeEstimateHorizon.LongHalfLife: - { - stats = this.longStats; - break; - } - default: - { - throw new ArgumentException(nameof(horizon)); - } - } - - lock(this.lockObject) - { - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || confTarget > stats.GetMaxConfirms()) - return new FeeRate(0); - - if (successThreshold > 1) - return new FeeRate(0); - - double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, this.nBestSeenHeight, result); - - return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); - } - } - /// public FeeRate EstimateFee(int confTarget) { @@ -402,152 +323,6 @@ public FeeRate EstimateFee(int confTarget) return confTarget <= 1 ? new FeeRate(0) : EstimateRawFee(confTarget, DoubleSuccessPct, FeeEstimateHorizon.MedHalfLife, null); } - /// - public int HighestTargetTracked(FeeEstimateHorizon horizon) - { - switch (horizon) - { - case FeeEstimateHorizon.ShortHalfLife: - { - return this.shortStats.GetMaxConfirms(); - } - case FeeEstimateHorizon.MedHalfLife: - { - return this.feeStats.GetMaxConfirms(); - } - case FeeEstimateHorizon.LongHalfLife: - { - return this.longStats.GetMaxConfirms(); - } - default: - { - throw new ArgumentException(nameof(horizon)); - } - } - } - - private int BlockSpan() - { - if (this.firstRecordedHeight == 0) - return 0; - - Guard.Assert(this.nBestSeenHeight >= this.firstRecordedHeight); - - return this.nBestSeenHeight - this.firstRecordedHeight; - } - - private int HistoricalBlockSpan() - { - if (this.historicalFirst == 0) - return 0; - - Guard.Assert(this.historicalBest >= this.historicalFirst); - - if (this.nBestSeenHeight - this.historicalBest > OldestEstimateHistory) - return 0; - - return this.historicalBest - this.historicalFirst; - } - - private int MaxUsableEstimate() - { - // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate - return Math.Min(this.longStats.GetMaxConfirms(), Math.Max(BlockSpan(), HistoricalBlockSpan()) / 2); - } - - /// - /// Return a fee estimate at the required successThreshold from the shortest - /// time horizon which tracks confirmations up to the desired target.If - /// checkShorterHorizon is requested, also allow short time horizon estimates - /// for a lower target to reduce the given answer - /// - private double EstimateCombinedFee(int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult result) - { - double estimate = -1; - - if (confTarget >= 1 && confTarget <= this.longStats.GetMaxConfirms()) - { - // Find estimate from shortest time horizon possible - if (confTarget <= this.shortStats.GetMaxConfirms()) - { - // short horizon - estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, result); - } - else if (confTarget <= this.feeStats.GetMaxConfirms()) - { - // medium horizon - estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); - } - else - { - // long horizon - estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); - } - - if (checkShorterHorizon) - { - var tempResult = new EstimationResult(); - - // If a lower confTarget from a more recent horizon returns a lower answer use it. - if (confTarget > this.feeStats.GetMaxConfirms()) - { - double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); - - if (medMax > 0 && (estimate == -1 || medMax < estimate)) - { - estimate = medMax; - - if (result != null) - result = tempResult; - } - } - - if (confTarget > this.shortStats.GetMaxConfirms()) - { - double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); - - if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) - { - estimate = shortMax; - - if (result != null) - result = tempResult; - } - } - } - } - - return estimate; - } - - /// - /// Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met - /// at 2 * target for any longer time horizons. - /// - private double EstimateConservativeFee(int doubleTarget, EstimationResult result) - { - double estimate = -1; - var tempResult = new EstimationResult(); - - if (doubleTarget <= this.shortStats.GetMaxConfirms()) - estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, result); - - if (doubleTarget <= this.feeStats.GetMaxConfirms()) - { - double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); - - if (longEstimate > estimate) - { - estimate = longEstimate; - - if (result != null) - result = tempResult; - } - } - - return estimate; - } - /// public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) { @@ -638,7 +413,7 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con if (feeCalc != null) { feeCalc.Estimation = tempResult; - feeCalc.Reason = FeeReason.Coservative; + feeCalc.Reason = FeeReason.Conservative; } } } @@ -647,6 +422,50 @@ public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool con } } + /// + public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) + { + TxConfirmStats stats; + double sufficientTxs = SufficientFeeTxs; + switch (horizon) + { + case FeeEstimateHorizon.ShortHalfLife: + { + stats = this.shortStats; + sufficientTxs = SufficientTxsShort; + break; + } + case FeeEstimateHorizon.MedHalfLife: + { + stats = this.feeStats; + break; + } + case FeeEstimateHorizon.LongHalfLife: + { + stats = this.longStats; + break; + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + + lock (this.lockObject) + { + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > stats.GetMaxConfirms()) + return new FeeRate(0); + + if (successThreshold > 1) + return new FeeRate(0); + + double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, this.nBestSeenHeight, result); + + return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); + } + } + /// public void Write() { @@ -738,5 +557,184 @@ public void FlushUnconfirmed() this.logger.LogInformation($"Recorded {numEntries} unconfirmed txs from mempool"); } } + + /// + public int HighestTargetTracked(FeeEstimateHorizon horizon) + { + switch (horizon) + { + case FeeEstimateHorizon.ShortHalfLife: + { + return this.shortStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.MedHalfLife: + { + return this.feeStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.LongHalfLife: + { + return this.longStats.GetMaxConfirms(); + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + } + + /// + /// Process a transaction confirmed in a block. + /// + /// Height of the block. + /// The memory pool entry. + /// Whether it was able to successfully process the transaction. + private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) + { + if (!this.RemoveTx(entry.TransactionHash, true)) + return false; + + // How many blocks did it take for miners to include this transaction? + // blocksToConfirm is 1-based, so a transaction included in the earliest + // possible block has confirmation count of 1 + int blocksToConfirm = nBlockHeight - entry.EntryHeight; + + if (blocksToConfirm <= 0) + { + // This can't happen because we don't process transactions from a block with a height lower than our greatest seen height. + this.logger.LogDebug($"Blockpolicy error Transaction had negative blocksToConfirm"); + return false; + } + + // Feerates are stored and reported as BTC-per-kb: + var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.shortStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.longStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + + return true; + } + + private int BlockSpan() + { + if (this.firstRecordedHeight == 0) + return 0; + + Guard.Assert(this.nBestSeenHeight >= this.firstRecordedHeight); + + return this.nBestSeenHeight - this.firstRecordedHeight; + } + + private int HistoricalBlockSpan() + { + if (this.historicalFirst == 0) + return 0; + + Guard.Assert(this.historicalBest >= this.historicalFirst); + + if (this.nBestSeenHeight - this.historicalBest > OldestEstimateHistory) + return 0; + + return this.historicalBest - this.historicalFirst; + } + + private int MaxUsableEstimate() + { + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return Math.Min(this.longStats.GetMaxConfirms(), Math.Max(BlockSpan(), HistoricalBlockSpan()) / 2); + } + + /// + /// Return a fee estimate at the required successThreshold from the shortest + /// time horizon which tracks confirmations up to the desired target.If + /// checkShorterHorizon is requested, also allow short time horizon estimates + /// for a lower target to reduce the given answer + /// + private double EstimateCombinedFee(int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult result) + { + double estimate = -1; + + if (confTarget >= 1 && confTarget <= this.longStats.GetMaxConfirms()) + { + // Find estimate from shortest time horizon possible + if (confTarget <= this.shortStats.GetMaxConfirms()) + { + // short horizon + estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, result); + } + else if (confTarget <= this.feeStats.GetMaxConfirms()) + { + // medium horizon + estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); + } + else + { + // long horizon + estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); + } + + if (checkShorterHorizon) + { + var tempResult = new EstimationResult(); + + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > this.feeStats.GetMaxConfirms()) + { + double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); + + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + { + estimate = medMax; + + if (result != null) + result = tempResult; + } + } + + if (confTarget > this.shortStats.GetMaxConfirms()) + { + double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); + + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + { + estimate = shortMax; + + if (result != null) + result = tempResult; + } + } + } + } + + return estimate; + } + + /// + /// Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + /// at 2 * target for any longer time horizons. + /// + private double EstimateConservativeFee(int doubleTarget, EstimationResult result) + { + double estimate = -1; + var tempResult = new EstimationResult(); + + if (doubleTarget <= this.shortStats.GetMaxConfirms()) + estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, result); + + if (doubleTarget <= this.feeStats.GetMaxConfirms()) + { + double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); + + if (longEstimate > estimate) + { + estimate = longEstimate; + + if (result != null) + result = tempResult; + } + } + + return estimate; + } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs index c53feef1a3d..a274466b323 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee +namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// /// Enumeration of reason for returned fee estimate @@ -13,7 +9,7 @@ public enum FeeReason HalfEstimate, FullEstimate, DoubleEstimate, - Coservative, + Conservative, MemPoolMin, PayTxFee, Fallback, diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs index fe43ee2f74d..37bf3dd1771 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs @@ -6,10 +6,18 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee public interface IBlockPolicyEstimator { /// - /// Return a feerate estimate (deprecated, per Bitcoin Core source). + /// Process all the transactions that have been included in a block. /// - /// The desired number of confirmations to be included in a block. - FeeRate EstimateFee(int confTarget); + /// The block height for the block. + /// Collection of memory pool entries. + void ProcessBlock(int nBlockHeight, List entries); + + /// + /// Process a transaction accepted to the mempool. + /// + /// Memory pool entry. + /// Whether to update fee estimate. + void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate); /// /// Remove a transaction from the mempool tracking stats. @@ -26,11 +34,10 @@ public interface IBlockPolicyEstimator bool RemoveTx(uint256 hash, bool inBlock); /// - /// Return a specific fee estimate calculation with a given success threshold and time horizon, - /// and optionally return detailed data about calculation. + /// Return a feerate estimate (deprecated, per Bitcoin Core source). /// - /// The desired number of confirmations to be included in a block - FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result); + /// The desired number of confirmations to be included in a block. + FeeRate EstimateFee(int confTarget); /// /// Returns the max of the feerates calculated with a 60% @@ -43,24 +50,11 @@ public interface IBlockPolicyEstimator FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative); /// - /// Process all the transactions that have been included in a block. - /// - /// The block height for the block. - /// Collection of memory pool entries. - void ProcessBlock(int nBlockHeight, List entries); - - /// - /// Process a transaction accepted to the mempool. - /// - /// Memory pool entry. - /// Whether to update fee estimate. - void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate); - - /// - /// Calculation of highest target that estimates are tracked for. + /// Return a specific fee estimate calculation with a given success threshold and time horizon, + /// and optionally return detailed data about calculation. /// - /// - int HighestTargetTracked(FeeEstimateHorizon horizon); + /// The desired number of confirmations to be included in a block + FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result); /// /// Write estimation data to a file. @@ -74,6 +68,12 @@ public interface IBlockPolicyEstimator /// Version number of the file. bool Read(); + /// + /// Calculation of highest target that estimates are tracked for. + /// + /// + int HighestTargetTracked(FeeEstimateHorizon horizon); + /// /// Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool. /// diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index e679a7ed30f..7d60adbb488 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -40,7 +40,7 @@ public class TxConfirmStats /// /// For each bucket X: /// Count the total # of txs in each bucket. - /// Track the historical moving average of this total over blocks + /// Track the historical moving average of this total over blocks. /// private List txCtAvg; @@ -49,7 +49,7 @@ public class TxConfirmStats /// /// /// Count the total # of txs confirmed within Y blocks in each bucket. - /// Track the historical moving average of theses totals over blocks. + /// Track the historical moving average of these totals over blocks. /// private List> confAvg; @@ -58,7 +58,7 @@ public class TxConfirmStats /// /// /// Track moving avg of txs which have been evicted from the mempool - /// after failing to be confirmed within Y blocks + /// after failing to be confirmed within Y blocks. /// private List> failAvg; @@ -86,7 +86,7 @@ public class TxConfirmStats /// /// /// For each bucket X, track the number of transactions in the mempool - /// that are unconfirmed for each possible confirmation value Y + /// that are unconfirmed for each possible confirmation value Y. /// unconfTxs[Y][X] /// private List> unconfTxs; @@ -104,8 +104,7 @@ public TxConfirmStats(ILogger logger) } /// - /// Initialize the data structures. This is called by BlockPolicyEstimator's - /// constructor with default values. + /// Initialize the data structures. This is called by BlockPolicyEstimator's constructor with default values. /// /// Contains the upper limits for the bucket boundaries. /// Max number of periods to track. @@ -176,68 +175,6 @@ public void Record(int blocksToConfirm, double val) this.avg[bucketindex] += val; } - /// - /// Record a new transaction entering the mempool. - /// - /// The block height. - /// - /// The feerate of the transaction. - public int NewTx(int nBlockHeight, double val) - { - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; - int blockIndex = nBlockHeight % this.unconfTxs.Count; - this.unconfTxs[blockIndex][bucketindex]++; - - return bucketindex; - } - - /// - /// Remove a transaction from mempool tracking stats. - /// - /// The height of the mempool entry. - /// The best sceen height. - /// The bucket index. - public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool inBlock) - { - //nBestSeenHeight is not updated yet for the new block - int blocksAgo = nBestSeenHeight - entryHeight; - - if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet - blocksAgo = 0; - - if (blocksAgo < 0) - { - this.logger.LogInformation($"Blockpolicy error, blocks ago is negative for mempool tx"); - return; //This can't happen because we call this with our best seen height, no entries can have higher - } - - if (blocksAgo >= this.unconfTxs.Count) - { - if (this.oldUnconfTxs[bucketIndex] > 0) - this.oldUnconfTxs[bucketIndex]--; - else - this.logger.LogInformation($"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); - } - else - { - int blockIndex = entryHeight % this.unconfTxs.Count; - - if (this.unconfTxs[blockIndex][bucketIndex] > 0) - this.unconfTxs[blockIndex][bucketIndex]--; - else - this.logger.LogInformation($"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); - } - - if(!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period - { - Guard.Assert(this.scale != 0); - int periodsAgo = blocksAgo / this.scale; - - for (int i = 0; i < periodsAgo && i < this.failAvg.Count; i++) - this.failAvg[i][bucketIndex]++; - } - } - /// /// Update our estimates by decaying our historical moving average and updating /// with the data gathered from the current block. @@ -267,7 +204,6 @@ public void UpdateMovingAverages() /// The success probability we require. /// Return the lowest feerate such that all higher values pass minSuccess OR return the highest feerate such that all lower values fail minSuccess. /// The current block height. - /// public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, int nBlockHeight, EstimationResult result) { // Counters for a bucket (or range of buckets) @@ -300,8 +236,8 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s int bins = this.unconfTxs.Count; bool newBucketRange = true; bool passing = true; - EstimatorBucket passBucket = new EstimatorBucket(); ; - EstimatorBucket failBucket = new EstimatorBucket(); + var passBucket = new EstimatorBucket(); + var failBucket = new EstimatorBucket(); // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) @@ -381,9 +317,7 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s int maxBucket = Math.Max(bestNearBucket, bestFarBucket); for (int j = minBucket; j <= maxBucket; j++) - { txSum += this.txCtAvg[j]; - } if (foundAnswer && txSum != 0) { @@ -470,24 +404,24 @@ public TxConfirmData Write() } /// - /// Read saved state of estimation data from a file and replace all internal data structures and - /// variables with this state. + /// Read saved state of estimation data from a file and replace all internal data structures and variables with this state. /// - /// Stream to read from. public void Read(TxConfirmData data) { + // Read data file and do some very basic sanity checking. + // We presume that Initialize has been called prior to this, so that this.buckets is set. try { if (data == null) throw new ArgumentNullException(nameof(data)); - if(data.Decay <= 0 || data.Decay >= 1) + if (data.Decay <= 0 || data.Decay >= 1) throw new ApplicationException("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); - if(data.Scale == 0) + if (data.Scale == 0) throw new ApplicationException("Corrupt estimates file. Scale must be non-zero"); - if(data.Avg.Count != this.buckets.Count) + if (data.Avg.Count != this.buckets.Count) throw new ApplicationException("Corrupt estimates file. Mismatch in feerate average bucket count"); if (data.TxCtAvg.Count != this.buckets.Count) @@ -497,7 +431,7 @@ public void Read(TxConfirmData data) double maxConfirms = data.Scale * maxPeriods; if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) - throw new ApplicationException("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + throw new ApplicationException("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); for (int i = 0; i < maxPeriods; i++) { @@ -526,5 +460,67 @@ public void Read(TxConfirmData data) this.logger.LogError("Error while reading tx confirm data from file", e); } } + + /// + /// Record a new transaction entering the mempool. + /// + /// The block height. + /// + /// The feerate of the transaction. + public int NewTx(int nBlockHeight, double val) + { + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; + int blockIndex = nBlockHeight % this.unconfTxs.Count; + this.unconfTxs[blockIndex][bucketindex]++; + + return bucketindex; + } + + /// + /// Remove a transaction from mempool tracking stats. + /// + /// The height of the mempool entry. + /// The best sceen height. + /// The bucket index. + public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool inBlock) + { + //nBestSeenHeight is not updated yet for the new block + int blocksAgo = nBestSeenHeight - entryHeight; + + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet + blocksAgo = 0; + + if (blocksAgo < 0) + { + this.logger.LogDebug($"Blockpolicy error, blocks ago is negative for mempool tx"); + return; //This can't happen because we call this with our best seen height, no entries can have higher + } + + if (blocksAgo >= this.unconfTxs.Count) + { + if (this.oldUnconfTxs[bucketIndex] > 0) + this.oldUnconfTxs[bucketIndex]--; + else + this.logger.LogDebug($"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); + } + else + { + int blockIndex = entryHeight % this.unconfTxs.Count; + + if (this.unconfTxs[blockIndex][bucketIndex] > 0) + this.unconfTxs[blockIndex][bucketIndex]--; + else + this.logger.LogDebug($"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); + } + + if (!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period + { + Guard.Assert(this.scale != 0); + int periodsAgo = blocksAgo / this.scale; + + for (int i = 0; i < periodsAgo && i < this.failAvg.Count; i++) + this.failAvg[i][bucketIndex]++; + } + } } } diff --git a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs index 68e4338d7b0..e0860ca55e0 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs @@ -205,7 +205,7 @@ public async Task InitializeAsync() }; this.DateTimeProvider = dateTimeProviderSet; - this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); + this.mempool = new TxMempool(dateTimeProvider, new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); this.mempoolLock = new MempoolSchedulerLock(); // We can't make transactions until we have inputs diff --git a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs index 40dab1a1ee1..02984021065 100644 --- a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs +++ b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs @@ -33,7 +33,7 @@ public class SignedMultisigTransactionBroadcasterTests : IDisposable private readonly IDateTimeProvider dateTimeProvider; private readonly MempoolSettings mempoolSettings; private readonly NodeSettings nodeSettings; - private readonly BlockPolicyEstimator blockPolicyEstimator; + private readonly BitcoinBlockPolicyEstimator blockPolicyEstimator; private readonly TxMempool txMempool; private readonly IMempoolValidator mempoolValidator; private readonly IMempoolPersistence mempoolPersistence; @@ -68,7 +68,7 @@ public SignedMultisigTransactionBroadcasterTests() MempoolExpiry = MempoolValidator.DefaultMempoolExpiry }; - this.blockPolicyEstimator = new BlockPolicyEstimator(this.loggerFactory, this.nodeSettings); + this.blockPolicyEstimator = new BitcoinBlockPolicyEstimator(this.loggerFactory, this.nodeSettings); this.txMempool = new TxMempool(this.dateTimeProvider, this.blockPolicyEstimator, this.loggerFactory, this.nodeSettings); diff --git a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs index 85437dd1369..c441ba8505d 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs @@ -269,7 +269,7 @@ public async Task InitializeAsync([CallerMemberName] string callingMethod = "") timeutc = DateTimeProvider.Default.GetUtcNow() }; - this.mempool = new TxMempool(dateTimeProviderSet, new BlockPolicyEstimator(this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); + this.mempool = new TxMempool(dateTimeProviderSet, new BitcoinBlockPolicyEstimator(this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); this.mempoolLock = new MempoolSchedulerLock(); var blocks = new List(); From 5d0fedcc9b32271e08ebf4d8f18c8097b8ffd7da Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Tue, 29 Oct 2019 19:49:34 +0200 Subject: [PATCH 14/18] Comments --- .../Fee/BitcoinBlockPolicyEstimator.cs | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs index 517aa516311..322551a4951 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs @@ -121,7 +121,6 @@ public class BitcoinBlockPolicyEstimator : IBlockPolicyEstimator private const string FileName = "fee.json"; - // TODO: How is this used? Should probably use ChainIndexer instead /// Best seen block height. private int nBestSeenHeight; @@ -157,9 +156,12 @@ public class BitcoinBlockPolicyEstimator : IBlockPolicyEstimator /// Map of bucket upper-bound to index into all vectors by bucket. private SortedDictionary bucketMap; - private object lockObject; + /// + /// Locks access to + /// + private readonly object lockObject; - private FileStorage fileStorage; + private readonly FileStorage fileStorage; /// /// Constructs an instance of the block policy estimator object. @@ -269,9 +271,15 @@ public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) } if (txHeight != this.nBestSeenHeight) + { + // Ignore side chains and re-orgs; assuming they are random they don't + // affect the estimate. We'll potentially double count transactions in 1-block reorgs. + // Ignore txs if BlockPolicyEstimator is not in sync with the chain tip. + // It will be synced next time a block is processed. return; + } - // Only want to be updating estimates when our blockchain is synced, otherwise we'll miscalculate how many blocks its taking to get included. + // Only want to be updating estimates when our blockchain is synced, otherwise we'll miscalculate how many blocks it's taking to get included. if (!validFeeEstimate) { this.untrackedTxs++; @@ -554,7 +562,7 @@ public void FlushUnconfirmed() RemoveTx(mi.Key, false); // this calls erase() on mapMemPoolTxs } - this.logger.LogInformation($"Recorded {numEntries} unconfirmed txs from mempool"); + this.logger.LogDebug($"Recorded {numEntries} unconfirmed txs from mempool"); } } @@ -591,7 +599,10 @@ public int HighestTargetTracked(FeeEstimateHorizon horizon) private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) { if (!this.RemoveTx(entry.TransactionHash, true)) + { + // This transaction wasn't being tracked for fee estimation return false; + } // How many blocks did it take for miners to include this transaction? // blocksToConfirm is 1-based, so a transaction included in the earliest From ec1c744464667e34763d3ae7eeb6c1a91f0c00d4 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Tue, 29 Oct 2019 20:54:20 +0200 Subject: [PATCH 15/18] Remove unnecessary algorithm --- .../Fee/Algorithm014/BlockPolicyEstimator.cs | 405 ------------------ .../Fee/Algorithm014/TxConfirmStats.cs | 358 ---------------- 2 files changed, 763 deletions(-) delete mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs delete mode 100644 src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs deleted file mode 100644 index b23d4254390..00000000000 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/BlockPolicyEstimator.cs +++ /dev/null @@ -1,405 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Features.MemoryPool.Interfaces; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee.Algorithm014 -{ - /// - /// The BlockPolicyEstimator is used for estimating the feerate needed - /// for a transaction to be included in a block within a certain number of - /// blocks. - /// - /// - /// At a high level the algorithm works by grouping transactions into buckets - /// based on having similar feerates and then tracking how long it - /// takes transactions in the various buckets to be mined. It operates under - /// the assumption that in general transactions of higher feerate will be - /// included in blocks before transactions of lower feerate. So for - /// example if you wanted to know what feerate you should put on a transaction to - /// be included in a block within the next 5 blocks, you would start by looking - /// at the bucket with the highest feerate transactions and verifying that a - /// sufficiently high percentage of them were confirmed within 5 blocks and - /// then you would look at the next highest feerate bucket, and so on, stopping at - /// the last bucket to pass the test. The average feerate of transactions in this - /// bucket will give you an indication of the lowest feerate you can put on a - /// transaction and still have a sufficiently high chance of being confirmed - /// within your desired 5 blocks. - /// - /// Here is a brief description of the implementation: - /// When a transaction enters the mempool, we - /// track the height of the block chain at entry. Whenever a block comes in, - /// we count the number of transactions in each bucket and the total amount of feerate - /// paid in each bucket. Then we calculate how many blocks Y it took each - /// transaction to be mined and we track an array of counters in each bucket - /// for how long it to took transactions to get confirmed from 1 to a max of 25 - /// and we increment all the counters from Y up to 25. This is because for any - /// number Z>=Y the transaction was successfully mined within Z blocks. We - /// want to save a history of this information, so at any time we have a - /// counter of the total number of transactions that happened in a given feerate - /// bucket and the total number that were confirmed in each number 1-25 blocks - /// or less for any bucket. We save this history by keeping an exponentially - /// decaying moving average of each one of these stats. Furthermore we also - /// keep track of the number unmined (in mempool) transactions in each bucket - /// and for how many blocks they have been outstanding and use that to increase - /// the number of transactions we've seen in that feerate bucket when calculating - /// an estimate for any number of confirmations below the number of blocks - /// they've been outstanding. - /// - /// We will instantiate an instance of this class to track transactions that were - /// included in a block. We will lump transactions into a bucket according to their - /// approximate feerate and then track how long it took for those txs to be included in a block - /// - /// The tracking of unconfirmed (mempool) transactions is completely independent of the - /// historical tracking of transactions that have been confirmed in a block. - /// - /// We want to be able to estimate feerates that are needed on tx's to be included in - /// a certain number of blocks.Every time a block is added to the best chain, this class records - /// stats on the transactions included in that block - /// - public class BlockPolicyEstimator - { - /// Require an avg of 1 tx in the combined feerate bucket per block to have stat significance. - private const double SufficientFeeTxs = 1; - - /// Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough. - private const double MinSuccessPct = .95; - - /// Minimum value for tracking feerates. - private const long MinFeeRate = 10; - - /// Maximum value for tracking feerates. - private const double MaxFeeRate = 1e7; - - /// - /// Spacing of FeeRate buckets. - /// - /// - /// We have to lump transactions into buckets based on feerate, but we want to be able - /// to give accurate estimates over a large range of potential feerates. - /// Therefore it makes sense to exponentially space the buckets. - /// - private const double FeeSpacing = 1.1; - - /// Track confirm delays up to 25 blocks, can't estimate beyond that. - private const int MaxBlockConfirms = 25; - - /// Decay of .998 is a half-life of 346 blocks or about 2.4 days. - private const double DefaultDecay = .998; - - /// Value for infinite priority. - public const double InfPriority = 1e9 * 21000000ul * Money.COIN; - - /// Maximum money value. - private static readonly Money MaxMoney = new Money(21000000 * Money.COIN); - - /// Value for infinite fee rate. - private static readonly double InfFeeRate = MaxMoney.Satoshi; - - /// Classes to track historical data on transaction confirmations. - private readonly TxConfirmStats feeStats; - - /// Map of txids to information about that transaction. - private readonly Dictionary mapMemPoolTxs; - - /// Minimum tracked Fee. Passed to constructor to avoid dependency on main./// - private readonly FeeRate minTrackedFee; - - /// Best seen block height. - private int nBestSeenHeight; - - /// Setting for the node. - private readonly MempoolSettings mempoolSettings; - - /// Logger for logging on this object. - private readonly ILogger logger; - - /// Count of tracked transactions. - private int trackedTxs; - - /// Count of untracked transactions. - private int untrackedTxs; - - /// - /// Constructs an instance of the block policy estimator object. - /// - /// Mempool settings. - /// Factory for creating loggers. - /// Full node settings. - public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory loggerFactory, NodeSettings nodeSettings) - { - this.mapMemPoolTxs = new Dictionary(); - this.mempoolSettings = mempoolSettings; - this.nBestSeenHeight = 0; - this.trackedTxs = 0; - this.untrackedTxs = 0; - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - - this.minTrackedFee = nodeSettings.MinRelayTxFeeRate < new FeeRate(new Money(MinFeeRate)) - ? new FeeRate(new Money(MinFeeRate)) - : nodeSettings.MinRelayTxFeeRate; - var vfeelist = new List(); - for (double bucketBoundary = this.minTrackedFee.FeePerK.Satoshi; - bucketBoundary <= MaxFeeRate; - bucketBoundary *= FeeSpacing) - vfeelist.Add(bucketBoundary); - vfeelist.Add(InfFeeRate); - this.feeStats = new TxConfirmStats(this.logger); - this.feeStats.Initialize(vfeelist, MaxBlockConfirms, DefaultDecay); - } - - /// - /// Process all the transactions that have been included in a block. - /// - /// The block height for the block. - /// Collection of memory pool entries. - public void ProcessBlock(int nBlockHeight, List entries) - { - if (nBlockHeight <= this.nBestSeenHeight) - return; - - // Must update nBestSeenHeight in sync with ClearCurrent so that - // calls to removeTx (via processBlockTx) correctly calculate age - // of unconfirmed txs to remove from tracking. - this.nBestSeenHeight = nBlockHeight; - - // Clear the current block state and update unconfirmed circular buffer - this.feeStats.ClearCurrent(nBlockHeight); - - int countedTxs = 0; - // Repopulate the current block states - for (int i = 0; i < entries.Count; i++) - if (this.ProcessBlockTx(nBlockHeight, entries[i])) - countedTxs++; - - // Update all exponential averages with the current block state - this.feeStats.UpdateMovingAverages(); - - // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) - // Logging.Logs.EstimateFee.LogInformation( - // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); - - this.trackedTxs = 0; - this.untrackedTxs = 0; - } - - /// - /// Process a transaction confirmed in a block. - /// - /// Height of the block. - /// The memory pool entry. - /// Whether it was able to successfully process the transaction. - private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) - { - if (!this.RemoveTx(entry.TransactionHash)) - return false; - - // How many blocks did it take for miners to include this transaction? - // blocksToConfirm is 1-based, so a transaction included in the earliest - // possible block has confirmation count of 1 - int blocksToConfirm = nBlockHeight - entry.EntryHeight; - if (blocksToConfirm <= 0) - { - // This can't happen because we don't process transactions from a block with a height - // lower than our greatest seen height - this.logger.LogInformation($"Blockpolicy error Transaction had negative blocksToConfirm"); - return false; - } - - // Feerates are stored and reported as BTC-per-kb: - FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); - return true; - } - - /// - /// Process a transaction accepted to the mempool. - /// - /// Memory pool entry. - /// Whether to update fee estimate. - public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) - { - int txHeight = entry.EntryHeight; - uint256 hash = entry.TransactionHash; - if (this.mapMemPoolTxs.ContainsKey(hash)) - { - this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); - return; - } - - if (txHeight != this.nBestSeenHeight) - return; - - // Only want to be updating estimates when our blockchain is synced, - // otherwise we'll miscalculate how many blocks its taking to get included. - if (!validFeeEstimate) - { - this.untrackedTxs++; - return; - } - this.trackedTxs++; - - // Feerates are stored and reported as BTC-per-kb: - FeeRate feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); - this.mapMemPoolTxs[hash].blockHeight = txHeight; - this.mapMemPoolTxs[hash].bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); - } - - /// - /// Remove a transaction from the mempool tracking stats. - /// - /// Transaction hash. - /// Whether the transaction was successfully removed. - /// - /// This function is called from TxMemPool.RemoveUnchecked to ensure - /// txs removed from the mempool for any reason are no longer - /// tracked. Txs that were part of a block have already been removed in - /// ProcessBlockTx to ensure they are never double tracked, but it is - /// of no harm to try to remove them again. - /// - public bool RemoveTx(uint256 hash) - { - TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); - if (pos != null) - { - this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex); - this.mapMemPoolTxs.Remove(hash); - return true; - } - return false; - } - - /// - /// Return a feerate estimate - /// - /// The desired number of confirmations to be included in a block. - public FeeRate EstimateFee(int confTarget) - { - // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || confTarget > this.feeStats.GetMaxConfirms()) - return new FeeRate(0); - - double median = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); - - if (median < 0) - return new FeeRate(0); - - return new FeeRate(new Money((int)median)); - } - - /// - /// Estimate feerate needed to be included in a block within - /// confTarget blocks. If no answer can be given at confTarget, return an - /// estimate at the lowest target where one can be given. - /// - public FeeRate EstimateSmartFee(int confTarget, ITxMempool pool, out int answerFoundAtTarget) - { - answerFoundAtTarget = confTarget; - - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || confTarget > this.feeStats.GetMaxConfirms()) - return new FeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; - - double median = -1; - while (median < 0 && confTarget <= this.feeStats.GetMaxConfirms()) - median = this.feeStats.EstimateMedianVal(confTarget++, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); - - answerFoundAtTarget = confTarget - 1; - - // If mempool is limiting txs , return at least the min feerate from the mempool - if (pool != null) - { - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0 && minPoolFee.Satoshi > median) - return new FeeRate(minPoolFee); - } - - if (median < 0) - return new FeeRate(0); - - return new FeeRate((int)median); - } - - /// - /// Write estimation data to a file. - /// - /// Stream to write to. - /// TODO: Implement write estimation - public void Write(Stream fileout) - { - } - - /// - /// Read estimation data from a file. - /// - /// Stream to read data from. - /// Version number of the file. - /// TODO: Implement read estimation - public void Read(Stream filein, int nFileVersion) - { - } - - /// - /// Return an estimate of the priority. - /// - /// The desired number of confirmations to be included in a block. - /// Estimate of the priority. - /// TODO: Implement priority estimation - public double EstimatePriority(int confTarget) - { - return -1; - } - - /// - /// Return an estimated smart priority. - /// - /// The desired number of confirmations to be included in a block. - /// Memory pool transactions. - /// Block height where answer was found. - /// The smart priority. - public double EstimateSmartPriority(int confTarget, ITxMempool pool, out int answerFoundAtTarget) - { - answerFoundAtTarget = confTarget; - - // If mempool is limiting txs, no priority txs are allowed - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0) - return InfPriority; - - return -1; - } - - /// - /// Transaction statistics information. - /// - public class TxStatsInfo - { - /// The block height. - public int blockHeight; - - /// The index into the confirmed transactions bucket map. - public int bucketIndex; - - /// - /// Constructs a instance of a transaction stats info object. - /// - public TxStatsInfo() - { - this.blockHeight = 0; - this.bucketIndex = 0; - } - } - } -} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs deleted file mode 100644 index 375f123612c..00000000000 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/Algorithm014/TxConfirmStats.cs +++ /dev/null @@ -1,358 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Extensions.Logging; -using NBitcoin; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee.Algorithm014 -{ - /// - /// Transation confirmation statistics. - /// - public class TxConfirmStats - { - /// Instance logger for logging messages. - private readonly ILogger logger; - - /// - /// Moving average of total fee rate of all transactions in each bucket. - /// - /// - /// Track the historical moving average of this total over blocks. - /// - private List avg; - - /// Map of bucket upper-bound to index into all vectors by bucket. - private Dictionary bucketMap; - - //Define the buckets we will group transactions into. - - /// The upper-bound of the range for the bucket (inclusive). - private List buckets; - - // Count the total # of txs confirmed within Y blocks in each bucket. - // Track the historical moving average of theses totals over blocks. - - /// Confirmation average. confAvg[Y][X]. - private List> confAvg; - - /// Current block confirmations. curBlockConf[Y][X]. - private List> curBlockConf; - - /// Current block transaction count. - private List curBlockTxCt; - - /// Current block fee rate. - private List curBlockVal; - - // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X - // Combine the total value with the tx counts to calculate the avg feerate per bucket - - /// Decay value to use. - private double decay; - - /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket - private List oldUnconfTxs; - - /// - /// Historical moving average of transaction counts. - /// - /// - /// For each bucket X: - /// Count the total # of txs in each bucket. - /// Track the historical moving average of this total over blocks - /// - private List txCtAvg; - - /// - /// Mempool counts of outstanding transactions. - /// - /// - /// For each bucket X, track the number of transactions in the mempool - /// that are unconfirmed for each possible confirmation value Y - /// unconfTxs[Y][X] - /// - private List> unconfTxs; - - /// - /// Constructs an instance of the transaction confirmation stats object. - /// - /// Instance logger to use for message logging. - public TxConfirmStats(ILogger logger) - { - this.logger = logger; - } - - /// - /// Initialize the data structures. This is called by BlockPolicyEstimator's - /// constructor with default values. - /// - /// Contains the upper limits for the bucket boundaries. - /// Max number of confirms to track. - /// How much to decay the historical moving average per block. - public void Initialize(List defaultBuckets, int maxConfirms, double decay) - { - this.buckets = new List(); - this.bucketMap = new Dictionary(); - - this.decay = decay; - for (int i = 0; i < defaultBuckets.Count; i++) - { - this.buckets.Add(defaultBuckets[i]); - this.bucketMap[defaultBuckets[i]] = i; - } - this.confAvg = new List>(); - this.curBlockConf = new List>(); - this.unconfTxs = new List>(); - - for (int i = 0; i < maxConfirms; i++) - { - this.confAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); - this.curBlockConf.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); - this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); - } - - this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); - this.curBlockTxCt = new List(Enumerable.Repeat(default(int), this.buckets.Count)); - this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - this.curBlockVal = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - } - - /// - /// Clear the state of the curBlock variables to start counting for the new block. - /// - /// Block height. - public void ClearCurrent(int nBlockHeight) - { - for (var j = 0; j < this.buckets.Count; j++) - { - this.oldUnconfTxs[j] += this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j]; - this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j] = 0; - for (int i = 0; i < this.curBlockConf.Count; i++) - this.curBlockConf[i][j] = 0; - this.curBlockTxCt[j] = 0; - this.curBlockVal[j] = 0; - } - } - - /// - /// Record a new transaction data point in the current block stats. - /// - /// The number of blocks it took this transaction to confirm. blocksToConfirm is 1-based and has to be >= 1. - /// The feerate of the transaction. - public void Record(int blocksToConfirm, double val) - { - // blocksToConfirm is 1-based - if (blocksToConfirm < 1) - return; - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; - for (int i = blocksToConfirm; i <= this.curBlockConf.Count; i++) - this.curBlockConf[i - 1][bucketindex]++; - this.curBlockTxCt[bucketindex]++; - this.curBlockVal[bucketindex] += val; - } - - /// - /// Record a new transaction entering the mempool. - /// - /// The block height. - /// - /// The feerate of the transaction. - public int NewTx(int nBlockHeight, double val) - { - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; - int blockIndex = nBlockHeight % this.unconfTxs.Count; - this.unconfTxs[blockIndex][bucketindex]++; - return bucketindex; - } - - /// - /// Remove a transaction from mempool tracking stats. - /// - /// The height of the mempool entry. - /// The best sceen height. - /// The bucket index. - public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex) - { - //nBestSeenHeight is not updated yet for the new block - int blocksAgo = nBestSeenHeight - entryHeight; - if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet - blocksAgo = 0; - if (blocksAgo < 0) - { - this.logger.LogInformation($"Blockpolicy error, blocks ago is negative for mempool tx"); - return; //This can't happen because we call this with our best seen height, no entries can have higher - } - - if (blocksAgo >= this.unconfTxs.Count) - { - if (this.oldUnconfTxs[bucketIndex] > 0) - this.oldUnconfTxs[bucketIndex]--; - else - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); - } - else - { - int blockIndex = entryHeight % this.unconfTxs.Count; - if (this.unconfTxs[blockIndex][bucketIndex] > 0) - this.unconfTxs[blockIndex][bucketIndex]--; - else - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); - } - } - - /// - /// Update our estimates by decaying our historical moving average and updating - /// with the data gathered from the current block. - /// - public void UpdateMovingAverages() - { - for (var j = 0; j < this.buckets.Count; j++) - { - for (var i = 0; i < this.confAvg.Count; i++) - this.confAvg[i][j] = this.confAvg[i][j] * this.decay + this.curBlockConf[i][j]; - this.avg[j] = this.avg[j] * this.decay + this.curBlockVal[j]; - this.txCtAvg[j] = this.txCtAvg[j] * this.decay + this.curBlockTxCt[j]; - } - } - - /// - /// Calculate a feerate estimate. Find the lowest value bucket (or range of buckets - /// to make sure we have enough data points) whose transactions still have sufficient likelihood - /// of being confirmed within the target number of confirmations. - /// - /// Target number of confirmations. - /// Required average number of transactions per block in a bucket range. - /// The success probability we require. - /// Return the lowest feerate such that all higher values pass minSuccess OR return the highest feerate such that all lower values fail minSuccess. - /// The current block height. - /// - public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, - bool requireGreater, - int nBlockHeight) - { - // Counters for a bucket (or range of buckets) - double nConf = 0; // Number of tx's confirmed within the confTarget - double totalNum = 0; // Total number of tx's that were ever confirmed - int extraNum = 0; // Number of tx's still in mempool for confTarget or longer - - int maxbucketindex = this.buckets.Count - 1; - - // requireGreater means we are looking for the lowest feerate such that all higher - // values pass, so we start at maxbucketindex (highest feerate) and look at successively - // smaller buckets until we reach failure. Otherwise, we are looking for the highest - // feerate such that all lower values fail, and we go in the opposite direction. - int startbucket = requireGreater ? maxbucketindex : 0; - int step = requireGreater ? -1 : 1; - - // We'll combine buckets until we have enough samples. - // The near and far variables will define the range we've combined - // The best variables are the last range we saw which still had a high - // enough confirmation rate to count as success. - // The cur variables are the current range we're counting. - int curNearBucket = startbucket; - int bestNearBucket = startbucket; - int curFarBucket = startbucket; - int bestFarBucket = startbucket; - - bool foundAnswer = false; - int bins = this.unconfTxs.Count; - - // Start counting from highest(default) or lowest feerate transactions - for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) - { - curFarBucket = bucket; - nConf += this.confAvg[confTarget - 1][bucket]; - totalNum += this.txCtAvg[bucket]; - for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) - extraNum += this.unconfTxs[(nBlockHeight - confct) % bins][bucket]; - extraNum += this.oldUnconfTxs[bucket]; - // If we have enough transaction data points in this range of buckets, - // we can test for success - // (Only count the confirmed data points, so that each confirmation count - // will be looking at the same amount of data and same bucket breaks) - if (totalNum >= sufficientTxVal / (1 - this.decay)) - { - double curPct = nConf / (totalNum + extraNum); - - // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - - // Otherwise update the cumulative stats, and the bucket variables - // and reset the counters - foundAnswer = true; - nConf = 0; - totalNum = 0; - extraNum = 0; - bestNearBucket = curNearBucket; - bestFarBucket = curFarBucket; - curNearBucket = bucket + step; - } - } - - double median = -1; - double txSum = 0; - - // Calculate the "average" feerate of the best bucket range that met success conditions - // Find the bucket with the median transaction and then report the average feerate from that bucket - // This is a compromise between finding the median which we can't since we don't save all tx's - // and reporting the average which is less accurate - int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; - for (int j = minBucket; j <= maxBucket; j++) - txSum += this.txCtAvg[j]; - if (foundAnswer && txSum != 0) - { - txSum = txSum / 2; - for (int j = minBucket; j <= maxBucket; j++) - if (this.txCtAvg[j] < txSum) - { - txSum -= this.txCtAvg[j]; - } - else - { - // we're in the right bucket - median = this.avg[j] / this.txCtAvg[j]; - break; - } - } - - this.logger.LogInformation( - $"{confTarget}: For conf success {(requireGreater ? $">" : $"<")} {successBreakPoint} need feerate {(requireGreater ? $">" : $"<")}: {median} from buckets {this.buckets[minBucket]} -{this.buckets[maxBucket]} Cur Bucket stats {100 * nConf / (totalNum + extraNum)} {nConf}/({totalNum}+{extraNum} mempool)"); - - return median; - } - - /// - /// Return the max number of confirms we're tracking. - /// - /// The max number of confirms. - public int GetMaxConfirms() - { - return this.confAvg.Count; - } - - /// - /// Write state of estimation data to a file. - /// - /// Stream to write to. - public void Write(BitcoinStream stream) - { - } - - /// - /// Read saved state of estimation data from a file and replace all internal data structures and - /// variables with this state. - /// - /// Stream to read from. - public void Read(Stream filein) - { - } - } -} From 610d40ee45bf1551d636bcd575efdc11c25701e5 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Tue, 29 Oct 2019 21:31:37 +0200 Subject: [PATCH 16/18] Fix DI --- src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs index 25f89f395c4..18a55474d67 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs @@ -158,7 +158,7 @@ public static IFullNodeBuilder UseMempool(this IFullNodeBuilder fullNodeBuilder) { services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton() From d3af270ed0826a814c5e262d8fabaf04b8ec5c75 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Tue, 29 Oct 2019 22:18:27 +0200 Subject: [PATCH 17/18] Remove unnecessary imports --- .../Fee/EstimationResult.cs | 9 ++++----- .../Fee/EstimatorBucket.cs | 11 ++++++----- .../Fee/FeeCalculation.cs | 9 ++++----- .../Fee/FeeEstimateHorizon.cs | 6 +----- .../Fee/SerializationEntity/TxConfirmData.cs | 9 ++++++--- .../Fee/TxConfirmStats.cs | 2 -- 6 files changed, 21 insertions(+), 25 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs index b8e67befe26..fc94589867a 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee +namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// /// Used to return detailed information about a fee estimate calculation @@ -10,8 +6,11 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee public class EstimationResult { public EstimatorBucket Pass { get; set; } + public EstimatorBucket Fail { get; set; } + public double Decay { get; set; } + public int Scale { get; set; } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs index a10541fb275..d56493d6584 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee +namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// /// Used to return detailed information about a feerate bucket @@ -10,10 +6,15 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Fee public class EstimatorBucket { public double Start { get; set; } + public double End { get; set; } + public double WithinTarget { get; set; } + public double TotalConfirmed { get; set; } + public double InMempool { get; set; } + public double LeftMempool { get; set; } public EstimatorBucket() diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs index 9d7e9c86b62..a3d1d4e61d6 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs @@ -1,14 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee +namespace Stratis.Bitcoin.Features.MemoryPool.Fee { public class FeeCalculation { public EstimationResult Estimation { get; set; } + public FeeReason Reason { get; set; } + public int DesiredTarget { get; set; } + public int ReturnedTarget { get; set; } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs index 283b8f73134..7e9956e0cae 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs @@ -1,8 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee +namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// /// Identifier for each of the 3 different TxConfirmStats which will track diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs index 472861cd742..644d7aa8572 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs @@ -1,16 +1,19 @@ -using System; -using System.Collections.Generic; -using System.Text; +using System.Collections.Generic; namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity { public class TxConfirmData { public double Decay { get; set; } + public int Scale { get; set; } + public List Avg { get; set; } + public List TxCtAvg { get; set; } + public List> ConfAvg { get; set; } + public List> FailAvg { get; set; } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 7d60adbb488..bfa89e90d44 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -1,8 +1,6 @@ using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Extensions.Logging; -using NBitcoin; using Stratis.Bitcoin.Utilities; using System; using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; From a8dd61c95fdef71ee00747510900b08b7e2c2731 Mon Sep 17 00:00:00 2001 From: Kevin Loubser Date: Tue, 29 Oct 2019 23:13:00 +0200 Subject: [PATCH 18/18] Test - try attach block estimator to wallet fee policy --- .../WalletFeePolicy.cs | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs index ab56d021956..4fbc8a56816 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs @@ -1,6 +1,7 @@ using System; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Features.MemoryPool.Fee; using Stratis.Bitcoin.Features.Wallet.Interfaces; namespace Stratis.Bitcoin.Features.Wallet @@ -33,17 +34,20 @@ public class WalletFeePolicy : IWalletFeePolicy /// private readonly FeeRate minRelayTxFee; + private readonly IBlockPolicyEstimator blockPolicyEstimator; + /// /// Constructs a wallet fee policy. /// /// Settings for the the node. - public WalletFeePolicy(NodeSettings nodeSettings) + public WalletFeePolicy(NodeSettings nodeSettings, IBlockPolicyEstimator blockPolicyEstimator) { this.minTxFee = nodeSettings.MinTxFeeRate; this.fallbackFee = nodeSettings.FallbackTxFeeRate; this.payTxFee = new FeeRate(0); this.maxTxFee = new Money(0.1M, MoneyUnit.BTC); this.minRelayTxFee = nodeSettings.MinRelayTxFeeRate; + this.blockPolicyEstimator = blockPolicyEstimator; } /// @@ -75,30 +79,33 @@ public Money GetMinimumFee(int txBytes, int confirmTarget) public Money GetMinimumFee(int txBytes, int confirmTarget, Money targetFee) { Money nFeeNeeded = targetFee; + // User didn't set: use -txconfirmtarget to estimate... if (nFeeNeeded == 0) { - int estimateFoundTarget = confirmTarget; - - // TODO: the fee estimation is not ready for release for now use the fall back fee - //nFeeNeeded = this.blockPolicyEstimator.EstimateSmartFee(confirmTarget, this.mempool, out estimateFoundTarget).GetFee(txBytes); + nFeeNeeded = this.blockPolicyEstimator.EstimateFee(confirmTarget).GetFee(txBytes); + // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = this.fallbackFee.GetFee(txBytes); } + // prevent user from paying a fee below minRelayTxFee or minTxFee nFeeNeeded = Math.Max(nFeeNeeded, this.GetRequiredFee(txBytes)); + // But always obey the maximum if (nFeeNeeded > this.maxTxFee) nFeeNeeded = this.maxTxFee; + return nFeeNeeded; } /// public FeeRate GetFeeRate(int confirmTarget) { - //this.blockPolicyEstimator.EstimateSmartFee(confirmTarget, this.mempool, out estimateFoundTarget).GetFee(txBytes); - return this.fallbackFee; + FeeRate feeRate = this.blockPolicyEstimator.EstimateFee(confirmTarget); + + return feeRate.FeePerK > 0 ? feeRate : this.fallbackFee; } } }