diff --git a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs index f0ba2ab0001..f68649125be 100644 --- a/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs +++ b/src/Stratis.Bitcoin.Features.ColdStaking.Tests/ColdStakingControllerTest.cs @@ -138,8 +138,7 @@ private void CreateMempoolManager() { this.mempoolSettings = new MempoolSettings(this.nodeSettings); this.consensusSettings = new ConsensusSettings(this.nodeSettings); - this.txMemPool = new TxMempool(this.dateTimeProvider, new BlockPolicyEstimator( - new MempoolSettings(this.nodeSettings), this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); + this.txMemPool = new TxMempool(this.dateTimeProvider, new BitcoinBlockPolicyEstimator(this.loggerFactory, this.nodeSettings), this.loggerFactory, this.nodeSettings); this.chainIndexer = new ChainIndexer(this.Network); this.nodeDeployments = new NodeDeployments(this.Network, this.chainIndexer); diff --git a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs index 1824abdccde..e6ab4841448 100644 --- a/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.Consensus.Tests/TestChainFactory.cs @@ -181,7 +181,7 @@ public static async Task> MineBlocksAsync(TestChainContext testChain /// private static async Task> MineBlocksAsync(TestChainContext testChainContext, int count, Script receiver, bool mutateLastBlock) { - var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(testChainContext.NodeSettings), testChainContext.LoggerFactory, testChainContext.NodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempool = new TxMempool(testChainContext.DateTimeProvider, blockPolicyEstimator, testChainContext.LoggerFactory, testChainContext.NodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs index 5d6de012162..72fcf7fbb27 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/FeeTests.cs @@ -17,8 +17,7 @@ public void BlockPolicyEstimates() { var dateTimeSet = new DateTimeProviderSet(); NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var mpool = new TxMempool(DateTimeProvider.Default, - new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var mpool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); var entry = new TestMemPoolEntryHelper(); var basefee = new Money(2000); var deltaFee = new Money(100); @@ -51,8 +50,8 @@ public void BlockPolicyEstimates() int answerFound; // Loop through 200 blocks - // At a decay .998 and 4 fee transactions per block - // This makes the tx count about 1.33 per bucket, above the 1 threshold + // At a decay .9952 and 4 fee transactions per block + // This makes the tx count about 2.5 per bucket, well above the 0.1 threshold while (blocknum < 200) { for (int j = 0; j < 10; j++) @@ -82,21 +81,15 @@ public void BlockPolicyEstimates() } mpool.RemoveForBlock(block, ++blocknum); block.Clear(); - if (blocknum == 30) + // Check after just a few txs that combining buckets works as expected + if (blocknum == 3) { - // At this point we should need to combine 5 buckets to get enough data points - // So estimateFee(1,2,3) should fail and estimateFee(4) should return somewhere around - // 8*baserate. estimateFee(4) %'s are 100,100,100,100,90 = average 98% + // At this point we should need to combine 3 buckets to get enough data points + // So estimateFee(1) should fail and estimateFee(2) should return somewhere around + // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(2) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(3) == new FeeRate(0)); - Assert.True(mpool.EstimateFee(4).FeePerK < 8 * baseRate.FeePerK + deltaFee); - Assert.True(mpool.EstimateFee(4).FeePerK > 8 * baseRate.FeePerK - deltaFee); - - Assert.True(mpool.EstimateSmartFee(1, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(3, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(4, out answerFound) == mpool.EstimateFee(4) && answerFound == 4); - Assert.True(mpool.EstimateSmartFee(8, out answerFound) == mpool.EstimateFee(8) && answerFound == 8); + Assert.True(mpool.EstimateFee(2).FeePerK < 9 * baseRate.FeePerK + deltaFee); + Assert.True(mpool.EstimateFee(2).FeePerK > 9 * baseRate.FeePerK - deltaFee); } } @@ -115,15 +108,16 @@ public void BlockPolicyEstimates() Assert.True(origFeeEst[i - 1] <= origFeeEst[i - 2]); } int mult = 11 - i; - if (i > 1) + if (i % 2 == 0) //At scale 2, test logic is only correct for even targets { Assert.True(origFeeEst[i - 1] < mult * baseRate.FeePerK + deltaFee); Assert.True(origFeeEst[i - 1] > mult * baseRate.FeePerK - deltaFee); } - else - { - Assert.True(origFeeEst[i - 1] == new FeeRate(0).FeePerK); - } + } + // Fill out rest of the original estimates + for (int i = 10; i <= 48; i++) + { + origFeeEst.Add(mpool.EstimateFee(i).FeePerK); } // Mine 50 more blocks with no transactions happening, estimates shouldn't change @@ -132,7 +126,7 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, ++blocknum); Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] + deltaFee); Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); @@ -156,10 +150,9 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, ++blocknum); } - for (int i = 1; i < 10; i++) + for (int i = 1; i < 9; i++) { Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK > origFeeEst[answerFound - 1] - deltaFee); } // Mine all those transactions @@ -177,14 +170,15 @@ public void BlockPolicyEstimates() mpool.RemoveForBlock(block, 265); block.Clear(); Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { - Assert.True(mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); + Assert.True(mpool.EstimateFee(i) == new FeeRate(0) || + mpool.EstimateFee(i).FeePerK > origFeeEst[i - 1] - deltaFee); } - // Mine 200 more blocks where everything is mined every block + // Mine 600 more blocks where everything is mined every block // Estimates should be below original estimates - while (blocknum < 465) + while (blocknum < 865) { for (int j = 0; j < 10; j++) { // For each fee multiple @@ -203,23 +197,10 @@ public void BlockPolicyEstimates() block.Clear(); } Assert.True(mpool.EstimateFee(1) == new FeeRate(0)); - for (int i = 2; i < 10; i++) + for (int i = 2; i < 9; i++) { Assert.True(mpool.EstimateFee(i).FeePerK < origFeeEst[i - 1] - deltaFee); - } - - // Test that if the mempool is limited, estimateSmartFee won't return a value below the mempool min fee - // and that estimateSmartPriority returns essentially an infinite value - mpool.AddUnchecked(txf.GetHash(), entry.Fee(feeV[5]).Time(dateTimeSet.GetTime()).Priority(0).Height(blocknum).FromTx(txf, mpool)); - // evict that transaction which should set a mempool min fee of minRelayTxFee + feeV[5] - mpool.TrimToSize(1); - Assert.True(mpool.GetMinFee(1).FeePerK > feeV[5]); - for (int i = 1; i < 10; i++) - { - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.EstimateFee(i).FeePerK); - Assert.True(mpool.EstimateSmartFee(i, out answerFound).FeePerK >= mpool.GetMinFee(1).FeePerK); - Assert.True(mpool.EstimateSmartPriority(i, out answerFound) == BlockPolicyEstimator.InfPriority); - } + } } public class DateTimeProviderSet : DateTimeProvider diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs index 3d9fc4a853f..2bb377c1400 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MemoryPoolTransactionTests.cs @@ -20,6 +20,7 @@ public void MempoolRemoveTest() // Parent transaction with three children, // and three grand-children: + // TODO: Use network factory methods for all Transaction instantiation var txParent = new Transaction(); txParent.AddInput(new TxIn()); @@ -47,7 +48,7 @@ public void MempoolRemoveTest() } NodeSettings settings = NodeSettings.Default(KnownNetworks.TestNet); - var testPool = new TxMempool(DateTimeProvider.Default, new BlockPolicyEstimator(new MempoolSettings(settings), settings.LoggerFactory, settings), settings.LoggerFactory, settings); + var testPool = new TxMempool(DateTimeProvider.Default, new BitcoinBlockPolicyEstimator(settings.LoggerFactory, settings), settings.LoggerFactory, settings); // Nothing in pool, remove should do nothing: long poolSize = testPool.Size; @@ -115,7 +116,7 @@ private void CheckSort(TxMempool pool, List sortedSource, List(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs index 74433c7b664..0c68db908d9 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/MempoolPersistenceTest.cs @@ -282,7 +282,7 @@ private MempoolManager CreateTestMempool(NodeSettings settings, out TxMempool tx NodeSettings nodeSettings = NodeSettings.Default(settings.Network); ILoggerFactory loggerFactory = nodeSettings.LoggerFactory; var consensusSettings = new ConsensusSettings(nodeSettings); - txMemPool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings); + txMemPool = new TxMempool(dateTimeProvider, new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); var coins = new InMemoryCoinView(settings.Network.GenesisHash); var chain = new ChainIndexer(settings.Network); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs index 509f9b41dce..1ec049f9580 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool.Tests/TestChainFactory.cs @@ -108,7 +108,7 @@ public static async Task CreatePosAsync(Network network, Scri await consensus.InitializeAsync(genesis).ConfigureAwait(false); var mempoolSettings = new MempoolSettings(nodeSettings); - var blockPolicyEstimator = new BlockPolicyEstimator(mempoolSettings, loggerFactory, nodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); @@ -188,7 +188,7 @@ public static async Task CreateAsync(Network network, Script chainState.BlockStoreTip = genesis; await consensus.InitializeAsync(genesis).ConfigureAwait(false); - var blockPolicyEstimator = new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings); + var blockPolicyEstimator = new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings); var mempool = new TxMempool(dateTimeProvider, blockPolicyEstimator, loggerFactory, nodeSettings); var mempoolLock = new MempoolSchedulerLock(); diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs new file mode 100644 index 00000000000..322551a4951 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BitcoinBlockPolicyEstimator.cs @@ -0,0 +1,751 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging; +using NBitcoin; +using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; +using Stratis.Bitcoin.Utilities; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// The BlockPolicyEstimator is used for estimating the feerate needed + /// for a transaction to be included in a block within a certain number of blocks. + /// + /// + /// This is based on the fee estimator used in Bitcoin Core 0.15 onwards; it operates + /// differently from the 0.14 version. + /// + /// At a high level the algorithm works by grouping transactions into buckets + /// based on having similar feerates and then tracking how long it + /// takes transactions in the various buckets to be mined. It operates under + /// the assumption that in general transactions of higher feerate will be + /// included in blocks before transactions of lower feerate. So for + /// example if you wanted to know what feerate you should put on a transaction to + /// be included in a block within the next 5 blocks, you would start by looking + /// at the bucket with the highest feerate transactions and verifying that a + /// sufficiently high percentage of them were confirmed within 5 blocks and + /// then you would look at the next highest feerate bucket, and so on, stopping at + /// the last bucket to pass the test.The average feerate of transactions in this + /// bucket will give you an indication of the lowest feerate you can put on a + /// transaction and still have a sufficiently high chance of being confirmed + /// within your desired 5 blocks. + /// + /// Here is a brief description of the implementation: + /// When a transaction enters the mempool, we track the height of the block chain + /// at entry. All further calculations are conducted only on this set of "seen" + /// transactions.Whenever a block comes in, we count the number of transactions + /// in each bucket and the total amount of feerate paid in each bucket. Then we + /// calculate how many blocks Y it took each transaction to be mined. We convert + /// from a number of blocks to a number of periods Y' each encompassing "scale" + /// blocks.This is tracked in 3 different data sets each up to a maximum + /// number of periods.Within each data set we have an array of counters in each + /// feerate bucket and we increment all the counters from Y' up to max periods + /// representing that a tx was successfully confirmed in less than or equal to + /// that many periods. We want to save a history of this information, so at any + /// time we have a counter of the total number of transactions that happened in a + /// given feerate bucket and the total number that were confirmed in each of the + /// periods or less for any bucket. We save this history by keeping an + /// exponentially decaying moving average of each one of these stats. This is + /// done for a different decay in each of the 3 data sets to keep relevant data + /// from different time horizons. Furthermore we also keep track of the number + /// unmined (in mempool or left mempool without being included in a block) + /// transactions in each bucket and for how many blocks they have been + /// outstanding and use both of these numbers to increase the number of transactions + /// we've seen in that feerate bucket when calculating an estimate for any number + /// of confirmations below the number of blocks they've been outstanding. + /// + public class BitcoinBlockPolicyEstimator : IBlockPolicyEstimator + { + /// Track confirm delays up to 12 blocks for short horizon + private const int ShortBlockPeriods = 12; + private const int ShortScale = 1; + + /// Track confirm delays up to 48 blocks for medium horizon + private const int MedBlockPeriods = 24; + private const int MedScale = 2; + + /// Track confirm delays up to 1008 blocks for long horizon + private const int LongBlockPeriods = 42; + private const int LongScale = 24; + + /// Historical estimates that are older than this aren't valid + private const int OldestEstimateHistory = 6 * 1008; + + /// Decay of .962 is a half-life of 18 blocks or about 3 hours + private const double ShortDecay = .962; + + /// Decay of .998 is a half-life of 144 blocks or about 1 day + private const double MedDecay = .9952; + + /// Decay of .9995 is a half-life of 1008 blocks or about 1 week + private const double LongDecay = .99931; + + /// Require greater than 60% of X feerate transactions to be confirmed within Y/2 blocks + private const double HalfSuccessPct = .6; + + /// Require greater than 85% of X feerate transactions to be confirmed within Y blocks + private const double SuccessPct = .85; + + /// Require greater than 95% of X feerate transactions to be confirmed within 2 * Y blocks + private const double DoubleSuccessPct = .95; + + /// Require an avg of 0.1 tx in the combined feerate bucket per block to have stat significance. + private const double SufficientFeeTxs = 0.1; + + /// Require an avg of 0.5 tx when using short decay since there are fewer blocks considered + private const double SufficientTxsShort = 0.5; + + /// + /// Minimum and maximum values for tracking feerates. + /// The should just be set to the lowest reasonable feerate we + /// might ever want to track. Historically this has been 1000 since it was inheriting + /// DEFAULT_MIN_RELAY_TX_FEE and changing it is disruptive as it invalidates old estimates + /// files. So leave it at 1000 unless it becomes necessary to lower it, and then lower it substantially. + /// + private const double MinBucketFeeRate = 1000; + private const double MaxBucketFeeRate = 1e7; + + /// + /// Spacing of FeeRate buckets. + /// + /// + /// We have to lump transactions into buckets based on feerate, but we want to be able + /// to give accurate estimates over a large range of potential feerates. + /// Therefore it makes sense to exponentially space the buckets. + /// + private const double FeeSpacing = 1.05; + + private const double InfFeeRate = 1e99; + + private const string FileName = "fee.json"; + + /// Best seen block height. + private int nBestSeenHeight; + + private int firstRecordedHeight; + private int historicalFirst; + private int historicalBest; + + /// Logger for logging on this object. + private readonly ILogger logger; + + /// Classes to track historical data on transaction confirmations. + private TxConfirmStats feeStats; + private TxConfirmStats shortStats; + private TxConfirmStats longStats; + + /// Map of txids to information about that transaction. + private readonly Dictionary mapMemPoolTxs; + + /// Count of tracked transactions. + private int trackedTxs; + + /// Count of untracked transactions. + private int untrackedTxs; + + /// + /// The upper-bound of the range for the bucket (inclusive). + /// + /// + /// Define the buckets we will group transactions into. + /// + private List buckets; + + /// Map of bucket upper-bound to index into all vectors by bucket. + private SortedDictionary bucketMap; + + /// + /// Locks access to + /// + private readonly object lockObject; + + private readonly FileStorage fileStorage; + + /// + /// Constructs an instance of the block policy estimator object. + /// + /// Factory for creating loggers. + /// Full node settings. + public BitcoinBlockPolicyEstimator(ILoggerFactory loggerFactory, NodeSettings nodeSettings) + { + Guard.Assert(MinBucketFeeRate > 0); + + this.lockObject = new object(); + this.logger = loggerFactory.CreateLogger(this.GetType().FullName); + this.mapMemPoolTxs = new Dictionary(); + this.buckets = new List(); + this.bucketMap = new SortedDictionary(); + this.nBestSeenHeight = 0; + this.firstRecordedHeight = 0; + this.historicalFirst = 0; + this.historicalBest = 0; + this.trackedTxs = 0; + this.untrackedTxs = 0; + int bucketIndex = 0; + + for (double bucketBoundary = MinBucketFeeRate; bucketBoundary <= MaxBucketFeeRate; bucketBoundary *= FeeSpacing, bucketIndex++) + { + this.buckets.Add(bucketBoundary); + this.bucketMap.Add(bucketBoundary, bucketIndex); + } + + this.buckets.Add(InfFeeRate); + this.bucketMap.Add(InfFeeRate, bucketIndex); + + Guard.Assert(this.bucketMap.Count == this.buckets.Count); + + this.feeStats = new TxConfirmStats(this.logger); + this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); + + this.shortStats = new TxConfirmStats(this.logger); + this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); + + this.longStats = new TxConfirmStats(this.logger); + this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); + + this.fileStorage = new FileStorage(nodeSettings.DataFolder.WalletPath); + } + + /// + public void ProcessBlock(int nBlockHeight, List entries) + { + lock (this.lockObject) + { + if (nBlockHeight <= this.nBestSeenHeight) + { + // Ignore side chains and re-orgs; assuming they are random they don't affect the estimate. + // And if an attacker can re-org the chain at will, then you've got much bigger problems than "attacker can influence transaction fees." + return; + } + + // Must update nBestSeenHeight in sync with ClearCurrent so that calls to removeTx (via processBlockTx) correctly calculate age + // of unconfirmed txs to remove from tracking. + this.nBestSeenHeight = nBlockHeight; + + // Update unconfirmed circular buffer + this.feeStats.ClearCurrent(nBlockHeight); + this.shortStats.ClearCurrent(nBlockHeight); + this.longStats.ClearCurrent(nBlockHeight); + + // Decay all exponential averages + this.feeStats.UpdateMovingAverages(); + this.shortStats.UpdateMovingAverages(); + this.longStats.UpdateMovingAverages(); + + int countedTxs = 0; + + // Repopulate the current block states + foreach (TxMempoolEntry entry in entries) + { + if (this.ProcessBlockTx(nBlockHeight, entry)) + countedTxs++; + } + + if (this.firstRecordedHeight == 0 && countedTxs > 0) + { + this.firstRecordedHeight = this.nBestSeenHeight; + this.logger.LogInformation("Blockpolicy first recorded height {0}", this.firstRecordedHeight); + } + + this.logger.LogDebug($"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); + + this.trackedTxs = 0; + this.untrackedTxs = 0; + } + } + + /// + public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) + { + lock (this.lockObject) + { + int txHeight = entry.EntryHeight; + uint256 hash = entry.TransactionHash; + + if (this.mapMemPoolTxs.ContainsKey(hash)) + { + this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); + return; + } + + if (txHeight != this.nBestSeenHeight) + { + // Ignore side chains and re-orgs; assuming they are random they don't + // affect the estimate. We'll potentially double count transactions in 1-block reorgs. + // Ignore txs if BlockPolicyEstimator is not in sync with the chain tip. + // It will be synced next time a block is processed. + return; + } + + // Only want to be updating estimates when our blockchain is synced, otherwise we'll miscalculate how many blocks it's taking to get included. + if (!validFeeEstimate) + { + this.untrackedTxs++; + return; + } + + this.trackedTxs++; + + // Feerates are stored and reported as BTC-per-kb: + var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); + this.mapMemPoolTxs[hash].blockHeight = txHeight; + + int bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + this.mapMemPoolTxs[hash].bucketIndex = bucketIndex; + + int bucketIndex2 = this.shortStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + Guard.Assert(bucketIndex == bucketIndex2); + + int bucketIndex3 = this.longStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); + Guard.Assert(bucketIndex == bucketIndex3); + } + } + + /// + public bool RemoveTx(uint256 hash, bool inBlock) + { + lock (this.lockObject) + { + TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); + + if (pos == null) + return false; + + this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.shortStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.longStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex, inBlock); + this.mapMemPoolTxs.Remove(hash); + + return true; + } + } + + /// + public FeeRate EstimateFee(int confTarget) + { + // It's not possible to get reasonable estimates for confTarget of 1 + return confTarget <= 1 ? new FeeRate(0) : EstimateRawFee(confTarget, DoubleSuccessPct, FeeEstimateHorizon.MedHalfLife, null); + } + + /// + public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) + { + lock (this.lockObject) + { + if (feeCalc != null) + { + feeCalc.DesiredTarget = confTarget; + feeCalc.ReturnedTarget = confTarget; + } + + double median = -1; + var tempResult = new EstimationResult(); + + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > this.longStats.GetMaxConfirms()) + return new FeeRate(0); // error condition + + // It's not possible to get reasonable estimates for confTarget of 1 + if (confTarget == 1) + confTarget = 2; + + int maxUsableEstimate = MaxUsableEstimate(); + + if (confTarget > maxUsableEstimate) + confTarget = maxUsableEstimate; + + if (feeCalc != null) + feeCalc.ReturnedTarget = confTarget; + + if (confTarget <= 1) + return new FeeRate(0); // error condition + + Guard.Assert(confTarget > 0); //estimateCombinedFee and estimateConservativeFee take unsigned ints + + // true is passed to estimateCombined fee for target/2 and target so + // that we check the max confirms for shorter time horizons as well. + // This is necessary to preserve monotonically increasing estimates. + // For non-conservative estimates we do the same thing for 2*target, but + // for conservative estimates we want to skip these shorter horizons + // checks for 2*target because we are taking the max over all time + // horizons so we already have monotonically increasing estimates and + // the purpose of conservative estimates is not to let short term + // fluctuations lower our estimates by too much. + double halfEst = EstimateCombinedFee(confTarget / 2, HalfSuccessPct, true, tempResult); + + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.HalfEstimate; + } + + median = halfEst; + double actualEst = EstimateCombinedFee(confTarget, SuccessPct, true, tempResult); + + if (actualEst > median) + { + median = actualEst; + + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.FullEstimate; + } + } + + double doubleEst = EstimateCombinedFee(2 * confTarget, DoubleSuccessPct, !conservative, tempResult); + + if (doubleEst > median) + { + median = doubleEst; + + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.DoubleEstimate; + } + } + + if (conservative || median == -1) + { + double consEst = EstimateConservativeFee(2 * confTarget, tempResult); + + if (consEst > median) + { + median = consEst; + + if (feeCalc != null) + { + feeCalc.Estimation = tempResult; + feeCalc.Reason = FeeReason.Conservative; + } + } + } + + return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); + } + } + + /// + public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) + { + TxConfirmStats stats; + double sufficientTxs = SufficientFeeTxs; + switch (horizon) + { + case FeeEstimateHorizon.ShortHalfLife: + { + stats = this.shortStats; + sufficientTxs = SufficientTxsShort; + break; + } + case FeeEstimateHorizon.MedHalfLife: + { + stats = this.feeStats; + break; + } + case FeeEstimateHorizon.LongHalfLife: + { + stats = this.longStats; + break; + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + + lock (this.lockObject) + { + // Return failure if trying to analyze a target we're not tracking + if (confTarget <= 0 || confTarget > stats.GetMaxConfirms()) + return new FeeRate(0); + + if (successThreshold > 1) + return new FeeRate(0); + + double median = stats.EstimateMedianVal(confTarget, sufficientTxs, successThreshold, true, this.nBestSeenHeight, result); + + return median < 0 ? new FeeRate(0) : new FeeRate(Convert.ToInt64(median)); + } + } + + /// + public void Write() + { + var data = new BlockPolicyData(); + + data.BestSeenHeight = this.nBestSeenHeight; + + if (BlockSpan() > HistoricalBlockSpan()) + { + data.HistoricalFirst = this.firstRecordedHeight; + data.HistoricalBest = this.nBestSeenHeight; + } + else + { + data.HistoricalFirst = this.historicalFirst; + data.HistoricalBest = this.historicalBest; + } + + data.Buckets = this.buckets; + data.ShortStats = this.shortStats.Write(); + data.MedStats = this.feeStats.Write(); + data.LongStats = this.longStats.Write(); + + this.fileStorage.SaveToFile(data, FileName); + } + + /// + public bool Read() + { + try + { + lock (this.lockObject) + { + var data = this.fileStorage.LoadByFileName(FileName); + if (data != null) + throw new ApplicationException("Corrupt estimates file or file not found"); + + if (data.HistoricalFirst > data.HistoricalBest || data.HistoricalBest > data.BestSeenHeight) + throw new ApplicationException("Corrupt estimates file. Historical block range for estimates is invalid"); + + if (data.Buckets.Count <= 1 || data.Buckets.Count > 1000) + throw new ApplicationException("Corrupt estimates file. Must have between 2 and 1000 feerate buckets"); + + this.nBestSeenHeight = data.BestSeenHeight; + this.historicalFirst = data.HistoricalFirst; + this.historicalBest = data.HistoricalBest; + this.buckets = data.Buckets; + this.bucketMap = new SortedDictionary(); + + for(int i = 0; i< this.buckets.Count; i++) + this.bucketMap.Add(this.buckets[i], i); + + this.feeStats = new TxConfirmStats(this.logger); + this.feeStats.Initialize(this.buckets, this.bucketMap, MedBlockPeriods, MedDecay, MedScale); + this.feeStats.Read(data.MedStats); + + this.shortStats = new TxConfirmStats(this.logger); + this.shortStats.Initialize(this.buckets, this.bucketMap, ShortBlockPeriods, ShortDecay, ShortScale); + this.shortStats.Read(data.ShortStats); + + this.longStats = new TxConfirmStats(this.logger); + this.longStats.Initialize(this.buckets, this.bucketMap, LongBlockPeriods, LongDecay, LongScale); + this.longStats.Read(data.LongStats); + } + } + catch (Exception e) + { + this.logger.LogError("Error while reading policy estimation data from file", e); + return false; + } + + return true; + } + + /// + public void FlushUnconfirmed() + { + lock (this.lockObject) + { + int numEntries = this.mapMemPoolTxs.Count; + + // Remove every entry in mapMemPoolTxs + while (this.mapMemPoolTxs.Count > 0) + { + var mi = this.mapMemPoolTxs.First(); ; + RemoveTx(mi.Key, false); // this calls erase() on mapMemPoolTxs + } + + this.logger.LogDebug($"Recorded {numEntries} unconfirmed txs from mempool"); + } + } + + /// + public int HighestTargetTracked(FeeEstimateHorizon horizon) + { + switch (horizon) + { + case FeeEstimateHorizon.ShortHalfLife: + { + return this.shortStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.MedHalfLife: + { + return this.feeStats.GetMaxConfirms(); + } + case FeeEstimateHorizon.LongHalfLife: + { + return this.longStats.GetMaxConfirms(); + } + default: + { + throw new ArgumentException(nameof(horizon)); + } + } + } + + /// + /// Process a transaction confirmed in a block. + /// + /// Height of the block. + /// The memory pool entry. + /// Whether it was able to successfully process the transaction. + private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) + { + if (!this.RemoveTx(entry.TransactionHash, true)) + { + // This transaction wasn't being tracked for fee estimation + return false; + } + + // How many blocks did it take for miners to include this transaction? + // blocksToConfirm is 1-based, so a transaction included in the earliest + // possible block has confirmation count of 1 + int blocksToConfirm = nBlockHeight - entry.EntryHeight; + + if (blocksToConfirm <= 0) + { + // This can't happen because we don't process transactions from a block with a height lower than our greatest seen height. + this.logger.LogDebug($"Blockpolicy error Transaction had negative blocksToConfirm"); + return false; + } + + // Feerates are stored and reported as BTC-per-kb: + var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); + + this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.shortStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + this.longStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); + + return true; + } + + private int BlockSpan() + { + if (this.firstRecordedHeight == 0) + return 0; + + Guard.Assert(this.nBestSeenHeight >= this.firstRecordedHeight); + + return this.nBestSeenHeight - this.firstRecordedHeight; + } + + private int HistoricalBlockSpan() + { + if (this.historicalFirst == 0) + return 0; + + Guard.Assert(this.historicalBest >= this.historicalFirst); + + if (this.nBestSeenHeight - this.historicalBest > OldestEstimateHistory) + return 0; + + return this.historicalBest - this.historicalFirst; + } + + private int MaxUsableEstimate() + { + // Block spans are divided by 2 to make sure there are enough potential failing data points for the estimate + return Math.Min(this.longStats.GetMaxConfirms(), Math.Max(BlockSpan(), HistoricalBlockSpan()) / 2); + } + + /// + /// Return a fee estimate at the required successThreshold from the shortest + /// time horizon which tracks confirmations up to the desired target.If + /// checkShorterHorizon is requested, also allow short time horizon estimates + /// for a lower target to reduce the given answer + /// + private double EstimateCombinedFee(int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult result) + { + double estimate = -1; + + if (confTarget >= 1 && confTarget <= this.longStats.GetMaxConfirms()) + { + // Find estimate from shortest time horizon possible + if (confTarget <= this.shortStats.GetMaxConfirms()) + { + // short horizon + estimate = this.shortStats.EstimateMedianVal(confTarget, SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, result); + } + else if (confTarget <= this.feeStats.GetMaxConfirms()) + { + // medium horizon + estimate = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); + } + else + { + // long horizon + estimate = this.longStats.EstimateMedianVal(confTarget, SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, result); + } + + if (checkShorterHorizon) + { + var tempResult = new EstimationResult(); + + // If a lower confTarget from a more recent horizon returns a lower answer use it. + if (confTarget > this.feeStats.GetMaxConfirms()) + { + double medMax = this.feeStats.EstimateMedianVal(this.feeStats.GetMaxConfirms(), SufficientFeeTxs, successThreshold, true, this.nBestSeenHeight, tempResult); + + if (medMax > 0 && (estimate == -1 || medMax < estimate)) + { + estimate = medMax; + + if (result != null) + result = tempResult; + } + } + + if (confTarget > this.shortStats.GetMaxConfirms()) + { + double shortMax = this.shortStats.EstimateMedianVal(this.shortStats.GetMaxConfirms(), SufficientTxsShort, successThreshold, true, this.nBestSeenHeight, tempResult); + + if (shortMax > 0 && (estimate == -1 || shortMax < estimate)) + { + estimate = shortMax; + + if (result != null) + result = tempResult; + } + } + } + } + + return estimate; + } + + /// + /// Ensure that for a conservative estimate, the DOUBLE_SUCCESS_PCT is also met + /// at 2 * target for any longer time horizons. + /// + private double EstimateConservativeFee(int doubleTarget, EstimationResult result) + { + double estimate = -1; + var tempResult = new EstimationResult(); + + if (doubleTarget <= this.shortStats.GetMaxConfirms()) + estimate = this.feeStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, result); + + if (doubleTarget <= this.feeStats.GetMaxConfirms()) + { + double longEstimate = this.longStats.EstimateMedianVal(doubleTarget, SufficientFeeTxs, DoubleSuccessPct, true, this.nBestSeenHeight, tempResult); + + if (longEstimate > estimate) + { + estimate = longEstimate; + + if (result != null) + result = tempResult; + } + } + + return estimate; + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs deleted file mode 100644 index 005af909a9f..00000000000 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/BlockPolicyEstimator.cs +++ /dev/null @@ -1,409 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Features.MemoryPool.Interfaces; - -namespace Stratis.Bitcoin.Features.MemoryPool.Fee -{ - /// - /// The BlockPolicyEstimator is used for estimating the feerate needed - /// for a transaction to be included in a block within a certain number of - /// blocks. - /// - /// - /// At a high level the algorithm works by grouping transactions into buckets - /// based on having similar feerates and then tracking how long it - /// takes transactions in the various buckets to be mined. It operates under - /// the assumption that in general transactions of higher feerate will be - /// included in blocks before transactions of lower feerate. So for - /// example if you wanted to know what feerate you should put on a transaction to - /// be included in a block within the next 5 blocks, you would start by looking - /// at the bucket with the highest feerate transactions and verifying that a - /// sufficiently high percentage of them were confirmed within 5 blocks and - /// then you would look at the next highest feerate bucket, and so on, stopping at - /// the last bucket to pass the test. The average feerate of transactions in this - /// bucket will give you an indication of the lowest feerate you can put on a - /// transaction and still have a sufficiently high chance of being confirmed - /// within your desired 5 blocks. - /// - /// Here is a brief description of the implementation: - /// When a transaction enters the mempool, we - /// track the height of the block chain at entry. Whenever a block comes in, - /// we count the number of transactions in each bucket and the total amount of feerate - /// paid in each bucket. Then we calculate how many blocks Y it took each - /// transaction to be mined and we track an array of counters in each bucket - /// for how long it to took transactions to get confirmed from 1 to a max of 25 - /// and we increment all the counters from Y up to 25. This is because for any - /// number Z>=Y the transaction was successfully mined within Z blocks. We - /// want to save a history of this information, so at any time we have a - /// counter of the total number of transactions that happened in a given feerate - /// bucket and the total number that were confirmed in each number 1-25 blocks - /// or less for any bucket. We save this history by keeping an exponentially - /// decaying moving average of each one of these stats. Furthermore we also - /// keep track of the number unmined (in mempool) transactions in each bucket - /// and for how many blocks they have been outstanding and use that to increase - /// the number of transactions we've seen in that feerate bucket when calculating - /// an estimate for any number of confirmations below the number of blocks - /// they've been outstanding. - /// - /// We will instantiate an instance of this class to track transactions that were - /// included in a block. We will lump transactions into a bucket according to their - /// approximate feerate and then track how long it took for those txs to be included in a block - /// - /// The tracking of unconfirmed (mempool) transactions is completely independent of the - /// historical tracking of transactions that have been confirmed in a block. - /// - /// We want to be able to estimate feerates that are needed on tx's to be included in - /// a certain number of blocks.Every time a block is added to the best chain, this class records - /// stats on the transactions included in that block - /// - public class BlockPolicyEstimator - { - /// Require an avg of 1 tx in the combined feerate bucket per block to have stat significance. - private const double SufficientFeeTxs = 1; - - /// Require greater than 95% of X feerate transactions to be confirmed within Y blocks for X to be big enough. - private const double MinSuccessPct = .95; - - /// Minimum value for tracking feerates. - private const long MinFeeRate = 10; - - /// Maximum value for tracking feerates. - private const double MaxFeeRate = 1e7; - - /// - /// Spacing of FeeRate buckets. - /// - /// - /// We have to lump transactions into buckets based on feerate, but we want to be able - /// to give accurate estimates over a large range of potential feerates. - /// Therefore it makes sense to exponentially space the buckets. - /// - private const double FeeSpacing = 1.1; - - /// Track confirm delays up to 25 blocks, can't estimate beyond that. - private const int MaxBlockConfirms = 25; - - /// Decay of .998 is a half-life of 346 blocks or about 2.4 days. - private const double DefaultDecay = .998; - - /// Value for infinite priority. - public const double InfPriority = 1e9 * 21000000ul * Money.COIN; - - /// Maximum money value. - private static readonly Money MaxMoney = new Money(21000000 * Money.COIN); - - /// Value for infinite fee rate. - private static readonly double InfFeeRate = MaxMoney.Satoshi; - - /// Classes to track historical data on transaction confirmations. - private readonly TxConfirmStats feeStats; - - /// Map of txids to information about that transaction. - private readonly Dictionary mapMemPoolTxs; - - /// Minimum tracked Fee. Passed to constructor to avoid dependency on main./// - private readonly FeeRate minTrackedFee; - - /// Best seen block height. - private int nBestSeenHeight; - - /// Setting for the node. - private readonly MempoolSettings mempoolSettings; - - /// Logger for logging on this object. - private readonly ILogger logger; - - /// Count of tracked transactions. - private int trackedTxs; - - /// Count of untracked transactions. - private int untrackedTxs; - - /// - /// Constructs an instance of the block policy estimator object. - /// - /// Mempool settings. - /// Factory for creating loggers. - /// Full node settings. - public BlockPolicyEstimator(MempoolSettings mempoolSettings, ILoggerFactory loggerFactory, NodeSettings nodeSettings) - { - this.mapMemPoolTxs = new Dictionary(); - this.mempoolSettings = mempoolSettings; - this.nBestSeenHeight = 0; - this.trackedTxs = 0; - this.untrackedTxs = 0; - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - - this.minTrackedFee = nodeSettings.MinRelayTxFeeRate < new FeeRate(new Money(MinFeeRate)) - ? new FeeRate(new Money(MinFeeRate)) - : nodeSettings.MinRelayTxFeeRate; - var vfeelist = new List(); - for (double bucketBoundary = this.minTrackedFee.FeePerK.Satoshi; - bucketBoundary <= MaxFeeRate; - bucketBoundary *= FeeSpacing) - vfeelist.Add(bucketBoundary); - vfeelist.Add(InfFeeRate); - this.feeStats = new TxConfirmStats(this.logger); - this.feeStats.Initialize(vfeelist, MaxBlockConfirms, DefaultDecay); - } - - /// - /// Process all the transactions that have been included in a block. - /// - /// The block height for the block. - /// Collection of memory pool entries. - public void ProcessBlock(int nBlockHeight, List entries) - { - if (nBlockHeight <= this.nBestSeenHeight) - return; - - // Must update nBestSeenHeight in sync with ClearCurrent so that - // calls to removeTx (via processBlockTx) correctly calculate age - // of unconfirmed txs to remove from tracking. - this.nBestSeenHeight = nBlockHeight; - - // Clear the current block state and update unconfirmed circular buffer - this.feeStats.ClearCurrent(nBlockHeight); - - int countedTxs = 0; - // Repopulate the current block states - for (int i = 0; i < entries.Count; i++) - { - if (this.ProcessBlockTx(nBlockHeight, entries[i])) - countedTxs++; - } - - // Update all exponential averages with the current block state - this.feeStats.UpdateMovingAverages(); - - // TODO: this makes too much noise right now, put it back when logging is can be switched on by categories (and also consider disabling during IBD) - // Logging.Logs.EstimateFee.LogInformation( - // $"Blockpolicy after updating estimates for {countedTxs} of {entries.Count} txs in block, since last block {trackedTxs} of {trackedTxs + untrackedTxs} tracked, new mempool map size {mapMemPoolTxs.Count}"); - - this.trackedTxs = 0; - this.untrackedTxs = 0; - } - - /// - /// Process a transaction confirmed in a block. - /// - /// Height of the block. - /// The memory pool entry. - /// Whether it was able to successfully process the transaction. - private bool ProcessBlockTx(int nBlockHeight, TxMempoolEntry entry) - { - if (!this.RemoveTx(entry.TransactionHash)) - return false; - - // How many blocks did it take for miners to include this transaction? - // blocksToConfirm is 1-based, so a transaction included in the earliest - // possible block has confirmation count of 1 - int blocksToConfirm = nBlockHeight - entry.EntryHeight; - if (blocksToConfirm <= 0) - { - // This can't happen because we don't process transactions from a block with a height - // lower than our greatest seen height - this.logger.LogInformation($"Blockpolicy error Transaction had negative blocksToConfirm"); - return false; - } - - // Feerates are stored and reported as BTC-per-kb: - var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.feeStats.Record(blocksToConfirm, feeRate.FeePerK.Satoshi); - return true; - } - - /// - /// Process a transaction accepted to the mempool. - /// - /// Memory pool entry. - /// Whether to update fee estimate. - public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) - { - int txHeight = entry.EntryHeight; - uint256 hash = entry.TransactionHash; - if (this.mapMemPoolTxs.ContainsKey(hash)) - { - this.logger.LogInformation($"Blockpolicy error mempool tx {hash} already being tracked"); - return; - } - - if (txHeight != this.nBestSeenHeight) - return; - - // Only want to be updating estimates when our blockchain is synced, - // otherwise we'll miscalculate how many blocks its taking to get included. - if (!validFeeEstimate) - { - this.untrackedTxs++; - return; - } - this.trackedTxs++; - - // Feerates are stored and reported as BTC-per-kb: - var feeRate = new FeeRate(entry.Fee, (int)entry.GetTxSize()); - - this.mapMemPoolTxs.Add(hash, new TxStatsInfo()); - this.mapMemPoolTxs[hash].blockHeight = txHeight; - this.mapMemPoolTxs[hash].bucketIndex = this.feeStats.NewTx(txHeight, feeRate.FeePerK.Satoshi); - } - - /// - /// Remove a transaction from the mempool tracking stats. - /// - /// Transaction hash. - /// Whether the transaction was successfully removed. - /// - /// This function is called from TxMemPool.RemoveUnchecked to ensure - /// txs removed from the mempool for any reason are no longer - /// tracked. Txs that were part of a block have already been removed in - /// ProcessBlockTx to ensure they are never double tracked, but it is - /// of no harm to try to remove them again. - /// - public bool RemoveTx(uint256 hash) - { - TxStatsInfo pos = this.mapMemPoolTxs.TryGet(hash); - if (pos != null) - { - this.feeStats.RemoveTx(pos.blockHeight, this.nBestSeenHeight, pos.bucketIndex); - this.mapMemPoolTxs.Remove(hash); - return true; - } - return false; - } - - /// - /// Return a feerate estimate - /// - /// The desired number of confirmations to be included in a block. - public FeeRate EstimateFee(int confTarget) - { - // Return failure if trying to analyze a target we're not tracking - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget <= 1 || confTarget > this.feeStats.GetMaxConfirms()) - return new FeeRate(0); - - double median = this.feeStats.EstimateMedianVal(confTarget, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); - - if (median < 0) - return new FeeRate(0); - - return new FeeRate(new Money((int)median)); - } - - /// - /// Estimate feerate needed to be included in a block within - /// confTarget blocks. If no answer can be given at confTarget, return an - /// estimate at the lowest target where one can be given. - /// - public FeeRate EstimateSmartFee(int confTarget, ITxMempool pool, out int answerFoundAtTarget) - { - answerFoundAtTarget = confTarget; - - // Return failure if trying to analyze a target we're not tracking - if (confTarget <= 0 || confTarget > this.feeStats.GetMaxConfirms()) - return new FeeRate(0); - - // It's not possible to get reasonable estimates for confTarget of 1 - if (confTarget == 1) - confTarget = 2; - - double median = -1; - while (median < 0 && confTarget <= this.feeStats.GetMaxConfirms()) - { - median = this.feeStats.EstimateMedianVal(confTarget++, SufficientFeeTxs, MinSuccessPct, true, - this.nBestSeenHeight); - } - - answerFoundAtTarget = confTarget - 1; - - // If mempool is limiting txs , return at least the min feerate from the mempool - if (pool != null) - { - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0 && minPoolFee.Satoshi > median) - return new FeeRate(minPoolFee); - } - - if (median < 0) - return new FeeRate(0); - - return new FeeRate((int)median); - } - - /// - /// Write estimation data to a file. - /// - /// Stream to write to. - /// TODO: Implement write estimation - public void Write(Stream fileout) - { - } - - /// - /// Read estimation data from a file. - /// - /// Stream to read data from. - /// Version number of the file. - /// TODO: Implement read estimation - public void Read(Stream filein, int nFileVersion) - { - } - - /// - /// Return an estimate of the priority. - /// - /// The desired number of confirmations to be included in a block. - /// Estimate of the priority. - /// TODO: Implement priority estimation - public double EstimatePriority(int confTarget) - { - return -1; - } - - /// - /// Return an estimated smart priority. - /// - /// The desired number of confirmations to be included in a block. - /// Memory pool transactions. - /// Block height where answer was found. - /// The smart priority. - public double EstimateSmartPriority(int confTarget, ITxMempool pool, out int answerFoundAtTarget) - { - answerFoundAtTarget = confTarget; - - // If mempool is limiting txs, no priority txs are allowed - Money minPoolFee = pool.GetMinFee(this.mempoolSettings.MaxMempool * 1000000).FeePerK; - if (minPoolFee > 0) - return InfPriority; - - return -1; - } - - /// - /// Transaction statistics information. - /// - public class TxStatsInfo - { - /// The block height. - public int blockHeight; - - /// The index into the confirmed transactions bucket map. - public int bucketIndex; - - /// - /// Constructs a instance of a transaction stats info object. - /// - public TxStatsInfo() - { - this.blockHeight = 0; - this.bucketIndex = 0; - } - } - } -} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs new file mode 100644 index 00000000000..fc94589867a --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimationResult.cs @@ -0,0 +1,16 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Used to return detailed information about a fee estimate calculation + /// + public class EstimationResult + { + public EstimatorBucket Pass { get; set; } + + public EstimatorBucket Fail { get; set; } + + public double Decay { get; set; } + + public int Scale { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs new file mode 100644 index 00000000000..d56493d6584 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/EstimatorBucket.cs @@ -0,0 +1,26 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Used to return detailed information about a feerate bucket + /// + public class EstimatorBucket + { + public double Start { get; set; } + + public double End { get; set; } + + public double WithinTarget { get; set; } + + public double TotalConfirmed { get; set; } + + public double InMempool { get; set; } + + public double LeftMempool { get; set; } + + public EstimatorBucket() + { + this.Start = -1; + this.End = -1; + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs new file mode 100644 index 00000000000..a3d1d4e61d6 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeCalculation.cs @@ -0,0 +1,13 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public class FeeCalculation + { + public EstimationResult Estimation { get; set; } + + public FeeReason Reason { get; set; } + + public int DesiredTarget { get; set; } + + public int ReturnedTarget { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs new file mode 100644 index 00000000000..7e9956e0cae --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeEstimateHorizon.cs @@ -0,0 +1,13 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Identifier for each of the 3 different TxConfirmStats which will track + /// history over different time horizons + /// + public enum FeeEstimateHorizon + { + ShortHalfLife, + MedHalfLife, + LongHalfLife + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs new file mode 100644 index 00000000000..a274466b323 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/FeeReason.cs @@ -0,0 +1,19 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Enumeration of reason for returned fee estimate + /// + public enum FeeReason + { + None, + HalfEstimate, + FullEstimate, + DoubleEstimate, + Conservative, + MemPoolMin, + PayTxFee, + Fallback, + Required, + MaxTxFee + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs new file mode 100644 index 00000000000..37bf3dd1771 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/IBlockPolicyEstimator.cs @@ -0,0 +1,82 @@ +using System.Collections.Generic; +using NBitcoin; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public interface IBlockPolicyEstimator + { + /// + /// Process all the transactions that have been included in a block. + /// + /// The block height for the block. + /// Collection of memory pool entries. + void ProcessBlock(int nBlockHeight, List entries); + + /// + /// Process a transaction accepted to the mempool. + /// + /// Memory pool entry. + /// Whether to update fee estimate. + void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate); + + /// + /// Remove a transaction from the mempool tracking stats. + /// + /// Transaction hash. + /// Whether the transaction was successfully removed. + /// + /// This function is called from TxMemPool.RemoveUnchecked to ensure + /// txs removed from the mempool for any reason are no longer + /// tracked. Txs that were part of a block have already been removed in + /// ProcessBlockTx to ensure they are never double tracked, but it is + /// of no harm to try to remove them again. + /// + bool RemoveTx(uint256 hash, bool inBlock); + + /// + /// Return a feerate estimate (deprecated, per Bitcoin Core source). + /// + /// The desired number of confirmations to be included in a block. + FeeRate EstimateFee(int confTarget); + + /// + /// Returns the max of the feerates calculated with a 60% + /// threshold required at target / 2, an 85% threshold required at target and a + /// 95% threshold required at 2 * target.Each calculation is performed at the + /// shortest time horizon which tracks the required target.Conservative + /// estimates, however, required the 95% threshold at 2 * target be met for any + /// longer time horizons also. + /// + FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative); + + /// + /// Return a specific fee estimate calculation with a given success threshold and time horizon, + /// and optionally return detailed data about calculation. + /// + /// The desired number of confirmations to be included in a block + FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result); + + /// + /// Write estimation data to a file. + /// + void Write(); + + /// + /// Read estimation data from a file. + /// + /// Stream to read data from. + /// Version number of the file. + bool Read(); + + /// + /// Calculation of highest target that estimates are tracked for. + /// + /// + int HighestTargetTracked(FeeEstimateHorizon horizon); + + /// + /// Empty mempool transactions on shutdown to record failure to confirm for txs still in mempool. + /// + void FlushUnconfirmed(); + } +} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs new file mode 100644 index 00000000000..04846c8ee58 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/BlockPolicyData.cs @@ -0,0 +1,21 @@ +using System.Collections.Generic; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity +{ + public class BlockPolicyData + { + public int BestSeenHeight { get; set; } + + public int HistoricalFirst { get; set; } + + public int HistoricalBest { get; set; } + + public List Buckets { get; set; } + + public TxConfirmData ShortStats { get; set; } + + public TxConfirmData MedStats { get; set; } + + public TxConfirmData LongStats { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs new file mode 100644 index 00000000000..644d7aa8572 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/SerializationEntity/TxConfirmData.cs @@ -0,0 +1,19 @@ +using System.Collections.Generic; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity +{ + public class TxConfirmData + { + public double Decay { get; set; } + + public int Scale { get; set; } + + public List Avg { get; set; } + + public List TxCtAvg { get; set; } + + public List> ConfAvg { get; set; } + + public List> FailAvg { get; set; } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs new file mode 100644 index 00000000000..002fbd43550 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/StratisBlockPolicyEstimator.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using NBitcoin; + +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + public class StratisBlockPolicyEstimator : IBlockPolicyEstimator + { + // TODO: Need to complete the Bitcoin version of the algorithm, then restructure it to compute its constants in a non-network-specific way + + public FeeRate EstimateFee(int confTarget) + { + throw new System.NotImplementedException(); + } + + public bool RemoveTx(uint256 hash, bool inBlock) + { + throw new System.NotImplementedException(); + } + + public FeeRate EstimateRawFee(int confTarget, double successThreshold, FeeEstimateHorizon horizon, EstimationResult result) + { + throw new System.NotImplementedException(); + } + + public FeeRate EstimateSmartFee(int confTarget, FeeCalculation feeCalc, bool conservative) + { + throw new System.NotImplementedException(); + } + + public void ProcessBlock(int nBlockHeight, List entries) + { + throw new System.NotImplementedException(); + } + + public void ProcessTransaction(TxMempoolEntry entry, bool validFeeEstimate) + { + throw new System.NotImplementedException(); + } + + public int HighestTargetTracked(FeeEstimateHorizon horizon) + { + throw new System.NotImplementedException(); + } + + public void Write() + { + throw new System.NotImplementedException(); + } + + public bool Read() + { + throw new System.NotImplementedException(); + } + + public void FlushUnconfirmed() + { + throw new System.NotImplementedException(); + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs index 70139ed3f09..bfa89e90d44 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxConfirmStats.cs @@ -1,49 +1,72 @@ using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Extensions.Logging; -using NBitcoin; +using Stratis.Bitcoin.Utilities; +using System; +using Stratis.Bitcoin.Features.MemoryPool.Fee.SerializationEntity; namespace Stratis.Bitcoin.Features.MemoryPool.Fee { /// - /// Transation confirmation statistics. + /// Transaction confirmation statistics. /// + /// + /// We will instantiate an instance of this class to track transactions that were + /// included in a block. We will lump transactions into a bucket according to their + /// approximate feerate and then track how long it took for those txs to be included in a block. + /// The tracking of unconfirmed (mempool) transactions is completely independent of the + /// historical tracking of transactions that have been confirmed in a block. + /// public class TxConfirmStats { - /// Instance logger for logging messages. private readonly ILogger logger; /// - /// Moving average of total fee rate of all transactions in each bucket. + /// The upper-bound of the range for the bucket (inclusive). /// /// - /// Track the historical moving average of this total over blocks. + /// Define the buckets we will group transactions into. /// - private List avg; + private List buckets; /// Map of bucket upper-bound to index into all vectors by bucket. - private Dictionary bucketMap; - - //Define the buckets we will group transactions into. - - /// The upper-bound of the range for the bucket (inclusive). - private List buckets; + private SortedDictionary bucketMap; - // Count the total # of txs confirmed within Y blocks in each bucket. - // Track the historical moving average of theses totals over blocks. + /// + /// Historical moving average of transaction counts. + /// + /// + /// For each bucket X: + /// Count the total # of txs in each bucket. + /// Track the historical moving average of this total over blocks. + /// + private List txCtAvg; - /// Confirmation average. confAvg[Y][X]. + /// + /// Confirmation average. confAvg[Y][X] + /// + /// + /// Count the total # of txs confirmed within Y blocks in each bucket. + /// Track the historical moving average of these totals over blocks. + /// private List> confAvg; - /// Current block confirmations. curBlockConf[Y][X]. - private List> curBlockConf; - - /// Current block transaction count. - private List curBlockTxCt; + /// + /// Failed average. failAvg[Y][X] + /// + /// + /// Track moving avg of txs which have been evicted from the mempool + /// after failing to be confirmed within Y blocks. + /// + private List> failAvg; - /// Current block fee rate. - private List curBlockVal; + /// + /// Moving average of total fee rate of all transactions in each bucket. + /// + /// + /// Track the historical moving average of this total over blocks. + /// + private List avg; // Combine the conf counts with tx counts to calculate the confirmation % for each Y,X // Combine the total value with the tx counts to calculate the avg feerate per bucket @@ -51,29 +74,24 @@ public class TxConfirmStats /// Decay value to use. private double decay; - /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket - private List oldUnconfTxs; - /// - /// Historical moving average of transaction counts. + /// Resolution (# of blocks) with which confirmations are tracked /// - /// - /// For each bucket X: - /// Count the total # of txs in each bucket. - /// Track the historical moving average of this total over blocks - /// - private List txCtAvg; + private int scale; /// /// Mempool counts of outstanding transactions. /// /// /// For each bucket X, track the number of transactions in the mempool - /// that are unconfirmed for each possible confirmation value Y + /// that are unconfirmed for each possible confirmation value Y. /// unconfTxs[Y][X] /// private List> unconfTxs; + /// Transactions still unconfirmed after MAX_CONFIRMS for each bucket + private List oldUnconfTxs; + /// /// Constructs an instance of the transaction confirmation stats object. /// @@ -84,39 +102,41 @@ public TxConfirmStats(ILogger logger) } /// - /// Initialize the data structures. This is called by BlockPolicyEstimator's - /// constructor with default values. + /// Initialize the data structures. This is called by BlockPolicyEstimator's constructor with default values. /// /// Contains the upper limits for the bucket boundaries. - /// Max number of confirms to track. + /// Max number of periods to track. /// How much to decay the historical moving average per block. - public void Initialize(List defaultBuckets, int maxConfirms, double decay) + public void Initialize(List defaultBuckets, IDictionary defaultBucketMap, int maxPeriods, double decay, int scale) { - this.buckets = new List(); - this.bucketMap = new Dictionary(); + Guard.Assert(scale != 0); this.decay = decay; - for (int i = 0; i < defaultBuckets.Count; i++) - { - this.buckets.Add(defaultBuckets[i]); - this.bucketMap[defaultBuckets[i]] = i; - } + this.scale = scale; this.confAvg = new List>(); - this.curBlockConf = new List>(); + this.failAvg = new List>(); + this.buckets = new List(defaultBuckets); + this.bucketMap = new SortedDictionary(defaultBucketMap); this.unconfTxs = new List>(); - for (int i = 0; i < maxConfirms; i++) + for (int i = 0; i < maxPeriods; i++) { this.confAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); - this.curBlockConf.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); - this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), this.buckets.Count).ToList()); + this.failAvg.Insert(i, Enumerable.Repeat(default(double), this.buckets.Count).ToList()); } - - this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), this.buckets.Count)); - this.curBlockTxCt = new List(Enumerable.Repeat(default(int), this.buckets.Count)); + this.txCtAvg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); - this.curBlockVal = new List(Enumerable.Repeat(default(double), this.buckets.Count)); this.avg = new List(Enumerable.Repeat(default(double), this.buckets.Count)); + + ClearInMemoryCounters(this.buckets.Count); + } + + private void ClearInMemoryCounters(int bucketsCount) + { + for (int i = 0; i < GetMaxConfirms(); i++) + this.unconfTxs.Insert(i, Enumerable.Repeat(default(int), bucketsCount).ToList()); + + this.oldUnconfTxs = new List(Enumerable.Repeat(default(int), bucketsCount)); } /// @@ -129,10 +149,6 @@ public void ClearCurrent(int nBlockHeight) { this.oldUnconfTxs[j] += this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j]; this.unconfTxs[nBlockHeight % this.unconfTxs.Count][j] = 0; - for (int i = 0; i < this.curBlockConf.Count; i++) - this.curBlockConf[i][j] = 0; - this.curBlockTxCt[j] = 0; - this.curBlockVal[j] = 0; } } @@ -146,66 +162,15 @@ public void Record(int blocksToConfirm, double val) // blocksToConfirm is 1-based if (blocksToConfirm < 1) return; - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; - for (int i = blocksToConfirm; i <= this.curBlockConf.Count; i++) - this.curBlockConf[i - 1][bucketindex]++; - this.curBlockTxCt[bucketindex]++; - this.curBlockVal[bucketindex] += val; - } - /// - /// Record a new transaction entering the mempool. - /// - /// The block height. - /// - /// The feerate of the transaction. - public int NewTx(int nBlockHeight, double val) - { - int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key > val).Value; - int blockIndex = nBlockHeight % this.unconfTxs.Count; - this.unconfTxs[blockIndex][bucketindex]++; - return bucketindex; - } + int periodsToConfirm = (blocksToConfirm + this.scale - 1) / this.scale; + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; - /// - /// Remove a transaction from mempool tracking stats. - /// - /// The height of the mempool entry. - /// The best sceen height. - /// The bucket index. - public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex) - { - //nBestSeenHeight is not updated yet for the new block - int blocksAgo = nBestSeenHeight - entryHeight; - if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet - blocksAgo = 0; - if (blocksAgo < 0) - { - this.logger.LogInformation($"Blockpolicy error, blocks ago is negative for mempool tx"); - return; //This can't happen because we call this with our best seen height, no entries can have higher - } + for (int i = periodsToConfirm; i <= this.confAvg.Count; i++) + this.confAvg[i - 1][bucketindex]++; - if (blocksAgo >= this.unconfTxs.Count) - { - if (this.oldUnconfTxs[bucketIndex] > 0) - this.oldUnconfTxs[bucketIndex]--; - else - { - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); - } - } - else - { - int blockIndex = entryHeight % this.unconfTxs.Count; - if (this.unconfTxs[blockIndex][bucketIndex] > 0) - this.unconfTxs[blockIndex][bucketIndex]--; - else - { - this.logger.LogInformation( - $"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); - } - } + this.txCtAvg[bucketindex]++; + this.avg[bucketindex] += val; } /// @@ -216,10 +181,14 @@ public void UpdateMovingAverages() { for (int j = 0; j < this.buckets.Count; j++) { - for (int i = 0; i < this.confAvg.Count; i++) - this.confAvg[i][j] = this.confAvg[i][j] * this.decay + this.curBlockConf[i][j]; - this.avg[j] = this.avg[j] * this.decay + this.curBlockVal[j]; - this.txCtAvg[j] = this.txCtAvg[j] * this.decay + this.curBlockTxCt[j]; + for (var i = 0; i < this.confAvg.Count; i++) + this.confAvg[i][j] = this.confAvg[i][j] * this.decay; + + for (var i = 0; i < this.failAvg.Count; i++) + this.failAvg[i][j] = this.failAvg[i][j] * this.decay; + + this.avg[j] = this.avg[j] * this.decay; + this.txCtAvg[j] = this.txCtAvg[j] * this.decay; } } @@ -233,15 +202,14 @@ public void UpdateMovingAverages() /// The success probability we require. /// Return the lowest feerate such that all higher values pass minSuccess OR return the highest feerate such that all lower values fail minSuccess. /// The current block height. - /// - public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, - bool requireGreater, - int nBlockHeight) + public double EstimateMedianVal(int confTarget, double sufficientTxVal, double successBreakPoint, bool requireGreater, int nBlockHeight, EstimationResult result) { // Counters for a bucket (or range of buckets) double nConf = 0; // Number of tx's confirmed within the confTarget double totalNum = 0; // Total number of tx's that were ever confirmed int extraNum = 0; // Number of tx's still in mempool for confTarget or longer + double failNum = 0; // Number of tx's that were never confirmed but removed from the mempool after confTarget + int periodTarget = (confTarget + this.scale - 1) / this.scale; int maxbucketindex = this.buckets.Count - 1; @@ -264,39 +232,75 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s bool foundAnswer = false; int bins = this.unconfTxs.Count; + bool newBucketRange = true; + bool passing = true; + var passBucket = new EstimatorBucket(); + var failBucket = new EstimatorBucket(); // Start counting from highest(default) or lowest feerate transactions for (int bucket = startbucket; bucket >= 0 && bucket <= maxbucketindex; bucket += step) { + if (newBucketRange) + { + curNearBucket = bucket; + newBucketRange = false; + } + curFarBucket = bucket; - nConf += this.confAvg[confTarget - 1][bucket]; + nConf += this.confAvg[periodTarget - 1][bucket]; totalNum += this.txCtAvg[bucket]; + failNum += this.failAvg[periodTarget - 1][bucket]; + for (int confct = confTarget; confct < this.GetMaxConfirms(); confct++) - extraNum += this.unconfTxs[(nBlockHeight - confct) % bins][bucket]; + extraNum += this.unconfTxs[Math.Abs(nBlockHeight - confct) % bins][bucket]; + extraNum += this.oldUnconfTxs[bucket]; + // If we have enough transaction data points in this range of buckets, // we can test for success // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) if (totalNum >= sufficientTxVal / (1 - this.decay)) { - double curPct = nConf / (totalNum + extraNum); + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate - if (requireGreater && curPct < successBreakPoint) - break; - if (!requireGreater && curPct > successBreakPoint) - break; - - // Otherwise update the cumulative stats, and the bucket variables - // and reset the counters - foundAnswer = true; - nConf = 0; - totalNum = 0; - extraNum = 0; - bestNearBucket = curNearBucket; - bestFarBucket = curFarBucket; - curNearBucket = bucket + step; + if ((requireGreater && curPct < successBreakPoint) || (!requireGreater && curPct > successBreakPoint)) + { + if (passing) + { + // First time we hit a failure record the failed bucket + int failMinBucket = Math.Min(curNearBucket, curFarBucket); + int failMaxBucket = Math.Max(curNearBucket, curFarBucket); + failBucket.Start = failMinBucket > 0 ? this.buckets[failMinBucket - 1] : 0; + failBucket.End = this.buckets[failMaxBucket]; + failBucket.WithinTarget = nConf; + failBucket.TotalConfirmed = totalNum; + failBucket.InMempool = extraNum; + failBucket.LeftMempool = failNum; + passing = false; + } + + continue; + } + else + { + // Otherwise update the cumulative stats, and the bucket variables and reset the counters + failBucket = new EstimatorBucket(); // Reset any failed bucket, currently passing + foundAnswer = true; + passing = true; + passBucket.WithinTarget = nConf; + nConf = 0; + passBucket.TotalConfirmed = totalNum; + totalNum = 0; + passBucket.InMempool = extraNum; + passBucket.LeftMempool = failNum; + failNum = 0; + extraNum = 0; + bestNearBucket = curNearBucket; + bestFarBucket = curFarBucket; + newBucketRange = true; + } } } @@ -307,13 +311,16 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s // Find the bucket with the median transaction and then report the average feerate from that bucket // This is a compromise between finding the median which we can't since we don't save all tx's // and reporting the average which is less accurate - int minBucket = bestNearBucket < bestFarBucket ? bestNearBucket : bestFarBucket; - int maxBucket = bestNearBucket > bestFarBucket ? bestNearBucket : bestFarBucket; + int minBucket = Math.Min(bestNearBucket, bestFarBucket); + int maxBucket = Math.Max(bestNearBucket, bestFarBucket); + for (int j = minBucket; j <= maxBucket; j++) txSum += this.txCtAvg[j]; + if (foundAnswer && txSum != 0) { txSum = txSum / 2; + for (int j = minBucket; j <= maxBucket; j++) { if (this.txCtAvg[j] < txSum) @@ -327,10 +334,43 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s break; } } + + passBucket.Start = minBucket > 0 ? this.buckets[minBucket - 1] : 0; + passBucket.End = this.buckets[maxBucket]; + } + + // If we were passing until we reached last few buckets with insufficient data, then report those as failed + if (passing && !newBucketRange) + { + int failMinBucket = Math.Min(curNearBucket, curFarBucket); + int failMaxBucket = Math.Max(curNearBucket, curFarBucket); + failBucket.Start = failMinBucket > 0 ? this.buckets[failMinBucket - 1] : 0; + failBucket.End = this.buckets[failMaxBucket]; + failBucket.WithinTarget = nConf; + failBucket.TotalConfirmed = totalNum; + failBucket.InMempool = extraNum; + failBucket.LeftMempool = failNum; } - this.logger.LogInformation( - $"{confTarget}: For conf success {(requireGreater ? $">" : $"<")} {successBreakPoint} need feerate {(requireGreater ? $">" : $"<")}: {median} from buckets {this.buckets[minBucket]} -{this.buckets[maxBucket]} Cur Bucket stats {100 * nConf / (totalNum + extraNum)} {nConf}/({totalNum}+{extraNum} mempool)"); + this.logger.LogDebug( + $"FeeEst: {confTarget} {(requireGreater ? $">" : $"<")} " + + $"{successBreakPoint} decay {this.decay} feerate: {median}" + + $" from ({passBucket.Start} - {passBucket.End}" + + $" {100 * passBucket.WithinTarget / (passBucket.TotalConfirmed + passBucket.InMempool + passBucket.LeftMempool)}" + + $" {passBucket.WithinTarget}/({passBucket.TotalConfirmed}" + + $" {passBucket.InMempool} mem {passBucket.LeftMempool} out) " + + $"Fail: ({failBucket.Start} - {failBucket.End} " + + $"{100 * failBucket.WithinTarget / (failBucket.TotalConfirmed + failBucket.InMempool + failBucket.LeftMempool)}" + + $" {failBucket.WithinTarget}/({failBucket.TotalConfirmed}" + + $" {failBucket.InMempool} mem {failBucket.LeftMempool} out)"); + + if (result != null) + { + result.Pass = passBucket; + result.Fail = failBucket; + result.Decay = this.decay; + result.Scale = this.scale; + } return median; } @@ -341,24 +381,144 @@ public double EstimateMedianVal(int confTarget, double sufficientTxVal, double s /// The max number of confirms. public int GetMaxConfirms() { - return this.confAvg.Count; + return this.scale * this.confAvg.Count; } /// /// Write state of estimation data to a file. /// /// Stream to write to. - public void Write(BitcoinStream stream) + public TxConfirmData Write() { + return new TxConfirmData + { + Decay = this.decay, + Scale = this.scale, + Avg = this.avg, + TxCtAvg = this.txCtAvg, + ConfAvg = this.confAvg, + FailAvg = this.failAvg + }; } /// - /// Read saved state of estimation data from a file and replace all internal data structures and - /// variables with this state. + /// Read saved state of estimation data from a file and replace all internal data structures and variables with this state. /// - /// Stream to read from. - public void Read(Stream filein) + public void Read(TxConfirmData data) { + // Read data file and do some very basic sanity checking. + // We presume that Initialize has been called prior to this, so that this.buckets is set. + try + { + if (data == null) + throw new ArgumentNullException(nameof(data)); + + if (data.Decay <= 0 || data.Decay >= 1) + throw new ApplicationException("Corrupt estimates file. Decay must be between 0 and 1 (non-inclusive)"); + + if (data.Scale == 0) + throw new ApplicationException("Corrupt estimates file. Scale must be non-zero"); + + if (data.Avg.Count != this.buckets.Count) + throw new ApplicationException("Corrupt estimates file. Mismatch in feerate average bucket count"); + + if (data.TxCtAvg.Count != this.buckets.Count) + throw new ApplicationException("Corrupt estimates file. Mismatch in tx count bucket count"); + + int maxPeriods = data.ConfAvg.Count; + double maxConfirms = data.Scale * maxPeriods; + + if (maxConfirms <= 0 || maxConfirms > 6 * 24 * 7) + throw new ApplicationException("Corrupt estimates file. Must maintain estimates for between 1 and 1008 (one week) confirms"); + + for (int i = 0; i < maxPeriods; i++) + { + if (data.ConfAvg[i].Count != this.buckets.Count) + throw new ApplicationException("Corrupt estimates file. Mismatch in feerate conf average bucket count"); + } + + if (maxPeriods != data.FailAvg.Count) + throw new ApplicationException("Corrupt estimates file. Mismatch in confirms tracked for failures"); + + for (int i = 0; i < maxPeriods; i++) + { + if (data.FailAvg[i].Count != this.buckets.Count) + throw new ApplicationException("Corrupt estimates file. Mismatch in one of failure average bucket counts"); + } + + this.decay = data.Decay; + this.scale = data.Scale; + this.avg = data.Avg; + this.txCtAvg = data.TxCtAvg; + this.confAvg = data.ConfAvg; + this.failAvg = data.FailAvg; + } + catch (Exception e) + { + this.logger.LogError("Error while reading tx confirm data from file", e); + } + } + + /// + /// Record a new transaction entering the mempool. + /// + /// The block height. + /// + /// The feerate of the transaction. + public int NewTx(int nBlockHeight, double val) + { + int bucketindex = this.bucketMap.FirstOrDefault(k => k.Key >= val).Value; + int blockIndex = nBlockHeight % this.unconfTxs.Count; + this.unconfTxs[blockIndex][bucketindex]++; + + return bucketindex; + } + + /// + /// Remove a transaction from mempool tracking stats. + /// + /// The height of the mempool entry. + /// The best sceen height. + /// The bucket index. + public void RemoveTx(int entryHeight, int nBestSeenHeight, int bucketIndex, bool inBlock) + { + //nBestSeenHeight is not updated yet for the new block + int blocksAgo = nBestSeenHeight - entryHeight; + + if (nBestSeenHeight == 0) // the BlockPolicyEstimator hasn't seen any blocks yet + blocksAgo = 0; + + if (blocksAgo < 0) + { + this.logger.LogDebug($"Blockpolicy error, blocks ago is negative for mempool tx"); + return; //This can't happen because we call this with our best seen height, no entries can have higher + } + + if (blocksAgo >= this.unconfTxs.Count) + { + if (this.oldUnconfTxs[bucketIndex] > 0) + this.oldUnconfTxs[bucketIndex]--; + else + this.logger.LogDebug($"Blockpolicy error, mempool tx removed from >25 blocks,bucketIndex={bucketIndex} already"); + } + else + { + int blockIndex = entryHeight % this.unconfTxs.Count; + + if (this.unconfTxs[blockIndex][bucketIndex] > 0) + this.unconfTxs[blockIndex][bucketIndex]--; + else + this.logger.LogDebug($"Blockpolicy error, mempool tx removed from blockIndex={blockIndex},bucketIndex={bucketIndex} already"); + } + + if (!inBlock && (blocksAgo >= this.scale)) // Only counts as a failure if not confirmed for entire period + { + Guard.Assert(this.scale != 0); + int periodsAgo = blocksAgo / this.scale; + + for (int i = 0; i < periodsAgo && i < this.failAvg.Count; i++) + this.failAvg[i][bucketIndex]++; + } } } } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs new file mode 100644 index 00000000000..56dc1909425 --- /dev/null +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Fee/TxStatsInfo.cs @@ -0,0 +1,23 @@ +namespace Stratis.Bitcoin.Features.MemoryPool.Fee +{ + /// + /// Transaction statistics information. + /// + public class TxStatsInfo + { + /// The block height. + public int blockHeight; + + /// The index into the confirmed transactions bucket map. + public int bucketIndex; + + /// + /// Constructs an instance of a transaction stats info object. + /// + public TxStatsInfo() + { + this.blockHeight = 0; + this.bucketIndex = 0; + } + } +} diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs index 059408f5662..bfd49ceecea 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/Interfaces/ITxMempool.cs @@ -11,7 +11,7 @@ namespace Stratis.Bitcoin.Features.MemoryPool.Interfaces public interface ITxMempool { /// Gets the miner policy estimator. - BlockPolicyEstimator MinerPolicyEstimator { get; } + IBlockPolicyEstimator MinerPolicyEstimator { get; } /// Get the number of transactions in the memory pool. long Size { get; } @@ -120,30 +120,7 @@ public interface ITxMempool /// The confirmation target blocks. /// The fee rate estimate. FeeRate EstimateFee(int nBlocks); - - /// - /// Estimates the priority using . - /// - /// The confirmation target blocks. - /// The estimated priority. - double EstimatePriority(int nBlocks); - - /// - /// Estimates the smart fee using . - /// - /// The confirmation target blocks. - /// The block where the fee was found. - /// The fee rate estimate. - FeeRate EstimateSmartFee(int nBlocks, out int answerFoundAtBlocks); - - /// - /// Estimates the smart priority using . - /// - /// The confirmation target blocks. - /// The block where the priority was found. - /// The estimated priority. - double EstimateSmartPriority(int nBlocks, out int answerFoundAtBlocks); - + /// /// Whether the transaction hash exists in the memory pool. /// @@ -195,7 +172,7 @@ public interface ITxMempool /// Read fee estimates from a stream. /// /// Stream to read from. - void ReadFeeEstimates(BitcoinStream stream); + void ReadFeeEstimates(); /// /// Called when a block is connected. Removes transactions from mempool and updates the miner fee estimator. @@ -240,7 +217,6 @@ public interface ITxMempool /// /// Write fee estimates to a stream. /// - /// Stream to write to. - void WriteFeeEstimates(BitcoinStream stream); + void WriteFeeEstimates(); } } \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs index ab27096055f..18a55474d67 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolFeature.cs @@ -142,12 +142,12 @@ public static class FullNodeBuilderMempoolExtension /// Include the memory pool feature and related services in the full node. /// /// Full node builder. - /// Whether or not to inject the mempool rules now, or defer it to another feature. /// Full node builder. public static IFullNodeBuilder UseMempool(this IFullNodeBuilder fullNodeBuilder) { + // TODO: Need to inject appropriate version of IBlockPolicyEstimator. Is it worth splitting this into two different extension methods? LoggingConfiguration.RegisterFeatureNamespace("mempool"); - LoggingConfiguration.RegisterFeatureNamespace("estimatefee"); + LoggingConfiguration.RegisterFeatureNamespace("estimatefee"); fullNodeBuilder.ConfigureFeature(features => { @@ -158,7 +158,7 @@ public static IFullNodeBuilder UseMempool(this IFullNodeBuilder fullNodeBuilder) { services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton() diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs index 136660bf0a8..182be0497eb 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/MempoolManager.cs @@ -279,6 +279,7 @@ internal async Task AddMempoolEntriesToMempoolAsync(IEnumerable toSave = memPool.MapTx.Values.ToArray().Select(tx => MempoolPersistenceEntry.FromTxMempoolEntry(tx)); return this.Save(network, toSave, fileName); } diff --git a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs index 30db87a48a8..f80cf9a6ef7 100644 --- a/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs +++ b/src/Stratis.Bitcoin.Features.MemoryPool/TxMemPool.cs @@ -182,7 +182,7 @@ public class TxMempool : ITxMempool /// The block policy estimator object. /// Factory for creating instance logger. /// Full node settings. - public TxMempool(IDateTimeProvider dateTimeProvider, BlockPolicyEstimator blockPolicyEstimator, ILoggerFactory loggerFactory, NodeSettings nodeSettings) + public TxMempool(IDateTimeProvider dateTimeProvider, IBlockPolicyEstimator blockPolicyEstimator, ILoggerFactory loggerFactory, NodeSettings nodeSettings) { this.MapTx = new IndexedTransactionSet(); this.mapLinks = new TxlinksMap(); @@ -203,7 +203,7 @@ public TxMempool(IDateTimeProvider dateTimeProvider, BlockPolicyEstimator blockP } /// Gets the miner policy estimator. - public BlockPolicyEstimator MinerPolicyEstimator { get; } + public IBlockPolicyEstimator MinerPolicyEstimator { get; } /// Get the number of transactions in the memory pool. public long Size @@ -280,24 +280,7 @@ public FeeRate EstimateFee(int nBlocks) { return this.MinerPolicyEstimator.EstimateFee(nBlocks); } - - /// - public FeeRate EstimateSmartFee(int nBlocks, out int answerFoundAtBlocks) - { - return this.MinerPolicyEstimator.EstimateSmartFee(nBlocks, this, out answerFoundAtBlocks); - } - - /// - public double EstimatePriority(int nBlocks) - { - return this.MinerPolicyEstimator.EstimatePriority(nBlocks); - } - - /// - public double EstimateSmartPriority(int nBlocks, out int answerFoundAtBlocks) - { - return this.MinerPolicyEstimator.EstimateSmartPriority(nBlocks, this, out answerFoundAtBlocks); - } + /// public void SetSanityCheck(double dFrequency = 1.0) @@ -706,7 +689,7 @@ private void RemoveUnchecked(TxMempoolEntry entry) this.mapLinks.Remove(entry); this.MapTx.Remove(entry); this.nTransactionsUpdated++; - this.MinerPolicyEstimator.RemoveTx(hash); + this.MinerPolicyEstimator.RemoveTx(hash, false); } /// @@ -1025,13 +1008,15 @@ public static bool AllowFree(double dPriority) } /// - public void WriteFeeEstimates(BitcoinStream stream) + public void WriteFeeEstimates() { + this.MinerPolicyEstimator.Write(); } /// - public void ReadFeeEstimates(BitcoinStream stream) + public void ReadFeeEstimates() { + this.MinerPolicyEstimator.Read(); } /// diff --git a/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs b/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs index ab56d021956..4fbc8a56816 100644 --- a/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs +++ b/src/Stratis.Bitcoin.Features.Wallet/WalletFeePolicy.cs @@ -1,6 +1,7 @@ using System; using NBitcoin; using Stratis.Bitcoin.Configuration; +using Stratis.Bitcoin.Features.MemoryPool.Fee; using Stratis.Bitcoin.Features.Wallet.Interfaces; namespace Stratis.Bitcoin.Features.Wallet @@ -33,17 +34,20 @@ public class WalletFeePolicy : IWalletFeePolicy /// private readonly FeeRate minRelayTxFee; + private readonly IBlockPolicyEstimator blockPolicyEstimator; + /// /// Constructs a wallet fee policy. /// /// Settings for the the node. - public WalletFeePolicy(NodeSettings nodeSettings) + public WalletFeePolicy(NodeSettings nodeSettings, IBlockPolicyEstimator blockPolicyEstimator) { this.minTxFee = nodeSettings.MinTxFeeRate; this.fallbackFee = nodeSettings.FallbackTxFeeRate; this.payTxFee = new FeeRate(0); this.maxTxFee = new Money(0.1M, MoneyUnit.BTC); this.minRelayTxFee = nodeSettings.MinRelayTxFeeRate; + this.blockPolicyEstimator = blockPolicyEstimator; } /// @@ -75,30 +79,33 @@ public Money GetMinimumFee(int txBytes, int confirmTarget) public Money GetMinimumFee(int txBytes, int confirmTarget, Money targetFee) { Money nFeeNeeded = targetFee; + // User didn't set: use -txconfirmtarget to estimate... if (nFeeNeeded == 0) { - int estimateFoundTarget = confirmTarget; - - // TODO: the fee estimation is not ready for release for now use the fall back fee - //nFeeNeeded = this.blockPolicyEstimator.EstimateSmartFee(confirmTarget, this.mempool, out estimateFoundTarget).GetFee(txBytes); + nFeeNeeded = this.blockPolicyEstimator.EstimateFee(confirmTarget).GetFee(txBytes); + // ... unless we don't have enough mempool data for estimatefee, then use fallbackFee if (nFeeNeeded == 0) nFeeNeeded = this.fallbackFee.GetFee(txBytes); } + // prevent user from paying a fee below minRelayTxFee or minTxFee nFeeNeeded = Math.Max(nFeeNeeded, this.GetRequiredFee(txBytes)); + // But always obey the maximum if (nFeeNeeded > this.maxTxFee) nFeeNeeded = this.maxTxFee; + return nFeeNeeded; } /// public FeeRate GetFeeRate(int confirmTarget) { - //this.blockPolicyEstimator.EstimateSmartFee(confirmTarget, this.mempool, out estimateFoundTarget).GetFee(txBytes); - return this.fallbackFee; + FeeRate feeRate = this.blockPolicyEstimator.EstimateFee(confirmTarget); + + return feeRate.FeePerK > 0 ? feeRate : this.fallbackFee; } } } diff --git a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs index 8c408d8334f..e0860ca55e0 100644 --- a/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs +++ b/src/Stratis.Bitcoin.IntegrationTests/MinerTests.cs @@ -205,7 +205,7 @@ public async Task InitializeAsync() }; this.DateTimeProvider = dateTimeProviderSet; - this.mempool = new TxMempool(dateTimeProvider, new BlockPolicyEstimator(new MempoolSettings(nodeSettings), loggerFactory, nodeSettings), loggerFactory, nodeSettings); + this.mempool = new TxMempool(dateTimeProvider, new BitcoinBlockPolicyEstimator(loggerFactory, nodeSettings), loggerFactory, nodeSettings); this.mempoolLock = new MempoolSchedulerLock(); // We can't make transactions until we have inputs diff --git a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs index d900569d776..02984021065 100644 --- a/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs +++ b/src/Stratis.Features.FederatedPeg.Tests/SignedMultisigTransactionBroadcasterTests.cs @@ -29,13 +29,11 @@ public class SignedMultisigTransactionBroadcasterTests : IDisposable private readonly IFederatedPegSettings federatedPegSettings; private readonly IBroadcasterManager broadcasterManager; - private readonly IAsyncProvider asyncProvider; - private readonly INodeLifetime nodeLifetime; private readonly MempoolManager mempoolManager; private readonly IDateTimeProvider dateTimeProvider; private readonly MempoolSettings mempoolSettings; private readonly NodeSettings nodeSettings; - private readonly BlockPolicyEstimator blockPolicyEstimator; + private readonly BitcoinBlockPolicyEstimator blockPolicyEstimator; private readonly TxMempool txMempool; private readonly IMempoolValidator mempoolValidator; private readonly IMempoolPersistence mempoolPersistence; @@ -55,8 +53,6 @@ public SignedMultisigTransactionBroadcasterTests() this.loggerFactory.CreateLogger(null).ReturnsForAnyArgs(this.logger); this.leaderReceiverSubscription = Substitute.For(); this.broadcasterManager = Substitute.For(); - this.asyncProvider = Substitute.For(); - this.nodeLifetime = Substitute.For(); this.ibdState = Substitute.For(); this.federationWalletManager = Substitute.For(); @@ -72,31 +68,15 @@ public SignedMultisigTransactionBroadcasterTests() MempoolExpiry = MempoolValidator.DefaultMempoolExpiry }; - this.blockPolicyEstimator = new BlockPolicyEstimator( - this.mempoolSettings, - this.loggerFactory, - this.nodeSettings); + this.blockPolicyEstimator = new BitcoinBlockPolicyEstimator(this.loggerFactory, this.nodeSettings); - this.txMempool = new TxMempool( - this.dateTimeProvider, - this.blockPolicyEstimator, - this.loggerFactory, - this.nodeSettings); + this.txMempool = new TxMempool(this.dateTimeProvider, this.blockPolicyEstimator, this.loggerFactory, this.nodeSettings); this.mempoolValidator = Substitute.For(); this.mempoolPersistence = Substitute.For(); this.coinView = Substitute.For(); - this.mempoolManager = new MempoolManager( - new MempoolSchedulerLock(), - this.txMempool, - this.mempoolValidator, - this.dateTimeProvider, - this.mempoolSettings, - this.mempoolPersistence, - this.coinView, - this.loggerFactory, - this.nodeSettings.Network); + this.mempoolManager = new MempoolManager(new MempoolSchedulerLock(), this.txMempool, this.mempoolValidator, this.dateTimeProvider, this.mempoolSettings, this.mempoolPersistence, this.coinView, this.loggerFactory, this.nodeSettings.Network); } [Fact] @@ -104,13 +84,7 @@ public async Task Call_GetSignedTransactionsAsync_Signed_Transactions_Broadcasts { this.federatedPegSettings.PublicKey.Returns(PublicKey); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); @@ -130,13 +104,7 @@ public async Task Dont_Do_Work_In_IBD() { this.ibdState.IsInitialBlockDownload().Returns(true); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); @@ -158,13 +126,7 @@ public async Task Dont_Do_Work_Inactive_Federation() this.ibdState.IsInitialBlockDownload().Returns(true); - using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster( - this.loggerFactory, - this.mempoolManager, - this.broadcasterManager, - this.ibdState, - this.federationWalletManager, - this.signals)) + using (var signedMultisigTransactionBroadcaster = new SignedMultisigTransactionBroadcaster(this.loggerFactory, this.mempoolManager, this.broadcasterManager, this.ibdState, this.federationWalletManager, this.signals)) { signedMultisigTransactionBroadcaster.Start(); diff --git a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs index b88f1e7a2ad..c441ba8505d 100644 --- a/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs +++ b/src/Stratis.SmartContracts.IntegrationTests/PoW/SmartContractMinerTests.cs @@ -269,7 +269,7 @@ public async Task InitializeAsync([CallerMemberName] string callingMethod = "") timeutc = DateTimeProvider.Default.GetUtcNow() }; - this.mempool = new TxMempool(dateTimeProviderSet, new BlockPolicyEstimator(new MempoolSettings(this.NodeSettings), this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); + this.mempool = new TxMempool(dateTimeProviderSet, new BitcoinBlockPolicyEstimator(this.loggerFactory, this.NodeSettings), this.loggerFactory, this.NodeSettings); this.mempoolLock = new MempoolSchedulerLock(); var blocks = new List();