From f54e2d743ff68d1c6092a1e134a68ac7cb4f0f9b Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 13:52:58 +0100 Subject: [PATCH 01/24] Log block height for balance changes --- src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index ac5fd7001..7fcce38fa 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -99,7 +99,7 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig if (amount > 0) { - logger.Info(() => $"Adding {payoutHandler.FormatAmount(amount)} to balance of {address} for {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares"); + logger.Info(() => $"Adding {payoutHandler.FormatAmount(amount)} to balance of {address} for {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares for block {block.BlockHeight}"); balanceRepo.AddAmount(con, tx, poolConfig.Id, poolConfig.Coin.Type, address, amount); } } From 29f1741154c830c0520bbaa59167fb71ad00fa90 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 14:04:25 +0100 Subject: [PATCH 02/24] Test --- src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index 7fcce38fa..a2d934505 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -111,7 +111,7 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig if (cutOffCount > 0) { - //LogObsoleteShares(poolConfig, block, shareCutOffDate.Value); + LogObsoleteShares(poolConfig, block, shareCutOffDate.Value); logger.Info(() => $"Deleting {cutOffCount} obsolete shares before {shareCutOffDate.Value}"); shareRepo.DeletePoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); @@ -135,6 +135,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var before = value; var pageSize = 50000; var currentPage = 0; + var lastId = (long?) null; var shares = new Dictionary(); while (true) @@ -144,6 +145,8 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); +logger.Info(() => $"page {currentPage}: {before} {lastId} {blockPage.Length} {(blockPage.Length > 0 ? blockPage[0].Id : 0)}"); + currentPage++; var start = blockPage.Length > 0 ? blockPage.Length - 1 : -1; @@ -175,7 +178,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var addressesByShares = shares.Keys.OrderByDescending(x => shares[x]); // compute summary - var summary = string.Join("\n", addressesByShares.Select(address => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares")); + var summary = string.Join("\n", addressesByShares.Select(address => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares for block {block.BlockHeight}")); logger.Info(() => $"{FormatUtil.FormatQuantity(shares.Values.Sum())} ({shares.Values.Sum()}) obsolete shares:\n" + summary); } From f3a61efa970f5cf5037c692fb1023f725fff9f0c Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 15:05:43 +0100 Subject: [PATCH 03/24] WIP --- src/MiningCore/Payments/PayoutManager.cs | 2 +- src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MiningCore/Payments/PayoutManager.cs b/src/MiningCore/Payments/PayoutManager.cs index 25886b802..fc2cde297 100644 --- a/src/MiningCore/Payments/PayoutManager.cs +++ b/src/MiningCore/Payments/PayoutManager.cs @@ -157,7 +157,7 @@ await cf.RunTxAsync(async (con, tx) => } else - logger.Info(() => $"No updated blocks for {pool.Id}"); + logger.Info(() => $"No updated blocks for pool {pool.Id}"); } private async Task PayoutPoolBalancesAsync(PoolConfig pool, IPayoutHandler handler) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index a2d934505..1490fa4b1 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -145,7 +145,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); -logger.Info(() => $"page {currentPage}: {before} {lastId} {blockPage.Length} {(blockPage.Length > 0 ? blockPage[0].Id : 0)}"); +logger.Info(() => $"page {currentPage}: {before:O} {lastId} {blockPage.Length} {(blockPage.Length > 0 ? blockPage[0].Id : 0)}"); currentPage++; var start = blockPage.Length > 0 ? blockPage.Length - 1 : -1; From 528741554f1d6844c5d2f3134a4ab4a701e6ae6b Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 15:37:14 +0100 Subject: [PATCH 04/24] Fix share traversal --- .../Payments/PayoutSchemes/PayPerLastNShares.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index 1490fa4b1..a7eb0dd6e 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -135,7 +135,6 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var before = value; var pageSize = 50000; var currentPage = 0; - var lastId = (long?) null; var shares = new Dictionary(); while (true) @@ -145,12 +144,12 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); -logger.Info(() => $"page {currentPage}: {before:O} {lastId} {blockPage.Length} {(blockPage.Length > 0 ? blockPage[0].Id : 0)}"); + if (blockPage.Length == 0) + break; currentPage++; - var start = blockPage.Length > 0 ? blockPage.Length - 1 : -1; - for (var i = start; i >= 0; i--) + for (var i = 0;i < blockPage.Length; i++) { var share = blockPage[i]; @@ -169,7 +168,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu if (blockPage.Length < pageSize) break; - before = blockPage[0].Created; + before = blockPage[blockPage.Length - 1].Created; } if (shares.Keys.Count > 0) @@ -212,9 +211,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu inclusive = false; currentPage++; - var start = blockPage.Length > 0 ? blockPage.Length - 1 : -1; - - for (var i = start; !done && i >= 0; i--) + for (var i = 0; !done && i < blockPage.Length; i++) { var share = blockPage[i]; @@ -261,7 +258,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu if (blockPage.Length < pageSize) break; - before = blockPage[0].Created; + before = blockPage[blockPage.Length - 1].Created; } logger.Info(() => $"Balance-calculation for pool {poolConfig.Id}, block {block.BlockHeight} completed with accumulated score {accumulatedScore:0.####} ({(accumulatedScore / window) * 100:0.#}%)"); From a5f614dc1607bd6c07a17852734d75218dcb42b7 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 15:47:49 +0100 Subject: [PATCH 05/24] Log one address per line --- src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index a7eb0dd6e..e26ca651a 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -176,10 +176,10 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu // sort addresses by shares var addressesByShares = shares.Keys.OrderByDescending(x => shares[x]); - // compute summary - var summary = string.Join("\n", addressesByShares.Select(address => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares for block {block.BlockHeight}")); + logger.Info(() => $"{FormatUtil.FormatQuantity(shares.Values.Sum())} ({shares.Values.Sum()}) obsolete shares total for block {block.BlockHeight}"); - logger.Info(() => $"{FormatUtil.FormatQuantity(shares.Values.Sum())} ({shares.Values.Sum()}) obsolete shares:\n" + summary); + foreach (var address in addressesByShares) + logger.Info(() => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares for block {block.BlockHeight}"); } } From 8b7c7852bf0b92a1caf41d3a0a1801706f597dea Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 16:33:10 +0100 Subject: [PATCH 06/24] Remove unnecessary checks --- src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index e26ca651a..576323eb6 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -144,9 +144,6 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); - if (blockPage.Length == 0) - break; - currentPage++; for (var i = 0;i < blockPage.Length; i++) @@ -205,9 +202,6 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, inclusive, pageSize))); //, sw, logger)); - if (blockPage.Length == 0) - break; - inclusive = false; currentPage++; From 93b6bcffdfe0bc961b08f8000aee519adf87268c Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 17:47:23 +0100 Subject: [PATCH 07/24] WIP --- src/MiningCore/Mining/Abstractions.cs | 1 + src/MiningCore/Mining/PoolStatsUpdater.cs | 172 ++++++++++++++++++ .../PayoutSchemes/PayPerLastNShares.cs | 12 +- .../Postgres/Repositories/ShareRepository.cs | 13 ++ .../Repositories/IShareRepository.cs | 1 + src/MiningCore/Program.cs | 12 +- 6 files changed, 202 insertions(+), 9 deletions(-) create mode 100644 src/MiningCore/Mining/PoolStatsUpdater.cs diff --git a/src/MiningCore/Mining/Abstractions.cs b/src/MiningCore/Mining/Abstractions.cs index 835995fdd..dc85e2bef 100644 --- a/src/MiningCore/Mining/Abstractions.cs +++ b/src/MiningCore/Mining/Abstractions.cs @@ -22,6 +22,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; using MiningCore.Blockchain; using MiningCore.Configuration; +using MiningCore.Persistence.Model; using MiningCore.Stratum; namespace MiningCore.Mining diff --git a/src/MiningCore/Mining/PoolStatsUpdater.cs b/src/MiningCore/Mining/PoolStatsUpdater.cs new file mode 100644 index 000000000..96bde1b0e --- /dev/null +++ b/src/MiningCore/Mining/PoolStatsUpdater.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using Autofac; +using MiningCore.Configuration; +using MiningCore.Contracts; +using MiningCore.Extensions; +using MiningCore.Payments; +using MiningCore.Persistence; +using MiningCore.Persistence.Repositories; +using MiningCore.Time; +using NLog; +using Polly; + +namespace MiningCore.Mining +{ + public class PoolStatsUpdater + { + public PoolStatsUpdater(IComponentContext ctx, + IMasterClock clock, + IConnectionFactory cf, + IShareRepository shareRepo, + IStatsRepository statsRepo) + { + Contract.RequiresNonNull(ctx, nameof(ctx)); + Contract.RequiresNonNull(clock, nameof(clock)); + Contract.RequiresNonNull(cf, nameof(cf)); + Contract.RequiresNonNull(shareRepo, nameof(shareRepo)); + Contract.RequiresNonNull(statsRepo, nameof(statsRepo)); + + this.ctx = ctx; + this.clock = clock; + this.cf = cf; + this.shareRepo = shareRepo; + this.statsRepo = statsRepo; + + BuildFaultHandlingPolicy(); + } + + private readonly IMasterClock clock; + private readonly IStatsRepository statsRepo; + private readonly IConnectionFactory cf; + private readonly IComponentContext ctx; + private readonly IShareRepository shareRepo; + private readonly AutoResetEvent stopEvent = new AutoResetEvent(false); + private readonly Dictionary pools = new Dictionary(); + private ClusterConfig clusterConfig; + private Thread thread; + private const int RetryCount = 4; + private Policy shareReadFaultPolicy; + + private static readonly ILogger logger = LogManager.GetCurrentClassLogger(); + + #region API-Surface + + public void Configure(ClusterConfig clusterConfig) + { + this.clusterConfig = clusterConfig; + } + + public void AttachPool(IMiningPool pool) + { + pools[pool.Config.Id] = pool; + } + + public void Start() + { + thread = new Thread(async () => + { + logger.Info(() => "Online"); + + var interval = TimeSpan.FromSeconds( + clusterConfig.PaymentProcessing.Interval > 0 ? clusterConfig.PaymentProcessing.Interval : 600); + + while (true) + { + try + { + await UpdatePoolsAsync(); + } + + catch (Exception ex) + { + logger.Error(ex); + } + + var waitResult = stopEvent.WaitOne(interval); + + // check if stop was signalled + if (waitResult) + break; + } + }); + + thread.Name = "Pool Stats Updater"; + thread.Start(); + } + + public void Stop() + { + logger.Info(() => "Stopping .."); + + stopEvent.Set(); + thread.Join(); + + logger.Info(() => "Stopped"); + } + + #endregion // API-Surface + + private Task UpdatePoolsAsync() + { + UpdateHashrates(); + + return Task.FromResult(true); + } + + private void UpdateHashrates() + { + var start = clock.Now; + var target = start.AddMinutes(-10); + var pageSize = 50000; + var currentPage = 0; + + foreach (var poolId in pools.Keys) + { + logger.Info(() => $"Updating hashrates for pool {poolId}"); + + var before = start; + var pool = pools[poolId]; + var accumulated = 0d; + + while (true) + { + logger.Info(() => $"Fetching page {currentPage} of shares for pool {poolId}"); + + var blockPage = shareReadFaultPolicy.Execute(() => + cf.Run(con => shareRepo.ReadSharesBeforeAndAfterCreated(con, poolId, before, target, true, pageSize))); + + currentPage++; + + // accumulate per pool, miner and worker + // accumulated += pool.HashrateAccumulate(blockPage); + + if (blockPage.Length < pageSize) + break; + + before = blockPage[blockPage.Length - 1].Created; + } + } + } + + private void BuildFaultHandlingPolicy() + { + var retry = Policy + .Handle() + .Or() + .Or() + .Retry(RetryCount, OnPolicyRetry); + + shareReadFaultPolicy = retry; + } + + private static void OnPolicyRetry(Exception ex, int retry, object context) + { + logger.Warn(() => $"Retry {retry} due to {ex.Source}: {ex.GetType().Name} ({ex.Message})"); + } + } +} diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index 576323eb6..e60d86061 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -104,7 +104,7 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig } } - // delete obsolete shares + // delete discarded shares if (shareCutOffDate.HasValue) { var cutOffCount = shareRepo.CountPoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); @@ -113,11 +113,9 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig { LogObsoleteShares(poolConfig, block, shareCutOffDate.Value); - logger.Info(() => $"Deleting {cutOffCount} obsolete shares before {shareCutOffDate.Value}"); + logger.Info(() => $"Deleting {cutOffCount} discarded shares before {shareCutOffDate.Value:O}"); shareRepo.DeletePoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); } - - //logger.Info(() => $"Shares before {shareCutOffDate.Value} can be deleted"); } // diagnostics @@ -139,7 +137,7 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu while (true) { - logger.Info(() => $"Fetching page {currentPage} of obsolete shares for pool {poolConfig.Id}, block {block.BlockHeight}"); + logger.Info(() => $"Fetching page {currentPage} of discarded shares for pool {poolConfig.Id}, block {block.BlockHeight}"); var blockPage = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); @@ -173,10 +171,10 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu // sort addresses by shares var addressesByShares = shares.Keys.OrderByDescending(x => shares[x]); - logger.Info(() => $"{FormatUtil.FormatQuantity(shares.Values.Sum())} ({shares.Values.Sum()}) obsolete shares total for block {block.BlockHeight}"); + logger.Info(() => $"{FormatUtil.FormatQuantity(shares.Values.Sum())} ({shares.Values.Sum()}) total discarded shares, block {block.BlockHeight}"); foreach (var address in addressesByShares) - logger.Info(() => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) shares for block {block.BlockHeight}"); + logger.Info(() => $"{address} = {FormatUtil.FormatQuantity(shares[address])} ({shares[address]}) discarded shares, block {block.BlockHeight}"); } } diff --git a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs index 79b2a5eec..710361b7d 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs @@ -67,6 +67,19 @@ public Share[] ReadSharesBeforeCreated(IDbConnection con, string poolId, DateTim .ToArray(); } + public Share[] ReadSharesBeforeAndAfterCreated(IDbConnection con, string poolId, DateTime before, DateTime after, bool inclusive, int pageSize) + { + logger.LogInvoke(new[] { poolId }); + + var query = $"SELECT * FROM shares WHERE poolid = @poolId AND created {(inclusive ? " <= " : " < ")} @before " + + $"AND created {(inclusive ? " >= " : " > ")} @after" + + "ORDER BY created DESC FETCH NEXT (@pageSize) ROWS ONLY"; + + return con.Query(query, new { poolId, before, after, pageSize }) + .Select(mapper.Map) + .ToArray(); + } + public Share[] PageSharesBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end, int page, int pageSize) { logger.LogInvoke(new[] { poolId }); diff --git a/src/MiningCore/Persistence/Repositories/IShareRepository.cs b/src/MiningCore/Persistence/Repositories/IShareRepository.cs index 533026184..5227a0f53 100644 --- a/src/MiningCore/Persistence/Repositories/IShareRepository.cs +++ b/src/MiningCore/Persistence/Repositories/IShareRepository.cs @@ -28,6 +28,7 @@ public interface IShareRepository { void Insert(IDbConnection con, IDbTransaction tx, Share share); Share[] ReadSharesBeforeCreated(IDbConnection con, string poolId, DateTime before, bool inclusive, int pageSize); + Share[] ReadSharesBeforeAndAfterCreated(IDbConnection con, string poolId, DateTime before, DateTime after, bool inclusive, int pageSize); Share[] PageSharesBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end, int page, int pageSize); long CountPoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before); diff --git a/src/MiningCore/Program.cs b/src/MiningCore/Program.cs index b894190d2..18ee877c5 100644 --- a/src/MiningCore/Program.cs +++ b/src/MiningCore/Program.cs @@ -70,6 +70,7 @@ public class Program private static CommandOption shareRecoveryOption; private static ShareRecorder shareRecorder; private static PayoutManager payoutManager; + private static PoolStatsUpdater poolStatsUpdater; private static ClusterConfig clusterConfig; private static ApiServer apiServer; @@ -545,8 +546,12 @@ private static async Task Start() else logger.Info("Payment processing is not enabled"); - // start pools - await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolConfig => + // start pool stats updater + poolStatsUpdater = container.Resolve(); + poolStatsUpdater.Configure(clusterConfig); + + // start pools + await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolConfig => { // resolve pool implementation var poolImpl = container.Resolve>>>() @@ -563,8 +568,11 @@ await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolCo // post-start attachments apiServer.AttachPool(pool); + poolStatsUpdater.AttachPool(pool); })); + poolStatsUpdater.Start(); + // keep running await Observable.Never().ToTask(); } From b4cf7f8848fa1f6c0fc7b31010e8c6e52d41fc4c Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 19 Dec 2017 18:00:30 +0100 Subject: [PATCH 08/24] WIP --- src/MiningCore/Mining/PoolStatsUpdater.cs | 44 +++++++++++++++++-- .../PayoutSchemes/PayPerLastNShares.cs | 24 +++++----- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/MiningCore/Mining/PoolStatsUpdater.cs b/src/MiningCore/Mining/PoolStatsUpdater.cs index 96bde1b0e..7fd2d44c4 100644 --- a/src/MiningCore/Mining/PoolStatsUpdater.cs +++ b/src/MiningCore/Mining/PoolStatsUpdater.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Data.Common; +using System.Linq; using System.Net.Sockets; using System.Threading; using System.Threading.Tasks; @@ -10,6 +11,7 @@ using MiningCore.Extensions; using MiningCore.Payments; using MiningCore.Persistence; +using MiningCore.Persistence.Model; using MiningCore.Persistence.Repositories; using MiningCore.Time; using NLog; @@ -137,18 +139,54 @@ private void UpdateHashrates() { logger.Info(() => $"Fetching page {currentPage} of shares for pool {poolId}"); - var blockPage = shareReadFaultPolicy.Execute(() => + var page = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeAndAfterCreated(con, poolId, before, target, true, pageSize))); currentPage++; + var sharesByMiner = page.GroupBy(x => x.Miner).ToArray(); + + foreach (var minerShares in sharesByMiner) + { + // Total hashrate + var miner = minerShares.Key; + var hashRate = HashrateFromShares(minerShares, interval); + + var sample = new MinerHashrateSample + { + PoolId = poolConfig.Id, + Miner = miner, + Hashrate = hashRate, + Created = clock.Now + }; + + // Per worker hashrates + var sharesPerWorker = minerShares + .GroupBy(x => x.Share.Worker) + .Where(x => !string.IsNullOrEmpty(x.Key)); + + foreach (var workerShares in sharesPerWorker) + { + var worker = workerShares.Key; + hashRate = HashrateFromShares(workerShares, interval); + + if (sample.WorkerHashrates == null) + sample.WorkerHashrates = new Dictionary(); + + sample.WorkerHashrates[worker] = hashRate; + } + + // Persist + cf.RunTx((con, tx) => { statsRepo.RecordMinerHashrateSample(con, tx, sample); }); + } + // accumulate per pool, miner and worker // accumulated += pool.HashrateAccumulate(blockPage); - if (blockPage.Length < pageSize) + if (page.Length < pageSize) break; - before = blockPage[blockPage.Length - 1].Created; + before = page[page.Length - 1].Created; } } } diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs index e60d86061..bb549acfc 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs @@ -111,7 +111,7 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig if (cutOffCount > 0) { - LogObsoleteShares(poolConfig, block, shareCutOffDate.Value); + LogDiscardedShares(poolConfig, block, shareCutOffDate.Value); logger.Info(() => $"Deleting {cutOffCount} discarded shares before {shareCutOffDate.Value:O}"); shareRepo.DeletePoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); @@ -128,7 +128,7 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig return Task.FromResult(true); } - private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime value) + private void LogDiscardedShares(PoolConfig poolConfig, Block block, DateTime value) { var before = value; var pageSize = 50000; @@ -139,14 +139,14 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu { logger.Info(() => $"Fetching page {currentPage} of discarded shares for pool {poolConfig.Id}, block {block.BlockHeight}"); - var blockPage = shareReadFaultPolicy.Execute(() => + var page = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, false, pageSize))); currentPage++; - for (var i = 0;i < blockPage.Length; i++) + for (var i = 0;i < page.Length; i++) { - var share = blockPage[i]; + var share = page[i]; // build address var address = share.Miner; @@ -160,10 +160,10 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu shares[address] += share.Difficulty; } - if (blockPage.Length < pageSize) + if (page.Length < pageSize) break; - before = blockPage[blockPage.Length - 1].Created; + before = page[page.Length - 1].Created; } if (shares.Keys.Count > 0) @@ -197,15 +197,15 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu { logger.Info(() => $"Fetching page {currentPage} of shares for pool {poolConfig.Id}, block {block.BlockHeight}"); - var blockPage = shareReadFaultPolicy.Execute(() => + var page = shareReadFaultPolicy.Execute(() => cf.Run(con => shareRepo.ReadSharesBeforeCreated(con, poolConfig.Id, before, inclusive, pageSize))); //, sw, logger)); inclusive = false; currentPage++; - for (var i = 0; !done && i < blockPage.Length; i++) + for (var i = 0; !done && i < page.Length; i++) { - var share = blockPage[i]; + var share = page[i]; // build address var address = share.Miner; @@ -247,10 +247,10 @@ private void LogObsoleteShares(PoolConfig poolConfig, Block block, DateTime valu } } - if (blockPage.Length < pageSize) + if (page.Length < pageSize) break; - before = blockPage[blockPage.Length - 1].Created; + before = page[page.Length - 1].Created; } logger.Info(() => $"Balance-calculation for pool {poolConfig.Id}, block {block.BlockHeight} completed with accumulated score {accumulatedScore:0.####} ({(accumulatedScore / window) * 100:0.#}%)"); From a4958ba648a14f5022a5876895da86d766565829 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Wed, 27 Dec 2017 21:22:06 +0100 Subject: [PATCH 09/24] - Compute miner/worker hashrates from database instead of buffering shares in memory - PPLNS fixes - ZCash shield coinbase fix --- src/MiningCore/Api/ApiServer.cs | 20 +-- .../Api/Responses/GetMinerStatsResponse.cs | 2 +- src/MiningCore/AutoMapperProfile.cs | 8 ++ src/MiningCore/AutofacModule.cs | 6 +- .../Blockchain/Bitcoin/BitcoinPoolBase.cs | 50 +------ .../Blockchain/Ethereum/EthereumPool.cs | 49 +------ .../Blockchain/Monero/MoneroPool.cs | 49 +------ .../Blockchain/ZCash/ZCashPayoutHandler.cs | 92 +----------- .../Blockchain/ZCash/ZCashPoolBase.cs | 6 +- src/MiningCore/Mining/Abstractions.cs | 2 +- src/MiningCore/Mining/PoolBase.cs | 131 ++---------------- .../{PoolStatsUpdater.cs => StatsRecorder.cs} | 119 ++++++++-------- src/MiningCore/MiningCore.csproj | 2 +- src/MiningCore/Payments/PayoutManager.cs | 5 - .../{PayPerLastNShares.cs => PPLNS.cs} | 14 +- .../Repositories/StatsRepositoryBase.cs | 55 -------- .../Persistence/Model/MinerHashrateSample.cs | 15 -- .../Model/MinerWorkerPerformanceStats.cs | 14 ++ .../Model/{ => Projections}/MinerStats.cs | 6 +- .../Model/Projections/MinerWorkerHashes.cs | 34 +++++ .../Entities/MinerWorkerPerformanceStats.cs | 35 +++++ .../Postgres/Repositories/BlockRepository.cs | 6 +- .../Postgres/Repositories/ShareRepository.cs | 21 ++- .../Postgres/Repositories/StatsRepository.cs | 44 ++++-- .../Persistence/Postgres/Scripts/createdb.sql | 18 ++- .../Repositories/IShareRepository.cs | 8 +- .../Repositories/IStatsRepository.cs | 8 +- src/MiningCore/Program.cs | 10 +- src/MiningCore/Stratum/StratumServer.cs | 5 +- src/MiningCore/Util/FormatUtil.cs | 2 +- 30 files changed, 296 insertions(+), 540 deletions(-) rename src/MiningCore/Mining/{PoolStatsUpdater.cs => StatsRecorder.cs} (54%) rename src/MiningCore/Payments/PayoutSchemes/{PayPerLastNShares.cs => PPLNS.cs} (94%) delete mode 100644 src/MiningCore/Persistence/Common/Repositories/StatsRepositoryBase.cs delete mode 100644 src/MiningCore/Persistence/Model/MinerHashrateSample.cs create mode 100644 src/MiningCore/Persistence/Model/MinerWorkerPerformanceStats.cs rename src/MiningCore/Persistence/Model/{ => Projections}/MinerStats.cs (89%) create mode 100644 src/MiningCore/Persistence/Model/Projections/MinerWorkerHashes.cs create mode 100644 src/MiningCore/Persistence/Postgres/Entities/MinerWorkerPerformanceStats.cs diff --git a/src/MiningCore/Api/ApiServer.cs b/src/MiningCore/Api/ApiServer.cs index 938c68083..ff90b14d5 100644 --- a/src/MiningCore/Api/ApiServer.cs +++ b/src/MiningCore/Api/ApiServer.cs @@ -20,6 +20,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System; using System.Collections.Generic; +using System.Data; using System.IO; using System.Linq; using System.Net; @@ -35,14 +36,12 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using MiningCore.Api.Extensions; using MiningCore.Api.Responses; using MiningCore.Blockchain; -using MiningCore.Buffers; using MiningCore.Configuration; using MiningCore.Extensions; using MiningCore.Mining; using MiningCore.Persistence; using MiningCore.Persistence.Model; using MiningCore.Persistence.Repositories; -using MiningCore.Stratum; using MiningCore.Time; using MiningCore.Util; using Newtonsoft.Json; @@ -171,7 +170,7 @@ private IMiningPool GetPool(HttpContext context, Match m) lock(pools) { var pool = pools.FirstOrDefault(x => x.Config.Id == poolId); - + if (pool != null) return pool; } @@ -206,7 +205,7 @@ private async Task HandleGetPoolAsync(HttpContext context, Match m) { Pool = pool.ToPoolInfo(mapper) }; - + await SendJson(context, response); } @@ -247,7 +246,7 @@ private async Task HandleGetBlocksPagedAsync(HttpContext context, Match m) } var blocks = cf.Run(con => blocksRepo.PageBlocks(con, pool.Config.Id, - new[] { BlockStatus.Confirmed, BlockStatus.Pending }, page, pageSize)) + new[] { BlockStatus.Confirmed, BlockStatus.Pending, BlockStatus.Orphaned }, page, pageSize)) .Select(mapper.Map) .ToArray(); @@ -320,12 +319,15 @@ private async Task HandleGetMinerStatsAsync(HttpContext context, Match m) return; } - var statsResult = cf.Run(con => statsRepo.GetMinerStats(con, pool.Config.Id, address)); - Responses.MinerStats stats = null; + var statsResult = cf.RunTx((con, tx) => + statsRepo.GetMinerStats(con, tx, pool.Config.Id, address), + true, IsolationLevel.Serializable); + + MinerStats stats = null; if (statsResult != null) { - stats = mapper.Map(statsResult); + stats = mapper.Map(statsResult); // optional fields if (statsResult.LastPayment != null) @@ -393,6 +395,6 @@ public void AttachPool(IMiningPool pool) } #endregion // API-Surface - + } } diff --git a/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs b/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs index 9422a2b7d..6417cbff5 100644 --- a/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs +++ b/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs @@ -30,6 +30,6 @@ public class MinerStats public decimal TotalPaid { get; set; } public DateTime? LastPayment { get; set; } public string LastPaymentLink { get; set; } - public MinerHashrateSample[] Hashrate { get; set; } + public MinerWorkerPerformanceStats[] PerformanceStats { get; set; } } } diff --git a/src/MiningCore/AutoMapperProfile.cs b/src/MiningCore/AutoMapperProfile.cs index 7d5348ccd..cbe6e2196 100644 --- a/src/MiningCore/AutoMapperProfile.cs +++ b/src/MiningCore/AutoMapperProfile.cs @@ -22,6 +22,8 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using MiningCore.Blockchain; using MiningCore.Configuration; using MiningCore.Persistence.Model; +using MiningCore.Persistence.Model.Projections; +using MinerStats = MiningCore.Persistence.Model.Projections.MinerStats; namespace MiningCore { @@ -63,6 +65,9 @@ public AutoMapperProfile() CreateMap(); CreateMap(); + CreateMap() + .ForMember(dest => dest.Id, opt => opt.Ignore()); + ////////////////////// // incoming mappings @@ -72,6 +77,9 @@ public AutoMapperProfile() CreateMap(); CreateMap(); CreateMap(); + CreateMap(); + + CreateMap(); } } } diff --git a/src/MiningCore/AutofacModule.cs b/src/MiningCore/AutofacModule.cs index 3a503b192..4172aecd7 100644 --- a/src/MiningCore/AutofacModule.cs +++ b/src/MiningCore/AutofacModule.cs @@ -35,7 +35,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using MiningCore.Blockchain.ZCash; using MiningCore.Blockchain.ZCash.DaemonResponses; using MiningCore.Configuration; -using MiningCore.JsonRpc; using MiningCore.Mining; using MiningCore.Notifications; using MiningCore.Payments; @@ -83,6 +82,9 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType() .SingleInstance(); + builder.RegisterType() + .AsSelf(); + builder.RegisterType() .SingleInstance(); @@ -98,7 +100,7 @@ protected override void Load(ContainerBuilder builder) ////////////////////// // Payment Schemes - builder.RegisterType() + builder.RegisterType() .Keyed(PayoutScheme.PPLNS) .SingleInstance(); diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs index 174012a92..0699478d9 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs @@ -343,58 +343,16 @@ protected override async Task OnRequestAsync(StratumClient client, } } - protected override void SetupStats() + public override ulong HashrateFromShares(double shares, double interval) { - base.SetupStats(); - - // Pool Hashrate - var poolHashRateSampleIntervalSeconds = 60 * 10; - - disposables.Add(Shares - .ObserveOn(ThreadPoolScheduler.Instance) - .Buffer(TimeSpan.FromSeconds(poolHashRateSampleIntervalSeconds)) - .Do(shares => UpdateMinerHashrates(shares, poolHashRateSampleIntervalSeconds)) - .Select(shares => - { - if (!shares.Any()) - return 0ul; - - try - { - return HashrateFromShares(shares, poolHashRateSampleIntervalSeconds); - } - - catch(Exception ex) - { - logger.Error(ex); - return 0ul; - } - }) - .Subscribe(hashRate => poolStats.PoolHashRate = hashRate)); - - // shares/sec - disposables.Add(Shares - .Buffer(TimeSpan.FromSeconds(1)) - .Do(shares => - { - poolStats.ValidSharesPerSecond = shares.Count; - - logger.Debug(() => $"[{LogCat}] Share/sec = {poolStats.ValidSharesPerSecond}"); - }) - .Subscribe()); - } - - protected override ulong HashrateFromShares(IEnumerable shares, int interval) - { - var sum = shares.Sum(share => Math.Max(0.00000001, share.Share.Difficulty)); var multiplier = BitcoinConstants.Pow2x32 / manager.ShareMultiplier; - var result = Math.Ceiling(sum * multiplier / interval); + var result = Math.Ceiling(shares * multiplier / interval); // OW: tmp hotfix if (poolConfig.Coin.Type == CoinType.MONA || poolConfig.Coin.Type == CoinType.VTC) result *= 1.3; - - return (ulong) result; + Console.WriteLine(result); + return (ulong)result; } protected override void OnVarDiffUpdate(StratumClient client, double newDiff) diff --git a/src/MiningCore/Blockchain/Ethereum/EthereumPool.cs b/src/MiningCore/Blockchain/Ethereum/EthereumPool.cs index 9b9d21de4..7e379719e 100644 --- a/src/MiningCore/Blockchain/Ethereum/EthereumPool.cs +++ b/src/MiningCore/Blockchain/Ethereum/EthereumPool.cs @@ -309,52 +309,11 @@ protected override async Task OnRequestAsync(StratumClient client, } } - protected override void SetupStats() + public override ulong HashrateFromShares(double shares, double interval) { - base.SetupStats(); - - // Pool Hashrate - var poolHashRateSampleIntervalSeconds = 60 * 10; - - disposables.Add(Shares - .ObserveOn(ThreadPoolScheduler.Instance) - .Buffer(TimeSpan.FromSeconds(poolHashRateSampleIntervalSeconds)) - .Do(shares => UpdateMinerHashrates(shares, poolHashRateSampleIntervalSeconds)) - .Select(shares => - { - if (!shares.Any()) - return 0ul; - - try - { - return HashrateFromShares(shares, poolHashRateSampleIntervalSeconds); - } - - catch(Exception ex) - { - logger.Error(ex); - return 0ul; - } - }) - .Subscribe(hashRate => poolStats.PoolHashRate = hashRate)); - - // shares/sec - disposables.Add(Shares - .Buffer(TimeSpan.FromSeconds(1)) - .Do(shares => - { - poolStats.ValidSharesPerSecond = shares.Count; - - logger.Debug(() => $"[{LogCat}] Share/sec = {poolStats.ValidSharesPerSecond}"); - }) - .Subscribe()); - } - - protected override ulong HashrateFromShares(IEnumerable shares, int interval) - { - var result = Math.Ceiling(shares.Sum(share => share.Share.Difficulty) / interval); - return (ulong) result; - } + var result = Math.Ceiling(shares / interval); + return (ulong)result; + } protected override void OnVarDiffUpdate(StratumClient client, double newDiff) { diff --git a/src/MiningCore/Blockchain/Monero/MoneroPool.cs b/src/MiningCore/Blockchain/Monero/MoneroPool.cs index 76e05075d..4cec21014 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPool.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPool.cs @@ -347,52 +347,11 @@ protected override async Task OnRequestAsync(StratumClient client, } } - protected override void SetupStats() + public override ulong HashrateFromShares(double shares, double interval) { - base.SetupStats(); - - // Pool Hashrate - var poolHashRateSampleIntervalSeconds = 60 * 10; - - disposables.Add(Shares - .ObserveOn(ThreadPoolScheduler.Instance) - .Buffer(TimeSpan.FromSeconds(poolHashRateSampleIntervalSeconds)) - .Do(shares => UpdateMinerHashrates(shares, poolHashRateSampleIntervalSeconds)) - .Select(shares => - { - if (!shares.Any()) - return 0ul; - - try - { - return HashrateFromShares(shares, poolHashRateSampleIntervalSeconds); - } - - catch(Exception ex) - { - logger.Error(ex); - return 0ul; - } - }) - .Subscribe(hashRate => poolStats.PoolHashRate = hashRate)); - - // shares/sec - disposables.Add(Shares - .Buffer(TimeSpan.FromSeconds(1)) - .Do(shares => - { - poolStats.ValidSharesPerSecond = shares.Count; - - logger.Debug(() => $"[{LogCat}] Share/sec = {poolStats.ValidSharesPerSecond}"); - }) - .Subscribe()); - } - - protected override ulong HashrateFromShares(IEnumerable shares, int interval) - { - var result = Math.Ceiling(shares.Sum(share => share.Share.Difficulty) / interval); - return (ulong) result; - } + var result = Math.Ceiling(shares / interval); + return (ulong)result; + } protected override void OnVarDiffUpdate(StratumClient client, double newDiff) { diff --git a/src/MiningCore/Blockchain/ZCash/ZCashPayoutHandler.cs b/src/MiningCore/Blockchain/ZCash/ZCashPayoutHandler.cs index 314c29999..6ea1826d8 100644 --- a/src/MiningCore/Blockchain/ZCash/ZCashPayoutHandler.cs +++ b/src/MiningCore/Blockchain/ZCash/ZCashPayoutHandler.cs @@ -147,7 +147,7 @@ public override async Task PayoutAsync(Balance[] balances) { case ZOperationStatus.Success: var txId = operationResult.Result?.Value("txid") ?? string.Empty; - logger.Info(() => $"[{LogCategory}] Payout completed with transaction id: {txId}"); + logger.Info(() => $"[{LogCategory}] completed with transaction id: {txId}"); PersistPayments(page, txId); NotifyPayoutSuccess(poolConfig.Id, page, new[] {txId}, null); @@ -191,95 +191,6 @@ private async Task ShieldCoinbaseAsync() { logger.Info(() => $"[{LogCategory}] Shielding ZCash Coinbase funds"); - // Emulate z_shieldcoinbase until its stable -#if true - // get t-addr balance - var balanceResult = await daemon.ExecuteCmdSingleAsync(BitcoinCommands.GetBalance); - - if (balanceResult.Error != null) - { - logger.Error(() => $"[{LogCategory}] {BitcoinCommands.GetBalance} returned error: {balanceResult.Error.Message} code {balanceResult.Error.Code}"); - return; - } - - var balance = (decimal)(double)balanceResult.Response; - - if (balance > 0) - { - logger.Info(() => $"[{LogCategory}] Transferring {FormatAmount(balance)} to pool's z-addr"); - - // transfer to z-addr - var recipient = new ZSendManyRecipient - { - Address = poolExtraConfig.ZAddress, - Amount = balance - TransferFee - }; - - var args = new object[] - { - poolConfig.Address, // default account - new object[] // addresses and associated amounts - { - recipient - }, - 10, // only spend funds covered by this many confirmations - TransferFee - }; - - // send command - var sendResult = await daemon.ExecuteCmdSingleAsync(ZCashCommands.ZSendMany, args); - - if (sendResult.Error != null) - { - logger.Error(() => $"[{LogCategory}] {ZCashCommands.ZSendMany} returned error: {balanceResult.Error.Message} code {balanceResult.Error.Code}"); - return; - } - - var operationId = sendResult.Response; - - logger.Info(() => $"[{LogCategory}] {ZCashCommands.ZSendMany} operation id: {operationId}"); - - var continueWaiting = true; - - while(continueWaiting) - { - var operationResultResponse = await daemon.ExecuteCmdSingleAsync( - ZCashCommands.ZGetOperationResult, new object[] { new object[] { operationId } }); - - if (operationResultResponse.Error == null && - operationResultResponse.Response?.Any(x => x.OperationId == operationId) == true) - { - var operationResult = operationResultResponse.Response.First(x => x.OperationId == operationId); - - if (!Enum.TryParse(operationResult.Status, true, out ZOperationStatus status)) - { - logger.Error(() => $"Unrecognized operation status: {operationResult.Status}"); - break; - } - - switch(status) - { - case ZOperationStatus.Success: - var txId = operationResult.Result?.Value("txid") ?? string.Empty; - logger.Info(() => $"[{LogCategory}] Transfer completed with transaction id: {txId}"); - - continueWaiting = false; - continue; - - case ZOperationStatus.Cancelled: - case ZOperationStatus.Failed: - logger.Error(() => $"{ZCashCommands.ZSendMany} failed: {operationResult.Error.Message} code {operationResult.Error.Code}"); - - continueWaiting = false; - continue; - } - } - - logger.Info(() => $"[{LogCategory}] Waiting for transfer completion: {operationId}"); - await Task.Delay(TimeSpan.FromSeconds(10)); - } - } -#else var args = new object[] { poolConfig.Address, // source: pool's t-addr receiving coinbase rewards @@ -336,7 +247,6 @@ private async Task ShieldCoinbaseAsync() logger.Info(() => $"[{LogCategory}] Waiting for operation completion: {operationId}"); await Task.Delay(TimeSpan.FromSeconds(10)); } -#endif } } } diff --git a/src/MiningCore/Blockchain/ZCash/ZCashPoolBase.cs b/src/MiningCore/Blockchain/ZCash/ZCashPoolBase.cs index a03e72b1b..9d1220619 100644 --- a/src/MiningCore/Blockchain/ZCash/ZCashPoolBase.cs +++ b/src/MiningCore/Blockchain/ZCash/ZCashPoolBase.cs @@ -29,6 +29,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using AutoMapper; using MiningCore.Blockchain.Bitcoin; using MiningCore.Blockchain.ZCash.DaemonResponses; +using MiningCore.Configuration; using MiningCore.Extensions; using MiningCore.JsonRpc; using MiningCore.Mining; @@ -213,11 +214,10 @@ protected override void OnNewJob(object jobParams) }); } - protected override ulong HashrateFromShares(IEnumerable shares, int interval) + public override ulong HashrateFromShares(double shares, double interval) { - var sum = shares.Sum(share => Math.Max(0.00000001, share.Share.Difficulty * manager.ShareMultiplier)); var multiplier = BitcoinConstants.Pow2x32 / manager.ShareMultiplier; - var result = Math.Ceiling(((sum * multiplier / interval) / 1000000) * 2); + var result = Math.Ceiling(((shares * multiplier / interval) / 1000000) * 2); return (ulong)result; } diff --git a/src/MiningCore/Mining/Abstractions.cs b/src/MiningCore/Mining/Abstractions.cs index dc85e2bef..a81fc28e2 100644 --- a/src/MiningCore/Mining/Abstractions.cs +++ b/src/MiningCore/Mining/Abstractions.cs @@ -22,7 +22,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System.Threading.Tasks; using MiningCore.Blockchain; using MiningCore.Configuration; -using MiningCore.Persistence.Model; using MiningCore.Stratum; namespace MiningCore.Mining @@ -46,6 +45,7 @@ public interface IMiningPool PoolStats PoolStats { get; } BlockchainStats NetworkStats { get; } void Configure(PoolConfig poolConfig, ClusterConfig clusterConfig); + ulong HashrateFromShares(double shares, double interval); Task StartAsync(); } } diff --git a/src/MiningCore/Mining/PoolBase.cs b/src/MiningCore/Mining/PoolBase.cs index 4d0855706..9528b7f08 100644 --- a/src/MiningCore/Mining/PoolBase.cs +++ b/src/MiningCore/Mining/PoolBase.cs @@ -81,7 +81,7 @@ protected PoolBase(IComponentContext ctx, .Synchronize(); } - protected readonly PoolStats poolStats = new PoolStats(); + protected PoolStats poolStats; protected readonly JsonSerializerSettings serializerSettings; protected readonly NotificationService notificationService; protected readonly IConnectionFactory cf; @@ -104,12 +104,6 @@ protected PoolBase(IComponentContext ctx, protected override void OnConnect(StratumClient client) { - // update stats - lock(clients) - { - poolStats.ConnectedMiners = clients.Count; - } - // client setup var context = CreateClientContext(); @@ -130,15 +124,6 @@ protected override void OnConnect(StratumClient client) EnsureNoZombieClient(client); } - protected override void OnDisconnect(string subscriptionId) - { - // update stats - lock(clients) - { - poolStats.ConnectedMiners = clients.Count; - } - } - private void EnsureNoZombieClient(StratumClient client) { Observable.Timer(clock.Now.AddSeconds(10)) @@ -310,39 +295,9 @@ protected void SetupBanning(ClusterConfig clusterConfig) } } - protected virtual void SetupStats() + protected virtual void InitStats() { LoadStats(); - - // Periodically persist pool- and blockchain-stats to persistent storage - disposables.Add(Observable.Interval(TimeSpan.FromSeconds(60)) - .Select(_ => Observable.FromAsync(async () => - { - try - { - await UpdateBlockChainStatsAsync(); - } - catch(Exception) - { - // ignored - } - })) - .Concat() - .Subscribe(_ => PersistStats())); - - // For external stratums, miner counts are derived from submitted shares - if (poolConfig.ExternalStratum) - { - disposables.Add(Shares - .Buffer(TimeSpan.FromMinutes(1)) - .Do(shares => - { - var sharesByMiner = shares.GroupBy(x => x.Share.Miner).ToArray(); - poolStats.ConnectedMiners = sharesByMiner.Length; - }) - .Subscribe()); - - } } protected abstract Task UpdateBlockChainStatsAsync(); @@ -356,10 +311,9 @@ private void LoadStats() var stats = cf.Run(con => statsRepo.GetLastPoolStats(con, poolConfig.Id)); if (stats != null) - { - poolStats.ConnectedMiners = stats.ConnectedMiners; - poolStats.PoolHashRate = (ulong) stats.PoolHashRate; - } + poolStats = mapper.Map(stats); + else + poolStats = new PoolStats(); } catch (Exception ex) @@ -368,28 +322,6 @@ private void LoadStats() } } - private void PersistStats() - { - try - { - logger.Debug(() => $"[{LogCat}] Persisting pool stats"); - - cf.RunTx((con, tx) => - { - var mapped = mapper.Map(poolStats); - mapped.PoolId = poolConfig.Id; - mapped.Created = clock.Now; - - statsRepo.InsertPoolStats(con, tx, mapped); - }); - } - - catch(Exception ex) - { - logger.Error(ex, () => $"[{LogCat}] Unable to persist pool stats"); - } - } - protected void ConsiderBan(StratumClient client, WorkerContextBase context, PoolShareBasedBanningConfig config) { var totalShares = context.Stats.ValidShares + context.Stats.InvalidShares; @@ -444,55 +376,6 @@ Stratum Port(s): {string.Join(", ", poolConfig.Ports.Keys)} logger.Info(() => msg); } - protected abstract ulong HashrateFromShares(IEnumerable shares, int interval); - - protected virtual void UpdateMinerHashrates(IList shares, int interval) - { - try - { - var sharesByMiner = shares.GroupBy(x => x.Share.Miner).ToArray(); - - foreach (var minerShares in sharesByMiner) - { - // Total hashrate - var miner = minerShares.Key; - var hashRate = HashrateFromShares(minerShares, interval); - - var sample = new MinerHashrateSample - { - PoolId = poolConfig.Id, - Miner = miner, - Hashrate = hashRate, - Created = clock.Now - }; - - // Per worker hashrates - var sharesPerWorker = minerShares - .GroupBy(x => x.Share.Worker) - .Where(x => !string.IsNullOrEmpty(x.Key)); - - foreach(var workerShares in sharesPerWorker) - { - var worker = workerShares.Key; - hashRate = HashrateFromShares(workerShares, interval); - - if (sample.WorkerHashrates == null) - sample.WorkerHashrates = new Dictionary(); - - sample.WorkerHashrates[worker] = hashRate; - } - - // Persist - cf.RunTx((con, tx) => { statsRepo.RecordMinerHashrateSample(con, tx, sample); }); - } - } - - catch(Exception ex) - { - logger.Error(ex); - } - } - #region API-Surface public IObservable Shares { get; } @@ -510,6 +393,8 @@ public virtual void Configure(PoolConfig poolConfig, ClusterConfig clusterConfig this.clusterConfig = clusterConfig; } + public abstract ulong HashrateFromShares(double shares, double interval); + public virtual async Task StartAsync() { Contract.RequiresNonNull(poolConfig, nameof(poolConfig)); @@ -540,7 +425,7 @@ public virtual async Task StartAsync() StartExternalStratumPublisherListener(); } - SetupStats(); + InitStats(); await UpdateBlockChainStatsAsync(); logger.Info(() => $"[{LogCat}] Online"); diff --git a/src/MiningCore/Mining/PoolStatsUpdater.cs b/src/MiningCore/Mining/StatsRecorder.cs similarity index 54% rename from src/MiningCore/Mining/PoolStatsUpdater.cs rename to src/MiningCore/Mining/StatsRecorder.cs index 7fd2d44c4..4faffb5bb 100644 --- a/src/MiningCore/Mining/PoolStatsUpdater.cs +++ b/src/MiningCore/Mining/StatsRecorder.cs @@ -6,10 +6,10 @@ using System.Threading; using System.Threading.Tasks; using Autofac; +using AutoMapper; using MiningCore.Configuration; using MiningCore.Contracts; using MiningCore.Extensions; -using MiningCore.Payments; using MiningCore.Persistence; using MiningCore.Persistence.Model; using MiningCore.Persistence.Repositories; @@ -19,23 +19,26 @@ namespace MiningCore.Mining { - public class PoolStatsUpdater + public class StatsRecorder { - public PoolStatsUpdater(IComponentContext ctx, + public StatsRecorder(IComponentContext ctx, IMasterClock clock, IConnectionFactory cf, + IMapper mapper, IShareRepository shareRepo, IStatsRepository statsRepo) { Contract.RequiresNonNull(ctx, nameof(ctx)); Contract.RequiresNonNull(clock, nameof(clock)); Contract.RequiresNonNull(cf, nameof(cf)); + Contract.RequiresNonNull(mapper, nameof(mapper)); Contract.RequiresNonNull(shareRepo, nameof(shareRepo)); Contract.RequiresNonNull(statsRepo, nameof(statsRepo)); this.ctx = ctx; this.clock = clock; this.cf = cf; + this.mapper = mapper; this.shareRepo = shareRepo; this.statsRepo = statsRepo; @@ -45,14 +48,16 @@ public PoolStatsUpdater(IComponentContext ctx, private readonly IMasterClock clock; private readonly IStatsRepository statsRepo; private readonly IConnectionFactory cf; + private readonly IMapper mapper; private readonly IComponentContext ctx; private readonly IShareRepository shareRepo; private readonly AutoResetEvent stopEvent = new AutoResetEvent(false); private readonly Dictionary pools = new Dictionary(); + private const int HashrateCalculationWindow = 1200; // seconds private ClusterConfig clusterConfig; private Thread thread; private const int RetryCount = 4; - private Policy shareReadFaultPolicy; + private Policy readFaultPolicy; private static readonly ILogger logger = LogManager.GetCurrentClassLogger(); @@ -74,8 +79,7 @@ public void Start() { logger.Info(() => "Online"); - var interval = TimeSpan.FromSeconds( - clusterConfig.PaymentProcessing.Interval > 0 ? clusterConfig.PaymentProcessing.Interval : 600); + var interval = TimeSpan.FromMinutes(1); while (true) { @@ -97,7 +101,7 @@ public void Start() } }); - thread.Name = "Pool Stats Updater"; + thread.Name = "StatsRecorder"; thread.Start(); } @@ -122,71 +126,74 @@ private Task UpdatePoolsAsync() private void UpdateHashrates() { - var start = clock.Now; - var target = start.AddMinutes(-10); - var pageSize = 50000; - var currentPage = 0; + var start = clock.Now; //DateTime.Parse("2017-12-22 15:26:48.925534");// clock.Now; + var target = start.AddSeconds(-HashrateCalculationWindow); + + var stats = new MinerWorkerPerformanceStats + { + Created = start + }; foreach (var poolId in pools.Keys) { + stats.PoolId = poolId; + logger.Info(() => $"Updating hashrates for pool {poolId}"); - var before = start; var pool = pools[poolId]; - var accumulated = 0d; - - while (true) - { - logger.Info(() => $"Fetching page {currentPage} of shares for pool {poolId}"); - var page = shareReadFaultPolicy.Execute(() => - cf.Run(con => shareRepo.ReadSharesBeforeAndAfterCreated(con, poolId, before, target, true, pageSize))); + // fetch stats + var result = readFaultPolicy.Execute(() => + cf.Run(con => shareRepo.GetHashAccumulationBetweenCreated(con, poolId, target, start))); - currentPage++; + if (result.Length == 0) + continue; - var sharesByMiner = page.GroupBy(x => x.Miner).ToArray(); + var byMiner = result.GroupBy(x => x.Miner).ToArray(); - foreach (var minerShares in sharesByMiner) - { - // Total hashrate - var miner = minerShares.Key; - var hashRate = HashrateFromShares(minerShares, interval); + // calculate pool stats + var windowActual = Math.Max(1, (result.Max(x => x.LastShare) - result.Max(x => x.FirstShare)).TotalSeconds); + var poolHashesAccumulated = result.Sum(x => x.Sum); + var poolHashesCountAccumulated = result.Sum(x => x.Count); + var poolHashrate = pool.HashrateFromShares(poolHashesAccumulated, windowActual); - var sample = new MinerHashrateSample - { - PoolId = poolConfig.Id, - Miner = miner, - Hashrate = hashRate, - Created = clock.Now - }; - - // Per worker hashrates - var sharesPerWorker = minerShares - .GroupBy(x => x.Share.Worker) - .Where(x => !string.IsNullOrEmpty(x.Key)); - - foreach (var workerShares in sharesPerWorker) - { - var worker = workerShares.Key; - hashRate = HashrateFromShares(workerShares, interval); + // update + pool.PoolStats.ConnectedMiners = byMiner.Length; + pool.PoolStats.PoolHashRate = poolHashrate; + pool.PoolStats.ValidSharesPerSecond = (int) (poolHashesCountAccumulated / windowActual); - if (sample.WorkerHashrates == null) - sample.WorkerHashrates = new Dictionary(); + // persist + cf.RunTx((con, tx) => + { + var mapped = mapper.Map(pool.PoolStats); + mapped.PoolId = poolId; + mapped.Created = start; - sample.WorkerHashrates[worker] = hashRate; - } + statsRepo.InsertPoolStats(con, tx, mapped); + }); - // Persist - cf.RunTx((con, tx) => { statsRepo.RecordMinerHashrateSample(con, tx, sample); }); - } + // calculate & update miner, worker hashrates + foreach (var minerHashes in byMiner) + { + cf.RunTx((con, tx) => + { + stats.Miner = minerHashes.Key; - // accumulate per pool, miner and worker - // accumulated += pool.HashrateAccumulate(blockPage); + foreach (var item in minerHashes) + { + // calculate miner/worker stats + windowActual = Math.Max(1, (minerHashes.Max(x => x.LastShare) - minerHashes.Max(x => x.FirstShare)).TotalSeconds); + var hashrate = pool.HashrateFromShares(item.Sum, windowActual); - if (page.Length < pageSize) - break; + // update + stats.Hashrate = hashrate; + stats.Worker = item.Worker; + stats.SharesPerSecond = (double) item.Count / HashrateCalculationWindow; - before = page[page.Length - 1].Created; + // persist + statsRepo.InsertMinerWorkerPerformanceStats(con, tx, stats); + } + }); } } } @@ -199,7 +206,7 @@ private void BuildFaultHandlingPolicy() .Or() .Retry(RetryCount, OnPolicyRetry); - shareReadFaultPolicy = retry; + readFaultPolicy = retry; } private static void OnPolicyRetry(Exception ex, int retry, object context) diff --git a/src/MiningCore/MiningCore.csproj b/src/MiningCore/MiningCore.csproj index d3dd16a5a..e242a094c 100644 --- a/src/MiningCore/MiningCore.csproj +++ b/src/MiningCore/MiningCore.csproj @@ -90,7 +90,7 @@ - + diff --git a/src/MiningCore/Payments/PayoutManager.cs b/src/MiningCore/Payments/PayoutManager.cs index fc2cde297..5d4e91d14 100644 --- a/src/MiningCore/Payments/PayoutManager.cs +++ b/src/MiningCore/Payments/PayoutManager.cs @@ -143,11 +143,6 @@ await cf.RunTxAsync(async (con, tx) => break; case BlockStatus.Orphaned: - logger.Info(() => $"Deleting orphaned block {block.BlockHeight} on pool {pool.Id}"); - - blockRepo.DeleteBlock(con, tx, block); - break; - case BlockStatus.Pending: blockRepo.UpdateBlock(con, tx, block); break; diff --git a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs b/src/MiningCore/Payments/PayoutSchemes/PPLNS.cs similarity index 94% rename from src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs rename to src/MiningCore/Payments/PayoutSchemes/PPLNS.cs index bb549acfc..63d90d551 100644 --- a/src/MiningCore/Payments/PayoutSchemes/PayPerLastNShares.cs +++ b/src/MiningCore/Payments/PayoutSchemes/PPLNS.cs @@ -22,7 +22,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System.Collections.Generic; using System.Data; using System.Data.Common; -using System.Diagnostics; using System.Linq; using System.Net.Sockets; using System.Threading.Tasks; @@ -34,8 +33,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using MiningCore.Util; using NLog; using Polly; -using Polly.CircuitBreaker; -using Polly.Wrap; using Contract = MiningCore.Contracts.Contract; namespace MiningCore.Payments.PayoutSchemes @@ -43,9 +40,10 @@ namespace MiningCore.Payments.PayoutSchemes /// /// PPLNS payout scheme implementation /// - public class PayPerLastNShares : IPayoutScheme + // ReSharper disable once InconsistentNaming + public class PPLNS : IPayoutScheme { - public PayPerLastNShares(IConnectionFactory cf, + public PPLNS(IConnectionFactory cf, IShareRepository shareRepo, IBlockRepository blockRepo, IBalanceRepository balanceRepo) @@ -107,14 +105,16 @@ public Task UpdateBalancesAsync(IDbConnection con, IDbTransaction tx, PoolConfig // delete discarded shares if (shareCutOffDate.HasValue) { - var cutOffCount = shareRepo.CountPoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); + var cutOffCount = shareRepo.CountSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); if (cutOffCount > 0) { LogDiscardedShares(poolConfig, block, shareCutOffDate.Value); +#if !DEBUG logger.Info(() => $"Deleting {cutOffCount} discarded shares before {shareCutOffDate.Value:O}"); - shareRepo.DeletePoolSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); + shareRepo.DeleteSharesBeforeCreated(con, tx, poolConfig.Id, shareCutOffDate.Value); +#endif } } diff --git a/src/MiningCore/Persistence/Common/Repositories/StatsRepositoryBase.cs b/src/MiningCore/Persistence/Common/Repositories/StatsRepositoryBase.cs deleted file mode 100644 index 401c57d25..000000000 --- a/src/MiningCore/Persistence/Common/Repositories/StatsRepositoryBase.cs +++ /dev/null @@ -1,55 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; -using System.Text; -using Microsoft.Extensions.Caching.Memory; -using MiningCore.Contracts; -using MiningCore.Persistence.Model; - -namespace MiningCore.Persistence.Common.Repositories -{ - public class StatsRepositoryBase - { - private static readonly IMemoryCache cache = new MemoryCache(new MemoryCacheOptions - { - ExpirationScanFrequency = TimeSpan.FromSeconds(60) - }); - - private const int MaxHistorySize = 6; - - private string BuildSampleKey(string poolId, string miner) - { - return $"mhr_{poolId}_{miner}"; - } - - public void RecordMinerHashrateSample(IDbConnection con, IDbTransaction tx, MinerHashrateSample sample) - { - Contract.RequiresNonNull(sample, nameof(sample)); - - var key = BuildSampleKey(sample.PoolId, sample.Miner); - var samples = cache.Get>(key) ?? new List(MaxHistorySize) { sample }; - - while(samples.Count >= MaxHistorySize) - samples.Remove(samples.Last()); - - samples.Insert(0, sample); - - cache.Set(key, samples, new MemoryCacheEntryOptions - { - AbsoluteExpiration = DateTime.Now.AddMinutes(15) - }); - } - - protected MinerHashrateSample[] GetMinerHashrateSamples(string poolId, string miner) - { - Contract.Requires(!string.IsNullOrEmpty(poolId), $"{nameof(poolId)} must not be empty"); - Contract.Requires(!string.IsNullOrEmpty(miner), $"{nameof(miner)} must not be empty"); - - var key = BuildSampleKey(poolId, miner); - var samples = cache.Get>(key); - - return samples?.ToArray(); - } - } -} diff --git a/src/MiningCore/Persistence/Model/MinerHashrateSample.cs b/src/MiningCore/Persistence/Model/MinerHashrateSample.cs deleted file mode 100644 index a48e478f9..000000000 --- a/src/MiningCore/Persistence/Model/MinerHashrateSample.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace MiningCore.Persistence.Model -{ - public class MinerHashrateSample - { - public string PoolId { get; set; } - public string Miner { get; set; } - public ulong Hashrate { get; set; } - public Dictionary WorkerHashrates { get; set; } - public DateTime Created { get; set; } - } -} diff --git a/src/MiningCore/Persistence/Model/MinerWorkerPerformanceStats.cs b/src/MiningCore/Persistence/Model/MinerWorkerPerformanceStats.cs new file mode 100644 index 000000000..c5630a3ef --- /dev/null +++ b/src/MiningCore/Persistence/Model/MinerWorkerPerformanceStats.cs @@ -0,0 +1,14 @@ +using System; + +namespace MiningCore.Persistence.Model +{ + public class MinerWorkerPerformanceStats + { + public string PoolId { get; set; } + public string Miner { get; set; } + public string Worker { get; set; } + public double Hashrate { get; set; } + public double SharesPerSecond { get; set; } + public DateTime Created { get; set; } + } +} diff --git a/src/MiningCore/Persistence/Model/MinerStats.cs b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs similarity index 89% rename from src/MiningCore/Persistence/Model/MinerStats.cs rename to src/MiningCore/Persistence/Model/Projections/MinerStats.cs index 09fc29e41..d50b61a15 100644 --- a/src/MiningCore/Persistence/Model/MinerStats.cs +++ b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs @@ -18,9 +18,7 @@ portions of the Software. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -using System; - -namespace MiningCore.Persistence.Model +namespace MiningCore.Persistence.Model.Projections { public class MinerStats { @@ -28,6 +26,6 @@ public class MinerStats public decimal PendingBalance { get; set; } public decimal TotalPaid { get; set; } public Payment LastPayment { get; set; } - public MinerHashrateSample[] Hashrate { get; set; } + public MinerWorkerPerformanceStats[] PerformanceStats { get; set; } } } diff --git a/src/MiningCore/Persistence/Model/Projections/MinerWorkerHashes.cs b/src/MiningCore/Persistence/Model/Projections/MinerWorkerHashes.cs new file mode 100644 index 000000000..9b3b73155 --- /dev/null +++ b/src/MiningCore/Persistence/Model/Projections/MinerWorkerHashes.cs @@ -0,0 +1,34 @@ +/* +Copyright 2017 Coin Foundry (coinfoundry.org) +Authors: Oliver Weichhold (oliver@weichhold.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +using System; + +namespace MiningCore.Persistence.Model.Projections +{ + public class MinerWorkerHashes + { + public double Sum { get; set; } + public long Count { get; set; } + public string Miner { get; set; } + public string Worker { get; set; } + public DateTime FirstShare { get; set; } + public DateTime LastShare { get; set; } + } +} diff --git a/src/MiningCore/Persistence/Postgres/Entities/MinerWorkerPerformanceStats.cs b/src/MiningCore/Persistence/Postgres/Entities/MinerWorkerPerformanceStats.cs new file mode 100644 index 000000000..65031643f --- /dev/null +++ b/src/MiningCore/Persistence/Postgres/Entities/MinerWorkerPerformanceStats.cs @@ -0,0 +1,35 @@ +/* +Copyright 2017 Coin Foundry (coinfoundry.org) +Authors: Oliver Weichhold (oliver@weichhold.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +using System; + +namespace MiningCore.Persistence.Postgres.Entities +{ + public class MinerWorkerPerformanceStats + { + public long Id { get; set; } + public string PoolId { get; set; } + public string Miner { get; set; } + public string Worker { get; set; } + public double Hashrate { get; set; } + public double SharesPerSecond { get; set; } + public DateTime Created { get; set; } + } +} diff --git a/src/MiningCore/Persistence/Postgres/Repositories/BlockRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/BlockRepository.cs index 3c42ce2ab..28608aea1 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/BlockRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/BlockRepository.cs @@ -48,8 +48,8 @@ public void Insert(IDbConnection con, IDbTransaction tx, Block block) var mapped = mapper.Map(block); var query = - "INSERT INTO blocks(poolid, blockheight, networkdifficulty, status, transactionconfirmationdata, reward, effort, confirmationprogress, created) " + - "VALUES(@poolid, @blockheight, @networkdifficulty, @status, @transactionconfirmationdata, @reward, @effort, @confirmationprogress, @created)"; + "INSERT INTO blocks(poolid, blockheight, networkdifficulty, status, type, transactionconfirmationdata, reward, effort, confirmationprogress, created) " + + "VALUES(@poolid, @blockheight, @networkdifficulty, @status, @type, @transactionconfirmationdata, @reward, @effort, @confirmationprogress, @created)"; con.Execute(query, mapped, tx); } @@ -68,7 +68,7 @@ public void UpdateBlock(IDbConnection con, IDbTransaction tx, Block block) var mapped = mapper.Map(block); - var query = "UPDATE blocks SET status = @status, reward = @reward, effort = @effort, confirmationprogress = @confirmationprogress WHERE id = @id"; + var query = "UPDATE blocks SET blockheight = @blockheight, status = @status, type = @type, reward = @reward, effort = @effort, confirmationprogress = @confirmationprogress WHERE id = @id"; con.Execute(query, mapped, tx); } diff --git a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs index 710361b7d..00e35cc6f 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs @@ -25,6 +25,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using Dapper; using MiningCore.Extensions; using MiningCore.Persistence.Model; +using MiningCore.Persistence.Model.Projections; using MiningCore.Persistence.Repositories; using MiningCore.Util; using NLog; @@ -72,7 +73,7 @@ public Share[] ReadSharesBeforeAndAfterCreated(IDbConnection con, string poolId, logger.LogInvoke(new[] { poolId }); var query = $"SELECT * FROM shares WHERE poolid = @poolId AND created {(inclusive ? " <= " : " < ")} @before " + - $"AND created {(inclusive ? " >= " : " > ")} @after" + + $"AND created {(inclusive ? " >= " : " > ")} @after" + "ORDER BY created DESC FETCH NEXT (@pageSize) ROWS ONLY"; return con.Query(query, new { poolId, before, after, pageSize }) @@ -92,7 +93,7 @@ public Share[] PageSharesBetweenCreated(IDbConnection con, string poolId, DateTi .ToArray(); } - public long CountPoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before) + public long CountSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before) { logger.LogInvoke(new[] { poolId }); @@ -101,7 +102,7 @@ public long CountPoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, s return con.QuerySingle(query, new { poolId, before }, tx); } - public void DeletePoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before) + public void DeleteSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before) { logger.LogInvoke(new[] { poolId }); @@ -110,7 +111,7 @@ public void DeletePoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, con.Execute(query, new { poolId, before }, tx); } - public long CountMinerSharesBetweenCreated(IDbConnection con, string poolId, string miner, DateTime? start, DateTime? end) + public long CountSharesBetweenCreated(IDbConnection con, string poolId, string miner, DateTime? start, DateTime? end) { logger.LogInvoke(new[] { poolId }); @@ -134,5 +135,17 @@ public long CountMinerSharesBetweenCreated(IDbConnection con, string poolId, str return con.QuerySingle(query, new { poolId, start, end }); } + + public MinerWorkerHashes[] GetHashAccumulationBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end) + { + logger.LogInvoke(new[] { poolId }); + + var query = "SELECT SUM(difficulty), COUNT(difficulty), min(created) as firstshare, max(created) as lastshare, miner, worker FROM shares " + + "WHERE poolid = @poolId AND created >= @start AND created <= @end " + + "GROUP BY miner, worker"; + + return con.Query(query, new { poolId, start, end }) + .ToArray(); + } } } diff --git a/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs index 337259025..6ced6bf24 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs @@ -24,16 +24,15 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using AutoMapper; using Dapper; using MiningCore.Extensions; -using MiningCore.Persistence.Common.Repositories; using MiningCore.Persistence.Model; +using MiningCore.Persistence.Model.Projections; using MiningCore.Persistence.Repositories; -using MiningCore.Util; using NLog; +using MinerStats = MiningCore.Persistence.Model.Projections.MinerStats; namespace MiningCore.Persistence.Postgres.Repositories { - public class StatsRepository : StatsRepositoryBase, - IStatsRepository + public class StatsRepository : IStatsRepository { public StatsRepository(IMapper mapper) { @@ -55,6 +54,18 @@ public void InsertPoolStats(IDbConnection con, IDbTransaction tx, PoolStats stat con.Execute(query, mapped, tx); } + public void InsertMinerWorkerPerformanceStats(IDbConnection con, IDbTransaction tx, MinerWorkerPerformanceStats stats) + { + logger.LogInvoke(); + + var mapped = mapper.Map(stats); + + var query = "INSERT INTO minerstats(poolid, miner, worker, hashrate, sharespersecond, created) " + + "VALUES(@poolid, @miner, @worker, @hashrate, @sharespersecond, @created)"; + + con.Execute(query, mapped, tx); + } + public PoolStats GetLastPoolStats(IDbConnection con, string poolId) { logger.LogInvoke(); @@ -97,7 +108,7 @@ public PoolStats[] GetPoolStatsBetweenHourly(IDbConnection con, string poolId, D .ToArray(); } - public MinerStats GetMinerStats(IDbConnection con, string poolId, string miner) + public MinerStats GetMinerStats(IDbConnection con, IDbTransaction tx, string poolId, string miner) { logger.LogInvoke(new[] { poolId, miner }); @@ -105,15 +116,32 @@ public MinerStats GetMinerStats(IDbConnection con, string poolId, string miner) "(SELECT amount FROM balances WHERE poolid = @poolId AND address = @miner) AS pendingbalance, " + "(SELECT SUM(amount) FROM payments WHERE poolid = @poolId and address = @miner) as totalpaid"; - var result = con.QuerySingleOrDefault(query, new { poolId, miner }); + var result = con.QuerySingleOrDefault(query, new { poolId, miner }, tx); if (result != null) { query = "SELECT * FROM payments WHERE poolid = @poolId AND address = @miner" + " ORDER BY created DESC LIMIT 1"; - result.LastPayment = con.QuerySingleOrDefault(query, new { poolId, miner }); - result.Hashrate = GetMinerHashrateSamples(poolId, miner); + result.LastPayment = con.QuerySingleOrDefault(query, new { poolId, miner }, tx); + + // query timestamp of last stats update + query = "SELECT created FROM minerstats WHERE poolid = @poolId AND miner = @miner" + + " ORDER BY created DESC LIMIT 1"; + + var lastUpdate = con.QuerySingleOrDefault(query, new { poolId, miner }, tx); + + if (lastUpdate.HasValue) + { + // load rows rows by timestamp + query = "SELECT * FROM minerstats WHERE poolid = @poolId AND miner = @miner AND created = @created"; + + var stats = con.Query(query, new { poolId, miner, created = lastUpdate }) + .Select(mapper.Map) + .ToArray(); + + result.PerformanceStats = stats; + } } return result; diff --git a/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql b/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql index 8910e1eed..0759b5fa8 100644 --- a/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql +++ b/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql @@ -66,12 +66,24 @@ CREATE TABLE poolstats ( id BIGSERIAL NOT NULL PRIMARY KEY, poolid TEXT NOT NULL, - connectedminers INT NOT NULL DEFAULT 0, - poolhashrate FLOAT NOT NULL DEFAULT 0, - + poolhashrate DOUBLE PRECISION NOT NULL DEFAULT 0, created TIMESTAMP NOT NULL ); CREATE INDEX IDX_POOLSTATS_POOL_CREATED on poolstats(poolid, created); CREATE INDEX IDX_POOLSTATS_POOL_CREATED_HOUR on poolstats(poolid, date_trunc('hour',created)); + +CREATE TABLE minerstats +( + id BIGSERIAL NOT NULL PRIMARY KEY, + poolid TEXT NOT NULL, + miner TEXT NOT NULL, + worker TEXT NULL, + hashrate DOUBLE PRECISION NOT NULL DEFAULT 0, + sharespersecond DOUBLE PRECISION NOT NULL DEFAULT 0, + created TIMESTAMP NOT NULL +); + +CREATE INDEX IDX_MINERSTATS_POOL_MINER_CREATED on minerstats(poolid, miner, created); +CREATE INDEX IDX_MINERSTATS_POOL_MINER_CREATED_HOUR on minerstats(poolid, miner, worker, date_trunc('hour',created)); diff --git a/src/MiningCore/Persistence/Repositories/IShareRepository.cs b/src/MiningCore/Persistence/Repositories/IShareRepository.cs index 5227a0f53..3178a6bc8 100644 --- a/src/MiningCore/Persistence/Repositories/IShareRepository.cs +++ b/src/MiningCore/Persistence/Repositories/IShareRepository.cs @@ -21,6 +21,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System; using System.Data; using MiningCore.Persistence.Model; +using MiningCore.Persistence.Model.Projections; namespace MiningCore.Persistence.Repositories { @@ -31,10 +32,11 @@ public interface IShareRepository Share[] ReadSharesBeforeAndAfterCreated(IDbConnection con, string poolId, DateTime before, DateTime after, bool inclusive, int pageSize); Share[] PageSharesBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end, int page, int pageSize); - long CountPoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before); - void DeletePoolSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before); + long CountSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before); + void DeleteSharesBeforeCreated(IDbConnection con, IDbTransaction tx, string poolId, DateTime before); - long CountMinerSharesBetweenCreated(IDbConnection con, string poolId, string miner, DateTime? start, DateTime? end); + long CountSharesBetweenCreated(IDbConnection con, string poolId, string miner, DateTime? start, DateTime? end); ulong? GetAccumulatedShareDifficultyBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end); + MinerWorkerHashes[] GetHashAccumulationBetweenCreated(IDbConnection con, string poolId, DateTime start, DateTime end); } } diff --git a/src/MiningCore/Persistence/Repositories/IStatsRepository.cs b/src/MiningCore/Persistence/Repositories/IStatsRepository.cs index 2522facce..80a24dcb5 100644 --- a/src/MiningCore/Persistence/Repositories/IStatsRepository.cs +++ b/src/MiningCore/Persistence/Repositories/IStatsRepository.cs @@ -22,16 +22,18 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System.Data; using Dapper; using MiningCore.Persistence.Model; +using MiningCore.Persistence.Model.Projections; +using MinerStats = MiningCore.Persistence.Model.Projections.MinerStats; namespace MiningCore.Persistence.Repositories { public interface IStatsRepository { - void InsertPoolStats(IDbConnection con, IDbTransaction tx, PoolStats share); + void InsertPoolStats(IDbConnection con, IDbTransaction tx, PoolStats stats); + void InsertMinerWorkerPerformanceStats(IDbConnection con, IDbTransaction tx, Model.MinerWorkerPerformanceStats stats); PoolStats GetLastPoolStats(IDbConnection con, string poolId); PoolStats[] PagePoolStatsBetween(IDbConnection con, string poolId, DateTime start, DateTime end, int page, int pageSize); PoolStats[] GetPoolStatsBetweenHourly(IDbConnection con, string poolId, DateTime start, DateTime end); - MinerStats GetMinerStats(IDbConnection con, string poolId, string miner); - void RecordMinerHashrateSample(IDbConnection con, IDbTransaction tx, MinerHashrateSample sample); + MinerStats GetMinerStats(IDbConnection con, IDbTransaction tx, string poolId, string miner); } } diff --git a/src/MiningCore/Program.cs b/src/MiningCore/Program.cs index 18ee877c5..b6ff01790 100644 --- a/src/MiningCore/Program.cs +++ b/src/MiningCore/Program.cs @@ -70,7 +70,7 @@ public class Program private static CommandOption shareRecoveryOption; private static ShareRecorder shareRecorder; private static PayoutManager payoutManager; - private static PoolStatsUpdater poolStatsUpdater; + private static StatsRecorder statsRecorder; private static ClusterConfig clusterConfig; private static ApiServer apiServer; @@ -547,8 +547,8 @@ private static async Task Start() logger.Info("Payment processing is not enabled"); // start pool stats updater - poolStatsUpdater = container.Resolve(); - poolStatsUpdater.Configure(clusterConfig); + statsRecorder = container.Resolve(); + statsRecorder.Configure(clusterConfig); // start pools await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolConfig => @@ -568,10 +568,10 @@ await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolCo // post-start attachments apiServer.AttachPool(pool); - poolStatsUpdater.AttachPool(pool); + statsRecorder.AttachPool(pool); })); - poolStatsUpdater.Start(); + statsRecorder.Start(); // keep running await Observable.Never().ToTask(); diff --git a/src/MiningCore/Stratum/StratumServer.cs b/src/MiningCore/Stratum/StratumServer.cs index 6076a341a..09e825299 100644 --- a/src/MiningCore/Stratum/StratumServer.cs +++ b/src/MiningCore/Stratum/StratumServer.cs @@ -295,7 +295,10 @@ protected void ForEachClient(Action action) } protected abstract void OnConnect(StratumClient client); - protected abstract void OnDisconnect(string subscriptionId); + + protected virtual void OnDisconnect(string subscriptionId) + { + } protected abstract Task OnRequestAsync(StratumClient client, Timestamped request); diff --git a/src/MiningCore/Util/FormatUtil.cs b/src/MiningCore/Util/FormatUtil.cs index c01aae7d8..175d4010e 100644 --- a/src/MiningCore/Util/FormatUtil.cs +++ b/src/MiningCore/Util/FormatUtil.cs @@ -9,7 +9,7 @@ public static class FormatUtil public static readonly string[] HashRateUnits = { " KH/s", " MH/s", " GH/s", " TH/s", " PH/s" }; public static readonly string[] DifficultyUnits = { " K", " M", " G", " T", " P" }; public static readonly string[] CapacityUnits = { " KB", " MB", " GB", " TB", " PB" }; - public static readonly string[] QuantityUnits = { " K", " M", " B", " T", " Q" }; + public static readonly string[] QuantityUnits = { "K", "M", "B", "T", "Q" }; public static string FormatHashRate(double hashrate) { From ce5d97532822dd362d23d93009790f695746a4d2 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Wed, 27 Dec 2017 21:30:45 +0100 Subject: [PATCH 10/24] Increase stats update interval --- src/MiningCore/Mining/StatsRecorder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MiningCore/Mining/StatsRecorder.cs b/src/MiningCore/Mining/StatsRecorder.cs index 4faffb5bb..8ba1e55fc 100644 --- a/src/MiningCore/Mining/StatsRecorder.cs +++ b/src/MiningCore/Mining/StatsRecorder.cs @@ -79,7 +79,7 @@ public void Start() { logger.Info(() => "Online"); - var interval = TimeSpan.FromMinutes(1); + var interval = TimeSpan.FromMinutes(5); while (true) { @@ -126,7 +126,7 @@ private Task UpdatePoolsAsync() private void UpdateHashrates() { - var start = clock.Now; //DateTime.Parse("2017-12-22 15:26:48.925534");// clock.Now; + var start = clock.Now; var target = start.AddSeconds(-HashrateCalculationWindow); var stats = new MinerWorkerPerformanceStats From d54132daeba264a833ddfaecd34d6266f3dae74c Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Wed, 27 Dec 2017 22:11:35 +0100 Subject: [PATCH 11/24] StatsRecorder start hot --- src/MiningCore/Mining/StatsRecorder.cs | 3 +++ src/MiningCore/Program.cs | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/MiningCore/Mining/StatsRecorder.cs b/src/MiningCore/Mining/StatsRecorder.cs index 8ba1e55fc..b3b18fa78 100644 --- a/src/MiningCore/Mining/StatsRecorder.cs +++ b/src/MiningCore/Mining/StatsRecorder.cs @@ -79,6 +79,9 @@ public void Start() { logger.Info(() => "Online"); + // warm-up delay + await Task.Delay(TimeSpan.FromSeconds(10)); + var interval = TimeSpan.FromMinutes(5); while (true) diff --git a/src/MiningCore/Program.cs b/src/MiningCore/Program.cs index b6ff01790..ab88afc02 100644 --- a/src/MiningCore/Program.cs +++ b/src/MiningCore/Program.cs @@ -549,6 +549,7 @@ private static async Task Start() // start pool stats updater statsRecorder = container.Resolve(); statsRecorder.Configure(clusterConfig); + statsRecorder.Start(); // start pools await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolConfig => @@ -563,16 +564,14 @@ await Task.WhenAll(clusterConfig.Pools.Where(x => x.Enabled).Select(async poolCo // pre-start attachments shareRecorder.AttachPool(pool); + statsRecorder.AttachPool(pool); await pool.StartAsync(); // post-start attachments apiServer.AttachPool(pool); - statsRecorder.AttachPool(pool); })); - statsRecorder.Start(); - // keep running await Observable.Never().ToTask(); } From 03d552aa3612d5abac672497e39529d78dc6aaf0 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Wed, 27 Dec 2017 22:16:54 +0100 Subject: [PATCH 12/24] WIP --- src/MiningCore/Mining/StatsRecorder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MiningCore/Mining/StatsRecorder.cs b/src/MiningCore/Mining/StatsRecorder.cs index b3b18fa78..17b1aacb1 100644 --- a/src/MiningCore/Mining/StatsRecorder.cs +++ b/src/MiningCore/Mining/StatsRecorder.cs @@ -155,7 +155,7 @@ private void UpdateHashrates() var byMiner = result.GroupBy(x => x.Miner).ToArray(); // calculate pool stats - var windowActual = Math.Max(1, (result.Max(x => x.LastShare) - result.Max(x => x.FirstShare)).TotalSeconds); + var windowActual = HashrateCalculationWindow;//Math.Max(1, (result.Max(x => x.LastShare) - result.Max(x => x.FirstShare)).TotalSeconds); var poolHashesAccumulated = result.Sum(x => x.Sum); var poolHashesCountAccumulated = result.Sum(x => x.Count); var poolHashrate = pool.HashrateFromShares(poolHashesAccumulated, windowActual); @@ -185,7 +185,7 @@ private void UpdateHashrates() foreach (var item in minerHashes) { // calculate miner/worker stats - windowActual = Math.Max(1, (minerHashes.Max(x => x.LastShare) - minerHashes.Max(x => x.FirstShare)).TotalSeconds); + windowActual = HashrateCalculationWindow;//Math.Max(1, (minerHashes.Max(x => x.LastShare) - minerHashes.Max(x => x.FirstShare)).TotalSeconds); var hashrate = pool.HashrateFromShares(item.Sum, windowActual); // update From 77f205bce0c284f7e58a1ab762317925f3f9d18c Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Wed, 27 Dec 2017 22:42:52 +0100 Subject: [PATCH 13/24] WIP --- src/MiningCore/Mining/StatsRecorder.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/MiningCore/Mining/StatsRecorder.cs b/src/MiningCore/Mining/StatsRecorder.cs index 17b1aacb1..81efb6525 100644 --- a/src/MiningCore/Mining/StatsRecorder.cs +++ b/src/MiningCore/Mining/StatsRecorder.cs @@ -155,7 +155,7 @@ private void UpdateHashrates() var byMiner = result.GroupBy(x => x.Miner).ToArray(); // calculate pool stats - var windowActual = HashrateCalculationWindow;//Math.Max(1, (result.Max(x => x.LastShare) - result.Max(x => x.FirstShare)).TotalSeconds); + var windowActual = Math.Max(1, (result.Max(x => x.LastShare) - result.Min(x => x.FirstShare)).TotalSeconds); var poolHashesAccumulated = result.Sum(x => x.Sum); var poolHashesCountAccumulated = result.Sum(x => x.Count); var poolHashrate = pool.HashrateFromShares(poolHashesAccumulated, windowActual); @@ -185,7 +185,7 @@ private void UpdateHashrates() foreach (var item in minerHashes) { // calculate miner/worker stats - windowActual = HashrateCalculationWindow;//Math.Max(1, (minerHashes.Max(x => x.LastShare) - minerHashes.Max(x => x.FirstShare)).TotalSeconds); + windowActual = Math.Max(1, (minerHashes.Max(x => x.LastShare) - minerHashes.Min(x => x.FirstShare)).TotalSeconds); var hashrate = pool.HashrateFromShares(item.Sum, windowActual); // update From 892b1bbcf147580c95684de875a08ff8c6a9f098 Mon Sep 17 00:00:00 2001 From: Dainel Vera Date: Fri, 29 Dec 2017 05:38:21 -0500 Subject: [PATCH 14/24] rename BCC to BCH (#127) --- src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs | 2 +- src/MiningCore/Blockchain/Bitcoin/BitcoinPool.cs | 2 +- src/MiningCore/Blockchain/Bitcoin/BitcoinProperties.cs | 2 +- src/MiningCore/Blockchain/CoinMetaData.cs | 6 +++--- src/MiningCore/Configuration/ClusterConfig.cs | 6 +++--- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs index 662c4e5c3..7cd030bc5 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs @@ -44,7 +44,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. namespace MiningCore.Blockchain.Bitcoin { [CoinMetadata( - CoinType.BTC, CoinType.BCC, CoinType.NMC, CoinType.PPC, + CoinType.BTC, CoinType.BCH, CoinType.NMC, CoinType.PPC, CoinType.LTC, CoinType.DOGE, CoinType.DGB, CoinType.VIA, CoinType.GRS, CoinType.MONA, CoinType.VTC, CoinType.BTG, CoinType.GLT, CoinType.STAK)] diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPool.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPool.cs index c2a6a9b6c..951f6e6e7 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPool.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPool.cs @@ -31,7 +31,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. namespace MiningCore.Blockchain.Bitcoin { [CoinMetadata( - CoinType.BTC, CoinType.BCC, CoinType.NMC, CoinType.PPC, + CoinType.BTC, CoinType.BCH, CoinType.NMC, CoinType.PPC, CoinType.LTC, CoinType.DOGE, CoinType.DGB, CoinType.VIA, CoinType.GRS, CoinType.MONA, CoinType.VTC, CoinType.GLT)] public class BitcoinPool : BitcoinPoolBase, BlockTemplate> diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinProperties.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinProperties.cs index 8a8ae6b34..9c1b689b6 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinProperties.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinProperties.cs @@ -74,7 +74,7 @@ public class BitcoinProperties { // SHA256 { CoinType.BTC, sha256Coin }, - { CoinType.BCC, sha256Coin }, + { CoinType.BCH, sha256Coin }, { CoinType.NMC, sha256Coin }, { CoinType.PPC, sha256Coin }, { CoinType.GLT, sha256Coin }, diff --git a/src/MiningCore/Blockchain/CoinMetaData.cs b/src/MiningCore/Blockchain/CoinMetaData.cs index dad8a726e..2f59cc48e 100644 --- a/src/MiningCore/Blockchain/CoinMetaData.cs +++ b/src/MiningCore/Blockchain/CoinMetaData.cs @@ -22,7 +22,7 @@ public static class CoinMetaData { CoinType.XMR, new Dictionary { { string.Empty, "https://chainradar.com/xmr/block/{0}" }}}, { CoinType.LTC, new Dictionary { { string.Empty, "http://explorer.litecoin.net/block/{0}" }}}, - { CoinType.BCC, new Dictionary { { string.Empty, "https://www.blocktrail.com/BCC/block/{0}" }}}, + { CoinType.BCH, new Dictionary { { string.Empty, "https://www.blocktrail.com/BCC/block/{0}" }}}, { CoinType.DASH, new Dictionary { { string.Empty, "https://chainz.cryptoid.info/dash/block.dws?{0}.htm" }}}, { CoinType.BTC, new Dictionary { { string.Empty, "https://blockchain.info/block/{0}" }}}, { CoinType.DOGE, new Dictionary { { string.Empty, "https://dogechain.info/block/{0}" }}}, @@ -46,7 +46,7 @@ public static class CoinMetaData { CoinType.ETH, "https://etherscan.io/tx/{0}" }, { CoinType.ETC, "https://gastracker.io/tx/{0}" }, { CoinType.LTC, "http://explorer.litecoin.net/tx/{0}" }, - { CoinType.BCC, "https://www.blocktrail.com/BCC/tx/{0}" }, + { CoinType.BCH, "https://www.blocktrail.com/BCC/tx/{0}" }, { CoinType.DASH, "https://chainz.cryptoid.info/dash/tx.dws?{0}.htm" }, { CoinType.BTC, "https://blockchain.info/tx/{0}" }, { CoinType.DOGE, "https://dogechain.info/tx/{0}" }, @@ -69,7 +69,7 @@ public static class CoinMetaData { CoinType.ETH, "https://etherscan.io/address/{0}" }, { CoinType.ETC, "https://gastracker.io/addr/{0}" }, { CoinType.LTC, "http://explorer.litecoin.net/address/{0}" }, - { CoinType.BCC, "https://www.blocktrail.com/BCC/address/{0}" }, + { CoinType.BCH, "https://www.blocktrail.com/BCC/address/{0}" }, { CoinType.DASH, "https://chainz.cryptoid.info/dash/address.dws?{0}.htm" }, { CoinType.BTC, "https://blockchain.info/address/{0}" }, { CoinType.DOGE, "https://dogechain.info/address/{0}" }, diff --git a/src/MiningCore/Configuration/ClusterConfig.cs b/src/MiningCore/Configuration/ClusterConfig.cs index 7915b3362..cb8d4c1f7 100644 --- a/src/MiningCore/Configuration/ClusterConfig.cs +++ b/src/MiningCore/Configuration/ClusterConfig.cs @@ -28,7 +28,7 @@ public enum CoinType { // ReSharper disable InconsistentNaming BTC = 1, // Bitcoin - BCC, // Bitcoin Cash + BCH, // Bitcoin Cash LTC, // Litecoin DOGE, // Dogecoin, XMR, // Monero @@ -46,9 +46,9 @@ public enum CoinType VTC, // Vertcoin BTG, // Bitcoin Gold GLT, // Globaltoken - ELLA, //Ellaism + ELLA, // Ellaism AEON, // AEON - STAK //Straks + STAK // Straks } public class CoinConfig From c990ebed9fd51c5b00b1c52ef7aa14c71014b5c5 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Fri, 29 Dec 2017 14:08:48 +0100 Subject: [PATCH 15/24] Electroneum Support (ETN) --- src/MiningCore/Blockchain/CoinMetaData.cs | 2 ++ src/MiningCore/Blockchain/Monero/MoneroConstants.cs | 1 + src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs | 2 +- src/MiningCore/Blockchain/Monero/MoneroPool.cs | 6 +----- src/MiningCore/Configuration/ClusterConfig.cs | 3 ++- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/MiningCore/Blockchain/CoinMetaData.cs b/src/MiningCore/Blockchain/CoinMetaData.cs index 2f59cc48e..f3f926de2 100644 --- a/src/MiningCore/Blockchain/CoinMetaData.cs +++ b/src/MiningCore/Blockchain/CoinMetaData.cs @@ -21,6 +21,7 @@ public static class CoinMetaData }}, { CoinType.XMR, new Dictionary { { string.Empty, "https://chainradar.com/xmr/block/{0}" }}}, + { CoinType.ETN, new Dictionary { { string.Empty, "https://blockexplorer.electroneum.com/block/{0}" } }}, { CoinType.LTC, new Dictionary { { string.Empty, "http://explorer.litecoin.net/block/{0}" }}}, { CoinType.BCH, new Dictionary { { string.Empty, "https://www.blocktrail.com/BCC/block/{0}" }}}, { CoinType.DASH, new Dictionary { { string.Empty, "https://chainz.cryptoid.info/dash/block.dws?{0}.htm" }}}, @@ -43,6 +44,7 @@ public static class CoinMetaData public static readonly Dictionary PaymentInfoLinks = new Dictionary { { CoinType.XMR, "https://chainradar.com/xmr/transaction/{0}" }, + { CoinType.ETN, "https://blockexplorer.electroneum.com/tx/{0}" }, { CoinType.ETH, "https://etherscan.io/tx/{0}" }, { CoinType.ETC, "https://gastracker.io/tx/{0}" }, { CoinType.LTC, "http://explorer.litecoin.net/tx/{0}" }, diff --git a/src/MiningCore/Blockchain/Monero/MoneroConstants.cs b/src/MiningCore/Blockchain/Monero/MoneroConstants.cs index 48ac21e0c..f8de9eb21 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroConstants.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroConstants.cs @@ -41,6 +41,7 @@ public class MoneroConstants public static readonly Dictionary AddressLength = new Dictionary { { CoinType.XMR, 95 }, + { CoinType.ETN, 98 }, { CoinType.AEON, 97 }, }; diff --git a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs index 60e2b04a6..47dbca103 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs @@ -45,7 +45,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. namespace MiningCore.Blockchain.Monero { - [CoinMetadata(CoinType.XMR, CoinType.AEON)] + [CoinMetadata(CoinType.XMR, CoinType.AEON, CoinType.ETN)] public class MoneroPayoutHandler : PayoutHandlerBase, IPayoutHandler { diff --git a/src/MiningCore/Blockchain/Monero/MoneroPool.cs b/src/MiningCore/Blockchain/Monero/MoneroPool.cs index 4cec21014..0d3a6b66d 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPool.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPool.cs @@ -19,18 +19,14 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ using System; -using System.Collections.Generic; -using System.Diagnostics; using System.Globalization; using System.Linq; using System.Reactive; -using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Threading.Tasks; using System.Threading; using System.Threading.Tasks; using Autofac; -using Autofac.Features.Metadata; using AutoMapper; using MiningCore.Blockchain.Monero.StratumRequests; using MiningCore.Blockchain.Monero.StratumResponses; @@ -46,7 +42,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. namespace MiningCore.Blockchain.Monero { - [CoinMetadata(CoinType.XMR, CoinType.AEON)] + [CoinMetadata(CoinType.XMR, CoinType.AEON, CoinType.ETN)] public class MoneroPool : PoolBase { public MoneroPool(IComponentContext ctx, diff --git a/src/MiningCore/Configuration/ClusterConfig.cs b/src/MiningCore/Configuration/ClusterConfig.cs index cb8d4c1f7..768ad7fdb 100644 --- a/src/MiningCore/Configuration/ClusterConfig.cs +++ b/src/MiningCore/Configuration/ClusterConfig.cs @@ -48,7 +48,8 @@ public enum CoinType GLT, // Globaltoken ELLA, // Ellaism AEON, // AEON - STAK // Straks + STAK, // Straks + ETN // Electroneum } public class CoinConfig From 5aa043d61d4d29f8f717afb385021e4cc334b3fb Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Fri, 29 Dec 2017 21:45:48 +0100 Subject: [PATCH 16/24] Lyra Hashrate Multiplier --- src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs index 0699478d9..230c47ac5 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs @@ -19,10 +19,8 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ using System; -using System.Collections.Generic; using System.Linq; using System.Reactive; -using System.Reactive.Concurrency; using System.Reactive.Linq; using System.Reactive.Threading.Tasks; using System.Threading.Tasks; @@ -350,8 +348,8 @@ public override ulong HashrateFromShares(double shares, double interval) // OW: tmp hotfix if (poolConfig.Coin.Type == CoinType.MONA || poolConfig.Coin.Type == CoinType.VTC) - result *= 1.3; - Console.WriteLine(result); + result *= 2; + return (ulong)result; } From 286d6cc989fa110d80ec64ea0d68d79b81cf0c39 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Sat, 30 Dec 2017 09:49:47 +0100 Subject: [PATCH 17/24] STAK hashrate fix --- src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs index 230c47ac5..6711febb3 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPoolBase.cs @@ -347,7 +347,7 @@ public override ulong HashrateFromShares(double shares, double interval) var result = Math.Ceiling(shares * multiplier / interval); // OW: tmp hotfix - if (poolConfig.Coin.Type == CoinType.MONA || poolConfig.Coin.Type == CoinType.VTC) + if (poolConfig.Coin.Type == CoinType.MONA || poolConfig.Coin.Type == CoinType.VTC || poolConfig.Coin.Type == CoinType.STAK) result *= 2; return (ulong)result; From 04b648c9402f3a715924f909160a806f99de1e13 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Sun, 31 Dec 2017 15:44:38 +0100 Subject: [PATCH 18/24] Change API to align with latest wiki docs --- .../Api/Responses/GetMinerStatsResponse.cs | 16 +++++- src/MiningCore/AutoMapperProfile.cs | 3 ++ .../Bitcoin/BitcoinPayoutHandler.cs | 5 +- .../Ethereum/EthereumPayoutHandler.cs | 1 + .../Blockchain/Monero/MoneroPayoutHandler.cs | 1 + .../Model/Projections/MinerStats.cs | 17 +++++- .../Postgres/Repositories/ShareRepository.cs | 2 +- .../Postgres/Repositories/StatsRepository.cs | 52 ++++++++++++++++--- .../Persistence/Postgres/Scripts/createdb.sql | 2 +- .../Repositories/IStatsRepository.cs | 4 +- 10 files changed, 86 insertions(+), 17 deletions(-) diff --git a/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs b/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs index 6417cbff5..16a080347 100644 --- a/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs +++ b/src/MiningCore/Api/Responses/GetMinerStatsResponse.cs @@ -19,10 +19,22 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ using System; -using MiningCore.Persistence.Model; +using System.Collections.Generic; namespace MiningCore.Api.Responses { + public class WorkerPerformanceStats + { + public double Hashrate { get; set; } + public double SharesPerSecond { get; set; } + } + + public class WorkerPerformanceStatsContainer + { + public DateTime Created { get; set; } + public Dictionary Workers { get; set; } + } + public class MinerStats { public ulong PendingShares { get; set; } @@ -30,6 +42,6 @@ public class MinerStats public decimal TotalPaid { get; set; } public DateTime? LastPayment { get; set; } public string LastPaymentLink { get; set; } - public MinerWorkerPerformanceStats[] PerformanceStats { get; set; } + public WorkerPerformanceStatsContainer Performance { get; set; } } } diff --git a/src/MiningCore/AutoMapperProfile.cs b/src/MiningCore/AutoMapperProfile.cs index cbe6e2196..a2f804026 100644 --- a/src/MiningCore/AutoMapperProfile.cs +++ b/src/MiningCore/AutoMapperProfile.cs @@ -58,6 +58,9 @@ public AutoMapperProfile() .ForMember(dest => dest.LastPayment, opt => opt.Ignore()) .ForMember(dest => dest.LastPaymentLink, opt => opt.Ignore()); + CreateMap(); + CreateMap(); + // PostgreSQL CreateMap(); CreateMap(); diff --git a/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs b/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs index 7cd030bc5..206fdb8d7 100644 --- a/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Bitcoin/BitcoinPayoutHandler.cs @@ -178,6 +178,7 @@ public virtual async Task ClassifyBlocksAsync(Block[] blocks) logger.Info(() => $"[{LogCategory}] Block {block.BlockHeight} classified as orphaned. Category: {transactionInfo.Details[0].Category}"); block.Status = BlockStatus.Orphaned; + block.Reward = 0; result.Add(block); break; } @@ -241,7 +242,7 @@ public virtual async Task PayoutAsync(Balance[] balances) args = new object[] { - string.Empty, // default account + string.Empty, // default account amounts, // addresses and associated amounts 1, // only spend funds covered by this many confirmations comment, // tx comment @@ -253,7 +254,7 @@ public virtual async Task PayoutAsync(Balance[] balances) { args = new object[] { - string.Empty, // default account + string.Empty, // default account amounts, // addresses and associated amounts }; } diff --git a/src/MiningCore/Blockchain/Ethereum/EthereumPayoutHandler.cs b/src/MiningCore/Blockchain/Ethereum/EthereumPayoutHandler.cs index 78d87d4f5..771ac68be 100644 --- a/src/MiningCore/Blockchain/Ethereum/EthereumPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Ethereum/EthereumPayoutHandler.cs @@ -225,6 +225,7 @@ public async Task ClassifyBlocksAsync(Block[] blocks) { // we've lost this one block.Status = BlockStatus.Orphaned; + block.Reward = 0; } } } diff --git a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs index 47dbca103..1919b0fe4 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs @@ -339,6 +339,7 @@ public async Task ClassifyBlocksAsync(Block[] blocks) if (blockHeader.IsOrphaned || blockHeader.Hash != block.TransactionConfirmationData) { block.Status = BlockStatus.Orphaned; + block.Reward = 0; continue; } diff --git a/src/MiningCore/Persistence/Model/Projections/MinerStats.cs b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs index d50b61a15..fbf30ec3f 100644 --- a/src/MiningCore/Persistence/Model/Projections/MinerStats.cs +++ b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs @@ -18,14 +18,29 @@ portions of the Software. SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +using System; +using System.Collections.Generic; + namespace MiningCore.Persistence.Model.Projections { + public class WorkerPerformanceStats + { + public double Hashrate { get; set; } + public double SharesPerSecond { get; set; } + } + + public class WorkerPerformanceStatsContainer + { + public DateTime Created { get; set; } + public Dictionary Workers { get; set; } + } + public class MinerStats { public ulong PendingShares { get; set; } public decimal PendingBalance { get; set; } public decimal TotalPaid { get; set; } public Payment LastPayment { get; set; } - public MinerWorkerPerformanceStats[] PerformanceStats { get; set; } + public WorkerPerformanceStatsContainer Performance { get; set; } } } diff --git a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs index 00e35cc6f..5bdd871c0 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/ShareRepository.cs @@ -140,7 +140,7 @@ public MinerWorkerHashes[] GetHashAccumulationBetweenCreated(IDbConnection con, { logger.LogInvoke(new[] { poolId }); - var query = "SELECT SUM(difficulty), COUNT(difficulty), min(created) as firstshare, max(created) as lastshare, miner, worker FROM shares " + + var query = "SELECT SUM(difficulty), COUNT(difficulty), MIN(created) AS firstshare, MAX(created) AS lastshare, miner, worker FROM shares " + "WHERE poolid = @poolId AND created >= @start AND created <= @end " + "GROUP BY miner, worker"; diff --git a/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs b/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs index 6ced6bf24..97167b328 100644 --- a/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs +++ b/src/MiningCore/Persistence/Postgres/Repositories/StatsRepository.cs @@ -96,12 +96,12 @@ public PoolStats[] GetPoolStatsBetweenHourly(IDbConnection con, string poolId, D logger.LogInvoke(new []{ poolId }); var query = "SELECT date_trunc('hour', created) AS created, " + - " AVG(poolhashrate) AS poolhashrate, " + - " CAST(AVG(connectedminers) AS BIGINT) AS connectedminers " + - "FROM poolstats " + - "WHERE poolid = @poolId AND created >= @start AND created <= @end " + - "GROUP BY date_trunc('hour', created) " + - "ORDER BY created;"; + "AVG(poolhashrate) AS poolhashrate, " + + "CAST(AVG(connectedminers) AS BIGINT) AS connectedminers " + + "FROM poolstats " + + "WHERE poolid = @poolId AND created >= @start AND created <= @end " + + "GROUP BY date_trunc('hour', created) " + + "ORDER BY created;"; return con.Query(query, new { poolId, start, end }) .Select(mapper.Map) @@ -140,11 +140,49 @@ public MinerStats GetMinerStats(IDbConnection con, IDbTransaction tx, string poo .Select(mapper.Map) .ToArray(); - result.PerformanceStats = stats; + if (stats.Any()) + { + // replace null worker with empty string + foreach(var stat in stats) + { + if (stat.Worker == null) + { + stat.Worker = string.Empty; + break; + } + } + + // transform to dictionary + result.Performance = new WorkerPerformanceStatsContainer + { + Workers = stats.ToDictionary(x => x.Worker, x => new WorkerPerformanceStats + { + Hashrate = x.Hashrate, + SharesPerSecond = x.SharesPerSecond + }), + + Created = stats.First().Created + }; + } } } return result; } + + public MinerWorkerPerformanceStats[] GetMinerStatsBetweenHourly(IDbConnection con, string poolId, string miner, DateTime start, DateTime end) + { + logger.LogInvoke(new[] { poolId }); + + var query = "SELECT worker, date_trunc('hour', created) AS created, AVG(hashrate) AS hashrate, " + + "AVG(sharespersecond) AS sharespersecond FROM minerstats " + + "WHERE poolid = @poolId AND miner = @miner AND created >= @start AND created <= @end " + + "GROUP BY date_trunc('hour', created), worker " + + "ORDER BY created, worker;"; + + return con.Query(query, new { poolId, miner, start, end }) + .Select(mapper.Map) + .ToArray(); + } } } diff --git a/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql b/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql index 0759b5fa8..525960a9f 100644 --- a/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql +++ b/src/MiningCore/Persistence/Postgres/Scripts/createdb.sql @@ -86,4 +86,4 @@ CREATE TABLE minerstats ); CREATE INDEX IDX_MINERSTATS_POOL_MINER_CREATED on minerstats(poolid, miner, created); -CREATE INDEX IDX_MINERSTATS_POOL_MINER_CREATED_HOUR on minerstats(poolid, miner, worker, date_trunc('hour',created)); +CREATE INDEX IDX_MINERSTATS_POOL_MINER_CREATED_HOUR on minerstats(poolid, miner, date_trunc('hour',created)); diff --git a/src/MiningCore/Persistence/Repositories/IStatsRepository.cs b/src/MiningCore/Persistence/Repositories/IStatsRepository.cs index 80a24dcb5..e16467229 100644 --- a/src/MiningCore/Persistence/Repositories/IStatsRepository.cs +++ b/src/MiningCore/Persistence/Repositories/IStatsRepository.cs @@ -20,9 +20,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. using System; using System.Data; -using Dapper; using MiningCore.Persistence.Model; -using MiningCore.Persistence.Model.Projections; using MinerStats = MiningCore.Persistence.Model.Projections.MinerStats; namespace MiningCore.Persistence.Repositories @@ -30,7 +28,7 @@ namespace MiningCore.Persistence.Repositories public interface IStatsRepository { void InsertPoolStats(IDbConnection con, IDbTransaction tx, PoolStats stats); - void InsertMinerWorkerPerformanceStats(IDbConnection con, IDbTransaction tx, Model.MinerWorkerPerformanceStats stats); + void InsertMinerWorkerPerformanceStats(IDbConnection con, IDbTransaction tx, MinerWorkerPerformanceStats stats); PoolStats GetLastPoolStats(IDbConnection con, string poolId); PoolStats[] PagePoolStatsBetween(IDbConnection con, string poolId, DateTime start, DateTime end, int page, int pageSize); PoolStats[] GetPoolStatsBetweenHourly(IDbConnection con, string poolId, DateTime start, DateTime end); From 07902b32e76ce0494936e29b1534fd33fbba9fcf Mon Sep 17 00:00:00 2001 From: Calvin Tam Date: Tue, 2 Jan 2018 14:25:03 +0800 Subject: [PATCH 19/24] Added Electroneum in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index dfcc6e94b..c5d9fb0de 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Namecoin | Yes | No | | Viacoin | Yes | No | | Peercoin | Yes | No | | Straks | Yes | Yes | | +Electroneum | Yes | No | | #### Ethereum Miningcore implements the [Ethereum stratum mining protocol](https://github.com/nicehash/Specifications/blob/master/EthereumStratum_NiceHash_v1.0.0.txt) authored by NiceHash. This protocol is implemented by all major Ethereum miners. From 2868c84d42eeea85b432cb0fb069122658df6cad Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 2 Jan 2018 12:32:04 +0100 Subject: [PATCH 20/24] Improved Monero transfer failure handling --- .../Blockchain/Monero/MoneroPayoutHandler.cs | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs index 1919b0fe4..d005babe3 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs @@ -156,27 +156,27 @@ private async Task PayoutBatch(Balance[] balances) // send command var transferResponse = await walletDaemon.ExecuteCmdSingleAsync(MWC.Transfer, request); - if (walletSupportsTransferSplit) + // gracefully handle error -4 (transaction would be too large. try /transfer_split) + if (transferResponse.Error?.Code == -4) { - // gracefully handle error -4 (transaction would be too large. try /transfer_split) - if (transferResponse.Error?.Code == -4) + if (walletSupportsTransferSplit) { logger.Info(() => $"[{LogCategory}] Retrying transfer using {MWC.TransferSplit}"); var transferSplitResponse = await walletDaemon.ExecuteCmdSingleAsync(MWC.TransferSplit, request); - HandleTransferResponse(transferSplitResponse, balances); - } - else - HandleTransferResponse(transferResponse, balances); - } + // gracefully handle error -4 (transaction would be too large. try /transfer_split) + if (transferResponse.Error?.Code != -4) + { + HandleTransferResponse(transferSplitResponse, balances); + return; + } + } - else - { // retry paged var validBalances = balances.Where(x => x.Amount > 0).ToArray(); var pageSize = 10; - var pageCount = (int) Math.Ceiling((double) validBalances.Length / pageSize); + var pageCount = (int)Math.Ceiling((double)validBalances.Length / pageSize); for (var i = 0; i < pageCount; i++) { @@ -191,7 +191,7 @@ private async Task PayoutBatch(Balance[] balances) .Select(x => new TransferDestination { Address = x.Address, - Amount = (ulong) Math.Floor(x.Amount * MoneroConstants.Piconero) + Amount = (ulong)Math.Floor(x.Amount * MoneroConstants.Piconero) }).ToArray(); transferResponse = await walletDaemon.ExecuteCmdSingleAsync(MWC.Transfer, request); @@ -201,6 +201,9 @@ private async Task PayoutBatch(Balance[] balances) break; } } + + else + HandleTransferResponse(transferResponse, balances); } private async Task PayoutToPaymentId(Balance balance) From 69d818cf1f8132cbe856e4a056cd811482709839 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 2 Jan 2018 12:36:56 +0100 Subject: [PATCH 21/24] WIP --- src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs index d005babe3..2dd396ab2 100644 --- a/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs +++ b/src/MiningCore/Blockchain/Monero/MoneroPayoutHandler.cs @@ -174,6 +174,8 @@ private async Task PayoutBatch(Balance[] balances) } // retry paged + logger.Info(() => $"[{LogCategory}] Retrying paged"); + var validBalances = balances.Where(x => x.Amount > 0).ToArray(); var pageSize = 10; var pageCount = (int)Math.Ceiling((double)validBalances.Length / pageSize); @@ -194,6 +196,8 @@ private async Task PayoutBatch(Balance[] balances) Amount = (ulong)Math.Floor(x.Amount * MoneroConstants.Piconero) }).ToArray(); + logger.Info(() => $"[{LogCategory}] Page {i + 1}: Paying out {FormatAmount(page.Sum(x => x.Amount))} to {page.Length} addresses"); + transferResponse = await walletDaemon.ExecuteCmdSingleAsync(MWC.Transfer, request); HandleTransferResponse(transferResponse, page); From 374f86adc61c0cc360886b12271725b7a7886b4f Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 2 Jan 2018 14:32:17 +0100 Subject: [PATCH 22/24] Do not emit UTF-8 BOM in Api responses (Fixes #124) --- src/MiningCore/Api/ApiServer.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/MiningCore/Api/ApiServer.cs b/src/MiningCore/Api/ApiServer.cs index ff90b14d5..88d2cb175 100644 --- a/src/MiningCore/Api/ApiServer.cs +++ b/src/MiningCore/Api/ApiServer.cs @@ -100,6 +100,7 @@ public ApiServer( private readonly List pools = new List(); private IWebHost webHost; private static readonly ILogger logger = LogManager.GetCurrentClassLogger(); + private static readonly Encoding encoding = new UTF8Encoding(false); private static readonly JsonSerializer serializer = new JsonSerializer { @@ -120,7 +121,7 @@ private async Task SendJson(HttpContext context, object response) using (var stream = context.Response.Body) { - using (var writer = new StreamWriter(stream, Encoding.UTF8)) + using (var writer = new StreamWriter(stream, encoding)) { serializer.Serialize(writer, response); From b8182b6c5795fc84e52fd1481711f22d87f04ce0 Mon Sep 17 00:00:00 2001 From: Andrew Camilleri Date: Tue, 2 Jan 2018 14:43:34 +0100 Subject: [PATCH 23/24] cleanup masternode code for straks (#140) --- .../StraksBlockTemplateResponse.cs | 9 -------- src/MiningCore/Blockchain/Straks/StraksJob.cs | 21 +++++++------------ 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/src/MiningCore/Blockchain/Straks/DaemonResponses/StraksBlockTemplateResponse.cs b/src/MiningCore/Blockchain/Straks/DaemonResponses/StraksBlockTemplateResponse.cs index c47f2f690..de06cea9b 100644 --- a/src/MiningCore/Blockchain/Straks/DaemonResponses/StraksBlockTemplateResponse.cs +++ b/src/MiningCore/Blockchain/Straks/DaemonResponses/StraksBlockTemplateResponse.cs @@ -22,13 +22,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. namespace MiningCore.Blockchain.Straks.DaemonResponses { - public class StraksMasternode - { - public string Payee { get; set; } - public string Script { get; set; } - public long Amount { get; set; } - } - public class StraksCoinbaseTransaction { public string Data { get; set; } @@ -56,8 +49,6 @@ public class StraksBlockTemplate : Bitcoin.DaemonResponses.BlockTemplate [JsonProperty("payee_amount")] public long? PayeeAmount { get; set; } - public StraksMasternode Masternode { get; set; } - [JsonProperty("masternode_payments")] public bool MasternodePaymentsStarted { get; set; } diff --git a/src/MiningCore/Blockchain/Straks/StraksJob.cs b/src/MiningCore/Blockchain/Straks/StraksJob.cs index 5b2c98b99..2a149a8a6 100644 --- a/src/MiningCore/Blockchain/Straks/StraksJob.cs +++ b/src/MiningCore/Blockchain/Straks/StraksJob.cs @@ -46,6 +46,12 @@ protected override Transaction CreateOutputTransaction() return tx; } + private bool ShouldHandleMasternodePayment() + { + return BlockTemplate.MasternodePaymentsStarted && + BlockTemplate.MasternodePaymentsEnforced && + !string.IsNullOrEmpty(BlockTemplate.Payee) && BlockTemplate.PayeeAmount.HasValue; + } private Money CreateStraksOutputs(Transaction tx, Money reward) { @@ -59,21 +65,10 @@ private Money CreateStraksOutputs(Transaction tx, Money reward) reward -= treasuryReward; } - if (!string.IsNullOrEmpty(BlockTemplate.Masternode?.Payee)) - { - var payeeAddress = BitcoinUtils.AddressToDestination(BlockTemplate.Masternode.Payee); - var payeeReward = BlockTemplate.Masternode.Amount; - - reward -= payeeReward; - rewardToPool -= payeeReward; - - tx.AddOutput(payeeReward, payeeAddress); - } - - if (!string.IsNullOrEmpty(BlockTemplate.Payee)) + if (ShouldHandleMasternodePayment()) { var payeeAddress = BitcoinUtils.AddressToDestination(BlockTemplate.Payee); - var payeeReward = BlockTemplate.PayeeAmount ?? reward / 5; + var payeeReward = BlockTemplate.PayeeAmount.Value; reward -= payeeReward; rewardToPool -= payeeReward; From 7531411597ebdeca92ccc67e7964c5c134acf862 Mon Sep 17 00:00:00 2001 From: Oliver Weichhold Date: Tue, 2 Jan 2018 14:48:34 +0100 Subject: [PATCH 24/24] Fix syntax error --- src/MiningCore/Persistence/Model/Projections/MinerStats.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/MiningCore/Persistence/Model/Projections/MinerStats.cs b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs index 5091541aa..5f0f5da61 100644 --- a/src/MiningCore/Persistence/Model/Projections/MinerStats.cs +++ b/src/MiningCore/Persistence/Model/Projections/MinerStats.cs @@ -35,8 +35,6 @@ public class WorkerPerformanceStatsContainer public Dictionary Workers { get; set; } } -namespace MiningCore.Persistence.Model.Projections -{ public class MinerStats { public ulong PendingShares { get; set; }