From 885948543b79739a4db44ee3250e7c29067e0d24 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 11 Aug 2018 07:44:59 +0900 Subject: [PATCH 01/17] support richlist * Account db added. * dynamic update accounts --- db.js | 12 +++ public/js/controllers/AccountsController.js | 72 ++++++++++++++ public/js/main.js | 21 ++++ public/views/accounts.html | 37 +++++++ routes/index.js | 2 + routes/richlist.js | 102 ++++++++++++++++++++ tools/sync.js | 76 ++++++++++++++- 7 files changed, 321 insertions(+), 1 deletion(-) create mode 100755 public/js/controllers/AccountsController.js create mode 100755 public/views/accounts.html create mode 100644 routes/richlist.js diff --git a/db.js b/db.js index 5496c1659..1d7a765ed 100644 --- a/db.js +++ b/db.js @@ -24,6 +24,14 @@ var Block = new Schema( "uncles": [String] }); +var Account = new Schema( +{ + "address": {type: String, index: {unique: true}}, + "balance": Number, + "blockNumber": Number, + "type": Number // address: 0x0, contract: 0x1 +}); + var Contract = new Schema( { "address": {type: String, index: {unique: true}}, @@ -70,16 +78,20 @@ var BlockStat = new Schema( Transaction.index({blockNumber:-1}); Transaction.index({from:1, blockNumber:-1}); Transaction.index({to:1, blockNumber:-1}); +Account.index({balance:-1}); +Account.index({balance:-1, blockNumber:-1}); Block.index({miner:1}); mongoose.model('BlockStat', BlockStat); mongoose.model('Block', Block); +mongoose.model('Account', Account); mongoose.model('Contract', Contract); mongoose.model('Transaction', Transaction); module.exports.BlockStat = mongoose.model('BlockStat'); module.exports.Block = mongoose.model('Block'); module.exports.Contract = mongoose.model('Contract'); module.exports.Transaction = mongoose.model('Transaction'); +module.exports.Account = mongoose.model('Account'); mongoose.Promise = global.Promise; mongoose.connect(process.env.MONGO_URI || 'mongodb://localhost/blockDB', { diff --git a/public/js/controllers/AccountsController.js b/public/js/controllers/AccountsController.js new file mode 100755 index 000000000..18fdb7a76 --- /dev/null +++ b/public/js/controllers/AccountsController.js @@ -0,0 +1,72 @@ +angular.module('BlocksApp').controller('AccountsController', function($stateParams, $rootScope, $scope, $http, $filter) { + $scope.settings = $rootScope.setup; + + // fetch accounts + var getAccounts = function() { + $("#table_accounts").DataTable({ + processing: true, + serverSide: true, + paging: true, + ajax: function(data, callback, settings) { + // get totalSupply only once. + data.totalSupply = $scope.totalSupply || -1; + data.recordsTotal = $scope.totalAccounts || 0; + $http.post('/richlist', data).then(function(resp) { + // set the totalSupply + if (resp.data.totalSupply) { + $scope.totalSupply = resp.data.totalSupply; + } + // set the number of total accounts + $scope.totalAccounts = resp.data.recordsTotal; + + // fixup data to show percentages + var newdata = resp.data.data.map(function(item) { + return [item[0], item[1], item[2], item[3], (item[3] / $scope.totalSupply) * 100, item[4]]; + }); + resp.data.data = newdata; + callback(resp.data); + }); + }, + lengthMenu: [ + [20, 50, 100, 150, -1], + [20, 50, 100, 150, "All"] // change per page values here + ], + pageLength: 20, + order: [ + [3, "desc"] + ], + language: { + lengthMenu: "_MENU_ accounts", + zeroRecords: "No accounts found", + infoEmpty: "", + infoFiltered: "(filtered from _MAX_ total accounts)" + }, + columnDefs: [ + { orderable: false, "targets": [0,1,2,4] }, + { + render: + function(data, type, row) { + return '' + data + '' + }, + targets: [1] + }, + { + render: + function(data, type, row) { + return $filter('number')(data, 8); + }, + targets: [3] + }, + { + render: + function(data, type, row) { + return $filter('number')(data, 4) + ' %'; + }, + targets: [4] + } + ] + }); + }; + + getAccounts(); +}); diff --git a/public/js/main.js b/public/js/main.js index 59d9bed2c..6ae01e19e 100755 --- a/public/js/main.js +++ b/public/js/main.js @@ -139,6 +139,27 @@ BlocksApp.config(['$stateProvider', '$urlRouterProvider', function($stateProvide }] } }) + .state('accounts', { + url: "/accounts", + templateUrl: "views/accounts.html", + data: {pageTitle: 'Accounts'}, + controller: "AccountsController", + resolve: { + deps: ['$ocLazyLoad', function($ocLazyLoad) { + return $ocLazyLoad.load({ + name: 'BlocksApp', + insertBefore: '#ng_load_plugins_before', // load the above css files before '#ng_load_plugins_before' + files: [ + '/js/controllers/AccountsController.js', + '/plugins/datatables/datatables.min.css', + '/plugins/datatables/datatables.bootstrap.css', + '/plugins/datatables/datatables.all.min.js', + '/plugins/datatables/datatable.min.js' + ] + }); + }] + } + }) .state('block', { url: "/block/{number}", templateUrl: "views/block.html", diff --git a/public/views/accounts.html b/public/views/accounts.html new file mode 100755 index 000000000..3fd5a238a --- /dev/null +++ b/public/views/accounts.html @@ -0,0 +1,37 @@ +
+
+
+
Overview
+
+
+
+
+
+ Total supply: {{ totalSupply | number: 2 }} {{ settings.symbol }} +
+ Total {{ totalAccounts | number }} accounts +
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + +
#AddressTypeBalancePercent
+
+
+
diff --git a/routes/index.js b/routes/index.js index e2b8b3680..46c230147 100644 --- a/routes/index.js +++ b/routes/index.js @@ -15,6 +15,7 @@ module.exports = function(app){ var compile = require('./compiler'); var fiat = require('./fiat'); var stats = require('./stats'); + var richList = require('./richlist'); /* Local DB: data request format @@ -22,6 +23,7 @@ module.exports = function(app){ { "tx": "0x1234blah" } { "block": "1234" } */ + app.post('/richlist', richList); app.post('/addr', getAddr); app.post('/addr_count', getAddrCounter); app.post('/tx', getTx); diff --git a/routes/richlist.js b/routes/richlist.js new file mode 100644 index 000000000..3826ab982 --- /dev/null +++ b/routes/richlist.js @@ -0,0 +1,102 @@ +#!/usr/bin/env node +/** + * Endpoint for richlist + */ + +var async = require('async'); +var mongoose = require('mongoose'); + +require( '../db.js' ); +var Account = mongoose.model('Account'); + +var getAccounts = function(req, res) { + // count accounts only once + var count = req.body.recordsTotal || 0; + count = parseInt(count); + if (count < 0) { + count = 0; + } + + // get totalSupply only once + var queryTotalSupply = req.body.totalSupply || null; + + async.waterfall([ + function(callback) { + if (queryTotalSupply < 0) { + Account.aggregate([ + { $group: { _id: null, totalSupply: { $sum: '$balance' } } } + ]).exec(function(err, docs) { + if (err) { + callbck(err); + return; + } + + var totalSupply = docs[0].totalSupply; + callback(null, totalSupply); + }); + } else { + callback(null, null); + } + }, + function(totalSupply, callback) { + if (!count) { + // get the number of all accounts + Account.count({}, function(err, count) { + if (err) { + callbck(err); + return; + } + + count = parseInt(count); + callback(null, totalSupply, count); + }); + } else { + callback(null, totalSupply, count); + } + } + ], function(error, totalSupply, count) { + if (error) { + res.write(JSON.stringify({"error": true})); + res.end(); + return; + } + + // check sort order + var sortOrder = '-balance'; + if (req.body.order && req.body.order[0] && req.body.order[0].column) { + // balance column + if (req.body.order[0].column == 3) { + if (req.body.order[0].dir == 'asc') { + sortOrder = 'balance'; + } + } + } + + // set datatable params + var limit = parseInt(req.body.length); + var start = parseInt(req.body.start); + + var data = { draw: parseInt(req.body.draw), recordsFiltered: count, recordsTotal: count }; + if (totalSupply > 0) { + data.totalSupply = totalSupply; + } + + Account.find({}).lean(true).sort(sortOrder).skip(start).limit(limit) + .exec(function (err, accounts) { + if (err) { + res.write(JSON.stringify({"error": true})); + res.end(); + return; + } + + data.data = accounts.map(function(account, i) { + return [i + 1 + start, account.address, account.type == 0 ? "Account" : "Contract", account.balance, account.blockNumber]; + }); + res.write(JSON.stringify(data)); + res.end(); + } + ); + }); +} + +module.exports = getAccounts; diff --git a/tools/sync.js b/tools/sync.js index 217d5b9c2..6a19a496d 100644 --- a/tools/sync.js +++ b/tools/sync.js @@ -9,11 +9,13 @@ var etherUnits = require("../lib/etherUnits.js"); var BigNumber = require('bignumber.js'); var _ = require('lodash'); +var async = require('async'); var Web3 = require('web3'); var mongoose = require( 'mongoose' ); var Block = mongoose.model( 'Block' ); var Transaction = mongoose.model( 'Transaction' ); +var Account = mongoose.model( 'Account' ); /** //Just listen for latest blocks and sync from the start of the app. @@ -137,6 +139,13 @@ var writeTransactionsToDB = function(config, blockData, flush) { self.bulkOps = []; self.blocks = 0; } + // save miner addresses + if (!self.miners) { + self.miners = []; + } + if (blockData) { + self.miners.push({ address: blockData.miner, blockNumber: blockData.blockNumber, type: 0 }); + } if (blockData && blockData.transactions.length > 0) { for (d in blockData.transactions) { var txData = blockData.transactions[d]; @@ -152,8 +161,73 @@ var writeTransactionsToDB = function(config, blockData, flush) { var bulk = self.bulkOps; self.bulkOps = []; self.blocks = 0; - if(bulk.length == 0) return; + var miners = self.miners; + self.miners = []; + + // setup accounts + var data = {}; + bulk.forEach(function(tx) { + data[tx.from] = { address: tx.from, blockNumber: tx.blockNumber, type: 0 }; + if (tx.to) { + data[tx.to] = { address: tx.to, blockNumber: tx.blockNumber, type: 0 }; + } + }); + + // setup miners + miners.forEach(function(miner) { + data[miner.address] = miner; + }); + + var accounts = Object.keys(data); + + if (bulk.length == 0 && accounts.length == 0) return; + + // update balances + if (accounts.length > 0) + async.eachSeries(accounts, function(account, eachCallback) { + var blockNumber = data[account].blockNumber; + // get contract account type + web3.eth.getCode(account, function(err, code) { + if (err) { + console.log("ERROR: fail to getCode(" + account + ")"); + return eachCallback(err); + } + if (code.length > 2) { + data[account].type = 1; // contract type + } + + web3.eth.getBalance(account, blockNumber, function(err, balance) { + if (err) { + console.log("ERROR: fail to getBalance(" + account + ")"); + return eachCallback(err); + } + + //data[account].balance = web3.fromWei(balance, 'ether'); + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether /= 1e18; + } + data[account].balance = ether; + eachCallback(); + }); + }); + }, function(err) { + var n = 0; + accounts.forEach(function(account) { + n++; + if (n <= 5) { + console.log(' - upsert ' + account + ' / balance = ' + data[account].balance); + } else if (n == 6) { + console.log(' (...) total ' + accounts.length + ' accounts updated.'); + } + // upsert account + Account.collection.update({ address: account }, { $set: data[account] }, { upsert: true }); + }); + }); + if (bulk.length > 0) Transaction.collection.insert(bulk, function( err, tx ){ if ( typeof err !== 'undefined' && err ) { if (err.code == 11000) { From 59fec5fff5f12060b734631524662133f3c03925 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 11 Aug 2018 16:07:56 +0900 Subject: [PATCH 02/17] tools/richlist.js added to support richlist feature using the parity --- tools/richlist.js | 184 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 tools/richlist.js diff --git a/tools/richlist.js b/tools/richlist.js new file mode 100644 index 000000000..bff5c398d --- /dev/null +++ b/tools/richlist.js @@ -0,0 +1,184 @@ +#!/usr/bin/env node +/** + * Tool for calculating richlist by hackyminer + */ + +var _ = require('lodash'); +var Web3 = require('web3'); +var async = require('async'); +var BigNumber = require('bignumber.js'); +var mongoose = require('mongoose'); + +var Account = require('../db.js').Account; + +mongoose.connect(process.env.MONGO_URI || 'mongodb://localhost/blockDB'); + +function makeParityRichList(number, offset, blockNumber, updateCallback) { + var self = makeParityRichList; + if (!self.index) { + self.index = 0; + } + number = number || 100; + offset = offset || null; + + async.waterfall([ + function(callback) { + web3.parity.listAccounts(number, offset, blockNumber, function(err, result) { + callback(err, result); + }); + }, function(accounts, callback) { + if (!accounts) { + return callback({ + error: true, + message: "No accounts found. Please restart Parity with --fat-db=on option to enable FatDB." + }); + } + + if (accounts.length === 0) { + return callback({ + error: true, + message: "No more accounts found." + }); + } + + var lastAccount = accounts[accounts.length - 1]; + var data = {}; + + // Please see https://github.com/gobitfly/etherchain-light by gobitfly + async.eachSeries(accounts, function(account, eachCallback) { + web3.eth.getCode(account, function(err, code) { + if (err) { + console.log("ERROR: fail to getCode(" + account + ")"); + return eachCallback(err); + } + data[account] = {}; + data[account].address = account; + data[account].type = code.length > 2 ? 1 : 0; // 0: address, 1: contract + + web3.eth.getBalance(account, blockNumber, function(err, balance) { + if (err) { + console.log("ERROR: fail to getBalance(" + account + ")"); + return eachCallback(err); + } + + //data[account].balance = web3.fromWei(balance, 'ether'); + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether /= 1e18; + } + data[account].balance = ether; + eachCallback(); + }); + }); + }, function(err) { + callback(err, data, lastAccount); + }); + } + ], function(error, accounts, lastAccount) { + if (error) { + console.log(error); + process.exit(9); + return; + } + + //console.log(JSON.stringify(accounts, null, 2)); + offset = lastAccount; + let j = Object.keys(accounts).length; + self.index += j; + console.log(' * ' + j + ' / ' + self.index + ' accounts, offset = ' + offset); + if (updateCallback) { + updateCallback(accounts, blockNumber); + } + setTimeout(function() { + makeParityRichList(number, lastAccount, blockNumber, updateCallback); + }, 300); + }); +} + +/** + * Write accounts to DB + */ +var updateAccounts = function(accounts, blockNumber) { + var bulk = Object.keys(accounts).map(function(j) { + let account = accounts[j]; + account.blockNumber = blockNumber; + return account; + }); + + Account.collection.insert(bulk, function(error, data) { + if (error) { + if (error.code == 11000) { + async.eachSeries(bulk, function(item, eachCallback) { + // upsert accounts + delete item._id; // remove _id field + Account.collection.update({ "address": item.address }, { $set: item }, { upsert: true }, function(err, updated) { + if (err) { + if (!config.quiet) { + console.log('WARN: Duplicate DB key : ' + error); + console.log('ERROR: Fail to update account: ' + err); + } + return eachCallback(err); + } + eachCallback(); + }); + }, function(err) { + if (err) { + console.log('ERROR: Aborted due to error: ' + err); + process.exit(9); + return; + } + console.log('* ' + bulk.length + ' accounts successfully updated.'); + }); + } else { + console.log('Error: Aborted due to error on DB: ' + error); + process.exit(9); + } + } else { + console.log('* ' + data.insertedCount + ' accounts successfully inserted.'); + } + }); +} + +/** + * Start config for node connection and sync + */ +var config = { nodeAddr: 'localhost', 'gethPort': 8545 }; +// load the config.json file +try { + var loaded = require('../config.json'); + _.extend(config, loaded); + console.log('config.json found.'); +} catch (error) { + console.log('No config file found.'); + throw error; + process.exit(1); +} + +// temporary turn on some debug +//config.quiet = false; +//mongoose.set('debug', true); + +console.log('Connecting ' + config.nodeAddr + ':' + config.gethPort + '...'); + +var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString())); + +var useParity = false; +if (web3.version.node.split('/')[0].toLowerCase().includes('parity')) { + // load parity extension + web3 = require("../lib/trace.js")(web3); + useParity = true; +} + +var latestBlock = web3.eth.blockNumber; + +// run +console.log("* latestBlock = " + latestBlock); + +if (useParity) { + makeParityRichList(100, null, latestBlock, updateAccounts); +} else { + console.log("Sorry, currently only Parity is supported."); + process.exit(1); +} From ad6e2348dc57d596623a82f157d4d014f1045282 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Thu, 28 Jun 2018 03:43:14 +0900 Subject: [PATCH 03/17] parity extensions added (web3.trace, web3.parity) --- lib/trace.js | 100 ++++++++++++++++++++++++++++++++++++++++++++ routes/web3relay.js | 5 +++ 2 files changed, 105 insertions(+) create mode 100644 lib/trace.js diff --git a/lib/trace.js b/lib/trace.js new file mode 100644 index 000000000..d5c81d516 --- /dev/null +++ b/lib/trace.js @@ -0,0 +1,100 @@ +/** + * @author Alexis Roussel + * @author Peter Pratscher + * @date 2017 + * @license LGPL + * @changelog 2018/05/19 - modified for web3.js 0.20.x using _extend() method. (by hackyminer ) + */ +module.exports = function(web3) { + /** + * @file trace.js + * @author Alexis Roussel + * @date 2017 + * @license LGPL + */ + web3._extend({ + property: 'trace', + methods: [ + new web3._extend.Method({ + name: 'call', + call: 'trace_call', + params: 3, + inputFormatter: [web3._extend.formatters.inputCallFormatter, null, web3._extend.formatters.inputDefaultBlockNumberFormatter] + }), + + new web3._extend.Method({ + name: 'rawTransaction', + call: 'trace_rawTransaction', + params: 2 + }), + + new web3._extend.Method({ + name: 'replayTransaction', + call: 'trace_replayTransaction', + params: 2 + }), + + new web3._extend.Method({ + name: 'block', + call: 'trace_block', + params: 1, + inputFormatter: [web3._extend.formatters.inputDefaultBlockNumberFormatter] + }), + + new web3._extend.Method({ + name: 'filter', + call: 'trace_filter', + params: 1 + }), + + new web3._extend.Method({ + name: 'get', + call: 'trace_get', + params: 2 + }), + + new web3._extend.Method({ + name: 'transaction', + call: 'trace_transaction', + params: 1 + }) + ] + }); + + /** + * @file parity.js + * @author Peter Pratscher + * @date 2017 + * @license LGPL + */ + web3._extend({ + property: 'parity', + methods: [ + new web3._extend.Method({ + name: 'pendingTransactions', + call: 'parity_pendingTransactions', + params: 0, + outputFormatter: web3._extend.formatters.outputTransactionFormatter + }), + + new web3._extend.Method({ + name: 'pendingTransactionsStats', + call: 'parity_pendingTransactionsStats', + params: 0 + }), + + new web3._extend.Method({ + name: 'listAccounts', + call: 'parity_listAccounts', + params: 2 + }), + + new web3._extend.Method({ + name: 'phraseToAddress', + call: 'parity_phraseToAddress', + params: 1 + }) + ] + }); + return web3; +}; diff --git a/routes/web3relay.js b/routes/web3relay.js index 1a284b158..c6fa91d30 100644 --- a/routes/web3relay.js +++ b/routes/web3relay.js @@ -46,6 +46,11 @@ if (web3.isConnected()) else throw "No connection, please specify web3host in conf.json"; +if (web3.version.node.split('/')[0].toLowerCase().includes('parity')) { + // parity extension + web3 = require("../lib/trace.js")(web3); +} + var newBlocks = web3.eth.filter("latest"); var newTxs = web3.eth.filter("pending"); From 1cda64e4d32a3efb614ec4a73f8c1559985c2408 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 11 Aug 2018 16:40:39 +0900 Subject: [PATCH 04/17] Accounts menu added. --- public/tpl/header.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/public/tpl/header.html b/public/tpl/header.html index a20316ca3..3d7bf5390 100755 --- a/public/tpl/header.html +++ b/public/tpl/header.html @@ -44,6 +44,9 @@
  • Home
    • +
  • + Accounts +
  • Tokens From e77c8a6323c8212171e84baaba01e8fe435b250f Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 11 Aug 2018 22:47:40 +0900 Subject: [PATCH 05/17] fixed parity_listAccounts() params --- lib/trace.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/trace.js b/lib/trace.js index d5c81d516..3feb4590c 100644 --- a/lib/trace.js +++ b/lib/trace.js @@ -86,7 +86,8 @@ module.exports = function(web3) { new web3._extend.Method({ name: 'listAccounts', call: 'parity_listAccounts', - params: 2 + params: 3, + inputFormatter: [null, null, web3._extend.formatters.inputDefaultBlockNumberFormatter] }), new web3._extend.Method({ From 3e373476fb8faf7b6d61446daff4805e8eb03d51 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Wed, 15 Aug 2018 13:55:10 +0900 Subject: [PATCH 06/17] tools: richlist support without parity * makeRichList() added to update accounts from Transaction/Block DB table. * config.settings.genesisAddress option added to get the genesis alloc addresses --- tools/richlist.js | 238 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 4 deletions(-) diff --git a/tools/richlist.js b/tools/richlist.js index bff5c398d..9b0b16fc4 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -10,9 +10,185 @@ var BigNumber = require('bignumber.js'); var mongoose = require('mongoose'); var Account = require('../db.js').Account; +var Transaction = require('../db.js').Transaction; +var Block = require('../db.js').Block; mongoose.connect(process.env.MONGO_URI || 'mongodb://localhost/blockDB'); +const ADDRESS_CACHE_MAX = 10000; // address cache threshold + +function makeRichList(toBlock, blocks, updateCallback) { + var self = makeRichList; + if (!self.cached) { + self.cached = {}; + self.index = 0; + } + if (!self.accounts) { + self.accounts = {}; + } + var fromBlock = toBlock - blocks; + if (fromBlock < 0) { + fromBlock = 0; + } + + console.log('Scan accounts from ' + fromBlock + ' to ' + toBlock + ' ...'); + if (fromBlock == toBlock) { + // scan ended + if (self.accounts && updateCallback) { + var data = self.accounts; + updateCallback(data, toBlock); + self.accounts = {}; + } + console.log("**DONE**"); + return; + } + + async.waterfall([ + function(callback) { + // Transaction.distinct("from", { blockNumber: { $lte: toBlock, $gt: fromBlock } }, function(err, docs) ... + // faster + // dictint("from") + Transaction.aggregate([ + { $match: { blockNumber: { $lte: toBlock, $gt: fromBlock } } }, + { $group: { _id: '$from' }}, + { $project: { "_id": 1 }} + ]).exec(function(err, docs) { + if (err) { + console.log(err); + return; + } + docs.forEach(function(doc) { + // check address cache + if (!self.cached[doc._id]) { + self.accounts[doc._id] = { address: doc._id, type: 0 }; + // increase cache counter + self.cached[doc._id] = 1; + } else { + self.cached[doc._id]++; + } + }); + callback(null); + }); + }, function(callback) { + // dictint("to") + Transaction.aggregate([ + { $match: { blockNumber: { $lte: toBlock, $gt: fromBlock } } }, + { $group: { _id: '$to' }}, + { $project: { "_id": 1 }} + ]).exec(function(err, docs) { + if (err) { + console.log(err); + return; + } + docs.forEach(function(doc) { + // to == null case + if (!doc._id) { + return; + } + if (!self.cached[doc._id]) { + self.accounts[doc._id] = { address: doc._id, type: 0 }; + self.cached[doc._id] = 1; + } else { + self.cached[doc._id]++; + } + }); + callback(null); + }); + }, function(callback) { + // aggregate miner's addresses + Block.aggregate([ + { $match: { blockNumber: { $lte: toBlock, $gt: fromBlock } } }, + { $group: { _id: '$miner' }}, + { $project: { "_id": 1 }} + ]).exec(function(err, docs) { + if (err) { + console.log(err); + return; + } + docs.forEach(function(doc) { + if (!self.cached[doc._id]) { + self.accounts[doc._id] = { address: doc._id, type: 0 }; + self.cached[doc._id] = 1; + } else { + self.cached[doc._id]++; + } + }); + callback(null); + }); + }, function(callback) { + let len = Object.keys(self.accounts).length; + console.info('* ' + len + ' / ' + (self.index + len) + ' total accounts.'); + if (updateCallback && len >= 100) { + self.index += len; + console.log("* update " + len + " accounts ..."); + + var accounts = Object.keys(self.accounts); + var data = self.accounts; + async.eachSeries(accounts, function(account, eachCallback) { + web3.eth.getCode(account, function(err, code) { + if (err) { + return eachCallback(err); + } + if (code.length > 2) { + data[account].type = 1; // contract type + } + + web3.eth.getBalance(account, toBlock, function(err, balance) { + if (err) { + return eachCallback(err); + } + + //data[account].balance = web3.fromWei(balance, 'ether'); + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether /= 1e18; + } + data[account].balance = ether; + eachCallback(); + }); + }); + }, function(err) { + if (err) { + console.log("FATAL: fail to call getBalance() " + err); + process.exit(1); + } + updateCallback(data, toBlock); + self.accounts = {}; + + // check the size of the cached accounts + if (Object.keys(self.cached).length > ADDRESS_CACHE_MAX) { + console.info("** reduce cached accounts ..."); + var sorted = Object.keys(self.cached).sort(function(a, b) { + return self.cached[b] - self.cached[a]; // descend order + }); + var newcached = {}; + var reduce = parseInt(ADDRESS_CACHE_MAX * 0.6); + for (var j = 0; j < reduce; j++) { + newcached[sorted[j]] = self.cached[sorted[j]]; + } + self.cached = newcached; + } + + callback(null); + }); + } else { + callback(null); + } + } + ], function(error) { + if (error) { + console.log(error); + return; + } + + setTimeout(function() { + makeRichList(fromBlock, blocks, updateCallback); + }, 300); + }); +} + function makeParityRichList(number, offset, blockNumber, updateCallback) { var self = makeParityRichList; if (!self.index) { @@ -101,13 +277,28 @@ function makeParityRichList(number, offset, blockNumber, updateCallback) { * Write accounts to DB */ var updateAccounts = function(accounts, blockNumber) { + // prepare var bulk = Object.keys(accounts).map(function(j) { let account = accounts[j]; account.blockNumber = blockNumber; return account; }); - Account.collection.insert(bulk, function(error, data) { + bulkInsert(bulk); +} + +var bulkInsert = function(bulk) { + if (!bulk.length) { + return; + } + + var localbulk; + if (bulk.length > 200) { + localbulk = bulk.splice(0, 100); + } else { + localbulk = bulk.splice(0, 200); + } + Account.collection.insert(localbulk, function(error, data) { if (error) { if (error.code == 11000) { async.eachSeries(bulk, function(item, eachCallback) { @@ -129,7 +320,8 @@ var updateAccounts = function(accounts, blockNumber) { process.exit(9); return; } - console.log('* ' + bulk.length + ' accounts successfully updated.'); + console.log('* ' + localbulk.length + ' accounts successfully updated.'); + bulkInsert(bulk); }); } else { console.log('Error: Aborted due to error on DB: ' + error); @@ -141,6 +333,34 @@ var updateAccounts = function(accounts, blockNumber) { }); } +function prepareJsonAddress(json, type) { + if (!type) { + type = 0; + } + var accounts = {}; + if (json.accounts) { + // genesis.json style + Object.keys(json.accounts).forEach(function(account) { + var key = account.toLowerCase(); + key = '0x' + key.replace(/^0x/, ''); + accounts[key] = { address: key, type: type }; + }); + } else if (typeof json === 'object') { + Object.keys(json).forEach(function(account) { + var key = account.toLowerCase(); + key = '0x' + key.replace(/^0x/, ''); + accounts[key] = { address: key, type: type }; + }); + } else { // normal array + json.forEach(function(account) { + var key = account.toLowerCase(); + key = '0x' + key.replace(/^0x/, ''); + accounts[key] = { address: key, type: type }; + }); + } + return accounts; +} + /** * Start config for node connection and sync */ @@ -179,6 +399,16 @@ console.log("* latestBlock = " + latestBlock); if (useParity) { makeParityRichList(100, null, latestBlock, updateAccounts); } else { - console.log("Sorry, currently only Parity is supported."); - process.exit(1); + // load genesis account + if (config.settings && config.settings.genesisAddress) { + try { + var genesis = require('../' + config.settings.genesisAddress); + var accounts = prepareJsonAddress(genesis); + console.log("* update genesis accounts..."); + updateAccounts(accounts, latestBlock); + } catch (e) { + console.log("Error: Fail to load genesis address (ignore)"); + } + } + makeRichList(latestBlock, 500, updateAccounts); } From 00d2044f6e68bacfbf92900c1ade7da5bb40e126 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Wed, 15 Aug 2018 17:07:38 +0900 Subject: [PATCH 07/17] read genesis accounts correctly --- tools/richlist.js | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/tools/richlist.js b/tools/richlist.js index 9b0b16fc4..aa5f2638c 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -329,6 +329,7 @@ var bulkInsert = function(bulk) { } } else { console.log('* ' + data.insertedCount + ' accounts successfully inserted.'); + bulkInsert(bulk); } }); } @@ -361,6 +362,36 @@ function prepareJsonAddress(json, type) { return accounts; } +function readJsonAccounts(json, blockNumber, callback) { + var data = prepareJsonAddress(json); + console.log("* update genesis accounts..."); + var accounts = Object.keys(data); + async.eachSeries(accounts, function(account, eachCallback) { + web3.eth.getBalance(account, blockNumber, function(err, balance) { + if (err) { + console.log("ERROR: fail to getBalance(" + account + ")"); + return eachCallback(err); + } + + //data[account].balance = web3.fromWei(balance, 'ether'); + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether /= 1e18; + } + data[account].balance = ether; + eachCallback(); + }); + }, function(err) { + if (err) { + console.log("ERROR: fail to getBalance()" + err); + return; + } + callback(data, blockNumber); + }); +} + /** * Start config for node connection and sync */ @@ -403,9 +434,7 @@ if (useParity) { if (config.settings && config.settings.genesisAddress) { try { var genesis = require('../' + config.settings.genesisAddress); - var accounts = prepareJsonAddress(genesis); - console.log("* update genesis accounts..."); - updateAccounts(accounts, latestBlock); + readJsonAccounts(genesis, latestBlock, updateAccounts); } catch (e) { console.log("Error: Fail to load genesis address (ignore)"); } From 4b6cc8369fcbd7d0191266d6af2b994244a26f25 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Thu, 16 Aug 2018 23:38:14 +0900 Subject: [PATCH 08/17] update accounts correctly. --- tools/richlist.js | 125 +++++++++++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/tools/richlist.js b/tools/richlist.js index aa5f2638c..fccc58f8a 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -32,15 +32,10 @@ function makeRichList(toBlock, blocks, updateCallback) { } console.log('Scan accounts from ' + fromBlock + ' to ' + toBlock + ' ...'); + + var ended = false; if (fromBlock == toBlock) { - // scan ended - if (self.accounts && updateCallback) { - var data = self.accounts; - updateCallback(data, toBlock); - self.accounts = {}; - } - console.log("**DONE**"); - return; + ended = true; } async.waterfall([ @@ -118,43 +113,65 @@ function makeRichList(toBlock, blocks, updateCallback) { }, function(callback) { let len = Object.keys(self.accounts).length; console.info('* ' + len + ' / ' + (self.index + len) + ' total accounts.'); - if (updateCallback && len >= 100) { + if (updateCallback && (len >= 100 || ended)) { self.index += len; console.log("* update " + len + " accounts ..."); var accounts = Object.keys(self.accounts); - var data = self.accounts; - async.eachSeries(accounts, function(account, eachCallback) { - web3.eth.getCode(account, function(err, code) { - if (err) { - return eachCallback(err); - } - if (code.length > 2) { - data[account].type = 1; // contract type - } + var chunks = []; + while (accounts.length > 200) { + var chunk = accounts.splice(0, 100); + chunks.push(chunk); + } + if (accounts.length > 0) { + chunks.push(accounts); + } - web3.eth.getBalance(account, toBlock, function(err, balance) { + async.eachSeries(chunks, function(chunk, outerCallback) { + var data = {}; + async.eachSeries(chunk, function(account, eachCallback) { + web3.eth.getCode(account, function(err, code) { if (err) { return eachCallback(err); } - - //data[account].balance = web3.fromWei(balance, 'ether'); - let ether; - if (typeof balance === 'object') { - ether = parseFloat(balance.div(1e18).toString()); + data[account] = { address: account }; + if (code.length > 2) { + data[account].type = 1; // contract type } else { - ether /= 1e18; + data[account].type = self.accounts[account].type; } - data[account].balance = ether; - eachCallback(); + + web3.eth.getBalance(account, function(err, balance) { + if (err) { + return eachCallback(err); + } + + //data[account].balance = web3.fromWei(balance, 'ether'); + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether = balance / 1e18; + } + data[account].balance = ether; + eachCallback(); + }); }); + }, function(err) { + if (err) { + return outerCallback(err); + } + if (data) { + updateCallback(data, toBlock); + } + + outerCallback(); }); - }, function(err) { - if (err) { - console.log("FATAL: fail to call getBalance() " + err); - process.exit(1); + }, function(error) { + if (error) { + console.log("WARN: fail to call getBalance() " + error); } - updateCallback(data, toBlock); + // reset accounts self.accounts = {}; // check the size of the cached accounts @@ -183,9 +200,13 @@ function makeRichList(toBlock, blocks, updateCallback) { return; } - setTimeout(function() { - makeRichList(fromBlock, blocks, updateCallback); - }, 300); + if (ended) { + console.log("**DONE**"); + } else { + setTimeout(function() { + makeRichList(fromBlock, blocks, updateCallback); + }, 300); + } }); } @@ -231,7 +252,7 @@ function makeParityRichList(number, offset, blockNumber, updateCallback) { data[account].address = account; data[account].type = code.length > 2 ? 1 : 0; // 0: address, 1: contract - web3.eth.getBalance(account, blockNumber, function(err, balance) { + web3.eth.getBalance(account, function(err, balance) { if (err) { console.log("ERROR: fail to getBalance(" + account + ")"); return eachCallback(err); @@ -242,7 +263,7 @@ function makeParityRichList(number, offset, blockNumber, updateCallback) { if (typeof balance === 'object') { ether = parseFloat(balance.div(1e18).toString()); } else { - ether /= 1e18; + ether = balance / 1e18; } data[account].balance = ether; eachCallback(); @@ -301,7 +322,7 @@ var bulkInsert = function(bulk) { Account.collection.insert(localbulk, function(error, data) { if (error) { if (error.code == 11000) { - async.eachSeries(bulk, function(item, eachCallback) { + async.eachSeries(localbulk, function(item, eachCallback) { // upsert accounts delete item._id; // remove _id field Account.collection.update({ "address": item.address }, { $set: item }, { upsert: true }, function(err, updated) { @@ -316,12 +337,20 @@ var bulkInsert = function(bulk) { }); }, function(err) { if (err) { - console.log('ERROR: Aborted due to error: ' + err); - process.exit(9); - return; + if (err.code != 11000) { + console.log('ERROR: Aborted due to error: ' + err); + process.exit(9); + return; + } else { + console.log('WARN: Fail to upsert (ignore) ' + err); + } } console.log('* ' + localbulk.length + ' accounts successfully updated.'); - bulkInsert(bulk); + if (bulk.length > 0) { + setTimeout(function() { + bulkInsert(bulk); + }, 200); + } }); } else { console.log('Error: Aborted due to error on DB: ' + error); @@ -329,7 +358,11 @@ var bulkInsert = function(bulk) { } } else { console.log('* ' + data.insertedCount + ' accounts successfully inserted.'); - bulkInsert(bulk); + if (bulk.length > 0) { + setTimeout(function() { + bulkInsert(bulk); + }, 200); + } } }); } @@ -364,10 +397,10 @@ function prepareJsonAddress(json, type) { function readJsonAccounts(json, blockNumber, callback) { var data = prepareJsonAddress(json); - console.log("* update genesis accounts..."); var accounts = Object.keys(data); + console.log("* update " + accounts.length + " genesis accounts..."); async.eachSeries(accounts, function(account, eachCallback) { - web3.eth.getBalance(account, blockNumber, function(err, balance) { + web3.eth.getBalance(account, function(err, balance) { if (err) { console.log("ERROR: fail to getBalance(" + account + ")"); return eachCallback(err); @@ -378,7 +411,7 @@ function readJsonAccounts(json, blockNumber, callback) { if (typeof balance === 'object') { ether = parseFloat(balance.div(1e18).toString()); } else { - ether /= 1e18; + ether = balance / 1e18; } data[account].balance = ether; eachCallback(); From 94c3ba076552841904623138a2898e8bba4dc2e3 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Fri, 31 Aug 2018 12:53:03 +0900 Subject: [PATCH 09/17] fixed field name bug / missing index added --- db.js | 1 + tools/richlist.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/db.js b/db.js index 1d7a765ed..fe393b56a 100644 --- a/db.js +++ b/db.js @@ -81,6 +81,7 @@ Transaction.index({to:1, blockNumber:-1}); Account.index({balance:-1}); Account.index({balance:-1, blockNumber:-1}); Block.index({miner:1}); +Block.index({miner:1, blockNumber:-1}); mongoose.model('BlockStat', BlockStat); mongoose.model('Block', Block); diff --git a/tools/richlist.js b/tools/richlist.js index fccc58f8a..bd79d16c6 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -92,7 +92,7 @@ function makeRichList(toBlock, blocks, updateCallback) { }, function(callback) { // aggregate miner's addresses Block.aggregate([ - { $match: { blockNumber: { $lte: toBlock, $gt: fromBlock } } }, + { $match: { number: { $lte: toBlock, $gt: fromBlock } } }, { $group: { _id: '$miner' }}, { $project: { "_id": 1 }} ]).exec(function(err, docs) { From ea82240430dd879101a9292f37dacd871f1143d9 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Fri, 31 Aug 2018 13:38:09 +0900 Subject: [PATCH 10/17] fixed per page counter: remove "All" / add 200, 500 --- public/js/controllers/AccountsController.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/public/js/controllers/AccountsController.js b/public/js/controllers/AccountsController.js index 18fdb7a76..639537126 100755 --- a/public/js/controllers/AccountsController.js +++ b/public/js/controllers/AccountsController.js @@ -28,8 +28,8 @@ angular.module('BlocksApp').controller('AccountsController', function($statePara }); }, lengthMenu: [ - [20, 50, 100, 150, -1], - [20, 50, 100, 150, "All"] // change per page values here + [20, 50, 100, 150, 200, 500], + [20, 50, 100, 150, 200, 500] // change per page values here ], pageLength: 20, order: [ From c0392cffad6c506c6cb9c08e412bb7f74fc1f7ab Mon Sep 17 00:00:00 2001 From: hackyminer Date: Mon, 3 Sep 2018 20:31:50 +0900 Subject: [PATCH 11/17] support user defined account type --- db.js | 3 ++- public/js/controllers/AccountsController.js | 20 ++++++++++++++- routes/richlist.js | 12 ++++++--- tools/richlist.js | 28 ++++++++++++++------- 4 files changed, 49 insertions(+), 14 deletions(-) diff --git a/db.js b/db.js index fe393b56a..178379032 100644 --- a/db.js +++ b/db.js @@ -29,7 +29,7 @@ var Account = new Schema( "address": {type: String, index: {unique: true}}, "balance": Number, "blockNumber": Number, - "type": Number // address: 0x0, contract: 0x1 + "type": {type: Number, default: 0} // address: 0x0, contract: 0x1 }); var Contract = new Schema( @@ -80,6 +80,7 @@ Transaction.index({from:1, blockNumber:-1}); Transaction.index({to:1, blockNumber:-1}); Account.index({balance:-1}); Account.index({balance:-1, blockNumber:-1}); +Account.index({type:-1, balance:-1}); Block.index({miner:1}); Block.index({miner:1, blockNumber:-1}); diff --git a/public/js/controllers/AccountsController.js b/public/js/controllers/AccountsController.js index 639537126..cf929183b 100755 --- a/public/js/controllers/AccountsController.js +++ b/public/js/controllers/AccountsController.js @@ -42,7 +42,7 @@ angular.module('BlocksApp').controller('AccountsController', function($statePara infoFiltered: "(filtered from _MAX_ total accounts)" }, columnDefs: [ - { orderable: false, "targets": [0,1,2,4] }, + { orderable: false, "targets": [0,1,4] }, { render: function(data, type, row) { @@ -50,6 +50,24 @@ angular.module('BlocksApp').controller('AccountsController', function($statePara }, targets: [1] }, + { + render: + function(data, type, row) { + if (data & 0x1) { + return "Contract"; + } + if (data & 0x4) { // user defined account type + var accountType = data >> 3; + accountType = accountType.toString(); + if ($scope.settings.accountTypes && $scope.settings.accountTypes[accountType]) { + return $scope.settings.accountTypes[accountType]; + } + return "Genesis Alloc"; + } + return "Account"; + }, + targets: [2] + }, { render: function(data, type, row) { diff --git a/routes/richlist.js b/routes/richlist.js index 3826ab982..acf9c337d 100644 --- a/routes/richlist.js +++ b/routes/richlist.js @@ -62,12 +62,18 @@ var getAccounts = function(req, res) { } // check sort order - var sortOrder = '-balance'; + var sortOrder = { balance: -1 }; if (req.body.order && req.body.order[0] && req.body.order[0].column) { // balance column if (req.body.order[0].column == 3) { if (req.body.order[0].dir == 'asc') { - sortOrder = 'balance'; + sortOrder = { balance: 1 }; + } + } + if (req.body.order[0].column == 2) { + // sort by account type and balance + if (req.body.order[0].dir == 'asc') { + sortOrder = { type: -1, balance: -1 }; } } } @@ -90,7 +96,7 @@ var getAccounts = function(req, res) { } data.data = accounts.map(function(account, i) { - return [i + 1 + start, account.address, account.type == 0 ? "Account" : "Contract", account.balance, account.blockNumber]; + return [i + 1 + start, account.address, account.type, account.balance, account.blockNumber]; }); res.write(JSON.stringify(data)); res.end(); diff --git a/tools/richlist.js b/tools/richlist.js index bd79d16c6..d5ec99657 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -137,7 +137,7 @@ function makeRichList(toBlock, blocks, updateCallback) { data[account] = { address: account }; if (code.length > 2) { data[account].type = 1; // contract type - } else { + } else if (self.accounts[account]) { data[account].type = self.accounts[account].type; } @@ -250,7 +250,10 @@ function makeParityRichList(number, offset, blockNumber, updateCallback) { } data[account] = {}; data[account].address = account; - data[account].type = code.length > 2 ? 1 : 0; // 0: address, 1: contract + if (code.length > 2) { + // 0: normal address, 1: contract + data[account].type = 1; //contract case + } web3.eth.getBalance(account, function(err, balance) { if (err) { @@ -324,7 +327,13 @@ var bulkInsert = function(bulk) { if (error.code == 11000) { async.eachSeries(localbulk, function(item, eachCallback) { // upsert accounts + item._id = undefined; delete item._id; // remove _id field + if (item.type == 0) { + // do not update for normal address cases + item.type = undefined; + delete item.type; + } Account.collection.update({ "address": item.address }, { $set: item }, { upsert: true }, function(err, updated) { if (err) { if (!config.quiet) { @@ -338,7 +347,7 @@ var bulkInsert = function(bulk) { }, function(err) { if (err) { if (err.code != 11000) { - console.log('ERROR: Aborted due to error: ' + err); + console.log('ERROR: Aborted due to error: ' + JSON.stringify(err, null, 2)); process.exit(9); return; } else { @@ -367,10 +376,7 @@ var bulkInsert = function(bulk) { }); } -function prepareJsonAddress(json, type) { - if (!type) { - type = 0; - } +function prepareJsonAddress(json, defaultType = 0) { var accounts = {}; if (json.accounts) { // genesis.json style @@ -383,6 +389,10 @@ function prepareJsonAddress(json, type) { Object.keys(json).forEach(function(account) { var key = account.toLowerCase(); key = '0x' + key.replace(/^0x/, ''); + var type = defaultType; + if (json[account].type) { + type = json[account].type; + } accounts[key] = { address: key, type: type }; }); } else { // normal array @@ -395,8 +405,8 @@ function prepareJsonAddress(json, type) { return accounts; } -function readJsonAccounts(json, blockNumber, callback) { - var data = prepareJsonAddress(json); +function readJsonAccounts(json, blockNumber, callback, defaultType = 0) { + var data = prepareJsonAddress(json, defaultType); var accounts = Object.keys(data); console.log("* update " + accounts.length + " genesis accounts..."); async.eachSeries(accounts, function(account, eachCallback) { From efc8f9cecdc174f990ae8ef1d0e3445df0f49cbe Mon Sep 17 00:00:00 2001 From: hackyminer Date: Wed, 12 Sep 2018 08:54:10 +0900 Subject: [PATCH 12/17] use json-rpc batch job to speed up --- tools/richlist.js | 188 ++++++++++++++++++++++++++++++++-------------- tools/sync.js | 93 +++++++++++++++-------- 2 files changed, 191 insertions(+), 90 deletions(-) diff --git a/tools/richlist.js b/tools/richlist.js index d5ec99657..016304fe0 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -113,14 +113,14 @@ function makeRichList(toBlock, blocks, updateCallback) { }, function(callback) { let len = Object.keys(self.accounts).length; console.info('* ' + len + ' / ' + (self.index + len) + ' total accounts.'); - if (updateCallback && (len >= 100 || ended)) { + if (updateCallback && (len >= 200 || ended)) { self.index += len; console.log("* update " + len + " accounts ..."); var accounts = Object.keys(self.accounts); var chunks = []; - while (accounts.length > 200) { - var chunk = accounts.splice(0, 100); + while (accounts.length > 800) { + var chunk = accounts.splice(0, 500); chunks.push(chunk); } if (accounts.length > 0) { @@ -129,35 +129,70 @@ function makeRichList(toBlock, blocks, updateCallback) { async.eachSeries(chunks, function(chunk, outerCallback) { var data = {}; - async.eachSeries(chunk, function(account, eachCallback) { - web3.eth.getCode(account, function(err, code) { + // get account type + getBalance using json rpc batch job + async.waterfall([ + function(innerCallback) { + var batch = web3.createBatch(); + + for (var i = 0; i < chunk.length; i++) { + var account = chunk[i]; + batch.add(web3.eth.getCode.request(account)); + } + + batch.requestManager.sendBatch(batch.requests, function(err, results) { if (err) { - return eachCallback(err); - } - data[account] = { address: account }; - if (code.length > 2) { - data[account].type = 1; // contract type - } else if (self.accounts[account]) { - data[account].type = self.accounts[account].type; + console.log("ERROR: fail to getCode batch job:", err); + innerCallback(err); + return; } - - web3.eth.getBalance(account, function(err, balance) { - if (err) { - return eachCallback(err); + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var code = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + var account = batch.requests[i].params[0]; + data[account] = { address: account }; + if (code.length > 2) { + // 0: normal address, 1: contract + data[account].type = 1; // contract case + } else if (self.accounts[account]) { + data[account].type = self.accounts[account].type; } - //data[account].balance = web3.fromWei(balance, 'ether'); + }); + innerCallback(null, data); + }); + }, function(data, innerCallback) { + // batch rpc job + var batch = web3.createBatch(); + for (var i = 0; i < chunk.length; i++) { + var account = chunk[i]; + batch.add(web3.eth.getBalance.request(account)); + } + + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getBalance batch job:", err); + innerCallback(err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + let ether; if (typeof balance === 'object') { ether = parseFloat(balance.div(1e18).toString()); } else { ether = balance / 1e18; } - data[account].balance = ether; - eachCallback(); + data[batch.requests[i].params[0]].balance = ether; }); + innerCallback(null, data); }); - }, function(err) { + }], function(err) { if (err) { return outerCallback(err); } @@ -241,38 +276,68 @@ function makeParityRichList(number, offset, blockNumber, updateCallback) { var lastAccount = accounts[accounts.length - 1]; var data = {}; - // Please see https://github.com/gobitfly/etherchain-light by gobitfly - async.eachSeries(accounts, function(account, eachCallback) { - web3.eth.getCode(account, function(err, code) { + // get account type + getBalance using json rpc batch job + async.waterfall([ + function(innerCallback) { + var batch = web3.createBatch(); + + for (var i = 0; i < accounts.length; i++) { + var account = accounts[i]; + batch.add(web3.eth.getCode.request(account)); + } + + batch.requestManager.sendBatch(batch.requests, function(err, results) { if (err) { - console.log("ERROR: fail to getCode(" + account + ")"); - return eachCallback(err); - } - data[account] = {}; - data[account].address = account; - if (code.length > 2) { - // 0: normal address, 1: contract - data[account].type = 1; //contract case + console.log("ERROR: fail to getCode batch job:", err); + innerCallback(err); + return; } - - web3.eth.getBalance(account, function(err, balance) { - if (err) { - console.log("ERROR: fail to getBalance(" + account + ")"); - return eachCallback(err); + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var code = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + var account = batch.requests[i].params[0]; + data[account] = { address: account }; + if (code.length > 2) { + // 0: normal address, 1: contract + data[account].type = 1; // contract case } - //data[account].balance = web3.fromWei(balance, 'ether'); + }); + innerCallback(null, data); + }); + }, function(data, innerCallback) { + // batch rpc job + var batch = web3.createBatch(); + for (var i = 0; i < accounts.length; i++) { + var account = accounts[i]; + batch.add(web3.eth.getBalance.request(account)); + } + + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getBalance batch job:", err); + innerCallback(err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + let ether; if (typeof balance === 'object') { ether = parseFloat(balance.div(1e18).toString()); } else { ether = balance / 1e18; } - data[account].balance = ether; - eachCallback(); + data[batch.requests[i].params[0]].balance = ether; }); + innerCallback(null, data); }); - }, function(err) { + }], function(err) { callback(err, data, lastAccount); }); } @@ -317,10 +382,10 @@ var bulkInsert = function(bulk) { } var localbulk; - if (bulk.length > 200) { - localbulk = bulk.splice(0, 100); - } else { + if (bulk.length > 300) { localbulk = bulk.splice(0, 200); + } else { + localbulk = bulk.splice(0, 300); } Account.collection.insert(localbulk, function(error, data) { if (error) { @@ -409,28 +474,35 @@ function readJsonAccounts(json, blockNumber, callback, defaultType = 0) { var data = prepareJsonAddress(json, defaultType); var accounts = Object.keys(data); console.log("* update " + accounts.length + " genesis accounts..."); - async.eachSeries(accounts, function(account, eachCallback) { - web3.eth.getBalance(account, function(err, balance) { - if (err) { - console.log("ERROR: fail to getBalance(" + account + ")"); - return eachCallback(err); - } - //data[account].balance = web3.fromWei(balance, 'ether'); + // batch rpc job + var batch = web3.createBatch(); + // normally, the request size of batch getBalance() of all accounts is not bigger than 128kB. + // simply getBalance at once using rpc batch job. + for (var i = 0; i < accounts.length; i++) { + var account = accounts[i]; + batch.add(web3.eth.getBalance.request(account)); + } + + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getBalance batch job:", err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + let ether; if (typeof balance === 'object') { ether = parseFloat(balance.div(1e18).toString()); } else { ether = balance / 1e18; } - data[account].balance = ether; - eachCallback(); + data[batch.requests[i].params[0]].balance = ether; }); - }, function(err) { - if (err) { - console.log("ERROR: fail to getBalance()" + err); - return; - } callback(data, blockNumber); }); } @@ -471,7 +543,7 @@ var latestBlock = web3.eth.blockNumber; console.log("* latestBlock = " + latestBlock); if (useParity) { - makeParityRichList(100, null, latestBlock, updateAccounts); + makeParityRichList(500, null, latestBlock, updateAccounts); } else { // load genesis account if (config.settings && config.settings.genesisAddress) { diff --git a/tools/sync.js b/tools/sync.js index 6a19a496d..1125bc96d 100644 --- a/tools/sync.js +++ b/tools/sync.js @@ -184,48 +184,77 @@ var writeTransactionsToDB = function(config, blockData, flush) { // update balances if (accounts.length > 0) - async.eachSeries(accounts, function(account, eachCallback) { - var blockNumber = data[account].blockNumber; + async.waterfall([ // get contract account type - web3.eth.getCode(account, function(err, code) { - if (err) { - console.log("ERROR: fail to getCode(" + account + ")"); - return eachCallback(err); + function(callback) { + var batch = web3.createBatch(); + + for (var i = 0; i < accounts.length; i++) { + var account = accounts[i]; + batch.add(web3.eth.getCode.request(account)); } - if (code.length > 2) { - data[account].type = 1; // contract type + + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getCode batch job:", err); + callback(err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var code = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + if (code.length > 2) { + data[batch.requests[i].params[0]].type = 1; // contract type + } + + }); + callback(null, data); + }); + }, function(data, callback) { + // batch rpc job + var batch = web3.createBatch(); + for (var i = 0; i < accounts.length; i++) { + var account = accounts[i]; + batch.add(web3.eth.getBalance.request(account)); } - web3.eth.getBalance(account, blockNumber, function(err, balance) { + batch.requestManager.sendBatch(batch.requests, function(err, results) { if (err) { - console.log("ERROR: fail to getBalance(" + account + ")"); - return eachCallback(err); + console.log("ERROR: fail to getBalance batch job:", err); + callback(err); + return; } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; - //data[account].balance = web3.fromWei(balance, 'ether'); - let ether; - if (typeof balance === 'object') { - ether = parseFloat(balance.div(1e18).toString()); - } else { - ether /= 1e18; + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether = balance / 1e18; + } + data[batch.requests[i].params[0]].balance = ether; + }); + callback(null, data); + }); + }], function(error, data) { + var n = 0; + accounts.forEach(function(account) { + n++; + if (n <= 5) { + console.log(' - upsert ' + account + ' / balance = ' + data[account].balance); + } else if (n == 6) { + console.log(' (...) total ' + accounts.length + ' accounts updated.'); } - data[account].balance = ether; - eachCallback(); + // upsert account + Account.collection.update({ address: account }, { $set: data[account] }, { upsert: true }); }); }); - }, function(err) { - var n = 0; - accounts.forEach(function(account) { - n++; - if (n <= 5) { - console.log(' - upsert ' + account + ' / balance = ' + data[account].balance); - } else if (n == 6) { - console.log(' (...) total ' + accounts.length + ' accounts updated.'); - } - // upsert account - Account.collection.update({ address: account }, { $set: data[account] }, { upsert: true }); - }); - }); if (bulk.length > 0) Transaction.collection.insert(bulk, function( err, tx ){ From 9bcd380519bc724e2bc1a6f6b2b58f3e606904e6 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Tue, 16 Oct 2018 20:22:41 +0900 Subject: [PATCH 13/17] reduce json-rpc batch size --- tools/sync.js | 157 +++++++++++++++++++++++++++++--------------------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/tools/sync.js b/tools/sync.js index 1125bc96d..cc50ed47c 100644 --- a/tools/sync.js +++ b/tools/sync.js @@ -21,6 +21,11 @@ var Account = mongoose.model( 'Account' ); //Just listen for latest blocks and sync from the start of the app. **/ var listenBlocks = function(config) { + if (web3.eth.syncing) { + console.log('Info: waiting until syncing finished... (currentBlock is #' + web3.eth.syncing.currentBlock + ')'); + setTimeout(function() { listenBlocks(config); }, 10000); + return; + } var newBlocks = web3.eth.filter("latest"); newBlocks.watch(function (error,latestBlock) { if(error) { @@ -52,6 +57,12 @@ var listenBlocks = function(config) { **/ var syncChain = function(config, nextBlock){ if(web3.isConnected()) { + if (web3.eth.syncing) { + console.log('Info: waiting until syncing finished... (currentBlock is #' + web3.eth.syncing.currentBlock + ')'); + setTimeout(function() { syncChain(config, nextBlock); }, 10000); + return; + } + if (typeof nextBlock === 'undefined') { prepareSync(config, function(error, startBlock) { if(error) { @@ -144,7 +155,7 @@ var writeTransactionsToDB = function(config, blockData, flush) { self.miners = []; } if (blockData) { - self.miners.push({ address: blockData.miner, blockNumber: blockData.blockNumber, type: 0 }); + self.miners.push({ address: blockData.miner, blockNumber: blockData.number, type: 0 }); } if (blockData && blockData.transactions.length > 0) { for (d in blockData.transactions) { @@ -183,78 +194,92 @@ var writeTransactionsToDB = function(config, blockData, flush) { if (bulk.length == 0 && accounts.length == 0) return; // update balances - if (accounts.length > 0) - async.waterfall([ - // get contract account type - function(callback) { - var batch = web3.createBatch(); - - for (var i = 0; i < accounts.length; i++) { - var account = accounts[i]; - batch.add(web3.eth.getCode.request(account)); - } + if (accounts.length > 0) { + var n = 0; + var chunks = []; + while (accounts.length > 800) { + var chunk = accounts.splice(0, 500); + chunks.push(chunk); + } + if (accounts.length > 0) { + chunks.push(accounts); + } + async.eachSeries(chunks, function(chunk, outerCallback) { + async.waterfall([ + // get contract account type + function(callback) { + var batch = web3.createBatch(); - batch.requestManager.sendBatch(batch.requests, function(err, results) { - if (err) { - console.log("ERROR: fail to getCode batch job:", err); - callback(err); - return; - } - results = results || []; - batch.requests.map(function (request, index) { - return results[index] || {}; - }).forEach(function (result, i) { - var code = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; - if (code.length > 2) { - data[batch.requests[i].params[0]].type = 1; // contract type + for (var i = 0; i < chunk.length; i++) { + var account = chunk[i]; + batch.add(web3.eth.getCode.request(account)); } - }); - callback(null, data); - }); - }, function(data, callback) { - // batch rpc job - var batch = web3.createBatch(); - for (var i = 0; i < accounts.length; i++) { - var account = accounts[i]; - batch.add(web3.eth.getBalance.request(account)); - } + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getCode batch job:", err); + callback(err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var code = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + if (code.length > 2) { + data[batch.requests[i].params[0]].type = 1; // contract type + } - batch.requestManager.sendBatch(batch.requests, function(err, results) { - if (err) { - console.log("ERROR: fail to getBalance batch job:", err); - callback(err); - return; - } - results = results || []; - batch.requests.map(function (request, index) { - return results[index] || {}; - }).forEach(function (result, i) { - var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; - - let ether; - if (typeof balance === 'object') { - ether = parseFloat(balance.div(1e18).toString()); - } else { - ether = balance / 1e18; + }); + callback(null); + }); + }, function(callback) { + // batch rpc job + var batch = web3.createBatch(); + for (var i = 0; i < chunk.length; i++) { + var account = chunk[i]; + if (account) { + batch.add(web3.eth.getBalance.request(account)); + } } - data[batch.requests[i].params[0]].balance = ether; - }); - callback(null, data); - }); - }], function(error, data) { - var n = 0; - accounts.forEach(function(account) { - n++; - if (n <= 5) { - console.log(' - upsert ' + account + ' / balance = ' + data[account].balance); - } else if (n == 6) { - console.log(' (...) total ' + accounts.length + ' accounts updated.'); - } - // upsert account - Account.collection.update({ address: account }, { $set: data[account] }, { upsert: true }); + + batch.requestManager.sendBatch(batch.requests, function(err, results) { + if (err) { + console.log("ERROR: fail to getBalance batch job:", err); + callback(err); + return; + } + results = results || []; + batch.requests.map(function (request, index) { + return results[index] || {}; + }).forEach(function (result, i) { + var balance = batch.requests[i].format ? batch.requests[i].format(result.result) : result.result; + + let ether; + if (typeof balance === 'object') { + ether = parseFloat(balance.div(1e18).toString()); + } else { + ether = balance / 1e18; + } + var account = batch.requests[i].params[0]; + data[account].balance = ether; + + if (n <= 5) { + console.log(' - upsert ' + account + ' / balance = ' + data[account].balance); + } else if (n == 6) { + console.log(' (...) total ' + accounts.length + ' accounts updated.'); + } + n++; + // upsert account + Account.collection.update({ address: account }, { $set: data[account] }, { upsert: true }); + }); + }); + callback(null); + }], function(error) { }); + }, function(error) { }); + } if (bulk.length > 0) Transaction.collection.insert(bulk, function( err, tx ){ From 6ff8f6b2e32b286ad21ff5089df3b90903ffd016 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 27 Oct 2018 16:51:45 +0900 Subject: [PATCH 14/17] support some configurable env variables - tools/richlist.js - check NOPARITY env to support parity without fat-db option - tools/sync.js - check NORICHLIST env to disable balance updater and check config.useRichList config variable --- tools/richlist.js | 3 ++- tools/sync.js | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/richlist.js b/tools/richlist.js index 016304fe0..e28822621 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -531,7 +531,8 @@ console.log('Connecting ' + config.nodeAddr + ':' + config.gethPort + '...'); var web3 = new Web3(new Web3.providers.HttpProvider('http://' + config.nodeAddr + ':' + config.gethPort.toString())); var useParity = false; -if (web3.version.node.split('/')[0].toLowerCase().includes('parity')) { +if (!process.env.NOPARITY + && web3.version.node.split('/')[0].toLowerCase().includes('parity')) { // load parity extension web3 = require("../lib/trace.js")(web3); useParity = true; diff --git a/tools/sync.js b/tools/sync.js index cc50ed47c..713018498 100644 --- a/tools/sync.js +++ b/tools/sync.js @@ -194,7 +194,7 @@ var writeTransactionsToDB = function(config, blockData, flush) { if (bulk.length == 0 && accounts.length == 0) return; // update balances - if (accounts.length > 0) { + if (config.useRichList && accounts.length > 0) { var n = 0; var chunks = []; while (accounts.length > 800) { @@ -448,6 +448,11 @@ if (config.patch === true){ runPatcher(config); } +// check NORICHLIST env +if (process.env.NORICHLIST) { + config.useRichList = false; +} + // Start listening for latest blocks listenBlocks(config); From a0d45d0d437b6a33af5246c6c4bb651b1fdecdb5 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Mon, 29 Oct 2018 18:26:39 +0900 Subject: [PATCH 15/17] richlist: cache totalSupply --- routes/richlist.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/routes/richlist.js b/routes/richlist.js index acf9c337d..8516645c2 100644 --- a/routes/richlist.js +++ b/routes/richlist.js @@ -10,6 +10,18 @@ require( '../db.js' ); var Account = mongoose.model('Account'); var getAccounts = function(req, res) { + var self = getAccounts; + if (!self.totalSupply) { + self.totalSupply = -1; + self.timestamp = 0; + } + + // check cached totalSupply + if (new Date() - self.timestamp > 30*60*1000) { + self.totalSupply = -1; + self.timestamp = 0; + } + // count accounts only once var count = req.body.recordsTotal || 0; count = parseInt(count); @@ -18,7 +30,7 @@ var getAccounts = function(req, res) { } // get totalSupply only once - var queryTotalSupply = req.body.totalSupply || null; + var queryTotalSupply = self.totalSupply || req.body.totalSupply || null; async.waterfall([ function(callback) { @@ -32,10 +44,13 @@ var getAccounts = function(req, res) { } var totalSupply = docs[0].totalSupply; + // update cache + self.timestamp = new Date(); + self.totalSupply = totalSupply; callback(null, totalSupply); }); } else { - callback(null, null); + callback(null, queryTotalSupply > 0 ? queryTotalSupply : null); } }, function(totalSupply, callback) { From a82c74ea1845e4a2598575777505f503c4609259 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Mon, 29 Oct 2018 18:48:52 +0900 Subject: [PATCH 16/17] check settings.showRichList config variable to show Accounts menu --- public/tpl/header.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/tpl/header.html b/public/tpl/header.html index 3d7bf5390..a4e300ffa 100755 --- a/public/tpl/header.html +++ b/public/tpl/header.html @@ -44,7 +44,7 @@
  • Home
    • -
  • +
  • Accounts
  • From de3ee7b3b8ea4aacf1149540db9afeeab5b8b2b5 Mon Sep 17 00:00:00 2001 From: hackyminer Date: Sat, 17 Nov 2018 06:56:32 +0900 Subject: [PATCH 17/17] added comments for clarity, etc * AccountsController: fixed for clarity --- public/js/controllers/AccountsController.js | 7 ++++++- tools/richlist.js | 7 +++++-- tools/sync.js | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/public/js/controllers/AccountsController.js b/public/js/controllers/AccountsController.js index cf929183b..38363b0e1 100755 --- a/public/js/controllers/AccountsController.js +++ b/public/js/controllers/AccountsController.js @@ -21,7 +21,12 @@ angular.module('BlocksApp').controller('AccountsController', function($statePara // fixup data to show percentages var newdata = resp.data.data.map(function(item) { - return [item[0], item[1], item[2], item[3], (item[3] / $scope.totalSupply) * 100, item[4]]; + var num = item[0]; + var addr = item[1]; + var type = item[2]; + var balance = item[3]; + var lastmod = item[4]; + return [num, addr, type, balance, (balance / $scope.totalSupply) * 100, lastmod]; }); resp.data.data = newdata; callback(resp.data); diff --git a/tools/richlist.js b/tools/richlist.js index e28822621..7e3c34258 100644 --- a/tools/richlist.js +++ b/tools/richlist.js @@ -13,10 +13,9 @@ var Account = require('../db.js').Account; var Transaction = require('../db.js').Transaction; var Block = require('../db.js').Block; -mongoose.connect(process.env.MONGO_URI || 'mongodb://localhost/blockDB'); - const ADDRESS_CACHE_MAX = 10000; // address cache threshold +// RichList for Geth Classic, Geth function makeRichList(toBlock, blocks, updateCallback) { var self = makeRichList; if (!self.cached) { @@ -117,8 +116,11 @@ function makeRichList(toBlock, blocks, updateCallback) { self.index += len; console.log("* update " + len + " accounts ..."); + // split accounts into chunks to make proper sized json-rpc batch job. var accounts = Object.keys(self.accounts); var chunks = []; + + // about ~1000 `eth_getBalance` json rpc calls are possible in one json-rpc batchjob. while (accounts.length > 800) { var chunk = accounts.splice(0, 500); chunks.push(chunk); @@ -390,6 +392,7 @@ var bulkInsert = function(bulk) { Account.collection.insert(localbulk, function(error, data) { if (error) { if (error.code == 11000) { + // For already exists case, try upsert method. async.eachSeries(localbulk, function(item, eachCallback) { // upsert accounts item._id = undefined; diff --git a/tools/sync.js b/tools/sync.js index 713018498..38ad3a4b6 100644 --- a/tools/sync.js +++ b/tools/sync.js @@ -449,6 +449,7 @@ if (config.patch === true){ } // check NORICHLIST env +// you can use it like as 'NORICHLIST=1 node tools/sync.js' to disable balance updater temporary. if (process.env.NORICHLIST) { config.useRichList = false; }