diff --git a/lib/bcoin-browser.js b/lib/bcoin-browser.js index 3cc9a1325..17e3277ed 100644 --- a/lib/bcoin-browser.js +++ b/lib/bcoin-browser.js @@ -59,8 +59,8 @@ bcoin.Mnemonic = require('./hd/mnemonic'); bcoin.indexer = require('./indexer'); bcoin.Indexer = require('./indexer/indexer'); bcoin.ChainClient = require('./indexer/chainclient'); -bcoin.TXIndexer = require('./indexer/txindexer/txindexer'); -bcoin.AddrIndexer = require('./indexer/addrindexer/addrindexer'); +bcoin.TXIndexer = require('./indexer/txindexer'); +bcoin.AddrIndexer = require('./indexer/addrindexer'); // Mempool bcoin.mempool = require('./mempool'); diff --git a/lib/bcoin.js b/lib/bcoin.js index f110fd2c3..0757ad94c 100644 --- a/lib/bcoin.js +++ b/lib/bcoin.js @@ -78,8 +78,8 @@ bcoin.define('Mnemonic', './hd/mnemonic'); bcoin.define('indexer', './indexer'); bcoin.define('Indexer', './indexer/indexer'); bcoin.define('ChainClient', './indexer/chainclient'); -bcoin.define('TXIndexer', './indexer/txindexer/txindexer'); -bcoin.define('AddrIndexer', './indexer/addrindexer/addrindexer'); +bcoin.define('TXIndexer', './indexer/txindexer'); +bcoin.define('AddrIndexer', './indexer/addrindexer'); // Mempool bcoin.define('mempool', './mempool'); diff --git a/lib/indexer/addrindexer/addrindexer.js b/lib/indexer/addrindexer.js similarity index 86% rename from lib/indexer/addrindexer/addrindexer.js rename to lib/indexer/addrindexer.js index 2c1f0b338..e27844b64 100644 --- a/lib/indexer/addrindexer/addrindexer.js +++ b/lib/indexer/addrindexer.js @@ -9,8 +9,25 @@ const assert = require('assert'); const bdb = require('bdb'); const layout = require('./layout'); -const Address = require('../../primitives/address'); -const Indexer = require('../indexer'); +const Address = require('../primitives/address'); +const Indexer = require('./indexer'); + +/* + * AddrIndexer Database Layout: + * T[addr-hash][hash] -> dummy (tx by address) + * C[addr-hash][hash][index] -> dummy (coin by address) +*/ + +Object.assign(layout, { + T: bdb.key('T', ['hash', 'hash256']), + C: bdb.key('C', ['hash', 'hash256', 'uint32']) +}); + +/** + * AddrIndexer + * @alias module:indexer.AddrIndexer + * @extends Indexer + */ class AddrIndexer extends Indexer { /** @@ -28,9 +45,9 @@ class AddrIndexer extends Indexer { /** * Index transactions by address. * @private - * @param (ChainEntry) entry - * @param (Block) block - * @param (CoinView) view + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view */ async indexBlock(entry, block, view) { @@ -74,9 +91,9 @@ class AddrIndexer extends Indexer { /** * Remove addresses from index. * @private - * @param (ChainEntry) entry - * @param (Block) block - * @param (CoinView) view + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view */ async unindexBlock(entry, block, view) { diff --git a/lib/indexer/addrindexer/layout.js b/lib/indexer/addrindexer/layout.js deleted file mode 100644 index a973cd8ea..000000000 --- a/lib/indexer/addrindexer/layout.js +++ /dev/null @@ -1,26 +0,0 @@ -/*! - * layout.js - addrindexer layout for bcoin - * Copyright (c) 2018, the bcoin developers (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const bdb = require('bdb'); - -/* - * AddrIndexer Database Layout: - * T[addr-hash][hash] -> dummy (tx by address) - * C[addr-hash][hash][index] -> dummy (coin by address) -*/ - -const layout = Object.assign({ - T: bdb.key('T', ['hash', 'hash256']), - C: bdb.key('C', ['hash', 'hash256', 'uint32']) -}, require('../layout')); - -/* - * Expose - */ - -module.exports = layout; diff --git a/lib/indexer/chainclient.js b/lib/indexer/chainclient.js index c5dae0d77..38f45f555 100644 --- a/lib/indexer/chainclient.js +++ b/lib/indexer/chainclient.js @@ -8,9 +8,11 @@ const assert = require('assert'); const AsyncEmitter = require('bevent'); +const Chain = require('../blockchain/chain'); /** * Chain Client + * @extends AsyncEmitter * @alias module:indexer.ChainClient */ @@ -18,14 +20,16 @@ class ChainClient extends AsyncEmitter { /** * Create a chain client. * @constructor + * @param {Chain} chain */ constructor(chain) { super(); + assert(chain instanceof Chain); + this.chain = chain; this.network = chain.network; - this.filter = null; this.opened = false; this.init(); @@ -92,7 +96,7 @@ class ChainClient extends AsyncEmitter { /** * Get chain entry. * @param {Hash} hash - * @returns {Promise} + * @returns {Promise} - Returns {@link ChainEntry}. */ async getEntry(hash) { @@ -132,7 +136,7 @@ class ChainClient extends AsyncEmitter { /** * Get block * @param {Hash} hash - * @returns {Promise} + * @returns {Promise} - Returns {@link Block} */ async getBlock(hash) { @@ -164,8 +168,10 @@ class ChainClient extends AsyncEmitter { async rescan(start) { for (let i = start; ; i++) { const entry = await this.getEntry(i); - if (!entry) + if (!entry) { + await this.emitAsync('chain tip'); break; + }; const block = await this.getBlock(entry.hash); assert(block); diff --git a/lib/indexer/index.js b/lib/indexer/index.js index d520a7a69..96ad09a3f 100644 --- a/lib/indexer/index.js +++ b/lib/indexer/index.js @@ -6,7 +6,11 @@ 'use strict'; +/** + * @module indexer + */ + exports.Indexer = require('./indexer'); -exports.TXIndexer = require('./txindexer/txindexer'); -exports.AddrIndexer = require('./addrindexer/addrindexer'); +exports.TXIndexer = require('./txindexer'); +exports.AddrIndexer = require('./addrindexer'); exports.ChainClient = require('./chainclient'); diff --git a/lib/indexer/indexer.js b/lib/indexer/indexer.js index c5144e014..fc02c003f 100644 --- a/lib/indexer/indexer.js +++ b/lib/indexer/indexer.js @@ -27,6 +27,10 @@ const { * Indexer * @alias module:indexer.Indexer * @extends EventEmitter + * @property {IndexerDB} db + * @property {Number} height + * @property {ChainState} state + * @emits Indexer#chain tip */ class Indexer extends EventEmitter { @@ -47,7 +51,6 @@ class Indexer extends EventEmitter { this.network = this.options.network; this.logger = this.options.logger.context(`${module}indexer`); - this.workers = this.options.workers; this.client = this.options.client || new NullClient(this); this.db = null; this.rescanning = false; @@ -122,6 +125,12 @@ class Indexer extends EventEmitter { this.emit('error', e); } }); + + this.client.on('chain tip', async () => { + this.logger.debug('Indexer: finished rescan'); + const tip = await this.getTip(); + this.emit('chain tip', tip); + }); } /** @@ -133,7 +142,7 @@ class Indexer extends EventEmitter { if (fs.unsupported) return undefined; - if (this.memory) + if (this.options.memory) return undefined; return fs.mkdirp(this.options.prefix); @@ -147,7 +156,7 @@ class Indexer extends EventEmitter { async open() { await this.ensure(); await this.db.open(); - await this.db.verify(layout.V.build(), 'index', 7); + await this.db.verify(layout.V.build(), 'index', 0); await this.verifyNetwork(); @@ -302,6 +311,7 @@ class Indexer extends EventEmitter { if (this.state.startHeight < height) height = this.state.startHeight; + this.logger.spam('Starting block rescan from: %d.', height); return this.scan(height); } @@ -314,6 +324,8 @@ class Indexer extends EventEmitter { */ async rescanBlock(entry, block, view) { + this.logger.spam('Rescanning block: %d.', entry.height); + if (!this.rescanning) { this.logger.warning('Unsolicited rescan block: %d.', entry.height); return; @@ -343,7 +355,7 @@ class Indexer extends EventEmitter { */ async scan(height) { - assert((height >>> 0) === height, 'WDB: Must pass in a height.'); + assert((height >>> 0) === height, 'Indexer: Must pass in a height.'); await this.rollback(height); @@ -465,7 +477,7 @@ class Indexer extends EventEmitter { const tip = await this.getBlock(this.state.height); if (!tip) - throw new Error('WDB: Tip not found!'); + throw new Error('Indexer: Tip not found!'); return tip; } @@ -478,7 +490,7 @@ class Indexer extends EventEmitter { async rollback(height) { if (height > this.state.height) - throw new Error('WDB: Cannot rollback to the future.'); + throw new Error('Indexer: Cannot rollback to the future.'); if (height === this.state.height) { this.logger.info('Rolled back to same height (%d).', height); @@ -526,6 +538,8 @@ class Indexer extends EventEmitter { if (tip.height >= this.network.block.slowHeight && !this.rescanning) this.logger.debug('Adding block: %d.', tip.height); + this.logger.spam('Adding block: %d.', entry.height); + if (tip.height === this.state.height) { // We let blocks of the same height // through specifically for rescans: @@ -539,6 +553,8 @@ class Indexer extends EventEmitter { return; } + this.logger.spam('Indexing block: %d.', entry.height); + await this.indexBlock(entry, block, view); // Sync the state to the new tip. @@ -549,6 +565,7 @@ class Indexer extends EventEmitter { /** * Process block indexing + * Indexers will implement this method to process the block for indexing * @param {ChainEntry} entry * @param {Block} block * @returns {Promise} @@ -560,6 +577,7 @@ class Indexer extends EventEmitter { /** * Undo block indexing + * Indexers will implement this method to undo indexing for the block * @param {ChainEntry} entry * @param {Block} block * @returns {Promise} @@ -605,8 +623,10 @@ class Indexer extends EventEmitter { async _removeBlock(entry, block, view) { const tip = BlockMeta.fromEntry(entry); + this.logger.spam('Removing block: %d.', entry.height); + if (tip.height === 0) - throw new Error('WDB: Bad disconnection (genesis block).'); + throw new Error('Indexer: Bad disconnection (genesis block).'); if (tip.height > this.state.height) { this.logger.warning( @@ -616,7 +636,9 @@ class Indexer extends EventEmitter { } if (tip.height !== this.state.height) - throw new Error('WDB: Bad disconnection (height mismatch).'); + throw new Error('Indexer: Bad disconnection (height mismatch).'); + + this.logger.spam('Unindexing block: %d.', entry.height); await this.unindexBlock(entry, block, view); @@ -653,7 +675,7 @@ class Indexer extends EventEmitter { async _resetChain(entry) { if (entry.height > this.state.height) - throw new Error('WDB: Bad reset height.'); + throw new Error('Indexer: Bad reset height.'); return this.rollback(entry.height); } @@ -676,7 +698,6 @@ class IndexOptions { this.module = module; this.network = Network.primary; this.logger = Logger.global; - this.workers = null; this.client = null; this.chain = null; this.indexers = null; @@ -708,11 +729,6 @@ class IndexOptions { this.logger = options.logger; } - if (options.workers != null) { - assert(typeof options.workers === 'object'); - this.workers = options.workers; - } - if (options.client != null) { assert(typeof options.client === 'object'); this.client = options.client; @@ -777,6 +793,12 @@ class IndexOptions { * Helpers */ +/** + * fromU32 + * read a 4 byte Uint32LE + * @param {Number} num number + * @returns {Buffer} buffer + */ function fromU32(num) { const data = Buffer.allocUnsafe(4); data.writeUInt32LE(num, 0, true); diff --git a/lib/indexer/layout.js b/lib/indexer/layout.js index 25a068dd5..e2bc243c3 100644 --- a/lib/indexer/layout.js +++ b/lib/indexer/layout.js @@ -10,6 +10,7 @@ const bdb = require('bdb'); /* * Index Database Layout: + * To be extended by indexer implementations * V -> db version * O -> flags * h[height] -> recent block hash diff --git a/lib/indexer/nullclient.js b/lib/indexer/nullclient.js index 3eb6876bf..f7c88ee76 100644 --- a/lib/indexer/nullclient.js +++ b/lib/indexer/nullclient.js @@ -19,13 +19,14 @@ class NullClient extends EventEmitter { /** * Create a client. * @constructor + * @param {Chain} chain */ - constructor(wdb) { + constructor(chain) { super(); - this.wdb = wdb; - this.network = wdb.network; + this.chain = chain; + this.network = chain.network; this.opened = false; } @@ -51,26 +52,6 @@ class NullClient extends EventEmitter { setImmediate(() => this.emit('disconnect')); } - /** - * Add a listener. - * @param {String} type - * @param {Function} handler - */ - - bind(type, handler) { - return this.on(type, handler); - } - - /** - * Add a listener. - * @param {String} type - * @param {Function} handler - */ - - hook(type, handler) { - return this.on(type, handler); - } - /** * Get chain tip. * @returns {Promise} @@ -84,7 +65,7 @@ class NullClient extends EventEmitter { /** * Get chain entry. * @param {Hash} hash - * @returns {Promise} + * @returns {Promise} - Returns {@link ChainEntry}. */ async getEntry(hash) { @@ -134,10 +115,8 @@ class NullClient extends EventEmitter { } /** - * Rescan for any missed transactions. - * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. + * Rescan for any missed blocks. + * @param {Number} start - Start block. * @returns {Promise} */ diff --git a/lib/indexer/records.js b/lib/indexer/records.js index c6778f363..3e67e8436 100644 --- a/lib/indexer/records.js +++ b/lib/indexer/records.js @@ -16,6 +16,7 @@ const consensus = require('../protocol/consensus'); /** * Chain State + * @alias module:indexer.ChainState */ class ChainState { @@ -83,6 +84,7 @@ class ChainState { /** * Block Meta + * @alias module:indexer.BlockMeta */ class BlockMeta { diff --git a/lib/indexer/txindexer/txindexer.js b/lib/indexer/txindexer.js similarity index 79% rename from lib/indexer/txindexer/txindexer.js rename to lib/indexer/txindexer.js index 538b60c5b..5050932b7 100644 --- a/lib/indexer/txindexer/txindexer.js +++ b/lib/indexer/txindexer.js @@ -8,8 +8,23 @@ const bdb = require('bdb'); const layout = require('./layout'); -const TXMeta = require('../../primitives/txmeta'); -const Indexer = require('../indexer'); +const TXMeta = require('../primitives/txmeta'); +const Indexer = require('./indexer'); + +/* + * TXIndexer Database Layout: + * t[hash] -> extended tx +*/ + +Object.assign(layout, { + t: bdb.key('t', ['hash256']) +}); + +/** + * TXIndexer + * @alias module:indexer.TXIndexer + * @extends Indexer + */ class TXIndexer extends Indexer { /** @@ -27,9 +42,9 @@ class TXIndexer extends Indexer { /** * Index transactions by txid. * @private - * @param (ChainEntry) entry - * @param (Block) block - * @param (CoinView) view + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view */ async indexBlock(entry, block, view) { @@ -48,9 +63,9 @@ class TXIndexer extends Indexer { /** * Remove transactions from index. * @private - * @param (ChainEntry) entry - * @param (Block) block - * @param (CoinView) view + * @param {ChainEntry} entry + * @param {Block} block + * @param {CoinView} view */ async unindexBlock(entry, block, view) { diff --git a/lib/indexer/txindexer/layout.js b/lib/indexer/txindexer/layout.js deleted file mode 100644 index defb43f14..000000000 --- a/lib/indexer/txindexer/layout.js +++ /dev/null @@ -1,24 +0,0 @@ -/*! - * layout.js - txindexer layout for bcoin - * Copyright (c) 2018, the bcoin developers (MIT License). - * https://github.com/bcoin-org/bcoin - */ - -'use strict'; - -const bdb = require('bdb'); - -/* - * TXIndexer Database Layout: - * t[hash] -> extended tx -*/ - -const layout = Object.assign({ - t: bdb.key('t', ['hash256']) -}, require('../layout')); - -/* - * Expose - */ - -module.exports = layout; diff --git a/lib/node/fullnode.js b/lib/node/fullnode.js index b2f9a74d3..ac1be1711 100644 --- a/lib/node/fullnode.js +++ b/lib/node/fullnode.js @@ -16,8 +16,8 @@ const Miner = require('../mining/miner'); const Node = require('./node'); const HTTP = require('./http'); const RPC = require('./rpc'); -const TXIndexer = require('../indexer/txindexer/txindexer'); -const AddrIndexer = require('../indexer/addrindexer/addrindexer'); +const TXIndexer = require('../indexer/txindexer'); +const AddrIndexer = require('../indexer/addrindexer'); /** * Full Node