Skip to content

Commit

Permalink
indexer: refactor layout, flatten
Browse files Browse the repository at this point in the history
tuxcanfly committed Jul 24, 2018
1 parent 4d706e8 commit 89af2bb
Showing 13 changed files with 117 additions and 121 deletions.
4 changes: 2 additions & 2 deletions lib/bcoin-browser.js
Original file line number Diff line number Diff line change
@@ -59,8 +59,8 @@ bcoin.Mnemonic = require('./hd/mnemonic');
bcoin.indexer = require('./indexer');
bcoin.Indexer = require('./indexer/indexer');
bcoin.ChainClient = require('./indexer/chainclient');
bcoin.TXIndexer = require('./indexer/txindexer/txindexer');
bcoin.AddrIndexer = require('./indexer/addrindexer/addrindexer');
bcoin.TXIndexer = require('./indexer/txindexer');
bcoin.AddrIndexer = require('./indexer/addrindexer');

// Mempool
bcoin.mempool = require('./mempool');
4 changes: 2 additions & 2 deletions lib/bcoin.js
Original file line number Diff line number Diff line change
@@ -78,8 +78,8 @@ bcoin.define('Mnemonic', './hd/mnemonic');
bcoin.define('indexer', './indexer');
bcoin.define('Indexer', './indexer/indexer');
bcoin.define('ChainClient', './indexer/chainclient');
bcoin.define('TXIndexer', './indexer/txindexer/txindexer');
bcoin.define('AddrIndexer', './indexer/addrindexer/addrindexer');
bcoin.define('TXIndexer', './indexer/txindexer');
bcoin.define('AddrIndexer', './indexer/addrindexer');

// Mempool
bcoin.define('mempool', './mempool');
Original file line number Diff line number Diff line change
@@ -9,8 +9,25 @@
const assert = require('assert');
const bdb = require('bdb');
const layout = require('./layout');
const Address = require('../../primitives/address');
const Indexer = require('../indexer');
const Address = require('../primitives/address');
const Indexer = require('./indexer');

/*
* AddrIndexer Database Layout:
* T[addr-hash][hash] -> dummy (tx by address)
* C[addr-hash][hash][index] -> dummy (coin by address)
*/

Object.assign(layout, {
T: bdb.key('T', ['hash', 'hash256']),
C: bdb.key('C', ['hash', 'hash256', 'uint32'])
});

/**
* AddrIndexer
* @alias module:indexer.AddrIndexer
* @extends Indexer
*/

class AddrIndexer extends Indexer {
/**
@@ -28,9 +45,9 @@ class AddrIndexer extends Indexer {
/**
* Index transactions by address.
* @private
* @param (ChainEntry) entry
* @param (Block) block
* @param (CoinView) view
* @param {ChainEntry} entry
* @param {Block} block
* @param {CoinView} view
*/

async indexBlock(entry, block, view) {
@@ -74,9 +91,9 @@ class AddrIndexer extends Indexer {
/**
* Remove addresses from index.
* @private
* @param (ChainEntry) entry
* @param (Block) block
* @param (CoinView) view
* @param {ChainEntry} entry
* @param {Block} block
* @param {CoinView} view
*/

async unindexBlock(entry, block, view) {
26 changes: 0 additions & 26 deletions lib/indexer/addrindexer/layout.js

This file was deleted.

14 changes: 10 additions & 4 deletions lib/indexer/chainclient.js
Original file line number Diff line number Diff line change
@@ -8,24 +8,28 @@

const assert = require('assert');
const AsyncEmitter = require('bevent');
const Chain = require('../blockchain/chain');

/**
* Chain Client
* @extends AsyncEmitter
* @alias module:indexer.ChainClient
*/

class ChainClient extends AsyncEmitter {
/**
* Create a chain client.
* @constructor
* @param {Chain} chain
*/

constructor(chain) {
super();

assert(chain instanceof Chain);

this.chain = chain;
this.network = chain.network;
this.filter = null;
this.opened = false;

this.init();
@@ -92,7 +96,7 @@ class ChainClient extends AsyncEmitter {
/**
* Get chain entry.
* @param {Hash} hash
* @returns {Promise}
* @returns {Promise} - Returns {@link ChainEntry}.
*/

async getEntry(hash) {
@@ -132,7 +136,7 @@ class ChainClient extends AsyncEmitter {
/**
* Get block
* @param {Hash} hash
* @returns {Promise}
* @returns {Promise} - Returns {@link Block}
*/

async getBlock(hash) {
@@ -164,8 +168,10 @@ class ChainClient extends AsyncEmitter {
async rescan(start) {
for (let i = start; ; i++) {
const entry = await this.getEntry(i);
if (!entry)
if (!entry) {
await this.emitAsync('chain tip');
break;
};

const block = await this.getBlock(entry.hash);
assert(block);
8 changes: 6 additions & 2 deletions lib/indexer/index.js
Original file line number Diff line number Diff line change
@@ -6,7 +6,11 @@

'use strict';

/**
* @module indexer
*/

exports.Indexer = require('./indexer');
exports.TXIndexer = require('./txindexer/txindexer');
exports.AddrIndexer = require('./addrindexer/addrindexer');
exports.TXIndexer = require('./txindexer');
exports.AddrIndexer = require('./addrindexer');
exports.ChainClient = require('./chainclient');
52 changes: 37 additions & 15 deletions lib/indexer/indexer.js
Original file line number Diff line number Diff line change
@@ -27,6 +27,10 @@ const {
* Indexer
* @alias module:indexer.Indexer
* @extends EventEmitter
* @property {IndexerDB} db
* @property {Number} height
* @property {ChainState} state
* @emits Indexer#chain tip
*/

class Indexer extends EventEmitter {
@@ -47,7 +51,6 @@ class Indexer extends EventEmitter {

this.network = this.options.network;
this.logger = this.options.logger.context(`${module}indexer`);
this.workers = this.options.workers;
this.client = this.options.client || new NullClient(this);
this.db = null;
this.rescanning = false;
@@ -122,6 +125,12 @@ class Indexer extends EventEmitter {
this.emit('error', e);
}
});

this.client.on('chain tip', async () => {
this.logger.debug('Indexer: finished rescan');
const tip = await this.getTip();
this.emit('chain tip', tip);
});
}

/**
@@ -133,7 +142,7 @@ class Indexer extends EventEmitter {
if (fs.unsupported)
return undefined;

if (this.memory)
if (this.options.memory)
return undefined;

return fs.mkdirp(this.options.prefix);
@@ -147,7 +156,7 @@ class Indexer extends EventEmitter {
async open() {
await this.ensure();
await this.db.open();
await this.db.verify(layout.V.build(), 'index', 7);
await this.db.verify(layout.V.build(), 'index', 0);

await this.verifyNetwork();

@@ -302,6 +311,7 @@ class Indexer extends EventEmitter {
if (this.state.startHeight < height)
height = this.state.startHeight;

this.logger.spam('Starting block rescan from: %d.', height);
return this.scan(height);
}

@@ -314,6 +324,8 @@ class Indexer extends EventEmitter {
*/

async rescanBlock(entry, block, view) {
this.logger.spam('Rescanning block: %d.', entry.height);

if (!this.rescanning) {
this.logger.warning('Unsolicited rescan block: %d.', entry.height);
return;
@@ -343,7 +355,7 @@ class Indexer extends EventEmitter {
*/

async scan(height) {
assert((height >>> 0) === height, 'WDB: Must pass in a height.');
assert((height >>> 0) === height, 'Indexer: Must pass in a height.');

await this.rollback(height);

@@ -465,7 +477,7 @@ class Indexer extends EventEmitter {
const tip = await this.getBlock(this.state.height);

if (!tip)
throw new Error('WDB: Tip not found!');
throw new Error('Indexer: Tip not found!');

return tip;
}
@@ -478,7 +490,7 @@ class Indexer extends EventEmitter {

async rollback(height) {
if (height > this.state.height)
throw new Error('WDB: Cannot rollback to the future.');
throw new Error('Indexer: Cannot rollback to the future.');

if (height === this.state.height) {
this.logger.info('Rolled back to same height (%d).', height);
@@ -526,6 +538,8 @@ class Indexer extends EventEmitter {
if (tip.height >= this.network.block.slowHeight && !this.rescanning)
this.logger.debug('Adding block: %d.', tip.height);

this.logger.spam('Adding block: %d.', entry.height);

if (tip.height === this.state.height) {
// We let blocks of the same height
// through specifically for rescans:
@@ -539,6 +553,8 @@ class Indexer extends EventEmitter {
return;
}

this.logger.spam('Indexing block: %d.', entry.height);

await this.indexBlock(entry, block, view);

// Sync the state to the new tip.
@@ -549,6 +565,7 @@ class Indexer extends EventEmitter {

/**
* Process block indexing
* Indexers will implement this method to process the block for indexing
* @param {ChainEntry} entry
* @param {Block} block
* @returns {Promise}
@@ -560,6 +577,7 @@ class Indexer extends EventEmitter {

/**
* Undo block indexing
* Indexers will implement this method to undo indexing for the block
* @param {ChainEntry} entry
* @param {Block} block
* @returns {Promise}
@@ -605,8 +623,10 @@ class Indexer extends EventEmitter {
async _removeBlock(entry, block, view) {
const tip = BlockMeta.fromEntry(entry);

this.logger.spam('Removing block: %d.', entry.height);

if (tip.height === 0)
throw new Error('WDB: Bad disconnection (genesis block).');
throw new Error('Indexer: Bad disconnection (genesis block).');

if (tip.height > this.state.height) {
this.logger.warning(
@@ -616,7 +636,9 @@ class Indexer extends EventEmitter {
}

if (tip.height !== this.state.height)
throw new Error('WDB: Bad disconnection (height mismatch).');
throw new Error('Indexer: Bad disconnection (height mismatch).');

this.logger.spam('Unindexing block: %d.', entry.height);

await this.unindexBlock(entry, block, view);

@@ -653,7 +675,7 @@ class Indexer extends EventEmitter {

async _resetChain(entry) {
if (entry.height > this.state.height)
throw new Error('WDB: Bad reset height.');
throw new Error('Indexer: Bad reset height.');

return this.rollback(entry.height);
}
@@ -676,7 +698,6 @@ class IndexOptions {
this.module = module;
this.network = Network.primary;
this.logger = Logger.global;
this.workers = null;
this.client = null;
this.chain = null;
this.indexers = null;
@@ -708,11 +729,6 @@ class IndexOptions {
this.logger = options.logger;
}

if (options.workers != null) {
assert(typeof options.workers === 'object');
this.workers = options.workers;
}

if (options.client != null) {
assert(typeof options.client === 'object');
this.client = options.client;
@@ -777,6 +793,12 @@ class IndexOptions {
* Helpers
*/

/**
* fromU32
* read a 4 byte Uint32LE
* @param {Number} num number
* @returns {Buffer} buffer
*/
function fromU32(num) {
const data = Buffer.allocUnsafe(4);
data.writeUInt32LE(num, 0, true);
1 change: 1 addition & 0 deletions lib/indexer/layout.js
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ const bdb = require('bdb');

/*
* Index Database Layout:
* To be extended by indexer implementations
* V -> db version
* O -> flags
* h[height] -> recent block hash
Loading

0 comments on commit 89af2bb

Please sign in to comment.