From e60c36c183e53477d44fd98489b9e62a456f7aca Mon Sep 17 00:00:00 2001 From: Cofresi Date: Wed, 5 Jun 2019 13:10:24 -0400 Subject: [PATCH] addHeaders with chunks out of sequence (#22) * add fn orphanChunksReconnect * add out of order tests * bump version * update package lock * use this instead of self * add jsdoc * more jsdoc * use while loop in checkPruneBlocks * group private & public methods * typo --- lib/spvchain.js | 179 +++++++++++++++++++++++++++++++++++----------- package-lock.json | 8 +-- package.json | 2 +- test/index.js | 112 +++++++++++++++++++++++++++-- 4 files changed, 249 insertions(+), 52 deletions(-) diff --git a/lib/spvchain.js b/lib/spvchain.js index 9cf1d40..40c3ad7 100644 --- a/lib/spvchain.js +++ b/lib/spvchain.js @@ -69,14 +69,11 @@ const SpvChain = class { this.setAllBranches(); } - getLongestChain() { - return this.allBranches.sort((b1, b2) => b1 < b2)[0]; - } - + /** @private */ checkPruneBlocks() { const longestChain = this.getLongestChain(); - if (longestChain.length > this.confirmsBeforeFinal) { + while (longestChain.length > this.confirmsBeforeFinal) { const pruneBlock = longestChain.splice(0, 1)[0]; // Children discarded as stale branches delete pruneBlock.orphan; @@ -84,25 +81,7 @@ const SpvChain = class { } } - getTipHash() { - return this.getLongestChain().slice(-1)[0].hash; - } - - getTipHeader() { - return this.getLongestChain().slice(-1)[0]; - } - - getHeader(hash) { - return this.store.get(hash) - .then((blockInDB) => { - if (blockInDB) { - return blockInDB; - } - - return this.getLongestChain().filter(h => h.hash === hash)[0]; - }); - } - + /** @private */ findConnection(newHeader) { const stack = [this.root]; while (stack.length > 0) { @@ -115,6 +94,7 @@ const SpvChain = class { return null; } + /** @private */ setAllBranches(node = this.root, branch = []) { this.allBranches = []; branch.push(node); @@ -128,22 +108,26 @@ const SpvChain = class { } } + /** @private */ appendHeadersToLongestChain(headers) { const newLongestChain = this.getLongestChain().concat(headers); this.allBranches = []; this.allBranches.push(newLongestChain); } + /** @private */ getAllBranches() { return this.allBranches; } + /** @private */ isDuplicate(compareHash) { return this.getAllBranches().map(branch => branch.map(node => node.hash)) .concat(this.orphanBlocks.map(orphan => orphan.hash)) .filter(hash => hash === compareHash).length > 0; } + /** @private */ orphanReconnect() { for (let i = 0; i < this.orphanBlocks.length; i += 1) { const connectionTip = this.findConnection(this.orphanBlocks[i]); @@ -154,10 +138,28 @@ const SpvChain = class { } } + /** @private */ + orphanChunksReconnect() { + this.orphanChunks.sort((a, b) => a[0].timestamp - b[0].timestamp); + this.orphanChunks.slice().forEach((chunk, index) => { + if (this.getTipHash() === utils.getCorrectedHash(chunk[0].prevHash)) { + this.appendHeadersToLongestChain(chunk); + this.orphanChunks.splice(index, 1); + } + }); + } + + /** @private */ getOrphans() { return this.orphanBlocks; } + /** @private */ + getOrphanChunks() { + return this.orphanChunks; + } + + /** @private */ processValidHeader(header) { const connection = this.findConnection(header); if (connection) { @@ -168,20 +170,27 @@ const SpvChain = class { } } - addHeader(header) { - const headerNormalised = utils.normalizeHeader(header); - - if (this.isValid(headerNormalised, this.getLongestChain())) { - headerNormalised.children = []; - this.processValidHeader(headerNormalised); - this.setAllBranches(); - this.checkPruneBlocks(); - return true; - } - return false; + /** @private + * validates a dashcore.BlockHeader object + * + * @param {Object} header + * @param {Object[]} previousHeaders + * @return {boolean} + */ + isValid(header, previousHeaders) { + return !!(Consensus.isValidBlockHeader(header, previousHeaders, this.network) + && !this.isDuplicate(header.hash)); } /* eslint-disable no-param-reassign */ + /** + * verifies the parent child connection + * between two adjacent dashcore.BlockHeader objects + * + * @param {Object} header + * @param {Object} previousHeader + * @return {boolean} + */ static isParentChild(header, previousHeader) { if (utils.getCorrectedHash(header.prevHash) !== previousHeader.hash) { return false; @@ -197,13 +206,87 @@ const SpvChain = class { } /* eslint-enable no-param-reassign */ - isValid(header, previousHeaders) { - return !!(Consensus.isValidBlockHeader(header, previousHeaders, this.network) - && !this.isDuplicate(header.hash)); + /** + * gets the longest chain + * + * @return {Object[]} + */ + getLongestChain() { + return this.allBranches.sort((b1, b2) => b1 < b2)[0]; + } + + /** + * gets the block hash of the longest chain tip + * + * @return {string} hash + */ + getTipHash() { + return this.getLongestChain().slice(-1)[0].hash; + } + + /** + * gets the dashcore.BlockHeader object the longest chain tip + * + * @return {Object} header + */ + getTipHeader() { + return this.getLongestChain().slice(-1)[0]; + } + + /** + * gets the dashcore.BlockHeader object for a specific block hash + * + * @param {string} hash + * @return {Object} header + */ + getHeader(hash) { + return this.store.get(hash) + .then((blockInDB) => { + if (blockInDB) { + return blockInDB; + } + + return this.getLongestChain().filter(h => h.hash === hash)[0]; + }); + } + + /** + * adds a valid header to the tip of the longest spv chain. + * If it cannot be connected to the tip it gets temporarily + * added to an orphan array for possible later reconnection + * + * @param {Object[]|string[]|buffer[]} header + * @return {boolean} + */ + addHeader(header) { + const headerNormalised = utils.normalizeHeader(header); + + if (this.isValid(headerNormalised, this.getLongestChain())) { + headerNormalised.children = []; + this.processValidHeader(headerNormalised); + this.setAllBranches(); + this.checkPruneBlocks(); + return true; + } + return false; } + /** + * adds an array of valid headers to the longest spv chain. + * If they cannot be connected to last tip they get temporarily + * added to an orphan array for possible later reconnection + * + * @param {Object[]|string[]|buffer[]} headers + * @return {boolean} + */ addHeaders(headers) { - const self = this; + if (headers.length === 1) { + if (!this.addHeader(headers[0])) { + throw new Error('Some headers are invalid'); + } else { + return true; + } + } const normalizedHeaders = headers.map(h => utils.normalizeHeader(h)); const isOrphan = !SpvChain.isParentChild(normalizedHeaders[0], this.getTipHeader()); @@ -212,12 +295,18 @@ const SpvChain = class { const previousHeaders = normalizedHeaders.slice(0, index); if (index !== 0) { if (!SpvChain.isParentChild(header, array[index - 1]) - || !self.isValid(header, previousHeaders)) { + || !this.isValid(header, previousHeaders)) { throw new Error('Some headers are invalid'); } return acc && true; } - if (!self.isValid(header, self.getLongestChain())) { + if (isOrphan) { + if (!this.isValid(header, previousHeaders)) { + throw new Error('Some headers are invalid'); + } + return acc && true; + } + if (!this.isValid(header, this.getLongestChain())) { throw new Error('Some headers are invalid'); } return acc && true; @@ -227,11 +316,15 @@ const SpvChain = class { throw new Error('Some headers are invalid'); } if (isOrphan) { - this.orphanChunks.push(headers); + this.orphanChunks.push(normalizedHeaders); } else { - self.appendHeadersToLongestChain(normalizedHeaders); + this.appendHeadersToLongestChain(normalizedHeaders); + } + if (this.orphanChunks.length > 0) { + this.orphanChunksReconnect(); } this.checkPruneBlocks(); + return true; } }; diff --git a/package-lock.json b/package-lock.json index d14c59f..8c493f6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@dashevo/dash-spv", - "version": "1.1.4", + "version": "1.1.5", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -1450,9 +1450,9 @@ "dev": true }, "resolve": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.0.tgz", - "integrity": "sha512-WL2pBDjqT6pGUNSUzMw00o4T7If+z4H2x3Gz893WoUQ5KW8Vr9txp00ykiP16VBaZF5+j/OcXJHZ9+PCvdiDKw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.11.1.tgz", + "integrity": "sha512-vIpgF6wfuJOZI7KKKSP+HmiKggadPQAdsp5HiC1mvqnfp0gF1vdwgBWZIdrVft9pgqoMFQN+R7BSWZiBxx+BBw==", "dev": true, "requires": { "path-parse": "^1.0.6" diff --git a/package.json b/package.json index 271f787..31e9a61 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@dashevo/dash-spv", - "version": "1.1.4", + "version": "1.1.5", "description": "Temporary repo until spv functions moved into dashcore-lib", "main": "index.js", "scripts": { diff --git a/test/index.js b/test/index.js index e626fbf..e4d27c0 100644 --- a/test/index.js +++ b/test/index.js @@ -177,9 +177,15 @@ describe('SPV-DASH (addHeaders) add many headers for testnet', () => { chain.getLongestChain().length.should.equal(500); }); - it('should throw an error if some of the headers is invalid', (done) => { + it('should orphan and not add invalid but consistent headers', () => { + chain.addHeaders([badRawHeaders[0], badRawHeaders[1]]); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(500); + }); + + it('should throw an error if some of the headers are inconsistent', (done) => { try { - chain.addHeaders([badRawHeaders[0], badRawHeaders[1]]); + chain.addHeaders([badRawHeaders[0], badRawHeaders[2]]); done(new Error('SPV chain failed to throw an error on invalid block')); } catch (e) { e.message.should.equal('Some headers are invalid'); @@ -188,6 +194,52 @@ describe('SPV-DASH (addHeaders) add many headers for testnet', () => { }); }); +describe('SPV-DASH (addHeaders) add testnet headers out of order', () => { + before(() => { + chain = new Blockchain('testnet', 10000, utils.normalizeHeader(testnet[0])); + }); + + it('should add the 1st 100 testnet headers', () => { + chain.addHeaders(testnet.slice(1, 100)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(0); + chain.getLongestChain().length.should.equal(100); + }); + + it('should orphan testnet headers 200 - 300', () => { + chain.addHeaders(testnet.slice(200, 300)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(100); + }); + + it('should orphan testnet headers 400 - 500', () => { + chain.addHeaders(testnet.slice(400, 500)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(2); + chain.getLongestChain().length.should.equal(100); + }); + + it('should reconnect orphaned chunks (testnet headers 1 - 100 and 200 - 300)', () => { + chain.addHeaders(testnet.slice(100, 200)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(300); + }); + + it('should reconnect orphaned chunks (testnet headers 1 - 300 and 400 - 500)', () => { + chain.addHeaders(testnet.slice(300, 400)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(0); + chain.getLongestChain().length.should.equal(500); + }); +}); + describe('SPV-DASH (addHeaders) add many headers for mainnet', () => { before(() => { chain = new Blockchain('mainnet', 10000, utils.normalizeHeader(mainnet[0])); @@ -219,9 +271,15 @@ describe('SPV-DASH (addHeaders) add many headers for mainnet', () => { chain.getLongestChain().length.should.equal(1500); }); - it('should throw an error if some of the headers is invalid', (done) => { + it('should orphan and not add invalid but consistent headers', () => { + chain.addHeaders([badRawHeaders[0], badRawHeaders[1]]); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(1500); + }); + + it('should throw an error if some of the headers are inconsistent', (done) => { try { - chain.addHeaders([badRawHeaders[0], badRawHeaders[1]]); + chain.addHeaders([badRawHeaders[0], badRawHeaders[2]]); done(new Error('SPV chain failed to throw an error on invalid block')); } catch (e) { e.message.should.equal('Some headers are invalid'); @@ -230,6 +288,52 @@ describe('SPV-DASH (addHeaders) add many headers for mainnet', () => { }); }); +describe('SPV-DASH (addHeaders) add mainnet headers out of order', () => { + before(() => { + chain = new Blockchain('mainnet', 10000, utils.normalizeHeader(mainnet[0])); + }); + + it('should add the 1st 100 mainnet headers', () => { + chain.addHeaders(mainnet.slice(1, 100)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(0); + chain.getLongestChain().length.should.equal(100); + }); + + it('should orphan mainnet headers 200 - 300', () => { + chain.addHeaders(mainnet.slice(200, 300)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(100); + }); + + it('should orphan mainnet headers 400 - 500', () => { + chain.addHeaders(mainnet.slice(400, 500)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(2); + chain.getLongestChain().length.should.equal(100); + }); + + it('should reconnect orphaned chunks (mainnet headers 1 - 100 and 200 - 300)', () => { + chain.addHeaders(mainnet.slice(100, 200)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(1); + chain.getLongestChain().length.should.equal(300); + }); + + it('should reconnect orphaned chunks (mainnet headers 1 - 300 and 400 - 500)', () => { + chain.addHeaders(mainnet.slice(300, 400)); + chain.getOrphans().length.should.equal(0); + chain.getAllBranches().length.should.equal(1); + chain.getOrphanChunks().length.should.equal(0); + chain.getLongestChain().length.should.equal(500); + }); +}); + let genesisHash = null; describe('Blockstore', () => { before(() => {