From 32d547a3d3115a21266426d807a824feab8da300 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 12:24:30 +0200 Subject: [PATCH 01/33] Write genesis block to filesystem asynchronously --- services/core/shared/core/compat/sdk_v5/blocksUtils.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 2a88fdb457..83149293be 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -54,8 +54,7 @@ const downloadGenesisBlock = async () => { request(genesisBlockURL) .then(async response => { const genesisBlock = typeof response === 'string' ? JSON.parse(response).data : response.data; - fs.writeFileSync(genesisBlockFilePath, JSON.stringify(genesisBlock)); - resolve(); + fs.writeFile(genesisBlockFilePath, JSON.stringify(genesisBlock), () => resolve()); }) .catch(err => reject(err)); } From b0cba2ed435eaa20108a945b0fc00586011fabd6 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 12:31:38 +0200 Subject: [PATCH 02/33] Keep downloaded genesis block in-memory for faster retrieval --- services/core/package-lock.json | 73 ++++++++++++++++++- services/core/package.json | 1 + .../shared/core/compat/sdk_v5/blocksUtils.js | 43 ++++++----- 3 files changed, 99 insertions(+), 18 deletions(-) diff --git a/services/core/package-lock.json b/services/core/package-lock.json index e73529aa6e..47a519f96a 100644 --- a/services/core/package-lock.json +++ b/services/core/package-lock.json @@ -2830,6 +2830,15 @@ "integrity": "sha512-37RSHht+gzzgYeobbG+KWryeAW8J33Nhr69cjTqSYymXVZEN9NbRYWoYlRtDhHKPVT1FyNKwaTPC1NynKZpzRA==", "dev": true }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "abab": { "version": "2.0.5", "resolved": "https://npm.lisk.io/abab/-/abab-2.0.5.tgz", @@ -3329,6 +3338,19 @@ } } }, + "big-json": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/big-json/-/big-json-3.1.0.tgz", + "integrity": "sha512-Xfzbp/LWMgOg41eO72u+TJnUflNsZf1U/Yhw9kNHP4J418xKzAxdVT5aHbZBQcAtjBj6KZasPefeAWclHyKrpQ==", + "requires": { + "JSONStream": "^1.3.1", + "assert-plus": "^1.0.0", + "into-stream": "^5.1.0", + "json-stream-stringify": "^2.0.1", + "once": "^1.4.0", + "through2": "^3.0.1" + } + }, "big-number": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/big-number/-/big-number-2.0.0.tgz", @@ -3341,7 +3363,7 @@ }, "block-stream": { "version": "0.0.9", - "resolved": "https://registry.npmjs.org/block-stream/-/block-stream-0.0.9.tgz", + "resolved": "https://npm.lisk.io/block-stream/-/block-stream-0.0.9.tgz", "integrity": "sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo=", "optional": true, "requires": { @@ -4793,6 +4815,15 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + } + }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -5236,6 +5267,15 @@ "resolved": "https://registry.npmjs.org/interpret/-/interpret-2.2.0.tgz", "integrity": "sha512-Ju0Bz/cEia55xDwUWEa8+olFpCiQoypjnQySseKtmjNrnps3P+xfpUmGr90T7yjlVJmOtybRvPXhKMbHr+fWnw==" }, + "into-stream": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-5.1.1.tgz", + "integrity": "sha512-krrAJ7McQxGGmvaYbB7Q1mcA+cRwg9Ij2RfWIeVesNBgVDZmzY/Fa4IpZUT3bmdRzMzdf/mzltCG2Dq99IZGBA==", + "requires": { + "from2": "^2.3.0", + "p-is-promise": "^3.0.0" + } + }, "ioredis": { "version": "4.27.1", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.27.1.tgz", @@ -8329,6 +8369,11 @@ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, + "json-stream-stringify": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/json-stream-stringify/-/json-stream-stringify-2.0.3.tgz", + "integrity": "sha512-Ry+1rZE1YVKlCMG1emCiReP/OfZrFrEGZn6TC7NPpIGO9NXf0KEqqBL0flgt2J59EiRPv+CKi7S7v31RAHrdXw==" + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -8351,6 +8396,11 @@ "graceful-fs": "^4.1.6" } }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" + }, "jsprim": { "version": "1.4.1", "resolved": "https://npm.lisk.io/jsprim/-/jsprim-1.4.1.tgz", @@ -9502,6 +9552,11 @@ "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" }, + "p-is-promise": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz", + "integrity": "sha512-Wo8VsW4IRQSKVXsJCn7TomUaVtyfjVDn3nUP7kE967BQk0CwFpdbZs0X0uk5sW9mkBa9eNM7hCMaG93WUAwxYQ==" + }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -11260,6 +11315,22 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, + "through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "requires": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + }, + "dependencies": { + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + } + } + }, "tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", diff --git a/services/core/package.json b/services/core/package.json index 6f514df5e1..1f1da70748 100644 --- a/services/core/package.json +++ b/services/core/package.json @@ -34,6 +34,7 @@ "@liskhq/lisk-transactions-v4": "npm:@liskhq/lisk-transactions@^4.0.0", "@liskhq/lisk-transactions-v5": "npm:@liskhq/lisk-transactions@^5.0.3", "async": "^3.2.0", + "big-json": "^3.1.0", "big-number": "=2.0.0", "bluebird": "^3.7.2", "body-parser": "^1.19.0", diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 83149293be..583603da33 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -15,6 +15,7 @@ */ const fs = require('fs'); const https = require('https'); +const json = require('big-json'); const path = require('path'); const tar = require('tar'); @@ -31,17 +32,22 @@ const genesisBlockURL = config.endpoints.genesisBlock; const genesisBlockFilePath = './shared/core/compat/sdk_v5/static/genesis_block.json'; -let genesisBlockId; +let readStream; +let genesisBlock = { header: {} }; -const setGenesisBlockId = (id) => genesisBlockId = id; +const parseStream = json.createParseStream(); -const getGenesisBlockId = () => genesisBlockId; +const setGenesisBlock = (block) => genesisBlock = block; + +const getGenesisBlock = () => genesisBlock; + +const getGenesisBlockId = () => genesisBlock.header.id; const downloadGenesisBlock = async () => { const directoryPath = path.dirname(genesisBlockFilePath); if (!fs.existsSync(directoryPath)) fs.mkdirSync(directoryPath); - logger.info(`Persisting genesis block to the filesystem. Downloading from: ${genesisBlockURL}`); + logger.info(`Downloading genesis block to the filesystem from: ${genesisBlockURL}`); return new Promise((resolve, reject) => { if (genesisBlockURL.endsWith('.tar.gz')) { @@ -62,23 +68,26 @@ const downloadGenesisBlock = async () => { }; const getGenesisBlockFromFS = async () => { - if (!fs.existsSync(genesisBlockFilePath)) await downloadGenesisBlock(); - - const genesisBlock = await new Promise((resolve, reject) => { - fs.readFile(genesisBlockFilePath, (err, data) => { - if (err) { - logger.error(err); - return reject(err); - } - const parsedGenesisBlock = JSON.parse(data.toString()); - return resolve(parsedGenesisBlock); + if (!getGenesisBlockId()) { + if (!fs.existsSync(genesisBlockFilePath)) { + await downloadGenesisBlock(); + readStream = fs.createReadStream(genesisBlockFilePath); + } + + const block = await new Promise((resolve, reject) => { + readStream.pipe(parseStream.on('data', (data) => resolve(data))); + parseStream.on('error', (err) => reject(err)); }); - }); - if (!getGenesisBlockId()) setGenesisBlockId(genesisBlock.header.id); - return genesisBlock; + if (!getGenesisBlockId()) setGenesisBlock(block); + } + + return getGenesisBlock(); }; +// If file exists, already create a read stream +if (fs.existsSync(genesisBlockFilePath)) readStream = fs.createReadStream(genesisBlockFilePath); + module.exports = { getGenesisBlockId, getGenesisBlockFromFS, From 0300d91f06633d52d4dd82a6d17e87346c7ebea9 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 12:46:27 +0200 Subject: [PATCH 03/33] Remove duplicate addresses when querying Core --- services/core/shared/core/compat/sdk_v5/accounts.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/accounts.js b/services/core/shared/core/compat/sdk_v5/accounts.js index 5bedc1dd1e..b2df85a262 100644 --- a/services/core/shared/core/compat/sdk_v5/accounts.js +++ b/services/core/shared/core/compat/sdk_v5/accounts.js @@ -108,7 +108,10 @@ const getAccountsFromCore = async (params) => { }; const indexAccountsbyAddress = async (addressesToIndex, isGenesisBlockAccount = false) => { - const { data: accountsToIndex } = await getAccountsFromCore({ addresses: addressesToIndex }); + const { data: accountsToIndex } = await getAccountsFromCore({ + addresses: addressesToIndex + .filter((v, i, a) => a.findIndex(t => (t === v)) === i), // Remove duplicates + }); const finalAccountsToIndex = await BluebirdPromise.map( accountsToIndex, async account => { @@ -224,7 +227,8 @@ const indexAccountsbyPublicKey = async (accountInfoArray) => { const { data: accountsToIndex } = await getAccountsFromCore({ addresses: accountInfoArray - .map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey)), + .map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey)) + .filter((v, i, a) => a.findIndex(t => (t === v)) === i), // Remove duplicates }); const finalAccountsToIndex = await BluebirdPromise.map( From 0b1935b1f17c93f61c7a1eb0f07cb31396e39bcd Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 14:55:23 +0200 Subject: [PATCH 04/33] Network specific config --- services/core/config.js | 16 ++++++++++- .../core/shared/core/compat/sdk_v5/blocks.js | 9 ++++++- .../shared/core/compat/sdk_v5/blocksUtils.js | 27 +++++++++++++------ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/services/core/config.js b/services/core/config.js index 756587cee2..8e665c6ae7 100644 --- a/services/core/config.js +++ b/services/core/config.js @@ -36,6 +36,21 @@ config.endpoints.geoip = process.env.GEOIP_JSON || 'https://geoip.lisk.io/json'; config.endpoints.mysql = process.env.SERVICE_CORE_MYSQL || 'mysql://lisk:password@localhost:3306/lisk'; config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz'; +config.network = [ + { + name: 'mainnet', + identifier: 'ed14889723f24ecc54871d058d98ce91ff2f973192075c0155ba2b7b70ad2511', + genesisHeight: Number(process.env.GENESIS_HEIGHT || 0), + genesisBlockUrl: 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz', + }, + { + name: 'testnet', + identifier: '15f0dacc1060e91818224a94286b13aa04279c640bd5d6f193182031d133df7c', + genesisHeight: 14075260, + genesisBlockUrl: 'https://downloads.lisk.io/lisk/testnet/genesis_block.json.tar.gz', + }, +]; + /** * Indexing * @@ -48,7 +63,6 @@ config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downlo * such as transactions, accounts, votes etc. */ config.indexNumOfBlocks = Number(process.env.INDEX_N_BLOCKS || 202); -config.genesisHeight = Number(process.env.GENESIS_HEIGHT || 0); config.transactionStatistics = { enabled: Boolean(String(process.env.ENABLE_TRANSACTION_STATS).toLowerCase() === 'true'), diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index f90a391d21..fb5cfbe4fc 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -56,10 +56,12 @@ const getBlocksIndex = () => mysqlIndex('blocks', blocksIndexSchema); const logger = Logger(); -const { genesisHeight } = config; +let genesisHeight; let finalizedHeight; let indexStartHeight; +const setGenesisHeight = (height) => genesisHeight = height; + const getGenesisHeight = () => genesisHeight; const setFinalizedHeight = (height) => finalizedHeight = height; @@ -519,6 +521,11 @@ const init = async () => { // Check state of index and perform update try { + // Determine genesis height + const { data: { networkIdentifier } } = await coreApi.getNetworkStatus(); + const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); + setGenesisHeight(networkConfig.genesisHeight); + await indexGenesisBlock().catch(err => { logger.error(err.message); logger.warn('Unable to index the Genesis block. Continuing with the remaining...'); diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 583603da33..9628f3daa0 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -24,15 +24,14 @@ const { HTTP: { request }, } = require('lisk-service-framework'); +const { getApiClient } = require('../common/wsRequest'); const config = require('../../../../config'); const logger = Logger(); -const genesisBlockURL = config.endpoints.genesisBlock; - -const genesisBlockFilePath = './shared/core/compat/sdk_v5/static/genesis_block.json'; - let readStream; +let genesisBlockURL; +let genesisBlockFilePath; let genesisBlock = { header: {} }; const parseStream = json.createParseStream(); @@ -43,9 +42,23 @@ const getGenesisBlock = () => genesisBlock; const getGenesisBlockId = () => genesisBlock.header.id; +const loadConfig = async () => { + // Direct invocation of action necessary to avoid circular dependency + const apiClient = await getApiClient(); + const { networkIdentifier } = await apiClient.node.getNodeInfo(); + + const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); + genesisBlockURL = networkConfig.genesisBlockUrl; + + genesisBlockFilePath = `./shared/core/compat/sdk_v5/static/${networkConfig.name}/genesis_block.json`; + + // If file exists, already create a read stream + if (fs.existsSync(genesisBlockFilePath)) readStream = fs.createReadStream(genesisBlockFilePath); +}; + const downloadGenesisBlock = async () => { const directoryPath = path.dirname(genesisBlockFilePath); - if (!fs.existsSync(directoryPath)) fs.mkdirSync(directoryPath); + if (!fs.existsSync(directoryPath)) fs.mkdirSync(directoryPath, { recursive: true }); logger.info(`Downloading genesis block to the filesystem from: ${genesisBlockURL}`); @@ -68,6 +81,7 @@ const downloadGenesisBlock = async () => { }; const getGenesisBlockFromFS = async () => { + if (!genesisBlockURL || !genesisBlockFilePath) await loadConfig(); if (!getGenesisBlockId()) { if (!fs.existsSync(genesisBlockFilePath)) { await downloadGenesisBlock(); @@ -85,9 +99,6 @@ const getGenesisBlockFromFS = async () => { return getGenesisBlock(); }; -// If file exists, already create a read stream -if (fs.existsSync(genesisBlockFilePath)) readStream = fs.createReadStream(genesisBlockFilePath); - module.exports = { getGenesisBlockId, getGenesisBlockFromFS, From f946614a2093745c10c40da3546e78e48f055423 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 15:57:23 +0200 Subject: [PATCH 05/33] Code refactor --- .../core/shared/core/compat/sdk_v5/blocks.js | 6 ++---- .../core/shared/core/compat/sdk_v5/coreApi.js | 19 +++++++++++++++---- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index fb5cfbe4fc..b17e506045 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -521,10 +521,8 @@ const init = async () => { // Check state of index and perform update try { - // Determine genesis height - const { data: { networkIdentifier } } = await coreApi.getNetworkStatus(); - const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); - setGenesisHeight(networkConfig.genesisHeight); + // Set the genesis height + setGenesisHeight(await coreApi.getGenesisHeight()); await indexGenesisBlock().catch(err => { logger.error(err.message); diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index 114b7c7230..aca32d291d 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -24,8 +24,9 @@ const { } = require('./blocksUtils'); const { getApiClient } = require('../common/wsRequest'); -const { genesisHeight } = require('../../../../config'); +const config = require('../../../../config'); +let genesisHeight; const logger = Logger(); const timeoutMessage = 'Response not received in'; @@ -43,6 +44,16 @@ const getNetworkStatus = async () => { } }; +const getGenesisHeight = async () => { + if (!genesisHeight) { + // Determine genesis height + const { data: { networkIdentifier } } = await getNetworkStatus(); + const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); + genesisHeight = Number(networkConfig.genesisHeight); + } + return genesisHeight; +}; + const getBlockByID = async id => { try { const apiClient = await getApiClient(); @@ -95,8 +106,7 @@ const getBlockByHeight = async height => { if (err.message.includes(timeoutMessage)) { await getApiClient(); // File based Genesis block handling - // eslint-disable-next-line max-len - if (Number(height) === Number(genesisHeight)) return { data: [await getGenesisBlockFromFS()] }; + if (Number(height) === getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlockByHeight' for height: ${height}`); } throw err; @@ -113,7 +123,7 @@ const getBlocksByHeightBetween = async (from, to) => { if (err.message.includes(timeoutMessage)) { await getApiClient(); // File based Genesis block handling - if (Number(from) === Number(genesisHeight)) { + if (Number(from) === getGenesisHeight()) { const genesisBlockResult = await getBlockByHeight(from); if (from < to) { const { data: [genesisBlock] } = genesisBlockResult; @@ -290,6 +300,7 @@ const getTransactionsSchemas = async () => { }; module.exports = { + getGenesisHeight, getBlockByID, getBlocksByIDs, getBlockByHeight, From ae9ceda99babc5412087f2ef1fde6cc050d7fa4c Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 16:04:06 +0200 Subject: [PATCH 06/33] Attempt to read genesis block from FS first if it exists --- .../core/shared/core/compat/sdk_v5/coreApi.js | 61 +++++++++++-------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index aca32d291d..0d1d36d643 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -22,7 +22,10 @@ const { getGenesisBlockId, getGenesisBlockFromFS, } = require('./blocksUtils'); -const { getApiClient } = require('../common/wsRequest'); + +const { + getApiClient, +} = require('../common/wsRequest'); const config = require('../../../../config'); @@ -56,14 +59,15 @@ const getGenesisHeight = async () => { const getBlockByID = async id => { try { + // File based Genesis block handling + if (id === getGenesisBlockId()) return { data: [await getGenesisBlockFromFS()] }; + const apiClient = await getApiClient(); const block = await apiClient.block.get(id); return { data: [block] }; } catch (err) { if (err.message.includes(timeoutMessage)) { await getApiClient(); - // File based Genesis block handling - if (id === getGenesisBlockId()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlocksByID' for ID: ${id}`); } throw err; @@ -72,6 +76,18 @@ const getBlockByID = async id => { const getBlocksByIDs = async ids => { try { + // File based Genesis block handling + if (ids.includes(genesisBlockId)) { + const remainingIds = ids.filter(id => id !== genesisBlockId); + const genesisBlockResult = await getBlockByID(genesisBlockId); + if (remainingIds.length) { + const { data: [genesisBlock] } = genesisBlockResult; + const { data: [...remainingBlocks] } = await getBlocksByIDs(remainingIds); + return { data: [genesisBlock, ...remainingBlocks] }; + } + return genesisBlockResult; + } + const apiClient = await getApiClient(); const encodedBlocks = await apiClient._channel.invoke('app:getBlocksByIDs', { ids }); const blocks = encodedBlocks.map(blk => apiClient.block.decode(Buffer.from(blk, 'hex'))); @@ -80,17 +96,6 @@ const getBlocksByIDs = async ids => { if (err.message.includes(timeoutMessage)) { await getApiClient(); const genesisBlockId = getGenesisBlockId(); - // File based Genesis block handling - if (ids.includes(genesisBlockId)) { - const remainingIds = ids.filter(id => id !== genesisBlockId); - const genesisBlockResult = await getBlockByID(genesisBlockId); - if (remainingIds.length) { - const { data: [genesisBlock] } = genesisBlockResult; - const { data: [...remainingBlocks] } = await getBlocksByIDs(remainingIds); - return { data: [genesisBlock, ...remainingBlocks] }; - } - return genesisBlockResult; - } throw new TimeoutException(`Request timed out when calling 'getBlocksByIDs' for IDs: ${ids}`); } throw err; @@ -99,14 +104,17 @@ const getBlocksByIDs = async ids => { const getBlockByHeight = async height => { try { + // File based Genesis block handling + if (getGenesisBlockId() && Number(height) === getGenesisHeight()) { + return { data: [await getGenesisBlockFromFS()] }; + } + const apiClient = await getApiClient(); const block = await apiClient.block.getByHeight(height); return { data: [block] }; } catch (err) { if (err.message.includes(timeoutMessage)) { await getApiClient(); - // File based Genesis block handling - if (Number(height) === getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlockByHeight' for height: ${height}`); } throw err; @@ -115,6 +123,17 @@ const getBlockByHeight = async height => { const getBlocksByHeightBetween = async (from, to) => { try { + // File based Genesis block handling + if (getGenesisBlockId() && Number(from) === getGenesisHeight()) { + const genesisBlockResult = await getBlockByHeight(from); + if (from < to) { + const { data: [genesisBlock] } = genesisBlockResult; + const { data: [...remainingBlocks] } = await getBlocksByHeightBetween(from + 1, to); + return { data: [genesisBlock, ...remainingBlocks] }; + } + return genesisBlockResult; + } + const apiClient = await getApiClient(); const encodedBlocks = await apiClient._channel.invoke('app:getBlocksByHeightBetween', { from, to }); const blocks = encodedBlocks.map(blk => apiClient.block.decode(Buffer.from(blk, 'hex'))); @@ -122,16 +141,6 @@ const getBlocksByHeightBetween = async (from, to) => { } catch (err) { if (err.message.includes(timeoutMessage)) { await getApiClient(); - // File based Genesis block handling - if (Number(from) === getGenesisHeight()) { - const genesisBlockResult = await getBlockByHeight(from); - if (from < to) { - const { data: [genesisBlock] } = genesisBlockResult; - const { data: [...remainingBlocks] } = await getBlocksByHeightBetween(from + 1, to); - return { data: [genesisBlock, ...remainingBlocks] }; - } - return genesisBlockResult; - } throw new TimeoutException(`Request timed out when calling 'getBlocksByHeightBetween' for heights: ${from} - ${to}`); } throw err; From fc97995dc3f5b3278138a7fe1375fa95b79ac98f Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 16:09:32 +0200 Subject: [PATCH 07/33] Fix linting errors --- services/core/shared/core/compat/sdk_v5/blocksUtils.js | 4 ++-- services/core/shared/core/compat/sdk_v5/coreApi.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 9628f3daa0..9ba002f0df 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -72,8 +72,8 @@ const downloadGenesisBlock = async () => { } else { request(genesisBlockURL) .then(async response => { - const genesisBlock = typeof response === 'string' ? JSON.parse(response).data : response.data; - fs.writeFile(genesisBlockFilePath, JSON.stringify(genesisBlock), () => resolve()); + const block = typeof response === 'string' ? JSON.parse(response).data : response.data; + fs.writeFile(genesisBlockFilePath, JSON.stringify(block), () => resolve()); }) .catch(err => reject(err)); } diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index 0d1d36d643..e139006e40 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -77,6 +77,7 @@ const getBlockByID = async id => { const getBlocksByIDs = async ids => { try { // File based Genesis block handling + const genesisBlockId = getGenesisBlockId(); if (ids.includes(genesisBlockId)) { const remainingIds = ids.filter(id => id !== genesisBlockId); const genesisBlockResult = await getBlockByID(genesisBlockId); @@ -95,7 +96,6 @@ const getBlocksByIDs = async ids => { } catch (err) { if (err.message.includes(timeoutMessage)) { await getApiClient(); - const genesisBlockId = getGenesisBlockId(); throw new TimeoutException(`Request timed out when calling 'getBlocksByIDs' for IDs: ${ids}`); } throw err; From 83a3e397000ea932df1a55fccfb7f607c9ec1b97 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 16:17:06 +0200 Subject: [PATCH 08/33] Download the genesis block if Core cannot respond and not file present on the FS --- services/core/shared/core/compat/sdk_v5/coreApi.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index e139006e40..00f530d315 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -115,6 +115,8 @@ const getBlockByHeight = async height => { } catch (err) { if (err.message.includes(timeoutMessage)) { await getApiClient(); + // Download to the FS & return the genesis block + if (Number(height) === getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlockByHeight' for height: ${height}`); } throw err; From 63015ac3274cbd250c50d41d9c46fee012c2f722 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 16:30:20 +0200 Subject: [PATCH 09/33] Await the promise to resolve --- services/core/shared/core/compat/sdk_v5/coreApi.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index 00f530d315..813aaffba9 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -105,7 +105,7 @@ const getBlocksByIDs = async ids => { const getBlockByHeight = async height => { try { // File based Genesis block handling - if (getGenesisBlockId() && Number(height) === getGenesisHeight()) { + if (getGenesisBlockId() && Number(height) === await getGenesisHeight()) { return { data: [await getGenesisBlockFromFS()] }; } @@ -116,7 +116,7 @@ const getBlockByHeight = async height => { if (err.message.includes(timeoutMessage)) { await getApiClient(); // Download to the FS & return the genesis block - if (Number(height) === getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; + if (Number(height) === await getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlockByHeight' for height: ${height}`); } throw err; @@ -126,7 +126,7 @@ const getBlockByHeight = async height => { const getBlocksByHeightBetween = async (from, to) => { try { // File based Genesis block handling - if (getGenesisBlockId() && Number(from) === getGenesisHeight()) { + if (getGenesisBlockId() && Number(from) === await getGenesisHeight()) { const genesisBlockResult = await getBlockByHeight(from); if (from < to) { const { data: [genesisBlock] } = genesisBlockResult; From 8c832312bdf446eedb8df0481d1314f2d0444c91 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 16:58:07 +0200 Subject: [PATCH 10/33] Index initDelegates when indexing genesis block --- services/core/shared/core/compat/sdk_v5/blocks.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index b17e506045..349d73e8cf 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -129,7 +129,7 @@ const deleteIndexedBlocksQueue = initializeQueue('deleteIndexedBlocksQueue', del const normalizeBlocks = async blocks => { const apiClient = await getApiClient(); - const normalizedBlocks = BluebirdPromise.map( + const normalizedBlocks = await BluebirdPromise.map( blocks.map(block => ({ ...block.header, payload: block.payload })), async block => { const account = await getIndexedAccountInfo({ publicKey: block.generatorPublicKey.toString('hex') }); @@ -362,10 +362,15 @@ const indexGenesisBlock = async () => { await indexTransactions([genesisBlock]); // Index the genesis block accounts next - const accountAddressesToIndex = genesisBlock.asset.accounts + const initDelegateAddresses = genesisBlock.asset.initDelegates; + const nonDelegateAddressesToIndex = genesisBlock.asset.accounts .filter(account => account.address.length > 16) // Filter out reclaim accounts .map(account => account.address); + const accountAddressesToIndex = initDelegateAddresses + .concat(nonDelegateAddressesToIndex) + .filter((v, i, a) => a.findIndex(t => (t === v)) === i); // Remove duplicates + const PAGE_SIZE = 20; const NUM_PAGES = Math.ceil(accountAddressesToIndex.length / PAGE_SIZE); for (let i = 0; i < NUM_PAGES; i++) { From c0bf41dff7277a947fa8ec6564079b6c10cf20ad Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 17:19:11 +0200 Subject: [PATCH 11/33] Filter out genesis assets by default when normalizing the genesis block --- .../core/shared/core/compat/sdk_v5/blocks.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index 349d73e8cf..c6d5fccb6b 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -126,7 +126,7 @@ const indexBlocksQueue = initializeQueue('indexBlocksQueue', indexBlocks); const updateBlockIndexQueue = initializeQueue('updateBlockIndexQueue', updateBlockIndex); const deleteIndexedBlocksQueue = initializeQueue('deleteIndexedBlocksQueue', deleteIndexedBlocks); -const normalizeBlocks = async blocks => { +const normalizeBlocks = async (blocks, isIgnoreGenesisAccounts = true) => { const apiClient = await getApiClient(); const normalizedBlocks = await BluebirdPromise.map( @@ -155,6 +155,17 @@ const normalizeBlocks = async blocks => { block.totalFee += Number(txn.fee) - txnMinFee; }); + if (isIgnoreGenesisAccounts) { + const { + accounts, + initRounds, + initDelegates, + ...otherAssets, + } = block.asset; + + block.asset = { ...otherAssets }; + } + return parseToJSONCompatObj(block); }, { concurrency: blocks.length }, @@ -173,9 +184,9 @@ const getBlocksByIDs = async ids => { return normalizeBlocks(response.data); }; -const getBlockByHeight = async height => { +const getBlockByHeight = async (height, isIgnoreGenesisAccounts = true) => { const response = await coreApi.getBlockByHeight(height); - return normalizeBlocks(response.data); + return normalizeBlocks(response.data, isIgnoreGenesisAccounts); }; const getBlocksByHeightBetween = async (from, to) => { @@ -356,7 +367,7 @@ const deleteBlock = async (block) => { const indexGenesisBlock = async () => { logger.info(`Ìndexing genesis block at height ${genesisHeight}`); - const [genesisBlock] = await getBlockByHeight(genesisHeight); + const [genesisBlock] = await getBlockByHeight(genesisHeight, false); // Index the genesis block transactions first await indexTransactions([genesisBlock]); From f03ea045d5f84c3d64b507e47094d52d175f2215 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 17:22:10 +0200 Subject: [PATCH 12/33] Fix linting errors --- services/core/shared/core/compat/sdk_v5/blocks.js | 2 +- services/core/shared/core/compat/sdk_v5/coreApi.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index c6d5fccb6b..f0fde7ccc0 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -160,7 +160,7 @@ const normalizeBlocks = async (blocks, isIgnoreGenesisAccounts = true) => { accounts, initRounds, initDelegates, - ...otherAssets, + ...otherAssets } = block.asset; block.asset = { ...otherAssets }; diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index 813aaffba9..39003de115 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -116,6 +116,7 @@ const getBlockByHeight = async height => { if (err.message.includes(timeoutMessage)) { await getApiClient(); // Download to the FS & return the genesis block + // eslint-disable-next-line max-len if (Number(height) === await getGenesisHeight()) return { data: [await getGenesisBlockFromFS()] }; throw new TimeoutException(`Request timed out when calling 'getBlockByHeight' for height: ${height}`); } From 0e07bd35b26408318d961a6e814b6555d8552bba Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 17:44:11 +0200 Subject: [PATCH 13/33] Remove array filter --- services/core/shared/core/compat/sdk_v5/blocks.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index f0fde7ccc0..e444abd3f9 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -378,9 +378,7 @@ const indexGenesisBlock = async () => { .filter(account => account.address.length > 16) // Filter out reclaim accounts .map(account => account.address); - const accountAddressesToIndex = initDelegateAddresses - .concat(nonDelegateAddressesToIndex) - .filter((v, i, a) => a.findIndex(t => (t === v)) === i); // Remove duplicates + const accountAddressesToIndex = initDelegateAddresses.concat(nonDelegateAddressesToIndex); const PAGE_SIZE = 20; const NUM_PAGES = Math.ceil(accountAddressesToIndex.length / PAGE_SIZE); From c97b96773a1fdd69675ad5c240283ca61ce2acbc Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 18:01:06 +0200 Subject: [PATCH 14/33] Mark instantiation to be complete after the signal is dispatched --- services/core/shared/core/compat/common/wsRequest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/common/wsRequest.js b/services/core/shared/core/compat/common/wsRequest.js index b0d6b671e6..deeebce79e 100644 --- a/services/core/shared/core/compat/common/wsRequest.js +++ b/services/core/shared/core/compat/common/wsRequest.js @@ -33,11 +33,11 @@ const instantiateClient = async () => { isInstantiating = true; if (clientCache) await clientCache.disconnect(); clientCache = await createWSClient(`${liskAddress}/ws`); - isInstantiating = false; // Inform listeners about the newly created ApiClient logger.debug(`============== 'newApiClient' signal: ${Signals.get('newApiClient')} ==============`); Signals.get('newApiClient').dispatch(); + isInstantiating = false; } return clientCache; } From e6500f052f0f34f7419463e01a60a1e651a83e07 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 18:49:13 +0200 Subject: [PATCH 15/33] Create arrayUtils --- services/core/shared/arrayUtils.js | 20 +++++++++++++++++++ .../shared/core/compat/sdk_v5/accounts.js | 12 ++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 services/core/shared/arrayUtils.js diff --git a/services/core/shared/arrayUtils.js b/services/core/shared/arrayUtils.js new file mode 100644 index 0000000000..d9c17835fd --- /dev/null +++ b/services/core/shared/arrayUtils.js @@ -0,0 +1,20 @@ +/* + * LiskHQ/lisk-service + * Copyright © 2021 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + * + */ +const dropDuplicates = arr => arr.filter((v, i, a) => a.findIndex(t => (t === v)) === i); + +module.exports = { + dropDuplicates, +}; diff --git a/services/core/shared/core/compat/sdk_v5/accounts.js b/services/core/shared/core/compat/sdk_v5/accounts.js index b2df85a262..2e9eca6f77 100644 --- a/services/core/shared/core/compat/sdk_v5/accounts.js +++ b/services/core/shared/core/compat/sdk_v5/accounts.js @@ -40,6 +40,10 @@ const { initializeQueue, } = require('../../queue'); +const { + dropDuplicates, +} = require('../../../arrayUtils'); + const { parseToJSONCompatObj, } = require('../../../jsonTools'); @@ -109,8 +113,7 @@ const getAccountsFromCore = async (params) => { const indexAccountsbyAddress = async (addressesToIndex, isGenesisBlockAccount = false) => { const { data: accountsToIndex } = await getAccountsFromCore({ - addresses: addressesToIndex - .filter((v, i, a) => a.findIndex(t => (t === v)) === i), // Remove duplicates + addresses: dropDuplicates(addressesToIndex), }); const finalAccountsToIndex = await BluebirdPromise.map( accountsToIndex, @@ -226,9 +229,8 @@ const indexAccountsbyPublicKey = async (accountInfoArray) => { const accountsDB = await getAccountsIndex(); const { data: accountsToIndex } = await getAccountsFromCore({ - addresses: accountInfoArray - .map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey)) - .filter((v, i, a) => a.findIndex(t => (t === v)) === i), // Remove duplicates + addresses: dropDuplicates(accountInfoArray + .map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey))), }); const finalAccountsToIndex = await BluebirdPromise.map( From 3ebb14982c261ec913d741847a4cc85627e317c7 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 18:56:29 +0200 Subject: [PATCH 16/33] Update genesis block download directory --- services/core/.dockerignore | 3 +++ services/core/.gitignore | 4 ++-- services/core/shared/core/compat/sdk_v5/blocksUtils.js | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/services/core/.dockerignore b/services/core/.dockerignore index 64a76cf84a..c6a7d2580d 100644 --- a/services/core/.dockerignore +++ b/services/core/.dockerignore @@ -64,3 +64,6 @@ db_data # Benchmark directory benchmark + +# Data download directory +data diff --git a/services/core/.gitignore b/services/core/.gitignore index d554aca42d..f3afa87985 100644 --- a/services/core/.gitignore +++ b/services/core/.gitignore @@ -61,5 +61,5 @@ xunit-report.xml # Embedded DB directory db_data -# Genesis block download -shared/core/compat/sdk_v5/static +# Data download directory +data diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 9ba002f0df..fe9d8d4741 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -50,7 +50,7 @@ const loadConfig = async () => { const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); genesisBlockURL = networkConfig.genesisBlockUrl; - genesisBlockFilePath = `./shared/core/compat/sdk_v5/static/${networkConfig.name}/genesis_block.json`; + genesisBlockFilePath = `./data/${networkConfig.name}/genesis_block.json`; // If file exists, already create a read stream if (fs.existsSync(genesisBlockFilePath)) readStream = fs.createReadStream(genesisBlockFilePath); From ac9b619f8739af345e28d8c5f0db9d5bbdbb805f Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 19:04:31 +0200 Subject: [PATCH 17/33] Set default genesis height to be '0' --- services/core/shared/core/compat/sdk_v5/coreApi.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/coreApi.js b/services/core/shared/core/compat/sdk_v5/coreApi.js index 39003de115..24fdbea554 100644 --- a/services/core/shared/core/compat/sdk_v5/coreApi.js +++ b/services/core/shared/core/compat/sdk_v5/coreApi.js @@ -51,8 +51,8 @@ const getGenesisHeight = async () => { if (!genesisHeight) { // Determine genesis height const { data: { networkIdentifier } } = await getNetworkStatus(); - const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); - genesisHeight = Number(networkConfig.genesisHeight); + const [networkConfig = {}] = config.network.filter(c => c.identifier === networkIdentifier); + genesisHeight = Number(networkConfig.genesisHeight || 0); } return genesisHeight; }; From 3355fcd2a0058abc1d1e4d39b56aa828882b7242 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 19:13:48 +0200 Subject: [PATCH 18/33] Code refactor --- services/core/shared/core/delegates.js | 52 ++++++++++++++------------ 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/services/core/shared/core/delegates.js b/services/core/shared/core/delegates.js index 5ccde95577..c0c8142d2e 100644 --- a/services/core/shared/core/delegates.js +++ b/services/core/shared/core/delegates.js @@ -250,32 +250,36 @@ const reload = async () => { }; // Keep the delegate cache up-to-date -Signals.get('newBlock').add(async data => { - const dposModuleId = 5; - const voteDelegateAssetId = 1; - const updatedDelegateAddresses = []; - const [block] = data.data; - if (block && block.payload) { - block.payload.forEach(tx => { - if (tx.moduleID === dposModuleId && tx.assetID === voteDelegateAssetId) { - tx.asset.votes.forEach(vote => updatedDelegateAddresses - .push(coreApi.getBase32AddressFromHex(vote.delegateAddress))); - } - }); - const { data: updatedDelegateAccounts } = await coreApi - .getAccounts({ addresses: updatedDelegateAddresses }); - updatedDelegateAccounts.forEach(delegate => { - const delegateIndex = delegateList.findIndex(acc => acc.address === delegate.address); - if (delegateIndex === -1) delegateList.push(delegate); - else delegateList[delegateIndex] = delegate; - }); - // Rank is impacted only when a delegate gets (un-)voted - if (updatedDelegateAddresses.length) await computeDelegateRank(); - } -}); +const updateDelegateListEveryBlock = () => Signals.get('newBlock') + .add(async data => { + const dposModuleId = 5; + const voteDelegateAssetId = 1; + const updatedDelegateAddresses = []; + const [block] = data.data; + if (block && block.payload) { + block.payload.forEach(tx => { + if (tx.moduleID === dposModuleId && tx.assetID === voteDelegateAssetId) { + tx.asset.votes.forEach(vote => updatedDelegateAddresses + .push(coreApi.getBase32AddressFromHex(vote.delegateAddress))); + } + }); + const { data: updatedDelegateAccounts } = await coreApi + .getAccounts({ addresses: updatedDelegateAddresses }); + updatedDelegateAccounts.forEach(delegate => { + const delegateIndex = delegateList.findIndex(acc => acc.address === delegate.address); + if (delegateIndex === -1) delegateList.push(delegate); + else delegateList[delegateIndex] = delegate; + }); + // Rank is impacted only when a delegate gets (un-)voted + if (updatedDelegateAddresses.length) await computeDelegateRank(); + } + }); // Reload the delegate cache when all the indexes are up-to-date -Signals.get('blockIndexReady').add(() => reload()); +const refreshDelegateListOnIndexReady = () => Signals.get('blockIndexReady').add(() => reload()); + +updateDelegateListEveryBlock(); +refreshDelegateListOnIndexReady(); module.exports = { reloadDelegateCache: reload, From d11ef9803cfa39bde7b07968adaf5f7e361e9ce3 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 19:33:19 +0200 Subject: [PATCH 19/33] Code refactor --- services/core/shared/knownAccounts.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/core/shared/knownAccounts.js b/services/core/shared/knownAccounts.js index a95795474c..5cf659dbf6 100644 --- a/services/core/shared/knownAccounts.js +++ b/services/core/shared/knownAccounts.js @@ -36,11 +36,11 @@ const getAccountKnowledge = (address) => { const reloadKnowledge = async () => { logger.debug('Reloading known accounts...'); - await waitForLastBlock(); - const netStatus = await getNetworkStatus(); - const { nethash } = netStatus.data.constants; - try { + await waitForLastBlock(); + const netStatus = await getNetworkStatus(); + const { nethash } = netStatus.data.constants; + const knownNetworks = await HTTP.request(`${staticUrl}/networks.json`); if (knownNetworks.data[nethash]) { const knownAccounts = await HTTP.request(`${staticUrl}/known_${knownNetworks.data[nethash]}.json`); From b74a6257ba8fb9bc8faf50117481bb41736fcb5c Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 20:00:13 +0200 Subject: [PATCH 20/33] Add error handling --- services/core/shared/core/peerCache.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/services/core/shared/core/peerCache.js b/services/core/shared/core/peerCache.js index 6c46dd707f..00f63b2d15 100644 --- a/services/core/shared/core/peerCache.js +++ b/services/core/shared/core/peerCache.js @@ -88,17 +88,21 @@ const getStatistics = () => peerStore.statistics; const reload = async () => { logger.debug('Refreshing peer cache...'); - if (sdkVersion <= 4) { - peerStore.peers = await requestAll(coreApi.getPeers, {}); - } else { - peerStore.peers = (await coreApi.getPeers()).data; + try { + if (sdkVersion <= 4) { + peerStore.peers = await requestAll(coreApi.getPeers, {}); + } else { + peerStore.peers = (await coreApi.getPeers()).data; + } + peerStore.connected = peerStore.peers.filter(o => o.state === peerStates.CONNECTED); + peerStore.disconnected = peerStore.peers.filter(o => o.state === peerStates.DISCONNECTED); + peerStore.statistics = await refreshStatistics(); + logger.debug('Updated peer cache.'); + logger.debug(`============== 'peerReload' signal: ${Signals.get('peerReload')} ==============`); + Signals.get('peerReload').dispatch(peerStore.peers); + } catch (err) { + logger.debug(`Unable to reload peer cache: ${err.message}`); } - peerStore.connected = peerStore.peers.filter(o => o.state === peerStates.CONNECTED); - peerStore.disconnected = peerStore.peers.filter(o => o.state === peerStates.DISCONNECTED); - peerStore.statistics = await refreshStatistics(); - logger.debug('Updated peer cache.'); - logger.debug(`============== 'peerReload' signal: ${Signals.get('peerReload')} ==============`); - Signals.get('peerReload').dispatch(peerStore.peers); }; module.exports = { From ece765de104a36a973ab1ef01eff71d19416edb3 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 20:15:09 +0200 Subject: [PATCH 21/33] Fix networkIdentifier for migrated mainnet --- services/core/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/config.js b/services/core/config.js index 8e665c6ae7..e103a7b728 100644 --- a/services/core/config.js +++ b/services/core/config.js @@ -39,7 +39,7 @@ config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downlo config.network = [ { name: 'mainnet', - identifier: 'ed14889723f24ecc54871d058d98ce91ff2f973192075c0155ba2b7b70ad2511', + identifier: '022097430f51c10da3146920f4c82005900ce5da83bd5d92c04dec2645e54325', genesisHeight: Number(process.env.GENESIS_HEIGHT || 0), genesisBlockUrl: 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz', }, From 5760a553b7b72e012e96432b14ce11a802775338 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 21:15:13 +0200 Subject: [PATCH 22/33] Use redis to cache network constants --- services/core/shared/core/compat/common/constants.js | 7 ++++++- services/core/shared/core/compat/sdk_v5/blocksUtils.js | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/services/core/shared/core/compat/common/constants.js b/services/core/shared/core/compat/common/constants.js index 539b4b6306..49b1eae4dc 100644 --- a/services/core/shared/core/compat/common/constants.js +++ b/services/core/shared/core/compat/common/constants.js @@ -13,7 +13,7 @@ * Removal or modification of this copyright notice is prohibited. * */ -const { Utils } = require('lisk-service-framework'); +const { CacheRedis, Utils } = require('lisk-service-framework'); const http = require('./httpRequest'); const { getApiClient } = require('./wsRequest'); @@ -21,6 +21,10 @@ const { getApiClient } = require('./wsRequest'); const ObjectUtilService = Utils.Data; const { isProperObject } = ObjectUtilService; +const config = require('../../../../config'); + +const constantsCache = CacheRedis('networkConstants', config.endpoints.redis); + let coreVersion = '1.0.0-alpha.0'; let constants; let readyStatus; @@ -82,6 +86,7 @@ const getNetworkConstants = async () => { } if (!isProperObject(result)) return {}; constants = result; + await constantsCache.set('networkConstants', JSON.stringify(constants)); } return constants; } catch (_) { diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index fe9d8d4741..8e408a47e5 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -20,6 +20,7 @@ const path = require('path'); const tar = require('tar'); const { + CacheRedis, Logger, HTTP: { request }, } = require('lisk-service-framework'); @@ -34,6 +35,8 @@ let genesisBlockURL; let genesisBlockFilePath; let genesisBlock = { header: {} }; +const constantsCache = CacheRedis('networkConstants', config.endpoints.redis); + const parseStream = json.createParseStream(); const setGenesisBlock = (block) => genesisBlock = block; @@ -43,14 +46,14 @@ const getGenesisBlock = () => genesisBlock; const getGenesisBlockId = () => genesisBlock.header.id; const loadConfig = async () => { - // Direct invocation of action necessary to avoid circular dependency - const apiClient = await getApiClient(); - const { networkIdentifier } = await apiClient.node.getNodeInfo(); + const { data: { networkIdentifier } } = JSON.parse(await constantsCache.get('networkConstants')); const [networkConfig] = config.network.filter(c => c.identifier === networkIdentifier); genesisBlockURL = networkConfig.genesisBlockUrl; + logger.debug(`genesisBlockURL set to ${genesisBlockURL}`); genesisBlockFilePath = `./data/${networkConfig.name}/genesis_block.json`; + logger.debug(`genesisBlockFilePath set to ${genesisBlockFilePath}`); // If file exists, already create a read stream if (fs.existsSync(genesisBlockFilePath)) readStream = fs.createReadStream(genesisBlockFilePath); From 3f2159ced5f4e4dac97398375e4dab29eccd872c Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Mon, 19 Jul 2021 21:24:38 +0200 Subject: [PATCH 23/33] Fix linting errors --- services/core/shared/core/compat/sdk_v5/blocksUtils.js | 1 - 1 file changed, 1 deletion(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 8e408a47e5..451e2df0e5 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -25,7 +25,6 @@ const { HTTP: { request }, } = require('lisk-service-framework'); -const { getApiClient } = require('../common/wsRequest'); const config = require('../../../../config'); const logger = Logger(); From ebb09c0eeca3bb776401eafc013726cb67f0f446 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 09:27:05 +0200 Subject: [PATCH 24/33] Genesis file download error handling --- .../core/shared/core/compat/sdk_v5/blocksUtils.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocksUtils.js b/services/core/shared/core/compat/sdk_v5/blocksUtils.js index 451e2df0e5..461404ec70 100644 --- a/services/core/shared/core/compat/sdk_v5/blocksUtils.js +++ b/services/core/shared/core/compat/sdk_v5/blocksUtils.js @@ -23,6 +23,7 @@ const { CacheRedis, Logger, HTTP: { request }, + Exceptions: { NotFoundException }, } = require('lisk-service-framework'); const config = require('../../../../config'); @@ -67,9 +68,16 @@ const downloadGenesisBlock = async () => { return new Promise((resolve, reject) => { if (genesisBlockURL.endsWith('.tar.gz')) { https.get(genesisBlockURL, (response) => { - response.pipe(tar.extract({ cwd: directoryPath })); - response.on('error', async (err) => reject(err)); - response.on('end', async () => setTimeout(resolve, 500)); + if (response.statusCode === 200) { + response.pipe(tar.extract({ cwd: directoryPath })); + response.on('error', async (err) => reject(err)); + response.on('end', async () => setTimeout(resolve, 500)); + } else { + const errMessage = `Download failed with HTTP status code: ${response.statusCode} (${response.statusMessage})`; + logger.error(errMessage); + if (response.statusCode === 404) throw new NotFoundException(errMessage); + throw new Error(errMessage); + } }); } else { request(genesisBlockURL) From c8095cb82c01736cc8fab98fb6d81fb9086c6c6c Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 10:13:37 +0200 Subject: [PATCH 25/33] Apply suggestions --- services/core/config.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/services/core/config.js b/services/core/config.js index e103a7b728..9a1256933a 100644 --- a/services/core/config.js +++ b/services/core/config.js @@ -34,20 +34,19 @@ config.endpoints.redis = process.env.SERVICE_CORE_REDIS || 'redis://localhost:63 config.endpoints.liskStatic = process.env.LISK_STATIC || 'https://static-data.lisk.io'; config.endpoints.geoip = process.env.GEOIP_JSON || 'https://geoip.lisk.io/json'; config.endpoints.mysql = process.env.SERVICE_CORE_MYSQL || 'mysql://lisk:password@localhost:3306/lisk'; -config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz'; config.network = [ { name: 'mainnet', identifier: '022097430f51c10da3146920f4c82005900ce5da83bd5d92c04dec2645e54325', genesisHeight: Number(process.env.GENESIS_HEIGHT || 0), - genesisBlockUrl: 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz', + genesisBlockUrl: process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz', }, { name: 'testnet', identifier: '15f0dacc1060e91818224a94286b13aa04279c640bd5d6f193182031d133df7c', - genesisHeight: 14075260, - genesisBlockUrl: 'https://downloads.lisk.io/lisk/testnet/genesis_block.json.tar.gz', + genesisHeight: Number(process.env.GENESIS_HEIGHT || 14075260), + genesisBlockUrl: process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/testnet/genesis_block.json.tar.gz', }, ]; From 36f5e43b10a2f38f859f8225d20fd3e4f90bda3b Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 10:22:10 +0200 Subject: [PATCH 26/33] Instantiation ends soon after the client is created --- services/core/shared/core/compat/common/wsRequest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/common/wsRequest.js b/services/core/shared/core/compat/common/wsRequest.js index deeebce79e..b0d6b671e6 100644 --- a/services/core/shared/core/compat/common/wsRequest.js +++ b/services/core/shared/core/compat/common/wsRequest.js @@ -33,11 +33,11 @@ const instantiateClient = async () => { isInstantiating = true; if (clientCache) await clientCache.disconnect(); clientCache = await createWSClient(`${liskAddress}/ws`); + isInstantiating = false; // Inform listeners about the newly created ApiClient logger.debug(`============== 'newApiClient' signal: ${Signals.get('newApiClient')} ==============`); Signals.get('newApiClient').dispatch(); - isInstantiating = false; } return clientCache; } From 84a372d0d782149df38e38a5ed1fbed8008cbd7e Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 11:05:54 +0200 Subject: [PATCH 27/33] Do not re-index genesis block, if already indexed --- .../core/shared/core/compat/sdk_v5/blocks.js | 59 +++++++++++-------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index e444abd3f9..9975fe1809 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -366,33 +366,40 @@ const deleteBlock = async (block) => { }; const indexGenesisBlock = async () => { - logger.info(`Ìndexing genesis block at height ${genesisHeight}`); - const [genesisBlock] = await getBlockByHeight(genesisHeight, false); - - // Index the genesis block transactions first - await indexTransactions([genesisBlock]); - - // Index the genesis block accounts next - const initDelegateAddresses = genesisBlock.asset.initDelegates; - const nonDelegateAddressesToIndex = genesisBlock.asset.accounts - .filter(account => account.address.length > 16) // Filter out reclaim accounts - .map(account => account.address); - - const accountAddressesToIndex = initDelegateAddresses.concat(nonDelegateAddressesToIndex); - - const PAGE_SIZE = 20; - const NUM_PAGES = Math.ceil(accountAddressesToIndex.length / PAGE_SIZE); - for (let i = 0; i < NUM_PAGES; i++) { - // eslint-disable-next-line no-await-in-loop - await indexAccountsbyAddress( - accountAddressesToIndex.slice(i * PAGE_SIZE, (i + 1) * PAGE_SIZE), - true, - ); - } + const blocksDB = await getBlocksIndex(); + const [indexedGenesisBlock] = await blocksDB.find({ height: genesisHeight }); + + if (indexedGenesisBlock !== undefined) { + logger.info(`Genesis block already indexed at height ${genesisHeight}`); + } else { + logger.info(`Ìndexing genesis block at height ${genesisHeight}`); + const [genesisBlock] = await getBlockByHeight(genesisHeight, false); + + // Index the genesis block transactions first + await indexTransactions([genesisBlock]); + + // Index the genesis block accounts next + const initDelegateAddresses = genesisBlock.asset.initDelegates; + const nonDelegateAddressesToIndex = genesisBlock.asset.accounts + .filter(account => account.address.length > 16) // Filter out reclaim accounts + .map(account => account.address); + + const accountAddressesToIndex = initDelegateAddresses.concat(nonDelegateAddressesToIndex); + + const PAGE_SIZE = 20; + const NUM_PAGES = Math.ceil(accountAddressesToIndex.length / PAGE_SIZE); + for (let i = 0; i < NUM_PAGES; i++) { + // eslint-disable-next-line no-await-in-loop + await indexAccountsbyAddress( + accountAddressesToIndex.slice(i * PAGE_SIZE, (i + 1) * PAGE_SIZE), + true, + ); + } - // Finally index the genesis block itself - await indexBlocksQueue.add('indexBlocksQueue', { blocks: [genesisBlock] }); - logger.info('Finished indexing the genesis block'); + // Finally index the genesis block itself + await indexBlocksQueue.add('indexBlocksQueue', { blocks: [genesisBlock] }); + logger.info('Finished indexing the genesis block'); + } }; const buildIndex = async (from, to) => { From c0b6eb471f2d3c54bb30b30f3da98dcd928010f4 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 12:00:21 +0200 Subject: [PATCH 28/33] Apply suggestion Co-authored-by: Himanshu Nagda <69150921+nagdahimanshu@users.noreply.github.com> --- services/core/shared/core/compat/sdk_v5/blocks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index 9975fe1809..e797970c5b 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -369,7 +369,7 @@ const indexGenesisBlock = async () => { const blocksDB = await getBlocksIndex(); const [indexedGenesisBlock] = await blocksDB.find({ height: genesisHeight }); - if (indexedGenesisBlock !== undefined) { + if (indexedGenesisBlock) { logger.info(`Genesis block already indexed at height ${genesisHeight}`); } else { logger.info(`Ìndexing genesis block at height ${genesisHeight}`); From 6a5b89171009782394913338aaae8d6aed5a4419 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 13:04:10 +0200 Subject: [PATCH 29/33] Error handling for 'checkIndexReadiness' --- .../core/shared/core/compat/sdk_v5/blocks.js | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index e797970c5b..ee5f5fb467 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -513,24 +513,28 @@ const indexPastBlocks = async () => { const checkIndexReadiness = async () => { logger.debug('============== Checking blocks index ready status =============='); if (!getIndexReadyStatus()) { - const blocksDB = await getBlocksIndex(); - const currentChainHeight = (await coreApi.getNetworkStatus()).data.height; - const numBlocksIndexed = await blocksDB.count(); - const [lastIndexedBlock] = await blocksDB.find({ sort: 'height:desc', limit: 1 }); - - logger.debug( - `\nnumBlocksIndexed: ${numBlocksIndexed}`, - `\nlastIndexedBlock: ${lastIndexedBlock.height}`, - `\ncurrentChainHeight: ${currentChainHeight}`, - ); - if (numBlocksIndexed >= currentChainHeight - genesisHeight - && lastIndexedBlock.height >= currentChainHeight - 1) { - setIndexReadyStatus(true); - logger.info('Blocks index is now ready'); - logger.debug(`============== 'blockIndexReady' signal: ${Signals.get('blockIndexReady')} ==============`); - Signals.get('blockIndexReady').dispatch(true); - } else { - logger.debug('Blocks index is not yet ready'); + try { + const blocksDB = await getBlocksIndex(); + const currentChainHeight = (await coreApi.getNetworkStatus()).data.height; + const numBlocksIndexed = await blocksDB.count(); + const [lastIndexedBlock] = await blocksDB.find({ sort: 'height:desc', limit: 1 }); + + logger.debug( + `\nnumBlocksIndexed: ${numBlocksIndexed}`, + `\nlastIndexedBlock: ${lastIndexedBlock.height}`, + `\ncurrentChainHeight: ${currentChainHeight}`, + ); + if (numBlocksIndexed >= currentChainHeight - genesisHeight + && lastIndexedBlock.height >= currentChainHeight - 1) { + setIndexReadyStatus(true); + logger.info('Blocks index is now ready'); + logger.debug(`============== 'blockIndexReady' signal: ${Signals.get('blockIndexReady')} ==============`); + Signals.get('blockIndexReady').dispatch(true); + } else { + logger.debug('Blocks index is not yet ready'); + } + } catch (err) { + logger.warn(`Error at checkIndexReadiness: ${err.message}`); } } return getIndexReadyStatus(); From 6b5eae7a1dc0e8480b6507bc9b2269c8a3216c7e Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 13:07:38 +0200 Subject: [PATCH 30/33] Update log --- services/core/shared/core/compat/sdk_v5/blocks.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index ee5f5fb467..3dc1a55cd5 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -534,7 +534,7 @@ const checkIndexReadiness = async () => { logger.debug('Blocks index is not yet ready'); } } catch (err) { - logger.warn(`Error at checkIndexReadiness: ${err.message}`); + logger.warn(`Error while checking index readiness: ${err.message}`); } } return getIndexReadyStatus(); From fc539cacee4f1b610248bb783a298d43b31e85fd Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 13:51:07 +0200 Subject: [PATCH 31/33] Use networkConstants cache when indexing past blocks --- services/core/shared/core/compat/sdk_v5/blocks.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index 3dc1a55cd5..214f41c3cb 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -16,6 +16,7 @@ const BluebirdPromise = require('bluebird'); const util = require('util'); const { + CacheRedis, Logger, Signals, Exceptions: { ValidationException, NotFoundException }, @@ -54,6 +55,8 @@ const blocksIndexSchema = require('./schema/blocks'); const getBlocksIndex = () => mysqlIndex('blocks', blocksIndexSchema); +const constantsCache = CacheRedis('networkConstants', config.endpoints.redis); + const logger = Logger(); let genesisHeight; @@ -492,7 +495,7 @@ const indexPastBlocks = async () => { if (config.indexNumOfBlocks === 0) setIsSyncFullBlockchain(true); // Lowest and highest block heights expected to be indexed - const blockIndexHigherRange = (await coreApi.getNetworkStatus()).data.height; + const blockIndexHigherRange = JSON.parse(await constantsCache.get('networkConstants')).data.height; const blockIndexLowerRange = config.indexNumOfBlocks > 0 ? blockIndexHigherRange - config.indexNumOfBlocks : genesisHeight; From 0ef77cef372ea3969295a94f5c27370ba39de0c0 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 15:07:58 +0200 Subject: [PATCH 32/33] Use cache from compat/common --- services/core/shared/core/compat/sdk_v5/blocks.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index 214f41c3cb..b9957a5d28 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -16,7 +16,6 @@ const BluebirdPromise = require('bluebird'); const util = require('util'); const { - CacheRedis, Logger, Signals, Exceptions: { ValidationException, NotFoundException }, @@ -42,6 +41,7 @@ const { const { getApiClient, + getNetworkConstants, getIndexReadyStatus, setIndexReadyStatus, setIsSyncFullBlockchain, @@ -55,8 +55,6 @@ const blocksIndexSchema = require('./schema/blocks'); const getBlocksIndex = () => mysqlIndex('blocks', blocksIndexSchema); -const constantsCache = CacheRedis('networkConstants', config.endpoints.redis); - const logger = Logger(); let genesisHeight; @@ -495,7 +493,7 @@ const indexPastBlocks = async () => { if (config.indexNumOfBlocks === 0) setIsSyncFullBlockchain(true); // Lowest and highest block heights expected to be indexed - const blockIndexHigherRange = JSON.parse(await constantsCache.get('networkConstants')).data.height; + const blockIndexHigherRange = (await getNetworkConstants()).data.height; const blockIndexLowerRange = config.indexNumOfBlocks > 0 ? blockIndexHigherRange - config.indexNumOfBlocks : genesisHeight; From ed464f8422f899cfbd3097afd42a9f86e9d94cb3 Mon Sep 17 00:00:00 2001 From: Sameer Kumar Subudhi <sameer.subudhi@lightcurve.io> Date: Tue, 20 Jul 2021 15:20:40 +0200 Subject: [PATCH 33/33] Revert to invoking networkStatus from Core --- services/core/shared/core/compat/sdk_v5/blocks.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/core/shared/core/compat/sdk_v5/blocks.js b/services/core/shared/core/compat/sdk_v5/blocks.js index b9957a5d28..ecd6024c03 100644 --- a/services/core/shared/core/compat/sdk_v5/blocks.js +++ b/services/core/shared/core/compat/sdk_v5/blocks.js @@ -41,7 +41,6 @@ const { const { getApiClient, - getNetworkConstants, getIndexReadyStatus, setIndexReadyStatus, setIsSyncFullBlockchain, @@ -487,13 +486,13 @@ const indexMissingBlocks = async (startHeight, endHeight) => { }; const indexPastBlocks = async () => { - logger.info('Building index of blocks'); + logger.info('Building the blocks index'); const blocksDB = await getBlocksIndex(); if (config.indexNumOfBlocks === 0) setIsSyncFullBlockchain(true); // Lowest and highest block heights expected to be indexed - const blockIndexHigherRange = (await getNetworkConstants()).data.height; + const blockIndexHigherRange = (await coreApi.getNetworkStatus()).data.height; const blockIndexLowerRange = config.indexNumOfBlocks > 0 ? blockIndexHigherRange - config.indexNumOfBlocks : genesisHeight; @@ -509,6 +508,7 @@ const indexPastBlocks = async () => { // Start building the block index await buildIndex(highestIndexedHeight, blockIndexHigherRange); await indexMissingBlocks(blockIndexLowerRange, blockIndexHigherRange); + logger.info('Finished building the blocks index'); }; const checkIndexReadiness = async () => {