Merge pull request #641 from LiskHQ/640-update-genesis-block-retrieva…

…l-configuation Update genesis block retrieval configuration - Closes #640
LiskArchive · Jul 20, 2021 · 56d0a74 · 56d0a74
2 parents 02b32ea + ed464f8
commit 56d0a74
Show file tree

Hide file tree

Showing 14 changed files with 340 additions and 127 deletions.
diff --git a/services/core/.dockerignore b/services/core/.dockerignore
@@ -64,3 +64,6 @@ db_data
 
 # Benchmark directory
 benchmark
+
+# Data download directory
+data
diff --git a/services/core/.gitignore b/services/core/.gitignore
@@ -61,5 +61,5 @@ xunit-report.xml
 # Embedded DB directory
 db_data
 
-# Genesis block download
-shared/core/compat/sdk_v5/static
+# Data download directory
+data
diff --git a/services/core/config.js b/services/core/config.js
@@ -34,7 +34,21 @@ config.endpoints.redis = process.env.SERVICE_CORE_REDIS || 'redis://localhost:63
 config.endpoints.liskStatic = process.env.LISK_STATIC || 'https://static-data.lisk.io';
 config.endpoints.geoip = process.env.GEOIP_JSON || 'https://geoip.lisk.io/json';
 config.endpoints.mysql = process.env.SERVICE_CORE_MYSQL || 'mysql://lisk:password@localhost:3306/lisk';
-config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz';
+
+config.network = [
+	{
+		name: 'mainnet',
+		identifier: '022097430f51c10da3146920f4c82005900ce5da83bd5d92c04dec2645e54325',
+		genesisHeight: Number(process.env.GENESIS_HEIGHT || 0),
+		genesisBlockUrl: process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/mainnet/genesis_block.json.tar.gz',
+	},
+	{
+		name: 'testnet',
+		identifier: '15f0dacc1060e91818224a94286b13aa04279c640bd5d6f193182031d133df7c',
+		genesisHeight: Number(process.env.GENESIS_HEIGHT || 14075260),
+		genesisBlockUrl: process.env.GENESIS_BLOCK_URL || 'https://downloads.lisk.io/lisk/testnet/genesis_block.json.tar.gz',
+	},
+];
 
 /**
  * Indexing
@@ -48,7 +62,6 @@ config.endpoints.genesisBlock = process.env.GENESIS_BLOCK_URL || 'https://downlo
  * such as transactions, accounts, votes etc.
  */
 config.indexNumOfBlocks = Number(process.env.INDEX_N_BLOCKS || 202);
-config.genesisHeight = Number(process.env.GENESIS_HEIGHT || 0);
 
 config.transactionStatistics = {
 	enabled: Boolean(String(process.env.ENABLE_TRANSACTION_STATS).toLowerCase() === 'true'),

diff --git a/services/core/package-lock.json b/services/core/package-lock.json
diff --git a/services/core/package.json b/services/core/package.json
@@ -34,6 +34,7 @@
     "@liskhq/lisk-transactions-v4": "npm:@liskhq/lisk-transactions@^4.0.0",
     "@liskhq/lisk-transactions-v5": "npm:@liskhq/lisk-transactions@^5.0.3",
     "async": "^3.2.0",
+    "big-json": "^3.1.0",
     "big-number": "=2.0.0",
     "bluebird": "^3.7.2",
     "body-parser": "^1.19.0",

diff --git a/services/core/shared/arrayUtils.js b/services/core/shared/arrayUtils.js
@@ -0,0 +1,20 @@
+/*
+ * LiskHQ/lisk-service
+ * Copyright © 2021 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ *
+ */
+const dropDuplicates = arr => arr.filter((v, i, a) => a.findIndex(t => (t === v)) === i);
+
+module.exports = {
+	dropDuplicates,
+};
diff --git a/services/core/shared/core/compat/common/constants.js b/services/core/shared/core/compat/common/constants.js
@@ -13,14 +13,18 @@
  * Removal or modification of this copyright notice is prohibited.
  *
  */
-const { Utils } = require('lisk-service-framework');
+const { CacheRedis, Utils } = require('lisk-service-framework');
 
 const http = require('./httpRequest');
 const { getApiClient } = require('./wsRequest');
 
 const ObjectUtilService = Utils.Data;
 const { isProperObject } = ObjectUtilService;
 
+const config = require('../../../../config');
+
+const constantsCache = CacheRedis('networkConstants', config.endpoints.redis);
+
 let coreVersion = '1.0.0-alpha.0';
 let constants;
 let readyStatus;
@@ -82,6 +86,7 @@ const getNetworkConstants = async () => {
 			}
 			if (!isProperObject(result)) return {};
 			constants = result;
+			await constantsCache.set('networkConstants', JSON.stringify(constants));
 		}
 		return constants;
 	} catch (_) {

diff --git a/services/core/shared/core/compat/sdk_v5/accounts.js b/services/core/shared/core/compat/sdk_v5/accounts.js
@@ -40,6 +40,10 @@ const {
 	initializeQueue,
 } = require('../../queue');
 
+const {
+	dropDuplicates,
+} = require('../../../arrayUtils');
+
 const {
 	parseToJSONCompatObj,
 } = require('../../../jsonTools');
@@ -108,7 +112,9 @@ const getAccountsFromCore = async (params) => {
 };
 
 const indexAccountsbyAddress = async (addressesToIndex, isGenesisBlockAccount = false) => {
-	const { data: accountsToIndex } = await getAccountsFromCore({ addresses: addressesToIndex });
+	const { data: accountsToIndex } = await getAccountsFromCore({
+		addresses: dropDuplicates(addressesToIndex),
+	});
 	const finalAccountsToIndex = await BluebirdPromise.map(
 		accountsToIndex,
 		async account => {
@@ -223,8 +229,8 @@ const indexAccountsbyPublicKey = async (accountInfoArray) => {
 	const accountsDB = await getAccountsIndex();
 
 	const { data: accountsToIndex } = await getAccountsFromCore({
-		addresses: accountInfoArray
-			.map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey)),
+		addresses: dropDuplicates(accountInfoArray
+			.map(accountInfo => getHexAddressFromPublicKey(accountInfo.publicKey))),
 	});
 
 	const finalAccountsToIndex = await BluebirdPromise.map(