From 099df0b46263925bf2eda2b2f3bb1d6efd2b6e0d Mon Sep 17 00:00:00 2001 From: Sonia Singla Date: Mon, 24 Jun 2024 14:39:21 +0200 Subject: [PATCH 01/51] Add LICENSE (#41) --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..b7a2b293 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Shardeum + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From b10cdd16a69644b5b580433d79e97991bfd3b5fc Mon Sep 17 00:00:00 2001 From: jairajdev Date: Wed, 19 Jun 2024 15:28:40 +0545 Subject: [PATCH 02/51] Added poq receipt signature verification --- src/Data/Collector.ts | 64 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 760bcdbc..c3ce97d3 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -389,12 +389,11 @@ export const verifyReceiptData = async ( receipt: Receipt.ArchiverReceipt, checkReceiptRobust = true ): Promise<{ success: boolean; newReceipt?: Receipt.ArchiverReceipt }> => { - if (config.newPOQReceipt === false) return { success: true } const result = { success: false } // Check the signed nodes are part of the execution group nodes of the tx const { executionShardKey, cycle, appliedReceipt, globalModification } = receipt if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true } - const { appliedVote, confirmOrChallenge } = appliedReceipt + const { appliedVote, signatures } = appliedReceipt const cycleShardData = shardValuesByCycle.get(cycle) if (!cycleShardData) { Logger.mainLogger.error('Cycle shard data not found') @@ -403,6 +402,67 @@ export const verifyReceiptData = async ( } // Determine the home partition index of the primary account (executionShardKey) const { homePartition } = ShardFunction.addressToPartition(cycleShardData.shardGlobals, executionShardKey) + if (config.newPOQReceipt === false) { + // Refer to https://github.com/shardeum/shardus-core/blob/f7000c36faa0cd1e0832aa1e5e3b1414d32dcf66/src/state-manager/TransactionConsensus.ts#L1406 + const requiredSignatures = Math.round(nodesPerConsensusGroup * (2 / 3.0)) + if (signatures.length < requiredSignatures) { + Logger.mainLogger.error( + 'Invalid receipt appliedReceipt signatures count is less than requiredSignatures' + ) + if (nestedCountersInstance) + nestedCountersInstance.countEvent( + 'receipt', + 'Invalid_receipt_appliedReceipt_signatures_count_less_than_requiredSignatures' + ) + return result + } + // Using a map to store the good signatures to avoid duplicates + const goodSignatures = new Map() + for (const signature of signatures) { + const { owner: nodePubKey } = signature + // Get the node id from the public key + const node = cycleShardData.nodes.find((node) => node.publicKey === nodePubKey) + if (node == null) { + Logger.mainLogger.error( + `The node with public key ${nodePubKey} of the receipt ${receipt.tx.txId}} is not in the active nodesList of cycle ${cycle}` + ) + if (nestedCountersInstance) + nestedCountersInstance.countEvent( + 'receipt', + 'appliedReceipt_signature_owner_not_in_active_nodesList' + ) + continue + } + // Check if the node is in the execution group + if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) { + Logger.mainLogger.error( + `The node with public key ${nodePubKey} of the receipt ${receipt.tx.txId}} is not in the execution group of the tx` + ) + if (nestedCountersInstance) + nestedCountersInstance.countEvent( + 'receipt', + 'IappliedReceipt_signature_node_not_in_execution_group_of_tx' + ) + continue + } + if (Crypto.verify({ appliedVote, node_id: node.id, sign: signature })) { + goodSignatures.set(signature.owner, signature) + } + } + if (goodSignatures.size < requiredSignatures) { + Logger.mainLogger.error( + 'Invalid receipt appliedReceipt valid signatures count is less than requiredSignatures' + ) + if (nestedCountersInstance) + nestedCountersInstance.countEvent( + 'receipt', + 'Invalid_receipt_appliedReceipt_valid_signatures_count_less_than_requiredSignatures' + ) + return result + } + return { success: true } + } + const { confirmOrChallenge } = appliedReceipt // Check if the appliedVote node is in the execution group if (!cycleShardData.nodeShardDataMap.has(appliedVote.node_id)) { Logger.mainLogger.error('Invalid receipt appliedReceipt appliedVote node is not in the active nodesList') From bdae355303e86410f9911d30b0daa5beb8b07a37 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 20 Jun 2024 15:13:54 +0545 Subject: [PATCH 03/51] Added correct appliedVote signature verification --- src/Data/Collector.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index c3ce97d3..213e96ae 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -394,6 +394,7 @@ export const verifyReceiptData = async ( const { executionShardKey, cycle, appliedReceipt, globalModification } = receipt if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true } const { appliedVote, signatures } = appliedReceipt + const txId = receipt.tx.txId const cycleShardData = shardValuesByCycle.get(cycle) if (!cycleShardData) { Logger.mainLogger.error('Cycle shard data not found') @@ -416,6 +417,13 @@ export const verifyReceiptData = async ( ) return result } + // Refer to https://github.com/shardeum/shardus-core/blob/50b6d00f53a35996cd69210ea817bee068a893d6/src/state-manager/TransactionConsensus.ts#L2799 + const voteHash = calculateVoteHash(appliedVote) + // Refer to https://github.com/shardeum/shardus-core/blob/50b6d00f53a35996cd69210ea817bee068a893d6/src/state-manager/TransactionConsensus.ts#L2663 + const appliedVoteHash = { + txid: txId, + voteHash, + } // Using a map to store the good signatures to avoid duplicates const goodSignatures = new Map() for (const signature of signatures) { @@ -424,7 +432,7 @@ export const verifyReceiptData = async ( const node = cycleShardData.nodes.find((node) => node.publicKey === nodePubKey) if (node == null) { Logger.mainLogger.error( - `The node with public key ${nodePubKey} of the receipt ${receipt.tx.txId}} is not in the active nodesList of cycle ${cycle}` + `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the active nodesList of cycle ${cycle}` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( @@ -436,17 +444,19 @@ export const verifyReceiptData = async ( // Check if the node is in the execution group if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) { Logger.mainLogger.error( - `The node with public key ${nodePubKey} of the receipt ${receipt.tx.txId}} is not in the execution group of the tx` + `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the execution group of the tx` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( 'receipt', - 'IappliedReceipt_signature_node_not_in_execution_group_of_tx' + 'appliedReceipt_signature_node_not_in_execution_group_of_tx' ) continue } - if (Crypto.verify({ appliedVote, node_id: node.id, sign: signature })) { + if (Crypto.verify({ ...appliedVoteHash, sign: signature })) { goodSignatures.set(signature.owner, signature) + // Break the loop if the required number of good signatures are found + if (goodSignatures.size >= requiredSignatures) break } } if (goodSignatures.size < requiredSignatures) { @@ -570,6 +580,10 @@ export const verifyReceiptData = async ( return { success: true } } +const calculateVoteHash = (vote: Receipt.AppliedVote): string => { + return Crypto.hashObj({ ...vote, node_id: '' }) +} + export const storeReceiptData = async ( receipts: Receipt.ArchiverReceipt[], senderInfo = '', From 882148dc12da845a375d946966d7b46a47be5120 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 20 Jun 2024 12:45:54 +0545 Subject: [PATCH 04/51] Added otherArchivers list --- src/API.ts | 5 +---- src/Data/Collector.ts | 36 ++++++++---------------------------- src/Data/GossipData.ts | 37 +++++++++++++++++++++++++++++++++++++ src/GlobalAccount.ts | 7 ++----- src/State.ts | 12 ++++++++---- 5 files changed, 56 insertions(+), 41 deletions(-) diff --git a/src/API.ts b/src/API.ts index 202f7997..5034ea9b 100644 --- a/src/API.ts +++ b/src/API.ts @@ -1201,11 +1201,8 @@ export const queryFromArchivers = async ( url = `/totalData` break } - const filteredArchivers = State.activeArchivers.filter( - (archiver) => archiver.publicKey !== config.ARCHIVER_PUBLIC_KEY - ) const maxNumberofArchiversToRetry = 3 - const randomArchivers = Utils.getRandomItemFromArr(filteredArchivers, 0, maxNumberofArchiversToRetry) + const randomArchivers = Utils.getRandomItemFromArr(State.otherArchivers, 0, maxNumberofArchiversToRetry) let retry = 0 while (retry < maxNumberofArchiversToRetry) { // eslint-disable-next-line security/detect-object-injection diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 213e96ae..380ff446 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -19,7 +19,14 @@ import { getCurrentCycleCounter, shardValuesByCycle, computeCycleMarker } from ' import { bulkInsertCycles, queryCycleByMarker, updateCycle } from '../dbstore/cycles' import * as State from '../State' import * as Utils from '../Utils' -import { DataType, GossipData, adjacentArchivers, sendDataToAdjacentArchivers, TxData } from './GossipData' +import { + DataType, + GossipData, + adjacentArchivers, + sendDataToAdjacentArchivers, + TxData, + getArchiversToUse, +} from './GossipData' import { postJson } from '../P2P' import { globalAccountsMap, setGlobalNetworkAccount } from '../GlobalAccount' import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Data/DataLogWriter' @@ -1180,33 +1187,6 @@ export const collectMissingReceipts = async (): Promise => { } } -export const getArchiversToUse = (): State.ArchiverNodeInfo[] => { - let archiversToUse: State.ArchiverNodeInfo[] = [] - const MAX_ARCHIVERS_TO_SELECT = 3 - // Choosing MAX_ARCHIVERS_TO_SELECT random archivers from the active archivers list - if (State.activeArchivers.length <= MAX_ARCHIVERS_TO_SELECT) { - State.activeArchivers.forEach( - (archiver) => archiver.publicKey !== State.getNodeInfo().publicKey && archiversToUse.push(archiver) - ) - } else { - // Filter out the adjacent archivers and self archiver from the active archivers list - const activeArchivers = [...State.activeArchivers].filter( - (archiver) => - adjacentArchivers.has(archiver.publicKey) || archiver.publicKey === State.getNodeInfo().publicKey - ) - archiversToUse = Utils.getRandomItemFromArr(activeArchivers, 0, MAX_ARCHIVERS_TO_SELECT) - if (archiversToUse.length < MAX_ARCHIVERS_TO_SELECT) { - const requiredArchivers = MAX_ARCHIVERS_TO_SELECT - archiversToUse.length - // If the required archivers are not selected, then get it from the adjacent archivers - archiversToUse = [ - ...archiversToUse, - ...Utils.getRandomItemFromArr([...adjacentArchivers.values()], requiredArchivers), - ] - } - } - return archiversToUse -} - type TxDataFromArchiversResponse = { receipts?: Receipt.Receipt[] originalTxs?: OriginalTxDB.OriginalTxData[] diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index 461dd0a7..a6800e02 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -4,6 +4,8 @@ import * as Crypto from '../Crypto' import { postJson } from '../P2P' import { Signature } from '@shardus/crypto-utils' import { P2P as P2PTypes } from '@shardus/types' +import { getRandomActiveNodes } from '../NodeList' +import * as Utils from '../Utils' // adjacentArchivers are one archiver from left and one archiver from right of the current archiver export let adjacentArchivers: Map = new Map() @@ -26,6 +28,12 @@ export interface GossipData { // For debugging purpose, set this to true to stop gossiping tx data const stopGossipTxData = false +const gossipToRandomArchivers = true // To gossip to random archivers in addition to adjacent archivers +const randomGossipArchiverCount = 2 // Number of random archivers to gossip to + +// List of archivers that are not adjacent to the current archiver +const remainingArchivers = [] + export const getAdjacentLeftAndRightArchivers = (): void => { if (State.activeArchivers.length <= 1) { adjacentArchivers = new Map() @@ -53,6 +61,12 @@ export const getAdjacentLeftAndRightArchivers = (): void => { adjacentArchivers = new Map() if (leftArchiver) adjacentArchivers.set(leftArchiver.publicKey, leftArchiver) if (rightArchiver) adjacentArchivers.set(rightArchiver.publicKey, rightArchiver) + remainingArchivers.length = 0 + for (const archiver of State.activeArchivers) { + if (!adjacentArchivers.has(archiver.publicKey) || archiver.publicKey !== State.getNodeInfo().publicKey) { + remainingArchivers.push(archiver) + } + } } export async function sendDataToAdjacentArchivers( @@ -87,6 +101,9 @@ export async function sendDataToAdjacentArchivers( Logger.mainLogger.error('Error', e) } } + if (gossipToRandomArchivers) { + const randomArchivers = getRandomActiveNodes + } try { await Promise.all(promises) } catch (err) { @@ -97,3 +114,23 @@ export async function sendDataToAdjacentArchivers( Logger.mainLogger.debug('Fail to gossip') } } + +export const getArchiversToUse = (): State.ArchiverNodeInfo[] => { + let archiversToUse: State.ArchiverNodeInfo[] = [] + const MAX_ARCHIVERS_TO_SELECT = 3 + // Choosing MAX_ARCHIVERS_TO_SELECT random archivers from the remaining archivers list + if (State.otherArchivers.length <= MAX_ARCHIVERS_TO_SELECT) { + archiversToUse = [...State.otherArchivers] + } else { + archiversToUse = Utils.getRandomItemFromArr(remainingArchivers, 0, MAX_ARCHIVERS_TO_SELECT) + if (archiversToUse.length < MAX_ARCHIVERS_TO_SELECT) { + const requiredArchivers = MAX_ARCHIVERS_TO_SELECT - archiversToUse.length + // If the required archivers are not selected, then get it from the adjacent archivers + archiversToUse = [ + ...archiversToUse, + ...Utils.getRandomItemFromArr([...adjacentArchivers.values()], requiredArchivers), + ] + } + } + return archiversToUse +} diff --git a/src/GlobalAccount.ts b/src/GlobalAccount.ts index 376fc8da..f722fe04 100644 --- a/src/GlobalAccount.ts +++ b/src/GlobalAccount.ts @@ -48,9 +48,6 @@ export const loadGlobalAccounts = async (): Promise => { } export const syncGlobalAccount = async (retry = 5): Promise => { - const filteredArchivers = State.activeArchivers.filter( - (archiver) => archiver.publicKey !== config.ARCHIVER_PUBLIC_KEY - ) while (retry > 0) { try { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -71,7 +68,7 @@ export const syncGlobalAccount = async (retry = 5): Promise => { return equivalent } - const globalAccsResponse = await robustQuery(filteredArchivers, queryFn, equalFn, 3, true) + const globalAccsResponse = await robustQuery(State.otherArchivers, queryFn, equalFn, 3, true) Logger.mainLogger.debug('syncGlobalAccount() - globalAccsResponse', globalAccsResponse) if (!globalAccsResponse) { Logger.mainLogger.warn('() - robustResponse is null') @@ -97,7 +94,7 @@ export const syncGlobalAccount = async (retry = 5): Promise => { const queryFn = async (node: Node): Promise => { return await getJson(`http://${node.ip}:${node.port}/get-network-account?hash=false`) } - const networkAccResponse = await robustQuery(filteredArchivers, queryFn, equalFn, 3, true) + const networkAccResponse = await robustQuery(State.otherArchivers, queryFn, equalFn, 3, true) Logger.mainLogger.debug('syncGlobalAccount() - networkAccResponse', networkAccResponse) if (!networkAccResponse) { Logger.mainLogger.warn('get-network-account() - robustResponse is null') diff --git a/src/State.ts b/src/State.ts index a98cc078..8423ff16 100644 --- a/src/State.ts +++ b/src/State.ts @@ -34,6 +34,8 @@ const nodeState: ArchiverNodeState = { export const joinedArchivers: ArchiverNodeInfo[] = [] // Add joined archivers to this list first and move to activeArchivers when they are active export let activeArchivers: ArchiverNodeInfo[] = [] export let activeArchiversByPublicKeySorted: ArchiverNodeInfo[] = [] +// archivers list without the current archiver +export let otherArchivers: ArchiverNodeInfo[] = [] export let isFirst = false export let isActive = false export const archiversReputation: Map = new Map() @@ -163,6 +165,9 @@ export function addArchiver(archiver: ArchiverNodeInfo): void { 'activeArchiversByPublicKeySorted', activeArchiversByPublicKeySorted.map((archiver) => archiver.publicKey) ) + if (archiver.publicKey !== config.ARCHIVER_PUBLIC_KEY) { + otherArchivers.push(archiver) + } Logger.mainLogger.debug('New archiver added to active list', archiver) } Logger.mainLogger.debug('archivers list', activeArchivers) @@ -173,6 +178,7 @@ export function removeActiveArchiver(publicKey: string): void { activeArchiversByPublicKeySorted = activeArchiversByPublicKeySorted.filter( (a: ArchiverNodeInfo) => a.publicKey !== publicKey ) + otherArchivers = otherArchivers.filter((a: ArchiverNodeInfo) => a.publicKey !== publicKey) archiversReputation.delete(publicKey) } @@ -180,6 +186,7 @@ export function resetActiveArchivers(archivers: ArchiverNodeInfo[]): void { Logger.mainLogger.debug('Resetting active archivers.', archivers) activeArchivers = archivers activeArchiversByPublicKeySorted = [...archivers.sort(NodeList.byAscendingPublicKey)] + otherArchivers = activeArchivers.filter((a) => a.publicKey !== config.ARCHIVER_PUBLIC_KEY) archiversReputation.clear() for (const archiver of activeArchivers) { archiversReputation.set(archiver.publicKey, 'up') @@ -252,9 +259,6 @@ export function setActive(): void { } export function getRandomArchiver(): ArchiverNodeInfo { - const filteredArchivers = activeArchivers.filter( - (archiver) => archiver.publicKey !== config.ARCHIVER_PUBLIC_KEY - ) - const randomArchiver = Utils.getRandomItemFromArr(filteredArchivers)[0] + const randomArchiver = Utils.getRandomItemFromArr(otherArchivers)[0] return randomArchiver } From 339d1ae24152c9596cc0e69b995e8cd4880de57c Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 20 Jun 2024 18:57:57 +0545 Subject: [PATCH 05/51] Updated to gossip more archivers in addition to adjacent archivers and the receiver to collect tx data from the sender archivers --- src/API.ts | 20 ++++- src/Data/Collector.ts | 190 ++++++++++++++++++++++++++++++++++------- src/Data/GossipData.ts | 24 +++--- 3 files changed, 187 insertions(+), 47 deletions(-) diff --git a/src/API.ts b/src/API.ts index 5034ea9b..0583cff4 100644 --- a/src/API.ts +++ b/src/API.ts @@ -400,6 +400,13 @@ export function registerRoutes(server: FastifyInstance MAX_ORIGINAL_TXS_PER_REQUEST) { + reply.send({ + success: false, + error: `Exceed maximum limit of ${MAX_ORIGINAL_TXS_PER_REQUEST} original transactions`, + }) + return + } for (const [txId, txTimestamp] of txIdList) { if (typeof txId !== 'string' || txId.length !== TXID_LENGTH || typeof txTimestamp !== 'number') { reply.send({ @@ -515,6 +522,13 @@ export function registerRoutes(server: FastifyInstance MAX_RECEIPTS_PER_REQUEST) { + reply.send({ + success: false, + error: `Exceed maximum limit of ${MAX_RECEIPTS_PER_REQUEST} receipts`, + }) + return + } for (const [txId, txTimestamp] of txIdList) { if (typeof txId !== 'string' || txId.length !== TXID_LENGTH || typeof txTimestamp !== 'number') { reply.send({ @@ -875,7 +889,8 @@ export function registerRoutes(server: FastifyInstance { const payload = _request.body as AccountDataProvider.AccountDataByListRequestSchema - if (config.VERBOSE) Logger.mainLogger.debug('Account Data By List received', StringUtils.safeStringify(payload)) + if (config.VERBOSE) + Logger.mainLogger.debug('Account Data By List received', StringUtils.safeStringify(payload)) const result = AccountDataProvider.validateAccountDataByListRequest(payload) // Logger.mainLogger.debug('Account Data By List validation result', result) if (!result.success) { @@ -893,7 +908,8 @@ export function registerRoutes(server: FastifyInstance { const payload = _request.body as AccountDataProvider.GlobalAccountReportRequestSchema - if (config.VERBOSE) Logger.mainLogger.debug('Global Account Report received', StringUtils.safeStringify(payload)) + if (config.VERBOSE) + Logger.mainLogger.debug('Global Account Report received', StringUtils.safeStringify(payload)) const result = AccountDataProvider.validateGlobalAccountReportRequest(payload) // Logger.mainLogger.debug('Global Account Report validation result', result) if (!result.success) { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 380ff446..0586cc6c 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -51,12 +51,13 @@ const collectingMissingOriginalTxsMap: Map = new Map() interface MissingTx { txTimestamp: number receivedTimestamp: number + senders: string[] } const WAIT_TIME_FOR_MISSING_TX_DATA = 2000 // in ms // For debugging gossip data, set this to true. This will save only the gossip data received from the adjacent archivers. -export const saveOnlyGossipData = false +export const saveOnlyGossipData = true type GET_TX_RECEIPT_RESPONSE = { success: boolean @@ -1040,7 +1041,6 @@ export const validateGossipData = (data: GossipData): validateResponse => { let err = Utils.validateTypes(data, { dataType: 's', data: 'a', - sender: 's', sign: 'o', }) if (err) { @@ -1052,13 +1052,9 @@ export const validateGossipData = (data: GossipData): validateResponse => { Logger.mainLogger.error('Invalid gossip data signature', err) return { success: false, reason: 'Invalid gossip data signature' + err } } - if (data.sign.owner !== data.sender) { - Logger.mainLogger.error('Data sender publicKey and sign owner key does not match') - return { success: false, error: 'Data sender publicKey and sign owner key does not match' } - } - if (!adjacentArchivers.has(data.sender)) { - Logger.mainLogger.error('Data sender is not the adjacent archiver') - return { success: false, error: 'Data sender not the adjacent archiver' } + if (!State.activeArchivers.some((archiver) => archiver.publicKey === data.sign.owner)) { + Logger.mainLogger.error('Data sender is not the active archivers') + return { success: false, error: 'Data sender not the active archivers' } } if ( data.dataType !== DataType.RECEIPT && @@ -1076,7 +1072,8 @@ export const validateGossipData = (data: GossipData): validateResponse => { } export const processGossipData = (gossipdata: GossipData): void => { - const { dataType, data, sender } = gossipdata + const { dataType, data, sign } = gossipdata + const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sign.owner) const receivedTimestamp = Date.now() if (dataType === DataType.RECEIPT) { for (const { txId, timestamp } of data as TxData[]) { @@ -1085,10 +1082,30 @@ export const processGossipData = (gossipdata: GossipData): void => { (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === timestamp) || (collectingMissingReceiptsMap.has(txId) && collectingMissingReceiptsMap.get(txId) === timestamp) ) { - // console.log('GOSSIP', 'RECEIPT', 'SKIP', txId, sender) + console.log('GOSSIP', 'RECEIPT', 'SKIP', txId, 'sender', sign.owner) continue - } else missingReceiptsMap.set(txId, { txTimestamp: timestamp, receivedTimestamp }) - // console.log('GOSSIP', 'RECEIPT', 'MISS', txId, sender) + } else { + if (missingReceiptsMap.has(txId)) { + if ( + missingReceiptsMap.get(txId).txTimestamp === timestamp && + !missingReceiptsMap.get(txId).senders.some((sender) => sender === sign.owner) + ) + missingReceiptsMap.get(txId).senders.push(sign.owner) + else { + // Not expected to happen, but log error if it happens <-- could be malicious act of the sender + if (missingReceiptsMap.get(txId).txTimestamp !== timestamp) + Logger.mainLogger.error( + `Received gossip for receipt ${txId} with different timestamp ${timestamp} from archiver ${sign.owner}` + ) + if (missingReceiptsMap.get(txId).senders.some((sender) => sender === sign.owner)) + Logger.mainLogger.error( + `Received gossip for receipt ${txId} from the same sender ${sign.owner}` + ) + } + } else + missingReceiptsMap.set(txId, { txTimestamp: timestamp, receivedTimestamp, senders: [sign.owner] }) + console.log('GOSSIP', 'RECEIPT', 'MISS', txId, 'sender', sign.owner) + } } } if (dataType === DataType.ORIGINAL_TX_DATA) { @@ -1098,28 +1115,51 @@ export const processGossipData = (gossipdata: GossipData): void => { (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === timestamp) || (collectingMissingOriginalTxsMap.has(txId) && collectingMissingOriginalTxsMap.get(txId) === timestamp) ) { - // console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'SKIP', txId, sender) + console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'SKIP', txId, 'sender', sign.owner) continue - } else missingOriginalTxsMap.set(txId, { txTimestamp: timestamp, receivedTimestamp }) - // console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'MISS', txId, sender) + } else { + if (missingOriginalTxsMap.has(txId)) { + if ( + missingOriginalTxsMap.get(txId).txTimestamp === timestamp && + !missingOriginalTxsMap.get(txId).senders.some((sender) => sender === sign.owner) + ) + missingOriginalTxsMap.get(txId).senders.push(sign.owner) + else { + // Not expected to happen, but log error if it happens <-- could be malicious act of the sender + if (missingOriginalTxsMap.get(txId).txTimestamp !== timestamp) + Logger.mainLogger.error( + `Received gossip for originalTxData ${txId} with different timestamp ${timestamp} from archiver ${sign.owner}` + ) + if (missingOriginalTxsMap.get(txId).senders.some((sender) => sender === sign.owner)) + Logger.mainLogger.error( + `Received gossip for originalTxData ${txId} from the same sender ${sign.owner}` + ) + } + } else + missingOriginalTxsMap.set(txId, { + txTimestamp: timestamp, + receivedTimestamp, + senders: [sign.owner], + }) + console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'MISS', txId, 'sender', sign.owner) + } } } if (dataType === DataType.CYCLE) { collectCycleData( data as P2PTypes.CycleCreatorTypes.CycleData[], - adjacentArchivers.get(sender).ip + ':' + adjacentArchivers.get(sender).port + senderArchiver?.ip + ':' + senderArchiver?.port ) } } export const collectMissingReceipts = async (): Promise => { if (missingReceiptsMap.size === 0) return - const bucketSize = 100 const currentTimestamp = Date.now() - const cloneMissingReceiptsMap: Map = new Map() - for (const [txId, { txTimestamp, receivedTimestamp }] of missingReceiptsMap) { + const cloneMissingReceiptsMap: Map> = new Map() + for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingReceiptsMap) { if (currentTimestamp - receivedTimestamp > WAIT_TIME_FOR_MISSING_TX_DATA) { - cloneMissingReceiptsMap.set(txId, txTimestamp) + cloneMissingReceiptsMap.set(txId, { txTimestamp, senders }) collectingMissingReceiptsMap.set(txId, txTimestamp) missingReceiptsMap.delete(txId) } @@ -1130,9 +1170,18 @@ export const collectMissingReceipts = async (): Promise => { cloneMissingReceiptsMap.size, cloneMissingReceiptsMap ) + const collectFromSenderArchivers = true + if (collectFromSenderArchivers) { + for (const [txId, { txTimestamp, senders }] of cloneMissingReceiptsMap) { + collectMissingTxDataFromSenderArchivers(DataType.RECEIPT, senders, txId, txTimestamp) + } + cloneMissingReceiptsMap.clear() + return + } // Try to get missing receipts from 3 different archivers if one archiver fails to return some receipts const maxRetry = 3 let retry = 0 + const bucketSize = config.REQUEST_LIMIT.MAX_RECEIPTS_PER_REQUEST const archiversToUse: State.ArchiverNodeInfo[] = getArchiversToUse() while (cloneMissingReceiptsMap.size > 0 && retry < maxRetry) { // eslint-disable-next-line security/detect-object-injection @@ -1140,7 +1189,7 @@ export const collectMissingReceipts = async (): Promise => { if (!archiver) archiver = archiversToUse[0] const txIdList: [string, number][] = [] let totalEntries = cloneMissingReceiptsMap.size - for (const [txId, txTimestamp] of cloneMissingReceiptsMap) { + for (const [txId, { txTimestamp }] of cloneMissingReceiptsMap) { totalEntries-- if ( (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || @@ -1157,20 +1206,20 @@ export const collectMissingReceipts = async (): Promise => { DataType.RECEIPT, txIdList )) as Receipt.Receipt[] - if (receipts && receipts.length > -1) { + if (receipts && receipts.length > 0) { const receiptsToSave = [] for (const receipt of receipts) { const { receiptId, timestamp } = receipt if ( cloneMissingReceiptsMap.has(receiptId) && - cloneMissingReceiptsMap.get(receiptId) === timestamp + cloneMissingReceiptsMap.get(receiptId).txTimestamp === timestamp ) { cloneMissingReceiptsMap.delete(receiptId) collectingMissingReceiptsMap.delete(txId) receiptsToSave.push(receipt) } } - await storeReceiptData(receiptsToSave, archiver.ip + ':' + archiver.port, true) + storeReceiptData(receiptsToSave, archiver.ip + ':' + archiver.port, true) } } retry++ @@ -1187,6 +1236,70 @@ export const collectMissingReceipts = async (): Promise => { } } +const collectMissingTxDataFromSenderArchivers = async ( + dataType: DataType, + senders: string[], + txId: string, + txTimestamp: number +): Promise => { + const txIdList: [string, number][] = [[txId, txTimestamp]] + let foundTxData = false + for (const sender of senders) { + if (dataType === DataType.ORIGINAL_TX_DATA) { + if ( + (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || + (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === txTimestamp) + ) { + foundTxData = true + break + } + } + if (dataType === DataType.RECEIPT) { + if ( + (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || + (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === txTimestamp) + ) { + foundTxData = true + break + } + } + const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sender) + if (!senderArchiver) continue + const response = (await queryTxDataFromArchivers( + senderArchiver, + dataType, + txIdList + )) as QueryTxDataFromArchiversResponse + if (dataType === DataType.RECEIPT) { + const receipts = response as Receipt.Receipt[] + if (receipts && receipts.length > 0) { + for (const receipt of receipts) { + const { receiptId, timestamp } = receipt + if (txId === receiptId && txTimestamp === timestamp) { + storeReceiptData([receipt], senderArchiver.ip + ':' + senderArchiver.port, true) + foundTxData = true + } + } + } + } + if (dataType === DataType.ORIGINAL_TX_DATA) { + const originalTxs = response as OriginalTxDB.OriginalTxData[] + if (originalTxs && originalTxs.length > 0) { + for (const originalTx of originalTxs) + if (txId === originalTx.txId && txTimestamp === originalTx.timestamp) { + storeOriginalTxData([originalTx], senderArchiver.ip + ':' + senderArchiver.port) + foundTxData = true + } + } + } + if (foundTxData) break + } + if (!foundTxData) { + Logger.mainLogger.error(`Failed to get missing ${dataType}`, txId, txTimestamp) + } + collectingMissingReceiptsMap.delete(txId) +} + type TxDataFromArchiversResponse = { receipts?: Receipt.Receipt[] originalTxs?: OriginalTxDB.OriginalTxData[] @@ -1228,12 +1341,11 @@ export const queryTxDataFromArchivers = async ( export const collectMissingOriginalTxsData = async (): Promise => { if (missingOriginalTxsMap.size === 0) return - const bucketSize = 100 const currentTimestamp = Date.now() - const cloneMissingOriginalTxsMap: Map = new Map() - for (const [txId, { txTimestamp, receivedTimestamp }] of missingOriginalTxsMap) { + const cloneMissingOriginalTxsMap: Map> = new Map() + for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingOriginalTxsMap) { if (currentTimestamp - receivedTimestamp > WAIT_TIME_FOR_MISSING_TX_DATA) { - cloneMissingOriginalTxsMap.set(txId, txTimestamp) + cloneMissingOriginalTxsMap.set(txId, { txTimestamp, senders }) collectingMissingOriginalTxsMap.set(txId, txTimestamp) missingOriginalTxsMap.delete(txId) } @@ -1244,9 +1356,18 @@ export const collectMissingOriginalTxsData = async (): Promise => { cloneMissingOriginalTxsMap.size, cloneMissingOriginalTxsMap ) + const collectFromSenderArchivers = true + if (collectFromSenderArchivers) { + for (const [txId, { txTimestamp, senders }] of cloneMissingOriginalTxsMap) { + collectMissingTxDataFromSenderArchivers(DataType.ORIGINAL_TX_DATA, senders, txId, txTimestamp) + } + cloneMissingOriginalTxsMap.clear() + return + } // Try to get missing originalTxs from 3 different archivers if one archiver fails to return some receipts const maxRetry = 3 let retry = 0 + const bucketSize = config.REQUEST_LIMIT.MAX_ORIGINAL_TXS_PER_REQUEST const archiversToUse: State.ArchiverNodeInfo[] = getArchiversToUse() while (cloneMissingOriginalTxsMap.size > 0 && retry < maxRetry) { // eslint-disable-next-line security/detect-object-injection @@ -1254,7 +1375,7 @@ export const collectMissingOriginalTxsData = async (): Promise => { if (!archiver) archiver = archiversToUse[0] const txIdList: [string, number][] = [] let totalEntries = cloneMissingOriginalTxsMap.size - for (const [txId, txTimestamp] of cloneMissingOriginalTxsMap) { + for (const [txId, { txTimestamp }] of cloneMissingOriginalTxsMap) { totalEntries-- if ( (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || @@ -1271,11 +1392,14 @@ export const collectMissingOriginalTxsData = async (): Promise => { DataType.ORIGINAL_TX_DATA, txIdList )) as OriginalTxDB.OriginalTxData[] - if (originalTxs && originalTxs.length > -1) { + if (originalTxs && originalTxs.length > 0) { const originalTxsDataToSave = [] for (const originalTx of originalTxs) { const { txId, timestamp } = originalTx - if (cloneMissingOriginalTxsMap.has(txId) && cloneMissingOriginalTxsMap.get(txId) === timestamp) { + if ( + cloneMissingOriginalTxsMap.has(txId) && + cloneMissingOriginalTxsMap.get(txId).txTimestamp === timestamp + ) { cloneMissingOriginalTxsMap.delete(txId) collectingMissingOriginalTxsMap.delete(txId) originalTxsDataToSave.push(originalTx) @@ -1329,7 +1453,7 @@ export function cleanOldOriginalTxsMap(timestamp: number): void { } export const scheduleMissingTxsDataQuery = (): void => { - // Set to collect missing txs data in every 5 seconds + // Set to collect missing txs data in every 1 second setInterval(() => { collectMissingReceipts() collectMissingOriginalTxsData() diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index a6800e02..e0f1522b 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -4,7 +4,6 @@ import * as Crypto from '../Crypto' import { postJson } from '../P2P' import { Signature } from '@shardus/crypto-utils' import { P2P as P2PTypes } from '@shardus/types' -import { getRandomActiveNodes } from '../NodeList' import * as Utils from '../Utils' // adjacentArchivers are one archiver from left and one archiver from right of the current archiver @@ -21,15 +20,14 @@ export type TxData = { txId: string; timestamp: number } export interface GossipData { dataType: DataType data: TxData[] | P2PTypes.CycleCreatorTypes.CycleData[] - sender: string sign: Signature } // For debugging purpose, set this to true to stop gossiping tx data const stopGossipTxData = false -const gossipToRandomArchivers = true // To gossip to random archivers in addition to adjacent archivers -const randomGossipArchiverCount = 2 // Number of random archivers to gossip to +const gossipToMoreArchivers = true // To gossip to more archivers in addition to adjacent archivers +const randomGossipArchiversCount = 2 // Number of random archivers to gossip to // List of archivers that are not adjacent to the current archiver const remainingArchivers = [] @@ -62,8 +60,8 @@ export const getAdjacentLeftAndRightArchivers = (): void => { if (leftArchiver) adjacentArchivers.set(leftArchiver.publicKey, leftArchiver) if (rightArchiver) adjacentArchivers.set(rightArchiver.publicKey, rightArchiver) remainingArchivers.length = 0 - for (const archiver of State.activeArchivers) { - if (!adjacentArchivers.has(archiver.publicKey) || archiver.publicKey !== State.getNodeInfo().publicKey) { + for (const archiver of State.otherArchivers) { + if (!adjacentArchivers.has(archiver.publicKey)) { remainingArchivers.push(archiver) } } @@ -78,7 +76,6 @@ export async function sendDataToAdjacentArchivers( const gossipPayload = { dataType, data, - sender: State.getNodeInfo().publicKey, } as GossipData const signedDataToSend = Crypto.sign(gossipPayload) try { @@ -88,7 +85,13 @@ export async function sendDataToAdjacentArchivers( )}` ) const promises = [] - for (const [, archiver] of adjacentArchivers) { + const archiversToSend = [...adjacentArchivers.values()] + if (gossipToMoreArchivers && remainingArchivers.length > 0) { + const randomArchivers = Utils.getRandomItemFromArr(remainingArchivers, 0, randomGossipArchiversCount) + if (randomArchivers.length > 0) archiversToSend.push(...randomArchivers) + } + console.log('archiversToSend 2', archiversToSend) + for (const archiver of archiversToSend) { const url = `http://${archiver.ip}:${archiver.port}/gossip-data` try { const GOSSIP_DATA_TIMEOUT_SECOND = 10 // 10 seconds @@ -101,11 +104,8 @@ export async function sendDataToAdjacentArchivers( Logger.mainLogger.error('Error', e) } } - if (gossipToRandomArchivers) { - const randomArchivers = getRandomActiveNodes - } try { - await Promise.all(promises) + await Promise.allSettled(promises) } catch (err) { Logger.mainLogger.error('Network: ' + err) } From bfbf33045ce66d96707fd7528ed1975816ed3d5a Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 20 Jun 2024 23:37:30 +0545 Subject: [PATCH 06/51] Updated to subscribed more nodes if there are less than 4 archivers and the consensusRadius is big --- src/Data/Collector.ts | 15 +++----- src/Data/Data.ts | 80 ++++++++++++++++++++----------------------- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 0586cc6c..30b60327 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -19,14 +19,7 @@ import { getCurrentCycleCounter, shardValuesByCycle, computeCycleMarker } from ' import { bulkInsertCycles, queryCycleByMarker, updateCycle } from '../dbstore/cycles' import * as State from '../State' import * as Utils from '../Utils' -import { - DataType, - GossipData, - adjacentArchivers, - sendDataToAdjacentArchivers, - TxData, - getArchiversToUse, -} from './GossipData' +import { DataType, GossipData, sendDataToAdjacentArchivers, TxData, getArchiversToUse } from './GossipData' import { postJson } from '../P2P' import { globalAccountsMap, setGlobalNetworkAccount } from '../GlobalAccount' import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Data/DataLogWriter' @@ -57,7 +50,7 @@ interface MissingTx { const WAIT_TIME_FOR_MISSING_TX_DATA = 2000 // in ms // For debugging gossip data, set this to true. This will save only the gossip data received from the adjacent archivers. -export const saveOnlyGossipData = true +export const saveOnlyGossipData = false type GET_TX_RECEIPT_RESPONSE = { success: boolean @@ -1295,7 +1288,9 @@ const collectMissingTxDataFromSenderArchivers = async ( if (foundTxData) break } if (!foundTxData) { - Logger.mainLogger.error(`Failed to get missing ${dataType}`, txId, txTimestamp) + Logger.mainLogger.error( + `Failed to collect ${dataType} data for txId ${txId} with timestamp ${txTimestamp} from archivers ${senders}` + ) } collectingMissingReceiptsMap.delete(txId) } diff --git a/src/Data/Data.ts b/src/Data/Data.ts index f98500c3..cdf56735 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -56,6 +56,7 @@ let subsetNodesMapByConsensusRadius: Map = const maxCyclesInCycleTracker = 5 const receivedCycleTracker = {} const QUERY_TIMEOUT_MAX = 30 // 30seconds +const subscribedToMoreConsensors = true const { MAX_ACCOUNTS_PER_REQUEST, @@ -479,25 +480,7 @@ export async function replaceDataSender(publicKey: NodeList.ConsensusNodeInfo['p ) return } - - // Check if there is any subscribed node from this subset - let foundSubscribedNodeFromThisSubset = false - for (const node of Object.values(subsetNodesList)) { - if (dataSenders.has(node.publicKey)) { - if (config.VERBOSE) Logger.mainLogger.debug('This node from the subset is in the subscribed list!') - if (foundSubscribedNodeFromThisSubset) { - // Unsubscribe the extra nodes from this subset - unsubscribeDataSender(node.publicKey) - } - foundSubscribedNodeFromThisSubset = true - } - } - - if (!foundSubscribedNodeFromThisSubset) { - Logger.mainLogger.debug('There is no subscribed node from this subset!') - // Pick a new dataSender from this subset - subscribeNodeFromThisSubset(subsetNodesList) - } + subscribeNodeFromThisSubset(subsetNodesList) } } } @@ -652,45 +635,56 @@ export async function subscribeConsensorsByConsensusRadius(): Promise { await createNodesGroupByConsensusRadius() for (const [i, subsetList] of subsetNodesMapByConsensusRadius) { if (config.VERBOSE) Logger.mainLogger.debug('Round', i, 'subsetList', subsetList, dataSenders.keys()) - let foundSubscribedNodeFromThisSubset = false - for (const node of Object.values(subsetList)) { - if (dataSenders.has(node.publicKey)) { - if (config.VERBOSE) Logger.mainLogger.debug('This node from the subset is in the subscribed list!') - if (foundSubscribedNodeFromThisSubset) { - // Unsubscribe the extra nodes from this subset - unsubscribeDataSender(node.publicKey) - } - foundSubscribedNodeFromThisSubset = true - } - } - - if (!foundSubscribedNodeFromThisSubset) { - Logger.mainLogger.debug('There is no subscribed node from this subset!') - // Pick a new dataSender from this subset - subscribeNodeFromThisSubset(subsetList) - } + subscribeNodeFromThisSubset(subsetList) } } export async function subscribeNodeFromThisSubset(nodeList: NodeList.ConsensusNodeInfo[]): Promise { + // First check if there is any subscribed node from this subset + const subscribedNodesFromThisSubset = [] + for (const node of nodeList) { + if (dataSenders.has(node.publicKey)) { + if (config.VERBOSE) + Logger.mainLogger.debug('This node from the subset is in the subscribed list!', node.publicKey) + subscribedNodesFromThisSubset.push(node.publicKey) + } + } + let numberOfNodesToSubsribe = 1 + // Only if there are less than 4 activeArchivers and the currentConsensusRadius is greater than 5 + if (subscribedToMoreConsensors && State.activeArchivers.length < 4 && currentConsensusRadius > 5) { + numberOfNodesToSubsribe = 2 + } + if (subscribedNodesFromThisSubset.length > numberOfNodesToSubsribe) { + // If there is more than one subscribed node from this subset, unsubscribe the extra ones + for (const publicKey of subscribedNodesFromThisSubset.splice(numberOfNodesToSubsribe)) { + unsubscribeDataSender(publicKey) + } + } + if (config.VERBOSE) + Logger.mainLogger.debug('Subscribed nodes from this subset', subscribedNodesFromThisSubset) + if (subscribedNodesFromThisSubset.length === numberOfNodesToSubsribe) return + Logger.mainLogger.debug('There is no subscribed node from this subset!') + // Pick a new dataSender from this subset let subsetList = [...nodeList] // Pick a random dataSender let newSenderInfo = nodeList[Math.floor(Math.random() * nodeList.length)] let connectionStatus = false let retry = 0 - const MAX_RETRY_SUBSCRIPTION = 3 - while (retry < MAX_RETRY_SUBSCRIPTION) { + const MAX_RETRY_SUBSCRIPTION = 3 * numberOfNodesToSubsribe + while (retry < MAX_RETRY_SUBSCRIPTION && subscribedNodesFromThisSubset.length < numberOfNodesToSubsribe) { if (!dataSenders.has(newSenderInfo.publicKey)) { connectionStatus = await createDataTransferConnection(newSenderInfo) if (connectionStatus) { - break - } else { - subsetList = subsetList.filter((node) => node.publicKey !== newSenderInfo.publicKey) + // Check if the newSender is in the subscribed nodes of this subset + if (!subscribedNodesFromThisSubset.includes(newSenderInfo.publicKey)) + subscribedNodesFromThisSubset.push(newSenderInfo.publicKey) } } else { - // This means there is already a subscribed node from this subset - break + // Add the newSender to the subscribed nodes of this subset + if (!subscribedNodesFromThisSubset.includes(newSenderInfo.publicKey)) + subscribedNodesFromThisSubset.push(newSenderInfo.publicKey) } + subsetList = subsetList.filter((node) => node.publicKey !== newSenderInfo.publicKey) if (subsetList.length > 0) { newSenderInfo = subsetList[Math.floor(Math.random() * subsetList.length)] } else { From f8f732c46c43fc0cae2804dc769851fca32d49dd Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 21 Jun 2024 18:38:48 +0545 Subject: [PATCH 07/51] More improvements --- src/Config.ts | 10 ++++++++++ src/Data/Collector.ts | 16 +++++++--------- src/Data/Data.ts | 17 ++++++++++------- src/Data/GossipData.ts | 38 +++++++++++++++++++------------------- 4 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 401af2c6..752d7b6a 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -56,6 +56,11 @@ export interface Config { enabled: boolean } newPOQReceipt: boolean + waitingTimeForMissingTxData: number // Wait time in ms for missing tx data before collecting from other archivers + gossipToMoreArchivers: true // To gossip to more archivers in addition to adjacent archivers + randomGossipArchiversCount: 2 // Number of random archivers to gossip to + subscribeToMoreConsensors: boolean // To subscribe to more consensors when the number of active archivers is less than 4 + extraConsensorsToSubscribe: 1 // Number of extra consensors to subscribe to } let config: Config = { @@ -110,6 +115,11 @@ let config: Config = { enabled: false, }, newPOQReceipt: false, + gossipToMoreArchivers: true, + randomGossipArchiversCount: 2, + subscribeToMoreConsensors: true, + extraConsensorsToSubscribe: 1, + waitingTimeForMissingTxData: 2000, // in ms } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 30b60327..22911551 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -47,9 +47,7 @@ interface MissingTx { senders: string[] } -const WAIT_TIME_FOR_MISSING_TX_DATA = 2000 // in ms - -// For debugging gossip data, set this to true. This will save only the gossip data received from the adjacent archivers. +// For debugging gossip data, set this to true. This will save only the gossip data received from the gossip archivers. export const saveOnlyGossipData = false type GET_TX_RECEIPT_RESPONSE = { @@ -1075,7 +1073,7 @@ export const processGossipData = (gossipdata: GossipData): void => { (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === timestamp) || (collectingMissingReceiptsMap.has(txId) && collectingMissingReceiptsMap.get(txId) === timestamp) ) { - console.log('GOSSIP', 'RECEIPT', 'SKIP', txId, 'sender', sign.owner) + // console.log('GOSSIP', 'RECEIPT', 'SKIP', txId, 'sender', sign.owner) continue } else { if (missingReceiptsMap.has(txId)) { @@ -1097,7 +1095,7 @@ export const processGossipData = (gossipdata: GossipData): void => { } } else missingReceiptsMap.set(txId, { txTimestamp: timestamp, receivedTimestamp, senders: [sign.owner] }) - console.log('GOSSIP', 'RECEIPT', 'MISS', txId, 'sender', sign.owner) + // console.log('GOSSIP', 'RECEIPT', 'MISS', txId, 'sender', sign.owner) } } } @@ -1108,7 +1106,7 @@ export const processGossipData = (gossipdata: GossipData): void => { (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === timestamp) || (collectingMissingOriginalTxsMap.has(txId) && collectingMissingOriginalTxsMap.get(txId) === timestamp) ) { - console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'SKIP', txId, 'sender', sign.owner) + // console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'SKIP', txId, 'sender', sign.owner) continue } else { if (missingOriginalTxsMap.has(txId)) { @@ -1134,7 +1132,7 @@ export const processGossipData = (gossipdata: GossipData): void => { receivedTimestamp, senders: [sign.owner], }) - console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'MISS', txId, 'sender', sign.owner) + // console.log('GOSSIP', 'ORIGINAL_TX_DATA', 'MISS', txId, 'sender', sign.owner) } } } @@ -1151,7 +1149,7 @@ export const collectMissingReceipts = async (): Promise => { const currentTimestamp = Date.now() const cloneMissingReceiptsMap: Map> = new Map() for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingReceiptsMap) { - if (currentTimestamp - receivedTimestamp > WAIT_TIME_FOR_MISSING_TX_DATA) { + if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { cloneMissingReceiptsMap.set(txId, { txTimestamp, senders }) collectingMissingReceiptsMap.set(txId, txTimestamp) missingReceiptsMap.delete(txId) @@ -1339,7 +1337,7 @@ export const collectMissingOriginalTxsData = async (): Promise => { const currentTimestamp = Date.now() const cloneMissingOriginalTxsMap: Map> = new Map() for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingOriginalTxsMap) { - if (currentTimestamp - receivedTimestamp > WAIT_TIME_FOR_MISSING_TX_DATA) { + if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { cloneMissingOriginalTxsMap.set(txId, { txTimestamp, senders }) collectingMissingOriginalTxsMap.set(txId, txTimestamp) missingOriginalTxsMap.delete(txId) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index cdf56735..3088c130 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -56,8 +56,6 @@ let subsetNodesMapByConsensusRadius: Map = const maxCyclesInCycleTracker = 5 const receivedCycleTracker = {} const QUERY_TIMEOUT_MAX = 30 // 30seconds -const subscribedToMoreConsensors = true - const { MAX_ACCOUNTS_PER_REQUEST, MAX_RECEIPTS_PER_REQUEST, @@ -618,7 +616,11 @@ export async function createNodesGroupByConsensusRadius(): Promise { currentConsensusRadius = consensusRadius const activeList = [...NodeList.activeListByIdSorted] if (config.VERBOSE) Logger.mainLogger.debug('activeList', activeList.length, activeList) - const totalNumberOfNodesToSubscribe = Math.ceil(activeList.length / consensusRadius) + let totalNumberOfNodesToSubscribe = Math.ceil(activeList.length / consensusRadius) + // Only if there are less than 4 activeArchivers and if the consensusRadius is greater than 5 + if (config.subscribeToMoreConsensors && State.activeArchivers.length < 4 && currentConsensusRadius > 5) { + totalNumberOfNodesToSubscribe += totalNumberOfNodesToSubscribe * config.extraConsensorsToSubscribe + } Logger.mainLogger.debug('totalNumberOfNodesToSubscribe', totalNumberOfNodesToSubscribe) subsetNodesMapByConsensusRadius = new Map() let round = 0 @@ -650,20 +652,21 @@ export async function subscribeNodeFromThisSubset(nodeList: NodeList.ConsensusNo } } let numberOfNodesToSubsribe = 1 - // Only if there are less than 4 activeArchivers and the currentConsensusRadius is greater than 5 - if (subscribedToMoreConsensors && State.activeArchivers.length < 4 && currentConsensusRadius > 5) { - numberOfNodesToSubsribe = 2 + // Only if there are less than 4 activeArchivers and if the consensusRadius is greater than 5 + if (config.subscribeToMoreConsensors && State.activeArchivers.length < 4 && currentConsensusRadius > 5) { + numberOfNodesToSubsribe += config.extraConsensorsToSubscribe } if (subscribedNodesFromThisSubset.length > numberOfNodesToSubsribe) { // If there is more than one subscribed node from this subset, unsubscribe the extra ones for (const publicKey of subscribedNodesFromThisSubset.splice(numberOfNodesToSubsribe)) { + Logger.mainLogger.debug('Unsubscribing extra node from this subset', publicKey) unsubscribeDataSender(publicKey) } } if (config.VERBOSE) Logger.mainLogger.debug('Subscribed nodes from this subset', subscribedNodesFromThisSubset) if (subscribedNodesFromThisSubset.length === numberOfNodesToSubsribe) return - Logger.mainLogger.debug('There is no subscribed node from this subset!') + Logger.mainLogger.debug('Subscribing node from this subset!') // Pick a new dataSender from this subset let subsetList = [...nodeList] // Pick a random dataSender diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index e0f1522b..5977f863 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -5,9 +5,10 @@ import { postJson } from '../P2P' import { Signature } from '@shardus/crypto-utils' import { P2P as P2PTypes } from '@shardus/types' import * as Utils from '../Utils' +import { config } from '../Config' // adjacentArchivers are one archiver from left and one archiver from right of the current archiver -export let adjacentArchivers: Map = new Map() +export let adjacentArchivers: State.ArchiverNodeInfo[] = [] export enum DataType { RECEIPT = 'RECEIPT', @@ -26,15 +27,12 @@ export interface GossipData { // For debugging purpose, set this to true to stop gossiping tx data const stopGossipTxData = false -const gossipToMoreArchivers = true // To gossip to more archivers in addition to adjacent archivers -const randomGossipArchiversCount = 2 // Number of random archivers to gossip to - // List of archivers that are not adjacent to the current archiver const remainingArchivers = [] export const getAdjacentLeftAndRightArchivers = (): void => { if (State.activeArchivers.length <= 1) { - adjacentArchivers = new Map() + adjacentArchivers = [] return } // Treat the archivers list as a circular list and get one left and one right archivers of the current archiver @@ -56,12 +54,12 @@ export const getAdjacentLeftAndRightArchivers = (): void => { rightArchiver = State.activeArchiversByPublicKeySorted[rightArchiverIndex] /* eslint-enable security/detect-object-injection */ } - adjacentArchivers = new Map() - if (leftArchiver) adjacentArchivers.set(leftArchiver.publicKey, leftArchiver) - if (rightArchiver) adjacentArchivers.set(rightArchiver.publicKey, rightArchiver) + adjacentArchivers.length = 0 + if (leftArchiver) adjacentArchivers.push(leftArchiver) + if (rightArchiver) adjacentArchivers.push(rightArchiver) remainingArchivers.length = 0 for (const archiver of State.otherArchivers) { - if (!adjacentArchivers.has(archiver.publicKey)) { + if (!adjacentArchivers.some((a) => a.publicKey === archiver.publicKey)) { remainingArchivers.push(archiver) } } @@ -72,25 +70,27 @@ export async function sendDataToAdjacentArchivers( data: GossipData['data'] ): Promise { if (stopGossipTxData) return - if (adjacentArchivers.size === 0) return + if (State.otherArchivers.length === 0) return const gossipPayload = { dataType, data, } as GossipData const signedDataToSend = Crypto.sign(gossipPayload) try { - Logger.mainLogger.debug( - `Sending ${dataType} data to the archivers: ${Array.from(adjacentArchivers.values()).map( - (n) => `${n.ip}:${n.port}` - )}` - ) const promises = [] - const archiversToSend = [...adjacentArchivers.values()] - if (gossipToMoreArchivers && remainingArchivers.length > 0) { - const randomArchivers = Utils.getRandomItemFromArr(remainingArchivers, 0, randomGossipArchiversCount) + const archiversToSend = [...adjacentArchivers] + if (config.gossipToMoreArchivers && remainingArchivers.length > 0) { + const randomArchivers = Utils.getRandomItemFromArr( + remainingArchivers, + 0, + config.randomGossipArchiversCount + ) if (randomArchivers.length > 0) archiversToSend.push(...randomArchivers) } - console.log('archiversToSend 2', archiversToSend) + if (config.VERBOSE) + Logger.mainLogger.debug( + `Sending ${dataType} data to the archivers: ${archiversToSend.map((n) => `${n.ip}:${n.port}`)}` + ) for (const archiver of archiversToSend) { const url = `http://${archiver.ip}:${archiver.port}/gossip-data` try { From 2bf45aed1b3905164d12f6bc9a3b14eda936f8a0 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 24 Jun 2024 16:56:43 +0545 Subject: [PATCH 08/51] clean up unused codes --- src/Config.ts | 2 +- src/Data/Collector.ts | 287 +++++++++++++---------------------------- src/Data/GossipData.ts | 20 --- 3 files changed, 90 insertions(+), 219 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 752d7b6a..b8a76d8f 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -115,11 +115,11 @@ let config: Config = { enabled: false, }, newPOQReceipt: false, + waitingTimeForMissingTxData: 2000, // in ms gossipToMoreArchivers: true, randomGossipArchiversCount: 2, subscribeToMoreConsensors: true, extraConsensorsToSubscribe: 1, - waitingTimeForMissingTxData: 2000, // in ms } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 22911551..4a25c2b7 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -19,7 +19,7 @@ import { getCurrentCycleCounter, shardValuesByCycle, computeCycleMarker } from ' import { bulkInsertCycles, queryCycleByMarker, updateCycle } from '../dbstore/cycles' import * as State from '../State' import * as Utils from '../Utils' -import { DataType, GossipData, sendDataToAdjacentArchivers, TxData, getArchiversToUse } from './GossipData' +import { DataType, GossipData, sendDataToAdjacentArchivers, TxData } from './GossipData' import { postJson } from '../P2P' import { globalAccountsMap, setGlobalNetworkAccount } from '../GlobalAccount' import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Data/DataLogWriter' @@ -1144,91 +1144,51 @@ export const processGossipData = (gossipdata: GossipData): void => { } } -export const collectMissingReceipts = async (): Promise => { - if (missingReceiptsMap.size === 0) return +export const collectMissingTxDataFromArchivers = async (): Promise => { const currentTimestamp = Date.now() - const cloneMissingReceiptsMap: Map> = new Map() - for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingReceiptsMap) { - if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { - cloneMissingReceiptsMap.set(txId, { txTimestamp, senders }) - collectingMissingReceiptsMap.set(txId, txTimestamp) - missingReceiptsMap.delete(txId) + if (missingReceiptsMap.size > 0) { + const cloneMissingReceiptsMap: Map> = new Map() + for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingReceiptsMap) { + if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { + cloneMissingReceiptsMap.set(txId, { txTimestamp, senders }) + collectingMissingReceiptsMap.set(txId, txTimestamp) + missingReceiptsMap.delete(txId) + } } - } - if (cloneMissingReceiptsMap.size === 0) return - Logger.mainLogger.debug( - 'Collecting missing receipts', - cloneMissingReceiptsMap.size, - cloneMissingReceiptsMap - ) - const collectFromSenderArchivers = true - if (collectFromSenderArchivers) { + if (cloneMissingReceiptsMap.size > 0) + Logger.mainLogger.debug( + 'Collecting missing receipts', + cloneMissingReceiptsMap.size, + cloneMissingReceiptsMap + ) for (const [txId, { txTimestamp, senders }] of cloneMissingReceiptsMap) { - collectMissingTxDataFromSenderArchivers(DataType.RECEIPT, senders, txId, txTimestamp) + collectMissingReceipts(senders, txId, txTimestamp) } cloneMissingReceiptsMap.clear() - return } - // Try to get missing receipts from 3 different archivers if one archiver fails to return some receipts - const maxRetry = 3 - let retry = 0 - const bucketSize = config.REQUEST_LIMIT.MAX_RECEIPTS_PER_REQUEST - const archiversToUse: State.ArchiverNodeInfo[] = getArchiversToUse() - while (cloneMissingReceiptsMap.size > 0 && retry < maxRetry) { - // eslint-disable-next-line security/detect-object-injection - let archiver = archiversToUse[retry] - if (!archiver) archiver = archiversToUse[0] - const txIdList: [string, number][] = [] - let totalEntries = cloneMissingReceiptsMap.size - for (const [txId, { txTimestamp }] of cloneMissingReceiptsMap) { - totalEntries-- - if ( - (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || - (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === txTimestamp) - ) { - cloneMissingReceiptsMap.delete(txId) - collectingMissingReceiptsMap.delete(txId) - if (totalEntries !== 0) continue - } else txIdList.push([txId, txTimestamp]) - if (txIdList.length !== bucketSize && totalEntries !== 0) continue - if (txIdList.length === 0) continue - const receipts = (await queryTxDataFromArchivers( - archiver, - DataType.RECEIPT, - txIdList - )) as Receipt.Receipt[] - if (receipts && receipts.length > 0) { - const receiptsToSave = [] - for (const receipt of receipts) { - const { receiptId, timestamp } = receipt - if ( - cloneMissingReceiptsMap.has(receiptId) && - cloneMissingReceiptsMap.get(receiptId).txTimestamp === timestamp - ) { - cloneMissingReceiptsMap.delete(receiptId) - collectingMissingReceiptsMap.delete(txId) - receiptsToSave.push(receipt) - } - } - storeReceiptData(receiptsToSave, archiver.ip + ':' + archiver.port, true) + if (missingOriginalTxsMap.size > 0) { + const cloneMissingOriginalTxsMap: Map> = new Map() + for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingOriginalTxsMap) { + if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { + cloneMissingOriginalTxsMap.set(txId, { txTimestamp, senders }) + collectingMissingOriginalTxsMap.set(txId, txTimestamp) + missingOriginalTxsMap.delete(txId) } } - retry++ - } - if (cloneMissingReceiptsMap.size > 0) { - Logger.mainLogger.debug( - 'Receipts TxId that are failed to get from other archivers', - cloneMissingReceiptsMap - ) - // Clear the failed txIds from the collectingMissingReceiptsMap - for (const [txId] of cloneMissingReceiptsMap) { - collectingMissingReceiptsMap.delete(txId) + if (cloneMissingOriginalTxsMap.size > 0) + Logger.mainLogger.debug( + 'Collecting missing originalTxsData', + cloneMissingOriginalTxsMap.size, + cloneMissingOriginalTxsMap + ) + for (const [txId, { txTimestamp, senders }] of cloneMissingOriginalTxsMap) { + collectMissingOriginalTxsData(senders, txId, txTimestamp) } + cloneMissingOriginalTxsMap.clear() } } -const collectMissingTxDataFromSenderArchivers = async ( - dataType: DataType, +export const collectMissingReceipts = async ( senders: string[], txId: string, txTimestamp: number @@ -1236,58 +1196,73 @@ const collectMissingTxDataFromSenderArchivers = async ( const txIdList: [string, number][] = [[txId, txTimestamp]] let foundTxData = false for (const sender of senders) { - if (dataType === DataType.ORIGINAL_TX_DATA) { - if ( - (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || - (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === txTimestamp) - ) { - foundTxData = true - break - } - } - if (dataType === DataType.RECEIPT) { - if ( - (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || - (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === txTimestamp) - ) { - foundTxData = true - break - } + if ( + (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || + (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === txTimestamp) + ) { + foundTxData = true + break } const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sender) if (!senderArchiver) continue - const response = (await queryTxDataFromArchivers( + const receipts = (await queryTxDataFromArchivers( senderArchiver, - dataType, + DataType.RECEIPT, txIdList - )) as QueryTxDataFromArchiversResponse - if (dataType === DataType.RECEIPT) { - const receipts = response as Receipt.Receipt[] - if (receipts && receipts.length > 0) { - for (const receipt of receipts) { - const { receiptId, timestamp } = receipt - if (txId === receiptId && txTimestamp === timestamp) { - storeReceiptData([receipt], senderArchiver.ip + ':' + senderArchiver.port, true) - foundTxData = true - } + )) as Receipt.Receipt[] + if (receipts && receipts.length > 0) { + for (const receipt of receipts) { + const { receiptId, timestamp } = receipt + if (txId === receiptId && txTimestamp === timestamp) { + storeReceiptData([receipt], senderArchiver.ip + ':' + senderArchiver.port, true) + foundTxData = true } } } - if (dataType === DataType.ORIGINAL_TX_DATA) { - const originalTxs = response as OriginalTxDB.OriginalTxData[] - if (originalTxs && originalTxs.length > 0) { - for (const originalTx of originalTxs) - if (txId === originalTx.txId && txTimestamp === originalTx.timestamp) { - storeOriginalTxData([originalTx], senderArchiver.ip + ':' + senderArchiver.port) - foundTxData = true - } - } + if (foundTxData) break + } + if (!foundTxData) { + Logger.mainLogger.error( + `Failed to collect receipt for txId ${txId} with timestamp ${txTimestamp} from archivers ${senders}` + ) + } + collectingMissingOriginalTxsMap.delete(txId) +} + +const collectMissingOriginalTxsData = async ( + senders: string[], + txId: string, + txTimestamp: number +): Promise => { + const txIdList: [string, number][] = [[txId, txTimestamp]] + let foundTxData = false + for (const sender of senders) { + if ( + (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || + (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === txTimestamp) + ) { + foundTxData = true + break + } + const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sender) + if (!senderArchiver) continue + const originalTxs = (await queryTxDataFromArchivers( + senderArchiver, + DataType.ORIGINAL_TX_DATA, + txIdList + )) as OriginalTxDB.OriginalTxData[] + if (originalTxs && originalTxs.length > 0) { + for (const originalTx of originalTxs) + if (txId === originalTx.txId && txTimestamp === originalTx.timestamp) { + storeOriginalTxData([originalTx], senderArchiver.ip + ':' + senderArchiver.port) + foundTxData = true + } } if (foundTxData) break } if (!foundTxData) { Logger.mainLogger.error( - `Failed to collect ${dataType} data for txId ${txId} with timestamp ${txTimestamp} from archivers ${senders}` + `Failed to collect originalTxData for txId ${txId} with timestamp ${txTimestamp} from archivers ${senders}` ) } collectingMissingReceiptsMap.delete(txId) @@ -1332,89 +1307,6 @@ export const queryTxDataFromArchivers = async ( return null } -export const collectMissingOriginalTxsData = async (): Promise => { - if (missingOriginalTxsMap.size === 0) return - const currentTimestamp = Date.now() - const cloneMissingOriginalTxsMap: Map> = new Map() - for (const [txId, { txTimestamp, receivedTimestamp, senders }] of missingOriginalTxsMap) { - if (currentTimestamp - receivedTimestamp > config.waitingTimeForMissingTxData) { - cloneMissingOriginalTxsMap.set(txId, { txTimestamp, senders }) - collectingMissingOriginalTxsMap.set(txId, txTimestamp) - missingOriginalTxsMap.delete(txId) - } - } - if (cloneMissingOriginalTxsMap.size === 0) return - Logger.mainLogger.debug( - 'Collecting missing originalTxsData', - cloneMissingOriginalTxsMap.size, - cloneMissingOriginalTxsMap - ) - const collectFromSenderArchivers = true - if (collectFromSenderArchivers) { - for (const [txId, { txTimestamp, senders }] of cloneMissingOriginalTxsMap) { - collectMissingTxDataFromSenderArchivers(DataType.ORIGINAL_TX_DATA, senders, txId, txTimestamp) - } - cloneMissingOriginalTxsMap.clear() - return - } - // Try to get missing originalTxs from 3 different archivers if one archiver fails to return some receipts - const maxRetry = 3 - let retry = 0 - const bucketSize = config.REQUEST_LIMIT.MAX_ORIGINAL_TXS_PER_REQUEST - const archiversToUse: State.ArchiverNodeInfo[] = getArchiversToUse() - while (cloneMissingOriginalTxsMap.size > 0 && retry < maxRetry) { - // eslint-disable-next-line security/detect-object-injection - let archiver = archiversToUse[retry] - if (!archiver) archiver = archiversToUse[0] - const txIdList: [string, number][] = [] - let totalEntries = cloneMissingOriginalTxsMap.size - for (const [txId, { txTimestamp }] of cloneMissingOriginalTxsMap) { - totalEntries-- - if ( - (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || - (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === txTimestamp) - ) { - cloneMissingOriginalTxsMap.delete(txId) - collectingMissingOriginalTxsMap.delete(txId) - if (totalEntries !== 0) continue - } else txIdList.push([txId, txTimestamp]) - if (txIdList.length !== bucketSize && totalEntries !== 0) continue - if (txIdList.length === 0) continue - const originalTxs = (await queryTxDataFromArchivers( - archiver, - DataType.ORIGINAL_TX_DATA, - txIdList - )) as OriginalTxDB.OriginalTxData[] - if (originalTxs && originalTxs.length > 0) { - const originalTxsDataToSave = [] - for (const originalTx of originalTxs) { - const { txId, timestamp } = originalTx - if ( - cloneMissingOriginalTxsMap.has(txId) && - cloneMissingOriginalTxsMap.get(txId).txTimestamp === timestamp - ) { - cloneMissingOriginalTxsMap.delete(txId) - collectingMissingOriginalTxsMap.delete(txId) - originalTxsDataToSave.push(originalTx) - } - } - await storeOriginalTxData(originalTxsDataToSave, archiver.ip + ':' + archiver.port) - } - } - retry++ - } - if (cloneMissingOriginalTxsMap.size > 0) { - Logger.mainLogger.debug( - 'OriginalTxsData TxId that are failed to get from other archivers', - cloneMissingOriginalTxsMap - ) - // Clear the failed txIds from the collectingMissingOriginalTxsMap - for (const [txId] of cloneMissingOriginalTxsMap) { - collectingMissingOriginalTxsMap.delete(txId) - } - } -} - export function cleanOldReceiptsMap(timestamp: number): void { let savedReceiptsCount = 0 for (const [key, value] of processedReceiptsMap) { @@ -1448,7 +1340,6 @@ export function cleanOldOriginalTxsMap(timestamp: number): void { export const scheduleMissingTxsDataQuery = (): void => { // Set to collect missing txs data in every 1 second setInterval(() => { - collectMissingReceipts() - collectMissingOriginalTxsData() + collectMissingTxDataFromArchivers() }, 1000) } diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index 5977f863..faceb79e 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -114,23 +114,3 @@ export async function sendDataToAdjacentArchivers( Logger.mainLogger.debug('Fail to gossip') } } - -export const getArchiversToUse = (): State.ArchiverNodeInfo[] => { - let archiversToUse: State.ArchiverNodeInfo[] = [] - const MAX_ARCHIVERS_TO_SELECT = 3 - // Choosing MAX_ARCHIVERS_TO_SELECT random archivers from the remaining archivers list - if (State.otherArchivers.length <= MAX_ARCHIVERS_TO_SELECT) { - archiversToUse = [...State.otherArchivers] - } else { - archiversToUse = Utils.getRandomItemFromArr(remainingArchivers, 0, MAX_ARCHIVERS_TO_SELECT) - if (archiversToUse.length < MAX_ARCHIVERS_TO_SELECT) { - const requiredArchivers = MAX_ARCHIVERS_TO_SELECT - archiversToUse.length - // If the required archivers are not selected, then get it from the adjacent archivers - archiversToUse = [ - ...archiversToUse, - ...Utils.getRandomItemFromArr([...adjacentArchivers.values()], requiredArchivers), - ] - } - } - return archiversToUse -} From c496addb0ee10943600eaa6f71baf8d6440273b0 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 24 Jun 2024 17:55:00 +0545 Subject: [PATCH 09/51] Fix required receipt signatures check --- src/Data/Collector.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 4a25c2b7..8a0d7dc0 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -404,10 +404,14 @@ export const verifyReceiptData = async ( const { homePartition } = ShardFunction.addressToPartition(cycleShardData.shardGlobals, executionShardKey) if (config.newPOQReceipt === false) { // Refer to https://github.com/shardeum/shardus-core/blob/f7000c36faa0cd1e0832aa1e5e3b1414d32dcf66/src/state-manager/TransactionConsensus.ts#L1406 - const requiredSignatures = Math.round(nodesPerConsensusGroup * (2 / 3.0)) + let votingGroupCount = cycleShardData.shardGlobals.nodesPerConsenusGroup + if (votingGroupCount > cycleShardData.nodes.length) { + votingGroupCount = nodesPerConsensusGroup + } + const requiredSignatures = Math.round(votingGroupCount * (2 / 3)) if (signatures.length < requiredSignatures) { Logger.mainLogger.error( - 'Invalid receipt appliedReceipt signatures count is less than requiredSignatures' + `Invalid receipt appliedReceipt signatures count is less than requiredSignatures, ${signatures.length}, ${requiredSignatures}` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( @@ -460,7 +464,7 @@ export const verifyReceiptData = async ( } if (goodSignatures.size < requiredSignatures) { Logger.mainLogger.error( - 'Invalid receipt appliedReceipt valid signatures count is less than requiredSignatures' + `Invalid receipt appliedReceipt valid signatures count is less than requiredSignatures ${goodSignatures.size}, ${requiredSignatures}` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( From c0f86885402dbfccbfef0ebfc845a0b0ea6a908b Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 24 Jun 2024 18:31:58 +0545 Subject: [PATCH 10/51] Clean up processedReceipts cache during archiver data syncing --- src/Data/Collector.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 8a0d7dc0..90d1ad08 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -406,7 +406,7 @@ export const verifyReceiptData = async ( // Refer to https://github.com/shardeum/shardus-core/blob/f7000c36faa0cd1e0832aa1e5e3b1414d32dcf66/src/state-manager/TransactionConsensus.ts#L1406 let votingGroupCount = cycleShardData.shardGlobals.nodesPerConsenusGroup if (votingGroupCount > cycleShardData.nodes.length) { - votingGroupCount = nodesPerConsensusGroup + votingGroupCount = cycleShardData.nodes.length } const requiredSignatures = Math.round(votingGroupCount * (2 / 3)) if (signatures.length < requiredSignatures) { @@ -812,6 +812,8 @@ export const storeReceiptData = async ( } if (combineAccounts.length > 0) await Account.bulkInsertAccounts(combineAccounts) if (combineTransactions.length > 0) await Transaction.bulkInsertTransactions(combineTransactions) + // If the archiver is not active, good to clean up the processed receipts map if it exceeds 2000 + if (!State.isActive && processedReceiptsMap.size > 2000) processedReceiptsMap.clear() } export const validateCycleData = (cycleRecord: P2PTypes.CycleCreatorTypes.CycleData): boolean => { @@ -1002,6 +1004,8 @@ export const storeOriginalTxData = async ( await OriginalTxsData.bulkInsertOriginalTxsData(combineOriginalTxsData) if (State.isActive) sendDataToAdjacentArchivers(DataType.ORIGINAL_TX_DATA, txDataList) } + // If the archiver is not active yet, good to clean up the processed originalTxs map if it exceeds 2000 + if (!State.isActive && processedOriginalTxsMap.size > 2000) processedOriginalTxsMap.clear() } interface validateResponse { success: boolean From c241ead32678809228c1a8b97c6077c4d0ef56d9 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 24 Jun 2024 22:17:29 +0545 Subject: [PATCH 11/51] Refactor debug log for collecting missing txs --- src/Data/Collector.ts | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 90d1ad08..b342d388 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -1164,11 +1164,7 @@ export const collectMissingTxDataFromArchivers = async (): Promise => { } } if (cloneMissingReceiptsMap.size > 0) - Logger.mainLogger.debug( - 'Collecting missing receipts', - cloneMissingReceiptsMap.size, - cloneMissingReceiptsMap - ) + Logger.mainLogger.debug('Collecting missing receipts', cloneMissingReceiptsMap.size) for (const [txId, { txTimestamp, senders }] of cloneMissingReceiptsMap) { collectMissingReceipts(senders, txId, txTimestamp) } @@ -1184,11 +1180,7 @@ export const collectMissingTxDataFromArchivers = async (): Promise => { } } if (cloneMissingOriginalTxsMap.size > 0) - Logger.mainLogger.debug( - 'Collecting missing originalTxsData', - cloneMissingOriginalTxsMap.size, - cloneMissingOriginalTxsMap - ) + Logger.mainLogger.debug('Collecting missing originalTxsData', cloneMissingOriginalTxsMap.size) for (const [txId, { txTimestamp, senders }] of cloneMissingOriginalTxsMap) { collectMissingOriginalTxsData(senders, txId, txTimestamp) } @@ -1203,7 +1195,12 @@ export const collectMissingReceipts = async ( ): Promise => { const txIdList: [string, number][] = [[txId, txTimestamp]] let foundTxData = false - for (const sender of senders) { + const senderArchivers = State.activeArchivers.filter((archiver) => senders.includes(archiver.publicKey)) + Logger.mainLogger.debug( + `Collecting missing receipt for txId ${txId} with timestamp ${txTimestamp} from archivers`, + senderArchivers.map((a) => a.ip + ':' + a.port) + ) + for (const senderArchiver of senderArchivers) { if ( (processedReceiptsMap.has(txId) && processedReceiptsMap.get(txId) === txTimestamp) || (receiptsInValidationMap.has(txId) && receiptsInValidationMap.get(txId) === txTimestamp) @@ -1211,8 +1208,6 @@ export const collectMissingReceipts = async ( foundTxData = true break } - const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sender) - if (!senderArchiver) continue const receipts = (await queryTxDataFromArchivers( senderArchiver, DataType.RECEIPT, @@ -1244,7 +1239,12 @@ const collectMissingOriginalTxsData = async ( ): Promise => { const txIdList: [string, number][] = [[txId, txTimestamp]] let foundTxData = false - for (const sender of senders) { + const senderArchivers = State.activeArchivers.filter((archiver) => senders.includes(archiver.publicKey)) + Logger.mainLogger.debug( + `Collecting missing originalTxData for txId ${txId} with timestamp ${txTimestamp} from archivers`, + senderArchivers.map((a) => a.ip + ':' + a.port) + ) + for (const senderArchiver of senderArchivers) { if ( (processedOriginalTxsMap.has(txId) && processedOriginalTxsMap.get(txId) === txTimestamp) || (originalTxsInValidationMap.has(txId) && originalTxsInValidationMap.get(txId) === txTimestamp) @@ -1252,8 +1252,6 @@ const collectMissingOriginalTxsData = async ( foundTxData = true break } - const senderArchiver = State.activeArchivers.find((archiver) => archiver.publicKey === sender) - if (!senderArchiver) continue const originalTxs = (await queryTxDataFromArchivers( senderArchiver, DataType.ORIGINAL_TX_DATA, From b80f5449ea37772fae7f30878bd56c98d702b8ad Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 24 Jun 2024 22:24:20 +0545 Subject: [PATCH 12/51] Move saveOnlyGossipData and stopGossipTxData flags to the config --- src/Config.ts | 6 ++++++ src/Data/Collector.ts | 3 --- src/Data/Data.ts | 5 ++--- src/Data/GossipData.ts | 5 +---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index b8a76d8f..0111fb26 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -61,6 +61,10 @@ export interface Config { randomGossipArchiversCount: 2 // Number of random archivers to gossip to subscribeToMoreConsensors: boolean // To subscribe to more consensors when the number of active archivers is less than 4 extraConsensorsToSubscribe: 1 // Number of extra consensors to subscribe to + // For debugging gossip data, set this to true. This will save only the gossip data received from the gossip archivers. + saveOnlyGossipData: boolean + // For debugging purpose, set this to true to stop gossiping tx data + stopGossipTxData: boolean } let config: Config = { @@ -120,6 +124,8 @@ let config: Config = { randomGossipArchiversCount: 2, subscribeToMoreConsensors: true, extraConsensorsToSubscribe: 1, + saveOnlyGossipData: false, + stopGossipTxData: false, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index b342d388..33956898 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -47,9 +47,6 @@ interface MissingTx { senders: string[] } -// For debugging gossip data, set this to true. This will save only the gossip data received from the gossip archivers. -export const saveOnlyGossipData = false - type GET_TX_RECEIPT_RESPONSE = { success: boolean receipt?: Receipt.ArchiverReceipt | Receipt.AppliedReceipt2 diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 3088c130..0fecbb03 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -27,7 +27,6 @@ import { storeAccountData, storingAccountData, storeOriginalTxData, - saveOnlyGossipData, } from './Collector' import * as CycleDB from '../dbstore/cycles' import * as ReceiptDB from '../dbstore/receipts' @@ -258,7 +257,7 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void { storeOriginalTxData( newData.responses.ORIGINAL_TX_DATA, sender.nodeInfo.ip + ':' + sender.nodeInfo.port, - saveOnlyGossipData + config.saveOnlyGossipData ) } if (newData.responses && newData.responses.RECEIPT) { @@ -274,7 +273,7 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void { newData.responses.RECEIPT, sender.nodeInfo.ip + ':' + sender.nodeInfo.port, true, - saveOnlyGossipData + config.saveOnlyGossipData ) } if (newData.responses && newData.responses.CYCLE) { diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index faceb79e..0f761e58 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -24,9 +24,6 @@ export interface GossipData { sign: Signature } -// For debugging purpose, set this to true to stop gossiping tx data -const stopGossipTxData = false - // List of archivers that are not adjacent to the current archiver const remainingArchivers = [] @@ -69,7 +66,7 @@ export async function sendDataToAdjacentArchivers( dataType: DataType, data: GossipData['data'] ): Promise { - if (stopGossipTxData) return + if (config.stopGossipTxData) return if (State.otherArchivers.length === 0) return const gossipPayload = { dataType, From 7d6f97fa63afe3d81df591e1816cbf63268bb9be Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Tue, 18 Jun 2024 14:44:04 +0530 Subject: [PATCH 13/51] BLUE-118(feat): Config-patch endpoint added --- src/API.ts | 33 ++++++++++++++++++++++++++++++++- src/Config.ts | 10 ++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/API.ts b/src/API.ts index 0583cff4..3d0206fb 100644 --- a/src/API.ts +++ b/src/API.ts @@ -1,7 +1,7 @@ import { Signature } from '@shardus/crypto-utils' import { FastifyInstance, FastifyRequest } from 'fastify' import { Server, IncomingMessage, ServerResponse } from 'http' -import { config } from './Config' +import { config, updateConfig, Config as ConfigInterface } from './Config' import * as Crypto from './Crypto' import * as State from './State' import * as NodeList from './NodeList' @@ -868,6 +868,9 @@ export function registerRoutes(server: FastifyInstance + type ConfigPatchRequest = FastifyRequest<{ + Body: Partial + }> server.post('/get_account_data_archiver', async (_request: AccountDataRequest, reply) => { const payload = _request.body as AccountDataProvider.AccountDataRequestSchema @@ -922,6 +925,34 @@ export function registerRoutes(server: FastifyInstance { + isDebugMiddleware(_request, reply) + }, + }, + async (_request: ConfigPatchRequest, reply) => { + try { + const { sign, ...newConfig } = _request.body + const validKeys = new Set(Object.keys(config)) + const payloadKeys = Object.keys(newConfig) + const invalidKeys = payloadKeys.filter((key) => !validKeys.has(key)) + + if (invalidKeys.length > 0) + throw new Error(`Invalid config properties provided: ${invalidKeys.join(', ')}`) + + if (config.VERBOSE) + Logger.mainLogger.debug('Archiver config update executed: ', JSON.stringify(newConfig)) + + const updatedConfig = updateConfig(newConfig) + reply.send({ success: true, updatedConfig }) + } catch (error) { + reply.status(400).send({ success: false, reason: error.message }) + } + } + ) + // Config Endpoint server.get( '/config', diff --git a/src/Config.ts b/src/Config.ts index 0111fb26..e07faed1 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -242,4 +242,14 @@ export async function overrideDefaultConfig(file: string): Promise { } } +export function updateConfig(newConfig: Partial): Config { + for (const key in newConfig) { + if (newConfig[key] === 'true') newConfig[key] = true + if (newConfig[key] === 'false') newConfig[key] = false + if (!Number.isNaN(Number(newConfig[key]))) newConfig[key] = Number(newConfig[key]) + } + config = merge(config, newConfig) + return config +} + export { config } From 47b4aa234cfa60951a0ed48368f058da37ae231e Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Tue, 18 Jun 2024 21:29:48 +0530 Subject: [PATCH 14/51] BLUE-119(chore): updating Archiver config in-sync with the validator config --- src/Data/Data.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 0fecbb03..cf8efaa4 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -17,7 +17,7 @@ import { ChangeSquasher, parse, totalNodeCount, activeNodeCount, applyNodeListCh import * as State from '../State' import * as P2P from '../P2P' import * as Utils from '../Utils' -import { config } from '../Config' +import { config, updateConfig } from '../Config' import { P2P as P2PTypes } from '@shardus/types' import * as Logger from '../Logger' import { nestedCountersInstance } from '../profiler/nestedCounters' @@ -558,6 +558,12 @@ async function getConsensusRadius(): Promise { if (tallyItem && tallyItem.value && tallyItem.value.config) { nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge + const devPublicKeys = tallyItem.value.config.debug.devPublicKeys + const updateConfigProps = { + newPOQReceipt: tallyItem.value.config.stateManager.useNewPOQ, + DevPublicKey: Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3), + } + updateConfig(updateConfigProps) // Upgrading consensus size to an odd number if (nodesPerConsensusGroup % 2 === 0) nodesPerConsensusGroup++ const consensusRadius = Math.floor((nodesPerConsensusGroup - 1) / 2) From 0eb43385e87d9d302c638f9c609d7b244678c619 Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Tue, 18 Jun 2024 21:57:30 +0530 Subject: [PATCH 15/51] fix: minor fixes --- src/Config.ts | 7 ++++--- src/Data/Data.ts | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index e07faed1..22284980 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -245,9 +245,10 @@ export async function overrideDefaultConfig(file: string): Promise { export function updateConfig(newConfig: Partial): Config { for (const key in newConfig) { if (newConfig[key] === 'true') newConfig[key] = true - if (newConfig[key] === 'false') newConfig[key] = false - if (!Number.isNaN(Number(newConfig[key]))) newConfig[key] = Number(newConfig[key]) - } + else if (newConfig[key] === 'false') newConfig[key] = false + else if (typeof newConfig[key] !== 'boolean' && !Number.isNaN(Number(newConfig[key]))) + newConfig[key] = Number(newConfig[key])} + config = merge(config, newConfig) return config } diff --git a/src/Data/Data.ts b/src/Data/Data.ts index cf8efaa4..242c6e74 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -558,9 +558,9 @@ async function getConsensusRadius(): Promise { if (tallyItem && tallyItem.value && tallyItem.value.config) { nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge - const devPublicKeys = tallyItem.value.config.debug.devPublicKeys + const devPublicKeys = tallyItem.value.config.devPublicKeys const updateConfigProps = { - newPOQReceipt: tallyItem.value.config.stateManager.useNewPOQ, + newPOQReceipt: tallyItem.value.config.useNewPOQ, DevPublicKey: Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3), } updateConfig(updateConfigProps) From 9429be98816afd9aaa3d7ab7950845ef42366340 Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Wed, 19 Jun 2024 21:02:21 +0530 Subject: [PATCH 16/51] chore: refactoring /netconfig API call function --- src/Config.ts | 4 ++-- src/Data/Data.ts | 28 +++++++++++++++++----------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 22284980..9ab213ac 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -247,8 +247,8 @@ export function updateConfig(newConfig: Partial): Config { if (newConfig[key] === 'true') newConfig[key] = true else if (newConfig[key] === 'false') newConfig[key] = false else if (typeof newConfig[key] !== 'boolean' && !Number.isNaN(Number(newConfig[key]))) - newConfig[key] = Number(newConfig[key])} - + newConfig[key] = Number(newConfig[key]) + } config = merge(config, newConfig) return config } diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 242c6e74..a6f66d30 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -515,10 +515,7 @@ export function addDataSender(sender: DataSender): void { dataSenders.set(sender.nodeInfo.publicKey, sender) } -async function getConsensusRadius(): Promise { - // If there is no node, return existing currentConsensusRadius - if (NodeList.isEmpty()) return currentConsensusRadius - +async function syncFromNetworkConfig(): Promise { // Define the query function to get the network config from a node const queryFn = async (node): Promise => { const REQUEST_NETCONFIG_TIMEOUT_SECOND = 2 // 2s timeout @@ -533,7 +530,6 @@ async function getConsensusRadius(): Promise { return null } } - // Define the equality function to compare two responses const equalityFn = (responseA, responseB): boolean => { return ( @@ -541,11 +537,9 @@ async function getConsensusRadius(): Promise { responseB?.config?.sharding?.nodesPerConsensusGroup ) } - // Get the list of 10 max random active nodes or the first node if no active nodes are available const nodes = NodeList.getActiveNodeCount() > 0 ? NodeList.getRandomActiveNodes(10) : [NodeList.getFirstNode()] - // Use robustQuery to get the consensusRadius from multiple nodes const tallyItem = await robustQuery( nodes, @@ -554,16 +548,28 @@ async function getConsensusRadius(): Promise { 3 // Redundancy (minimum 3 nodes should return the same result to reach consensus) ) - // Check if a consensus was reached - if (tallyItem && tallyItem.value && tallyItem.value.config) { - nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup - nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge + if (tallyItem?.value?.config) { + // Updating the Archiver Config as per the latest Network Config const devPublicKeys = tallyItem.value.config.devPublicKeys const updateConfigProps = { newPOQReceipt: tallyItem.value.config.useNewPOQ, DevPublicKey: Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3), } updateConfig(updateConfigProps) + return tallyItem + } + return null +} + +async function getConsensusRadius(): Promise { + // If there is no node, return existing currentConsensusRadius + if (NodeList.isEmpty()) return currentConsensusRadius + + const tallyItem = await syncFromNetworkConfig() + // Check if a consensus was reached + if (tallyItem?.value?.config) { + nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge + nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup // Upgrading consensus size to an odd number if (nodesPerConsensusGroup % 2 === 0) nodesPerConsensusGroup++ const consensusRadius = Math.floor((nodesPerConsensusGroup - 1) / 2) From b4ecd33afdb4fba9b3ec70276ef82bfb295b785f Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Mon, 24 Jun 2024 21:12:18 +0530 Subject: [PATCH 17/51] chore: hardening added for some params in /set-config and type checks in updateConfig --- src/API.ts | 15 +++++-- src/Config.ts | 10 +++-- src/Data/Data.ts | 92 ++++++++++++++++++++++++++----------------- src/ShardFunctions.ts | 2 +- 4 files changed, 74 insertions(+), 45 deletions(-) diff --git a/src/API.ts b/src/API.ts index 3d0206fb..a76f5891 100644 --- a/src/API.ts +++ b/src/API.ts @@ -933,20 +933,29 @@ export function registerRoutes(server: FastifyInstance { + const RESTRICTED_PARAMS = [ + 'ARCHIVER_IP', + 'ARCHIVER_PORT', + 'ARCHIVER_HASH_KEY', + 'ARCHIVER_SECRET_KEY', + 'ARCHIVER_PUBLIC_KEY', + ] try { const { sign, ...newConfig } = _request.body const validKeys = new Set(Object.keys(config)) const payloadKeys = Object.keys(newConfig) - const invalidKeys = payloadKeys.filter((key) => !validKeys.has(key)) + const invalidKeys = payloadKeys.filter( + (key) => !validKeys.has(key) || RESTRICTED_PARAMS.includes(key) + ) if (invalidKeys.length > 0) - throw new Error(`Invalid config properties provided: ${invalidKeys.join(', ')}`) + throw new Error(`Invalid/Unauthorised config properties provided: ${invalidKeys.join(', ')}`) if (config.VERBOSE) Logger.mainLogger.debug('Archiver config update executed: ', JSON.stringify(newConfig)) const updatedConfig = updateConfig(newConfig) - reply.send({ success: true, updatedConfig }) + reply.send({ success: true, ...updatedConfig, ARCHIVER_SECRET_KEY: '' }) } catch (error) { reply.status(400).send({ success: false, reason: error.message }) } diff --git a/src/Config.ts b/src/Config.ts index 9ab213ac..1d49d0e6 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -244,10 +244,12 @@ export async function overrideDefaultConfig(file: string): Promise { export function updateConfig(newConfig: Partial): Config { for (const key in newConfig) { - if (newConfig[key] === 'true') newConfig[key] = true - else if (newConfig[key] === 'false') newConfig[key] = false - else if (typeof newConfig[key] !== 'boolean' && !Number.isNaN(Number(newConfig[key]))) - newConfig[key] = Number(newConfig[key]) + if (typeof newConfig[key] !== typeof config[key]) + throw new Error( + `Value with incorrect type passed to update the Archiver Config: ${key}:${ + newConfig[key] + } of type ${typeof newConfig[key]}` + ) } config = merge(config, newConfig) return config diff --git a/src/Data/Data.ts b/src/Data/Data.ts index a6f66d30..38a1d3c4 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -516,49 +516,57 @@ export function addDataSender(sender: DataSender): void { } async function syncFromNetworkConfig(): Promise { - // Define the query function to get the network config from a node - const queryFn = async (node): Promise => { - const REQUEST_NETCONFIG_TIMEOUT_SECOND = 2 // 2s timeout - try { - const response = await P2P.getJson( - `http://${node.ip}:${node.port}/netconfig`, - REQUEST_NETCONFIG_TIMEOUT_SECOND + try { + // Define the query function to get the network config from a node + const queryFn = async (node): Promise => { + const REQUEST_NETCONFIG_TIMEOUT_SECOND = 2 // 2s timeout + try { + const response = await P2P.getJson( + `http://${node.ip}:${node.port}/netconfig`, + REQUEST_NETCONFIG_TIMEOUT_SECOND + ) + return response + } catch (error) { + Logger.mainLogger.error(`Error querying node ${node.ip}:${node.port}: ${error}`) + return null + } + } + // Define the equality function to compare two responses + const equalityFn = (responseA, responseB): boolean => { + return ( + responseA?.config?.sharding?.nodesPerConsensusGroup === + responseB?.config?.sharding?.nodesPerConsensusGroup ) - return response - } catch (error) { - Logger.mainLogger.error(`Error querying node ${node.ip}:${node.port}: ${error}`) - return null } - } - // Define the equality function to compare two responses - const equalityFn = (responseA, responseB): boolean => { - return ( - responseA?.config?.sharding?.nodesPerConsensusGroup === - responseB?.config?.sharding?.nodesPerConsensusGroup + // Get the list of 10 max random active nodes or the first node if no active nodes are available + const nodes = + NodeList.getActiveNodeCount() > 0 ? NodeList.getRandomActiveNodes(10) : [NodeList.getFirstNode()] + // Use robustQuery to get the consensusRadius from multiple nodes + const tallyItem = await robustQuery( + nodes, + queryFn, + equalityFn, + 3 // Redundancy (minimum 3 nodes should return the same result to reach consensus) ) - } - // Get the list of 10 max random active nodes or the first node if no active nodes are available - const nodes = - NodeList.getActiveNodeCount() > 0 ? NodeList.getRandomActiveNodes(10) : [NodeList.getFirstNode()] - // Use robustQuery to get the consensusRadius from multiple nodes - const tallyItem = await robustQuery( - nodes, - queryFn, - equalityFn, - 3 // Redundancy (minimum 3 nodes should return the same result to reach consensus) - ) - if (tallyItem?.value?.config) { - // Updating the Archiver Config as per the latest Network Config - const devPublicKeys = tallyItem.value.config.devPublicKeys - const updateConfigProps = { - newPOQReceipt: tallyItem.value.config.useNewPOQ, - DevPublicKey: Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3), + if (tallyItem?.value?.config) { + // Updating the Archiver Config as per the latest Network Config + const devPublicKeys = tallyItem.value.config?.devPublicKeys + const updateConfigProps = { + newPOQReceipt: tallyItem.value.config?.useNewPOQ, + DevPublicKey: + devPublicKeys && Object.keys(devPublicKeys).length >= 3 + ? Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) + : '', + } + updateConfig(updateConfigProps) + return tallyItem } - updateConfig(updateConfigProps) - return tallyItem + return null + } catch (error) { + Logger.mainLogger.error('❌ Error in syncFromNetworkConfig: ', error) + return null } - return null } async function getConsensusRadius(): Promise { @@ -570,6 +578,16 @@ async function getConsensusRadius(): Promise { if (tallyItem?.value?.config) { nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup + + if (!Number.isInteger(nodesPerConsensusGroup) || nodesPerConsensusGroup <= 0) { + Logger.mainLogger.error('nodesPerConsensusGroup is not a valid number:', nodesPerConsensusGroup) + return currentConsensusRadius + } + + if (!Number.isInteger(nodesPerEdge) || nodesPerEdge <= 0) { + Logger.mainLogger.error('nodesPerEdge is not a valid number:', nodesPerEdge) + return currentConsensusRadius + } // Upgrading consensus size to an odd number if (nodesPerConsensusGroup % 2 === 0) nodesPerConsensusGroup++ const consensusRadius = Math.floor((nodesPerConsensusGroup - 1) / 2) diff --git a/src/ShardFunctions.ts b/src/ShardFunctions.ts index 616b15d7..d0d62564 100644 --- a/src/ShardFunctions.ts +++ b/src/ShardFunctions.ts @@ -52,7 +52,7 @@ class ShardFunctions { //make sure nodesPerConsenusGroup is an odd number >= 3 if (nodesPerConsenusGroup % 2 === 0 || nodesPerConsenusGroup < 3) { - throw new Error(`nodesPerConsenusGroup:${nodesPerConsenusGroup} must be odd and >= 3`) + throw new Error(`nodesPerConsenusGroup: ${nodesPerConsenusGroup} must be odd and >= 3`) } shardGlobals.consensusRadius = Math.floor((nodesPerConsenusGroup - 1) / 2) From f8cdcb0ea22c411fba5ffa089413e1139071e77b Mon Sep 17 00:00:00 2001 From: Achal Singh Date: Mon, 24 Jun 2024 22:40:58 +0530 Subject: [PATCH 18/51] chore: Update-Config script added --- scripts/update_config.ts | 50 ++++++++++++++++++++++++++++++++++++++++ src/API.ts | 4 ++++ 2 files changed, 54 insertions(+) create mode 100644 scripts/update_config.ts diff --git a/scripts/update_config.ts b/scripts/update_config.ts new file mode 100644 index 00000000..f78657a4 --- /dev/null +++ b/scripts/update_config.ts @@ -0,0 +1,50 @@ +import axios from 'axios' +import { join } from 'path' +import { Utils } from '@shardus/types' +import * as crypto from '@shardus/crypto-utils' +import { config, overrideDefaultConfig } from '../src/Config' + +const configFile = join(process.cwd(), 'archiver-config.json') +overrideDefaultConfig(configFile) + +crypto.init(config.ARCHIVER_HASH_KEY) + +const DEV_KEYS = { + pk: config.ARCHIVER_PUBLIC_KEY, + sk: config.ARCHIVER_SECRET_KEY, +} + +function sign(obj: T, sk: string, pk: string): T & any { + const objCopy = JSON.parse(crypto.stringify(obj)) + crypto.signObj(objCopy, sk, pk) + return objCopy +} + +function createSignature(data: any, pk: string, sk: string): any { + return sign({ ...data }, sk, pk) +} + +const UPDATE_CONFIG = { + /* Add Config properties that need to be updated here */ + VERBOSE: true, + RATE_LIMIT: 200, +} + +const INPUT = Utils.safeStringify(createSignature(UPDATE_CONFIG, DEV_KEYS.pk, DEV_KEYS.sk)) + +axios + .patch('http://127.0.0.1:4000/set-config', INPUT, { + headers: { + 'Content-Type': 'application/json', + }, + }) + .then((response) => { + console.log(response.data) + }) + .catch((error) => { + if (error.response) { + console.error(error.response) + } else { + console.error(error.message) + } + }) diff --git a/src/API.ts b/src/API.ts index a76f5891..5335754c 100644 --- a/src/API.ts +++ b/src/API.ts @@ -925,6 +925,10 @@ export function registerRoutes(server: FastifyInstance Date: Tue, 25 Jun 2024 12:51:49 +0545 Subject: [PATCH 19/51] Update devPublicKey and newPOQReceipt if only exist and not the same --- src/Config.ts | 1 + src/Data/Data.ts | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 1d49d0e6..a8120034 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -252,6 +252,7 @@ export function updateConfig(newConfig: Partial): Config { ) } config = merge(config, newConfig) + Logger.mainLogger.info('Updated Archiver Config:', config) return config } diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 38a1d3c4..373e7b92 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -552,14 +552,20 @@ async function syncFromNetworkConfig(): Promise { if (tallyItem?.value?.config) { // Updating the Archiver Config as per the latest Network Config const devPublicKeys = tallyItem.value.config?.devPublicKeys - const updateConfigProps = { - newPOQReceipt: tallyItem.value.config?.useNewPOQ, - DevPublicKey: - devPublicKeys && Object.keys(devPublicKeys).length >= 3 - ? Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) - : '', - } - updateConfig(updateConfigProps) + const devPublicKey = devPublicKeys && Object.keys(devPublicKeys).length >= 3 && devPublicKeys[3] + const newPOQReceipt = tallyItem.value.config?.useNewPOQ + if ( + devPublicKey && + typeof devPublicKey === typeof config.DevPublicKey && + devPublicKey !== config.DevPublicKey + ) + updateConfig({ DevPublicKey: devPublicKey }) + if ( + newPOQReceipt !== null && + typeof newPOQReceipt === typeof config.newPOQReceipt && + newPOQReceipt !== config.newPOQReceipt + ) + updateConfig({ newPOQReceipt }) return tallyItem } return null @@ -576,8 +582,8 @@ async function getConsensusRadius(): Promise { const tallyItem = await syncFromNetworkConfig() // Check if a consensus was reached if (tallyItem?.value?.config) { - nodesPerEdge = tallyItem.value.config.sharding.nodesPerEdge - nodesPerConsensusGroup = tallyItem.value.config.sharding.nodesPerConsensusGroup + nodesPerEdge = tallyItem.value.config.sharding?.nodesPerEdge + nodesPerConsensusGroup = tallyItem.value.config?.sharding.nodesPerConsensusGroup if (!Number.isInteger(nodesPerConsensusGroup) || nodesPerConsensusGroup <= 0) { Logger.mainLogger.error('nodesPerConsensusGroup is not a valid number:', nodesPerConsensusGroup) From 3f6b0596e4989dcad8995e840eee830ae66ff444 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Tue, 25 Jun 2024 14:30:33 +0545 Subject: [PATCH 20/51] Fix devKey extraction from validator netconfig; --- src/Data/Data.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 373e7b92..f74bd1f5 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -552,7 +552,10 @@ async function syncFromNetworkConfig(): Promise { if (tallyItem?.value?.config) { // Updating the Archiver Config as per the latest Network Config const devPublicKeys = tallyItem.value.config?.devPublicKeys - const devPublicKey = devPublicKeys && Object.keys(devPublicKeys).length >= 3 && devPublicKeys[3] + const devPublicKey = + devPublicKeys && + Object.keys(devPublicKeys).length >= 3 && + Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) const newPOQReceipt = tallyItem.value.config?.useNewPOQ if ( devPublicKey && @@ -561,7 +564,7 @@ async function syncFromNetworkConfig(): Promise { ) updateConfig({ DevPublicKey: devPublicKey }) if ( - newPOQReceipt !== null && + !Utils.isUndefined(newPOQReceipt) && typeof newPOQReceipt === typeof config.newPOQReceipt && newPOQReceipt !== config.newPOQReceipt ) From 0725b86db8f34a226076c93a1a8b064cdd1ea2f4 Mon Sep 17 00:00:00 2001 From: chrypnotoad Date: Tue, 9 Jul 2024 16:54:04 -0500 Subject: [PATCH 21/51] Create ci.yml (#30) Co-authored-by: municorn --- .github/workflows/ci.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..58d5b64b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,36 @@ +name: Node CI Workflow +# The parameters are defaulted at the org level but can be overridden on the repository. +on: + push: + branches: + - dev + - main + pull_request: + branches: + - dev + - main + workflow_dispatch: + inputs: + workflowBranch: + description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.' + required: true + default: 'main' + type: choice + options: + - dev + - main + +permissions: + issues: write + pull-requests: write + +jobs: + ci-dev: + if: ${{ github.event.inputs.workflowBranch == 'dev' }} + uses: shardeum/github-automation/.github/workflows/node-ci-shared.yml@dev + secrets: inherit + + ci-main: + if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }} + uses: shardeum/github-automation/.github/workflows/node-ci-shared.yml@main + secrets: inherit From 8270da784f4fd27eb8d73fd2c54b777b14e353c9 Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Tue, 9 Jul 2024 17:03:20 -0500 Subject: [PATCH 22/51] SYS-188 add github ci (#48) --- .github/workflows/ci.yml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58d5b64b..bd81bdf2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,8 @@ name: Node CI Workflow -# The parameters are defaulted at the org level but can be overridden on the repository. +# +# The parameters are defaulted at the org level but can be overridden on the repository. +# See the github-automation repo for more documentation +# on: push: branches: @@ -25,12 +28,37 @@ permissions: pull-requests: write jobs: + echo-inputs: + name: Repo Workflow Debugging + runs-on: ubuntu-latest + steps: + - name: Check Repo Vars + run: | + echo "*** Start - Check inputs in repo workflow ***" + echo "Node Version: ${{ vars.NODE_VERSION }}" + echo "Lint Required: ${{ vars.IS_LINT_REQUIRED }}" + echo "Format Check Required: ${{ vars.IS_FORMAT_CHECK_REQUIRED }}" + echo "Apply Patches Required: ${{ vars.IS_APPLY_PATCHES_REQUIRED }}" + echo "Unit Tests Required: ${{ vars.IS_UNIT_TESTS_REQUIRED }}" + echo "*** End - Check inputs in repo workflow ***" ci-dev: if: ${{ github.event.inputs.workflowBranch == 'dev' }} - uses: shardeum/github-automation/.github/workflows/node-ci-shared.yml@dev + uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev + with: + node-version: ${{ vars.NODE_VERSION }} + lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }} + format-check-required: ${{ vars.IS_FORMAT_CHECK_REQUIRED == 'true' }} + apply-patches-required: ${{ vars.IS_APPLY_PATCHES_REQUIRED == 'true' }} + unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }} secrets: inherit ci-main: if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }} - uses: shardeum/github-automation/.github/workflows/node-ci-shared.yml@main + uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main + with: + node-version: ${{ vars.NODE_VERSION }} + lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }} + format-check-required: ${{ vars.IS_FORMAT_CHECK_REQUIRED == 'true' }} + apply-patches-required: ${{ vars.IS_APPLY_PATCHES_REQUIRED == 'true' }} + unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }} secrets: inherit From edc0074f5d366b562334047df488c056572dc87f Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Tue, 9 Jul 2024 18:20:21 -0500 Subject: [PATCH 23/51] fixing issue comment Pr review stuff --- .github/workflows/ci.yml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd81bdf2..49ff74bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,16 @@ on: branches: - dev - main + issue_comment: + inputs: + workflowBranch: + description: 'Branch of the reusable workflow. Defaults to main, select dev for testing only.' + required: true + default: 'main' + type: choice + options: + - dev + - main workflow_dispatch: inputs: workflowBranch: @@ -22,11 +32,6 @@ on: options: - dev - main - -permissions: - issues: write - pull-requests: write - jobs: echo-inputs: name: Repo Workflow Debugging @@ -44,6 +49,10 @@ jobs: ci-dev: if: ${{ github.event.inputs.workflowBranch == 'dev' }} uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev + permissions: + issues: write + pull-requests: write + contents: write with: node-version: ${{ vars.NODE_VERSION }} lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }} @@ -55,6 +64,10 @@ jobs: ci-main: if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }} uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main + permissions: + issues: write + pull-requests: write + contents: write with: node-version: ${{ vars.NODE_VERSION }} lint-required: ${{ vars.IS_LINT_REQUIRED == 'true' }} From b8f3dcfe458a7384ad8d7bd96d3c0e693971c2d3 Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Fri, 19 Jul 2024 12:13:53 -0500 Subject: [PATCH 24/51] add codeowners and sync ci.yml --- .github/workflows/ci.yml | 4 ++-- CODEOWNERS | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 CODEOWNERS diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49ff74bb..4597d442 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: echo "Apply Patches Required: ${{ vars.IS_APPLY_PATCHES_REQUIRED }}" echo "Unit Tests Required: ${{ vars.IS_UNIT_TESTS_REQUIRED }}" echo "*** End - Check inputs in repo workflow ***" - ci-dev: + ci-test-only: if: ${{ github.event.inputs.workflowBranch == 'dev' }} uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@dev permissions: @@ -61,7 +61,7 @@ jobs: unit-tests-required: ${{ vars.IS_UNIT_TESTS_REQUIRED == 'true' }} secrets: inherit - ci-main: + ci: if: ${{ github.event.inputs.workflowBranch == 'main' || !github.event.inputs.workflowBranch }} uses: shardeum/github-automation/.github/workflows/reusable-node-ci.yml@main permissions: diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000..200b7372 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,13 @@ +# CODEOWNERS file +# To add additional teams to any approval, include them on the same line separated by spaces +# It is best practice to assign a team as a code owner and not an invidual. +# Please submit requests for new teams to Systems and Automation + +# Global approval (all files) +# * @shardeum/team-name + +# Directory-level approval +/.github/ @shardeum/systems-and-automation + +# Specific file rules +# README.md @shardeum/team-name From 318f403b3c16d911fb413626664977755322971b Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Fri, 19 Jul 2024 12:39:16 -0500 Subject: [PATCH 25/51] fix lint and add test scripts --- package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index a3bf835b..049da770 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,11 @@ "scripts": { "start": "npm run prepare && node build/server.js", "release": "npm run prepare && np --no-cleanup --no-tests --no-yarn --any-branch", - "test": "echo \"Error: no test specified\" && exit 1", "check": "gts check", "clean": "npm-run-all clean:*", "clean:typescript": "gts clean", - "lint": "eslint './src/**/*.ts'", - "lint-windows": "eslint ./src/**/*.ts", + "lint": "eslint \"./src/**/*.ts\"", + "test": "echo \"Error: no test specified\" && exit 1", "format-check": "prettier --check './src/**/*.ts'", "format-fix": "prettier --write './src/**/*.ts'", "clean:artifacts": "shx rm -rf archiver-logs/ archiver-db/ data-logs/", From 776a128a3af48f0537ba4138c536294aa14d9a9d Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Fri, 19 Jul 2024 12:50:30 -0500 Subject: [PATCH 26/51] prettier fixes --- src/dbstore/types.ts | 12 ++++++------ src/profiler/StringifyReduce.ts | 2 +- src/sync-v2/queries.ts | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/dbstore/types.ts b/src/dbstore/types.ts index 179bc525..fabc7ef0 100644 --- a/src/dbstore/types.ts +++ b/src/dbstore/types.ts @@ -1,11 +1,11 @@ -import { P2P, StateManager } from "@shardus/types" +import { P2P, StateManager } from '@shardus/types' export interface Cycle { - counter: P2P.CycleCreatorTypes.CycleData['counter'] - cycleRecord: P2P.CycleCreatorTypes.CycleData - cycleMarker: StateManager.StateMetaDataTypes.CycleMarker + counter: P2P.CycleCreatorTypes.CycleData['counter'] + cycleRecord: P2P.CycleCreatorTypes.CycleData + cycleMarker: StateManager.StateMetaDataTypes.CycleMarker } export type DbCycle = Cycle & { - cycleRecord: string -} \ No newline at end of file + cycleRecord: string +} diff --git a/src/profiler/StringifyReduce.ts b/src/profiler/StringifyReduce.ts index f0092ee8..6f24eb0a 100644 --- a/src/profiler/StringifyReduce.ts +++ b/src/profiler/StringifyReduce.ts @@ -1,4 +1,4 @@ -import { Utils as StringUtils } from "@shardus/types" +import { Utils as StringUtils } from '@shardus/types' export const makeShortHash = (x: string, n = 4): string => { if (!x) { diff --git a/src/sync-v2/queries.ts b/src/sync-v2/queries.ts index 46e1b96b..82731e86 100644 --- a/src/sync-v2/queries.ts +++ b/src/sync-v2/queries.ts @@ -119,7 +119,9 @@ function attemptSimpleFetch( } /** Executes a robust query to retrieve the cycle marker from the network. */ -export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync<{ currentCycleHash: hexstring }> { +export function robustQueryForCycleRecordHash( + nodes: ActiveNode[] +): RobustQueryResultAsync<{ currentCycleHash: hexstring }> { return makeRobustQueryCall(nodes, 'current-cycle-hash') } From 01faa5e87e0792afb1a97b8345b5b15eddf9dbb7 Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Fri, 19 Jul 2024 19:08:54 -0500 Subject: [PATCH 27/51] cleanup gitlab.ci --- .gitlab-ci.yml | 69 -------------------------------------------------- 1 file changed, 69 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 1de80e8e..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,69 +0,0 @@ -# You can override the included template(s) by including variable overrides -# SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings -# Secret Detection customization: https://docs.gitlab.com/ee/user/application_security/secret_detection/#customizing-settings -# Dependency Scanning customization: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#customizing-the-dependency-scanning-settings -# Container Scanning customization: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings -# Note that environment variables can be set in several places -# See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence - -# Include security-related templates -include: - - template: Security/Dependency-Scanning.gitlab-ci.yml - - template: Security/Secret-Detection.gitlab-ci.yml - - template: Security/SAST.gitlab-ci.yml - - remote: 'https://gitlab.com/pod_security/shared-ci/-/raw/main/security.yml' - -# Define the default Docker image for all jobs -image: registry.gitlab.com/shardus/dev-container - -# Define global cache settings for all jobs -cache: - key: '$CI_COMMIT_REF_SLUG-node-modules' - paths: - - node_modules/ - -# Define the stages for the pipeline -stages: - - prepare - - build - - appsec - - test - -# Prepare Job: Install Node.js dependencies -prepare-job: - stage: prepare - script: - - npm ci - -# Build Job: Compiles the code -compile-job: - stage: build - needs: ['prepare-job'] - script: - - echo "Running Compiler..." - - npm run compile - - echo "Compilation complete." - -# Format Checker Job: Runs Prettier for code formatting -format-checker-job: - stage: build - needs: ['prepare-job'] - script: - - echo "Running Prettier..." - - npm run format-check - - echo "Running Prettier complete." - -# Lint Checker Job: Runs ESlint for code linting -lint-checker-job: - stage: build - needs: ['prepare-job'] - script: - - echo "Running ESlint..." - - npm run lint - - echo "Running ESlint complete." - -# SAST Job: Performs static application security testing -sast: - variables: - SAST_EXCLUDED_ANALYZERS: bandit, brakeman, flawfinder, gosec, kubesec, phpcs-security-audit, pmd-apex, security-code-scan, semgrep, sobelow, spotbugs - stage: test From 957856d71b12ad19a2e0aa9d8321d157f78a227f Mon Sep 17 00:00:00 2001 From: Caralee Jackson Date: Mon, 22 Jul 2024 12:19:25 -0500 Subject: [PATCH 28/51] Revert "prettier fixes" This reverts commit 776a128a3af48f0537ba4138c536294aa14d9a9d. --- src/dbstore/types.ts | 12 ++++++------ src/profiler/StringifyReduce.ts | 2 +- src/sync-v2/queries.ts | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/dbstore/types.ts b/src/dbstore/types.ts index fabc7ef0..179bc525 100644 --- a/src/dbstore/types.ts +++ b/src/dbstore/types.ts @@ -1,11 +1,11 @@ -import { P2P, StateManager } from '@shardus/types' +import { P2P, StateManager } from "@shardus/types" export interface Cycle { - counter: P2P.CycleCreatorTypes.CycleData['counter'] - cycleRecord: P2P.CycleCreatorTypes.CycleData - cycleMarker: StateManager.StateMetaDataTypes.CycleMarker + counter: P2P.CycleCreatorTypes.CycleData['counter'] + cycleRecord: P2P.CycleCreatorTypes.CycleData + cycleMarker: StateManager.StateMetaDataTypes.CycleMarker } export type DbCycle = Cycle & { - cycleRecord: string -} + cycleRecord: string +} \ No newline at end of file diff --git a/src/profiler/StringifyReduce.ts b/src/profiler/StringifyReduce.ts index 6f24eb0a..f0092ee8 100644 --- a/src/profiler/StringifyReduce.ts +++ b/src/profiler/StringifyReduce.ts @@ -1,4 +1,4 @@ -import { Utils as StringUtils } from '@shardus/types' +import { Utils as StringUtils } from "@shardus/types" export const makeShortHash = (x: string, n = 4): string => { if (!x) { diff --git a/src/sync-v2/queries.ts b/src/sync-v2/queries.ts index 82731e86..46e1b96b 100644 --- a/src/sync-v2/queries.ts +++ b/src/sync-v2/queries.ts @@ -119,9 +119,7 @@ function attemptSimpleFetch( } /** Executes a robust query to retrieve the cycle marker from the network. */ -export function robustQueryForCycleRecordHash( - nodes: ActiveNode[] -): RobustQueryResultAsync<{ currentCycleHash: hexstring }> { +export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync<{ currentCycleHash: hexstring }> { return makeRobustQueryCall(nodes, 'current-cycle-hash') } From 5f61735113b436c4bfcbc9a4ea651f2c0e2e98f8 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 4 Jul 2024 20:02:35 +0545 Subject: [PATCH 29/51] Added the voteHashCalculation for new POQ-O receipt --- src/Config.ts | 2 ++ src/Data/Collector.ts | 27 ++++++++++++++++++++++++++- src/Data/Cycles.ts | 6 +++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index a8120034..9fcd6049 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -65,6 +65,7 @@ export interface Config { saveOnlyGossipData: boolean // For debugging purpose, set this to true to stop gossiping tx data stopGossipTxData: boolean + usePOQo: boolean } let config: Config = { @@ -126,6 +127,7 @@ let config: Config = { extraConsensorsToSubscribe: 1, saveOnlyGossipData: false, stopGossipTxData: false, + usePOQo: true, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 33956898..f45dce5d 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -581,7 +581,32 @@ export const verifyReceiptData = async ( } const calculateVoteHash = (vote: Receipt.AppliedVote): string => { - return Crypto.hashObj({ ...vote, node_id: '' }) + try { + if (config.usePOQo === true) { + const appliedHash = { + applied: vote.transaction_result, + cantApply: vote.cant_apply, + } + const stateHash = { + account_id: vote.account_id, + account_state_hash_after: vote.account_state_hash_after, + account_state_hash_before: vote.account_state_hash_before, + } + const appDataHash = { + app_data_hash: vote.app_data_hash, + } + const voteToHash = { + appliedHash: Crypto.hashObj(appliedHash), + stateHash: Crypto.hashObj(stateHash), + appDataHash: Crypto.hashObj(appDataHash), + } + return Crypto.hashObj(voteToHash) + } + return Crypto.hashObj({ ...vote, node_id: '' }) + } catch { + Logger.mainLogger.error('Error in calculateVoteHash', vote) + return '' + } } export const storeReceiptData = async ( diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index 005eb41f..25748446 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -52,7 +52,7 @@ export let cycleRecordWithShutDownMode = null as P2PTypes.CycleCreatorTypes.Cycl export let currentNetworkMode: P2PTypes.ModesTypes.Record['mode'] = 'forming' export const shardValuesByCycle = new Map() -const CYCLE_SHARD_STORAGE_LIMIT = 3 +const CYCLE_SHARD_STORAGE_LIMIT = 20 export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData[]): Promise { if (profilerInstance) profilerInstance.profileSectionStart('process_cycle', false) @@ -93,8 +93,8 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData setShutdownCycleRecord(cycle) NodeList.toggleFirstNode() } - // Clean receipts/originalTxs cache that are older than 5 minutes - const cleanupTimestamp = Date.now() - 5 * 60 * 1000 + // Clean receipts/originalTxs cache that are older than 20 minutes to match with CYCLE_SHARD_STORAGE_LIMIT ( actual fix is on another branch ) + const cleanupTimestamp = Date.now() - 20 * 60 * 1000 cleanOldOriginalTxsMap(cleanupTimestamp) cleanOldReceiptsMap(cleanupTimestamp) } From 4bd1610a4781b35af00f7412a1bd84b25e7a97aa Mon Sep 17 00:00:00 2001 From: jairajdev Date: Tue, 9 Jul 2024 15:21:48 +0545 Subject: [PATCH 30/51] Added requiredVotesPercentage config --- src/Config.ts | 3 +++ src/Data/Collector.ts | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Config.ts b/src/Config.ts index 9fcd6049..5641109d 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -66,6 +66,8 @@ export interface Config { // For debugging purpose, set this to true to stop gossiping tx data stopGossipTxData: boolean usePOQo: boolean + // The percentage of votes required to confirm transaction + requiredVotesPercentage: number } let config: Config = { @@ -128,6 +130,7 @@ let config: Config = { saveOnlyGossipData: false, stopGossipTxData: false, usePOQo: true, + requiredVotesPercentage: 2 / 3, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index f45dce5d..6f7b4f9c 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -405,7 +405,10 @@ export const verifyReceiptData = async ( if (votingGroupCount > cycleShardData.nodes.length) { votingGroupCount = cycleShardData.nodes.length } - const requiredSignatures = Math.round(votingGroupCount * (2 / 3)) + const requiredSignatures = + config.usePOQo === true + ? Math.ceil(votingGroupCount * config.requiredVotesPercentage) + : Math.round(votingGroupCount * config.requiredVotesPercentage) if (signatures.length < requiredSignatures) { Logger.mainLogger.error( `Invalid receipt appliedReceipt signatures count is less than requiredSignatures, ${signatures.length}, ${requiredSignatures}` From 1feccc0364c3df94edf8187bbab2e33643dc6245 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 19 Jul 2024 18:20:42 +0545 Subject: [PATCH 31/51] Updated to use single index for each field --- src/dbstore/index.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts index 9d5f8977..bed6256d 100644 --- a/src/dbstore/index.ts +++ b/src/dbstore/index.ts @@ -7,7 +7,10 @@ export const initializeDB = async (config: Config): Promise => { 'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)' ) await db.runCreate( - 'CREATE INDEX if not exists `transactions_idx` ON `transactions` (`cycleNumber` DESC, `timestamp` DESC)' + 'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` DESC)' + ) + await db.runCreate( + 'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` DESC)' ) await db.runCreate( 'CREATE INDEX if not exists `transactions_appReceiptId_idx` ON `transactions` (`appReceiptId`)' @@ -19,19 +22,20 @@ export const initializeDB = async (config: Config): Promise => { await db.runCreate( 'CREATE TABLE if not exists `accounts` (`accountId` TEXT NOT NULL UNIQUE PRIMARY KEY, `data` JSON NOT NULL, `timestamp` BIGINT NOT NULL, `hash` TEXT NOT NULL, `cycleNumber` NUMBER NOT NULL, `isGlobal` BOOLEAN NOT NULL)' ) - await db.runCreate( - 'CREATE INDEX if not exists `accounts_idx` ON `accounts` (`cycleNumber` DESC, `timestamp` DESC)' - ) + await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` DESC)') + await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` DESC)') await db.runCreate( 'CREATE TABLE if not exists `receipts` (`receiptId` TEXT NOT NULL UNIQUE PRIMARY KEY, `tx` JSON NOT NULL, `cycle` NUMBER NOT NULL, `timestamp` BIGINT NOT NULL, `beforeStateAccounts` JSON, `accounts` JSON NOT NULL, `appliedReceipt` JSON NOT NULL, `appReceiptData` JSON, `executionShardKey` TEXT NOT NULL, `globalModification` BOOLEAN NOT NULL)' ) - await db.runCreate('CREATE INDEX if not exists `receipts_idx` ON `receipts` (`cycle` ASC, `timestamp` ASC)') + await db.runCreate('CREATE INDEX if not exists `receipts_cycle` ON `receipts` (`cycle` ASC)') + await db.runCreate('CREATE INDEX if not exists `receipts_timestamp` ON `receipts` (`timestamp` ASC)') + await db.runCreate( 'CREATE TABLE if not exists `originalTxsData` (`txId` TEXT NOT NULL, `timestamp` BIGINT NOT NULL, `cycle` NUMBER NOT NULL, `originalTxData` JSON NOT NULL, PRIMARY KEY (`txId`, `timestamp`))' ) - // await db.runCreate('Drop INDEX if exists `originalTxData_idx`'); + await db.runCreate('CREATE INDEX if not exists `originalTxsData_cycle` ON `originalTxsData` (`cycle` ASC)') await db.runCreate( - 'CREATE INDEX if not exists `originalTxsData_idx` ON `originalTxsData` (`cycle` ASC, `timestamp` ASC)' + 'CREATE INDEX if not exists `originalTxsData_timestamp` ON `originalTxsData` (`timestamp` ASC)' ) await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId_idx` ON `originalTxsData` (`txId`)') } From 9916d909f6bfe59c1672a839644943020cac3ab0 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 19 Jul 2024 19:51:55 +0545 Subject: [PATCH 32/51] rename originalTxsData_txId index --- src/dbstore/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts index bed6256d..1bb31108 100644 --- a/src/dbstore/index.ts +++ b/src/dbstore/index.ts @@ -37,7 +37,7 @@ export const initializeDB = async (config: Config): Promise => { await db.runCreate( 'CREATE INDEX if not exists `originalTxsData_timestamp` ON `originalTxsData` (`timestamp` ASC)' ) - await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId_idx` ON `originalTxsData` (`txId`)') + await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId` ON `originalTxsData` (`txId`)') } export const closeDatabase = async (): Promise => { From 0512864e597827f54bdb47fc0a7bbda48286f317 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 19 Jul 2024 20:11:18 +0545 Subject: [PATCH 33/51] Updated to use ascending order index --- src/dbstore/index.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts index 1bb31108..0e5ae727 100644 --- a/src/dbstore/index.ts +++ b/src/dbstore/index.ts @@ -7,10 +7,10 @@ export const initializeDB = async (config: Config): Promise => { 'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)' ) await db.runCreate( - 'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` DESC)' + 'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` ASC)' ) await db.runCreate( - 'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` DESC)' + 'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` ASC)' ) await db.runCreate( 'CREATE INDEX if not exists `transactions_appReceiptId_idx` ON `transactions` (`appReceiptId`)' @@ -18,12 +18,12 @@ export const initializeDB = async (config: Config): Promise => { await db.runCreate( 'CREATE TABLE if not exists `cycles` (`cycleMarker` TEXT NOT NULL UNIQUE PRIMARY KEY, `counter` NUMBER NOT NULL, `cycleRecord` JSON NOT NULL)' ) - await db.runCreate('CREATE INDEX if not exists `cycles_idx` ON `cycles` (`counter` DESC)') + await db.runCreate('CREATE INDEX if not exists `cycles_idx` ON `cycles` (`counter` ASC)') await db.runCreate( 'CREATE TABLE if not exists `accounts` (`accountId` TEXT NOT NULL UNIQUE PRIMARY KEY, `data` JSON NOT NULL, `timestamp` BIGINT NOT NULL, `hash` TEXT NOT NULL, `cycleNumber` NUMBER NOT NULL, `isGlobal` BOOLEAN NOT NULL)' ) - await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` DESC)') - await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` DESC)') + await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` ASC)') + await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` ASC)') await db.runCreate( 'CREATE TABLE if not exists `receipts` (`receiptId` TEXT NOT NULL UNIQUE PRIMARY KEY, `tx` JSON NOT NULL, `cycle` NUMBER NOT NULL, `timestamp` BIGINT NOT NULL, `beforeStateAccounts` JSON, `accounts` JSON NOT NULL, `appliedReceipt` JSON NOT NULL, `appReceiptData` JSON, `executionShardKey` TEXT NOT NULL, `globalModification` BOOLEAN NOT NULL)' ) From 732ab1e94d046605e4f1d0377b1bb57f9e3feef4 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 19 Jul 2024 20:22:28 +0545 Subject: [PATCH 34/51] Added db profiler --- src/dbstore/sqlite3storage.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/dbstore/sqlite3storage.ts b/src/dbstore/sqlite3storage.ts index c5302e4b..667ff841 100644 --- a/src/dbstore/sqlite3storage.ts +++ b/src/dbstore/sqlite3storage.ts @@ -22,6 +22,13 @@ export async function init(config: Config): Promise { } }) await run('PRAGMA journal_mode=WAL') + db.on('profile', (sql, time) => { + if (time > 500 && time < 1000) { + console.log('SLOW QUERY', sql, time) + } else if (time > 1000) { + console.log('VERY SLOW QUERY', sql, time) + } + }) console.log('Database initialized.') } From c1c4201e175386105839c10b0403758e03be6b1b Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 18 Jul 2024 23:41:31 +0545 Subject: [PATCH 35/51] Fix StringUtils issue in the update-network-account script --- scripts/update_network_account.ts | 1 + src/Data/Collector.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/update_network_account.ts b/scripts/update_network_account.ts index b72c5974..90cf1671 100644 --- a/scripts/update_network_account.ts +++ b/scripts/update_network_account.ts @@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput' import * as Logger from '../src/Logger' import { accountSpecificHash } from '../src/shardeum/calculateAccountHash' import { addSigListeners } from '../src/State' +import { Utils as StringUtils } from '@shardus/types' const activeVersion = '1.9.0' const latestVersion = '1.9.0' diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 6f7b4f9c..ec7ee857 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -447,7 +447,7 @@ export const verifyReceiptData = async ( // Check if the node is in the execution group if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) { Logger.mainLogger.error( - `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the execution group of the tx` + `The node with public key ${nodePubKey} of the receipt ${txId} is not in the execution group of the tx` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( From 7175d87d053c1bb5446ea43e54769e7fac19c3f6 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 22 Jul 2024 11:55:15 +0545 Subject: [PATCH 36/51] Fix StringUtils missing issue in scripts --- scripts/archiver_data_sync_check.ts | 1 + scripts/create_shut_down_cycle.ts | 1 + scripts/repair_missing_cycle.ts | 4 ++-- scripts/verify_account_hash.ts | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/archiver_data_sync_check.ts b/scripts/archiver_data_sync_check.ts index 0d5555a4..704bf35d 100644 --- a/scripts/archiver_data_sync_check.ts +++ b/scripts/archiver_data_sync_check.ts @@ -4,6 +4,7 @@ import { join } from 'path' import { postJson } from '../src/P2P' import { config, overrideDefaultConfig } from '../src/Config' import { ArchiverNodeInfo } from '../src/State' +import { Utils as StringUtils } from '@shardus/types' const configFile = join(process.cwd(), 'archiver-config.json') overrideDefaultConfig(configFile) diff --git a/scripts/create_shut_down_cycle.ts b/scripts/create_shut_down_cycle.ts index ed793ee6..cf575733 100644 --- a/scripts/create_shut_down_cycle.ts +++ b/scripts/create_shut_down_cycle.ts @@ -10,6 +10,7 @@ import * as Logger from '../src/Logger' import { P2P } from '@shardus/types' import { addSigListeners } from '../src/State' import { computeCycleMarker } from '../src/Data/Cycles' +import { Utils as StringUtils } from '@shardus/types' const archiversAtShutdown = [ { diff --git a/scripts/repair_missing_cycle.ts b/scripts/repair_missing_cycle.ts index 43188aea..a35b657d 100644 --- a/scripts/repair_missing_cycle.ts +++ b/scripts/repair_missing_cycle.ts @@ -1,6 +1,5 @@ import { readFileSync } from 'fs' -import { resolve } from 'path' -import { join } from 'path' +import { resolve, join } from 'path' import { overrideDefaultConfig, config } from '../src/Config' import * as Crypto from '../src/Crypto' import * as db from '../src/dbstore/sqlite3storage' @@ -8,6 +7,7 @@ import * as dbstore from '../src/dbstore' import * as CycleDB from '../src/dbstore/cycles' import { startSaving } from '../src/saveConsoleOutput' import * as Logger from '../src/Logger' +import { Utils as StringUtils } from '@shardus/types' const patchCycleData = false diff --git a/scripts/verify_account_hash.ts b/scripts/verify_account_hash.ts index 5eaf8686..edd760d3 100644 --- a/scripts/verify_account_hash.ts +++ b/scripts/verify_account_hash.ts @@ -9,6 +9,7 @@ import { startSaving } from '../src/saveConsoleOutput' import * as Logger from '../src/Logger' import { AccountType, fixAccountUint8Arrays, accountSpecificHash } from '../src/shardeum/calculateAccountHash' import { addSigListeners } from '../src/State' +import { Utils as StringUtils } from '@shardus/types' const updateHash = false const runProgram = async (): Promise => { From 1821e3fe5fffdfedfa67088a7cf6a2bda1969971 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Wed, 26 Jun 2024 22:56:57 +0545 Subject: [PATCH 37/51] - Restricted to only the first validator can forward the accounts data to the archiver during early cycles - Remove accounts transformation used in account verification --- src/Data/Collector.ts | 39 ++++++++++++---- src/Data/Data.ts | 70 +++++++++++++++------------- src/shardeum/calculateAccountHash.ts | 21 --------- 3 files changed, 67 insertions(+), 63 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index ec7ee857..a26d233d 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -26,7 +26,7 @@ import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Da import * as OriginalTxDB from '../dbstore/originalTxsData' import ShardFunction from '../ShardFunctions' import { ConsensusNodeInfo } from '../NodeList' -import { verifyAccountHash } from '../shardeum/calculateAccountHash' +import { accountSpecificHash, verifyAccountHash } from '../shardeum/calculateAccountHash' import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData' import { Cycle as DbCycle } from '../dbstore/types' import { Utils as StringUtils } from '@shardus/types' @@ -927,12 +927,11 @@ interface StoreAccountParam { } export const storeAccountData = async (restoreData: StoreAccountParam = {}): Promise => { - console.log( - 'RestoreData', - 'accounts', - restoreData.accounts ? restoreData.accounts.length : 0, - 'receipts', - restoreData.receipts ? restoreData.receipts.length : 0 + Logger.mainLogger.debug( + `storeAccountData: ${restoreData.accounts ? restoreData.accounts.length : 0} accounts` + ) + Logger.mainLogger.debug( + `storeAccountData: ${restoreData.receipts ? restoreData.receipts.length : 0} receipts` ) const { accounts, receipts } = restoreData if (profilerInstance) profilerInstance.profileSectionStart('store_account_data') @@ -953,7 +952,28 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro // // await Account.insertAccount(account) // // } // } - if (accounts && accounts.length > 0) await Account.bulkInsertAccounts(accounts) + // + if (accounts && accounts.length > 0) { + const combineAccounts = [] + for (const account of accounts) { + try { + const calculatedAccountHash = accountSpecificHash(account.data) + if (calculatedAccountHash !== account.hash) { + Logger.mainLogger.error( + 'Invalid account hash', + account.accountId, + account.hash, + calculatedAccountHash + ) + continue + } + combineAccounts.push(account) + } catch (error) { + Logger.mainLogger.error('Error in calculating genesis account hash', error) + } + } + if (combineAccounts.length > 0) await Account.bulkInsertAccounts(accounts) + } if (receipts && receipts.length > 0) { Logger.mainLogger.debug('Received receipts Size', receipts.length) const combineTransactions = [] @@ -971,10 +991,9 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro await Transaction.bulkInsertTransactions(combineTransactions) } if (profilerInstance) profilerInstance.profileSectionEnd('store_account_data') - console.log('Combined Accounts Data', combineAccountsData.accounts.length) Logger.mainLogger.debug('Combined Accounts Data', combineAccountsData.accounts.length) if (combineAccountsData.accounts.length > 0 || combineAccountsData.receipts.length > 0) { - console.log('Found combine accountsData') + Logger.mainLogger.debug('Found combine accountsData', combineAccountsData.accounts.length) const accountData = { ...combineAccountsData } clearCombinedAccountsData() storeAccountData(accountData) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index f74bd1f5..a3de8cef 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -63,6 +63,11 @@ const { MAX_BETWEEN_CYCLES_PER_REQUEST, } = config.REQUEST_LIMIT +const GENESIS_ACCOUNTS_CYCLE_RANGE = { + startCycle: 0, + endCycle: 5, +} + export enum DataRequestTypes { SUBSCRIBE = 'SUBSCRIBE', UNSUBSCRIBE = 'UNSUBSCRIBE', @@ -280,26 +285,30 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void { collectCycleData(newData.responses.CYCLE, sender.nodeInfo.ip + ':' + sender.nodeInfo.port) } if (newData.responses && newData.responses.ACCOUNT) { - console.log( - 'RECEIVED ACCOUNTS DATA', - sender.nodeInfo.publicKey, - sender.nodeInfo.ip, - sender.nodeInfo.port - ) - Logger.mainLogger.debug( - 'RECEIVED ACCOUNTS DATA', - sender.nodeInfo.publicKey, - sender.nodeInfo.ip, - sender.nodeInfo.port - ) + if (getCurrentCycleCounter() > GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle) { + Logger.mainLogger.error( + 'Account data is not meant to be received after the genesis cycle', + getCurrentCycleCounter() + ) + unsubscribeDataSender(sender.nodeInfo.publicKey) + return + } + if (NodeList.byPublicKey.size > 1 || !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)) { + Logger.mainLogger.error( + 'Account data is not meant to be received by the first validator', + `Number of nodes in the network ${NodeList.byPublicKey.size}` + ) + unsubscribeDataSender(sender.nodeInfo.publicKey) + return + } + Logger.mainLogger.debug(`RECEIVED ACCOUNTS DATA FROM ${sender.nodeInfo.ip}:${sender.nodeInfo.port}`) nestedCountersInstance.countEvent('genesis', 'accounts', 1) if (!forwardGenesisAccounts) { - console.log('Genesis Accounts To Sycn', newData.responses.ACCOUNT) Logger.mainLogger.debug('Genesis Accounts To Sycn', newData.responses.ACCOUNT) syncGenesisAccountsFromConsensor(newData.responses.ACCOUNT, sender.nodeInfo) } else { if (storingAccountData) { - console.log('Storing Data') + Logger.mainLogger.debug('Storing Account Data') let newCombineAccountsData = { ...combineAccountsData } if (newData.responses.ACCOUNT.accounts) newCombineAccountsData.accounts = [ @@ -321,17 +330,14 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void { } // Set new contactTimeout for sender. Postpone sender removal because data is still received from consensor - if (currentCycleDuration > 0) { - nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout') - // To make sure that the sender is still in the subscribed list - sender = dataSenders.get(newData.publicKey) - if (sender) - sender.contactTimeout = createContactTimeout( - sender.nodeInfo.publicKey, - 'This timeout is created after processing data' - ) - } - return + nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout') + // To make sure that the sender is still in the subscribed list + sender = dataSenders.get(newData.publicKey) + if (sender) + sender.contactTimeout = createContactTimeout( + sender.nodeInfo.publicKey, + 'This timeout is created after processing data' + ) } }) } @@ -1009,7 +1015,7 @@ export async function syncGenesisAccountsFromArchiver(): Promise { // } const res = (await queryFromArchivers( RequestDataType.ACCOUNT, - { startCycle: 0, endCycle: 5 }, + { startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle }, QUERY_TIMEOUT_MAX )) as ArchiverAccountResponse if (res && (res.totalAccounts || res.totalAccounts === 0)) { @@ -1026,8 +1032,8 @@ export async function syncGenesisAccountsFromArchiver(): Promise { const response = (await queryFromArchivers( RequestDataType.ACCOUNT, { - startCycle: 0, - endCycle: 5, + startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, + endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle, page, }, QUERY_TIMEOUT_MAX @@ -1059,8 +1065,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise { const res = (await queryFromArchivers( RequestDataType.TRANSACTION, { - startCycle: 0, - endCycle: 5, + startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, + endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle, }, QUERY_TIMEOUT_MAX )) as ArchiverTransactionResponse @@ -1078,8 +1084,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise { const response = (await queryFromArchivers( RequestDataType.TRANSACTION, { - startCycle: 0, - endCycle: 5, + startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, + endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle, page, }, QUERY_TIMEOUT_MAX diff --git a/src/shardeum/calculateAccountHash.ts b/src/shardeum/calculateAccountHash.ts index 61e6853e..326d08f9 100644 --- a/src/shardeum/calculateAccountHash.ts +++ b/src/shardeum/calculateAccountHash.ts @@ -57,17 +57,6 @@ export const accountSpecificHash = (account: any): string => { return hash } -// Converting the correct account data format to get the correct hash -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export const fixAccountUint8Arrays = (account: any): void => { - if (!account) return // if account is null, return - if (account.storageRoot) account.storageRoot = Uint8Array.from(Object.values(account.storageRoot)) // Account - if (account.codeHash) account.codeHash = Uint8Array.from(Object.values(account.codeHash)) // - //Account and ContractCode - if (account.codeByte) account.codeByte = Uint8Array.from(Object.values(account.codeByte)) // ContractCode - if (account.value) account.value = Uint8Array.from(Object.values(account.value)) // ContractByte -} - export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => { try { if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification @@ -87,16 +76,6 @@ export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => { return false } for (const account of receipt.accounts) { - if (account.data.accountType === AccountType.Account) { - fixAccountUint8Arrays(account.data.account) - // console.dir(acc, { depth: null }) - } else if ( - account.data.accountType === AccountType.ContractCode || - account.data.accountType === AccountType.ContractStorage - ) { - fixAccountUint8Arrays(account.data) - // console.dir(acc, { depth: null }) - } const calculatedAccountHash = accountSpecificHash(account.data) const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId) if (indexOfAccount === -1) { From 7c545c91e22246219ad29dfd5e6ffc4c82586b59 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 22 Jul 2024 12:11:30 +0545 Subject: [PATCH 38/51] Remove fixAccountUint8Arrays from the scripts --- scripts/validate_archiver_receipt.ts | 12 +----------- scripts/verify_account_hash.ts | 12 +----------- src/Data/Data.ts | 8 ++++++-- 3 files changed, 8 insertions(+), 24 deletions(-) diff --git a/scripts/validate_archiver_receipt.ts b/scripts/validate_archiver_receipt.ts index a7682f02..1785fef3 100644 --- a/scripts/validate_archiver_receipt.ts +++ b/scripts/validate_archiver_receipt.ts @@ -3,7 +3,7 @@ import { overrideDefaultConfig, config } from '../src/Config' import * as Crypto from '../src/Crypto' import * as Utils from '../src/Utils' import * as Receipt from '../src/dbstore/receipts' -import { AccountType, accountSpecificHash, fixAccountUint8Arrays } from '../src/shardeum/calculateAccountHash' +import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash' import { ShardeumReceipt } from '../src/shardeum/verifyAppReceiptData' // Add the full receipt data here @@ -243,16 +243,6 @@ export const verifyAccountHash = (receipt: Receipt.ArchiverReceipt): boolean => try { if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification for (const account of receipt.accounts) { - if (account.data.accountType === AccountType.Account) { - fixAccountUint8Arrays(account.data.account) - // console.dir(acc, { depth: null }) - } else if ( - account.data.accountType === AccountType.ContractCode || - account.data.accountType === AccountType.ContractStorage - ) { - fixAccountUint8Arrays(account.data) - // console.dir(acc, { depth: null }) - } const calculatedAccountHash = accountSpecificHash(account.data) const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId) if (indexOfAccount === -1) { diff --git a/scripts/verify_account_hash.ts b/scripts/verify_account_hash.ts index edd760d3..e0be9b99 100644 --- a/scripts/verify_account_hash.ts +++ b/scripts/verify_account_hash.ts @@ -7,7 +7,7 @@ import * as dbstore from '../src/dbstore' import * as AccountDB from '../src/dbstore/accounts' import { startSaving } from '../src/saveConsoleOutput' import * as Logger from '../src/Logger' -import { AccountType, fixAccountUint8Arrays, accountSpecificHash } from '../src/shardeum/calculateAccountHash' +import { AccountType, accountSpecificHash } from '../src/shardeum/calculateAccountHash' import { addSigListeners } from '../src/State' import { Utils as StringUtils } from '@shardus/types' @@ -48,16 +48,6 @@ const runProgram = async (): Promise => { if (accountHash1 !== accountHash2) { console.log(account.accountId, 'accountHash', accountHash1, 'accountHash2', accountHash2) } - if (account.data.accountType === AccountType.Account) { - fixAccountUint8Arrays(account.data.account) - // console.dir(acc, { depth: null }) - } else if ( - account.data.accountType === AccountType.ContractCode || - account.data.accountType === AccountType.ContractStorage - ) { - fixAccountUint8Arrays(account.data) - // console.dir(acc, { depth: null }) - } const calculatedAccountHash = accountSpecificHash(account.data) if (accountHash1 !== calculatedAccountHash) { diff --git a/src/Data/Data.ts b/src/Data/Data.ts index a3de8cef..ab89bdcf 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -287,13 +287,17 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void { if (newData.responses && newData.responses.ACCOUNT) { if (getCurrentCycleCounter() > GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle) { Logger.mainLogger.error( - 'Account data is not meant to be received after the genesis cycle', + 'Account data is not meant to be received after the genesis accounts cycle range', getCurrentCycleCounter() ) unsubscribeDataSender(sender.nodeInfo.publicKey) return } - if (NodeList.byPublicKey.size > 1 || !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)) { + if ( + Cycles.currentNetworkMode !== 'forming' || + NodeList.byPublicKey.size > 1 || + !NodeList.byPublicKey.has(sender.nodeInfo.publicKey) + ) { Logger.mainLogger.error( 'Account data is not meant to be received by the first validator', `Number of nodes in the network ${NodeList.byPublicKey.size}` From 8ec0f5bff98aeb50d540f936eb92dfb38663d45a Mon Sep 17 00:00:00 2001 From: jairajdev Date: Tue, 2 Jul 2024 23:00:13 +0545 Subject: [PATCH 39/51] Created config for the number of cycles to keep shard calculations data --- src/Config.ts | 3 +++ src/Data/Cycles.ts | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 5641109d..2ec120cf 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -68,6 +68,8 @@ export interface Config { usePOQo: boolean // The percentage of votes required to confirm transaction requiredVotesPercentage: number + // number of recent cycles of shard data to keep + CYCLE_SHARD_STORAGE_LIMIT: number } let config: Config = { @@ -131,6 +133,7 @@ let config: Config = { stopGossipTxData: false, usePOQo: true, requiredVotesPercentage: 2 / 3, + CYCLE_SHARD_STORAGE_LIMIT: 3, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index 25748446..139fe7f6 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -52,8 +52,6 @@ export let cycleRecordWithShutDownMode = null as P2PTypes.CycleCreatorTypes.Cycl export let currentNetworkMode: P2PTypes.ModesTypes.Record['mode'] = 'forming' export const shardValuesByCycle = new Map() -const CYCLE_SHARD_STORAGE_LIMIT = 20 - export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData[]): Promise { if (profilerInstance) profilerInstance.profileSectionStart('process_cycle', false) try { @@ -518,7 +516,7 @@ function updateShardValues(cycle: P2PTypes.CycleCreatorTypes.CycleData): void { const list = cycleShardData.nodes.map((n) => n['ip'] + ':' + n['port']) Logger.mainLogger.debug('cycleShardData', cycleShardData.cycleNumber, list.length, stringifyReduce(list)) shardValuesByCycle.set(cycleShardData.cycleNumber, cycleShardData) - if (shardValuesByCycle.size > CYCLE_SHARD_STORAGE_LIMIT) { + if (shardValuesByCycle.size > config.CYCLE_SHARD_STORAGE_LIMIT) { shardValuesByCycle.delete(shardValuesByCycle.keys().next().value) } } From 333754952e8f74594aefe6b9aa731606d6e3ffa3 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Wed, 3 Jul 2024 13:21:09 +0545 Subject: [PATCH 40/51] Added debug logs for receipt processing time taken --- src/Data/Collector.ts | 17 ++++++++++++++--- src/Data/GossipData.ts | 4 ++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index a26d233d..cfb2a4ac 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -390,7 +390,18 @@ export const verifyReceiptData = async ( const { executionShardKey, cycle, appliedReceipt, globalModification } = receipt if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true } const { appliedVote, signatures } = appliedReceipt - const txId = receipt.tx.txId + const { txId, timestamp } = receipt.tx + const currentTimestamp = Date.now() + // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s) + console.log( + `Time taken between receipt timestamp and current time: ${txId}`, + `${currentTimestamp - timestamp} ms`, + `${currentTimestamp - timestamp / 1000} s` + ) + if (getCurrentCycleCounter() - cycle > 2) { + Logger.mainLogger.error(`Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}`) + console.dir(receipt, { depth: null }) + } const cycleShardData = shardValuesByCycle.get(cycle) if (!cycleShardData) { Logger.mainLogger.error('Cycle shard data not found') @@ -435,7 +446,7 @@ export const verifyReceiptData = async ( const node = cycleShardData.nodes.find((node) => node.publicKey === nodePubKey) if (node == null) { Logger.mainLogger.error( - `The node with public key ${nodePubKey} of the receipt ${txId}} is not in the active nodesList of cycle ${cycle}` + `The node with public key ${nodePubKey} of the receipt ${txId}} with ${timestamp} is not in the active nodesList of cycle ${cycle}` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( @@ -447,7 +458,7 @@ export const verifyReceiptData = async ( // Check if the node is in the execution group if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) { Logger.mainLogger.error( - `The node with public key ${nodePubKey} of the receipt ${txId} is not in the execution group of the tx` + `The node with public key ${nodePubKey} of the receipt ${txId} with ${timestamp} is not in the execution group of the tx` ) if (nestedCountersInstance) nestedCountersInstance.countEvent( diff --git a/src/Data/GossipData.ts b/src/Data/GossipData.ts index 0f761e58..d2414e6d 100644 --- a/src/Data/GossipData.ts +++ b/src/Data/GossipData.ts @@ -98,13 +98,13 @@ export async function sendDataToAdjacentArchivers( }) promises.push(promise) } catch (e) { - Logger.mainLogger.error('Error', e) + Logger.mainLogger.error(`Gossip Error to archiver ${archiver.ip}: ${archiver.port}`, e) } } try { await Promise.allSettled(promises) } catch (err) { - Logger.mainLogger.error('Network: ' + err) + Logger.mainLogger.error('Gossip Error: ' + err) } } catch (ex) { Logger.mainLogger.debug(ex) From ab04c94be03ae754e8519254ae3294b5ba44ce02 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Wed, 3 Jul 2024 14:34:30 +0545 Subject: [PATCH 41/51] Updated to keep 10 CYCLE_SHARD_STORAGE_LIMIT --- src/Config.ts | 2 +- src/Data/Collector.ts | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 2ec120cf..b4abb154 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -133,7 +133,7 @@ let config: Config = { stopGossipTxData: false, usePOQo: true, requiredVotesPercentage: 2 / 3, - CYCLE_SHARD_STORAGE_LIMIT: 3, + CYCLE_SHARD_STORAGE_LIMIT: 10, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index cfb2a4ac..a668a916 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -396,10 +396,13 @@ export const verifyReceiptData = async ( console.log( `Time taken between receipt timestamp and current time: ${txId}`, `${currentTimestamp - timestamp} ms`, - `${currentTimestamp - timestamp / 1000} s` + `${(currentTimestamp - timestamp) / 1000} s` ) - if (getCurrentCycleCounter() - cycle > 2) { - Logger.mainLogger.error(`Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}`) + const currentCycle = getCurrentCycleCounter() + if (currentCycle - cycle > 2) { + Logger.mainLogger.error( + `Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}, ${currentCycle}` + ) console.dir(receipt, { depth: null }) } const cycleShardData = shardValuesByCycle.get(cycle) From dddd88a67774c5aa72c378f11c033f8cf1277e97 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 22 Jul 2024 13:06:44 +0545 Subject: [PATCH 42/51] Put the console logs behind the verbose flag --- src/Data/Collector.ts | 17 +++++++++-------- src/Data/Cycles.ts | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index a668a916..3335af16 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -391,19 +391,20 @@ export const verifyReceiptData = async ( if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true } const { appliedVote, signatures } = appliedReceipt const { txId, timestamp } = receipt.tx - const currentTimestamp = Date.now() - // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s) - console.log( - `Time taken between receipt timestamp and current time: ${txId}`, - `${currentTimestamp - timestamp} ms`, - `${(currentTimestamp - timestamp) / 1000} s` - ) + if (config.VERBOSE) { + const currentTimestamp = Date.now() + // Console log the timetaken between the receipt timestamp and the current time ( both in ms and s) + console.log( + `Time taken between receipt timestamp and current time: ${txId}`, + `${currentTimestamp - timestamp} ms`, + `${(currentTimestamp - timestamp) / 1000} s` + ) + } const currentCycle = getCurrentCycleCounter() if (currentCycle - cycle > 2) { Logger.mainLogger.error( `Found receipt with cycle older than 2 cycles ${txId}, ${cycle}, ${timestamp}, ${currentCycle}` ) - console.dir(receipt, { depth: null }) } const cycleShardData = shardValuesByCycle.get(cycle) if (!cycleShardData) { diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index 139fe7f6..88f9a0a0 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -91,8 +91,8 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData setShutdownCycleRecord(cycle) NodeList.toggleFirstNode() } - // Clean receipts/originalTxs cache that are older than 20 minutes to match with CYCLE_SHARD_STORAGE_LIMIT ( actual fix is on another branch ) - const cleanupTimestamp = Date.now() - 20 * 60 * 1000 + // Clean receipts/originalTxs cache that are older than minutes + const cleanupTimestamp = (cycle.start - config.CYCLE_SHARD_STORAGE_LIMIT * 60) * 1000 cleanOldOriginalTxsMap(cleanupTimestamp) cleanOldReceiptsMap(cleanupTimestamp) } From 6098cf7a6cf39a4624f5dbb23eb50e514e5c8724 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Thu, 27 Jun 2024 23:00:42 +0545 Subject: [PATCH 43/51] Port the logic from shardus core to modify the network account itself on the change cycle --- src/Config.ts | 6 ++ src/Data/Cycles.ts | 3 + src/GlobalAccount.ts | 136 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/src/Config.ts b/src/Config.ts index b4abb154..22bc220d 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -70,6 +70,10 @@ export interface Config { requiredVotesPercentage: number // number of recent cycles of shard data to keep CYCLE_SHARD_STORAGE_LIMIT: number + // the number of cycles within which we want to keep \changes to a config*/ + configChangeMaxCyclesToKeep: number + // the number of config changes to keep*/ + configChangeMaxChangesToKeep: number } let config: Config = { @@ -134,6 +138,8 @@ let config: Config = { usePOQo: true, requiredVotesPercentage: 2 / 3, CYCLE_SHARD_STORAGE_LIMIT: 10, + configChangeMaxCyclesToKeep: 5, + configChangeMaxChangesToKeep: 1000, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index 88f9a0a0..95cd97b2 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -29,6 +29,7 @@ import { RequestDataType, queryFromArchivers } from '../API' import { stringifyReduce } from '../profiler/StringifyReduce' import { addCyclesToCache } from '../cache/cycleRecordsCache' import { queryLatestCycleRecords } from '../dbstore/cycles' +import { updateGlobalNetworkAccount } from '../GlobalAccount' export interface ArchiverCycleResponse { cycleInfo: P2PTypes.CycleCreatorTypes.CycleData[] @@ -83,6 +84,8 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData // Check the archivers reputaion in every new cycle & record the status recordArchiversReputation() } + await updateGlobalNetworkAccount(cycle.counter) + if (currentNetworkMode === 'shutdown') { Logger.mainLogger.debug(Date.now(), `❌ Shutdown Cycle Record received at Cycle #: ${cycle.counter}`) await Utils.sleep(currentCycleDuration) diff --git a/src/GlobalAccount.ts b/src/GlobalAccount.ts index f722fe04..9a7adb86 100644 --- a/src/GlobalAccount.ts +++ b/src/GlobalAccount.ts @@ -7,8 +7,9 @@ import { config } from './Config' import { postJson, getJson } from './P2P' import { robustQuery, deepCopy } from './Utils' import { isDeepStrictEqual } from 'util' +import { accountSpecificHash } from './shardeum/calculateAccountHash' -let cachedGlobalNetworkAccount: object +let cachedGlobalNetworkAccount: AccountDB.AccountCopy let cachedGlobalNetworkAccountHash: string interface Node { @@ -21,6 +22,7 @@ export interface GlobalAccountsHashAndTimestamp { timestamp: number } export const globalAccountsMap = new Map() +const appliedConfigChanges = new Set() export function getGlobalNetworkAccount(hash: boolean): object | string { if (hash) { @@ -35,6 +37,138 @@ export function setGlobalNetworkAccount(account: AccountDB.AccountCopy): void { cachedGlobalNetworkAccountHash = account.hash } +interface NetworkConfigChanges { + cycle: number + change: any + appData: any +} + +export const updateGlobalNetworkAccount = async (cycleNumber: number): Promise => { + if (!cachedGlobalNetworkAccountHash) return + const networkAccount = rfdc()(cachedGlobalNetworkAccount) + const changes = networkAccount.data.listOfChanges as NetworkConfigChanges[] + if (!changes || !Array.isArray(changes)) { + return + } + for (const change of changes) { + // skip future changes + if (change.cycle > cycleNumber) { + continue + } + const changeHash = Crypto.hashObj(change) + // skip handled changes + if (appliedConfigChanges.has(changeHash)) { + continue + } + // apply this change + appliedConfigChanges.add(changeHash) + const changeObj = change.change + const appData = change.appData + + // If there is initShutdown change, if the latest cycle is greater than the cycle of the change, then skip it + if (changeObj['p2p'] && changeObj['p2p']['initShutdown'] && change.cycle !== cycleNumber) continue + + const newChanges = pruneNetworkChangeQueue(changes, cycleNumber) + networkAccount.data.listOfChanges = newChanges + // Increase the timestamp by 1 second + networkAccount.data.timestamp += 1000 + + if (appData) { + updateNetworkChangeQueue(networkAccount.data, appData) + console.dir(networkAccount.data, { depth: null }) + networkAccount.data.timestamp += 1000 + } + + // Increase the timestamp by 1 second + networkAccount.hash = accountSpecificHash(networkAccount.data) + networkAccount.timestamp = networkAccount.data.timestamp + Logger.mainLogger.debug('updateGlobalNetworkAccount', networkAccount) + await AccountDB.updateAccount(networkAccount) + setGlobalNetworkAccount(networkAccount) + } +} + +const generatePathKeys = (obj: any, prefix = ''): string[] => { + /* eslint-disable security/detect-object-injection */ + let paths: string[] = [] + + // Loop over each key in the object + for (const key of Object.keys(obj)) { + // If the value corresponding to this key is an object (and not an array or null), + // then recurse into it. + if (obj[key] !== null && typeof obj[key] === 'object' && !Array.isArray(obj[key])) { + paths = paths.concat(generatePathKeys(obj[key], prefix + key + '.')) + } else { + // Otherwise, just append this key to the path. + paths.push(prefix + key) + } + } + return paths + /* eslint-enable security/detect-object-injection */ +} + +const pruneNetworkChangeQueue = ( + changes: NetworkConfigChanges[], + currentCycle: number +): NetworkConfigChanges[] => { + const configsMap = new Map() + const keepAliveCount = config.configChangeMaxChangesToKeep + console.log('pruneNetworkChangeQueue', changes.length, keepAliveCount) + for (let i = changes.length - 1; i >= 0; i--) { + const thisChange = changes[i] + let keepAlive = false + + let appConfigs = [] + if (thisChange.appData) { + appConfigs = generatePathKeys(thisChange.appData, 'appdata.') + console.log('pruneNetworkChangeQueue appConfigs', appConfigs) + } + const shardusConfigs: string[] = generatePathKeys(thisChange.change) + console.log('pruneNetworkChangeQueue shardusConfigs', shardusConfigs) + + const allConfigs = appConfigs.concat(shardusConfigs) + console.log('pruneNetworkChangeQueue allConfigs', allConfigs) + + for (const config of allConfigs) { + if (!configsMap.has(config)) { + configsMap.set(config, 1) + keepAlive = true + } else if (configsMap.get(config) < keepAliveCount) { + configsMap.set(config, configsMap.get(config) + 1) + keepAlive = true + } + } + + if (currentCycle - thisChange.cycle <= config.configChangeMaxCyclesToKeep) { + keepAlive = true + } + console.log('pruneNetworkChangeQueue keepAlive', keepAlive, thisChange, configsMap) + + if (keepAlive == false) { + changes.splice(i, 1) + } + } + return changes +} + +const updateNetworkChangeQueue = (data: object, appData: object): void => { + if ('current' in data) patchAndUpdate(data?.current, appData) + console.log('updateNetworkChangeQueue', data) +} + +const patchAndUpdate = (existingObject: any, changeObj: any, parentPath = ''): void => { + /* eslint-disable security/detect-object-injection */ + for (const [key, value] of Object.entries(changeObj)) { + if (existingObject[key] != null) { + if (typeof value === 'object') { + patchAndUpdate(existingObject[key], value, parentPath === '' ? key : parentPath + '.' + key) + } else { + existingObject[key] = value + } + } + } +} + export const loadGlobalAccounts = async (): Promise => { const sql = `SELECT * FROM accounts WHERE isGlobal=1` const values = [] From 1e548486ec091f76387dc728dd676d541548034c Mon Sep 17 00:00:00 2001 From: jairajdev Date: Fri, 28 Jun 2024 16:52:16 +0545 Subject: [PATCH 44/51] Updated to adapt `configChangeMaxChangesToKeep` and `configChangeMaxCyclesToKeep` config from the network and to clear the entries from appliedConfigChanges that are no longer in the changes list --- src/Data/Data.ts | 46 +++++++++++++++++++++++++++++++++----------- src/GlobalAccount.ts | 13 ++++++++++++- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index ab89bdcf..38a71a06 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -558,15 +558,18 @@ async function syncFromNetworkConfig(): Promise { equalityFn, 3 // Redundancy (minimum 3 nodes should return the same result to reach consensus) ) - - if (tallyItem?.value?.config) { + if (tallyItem?.value?.config?.stateManager) { // Updating the Archiver Config as per the latest Network Config - const devPublicKeys = tallyItem.value.config?.devPublicKeys + const { + devPublicKeys, + useNewPOQ: newPOQReceipt, + configChangeMaxChangesToKeep, + configChangeMaxCyclesToKeep, + } = tallyItem.value.config.stateManager const devPublicKey = devPublicKeys && Object.keys(devPublicKeys).length >= 3 && Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) - const newPOQReceipt = tallyItem.value.config?.useNewPOQ if ( devPublicKey && typeof devPublicKey === typeof config.DevPublicKey && @@ -579,6 +582,18 @@ async function syncFromNetworkConfig(): Promise { newPOQReceipt !== config.newPOQReceipt ) updateConfig({ newPOQReceipt }) + if ( + !Utils.isUndefined(configChangeMaxChangesToKeep) && + typeof configChangeMaxChangesToKeep === typeof config.configChangeMaxChangesToKeep && + configChangeMaxChangesToKeep !== config.configChangeMaxChangesToKeep + ) + updateConfig({ configChangeMaxChangesToKeep }) + if ( + !Utils.isUndefined(configChangeMaxCyclesToKeep) && + typeof configChangeMaxCyclesToKeep === typeof config.configChangeMaxCyclesToKeep && + configChangeMaxCyclesToKeep !== config.configChangeMaxCyclesToKeep + ) + updateConfig({ configChangeMaxCyclesToKeep }) return tallyItem } return null @@ -593,20 +608,29 @@ async function getConsensusRadius(): Promise { if (NodeList.isEmpty()) return currentConsensusRadius const tallyItem = await syncFromNetworkConfig() - // Check if a consensus was reached if (tallyItem?.value?.config) { - nodesPerEdge = tallyItem.value.config.sharding?.nodesPerEdge - nodesPerConsensusGroup = tallyItem.value.config?.sharding.nodesPerConsensusGroup + const nodesPerEdgeFromConfig = tallyItem.value.config.sharding?.nodesPerEdge + const nodesPerConsensusGroupFromConfig = tallyItem.value.config.sharding?.nodesPerConsensusGroup - if (!Number.isInteger(nodesPerConsensusGroup) || nodesPerConsensusGroup <= 0) { - Logger.mainLogger.error('nodesPerConsensusGroup is not a valid number:', nodesPerConsensusGroup) + if (!Number.isInteger(nodesPerConsensusGroupFromConfig) || nodesPerConsensusGroupFromConfig <= 0) { + Logger.mainLogger.error( + 'nodesPerConsensusGroup is not a valid number:', + nodesPerConsensusGroupFromConfig + ) return currentConsensusRadius } - if (!Number.isInteger(nodesPerEdge) || nodesPerEdge <= 0) { - Logger.mainLogger.error('nodesPerEdge is not a valid number:', nodesPerEdge) + if (!Number.isInteger(nodesPerEdgeFromConfig) || nodesPerEdgeFromConfig <= 0) { + Logger.mainLogger.error('nodesPerEdge is not a valid number:', nodesPerEdgeFromConfig) return currentConsensusRadius } + if ( + nodesPerConsensusGroup === nodesPerConsensusGroupFromConfig && + nodesPerEdge === nodesPerEdgeFromConfig + ) + return currentConsensusRadius + nodesPerConsensusGroup = nodesPerConsensusGroupFromConfig + nodesPerEdge = nodesPerEdgeFromConfig // Upgrading consensus size to an odd number if (nodesPerConsensusGroup % 2 === 0) nodesPerConsensusGroup++ const consensusRadius = Math.floor((nodesPerConsensusGroup - 1) / 2) diff --git a/src/GlobalAccount.ts b/src/GlobalAccount.ts index 9a7adb86..95d0b98a 100644 --- a/src/GlobalAccount.ts +++ b/src/GlobalAccount.ts @@ -22,7 +22,7 @@ export interface GlobalAccountsHashAndTimestamp { timestamp: number } export const globalAccountsMap = new Map() -const appliedConfigChanges = new Set() +const appliedConfigChanges = new Set() export function getGlobalNetworkAccount(hash: boolean): object | string { if (hash) { @@ -50,6 +50,7 @@ export const updateGlobalNetworkAccount = async (cycleNumber: number): Promise() for (const change of changes) { // skip future changes if (change.cycle > cycleNumber) { @@ -58,10 +59,12 @@ export const updateGlobalNetworkAccount = async (cycleNumber: number): Promise 0) { + // clear the entries from appliedConfigChanges that are no longer in the changes list + for (const changeHash of appliedConfigChanges) { + if (!activeConfigChanges.has(changeHash)) { + appliedConfigChanges.delete(changeHash) + } + } + } } const generatePathKeys = (obj: any, prefix = ''): string[] => { From e1eff936211536e233518e110b724da17fd9c87c Mon Sep 17 00:00:00 2001 From: jairajdev Date: Tue, 23 Jul 2024 00:25:04 +0545 Subject: [PATCH 45/51] Avoid timestamp update in network config change --- src/GlobalAccount.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/GlobalAccount.ts b/src/GlobalAccount.ts index 95d0b98a..663cf920 100644 --- a/src/GlobalAccount.ts +++ b/src/GlobalAccount.ts @@ -73,16 +73,18 @@ export const updateGlobalNetworkAccount = async (cycleNumber: number): Promise { const configsMap = new Map() const keepAliveCount = config.configChangeMaxChangesToKeep - console.log('pruneNetworkChangeQueue', changes.length, keepAliveCount) for (let i = changes.length - 1; i >= 0; i--) { const thisChange = changes[i] let keepAlive = false @@ -132,13 +133,10 @@ const pruneNetworkChangeQueue = ( let appConfigs = [] if (thisChange.appData) { appConfigs = generatePathKeys(thisChange.appData, 'appdata.') - console.log('pruneNetworkChangeQueue appConfigs', appConfigs) } const shardusConfigs: string[] = generatePathKeys(thisChange.change) - console.log('pruneNetworkChangeQueue shardusConfigs', shardusConfigs) const allConfigs = appConfigs.concat(shardusConfigs) - console.log('pruneNetworkChangeQueue allConfigs', allConfigs) for (const config of allConfigs) { if (!configsMap.has(config)) { @@ -153,7 +151,6 @@ const pruneNetworkChangeQueue = ( if (currentCycle - thisChange.cycle <= config.configChangeMaxCyclesToKeep) { keepAlive = true } - console.log('pruneNetworkChangeQueue keepAlive', keepAlive, thisChange, configsMap) if (keepAlive == false) { changes.splice(i, 1) @@ -164,7 +161,6 @@ const pruneNetworkChangeQueue = ( const updateNetworkChangeQueue = (data: object, appData: object): void => { if ('current' in data) patchAndUpdate(data?.current, appData) - console.log('updateNetworkChangeQueue', data) } const patchAndUpdate = (existingObject: any, changeObj: any, parentPath = ''): void => { From f94de77df65575691a1989dbccce9f4bc84ae6c9 Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:56:42 -0500 Subject: [PATCH 46/51] 3.4.22 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 194a52a4..811252de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shardus/archiver", - "version": "3.4.21", + "version": "3.4.22", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shardus/archiver", - "version": "3.4.21", + "version": "3.4.22", "license": "ISC", "dependencies": { "@fastify/cors": "^8.2.0", diff --git a/package.json b/package.json index 049da770..7c756f88 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shardus/archiver", - "version": "3.4.21", + "version": "3.4.22", "engines": { "node": "18.16.1" }, From e1f998a0564fcd5767d5ed4567f6acb2e86a366c Mon Sep 17 00:00:00 2001 From: jairajdev Date: Wed, 31 Jul 2024 18:41:24 +0545 Subject: [PATCH 47/51] Fixed devPubKeys update and added maxCyclesShardDataToKeep from shardus core --- src/Config.ts | 6 +++--- src/Data/Cycles.ts | 14 ++++++++++---- src/Data/Data.ts | 9 ++++++++- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Config.ts b/src/Config.ts index 22bc220d..4df699e7 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -68,8 +68,8 @@ export interface Config { usePOQo: boolean // The percentage of votes required to confirm transaction requiredVotesPercentage: number - // number of recent cycles of shard data to keep - CYCLE_SHARD_STORAGE_LIMIT: number + // max number of recent cycle shard data to keep + maxCyclesShardDataToKeep: number // the number of cycles within which we want to keep \changes to a config*/ configChangeMaxCyclesToKeep: number // the number of config changes to keep*/ @@ -137,7 +137,7 @@ let config: Config = { stopGossipTxData: false, usePOQo: true, requiredVotesPercentage: 2 / 3, - CYCLE_SHARD_STORAGE_LIMIT: 10, + maxCyclesShardDataToKeep: 10, configChangeMaxCyclesToKeep: 5, configChangeMaxChangesToKeep: 1000, } diff --git a/src/Data/Cycles.ts b/src/Data/Cycles.ts index 95cd97b2..d0e5a324 100644 --- a/src/Data/Cycles.ts +++ b/src/Data/Cycles.ts @@ -94,10 +94,11 @@ export async function processCycles(cycles: P2PTypes.CycleCreatorTypes.CycleData setShutdownCycleRecord(cycle) NodeList.toggleFirstNode() } - // Clean receipts/originalTxs cache that are older than minutes - const cleanupTimestamp = (cycle.start - config.CYCLE_SHARD_STORAGE_LIMIT * 60) * 1000 + // Clean receipts/originalTxs cache that are older than minutes + const cleanupTimestamp = (cycle.start - config.maxCyclesShardDataToKeep * 60) * 1000 cleanOldOriginalTxsMap(cleanupTimestamp) cleanOldReceiptsMap(cleanupTimestamp) + cleanShardCycleData(cycle.counter - config.maxCyclesShardDataToKeep) } } finally { if (profilerInstance) profilerInstance.profileSectionEnd('process_cycle', false) @@ -519,8 +520,13 @@ function updateShardValues(cycle: P2PTypes.CycleCreatorTypes.CycleData): void { const list = cycleShardData.nodes.map((n) => n['ip'] + ':' + n['port']) Logger.mainLogger.debug('cycleShardData', cycleShardData.cycleNumber, list.length, stringifyReduce(list)) shardValuesByCycle.set(cycleShardData.cycleNumber, cycleShardData) - if (shardValuesByCycle.size > config.CYCLE_SHARD_STORAGE_LIMIT) { - shardValuesByCycle.delete(shardValuesByCycle.keys().next().value) +} + +const cleanShardCycleData = (cycleNumber: number): void => { + for (const [key] of shardValuesByCycle) { + if (key < cycleNumber) { + shardValuesByCycle.delete(key) + } } } diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 38a71a06..0a352aa5 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -561,11 +561,12 @@ async function syncFromNetworkConfig(): Promise { if (tallyItem?.value?.config?.stateManager) { // Updating the Archiver Config as per the latest Network Config const { - devPublicKeys, useNewPOQ: newPOQReceipt, configChangeMaxChangesToKeep, configChangeMaxCyclesToKeep, + maxCyclesShardDataToKeep, } = tallyItem.value.config.stateManager + const devPublicKeys = tallyItem.value.config.debug.devPublicKeys const devPublicKey = devPublicKeys && Object.keys(devPublicKeys).length >= 3 && @@ -594,6 +595,12 @@ async function syncFromNetworkConfig(): Promise { configChangeMaxCyclesToKeep !== config.configChangeMaxCyclesToKeep ) updateConfig({ configChangeMaxCyclesToKeep }) + if ( + !Utils.isUndefined(maxCyclesShardDataToKeep) && + typeof maxCyclesShardDataToKeep === typeof config.maxCyclesShardDataToKeep && + maxCyclesShardDataToKeep !== config.maxCyclesShardDataToKeep + ) + updateConfig({ maxCyclesShardDataToKeep }) return tallyItem } return null From 039e09910aae73cf845bac49df27716277e4b232 Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:12:13 -0500 Subject: [PATCH 48/51] 3.4.23 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 811252de..8afc5e77 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shardus/archiver", - "version": "3.4.22", + "version": "3.4.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shardus/archiver", - "version": "3.4.22", + "version": "3.4.23", "license": "ISC", "dependencies": { "@fastify/cors": "^8.2.0", diff --git a/package.json b/package.json index 7c756f88..8450e652 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shardus/archiver", - "version": "3.4.22", + "version": "3.4.23", "engines": { "node": "18.16.1" }, From 6798be2d515469944b839d44aba34fa292214e6d Mon Sep 17 00:00:00 2001 From: Tanuj Soni Date: Fri, 12 Jul 2024 16:01:20 +0530 Subject: [PATCH 49/51] update default archiver mode to release add patch file to run archiver in debug mode --- archiver-config.json | 2 +- debug_mode.patch | 27 +++++++++++++++++++++++++++ src/Config.ts | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 debug_mode.patch diff --git a/archiver-config.json b/archiver-config.json index ffac25a3..a892baf9 100644 --- a/archiver-config.json +++ b/archiver-config.json @@ -56,7 +56,7 @@ "publicKey": "aec5d2b663869d9c22ba99d8de76f3bff0f54fa5e39d2899ec1f3f4543422ec7" } ], - "ARCHIVER_MODE": "debug", + "ARCHIVER_MODE": "release", "DevPublicKey": "", "EXISTING_ARCHIVER_DB_PATH": "" } \ No newline at end of file diff --git a/debug_mode.patch b/debug_mode.patch new file mode 100644 index 00000000..cc6058c0 --- /dev/null +++ b/debug_mode.patch @@ -0,0 +1,27 @@ +diff --git a/archiver-config.json b/archiver-config.json +index a892baf..ffac25a 100644 +--- a/archiver-config.json ++++ b/archiver-config.json +@@ -56,7 +56,7 @@ + "publicKey": "aec5d2b663869d9c22ba99d8de76f3bff0f54fa5e39d2899ec1f3f4543422ec7" + } + ], +- "ARCHIVER_MODE": "release", ++ "ARCHIVER_MODE": "debug", + "DevPublicKey": "", + "EXISTING_ARCHIVER_DB_PATH": "" + } +\ No newline at end of file +diff --git a/src/Config.ts b/src/Config.ts +index 6b41ee4..a812003 100644 +--- a/src/Config.ts ++++ b/src/Config.ts +@@ -86,7 +86,7 @@ let config: Config = { + save: true, + interval: 1, + }, +- ARCHIVER_MODE: 'release', // 'debug'/'release' ++ ARCHIVER_MODE: 'debug', // 'debug'/'release' + DevPublicKey: '', + dataLogWrite: true, + dataLogWriter: { diff --git a/src/Config.ts b/src/Config.ts index 4df699e7..9e47d7eb 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -95,7 +95,7 @@ let config: Config = { save: true, interval: 1, }, - ARCHIVER_MODE: 'debug', // 'debug'/'release' + ARCHIVER_MODE: 'release', // 'debug'/'release' DevPublicKey: '', dataLogWrite: true, dataLogWriter: { From 9896f3ce94d7d072342ad8d3f1eff077070cc908 Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 5 Aug 2024 20:49:34 +0545 Subject: [PATCH 50/51] Revert to the original index applied in the tables --- src/Data/Data.ts | 24 ++++++++++++------------ src/dbstore/index.ts | 19 +++++++------------ 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/Data/Data.ts b/src/Data/Data.ts index 0a352aa5..d7a44150 100644 --- a/src/Data/Data.ts +++ b/src/Data/Data.ts @@ -529,7 +529,7 @@ async function syncFromNetworkConfig(): Promise { try { // Define the query function to get the network config from a node const queryFn = async (node): Promise => { - const REQUEST_NETCONFIG_TIMEOUT_SECOND = 2 // 2s timeout + const REQUEST_NETCONFIG_TIMEOUT_SECOND = 3 // 3s timeout try { const response = await P2P.getJson( `http://${node.ip}:${node.port}/netconfig`, @@ -566,17 +566,17 @@ async function syncFromNetworkConfig(): Promise { configChangeMaxCyclesToKeep, maxCyclesShardDataToKeep, } = tallyItem.value.config.stateManager - const devPublicKeys = tallyItem.value.config.debug.devPublicKeys - const devPublicKey = - devPublicKeys && - Object.keys(devPublicKeys).length >= 3 && - Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) - if ( - devPublicKey && - typeof devPublicKey === typeof config.DevPublicKey && - devPublicKey !== config.DevPublicKey - ) - updateConfig({ DevPublicKey: devPublicKey }) + // const devPublicKeys = tallyItem.value.config.debug.devPublicKeys + // const devPublicKey = + // devPublicKeys && + // Object.keys(devPublicKeys).length >= 3 && + // Object.keys(devPublicKeys).find((key) => devPublicKeys[key] === 3) + // if ( + // devPublicKey && + // typeof devPublicKey === typeof config.DevPublicKey && + // devPublicKey !== config.DevPublicKey + // ) + // updateConfig({ DevPublicKey: devPublicKey }) if ( !Utils.isUndefined(newPOQReceipt) && typeof newPOQReceipt === typeof config.newPOQReceipt && diff --git a/src/dbstore/index.ts b/src/dbstore/index.ts index 0e5ae727..9870e439 100644 --- a/src/dbstore/index.ts +++ b/src/dbstore/index.ts @@ -7,10 +7,7 @@ export const initializeDB = async (config: Config): Promise => { 'CREATE TABLE if not exists `transactions` (`txId` TEXT NOT NULL UNIQUE PRIMARY KEY, `appReceiptId` TEXT, `timestamp` BIGINT NOT NULL, `cycleNumber` NUMBER NOT NULL, `data` JSON NOT NULL, `originalTxData` JSON NOT NULL)' ) await db.runCreate( - 'CREATE INDEX if not exists `transactions_cycleNumber` ON `transactions` (`cycleNumber` ASC)' - ) - await db.runCreate( - 'CREATE INDEX if not exists `transactions_timestamp` ON `transactions` (`timestamp` ASC)' + 'CREATE INDEX if not exists `transactions_idx` ON `transactions` (`cycleNumber` DESC, `timestamp` DESC)' ) await db.runCreate( 'CREATE INDEX if not exists `transactions_appReceiptId_idx` ON `transactions` (`appReceiptId`)' @@ -22,22 +19,20 @@ export const initializeDB = async (config: Config): Promise => { await db.runCreate( 'CREATE TABLE if not exists `accounts` (`accountId` TEXT NOT NULL UNIQUE PRIMARY KEY, `data` JSON NOT NULL, `timestamp` BIGINT NOT NULL, `hash` TEXT NOT NULL, `cycleNumber` NUMBER NOT NULL, `isGlobal` BOOLEAN NOT NULL)' ) - await db.runCreate('CREATE INDEX if not exists `accounts_cycleNumber` ON `accounts` (`cycleNumber` ASC)') - await db.runCreate('CREATE INDEX if not exists `accounts_timestamp` ON `accounts` (`timestamp` ASC)') + await db.runCreate( + 'CREATE INDEX if not exists `accounts_idx` ON `accounts` (`cycleNumber` DESC, `timestamp` DESC)' + ) await db.runCreate( 'CREATE TABLE if not exists `receipts` (`receiptId` TEXT NOT NULL UNIQUE PRIMARY KEY, `tx` JSON NOT NULL, `cycle` NUMBER NOT NULL, `timestamp` BIGINT NOT NULL, `beforeStateAccounts` JSON, `accounts` JSON NOT NULL, `appliedReceipt` JSON NOT NULL, `appReceiptData` JSON, `executionShardKey` TEXT NOT NULL, `globalModification` BOOLEAN NOT NULL)' ) - await db.runCreate('CREATE INDEX if not exists `receipts_cycle` ON `receipts` (`cycle` ASC)') - await db.runCreate('CREATE INDEX if not exists `receipts_timestamp` ON `receipts` (`timestamp` ASC)') - + await db.runCreate('CREATE INDEX if not exists `receipts_idx` ON `receipts` (`cycle` ASC, `timestamp` ASC)') await db.runCreate( 'CREATE TABLE if not exists `originalTxsData` (`txId` TEXT NOT NULL, `timestamp` BIGINT NOT NULL, `cycle` NUMBER NOT NULL, `originalTxData` JSON NOT NULL, PRIMARY KEY (`txId`, `timestamp`))' ) - await db.runCreate('CREATE INDEX if not exists `originalTxsData_cycle` ON `originalTxsData` (`cycle` ASC)') await db.runCreate( - 'CREATE INDEX if not exists `originalTxsData_timestamp` ON `originalTxsData` (`timestamp` ASC)' + 'CREATE INDEX if not exists `originalTxsData_idx` ON `originalTxsData` (`cycle` ASC, `timestamp` ASC)' ) - await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId` ON `originalTxsData` (`txId`)') + await db.runCreate('CREATE INDEX if not exists `originalTxsData_txId_idx` ON `originalTxsData` (`txId`)') } export const closeDatabase = async (): Promise => { From d1b989b29a7fb2c0847be18ad4205dcf8b74b952 Mon Sep 17 00:00:00 2001 From: asyed94 Date: Fri, 2 Aug 2024 17:11:01 -0500 Subject: [PATCH 51/51] Introduced two new endpoints, `/is-alive` and `/is-healthy`, for health checks. Documentation updated accordingly. --- README.md | 5 +++++ src/routes/healthCheck.ts | 14 ++++++++++++++ src/server.ts | 2 ++ 3 files changed, 21 insertions(+) create mode 100644 src/routes/healthCheck.ts diff --git a/README.md b/README.md index f630e2f3..da3f96a7 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,11 @@ This is a node that runs as part of the shardus network, with the function of re To release, just run `npm run release` +## Health Check + +GET `/is-alive` this endpoint returns 200 if the server is running. +GET `/is-healthy` currently the same as `/is-alive` but will be expanded. + ## Contributing Contributions are very welcome! Everyone interacting in our codebases, issue trackers, and any other form of communication, including chat rooms and mailing lists, is expected to follow our [code of conduct](./CODE_OF_CONDUCT.md) so we can all enjoy the effort we put into this project. diff --git a/src/routes/healthCheck.ts b/src/routes/healthCheck.ts new file mode 100644 index 00000000..69ab6e2f --- /dev/null +++ b/src/routes/healthCheck.ts @@ -0,0 +1,14 @@ +import { FastifyPluginCallback } from 'fastify' + +export const healthCheckRouter: FastifyPluginCallback = function (fastify, opts, done) { + fastify.get('/is-alive', (req, res) => { + return res.status(200).send('OK') + }) + + fastify.get('/is-healthy', (req, res) => { + // TODO: Add actual health check logic + return res.status(200).send('OK') + }) + + done() +} \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index 80f81529..f05463b7 100644 --- a/src/server.ts +++ b/src/server.ts @@ -40,6 +40,7 @@ import { loadGlobalAccounts, syncGlobalAccount } from './GlobalAccount' import { setShutdownCycleRecord, cycleRecordWithShutDownMode } from './Data/Cycles' import { registerRoutes } from './API' import { Utils as StringUtils } from '@shardus/types' +import { healthCheckRouter } from './routes/healthCheck' const configFile = join(process.cwd(), 'archiver-config.json') let logDir: string @@ -438,6 +439,7 @@ async function startServer(): Promise { timeWindow: 10, allowList: ['127.0.0.1', '0.0.0.0'], // Excludes local IPs from rate limits }) + await server.register(healthCheckRouter) server.addContentTypeParser('application/json', { parseAs: 'string' }, (req, body, done) => { try {