From 05b4d75e9bcc5396de6a961d308f11d29f0c5420 Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Sun, 16 Jun 2024 09:33:50 -0500 Subject: [PATCH 1/6] baked in newPOQReceipt = false --- src/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.ts b/src/Config.ts index 825e00b7..401af2c6 100644 --- a/src/Config.ts +++ b/src/Config.ts @@ -109,7 +109,7 @@ let config: Config = { cycleRecordsCache: { enabled: false, }, - newPOQReceipt: true, + newPOQReceipt: false, } // Override default config params from config file, env vars, and cli args export async function overrideDefaultConfig(file: string): Promise { From e4e040571fd229a65743701eb6fadc311290be78 Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Sun, 16 Jun 2024 09:37:03 -0500 Subject: [PATCH 2/6] 3.4.20 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b153085b..b761f434 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shardus/archiver", - "version": "3.4.19", + "version": "3.4.20", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shardus/archiver", - "version": "3.4.19", + "version": "3.4.20", "license": "ISC", "dependencies": { "@fastify/cors": "^8.2.0", diff --git a/package.json b/package.json index 87d86a8e..5ca1e07b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shardus/archiver", - "version": "3.4.19", + "version": "3.4.20", "engines": { "node": "18.16.1" }, From a270eade92d94e2cdb0616a898b5083ef58e56ce Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 17 Jun 2024 12:28:34 +0545 Subject: [PATCH 3/6] Added the before/after account hash exist check in the appliedReceipt --- src/shardeum/verifyAppReceiptData.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/shardeum/verifyAppReceiptData.ts b/src/shardeum/verifyAppReceiptData.ts index 803ccb88..0a9e78bd 100644 --- a/src/shardeum/verifyAppReceiptData.ts +++ b/src/shardeum/verifyAppReceiptData.ts @@ -26,6 +26,14 @@ export const verifyAppReceiptData = async ( appliedReceipt.appliedVote.account_state_hash_after.length > 0 ) { for (let i = 0; i < appliedReceipt.appliedVote.account_id.length; i++) { + if ( + !appliedReceipt.appliedVote.account_state_hash_before[i] || + !appliedReceipt.appliedVote.account_state_hash_after[i] + ) { + Logger.mainLogger.error( + `The account state hash before or after is missing in the receipt! ${receipt.tx.txId} , ${receipt.cycle} , ${receipt.tx.timestamp}` + ) + } if ( // eslint-disable-next-line security/detect-object-injection appliedReceipt.appliedVote.account_state_hash_before[i] !== From b5333f96d9b5515dd99fd6055778a7fde7aee0ed Mon Sep 17 00:00:00 2001 From: jairajdev Date: Mon, 17 Jun 2024 17:36:08 +0545 Subject: [PATCH 4/6] Additional hardening for old POQ --- src/Data/Collector.ts | 27 +++++++++++++++++---------- src/shardeum/calculateAccountHash.ts | 9 +++++++++ src/shardeum/verifyAppReceiptData.ts | 3 +++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/Data/Collector.ts b/src/Data/Collector.ts index 0fde302a..760bcdbc 100644 --- a/src/Data/Collector.ts +++ b/src/Data/Collector.ts @@ -123,7 +123,7 @@ const isReceiptRobust = async ( if (config.VERBOSE) Logger.mainLogger.debug('robustQuery', receipt.tx.txId, robustQuery) if (!robustQuery || !robustQuery.value || !(robustQuery.value as any).receipt) { Logger.mainLogger.error( - `❌ 'null' response from all nodes in receipt-validation for txId: ${receipt.tx.txId} , ${receipt.cycle}, ${receipt.tx.timestamp}) + `❌ 'null' response from all nodes in receipt-validation for txId: ${receipt.tx.txId} , ${receipt.cycle}, ${receipt.tx.timestamp} }` ) if (nestedCountersInstance) @@ -235,7 +235,10 @@ const isReceiptRobust = async ( Logger.mainLogger.error( `The receipt validation failed from robustQuery nodes ${receipt.tx.txId} , ${receipt.cycle}, ${receipt.tx.timestamp}` ) - Logger.mainLogger.error(StringUtils.safeStringify(robustQueryReceipt), StringUtils.safeStringify(fullReceipt)) + Logger.mainLogger.error( + StringUtils.safeStringify(robustQueryReceipt), + StringUtils.safeStringify(fullReceipt) + ) } } Logger.mainLogger.error( @@ -384,7 +387,7 @@ export const validateReceiptData = (receipt: Receipt.ArchiverReceipt): boolean = export const verifyReceiptData = async ( receipt: Receipt.ArchiverReceipt, - checkReceiptRobust: boolean = true + checkReceiptRobust = true ): Promise<{ success: boolean; newReceipt?: Receipt.ArchiverReceipt }> => { if (config.newPOQReceipt === false) return { success: true } const result = { success: false } @@ -641,6 +644,7 @@ export const storeReceiptData = async ( txDataList.push({ txId, timestamp }) // If the receipt is a challenge, then skip updating its accounts data or transaction data if ( + config.newPOQReceipt === true && appliedReceipt && appliedReceipt.confirmOrChallenge && appliedReceipt.confirmOrChallenge.message === 'challenge' @@ -796,7 +800,8 @@ export const storeCycleData = async (cycles: P2PTypes.CycleCreatorTypes.CycleDat cycleMarker: cycleRecord.marker, cycleRecord, } - if (config.dataLogWrite && CycleLogWriter) CycleLogWriter.writeToLog(`${StringUtils.safeStringify(cycleObj)}\n`) + if (config.dataLogWrite && CycleLogWriter) + CycleLogWriter.writeToLog(`${StringUtils.safeStringify(cycleObj)}\n`) const cycleExist = await queryCycleByMarker(cycleObj.cycleMarker) if (cycleExist) { if (StringUtils.safeStringify(cycleObj) !== StringUtils.safeStringify(cycleExist)) @@ -1247,9 +1252,10 @@ export function cleanOldReceiptsMap(timestamp: number): void { savedReceiptsCount++ } } - Logger.mainLogger.debug( - `Clean ${savedReceiptsCount} old receipts from the processed receipts cache on cycle ${getCurrentCycleCounter()}` - ) + if (savedReceiptsCount > 0) + Logger.mainLogger.debug( + `Clean ${savedReceiptsCount} old receipts from the processed receipts cache on cycle ${getCurrentCycleCounter()}` + ) } export function cleanOldOriginalTxsMap(timestamp: number): void { @@ -1262,9 +1268,10 @@ export function cleanOldOriginalTxsMap(timestamp: number): void { savedOriginalTxsCount++ } } - Logger.mainLogger.debug( - `Clean ${savedOriginalTxsCount} old originalTxsData from the processed originalTxsData cache on cycle ${getCurrentCycleCounter()}` - ) + if (savedOriginalTxsCount > 0) + Logger.mainLogger.debug( + `Clean ${savedOriginalTxsCount} old originalTxsData from the processed originalTxsData cache on cycle ${getCurrentCycleCounter()}` + ) } export const scheduleMissingTxsDataQuery = (): void => { diff --git a/src/shardeum/calculateAccountHash.ts b/src/shardeum/calculateAccountHash.ts index c2f5b4ec..61e6853e 100644 --- a/src/shardeum/calculateAccountHash.ts +++ b/src/shardeum/calculateAccountHash.ts @@ -77,6 +77,15 @@ export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => { ) return false } + if ( + receipt.appliedReceipt.appliedVote.account_state_hash_before.length !== + receipt.appliedReceipt.appliedVote.account_state_hash_after.length + ) { + Logger.mainLogger.error( + `Account state hash before and after count does not match! ${receipt.tx.txId} , ${receipt.cycle} , ${receipt.tx.timestamp}` + ) + return false + } for (const account of receipt.accounts) { if (account.data.accountType === AccountType.Account) { fixAccountUint8Arrays(account.data.account) diff --git a/src/shardeum/verifyAppReceiptData.ts b/src/shardeum/verifyAppReceiptData.ts index 0a9e78bd..df8b50bf 100644 --- a/src/shardeum/verifyAppReceiptData.ts +++ b/src/shardeum/verifyAppReceiptData.ts @@ -27,7 +27,9 @@ export const verifyAppReceiptData = async ( ) { for (let i = 0; i < appliedReceipt.appliedVote.account_id.length; i++) { if ( + // eslint-disable-next-line security/detect-object-injection !appliedReceipt.appliedVote.account_state_hash_before[i] || + // eslint-disable-next-line security/detect-object-injection !appliedReceipt.appliedVote.account_state_hash_after[i] ) { Logger.mainLogger.error( @@ -52,6 +54,7 @@ export const verifyAppReceiptData = async ( // If the existing receipt is challenged and the new receipt is confirmed, overwrite the existing receipt let skipAppReceiptCheck = false if ( + config.newPOQReceipt === true && existingReceipt.appliedReceipt && existingReceipt.appliedReceipt.confirmOrChallenge && receipt.appliedReceipt && From 00e6712ca7df428759976a66906836db3fc33e31 Mon Sep 17 00:00:00 2001 From: Tanuj Soni Date: Mon, 17 Jun 2024 14:45:38 +0530 Subject: [PATCH 5/6] fix: current-cycle-hash resp type issue --- src/sync-v2/index.ts | 8 ++++---- src/sync-v2/queries.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sync-v2/index.ts b/src/sync-v2/index.ts index d13359c4..cfc1ab13 100644 --- a/src/sync-v2/index.ts +++ b/src/sync-v2/index.ts @@ -192,11 +192,11 @@ function syncLatestCycleRecordAndMarker( activeNodes: P2PTypes.SyncTypes.ActiveNode[] ): ResultAsync<[P2PTypes.CycleCreatorTypes.CycleData, hexstring], Error> { // run a robust query for the latest cycle record hash - return robustQueryForCycleRecordHash(activeNodes).andThen(({ value: cycleRecordHash, winningNodes }) => + return robustQueryForCycleRecordHash(activeNodes).andThen(({ value, winningNodes }) => // get current cycle record from node - getCurrentCycleDataFromNode(winningNodes[0], cycleRecordHash).andThen((cycleRecord) => - verifyCycleRecord(cycleRecord, cycleRecordHash).map( - () => [cycleRecord, cycleRecordHash] as [P2PTypes.CycleCreatorTypes.CycleData, hexstring] + getCurrentCycleDataFromNode(winningNodes[0], value.currentCycleHash).andThen((cycleRecord) => + verifyCycleRecord(cycleRecord, value.currentCycleHash).map( + () => [cycleRecord, value.currentCycleHash] as [P2PTypes.CycleCreatorTypes.CycleData, hexstring] ) ) ) diff --git a/src/sync-v2/queries.ts b/src/sync-v2/queries.ts index 57e4dae5..46e1b96b 100644 --- a/src/sync-v2/queries.ts +++ b/src/sync-v2/queries.ts @@ -119,7 +119,7 @@ function attemptSimpleFetch( } /** Executes a robust query to retrieve the cycle marker from the network. */ -export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync { +export function robustQueryForCycleRecordHash(nodes: ActiveNode[]): RobustQueryResultAsync<{ currentCycleHash: hexstring }> { return makeRobustQueryCall(nodes, 'current-cycle-hash') } From 13cfc2e90940339ca6c96e185c40dddb97be381a Mon Sep 17 00:00:00 2001 From: andrew <44451818+afostr@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:13:16 -0500 Subject: [PATCH 6/6] 3.4.21 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index b761f434..194a52a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@shardus/archiver", - "version": "3.4.20", + "version": "3.4.21", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@shardus/archiver", - "version": "3.4.20", + "version": "3.4.21", "license": "ISC", "dependencies": { "@fastify/cors": "^8.2.0", diff --git a/package.json b/package.json index 5ca1e07b..a3bf835b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@shardus/archiver", - "version": "3.4.20", + "version": "3.4.21", "engines": { "node": "18.16.1" },