Added composite indexes ( cycle and timesamp ) on each data and added…

… debug logs for account query

Adding global-txreceipt-verification

Added global tx receipt validation and verification

Added account verification for global tx receipt

Updated the new poqo receipt change

src/API.ts

@@ -145,8 +145,6 @@ export function registerRoutes(server: FastifyInstance<Server, IncomingMessage,
         publicKey,
       }
 
-      Data.initSocketClient(firstNode)
-
       // Add first node to NodeList
       NodeList.addNodes(NodeList.NodeStatus.SYNCING, [firstNode])
       // Setting current time for realUpdatedTimes to refresh the nodelist and full-nodelist cache

src/Data/Collector.ts

@@ -219,7 +219,7 @@ const isReceiptRobust = async (
             return { success: false }
           }
         }
-        if (config.verifyAccountData) {
+        if (config.verifyAccountData && receipt.globalModification === false) {
           if (profilerInstance) profilerInstance.profileSectionStart('Verify_receipt_account_data')
           if (nestedCountersInstance)
             nestedCountersInstance.countEvent('receipt', 'Verify_receipt_account_data')
@@ -328,17 +328,51 @@ export const validateArchiverReceipt = (receipt: Receipt.ArchiverReceipt): boole
       return false
     }
   }
-  if (receipt.globalModification) return true
+  if (receipt.globalModification) {
+    const signedReceipt = receipt.signedReceipt as P2PTypes.GlobalAccountsTypes.GlobalTxReceipt
+    err = Utils.validateTypes(signedReceipt, {
+      tx: 'o',
+      signs: 'a',
+    })
+    if (err) {
+      Logger.mainLogger.error('Invalid receipt globalModification data', err)
+      return false
+    }
+    err = Utils.validateTypes(signedReceipt.tx, {
+      address: 's',
+      addressHash: 's',
+      value: 'o',
+      when: 'n',
+      source: 's',
+    })
+    if (err) {
+      Logger.mainLogger.error('Invalid receipt globalModification tx data', err)
+      return false
+    }
+    for (const sign of signedReceipt.signs) {
+      err = Utils.validateTypes(sign, {
+        owner: 's',
+        sig: 's',
+      })
+      if (err) {
+        Logger.mainLogger.error('Invalid receipt globalModification signs data', err)
+        return false
+      }
+    }
+    return true
+  }
   // Global Modification Tx does not have appliedReceipt
+  const signedReceipt = receipt.signedReceipt as Receipt.SignedReceipt
   const signedReceiptToValidate = {
     proposal: 'o',
     proposalHash: 's',
     signaturePack: 'a',
+    voteOffsets: 'a',
   }
   // if (config.newPOQReceipt === false) delete appliedReceiptToValidate.confirmOrChallenge
-  err = Utils.validateTypes(receipt.signedReceipt, signedReceiptToValidate)
+  err = Utils.validateTypes(signedReceipt, signedReceiptToValidate)
   if (err) {
-    Logger.mainLogger.error('Invalid receipt appliedReceipt data', err)
+    Logger.mainLogger.error('Invalid receipt signedReceipt data', err)
     return false
   }
   const proposalToValidate = {
@@ -354,18 +388,25 @@ export const validateArchiverReceipt = (receipt: Receipt.ArchiverReceipt): boole
   // delete appliedVoteToValidate.node_id
   // delete appliedVoteToValidate.sign
   // }
-  err = Utils.validateTypes(receipt.signedReceipt.proposal, proposalToValidate)
+  err = Utils.validateTypes(signedReceipt.proposal, proposalToValidate)
   if (err) {
-    Logger.mainLogger.error('Invalid receipt appliedReceipt appliedVote data', err)
+    Logger.mainLogger.error('Invalid receipt signedReceipt appliedVote data', err)
     return false
   }
-  for (const signature of receipt.signedReceipt.signaturePack) {
+  for (const signature of signedReceipt.signaturePack) {
     err = Utils.validateTypes(signature, {
       owner: 's',
       sig: 's',
     })
     if (err) {
-      Logger.mainLogger.error('Invalid receipt appliedReceipt signatures data', err)
+      Logger.mainLogger.error('Invalid receipt signedReceipt signatures data', err)
+      return false
+    }
+  }
+  for (const voteOffset of signedReceipt.voteOffsets) {
+    const isValid = typeof voteOffset === 'number' || !isNaN(voteOffset)
+    if (isValid) {
+      Logger.mainLogger.error('Invalid receipt signedReceipt voteOffsets data', voteOffset)
       return false
     }
   }
@@ -405,9 +446,7 @@ export const verifyReceiptData = async (
 ): Promise<{ success: boolean; requiredSignatures?: number; newReceipt?: Receipt.ArchiverReceipt }> => {
   const result = { success: false }
   // Check the signed nodes are part of the execution group nodes of the tx
-  const { executionShardKey, cycle, signedReceipt, globalModification } = receipt
-  if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true }
-  const { signaturePack } = signedReceipt
+  const { executionShardKey, cycle, globalModification } = receipt
   const { txId, timestamp } = receipt.tx
   if (config.VERBOSE) {
     const currentTimestamp = Date.now()
@@ -432,6 +471,76 @@ export const verifyReceiptData = async (
   }
   // Determine the home partition index of the primary account (executionShardKey)
   const { homePartition } = ShardFunction.addressToPartition(cycleShardData.shardGlobals, executionShardKey)
+  if (globalModification) {
+    const appliedReceipt = receipt.signedReceipt as P2PTypes.GlobalAccountsTypes.GlobalTxReceipt
+    if (config.skipGlobalTxReceiptVerification) return { success: true }
+    else {
+      const { signs } = appliedReceipt
+      // Refer to https://github.com/shardeum/shardus-core/blob/7d8877b7e1a5b18140f898a64b932182d8a35298/src/p2p/GlobalAccounts.ts#L397
+      let votingGroupCount = cycleShardData.shardGlobals.nodesPerConsenusGroup
+      if (votingGroupCount > cycleShardData.nodes.length) {
+        votingGroupCount = cycleShardData.nodes.length
+      }
+      let isReceiptMajority = (signs.length / votingGroupCount) * 100 >= 60
+      if (!isReceiptMajority) {
+        Logger.mainLogger.error(
+          `Invalid receipt globalModification signs count is less than 60% of the votingGroupCount, ${signs.length}, ${votingGroupCount}`
+        )
+        if (nestedCountersInstance)
+          nestedCountersInstance.countEvent(
+            'receipt',
+            'Invalid_receipt_globalModification_signs_count_less_than_60%'
+          )
+        return result
+      }
+      // Using a set to store the unique signers to avoid duplicates
+      const uniqueSigners = new Set()
+      for (const sign of signs) {
+        const { owner: nodePubKey } = sign
+        // Get the node id from the public key
+        const node = cycleShardData.nodes.find((node) => node.publicKey === nodePubKey)
+        if (node == null) {
+          Logger.mainLogger.error(
+            `The node with public key ${nodePubKey} of the receipt ${txId} with ${timestamp} is not in the active nodesList of cycle ${cycle}`
+          )
+          if (nestedCountersInstance)
+            nestedCountersInstance.countEvent(
+              'receipt',
+              'globalModification_sign_owner_not_in_active_nodesList'
+            )
+          continue
+        }
+        // Check if the node is in the execution group
+        if (!cycleShardData.parititionShardDataMap.get(homePartition).coveredBy[node.id]) {
+          Logger.mainLogger.error(
+            `The node with public key ${nodePubKey} of the receipt ${txId} with ${timestamp} is not in the execution group of the tx`
+          )
+          if (nestedCountersInstance)
+            nestedCountersInstance.countEvent(
+              'receipt',
+              'globalModification_sign_node_not_in_execution_group_of_tx'
+            )
+          continue
+        }
+        uniqueSigners.add(nodePubKey)
+      }
+      isReceiptMajority = (uniqueSigners.size / votingGroupCount) * 100 >= 60
+      if (isReceiptMajority) {
+        Logger.mainLogger.error(
+          `Invalid receipt globalModification valid signs count is less than votingGroupCount ${uniqueSigners.size}, ${votingGroupCount}`
+        )
+        if (nestedCountersInstance)
+          nestedCountersInstance.countEvent(
+            'receipt',
+            'Invalid_receipt_globalModification_valid_signs_count_less_than_votingGroupCount'
+          )
+        return result
+      }
+      const requiredSignatures = Math.floor(votingGroupCount * (60 / 100))
+      return { success: true, requiredSignatures }
+    }
+  }
+  const { signaturePack } = receipt.signedReceipt as Receipt.SignedReceipt
   if (config.newPOQReceipt === false) {
     // Refer to https://github.com/shardeum/shardus-core/blob/f7000c36faa0cd1e0832aa1e5e3b1414d32dcf66/src/state-manager/TransactionConsensus.ts#L1406
     let votingGroupCount = cycleShardData.shardGlobals.nodesPerConsenusGroup
@@ -497,7 +606,8 @@ export const verifyReceiptData = async (
     }
     return { success: true, requiredSignatures }
   }
-  // const { confirmOrChallenge } = appliedReceipt
+
+  // const { confirmOrChallenge } = appliedReceipt as Receipt.AppliedReceipt2
   // // Check if the appliedVote node is in the execution group
   // if (!cycleShardData.nodeShardDataMap.has(appliedVote.node_id)) {
   //   Logger.mainLogger.error('Invalid receipt appliedReceipt appliedVote node is not in the active nodesList')
@@ -612,10 +722,34 @@ const verifyAppliedReceiptSignatures = (
   nestedCounterMessages = []
 ): { success: boolean } => {
   const result = { success: false, failedReasons, nestedCounterMessages }
-  const { signedReceipt, globalModification } = receipt
-  if (globalModification && config.skipGlobalTxReceiptVerification) return { success: true }
-  const { proposal, signaturePack, voteOffsets } = signedReceipt
+  const { globalModification } = receipt
   const { txId: txid } = receipt.tx
+  if (globalModification) {
+    const appliedReceipt = receipt.signedReceipt as P2PTypes.GlobalAccountsTypes.GlobalTxReceipt
+    // Refer to https://github.com/shardeum/shardus-core/blob/7d8877b7e1a5b18140f898a64b932182d8a35298/src/p2p/GlobalAccounts.ts#L294
+
+    const { signs, tx } = appliedReceipt
+    // Using a map to store the good signatures to avoid duplicates
+    const goodSignatures = new Map()
+    for (const sign of signs) {
+      if (Crypto.verify({ ...tx, sign: sign })) {
+        goodSignatures.set(sign.owner, sign)
+        // Break the loop if the required number of good signatures are found
+        if (goodSignatures.size >= requiredSignatures) break
+      }
+    }
+    if (goodSignatures.size < requiredSignatures) {
+      failedReasons.push(
+        `Invalid receipt globalModification valid signs count is less than requiredSignatures ${txid}, ${goodSignatures.size}, ${requiredSignatures}`
+      )
+      nestedCounterMessages.push(
+        'Invalid_receipt_globalModification_valid_signs_count_less_than_requiredSignatures'
+      )
+      return result
+    }
+    return { success: true }
+  }
+  const { proposal, signaturePack, voteOffsets } = receipt.signedReceipt as Receipt.SignedReceipt
   // Refer to https://github.com/shardeum/shardus-core/blob/50b6d00f53a35996cd69210ea817bee068a893d6/src/state-manager/TransactionConsensus.ts#L2799
   const voteHash = calculateVoteHash(proposal, failedReasons, nestedCounterMessages)
   // Refer to https://github.com/shardeum/shardus-core/blob/50b6d00f53a35996cd69210ea817bee068a893d6/src/state-manager/TransactionConsensus.ts#L2663
@@ -639,7 +773,7 @@ const verifyAppliedReceiptSignatures = (
   }
   if (goodSignatures.size < requiredSignatures) {
     failedReasons.push(
-      `Invalid receipt signedReceipt valid signatures count is less than requiredSignatures ${goodSignatures.size}, ${requiredSignatures}`
+      `Invalid receipt signedReceipt valid signatures count is less than requiredSignatures ${txid}, ${goodSignatures.size}, ${requiredSignatures}`
     )
     nestedCounterMessages.push(
       'Invalid_receipt_signedReceipt_valid_signatures_count_less_than_requiredSignatures'
@@ -869,16 +1003,17 @@ export const storeReceiptData = async (
     //   receiptId: tx.txId,
     //   timestamp: tx.timestamp,
     // })
-    const { afterStates, cycle, tx, appReceiptData, signedReceipt } = receipt
-    receipt.beforeStates = config.storeReceiptBeforeStates ? receipt.beforeStates : []
-
-    const sortedVoteOffsets = (signedReceipt.voteOffsets ?? []).sort()
+    const { afterStates, cycle, tx, appReceiptData, signedReceipt, globalModification } = receipt
+    const sortedVoteOffsets = globalModification
+      ? []
+      : (signedReceipt as Receipt.SignedReceipt).voteOffsets.sort()
     const medianOffset = sortedVoteOffsets[Math.floor(sortedVoteOffsets.length / 2)] ?? 0
     const applyTimestamp = tx.timestamp + medianOffset * 1000
     if (config.VERBOSE) console.log('RECEIPT', 'Save', txId, timestamp, senderInfo)
     processedReceiptsMap.set(tx.txId, tx.timestamp)
     receiptsInValidationMap.delete(tx.txId)
     if (missingReceiptsMap.has(tx.txId)) missingReceiptsMap.delete(tx.txId)
+    receipt.beforeStates = globalModification || config.storeReceiptBeforeStates ? receipt.beforeStates : [] // Store beforeStates for globalModification tx, or if config.storeReceiptBeforeStates is true
     combineReceipts.push({
       ...receipt,
       receiptId: tx.txId,

src/State.ts

@@ -40,7 +40,11 @@ export let isFirst = false
 export let isActive = false
 export const archiversReputation: Map<string, string> = new Map()
 
-export async function initFromConfig(config: Config, shutDownMode = false, useArchiverDiscovery = true): Promise<void> {
+export async function initFromConfig(
+  config: Config,
+  shutDownMode = false,
+  useArchiverDiscovery = true
+): Promise<void> {
   // Get own nodeInfo from config
   nodeState.ip = config.ARCHIVER_IP
   nodeState.port = config.ARCHIVER_PORT

src/dbstore/index.ts

@@ -73,6 +73,7 @@ export const initializeDB = async (config: Config): Promise<void> => {
     receiptDatabase,
     'CREATE INDEX if not exists `receipts_timestamp` ON `receipts` (`timestamp` ASC)'
   )
+  await runCreate(receiptDatabase, 'CREATE INDEX if not exists `receipts_cycle` ON `receipts` (`cycle` ASC)')
   await runCreate(
     receiptDatabase,
     'CREATE INDEX if not exists `receipts_cycle_timestamp` ON `receipts` (`cycle` ASC, `timestamp` ASC)'

src/dbstore/receipts.ts

@@ -1,4 +1,5 @@
 import { Signature } from '@shardus/crypto-utils'
+import { P2P } from '@shardus/types'
 import * as db from './sqlite3storage'
 import { receiptDatabase } from '.'
 import * as Logger from '../Logger'
@@ -34,7 +35,7 @@ export interface ArchiverReceipt {
     timestamp: number
   }
   cycle: number
-  signedReceipt: SignedReceipt
+  signedReceipt: SignedReceipt | P2P.GlobalAccountsTypes.GlobalTxReceipt
   afterStates?: AccountsCopy[]
   beforeStates?: AccountsCopy[]
   appReceiptData: object & { accountId?: string; data: object }

src/primary-process/index.ts

@@ -160,7 +160,7 @@ const setupWorkerListeners = (worker: Worker): void => {
         }
         break
       default:
-        if (type && type.includes('axm')) {
+        if (type && typeof type === 'string' && type.includes('axm')) {
           if (config.VERBOSE) {
             console.log(`Worker ${workerId} is sending axm message: ${type}`)
             console.log(data)

src/shardeum/calculateAccountHash.ts

@@ -1,7 +1,6 @@
-import { config } from '../Config'
 import * as crypto from '../Crypto'
-import * as Logger from '../Logger'
-import { ArchiverReceipt } from '../dbstore/receipts'
+import { ArchiverReceipt, SignedReceipt } from '../dbstore/receipts'
+import { verifyGlobalTxAccountChange } from './verifyGlobalTxReceipt'
 
 // account types in Shardeum
 export enum AccountType {
@@ -63,18 +62,27 @@ export const verifyAccountHash = (
   nestedCounterMessages = []
 ): boolean => {
   try {
-    if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
-    const { accountIDs, afterStateHashes, beforeStateHashes } = receipt.signedReceipt.proposal
+    if (receipt.globalModification) {
+      const result = verifyGlobalTxAccountChange(receipt, failedReasons, nestedCounterMessages)
+      if (!result) return false
+      return true
+    }
+    const signedReceipt = receipt.signedReceipt as SignedReceipt
+    const { accountIDs, afterStateHashes, beforeStateHashes } = signedReceipt.proposal
     if (accountIDs.length !== afterStateHashes.length) {
       failedReasons.push(
         `Modified account count specified in the receipt and the actual updated account count does not match! ${receipt.tx.txId} , ${receipt.cycle} , ${receipt.tx.timestamp}`
       )
+      nestedCounterMessages.push(
+        `Modified account count specified in the receipt and the actual updated account count does not match!`
+      )
       return false
     }
     if (beforeStateHashes.length !== afterStateHashes.length) {
       failedReasons.push(
         `Account state hash before and after count does not match! ${receipt.tx.txId} , ${receipt.cycle} , ${receipt.tx.timestamp}`
       )
+      nestedCounterMessages.push(`Account state hash before and after count does not match!`)
       return false
     }
     for (const [index, accountId] of accountIDs.entries()) {
@@ -83,6 +91,7 @@ export const verifyAccountHash = (
         failedReasons.push(
           `Account not found in the receipt's afterStates | Acc-ID: ${accountId}, txId: ${receipt.tx.txId}, Cycle: ${receipt.cycle}, timestamp: ${receipt.tx.timestamp}`
         )
+        nestedCounterMessages.push(`Account not found in the receipt`)
         return false
       }
       const calculatedAccountHash = accountSpecificHash(accountData.data)
@@ -92,12 +101,15 @@ export const verifyAccountHash = (
         failedReasons.push(
           `Account hash does not match | Acc-ID: ${accountId}, txId: ${receipt.tx.txId}, Cycle: ${receipt.cycle}, timestamp: ${receipt.tx.timestamp}`
         )
+        nestedCounterMessages.push(`Account hash does not match`)
         return false
       }
     }
     return true
   } catch (e) {
-    failedReasons.push('Error in verifyAccountHash', e)
+    console.error(`Error in verifyAccountHash`, e)
+    failedReasons.push(`Error in verifyAccountHash ${e}`)
+    nestedCounterMessages.push('Error in verifyAccountHash')
     return false
   }
 }