Skip to content

Commit

Permalink
- Restricted to only the first validator can forward the accounts dat…
Browse files Browse the repository at this point in the history 
…a to the archiver during early cycles

- Remove accounts transformation used in account verification
  • Loading branch information
@jairajdev
jairajdev committed Jul 22, 2024
1 parent 8756e0a commit 30ee7bc
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 63 deletions.
39 changes: 29 additions & 10 deletions src/Data/Collector.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import { CycleLogWriter, ReceiptLogWriter, OriginalTxDataLogWriter } from '../Da
import * as OriginalTxDB from '../dbstore/originalTxsData'
import ShardFunction from '../ShardFunctions'
import { ConsensusNodeInfo } from '../NodeList'
import { verifyAccountHash } from '../shardeum/calculateAccountHash'
import { accountSpecificHash, verifyAccountHash } from '../shardeum/calculateAccountHash'
import { verifyAppReceiptData } from '../shardeum/verifyAppReceiptData'
import { Cycle as DbCycle } from '../dbstore/types'
import { Utils as StringUtils } from '@shardus/types'
Expand Down Expand Up @@ -927,12 +927,11 @@ interface StoreAccountParam {
}

export const storeAccountData = async (restoreData: StoreAccountParam = {}): Promise<void> => {
console.log(
'RestoreData',
'accounts',
restoreData.accounts ? restoreData.accounts.length : 0,
'receipts',
restoreData.receipts ? restoreData.receipts.length : 0
Logger.mainLogger.debug(
`storeAccountData: ${restoreData.accounts ? restoreData.accounts.length : 0} accounts`
)
Logger.mainLogger.debug(
`storeAccountData: ${restoreData.receipts ? restoreData.receipts.length : 0} receipts`
)
const { accounts, receipts } = restoreData
if (profilerInstance) profilerInstance.profileSectionStart('store_account_data')
Expand All @@ -953,7 +952,28 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro
// // await Account.insertAccount(account)
// // }
// }
if (accounts && accounts.length > 0) await Account.bulkInsertAccounts(accounts)
//
if (accounts && accounts.length > 0) {
const combineAccounts = []
for (const account of accounts) {
try {
const calculatedAccountHash = accountSpecificHash(account.data)
if (calculatedAccountHash !== account.hash) {
Logger.mainLogger.error(
'Invalid account hash',
account.accountId,
account.hash,
calculatedAccountHash
)
continue
}
combineAccounts.push(account)
} catch (error) {
Logger.mainLogger.error('Error in calculating genesis account hash', error)
}
}
if (combineAccounts.length > 0) await Account.bulkInsertAccounts(accounts)
}
if (receipts && receipts.length > 0) {
Logger.mainLogger.debug('Received receipts Size', receipts.length)
const combineTransactions = []
Expand All @@ -971,10 +991,9 @@ export const storeAccountData = async (restoreData: StoreAccountParam = {}): Pro
await Transaction.bulkInsertTransactions(combineTransactions)
}
if (profilerInstance) profilerInstance.profileSectionEnd('store_account_data')
console.log('Combined Accounts Data', combineAccountsData.accounts.length)
Logger.mainLogger.debug('Combined Accounts Data', combineAccountsData.accounts.length)
if (combineAccountsData.accounts.length > 0 || combineAccountsData.receipts.length > 0) {
console.log('Found combine accountsData')
Logger.mainLogger.debug('Found combine accountsData', combineAccountsData.accounts.length)
const accountData = { ...combineAccountsData }
clearCombinedAccountsData()
storeAccountData(accountData)
Expand Down
70 changes: 38 additions & 32 deletions src/Data/Data.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ const {
MAX_BETWEEN_CYCLES_PER_REQUEST,
} = config.REQUEST_LIMIT

const GENESIS_ACCOUNTS_CYCLE_RANGE = {
startCycle: 0,
endCycle: 5,
}

export enum DataRequestTypes {
SUBSCRIBE = 'SUBSCRIBE',
UNSUBSCRIBE = 'UNSUBSCRIBE',
Expand Down Expand Up @@ -280,26 +285,30 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void {
collectCycleData(newData.responses.CYCLE, sender.nodeInfo.ip + ':' + sender.nodeInfo.port)
}
if (newData.responses && newData.responses.ACCOUNT) {
console.log(
'RECEIVED ACCOUNTS DATA',
sender.nodeInfo.publicKey,
sender.nodeInfo.ip,
sender.nodeInfo.port
)
Logger.mainLogger.debug(
'RECEIVED ACCOUNTS DATA',
sender.nodeInfo.publicKey,
sender.nodeInfo.ip,
sender.nodeInfo.port
)
if (getCurrentCycleCounter() > GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle) {
Logger.mainLogger.error(
'Account data is not meant to be received after the genesis cycle',
getCurrentCycleCounter()
)
unsubscribeDataSender(sender.nodeInfo.publicKey)
return
}
if (NodeList.byPublicKey.size > 1 || !NodeList.byPublicKey.has(sender.nodeInfo.publicKey)) {
Logger.mainLogger.error(
'Account data is not meant to be received by the first validator',
`Number of nodes in the network ${NodeList.byPublicKey.size}`
)
unsubscribeDataSender(sender.nodeInfo.publicKey)
return
}
Logger.mainLogger.debug(`RECEIVED ACCOUNTS DATA FROM ${sender.nodeInfo.ip}:${sender.nodeInfo.port}`)
nestedCountersInstance.countEvent('genesis', 'accounts', 1)
if (!forwardGenesisAccounts) {
console.log('Genesis Accounts To Sycn', newData.responses.ACCOUNT)
Logger.mainLogger.debug('Genesis Accounts To Sycn', newData.responses.ACCOUNT)
syncGenesisAccountsFromConsensor(newData.responses.ACCOUNT, sender.nodeInfo)
} else {
if (storingAccountData) {
console.log('Storing Data')
Logger.mainLogger.debug('Storing Account Data')
let newCombineAccountsData = { ...combineAccountsData }
if (newData.responses.ACCOUNT.accounts)
newCombineAccountsData.accounts = [
Expand All @@ -321,17 +330,14 @@ export function initSocketClient(node: NodeList.ConsensusNodeInfo): void {
}

// Set new contactTimeout for sender. Postpone sender removal because data is still received from consensor
if (currentCycleDuration > 0) {
nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout')
// To make sure that the sender is still in the subscribed list
sender = dataSenders.get(newData.publicKey)
if (sender)
sender.contactTimeout = createContactTimeout(
sender.nodeInfo.publicKey,
'This timeout is created after processing data'
)
}
return
nestedCountersInstance.countEvent('archiver', 'postpone_contact_timeout')
// To make sure that the sender is still in the subscribed list
sender = dataSenders.get(newData.publicKey)
if (sender)
sender.contactTimeout = createContactTimeout(
sender.nodeInfo.publicKey,
'This timeout is created after processing data'
)
}
})
}
Expand Down Expand Up @@ -1009,7 +1015,7 @@ export async function syncGenesisAccountsFromArchiver(): Promise<void> {
// }
const res = (await queryFromArchivers(
RequestDataType.ACCOUNT,
{ startCycle: 0, endCycle: 5 },
{ startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle, endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle },
QUERY_TIMEOUT_MAX
)) as ArchiverAccountResponse
if (res && (res.totalAccounts || res.totalAccounts === 0)) {
Expand All @@ -1026,8 +1032,8 @@ export async function syncGenesisAccountsFromArchiver(): Promise<void> {
const response = (await queryFromArchivers(
RequestDataType.ACCOUNT,
{
startCycle: 0,
endCycle: 5,
startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
page,
},
QUERY_TIMEOUT_MAX
Expand Down Expand Up @@ -1059,8 +1065,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise<void> {
const res = (await queryFromArchivers(
RequestDataType.TRANSACTION,
{
startCycle: 0,
endCycle: 5,
startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
},
QUERY_TIMEOUT_MAX
)) as ArchiverTransactionResponse
Expand All @@ -1078,8 +1084,8 @@ export async function syncGenesisTransactionsFromArchiver(): Promise<void> {
const response = (await queryFromArchivers(
RequestDataType.TRANSACTION,
{
startCycle: 0,
endCycle: 5,
startCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.startCycle,
endCycle: GENESIS_ACCOUNTS_CYCLE_RANGE.endCycle,
page,
},
QUERY_TIMEOUT_MAX
Expand Down
21 changes: 0 additions & 21 deletions src/shardeum/calculateAccountHash.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,17 +57,6 @@ export const accountSpecificHash = (account: any): string => {
return hash
}

// Converting the correct account data format to get the correct hash
// eslint-disable-next-line @typescript-eslint/no-explicit-any
export const fixAccountUint8Arrays = (account: any): void => {
if (!account) return // if account is null, return
if (account.storageRoot) account.storageRoot = Uint8Array.from(Object.values(account.storageRoot)) // Account
if (account.codeHash) account.codeHash = Uint8Array.from(Object.values(account.codeHash)) //
//Account and ContractCode
if (account.codeByte) account.codeByte = Uint8Array.from(Object.values(account.codeByte)) // ContractCode
if (account.value) account.value = Uint8Array.from(Object.values(account.value)) // ContractByte
}

export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => {
try {
if (receipt.globalModification && config.skipGlobalTxReceiptVerification) return true // return true if global modification
Expand All @@ -87,16 +76,6 @@ export const verifyAccountHash = (receipt: ArchiverReceipt): boolean => {
return false
}
for (const account of receipt.accounts) {
if (account.data.accountType === AccountType.Account) {
fixAccountUint8Arrays(account.data.account)
// console.dir(acc, { depth: null })
} else if (
account.data.accountType === AccountType.ContractCode ||
account.data.accountType === AccountType.ContractStorage
) {
fixAccountUint8Arrays(account.data)
// console.dir(acc, { depth: null })
}
const calculatedAccountHash = accountSpecificHash(account.data)
const indexOfAccount = receipt.appliedReceipt.appliedVote.account_id.indexOf(account.accountId)
if (indexOfAccount === -1) {
Expand Down

0 comments on commit 30ee7bc

Please sign in to comment.