From 0aeae267b5409eba4f2dbf476c52a71c163c204d Mon Sep 17 00:00:00 2001 From: twoeths Date: Thu, 12 Sep 2024 14:58:36 +0700 Subject: [PATCH] feat: consume forEach api of ssz (#7079) * fix: remove get*Iter() apis from ssz * feat: decompose validators at epoch transition * fix: improve beforeProcessEpoch() using state.validators.forEachValue() * fix: only populate isCompoundingValidatorArr if electra * fix: use forEach() to improve processEth1Data * fix: use forEach() for getEffectiveBalanceIncrementsZeroInactive() * chore: minimal change of beforeProcessEpoch() compared to te/batch_hash_tree_root --- .../src/block/processEth1Data.ts | 13 ++--- .../src/cache/epochTransitionCache.ts | 48 +++++++++---------- .../epoch/processEffectiveBalanceUpdates.ts | 7 +-- .../state-transition/src/stateTransition.ts | 1 - packages/state-transition/src/util/balance.ts | 20 ++------ 5 files changed, 32 insertions(+), 57 deletions(-) diff --git a/packages/state-transition/src/block/processEth1Data.ts b/packages/state-transition/src/block/processEth1Data.ts index fe40f026028..5523b1dbb6d 100644 --- a/packages/state-transition/src/block/processEth1Data.ts +++ b/packages/state-transition/src/block/processEth1Data.ts @@ -1,5 +1,5 @@ import {Node} from "@chainsafe/persistent-merkle-tree"; -import {CompositeViewDU, ReusableListIterator} from "@chainsafe/ssz"; +import {CompositeViewDU} from "@chainsafe/ssz"; import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, ssz} from "@lodestar/types"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js"; @@ -23,10 +23,6 @@ export function processEth1Data(state: CachedBeaconStateAllForks, eth1Data: phas state.eth1DataVotes.push(eth1DataView); } -/** - * This data is reused and never gc. - */ -const eth1DataVotes = new ReusableListIterator>(); /** * Returns true if adding the given `eth1Data` to `state.eth1DataVotes` would * result in a change to `state.eth1Data`. @@ -52,14 +48,11 @@ export function becomesNewEth1Data( // Then isEqualEth1DataView compares cached roots (HashObject as of Jan 2022) which is much cheaper // than doing structural equality, which requires tree -> value conversions let sameVotesCount = 0; - eth1DataVotes.reset(); - state.eth1DataVotes.getAllReadonlyIter(eth1DataVotes); - eth1DataVotes.clean(); - for (const eth1DataVote of eth1DataVotes) { + state.eth1DataVotes.forEach((eth1DataVote) => { if (isEqualEth1DataView(eth1DataVote, newEth1Data)) { sameVotesCount++; } - } + }); // The +1 is to account for the `eth1Data` supplied to the function. if ((sameVotesCount + 1) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD) { diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 7e6aea0e92e..435291eafe6 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex, phase0} from "@lodestar/types"; +import {Epoch, ValidatorIndex} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; @@ -13,7 +13,12 @@ import { FLAG_CURR_TARGET_ATTESTER, FLAG_CURR_HEAD_ATTESTER, } from "../util/attesterStatus.js"; -import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js"; +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStatePhase0, + hasCompoundingWithdrawalCredential, +} from "../index.js"; import {computeBaseRewardPerIncrement} from "../util/altair.js"; import {processPendingAttestations} from "../epoch/processPendingAttestations.js"; @@ -127,11 +132,7 @@ export interface EpochTransitionCache { flags: number[]; - /** - * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly. - * Note that during epoch processing, validators could be updated so need to use it with care. - */ - validators: phase0.Validator[]; + isCompoundingValidatorArr: boolean[]; /** * This is for electra only @@ -210,10 +211,9 @@ const inclusionDelays = new Array(); const flags = new Array(); /** WARNING: reused, never gc'd */ const nextEpochShufflingActiveValidatorIndices = new Array(); -/** - * This data is reused and never gc. - */ -const validators: phase0.Validator[] = []; +/** WARNING: reused, never gc'd */ +const isCompoundingValidatorArr = new Array(); + const previousEpochParticipation = new Array(); const currentEpochParticipation = new Array(); @@ -237,15 +237,10 @@ export function beforeProcessEpoch( const indicesToEject: ValidatorIndex[] = []; let totalActiveStakeByIncrement = 0; - - // To optimize memory each validator node in `state.validators` is represented with a special node type - // `BranchNodeStruct` that represents the data as struct internally. This utility grabs the struct data directly - // from the nodes without any extra transformation. The returned `validators` array contains native JS objects. - validators.length = state.validators.length; - state.validators.getAllReadonlyValues(validators); - - const validatorCount = validators.length; - + const validatorCount = state.validators.length; + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr.length = validatorCount; + } nextEpochShufflingActiveValidatorIndices.length = validatorCount; let nextEpochShufflingActiveIndicesLength = 0; // pre-fill with true (most validators are active) @@ -275,11 +270,13 @@ export function beforeProcessEpoch( const effectiveBalancesByIncrements = epochCtx.effectiveBalanceIncrements; - // for (let i = 0; i < validatorCount; i++) { - let i = 0; - for (const validator of validators) { + state.validators.forEachValue((validator, i) => { let flag = 0; + if (forkSeq >= ForkSeq.electra) { + isCompoundingValidatorArr[i] = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials); + } + if (validator.slashed) { if (slashingsEpoch === validator.withdrawableEpoch) { indicesToSlash.push(i); @@ -370,8 +367,7 @@ export function beforeProcessEpoch( if (isActiveNext2) { nextEpochShufflingActiveValidatorIndices[nextEpochShufflingActiveIndicesLength++] = i; } - i++; - } + }); if (totalActiveStakeByIncrement < 1) { totalActiveStakeByIncrement = 1; @@ -508,7 +504,7 @@ export function beforeProcessEpoch( proposerIndices, inclusionDelays, flags, - validators, + isCompoundingValidatorArr, // will be assigned in processPendingConsolidations() newCompoundingValidators: undefined, // Will be assigned in processRewardsAndPenalties() diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index 0ea4b49dddf..9203c0419a9 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -10,7 +10,6 @@ import { TIMELY_TARGET_FLAG_INDEX, } from "@lodestar/params"; import {EpochTransitionCache, CachedBeaconStateAllForks, BeaconStateAltair} from "../types.js"; -import {hasCompoundingWithdrawalCredential} from "../util/electra.js"; /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; @@ -45,7 +44,7 @@ export function processEffectiveBalanceUpdates( // and updated in processPendingBalanceDeposits() and processPendingConsolidations() // so it's recycled here for performance. const balances = cache.balances ?? state.balances.getAll(); - const currentEpochValidators = cache.validators; + const {isCompoundingValidatorArr} = cache; const newCompoundingValidators = cache.newCompoundingValidators ?? new Set(); let numUpdate = 0; @@ -61,9 +60,7 @@ export function processEffectiveBalanceUpdates( effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; } else { // from electra, effectiveBalanceLimit is per validator - const isCompoundingValidator = - hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) || - newCompoundingValidators.has(i); + const isCompoundingValidator = isCompoundingValidatorArr[i] || newCompoundingValidators.has(i); effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE; } diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index dcbb8067d94..171ef089b5a 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -124,7 +124,6 @@ export function stateTransition( const stateRoot = postState.batchHashTreeRoot(hcGroup); hashTreeRootTimer?.(); - if (metrics) { onPostStateMetrics(postState, metrics); } diff --git a/packages/state-transition/src/util/balance.ts b/packages/state-transition/src/util/balance.ts index eca6372c220..a1b086cbd59 100644 --- a/packages/state-transition/src/util/balance.ts +++ b/packages/state-transition/src/util/balance.ts @@ -1,6 +1,5 @@ -import {ReusableListIterator} from "@chainsafe/ssz"; import {EFFECTIVE_BALANCE_INCREMENT} from "@lodestar/params"; -import {Gwei, ValidatorIndex, phase0} from "@lodestar/types"; +import {Gwei, ValidatorIndex} from "@lodestar/types"; import {bigIntMax} from "@lodestar/utils"; import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js"; import {BeaconStateAllForks} from ".."; @@ -44,11 +43,6 @@ export function decreaseBalance(state: BeaconStateAllForks, index: ValidatorInde state.balances.set(index, Math.max(0, newBalance)); } -/** - * This data is reused and never gc. - */ -const validators = new ReusableListIterator(); - /** * This method is used to get justified balances from a justified state. * This is consumed by forkchoice which based on delta so we return "by increment" (in ether) value, @@ -69,16 +63,13 @@ export function getEffectiveBalanceIncrementsZeroInactive( validatorCount ); - validators.reset(); - justifiedState.validators.getAllReadonlyValuesIter(validators); - validators.clean(); - let i = 0; let j = 0; - for (const validator of validators) { + justifiedState.validators.forEachValue((validator, i) => { + const {slashed} = validator; if (i === activeIndices[j]) { // active validator j++; - if (validator.slashed) { + if (slashed) { // slashed validator effectiveBalanceIncrementsZeroInactive[i] = 0; } @@ -86,8 +77,7 @@ export function getEffectiveBalanceIncrementsZeroInactive( // inactive validator effectiveBalanceIncrementsZeroInactive[i] = 0; } - i++; - } + }); return effectiveBalanceIncrementsZeroInactive; }