feat: consume forEach api of ssz (#7079)

* fix: remove get*Iter() apis from ssz

* feat: decompose validators at epoch transition

* fix: improve beforeProcessEpoch() using state.validators.forEachValue()

* fix: only populate isCompoundingValidatorArr if electra

* fix: use forEach() to improve processEth1Data

* fix: use forEach() for getEffectiveBalanceIncrementsZeroInactive()

* chore: minimal change of beforeProcessEpoch() compared to te/batch_hash_tree_root

packages/state-transition/src/block/processEth1Data.ts

@@ -1,5 +1,5 @@
 import {Node} from "@chainsafe/persistent-merkle-tree";
-import {CompositeViewDU, ReusableListIterator} from "@chainsafe/ssz";
+import {CompositeViewDU} from "@chainsafe/ssz";
 import {EPOCHS_PER_ETH1_VOTING_PERIOD, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {phase0, ssz} from "@lodestar/types";
 import {BeaconStateAllForks, CachedBeaconStateAllForks} from "../types.js";
@@ -23,10 +23,6 @@ export function processEth1Data(state: CachedBeaconStateAllForks, eth1Data: phas
   state.eth1DataVotes.push(eth1DataView);
 }
 
-/**
- * This data is reused and never gc.
- */
-const eth1DataVotes = new ReusableListIterator<CompositeViewDU<typeof ssz.phase0.Eth1Data>>();
 /**
  * Returns true if adding the given `eth1Data` to `state.eth1DataVotes` would
  * result in a change to `state.eth1Data`.
@@ -52,14 +48,11 @@ export function becomesNewEth1Data(
   // Then isEqualEth1DataView compares cached roots (HashObject as of Jan 2022) which is much cheaper
   // than doing structural equality, which requires tree -> value conversions
   let sameVotesCount = 0;
-  eth1DataVotes.reset();
-  state.eth1DataVotes.getAllReadonlyIter(eth1DataVotes);
-  eth1DataVotes.clean();
-  for (const eth1DataVote of eth1DataVotes) {
+  state.eth1DataVotes.forEach((eth1DataVote) => {
     if (isEqualEth1DataView(eth1DataVote, newEth1Data)) {
       sameVotesCount++;
     }
-  }
+  });
 
   // The +1 is to account for the `eth1Data` supplied to the function.
   if ((sameVotesCount + 1) * 2 > SLOTS_PER_ETH1_VOTING_PERIOD) {

packages/state-transition/src/cache/epochTransitionCache.ts

@@ -1,4 +1,4 @@
-import {Epoch, ValidatorIndex, phase0} from "@lodestar/types";
+import {Epoch, ValidatorIndex} from "@lodestar/types";
 import {intDiv} from "@lodestar/utils";
 import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params";
 
@@ -13,7 +13,12 @@ import {
   FLAG_CURR_TARGET_ATTESTER,
   FLAG_CURR_HEAD_ATTESTER,
 } from "../util/attesterStatus.js";
-import {CachedBeaconStateAllForks, CachedBeaconStateAltair, CachedBeaconStatePhase0} from "../index.js";
+import {
+  CachedBeaconStateAllForks,
+  CachedBeaconStateAltair,
+  CachedBeaconStatePhase0,
+  hasCompoundingWithdrawalCredential,
+} from "../index.js";
 import {computeBaseRewardPerIncrement} from "../util/altair.js";
 import {processPendingAttestations} from "../epoch/processPendingAttestations.js";
 
@@ -127,11 +132,7 @@ export interface EpochTransitionCache {
 
   flags: number[];
 
-  /**
-   * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly.
-   * Note that during epoch processing, validators could be updated so need to use it with care.
-   */
-  validators: phase0.Validator[];
+  isCompoundingValidatorArr: boolean[];
 
   /**
    * This is for electra only
@@ -210,10 +211,9 @@ const inclusionDelays = new Array<number>();
 const flags = new Array<number>();
 /** WARNING: reused, never gc'd */
 const nextEpochShufflingActiveValidatorIndices = new Array<number>();
-/**
- * This data is reused and never gc.
- */
-const validators: phase0.Validator[] = [];
+/** WARNING: reused, never gc'd */
+const isCompoundingValidatorArr = new Array<boolean>();
+
 const previousEpochParticipation = new Array<number>();
 const currentEpochParticipation = new Array<number>();
 
@@ -237,15 +237,10 @@ export function beforeProcessEpoch(
   const indicesToEject: ValidatorIndex[] = [];
 
   let totalActiveStakeByIncrement = 0;
-
-  // To optimize memory each validator node in `state.validators` is represented with a special node type
-  // `BranchNodeStruct` that represents the data as struct internally. This utility grabs the struct data directly
-  // from the nodes without any extra transformation. The returned `validators` array contains native JS objects.
-  validators.length = state.validators.length;
-  state.validators.getAllReadonlyValues(validators);
-
-  const validatorCount = validators.length;
-
+  const validatorCount = state.validators.length;
+  if (forkSeq >= ForkSeq.electra) {
+    isCompoundingValidatorArr.length = validatorCount;
+  }
   nextEpochShufflingActiveValidatorIndices.length = validatorCount;
   let nextEpochShufflingActiveIndicesLength = 0;
   // pre-fill with true (most validators are active)
@@ -275,11 +270,13 @@ export function beforeProcessEpoch(
 
   const effectiveBalancesByIncrements = epochCtx.effectiveBalanceIncrements;
 
-  // for (let i = 0; i < validatorCount; i++) {
-  let i = 0;
-  for (const validator of validators) {
+  state.validators.forEachValue((validator, i) => {
     let flag = 0;
 
+    if (forkSeq >= ForkSeq.electra) {
+      isCompoundingValidatorArr[i] = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials);
+    }
+
     if (validator.slashed) {
       if (slashingsEpoch === validator.withdrawableEpoch) {
         indicesToSlash.push(i);
@@ -370,8 +367,7 @@ export function beforeProcessEpoch(
     if (isActiveNext2) {
       nextEpochShufflingActiveValidatorIndices[nextEpochShufflingActiveIndicesLength++] = i;
     }
-    i++;
-  }
+  });
 
   if (totalActiveStakeByIncrement < 1) {
     totalActiveStakeByIncrement = 1;
@@ -508,7 +504,7 @@ export function beforeProcessEpoch(
     proposerIndices,
     inclusionDelays,
     flags,
-    validators,
+    isCompoundingValidatorArr,
     // will be assigned in processPendingConsolidations()
     newCompoundingValidators: undefined,
     // Will be assigned in processRewardsAndPenalties()

packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts

@@ -10,7 +10,6 @@ import {
   TIMELY_TARGET_FLAG_INDEX,
 } from "@lodestar/params";
 import {EpochTransitionCache, CachedBeaconStateAllForks, BeaconStateAltair} from "../types.js";
-import {hasCompoundingWithdrawalCredential} from "../util/electra.js";
 
 /** Same to https://github.com/ethereum/eth2.0-specs/blob/v1.1.0-alpha.5/specs/altair/beacon-chain.md#has_flag */
 const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX;
@@ -45,7 +44,7 @@ export function processEffectiveBalanceUpdates(
   // and updated in processPendingBalanceDeposits() and processPendingConsolidations()
   // so it's recycled here for performance.
   const balances = cache.balances ?? state.balances.getAll();
-  const currentEpochValidators = cache.validators;
+  const {isCompoundingValidatorArr} = cache;
   const newCompoundingValidators = cache.newCompoundingValidators ?? new Set();
 
   let numUpdate = 0;
@@ -61,9 +60,7 @@ export function processEffectiveBalanceUpdates(
       effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE;
     } else {
       // from electra, effectiveBalanceLimit is per validator
-      const isCompoundingValidator =
-        hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) ||
-        newCompoundingValidators.has(i);
+      const isCompoundingValidator = isCompoundingValidatorArr[i] || newCompoundingValidators.has(i);
       effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE;
     }
 

packages/state-transition/src/stateTransition.ts

@@ -124,7 +124,6 @@ export function stateTransition(
   const stateRoot = postState.batchHashTreeRoot(hcGroup);
   hashTreeRootTimer?.();
 
-
   if (metrics) {
     onPostStateMetrics(postState, metrics);
   }

packages/state-transition/src/util/balance.ts

@@ -1,6 +1,5 @@
-import {ReusableListIterator} from "@chainsafe/ssz";
 import {EFFECTIVE_BALANCE_INCREMENT} from "@lodestar/params";
-import {Gwei, ValidatorIndex, phase0} from "@lodestar/types";
+import {Gwei, ValidatorIndex} from "@lodestar/types";
 import {bigIntMax} from "@lodestar/utils";
 import {EffectiveBalanceIncrements} from "../cache/effectiveBalanceIncrements.js";
 import {BeaconStateAllForks} from "..";
@@ -44,11 +43,6 @@ export function decreaseBalance(state: BeaconStateAllForks, index: ValidatorInde
   state.balances.set(index, Math.max(0, newBalance));
 }
 
-/**
- * This data is reused and never gc.
- */
-const validators = new ReusableListIterator<phase0.Validator>();
-
 /**
  * This method is used to get justified balances from a justified state.
  * This is consumed by forkchoice which based on delta so we return "by increment" (in ether) value,
@@ -69,25 +63,21 @@ export function getEffectiveBalanceIncrementsZeroInactive(
     validatorCount
   );
 
-  validators.reset();
-  justifiedState.validators.getAllReadonlyValuesIter(validators);
-  validators.clean();
-  let i = 0;
   let j = 0;
-  for (const validator of validators) {
+  justifiedState.validators.forEachValue((validator, i) => {
+    const {slashed} = validator;
     if (i === activeIndices[j]) {
       // active validator
       j++;
-      if (validator.slashed) {
+      if (slashed) {
         // slashed validator
         effectiveBalanceIncrementsZeroInactive[i] = 0;
       }
     } else {
       // inactive validator
       effectiveBalanceIncrementsZeroInactive[i] = 0;
     }
-    i++;
-  }
+  });
 
   return effectiveBalanceIncrementsZeroInactive;
 }