From df82482b472d357817e630e4b52260e01a368738 Mon Sep 17 00:00:00 2001 From: twoeths Date: Tue, 27 Aug 2024 21:39:38 +0700 Subject: [PATCH] fix: improve processEffectiveBalanceUpdates (#7043) --- .../src/chain/historicalState/worker.ts | 4 +++ .../src/metrics/metrics/lodestar.ts | 4 +++ .../src/cache/epochTransitionCache.ts | 18 +++++++++-- packages/state-transition/src/epoch/index.ts | 3 +- .../epoch/processEffectiveBalanceUpdates.ts | 32 ++++++++++++------- .../src/epoch/processPendingConsolidations.ts | 4 +++ packages/state-transition/src/metrics.ts | 1 + .../test/perf/epoch/epochAltair.test.ts | 4 ++- .../test/perf/epoch/epochCapella.test.ts | 4 ++- .../test/perf/epoch/epochPhase0.test.ts | 4 ++- .../processEffectiveBalanceUpdates.test.ts | 4 ++- 11 files changed, 63 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/src/chain/historicalState/worker.ts b/packages/beacon-node/src/chain/historicalState/worker.ts index 9a9f9cc9cd0e..a07207cac5f5 100644 --- a/packages/beacon-node/src/chain/historicalState/worker.ts +++ b/packages/beacon-node/src/chain/historicalState/worker.ts @@ -82,6 +82,10 @@ if (metricsRegister) { buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], labelNames: ["source"], }), + numEffectiveBalanceUpdates: metricsRegister.gauge({ + name: "lodestar_historical_state_stfn_num_effective_balance_updates_count", + help: "Count of effective balance updates in epoch transition", + }), preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({ name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total", help: "Total count state.balances nodesPopulated is false on stfn", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 6b4bdb111f41..a0cf0a185c2f 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -332,6 +332,10 @@ export function createLodestarMetrics( buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5], labelNames: ["source"], }), + numEffectiveBalanceUpdates: register.gauge({ + name: "lodestar_stfn_effective_balance_updates_count", + help: "Total count of effective balance updates", + }), preStateBalancesNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({ name: "lodestar_stfn_balances_nodes_populated_miss_total", help: "Total count state.balances nodesPopulated is false on stfn", diff --git a/packages/state-transition/src/cache/epochTransitionCache.ts b/packages/state-transition/src/cache/epochTransitionCache.ts index 280524ec7beb..6f27ad96d1c8 100644 --- a/packages/state-transition/src/cache/epochTransitionCache.ts +++ b/packages/state-transition/src/cache/epochTransitionCache.ts @@ -1,4 +1,4 @@ -import {Epoch, ValidatorIndex} from "@lodestar/types"; +import {Epoch, ValidatorIndex, phase0} from "@lodestar/types"; import {intDiv} from "@lodestar/utils"; import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params"; @@ -127,6 +127,18 @@ export interface EpochTransitionCache { flags: number[]; + /** + * Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly. + * Note that during epoch processing, validators could be updated so need to use it with care. + */ + validators: phase0.Validator[]; + + /** + * This is for electra only + * Validators that're switched to compounding during processPendingConsolidations(), not available in beforeProcessEpoch() + */ + newCompoundingValidators?: Set; + /** * balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates(). * processRewardsAndPenalties() already has a regular Javascript array of balances. @@ -481,7 +493,9 @@ export function beforeProcessEpoch( proposerIndices, inclusionDelays, flags, - + validators, + // will be assigned in processPendingConsolidations() + newCompoundingValidators: undefined, // Will be assigned in processRewardsAndPenalties() balances: undefined, }; diff --git a/packages/state-transition/src/epoch/index.ts b/packages/state-transition/src/epoch/index.ts index 85e7c348dad3..bfb415b9ed6a 100644 --- a/packages/state-transition/src/epoch/index.ts +++ b/packages/state-transition/src/epoch/index.ts @@ -150,8 +150,9 @@ export function processEpoch( const timer = metrics?.epochTransitionStepTime.startTimer({ step: EpochTransitionStep.processEffectiveBalanceUpdates, }); - processEffectiveBalanceUpdates(fork, state, cache); + const numUpdate = processEffectiveBalanceUpdates(fork, state, cache); timer?.(); + metrics?.numEffectiveBalanceUpdates.set(numUpdate); } processSlashingsReset(state, cache); diff --git a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts index fdbc99b9265e..0ea4b49dddf4 100644 --- a/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts +++ b/packages/state-transition/src/epoch/processEffectiveBalanceUpdates.ts @@ -23,12 +23,14 @@ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX; * * - On normal mainnet conditions 0 validators change their effective balance * - In case of big innactivity event a medium portion of validators may have their effectiveBalance updated + * + * Return number of validators updated */ export function processEffectiveBalanceUpdates( fork: ForkSeq, state: CachedBeaconStateAllForks, cache: EpochTransitionCache -): void { +): number { const HYSTERESIS_INCREMENT = EFFECTIVE_BALANCE_INCREMENT / HYSTERESIS_QUOTIENT; const DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER; const UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER; @@ -43,34 +45,38 @@ export function processEffectiveBalanceUpdates( // and updated in processPendingBalanceDeposits() and processPendingConsolidations() // so it's recycled here for performance. const balances = cache.balances ?? state.balances.getAll(); + const currentEpochValidators = cache.validators; + const newCompoundingValidators = cache.newCompoundingValidators ?? new Set(); + let numUpdate = 0; for (let i = 0, len = balances.length; i < len; i++) { const balance = balances[i]; // PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms) let effectiveBalanceIncrement = effectiveBalanceIncrements[i]; let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT; - let effectiveBalanceLimit; + + let effectiveBalanceLimit: number; + if (fork < ForkSeq.electra) { + effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; + } else { + // from electra, effectiveBalanceLimit is per validator + const isCompoundingValidator = + hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) || + newCompoundingValidators.has(i); + effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE; + } if ( // Too big effectiveBalance > balance + DOWNWARD_THRESHOLD || // Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates - effectiveBalance + UPWARD_THRESHOLD < balance + (effectiveBalance < effectiveBalanceLimit && effectiveBalance + UPWARD_THRESHOLD < balance) ) { // Update the state tree // Should happen rarely, so it's fine to update the tree const validator = validators.get(i); - if (fork < ForkSeq.electra) { - effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE; - } else { - // Electra or after - effectiveBalanceLimit = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials) - ? MAX_EFFECTIVE_BALANCE_ELECTRA - : MIN_ACTIVATION_BALANCE; - } - effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), effectiveBalanceLimit); validator.effectiveBalance = effectiveBalance; // Also update the fast cached version @@ -95,6 +101,7 @@ export function processEffectiveBalanceUpdates( effectiveBalanceIncrement = newEffectiveBalanceIncrement; effectiveBalanceIncrements[i] = effectiveBalanceIncrement; + numUpdate++; } // TODO: Do this in afterEpochTransitionCache, looping a Uint8Array should be very cheap @@ -105,4 +112,5 @@ export function processEffectiveBalanceUpdates( } cache.nextEpochTotalActiveBalanceByIncrement = nextEpochTotalActiveBalanceByIncrement; + return numUpdate; } diff --git a/packages/state-transition/src/epoch/processPendingConsolidations.ts b/packages/state-transition/src/epoch/processPendingConsolidations.ts index d6f760a6b27a..7d8548046e79 100644 --- a/packages/state-transition/src/epoch/processPendingConsolidations.ts +++ b/packages/state-transition/src/epoch/processPendingConsolidations.ts @@ -1,3 +1,4 @@ +import {ValidatorIndex} from "@lodestar/types"; import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js"; import {decreaseBalance, increaseBalance} from "../util/balance.js"; import {getActiveBalance} from "../util/validator.js"; @@ -20,6 +21,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca let nextPendingConsolidation = 0; const validators = state.validators; const cachedBalances = cache.balances; + const newCompoundingValidators = new Set(); for (const pendingConsolidation of state.pendingConsolidations.getAllReadonly()) { const {sourceIndex, targetIndex} = pendingConsolidation; @@ -35,6 +37,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca } // Churn any target excess active balance of target and raise its max switchToCompoundingValidator(state, targetIndex); + newCompoundingValidators.add(targetIndex); // Move active balance to target. Excess balance is withdrawable. const activeBalance = getActiveBalance(state, sourceIndex); decreaseBalance(state, sourceIndex, activeBalance); @@ -47,5 +50,6 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca nextPendingConsolidation++; } + cache.newCompoundingValidators = newCompoundingValidators; state.pendingConsolidations = state.pendingConsolidations.sliceFrom(nextPendingConsolidation); } diff --git a/packages/state-transition/src/metrics.ts b/packages/state-transition/src/metrics.ts index 48e0bc921876..a5e5463231fa 100644 --- a/packages/state-transition/src/metrics.ts +++ b/packages/state-transition/src/metrics.ts @@ -11,6 +11,7 @@ export type BeaconStateTransitionMetrics = { processBlockTime: Histogram; processBlockCommitTime: Histogram; stateHashTreeRootTime: Histogram<{source: StateHashTreeRootSource}>; + numEffectiveBalanceUpdates: Gauge; preStateBalancesNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; preStateBalancesNodesPopulatedHit: Gauge<{source: StateCloneSource}>; preStateValidatorsNodesPopulatedMiss: Gauge<{source: StateCloneSource}>; diff --git a/packages/state-transition/test/perf/epoch/epochAltair.test.ts b/packages/state-transition/test/perf/epoch/epochAltair.test.ts index 39e0a1b4c5c3..15cde849ce9f 100644 --- a/packages/state-transition/test/perf/epoch/epochAltair.test.ts +++ b/packages/state-transition/test/perf/epoch/epochAltair.test.ts @@ -141,7 +141,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - altair processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochCapella.test.ts b/packages/state-transition/test/perf/epoch/epochCapella.test.ts index 5b8300df3f18..61bfad20b1ee 100644 --- a/packages/state-transition/test/perf/epoch/epochCapella.test.ts +++ b/packages/state-transition/test/perf/epoch/epochCapella.test.ts @@ -120,7 +120,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - capella processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts index 411577878102..3af3d4d4a832 100644 --- a/packages/state-transition/test/perf/epoch/epochPhase0.test.ts +++ b/packages/state-transition/test/perf/epoch/epochPhase0.test.ts @@ -123,7 +123,9 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue itBench({ id: `${stateId} - phase0 processEffectiveBalanceUpdates`, beforeEach: () => stateOg.value.clone(), - fn: (state) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value), + fn: (state) => { + processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value); + }, }); itBench({ diff --git a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts index d94daac9e59b..19f18df86c2e 100644 --- a/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts +++ b/packages/state-transition/test/perf/epoch/processEffectiveBalanceUpdates.test.ts @@ -36,7 +36,9 @@ describe("phase0 processEffectiveBalanceUpdates", () => { minRuns: 5, // Worst case is very slow before: () => getEffectiveBalanceTestData(vc, changeRatio), beforeEach: ({state, cache}) => ({state: state.clone(), cache}), - fn: ({state, cache}) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache), + fn: ({state, cache}) => { + processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache); + }, }); } });