Skip to content

Commit

Permalink
fix: improve processEffectiveBalanceUpdates (#7043)
Browse files Browse the repository at this point in the history
  • Loading branch information
twoeths authored and g11tech committed Aug 27, 2024
1 parent 8ccbed7 commit ed18ff0
Show file tree
Hide file tree
Showing 11 changed files with 63 additions and 19 deletions.
4 changes: 4 additions & 0 deletions packages/beacon-node/src/chain/historicalState/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,10 @@ if (metricsRegister) {
buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5],
labelNames: ["source"],
}),
numEffectiveBalanceUpdates: metricsRegister.gauge({
name: "lodestar_historical_state_stfn_num_effective_balance_updates_count",
help: "Count of effective balance updates in epoch transition",
}),
preStateBalancesNodesPopulatedMiss: metricsRegister.gauge<{source: StateCloneSource}>({
name: "lodestar_historical_state_stfn_balances_nodes_populated_miss_total",
help: "Total count state.balances nodesPopulated is false on stfn",
Expand Down
4 changes: 4 additions & 0 deletions packages/beacon-node/src/metrics/metrics/lodestar.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,10 @@ export function createLodestarMetrics(
buckets: [0.05, 0.1, 0.2, 0.5, 1, 1.5],
labelNames: ["source"],
}),
numEffectiveBalanceUpdates: register.gauge({
name: "lodestar_stfn_effective_balance_updates_count",
help: "Total count of effective balance updates",
}),
preStateBalancesNodesPopulatedMiss: register.gauge<{source: StateCloneSource}>({
name: "lodestar_stfn_balances_nodes_populated_miss_total",
help: "Total count state.balances nodesPopulated is false on stfn",
Expand Down
18 changes: 16 additions & 2 deletions packages/state-transition/src/cache/epochTransitionCache.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import {Epoch, ValidatorIndex} from "@lodestar/types";
import {Epoch, ValidatorIndex, phase0} from "@lodestar/types";
import {intDiv} from "@lodestar/utils";
import {EPOCHS_PER_SLASHINGS_VECTOR, FAR_FUTURE_EPOCH, ForkSeq, MIN_ACTIVATION_BALANCE} from "@lodestar/params";

Expand Down Expand Up @@ -127,6 +127,18 @@ export interface EpochTransitionCache {

flags: number[];

/**
* Validators in the current epoch, should use it for read-only value instead of accessing state.validators directly.
* Note that during epoch processing, validators could be updated so need to use it with care.
*/
validators: phase0.Validator[];

/**
* This is for electra only
* Validators that're switched to compounding during processPendingConsolidations(), not available in beforeProcessEpoch()
*/
newCompoundingValidators?: Set<ValidatorIndex>;

/**
* balances array will be populated by processRewardsAndPenalties() and consumed by processEffectiveBalanceUpdates().
* processRewardsAndPenalties() already has a regular Javascript array of balances.
Expand Down Expand Up @@ -481,7 +493,9 @@ export function beforeProcessEpoch(
proposerIndices,
inclusionDelays,
flags,

validators,
// will be assigned in processPendingConsolidations()
newCompoundingValidators: undefined,
// Will be assigned in processRewardsAndPenalties()
balances: undefined,
};
Expand Down
3 changes: 2 additions & 1 deletion packages/state-transition/src/epoch/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,9 @@ export function processEpoch(
const timer = metrics?.epochTransitionStepTime.startTimer({
step: EpochTransitionStep.processEffectiveBalanceUpdates,
});
processEffectiveBalanceUpdates(fork, state, cache);
const numUpdate = processEffectiveBalanceUpdates(fork, state, cache);
timer?.();
metrics?.numEffectiveBalanceUpdates.set(numUpdate);
}

processSlashingsReset(state, cache);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,14 @@ const TIMELY_TARGET = 1 << TIMELY_TARGET_FLAG_INDEX;
*
* - On normal mainnet conditions 0 validators change their effective balance
* - In case of big innactivity event a medium portion of validators may have their effectiveBalance updated
*
* Return number of validators updated
*/
export function processEffectiveBalanceUpdates(
fork: ForkSeq,
state: CachedBeaconStateAllForks,
cache: EpochTransitionCache
): void {
): number {
const HYSTERESIS_INCREMENT = EFFECTIVE_BALANCE_INCREMENT / HYSTERESIS_QUOTIENT;
const DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER;
const UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER;
Expand All @@ -43,34 +45,38 @@ export function processEffectiveBalanceUpdates(
// and updated in processPendingBalanceDeposits() and processPendingConsolidations()
// so it's recycled here for performance.
const balances = cache.balances ?? state.balances.getAll();
const currentEpochValidators = cache.validators;
const newCompoundingValidators = cache.newCompoundingValidators ?? new Set();

let numUpdate = 0;
for (let i = 0, len = balances.length; i < len; i++) {
const balance = balances[i];

// PERF: It's faster to access to get() every single element (4ms) than to convert to regular array then loop (9ms)
let effectiveBalanceIncrement = effectiveBalanceIncrements[i];
let effectiveBalance = effectiveBalanceIncrement * EFFECTIVE_BALANCE_INCREMENT;
let effectiveBalanceLimit;

let effectiveBalanceLimit: number;
if (fork < ForkSeq.electra) {
effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE;
} else {
// from electra, effectiveBalanceLimit is per validator
const isCompoundingValidator =
hasCompoundingWithdrawalCredential(currentEpochValidators[i].withdrawalCredentials) ||
newCompoundingValidators.has(i);
effectiveBalanceLimit = isCompoundingValidator ? MAX_EFFECTIVE_BALANCE_ELECTRA : MIN_ACTIVATION_BALANCE;
}

if (
// Too big
effectiveBalance > balance + DOWNWARD_THRESHOLD ||
// Too small. Check effectiveBalance < MAX_EFFECTIVE_BALANCE to prevent unnecessary updates
effectiveBalance + UPWARD_THRESHOLD < balance
(effectiveBalance < effectiveBalanceLimit && effectiveBalance + UPWARD_THRESHOLD < balance)
) {
// Update the state tree
// Should happen rarely, so it's fine to update the tree
const validator = validators.get(i);

if (fork < ForkSeq.electra) {
effectiveBalanceLimit = MAX_EFFECTIVE_BALANCE;
} else {
// Electra or after
effectiveBalanceLimit = hasCompoundingWithdrawalCredential(validator.withdrawalCredentials)
? MAX_EFFECTIVE_BALANCE_ELECTRA
: MIN_ACTIVATION_BALANCE;
}

effectiveBalance = Math.min(balance - (balance % EFFECTIVE_BALANCE_INCREMENT), effectiveBalanceLimit);
validator.effectiveBalance = effectiveBalance;
// Also update the fast cached version
Expand All @@ -95,6 +101,7 @@ export function processEffectiveBalanceUpdates(

effectiveBalanceIncrement = newEffectiveBalanceIncrement;
effectiveBalanceIncrements[i] = effectiveBalanceIncrement;
numUpdate++;
}

// TODO: Do this in afterEpochTransitionCache, looping a Uint8Array should be very cheap
Expand All @@ -105,4 +112,5 @@ export function processEffectiveBalanceUpdates(
}

cache.nextEpochTotalActiveBalanceByIncrement = nextEpochTotalActiveBalanceByIncrement;
return numUpdate;
}
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import {ValidatorIndex} from "@lodestar/types";
import {CachedBeaconStateElectra, EpochTransitionCache} from "../types.js";
import {decreaseBalance, increaseBalance} from "../util/balance.js";
import {getActiveBalance} from "../util/validator.js";
Expand All @@ -20,6 +21,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca
let nextPendingConsolidation = 0;
const validators = state.validators;
const cachedBalances = cache.balances;
const newCompoundingValidators = new Set<ValidatorIndex>();

for (const pendingConsolidation of state.pendingConsolidations.getAllReadonly()) {
const {sourceIndex, targetIndex} = pendingConsolidation;
Expand All @@ -35,6 +37,7 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca
}
// Churn any target excess active balance of target and raise its max
switchToCompoundingValidator(state, targetIndex);
newCompoundingValidators.add(targetIndex);
// Move active balance to target. Excess balance is withdrawable.
const activeBalance = getActiveBalance(state, sourceIndex);
decreaseBalance(state, sourceIndex, activeBalance);
Expand All @@ -47,5 +50,6 @@ export function processPendingConsolidations(state: CachedBeaconStateElectra, ca
nextPendingConsolidation++;
}

cache.newCompoundingValidators = newCompoundingValidators;
state.pendingConsolidations = state.pendingConsolidations.sliceFrom(nextPendingConsolidation);
}
1 change: 1 addition & 0 deletions packages/state-transition/src/metrics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ export type BeaconStateTransitionMetrics = {
processBlockTime: Histogram;
processBlockCommitTime: Histogram;
stateHashTreeRootTime: Histogram<{source: StateHashTreeRootSource}>;
numEffectiveBalanceUpdates: Gauge;
preStateBalancesNodesPopulatedMiss: Gauge<{source: StateCloneSource}>;
preStateBalancesNodesPopulatedHit: Gauge<{source: StateCloneSource}>;
preStateValidatorsNodesPopulatedMiss: Gauge<{source: StateCloneSource}>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue<CachedBeaconStateAllForks>
itBench({
id: `${stateId} - altair processEffectiveBalanceUpdates`,
beforeEach: () => stateOg.value.clone(),
fn: (state) => processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value),
fn: (state) => {
processEffectiveBalanceUpdates(ForkSeq.altair, state, cache.value);
},
});

itBench({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,9 @@ function benchmarkAltairEpochSteps(stateOg: LazyValue<CachedBeaconStateAllForks>
itBench({
id: `${stateId} - capella processEffectiveBalanceUpdates`,
beforeEach: () => stateOg.value.clone(),
fn: (state) => processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value),
fn: (state) => {
processEffectiveBalanceUpdates(ForkSeq.capella, state, cache.value);
},
});

itBench({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,9 @@ function benchmarkPhase0EpochSteps(stateOg: LazyValue<CachedBeaconStateAllForks>
itBench({
id: `${stateId} - phase0 processEffectiveBalanceUpdates`,
beforeEach: () => stateOg.value.clone(),
fn: (state) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value),
fn: (state) => {
processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache.value);
},
});

itBench({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ describe("phase0 processEffectiveBalanceUpdates", () => {
minRuns: 5, // Worst case is very slow
before: () => getEffectiveBalanceTestData(vc, changeRatio),
beforeEach: ({state, cache}) => ({state: state.clone(), cache}),
fn: ({state, cache}) => processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache),
fn: ({state, cache}) => {
processEffectiveBalanceUpdates(ForkSeq.phase0, state, cache);
},
});
}
});
Expand Down

0 comments on commit ed18ff0

Please sign in to comment.