From 66473e26e5e0a15bcb9298fb607413f93396e827 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 21:24:12 +0530 Subject: [PATCH] Persist invalid SSZ objects only if enabled --- packages/cli/src/cmds/beacon/options.ts | 2 +- .../src/options/beaconNodeOptions/chain.ts | 4 +- .../src/api/impl/beacon/pool/index.ts | 15 +----- .../lodestar/src/api/impl/validator/index.ts | 14 +----- packages/lodestar/src/chain/chain.ts | 46 +++++++++++++------ packages/lodestar/src/chain/eventHandlers.ts | 36 ++++----------- packages/lodestar/src/chain/interface.ts | 6 ++- packages/lodestar/src/chain/options.ts | 4 +- .../src/network/gossip/handlers/index.ts | 38 +++------------ .../lodestar/test/utils/mocks/chain/chain.ts | 8 +++- 10 files changed, 68 insertions(+), 105 deletions(-) diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 28748ca1271e..a34ba68db2b8 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -98,7 +98,7 @@ export const beaconPathsOptions: ICliCommandOptions = { }, persistInvalidSszObjectsDir: { - description: "Directory to persist invalid ssz objects", + description: "Enable and specify a directory to persist invalid ssz objects", defaultDescription: defaultBeaconPaths.persistInvalidSszObjectsDir, hidden: true, type: "string", diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index eb018fee3ab9..98964ea3ed90 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -5,10 +5,10 @@ export interface IChainArgs { "chain.useSingleThreadVerifier": boolean; "chain.disableBlsBatchVerify": boolean; "chain.persistInvalidSszObjects": boolean; - "chain.proposerBoostEnabled": boolean; - "safe-slots-to-import-optimistically": number; // this is defined as part of IBeaconPaths // "chain.persistInvalidSszObjectsDir": string; + "chain.proposerBoostEnabled": boolean; + "safe-slots-to-import-optimistically": number; } export function parseArgs(args: IChainArgs): IBeaconNodeOptions["chain"] { diff --git a/packages/lodestar/src/api/impl/beacon/pool/index.ts b/packages/lodestar/src/api/impl/beacon/pool/index.ts index 3f0dbdc76fe2..8038eba9cb9e 100644 --- a/packages/lodestar/src/api/impl/beacon/pool/index.ts +++ b/packages/lodestar/src/api/impl/beacon/pool/index.ts @@ -9,7 +9,6 @@ import {validateGossipVoluntaryExit} from "../../../../chain/validation/voluntar import {validateSyncCommitteeSigOnly} from "../../../../chain/validation/syncCommittee"; import {ApiModules} from "../../types"; import {OpSource} from "../../../../metrics/validatorMonitor"; -import {toHexString} from "@chainsafe/ssz"; import {AttestationError, GossipAction, SyncCommitteeError} from "../../../../chain/errors"; export function getBeaconPoolApi({ @@ -65,12 +64,7 @@ export function getBeaconPoolApi({ e as Error ); if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "attestation", - ssz.phase0.Attestation.serialize(attestation), - toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation)) - ); - logger.debug("Submitted invalid attestation was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "api_reject"); } } }) @@ -153,12 +147,7 @@ export function getBeaconPoolApi({ e as Error ); if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "syncCommittee", - ssz.altair.SyncCommitteeMessage.serialize(signature), - toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(signature)) - ); - logger.debug("The submitted sync committee message was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, signature, "api_reject"); } } }) diff --git a/packages/lodestar/src/api/impl/validator/index.ts b/packages/lodestar/src/api/impl/validator/index.ts index 93e4651748f7..077f864823d4 100644 --- a/packages/lodestar/src/api/impl/validator/index.ts +++ b/packages/lodestar/src/api/impl/validator/index.ts @@ -467,12 +467,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: e as Error ); if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedAggregatedAndProof", - ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof), - toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof)) - ); - logger.debug("The submitted signed aggregate and proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "api_reject"); } } }) @@ -518,12 +513,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: e as Error ); if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "contributionAndProof", - ssz.altair.SignedContributionAndProof.serialize(contributionAndProof), - toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof)) - ); - logger.debug("The submitted contribution adn proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "api_reject"); } } }) diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index 5cd2705f3a9d..d56efa729f5b 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -3,12 +3,13 @@ */ import fs from "node:fs"; +import path from "node:path"; import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types"; -import {ILogger} from "@chainsafe/lodestar-utils"; -import {fromHexString, TreeBacked} from "@chainsafe/ssz"; +import {ILogger, toHex} from "@chainsafe/lodestar-utils"; +import {fromHexString, TreeBacked, TreeView, Type} from "@chainsafe/ssz"; import {AbortController} from "@chainsafe/abort-controller"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants"; import {IBeaconDb} from "../db"; @@ -44,6 +45,7 @@ import {IEth1ForBlockProduction} from "../eth1"; import {IExecutionEngine} from "../executionEngine"; import {PrecomputeNextEpochTransitionScheduler} from "./precomputeNextEpochTransition"; import {ReprocessController} from "./reprocess"; +import {CompositeTypeAny} from "@chainsafe/ssz/lib/type/composite"; export class BeaconChain implements IBeaconChain { readonly genesisTime: Number64; @@ -272,23 +274,41 @@ export class BeaconChain implements IBeaconChain { return this.reprocessController.waitForBlockOfAttestation(slot, root); } - persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix = ""): string | null { + persistInvalidSszValue(type: Type, sszObject: T, suffix?: string): void { + if (this.opts.persistInvalidSszObjects) { + this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); + } + } + + persistInvalidSszView(view: TreeView, suffix?: string): void { + if (this.opts.persistInvalidSszObjects) { + this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); + } + } + + private persistInvalidSszObject(typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string): void { + if (!this.opts.persistInvalidSszObjects) { + return; + } + const now = new Date(); // yyyy-MM-dd - const date = now.toISOString().split("T")[0]; + const dateStr = now.toISOString().split("T")[0]; + // by default store to lodestar_archive of current dir - const byDate = this.opts.persistInvalidSszObjectsDir - ? `${this.opts.persistInvalidSszObjectsDir}/${date}` - : `invalidSszObjects/${date}`; - if (!fs.existsSync(byDate)) { - fs.mkdirSync(byDate, {recursive: true}); + const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr); + const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`); + + if (!fs.existsSync(dirpath)) { + fs.mkdirSync(dirpath, {recursive: true}); } - const fileName = `${byDate}/${type}_${suffix}.ssz`; + // as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes // remove date suffixes in file name, and check duplicate to avoid redundant persistence - if (!fs.existsSync(fileName)) { - fs.writeFileSync(fileName, bytes); + if (!fs.existsSync(filepath)) { + fs.writeFileSync(filepath, bytes); } - return fileName; + + this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath}); } } diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 4b69484d344f..4aef25377bf0 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -212,37 +212,19 @@ export async function onErrorBlock(this: BeaconChain, err: BlockError): Promise< const {signedBlock} = err; const blockSlot = signedBlock.message.slot; const {state} = err.type; - const blockPath = this.persistInvalidSszObject( - "signedBlock", - this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock), - `${blockSlot}_invalid_signature` - ); - const statePath = this.persistInvalidSszObject("state", state.serialize(), `${state.slot}_invalid_signature`); - this.logger.debug("Invalid signature block and state were written to disc", {blockPath, statePath}); + const forkTypes = this.config.getForkTypes(blockSlot); + this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`); + this.persistInvalidSszView(state, `${state.slot}_invalid_signature`); } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) { const {signedBlock} = err; const blockSlot = signedBlock.message.slot; const {preState, postState} = err.type; + const forkTypes = this.config.getForkTypes(blockSlot); const invalidRoot = toHexString(postState.hashTreeRoot()); - const blockPath = this.persistInvalidSszObject( - "signedBlock", - this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock), - `${blockSlot}_invalid_state_root_${invalidRoot}` - ); - const preStatePath = this.persistInvalidSszObject( - "state", - preState.serialize(), - `${blockSlot}_invalid_state_root_preState_${invalidRoot}` - ); - const postStatePath = this.persistInvalidSszObject( - "state", - postState.serialize(), - `${blockSlot}_invalid_state_root_postState_${invalidRoot}` - ); - this.logger.debug("Invalid state root block and states were written to disc", { - blockPath, - preStatePath, - postStatePath, - }); + + const suffix = `slot_${blockSlot}_invalid_state_root_${invalidRoot}`; + this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, suffix); + this.persistInvalidSszView(preState, `${suffix}_preState`); + this.persistInvalidSszView(postState, `${suffix}_postState`); } } diff --git a/packages/lodestar/src/chain/interface.ts b/packages/lodestar/src/chain/interface.ts index e16d6d527bd0..e2d13dfc3622 100644 --- a/packages/lodestar/src/chain/interface.ts +++ b/packages/lodestar/src/chain/interface.ts @@ -2,6 +2,8 @@ import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodest import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {TreeView, Type} from "@chainsafe/ssz"; +import {CompositeTypeAny} from "@chainsafe/ssz/lib/type/composite"; import {IEth1ForBlockProduction} from "../eth1"; import {IExecutionEngine} from "../executionEngine"; @@ -97,7 +99,9 @@ export interface IBeaconChain { waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise; /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ - persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix: string): string | null; + persistInvalidSszValue(type: Type, sszObject: T | Uint8Array, suffix?: string): void; + /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ + persistInvalidSszView(view: TreeView, suffix?: string): void; } export type SSZObjectType = diff --git a/packages/lodestar/src/chain/options.ts b/packages/lodestar/src/chain/options.ts index f2d90dba0ac3..37911d8707f5 100644 --- a/packages/lodestar/src/chain/options.ts +++ b/packages/lodestar/src/chain/options.ts @@ -6,7 +6,7 @@ export type IChainOptions = BlockProcessOpts & ForkChoiceOpts & { useSingleThreadVerifier?: boolean; persistInvalidSszObjects?: boolean; - persistInvalidSszObjectsDir: string; + persistInvalidSszObjectsDir?: string; }; export type BlockProcessOpts = { @@ -24,8 +24,6 @@ export type BlockProcessOpts = { export const defaultChainOptions: IChainOptions = { useSingleThreadVerifier: false, disableBlsBatchVerify: false, - persistInvalidSszObjects: true, - persistInvalidSszObjectsDir: "", proposerBoostEnabled: false, safeSlotsToImportOptimistically: SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, }; diff --git a/packages/lodestar/src/network/gossip/handlers/index.ts b/packages/lodestar/src/network/gossip/handlers/index.ts index 5feebcfe2260..3b7d91026bc9 100644 --- a/packages/lodestar/src/network/gossip/handlers/index.ts +++ b/packages/lodestar/src/network/gossip/handlers/index.ts @@ -75,7 +75,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH return { [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { const slot = signedBlock.message.slot; - const blockHex = prettyBytes(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + const forkTypes = config.getForkTypes(slot); + const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); logger.verbose("Received gossip block", { slot: slot, root: blockHex, @@ -94,12 +95,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } if (e instanceof BlockGossipError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedBlock", - config.getForkTypes(slot).SignedBeaconBlock.serialize(signedBlock), - `gossip_slot_${slot}` - ); - logger.debug("The invalid gossip block was written to", archivedPath); + chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`); } throw e; @@ -143,12 +139,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH validationResult = await validateGossipAggregateAndProofRetryUnknownRoot(chain, signedAggregateAndProof); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedAggregatedAndProof", - ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof), - toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof)) - ); - logger.debug("The invalid gossip aggregate and proof was written to", archivedPath, e); + chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject"); } throw e; } @@ -188,12 +179,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH validationResult = await validateGossipAttestationRetryUnknownRoot(chain, attestation, subnet); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "attestation", - ssz.phase0.Attestation.serialize(attestation), - toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation)) - ); - logger.debug("The invalid gossip attestation was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "gossip_reject"); } throw e; } @@ -265,12 +251,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH contributionAndProof ).catch((e) => { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "contributionAndProof", - ssz.altair.SignedContributionAndProof.serialize(contributionAndProof), - toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof)) - ); - logger.debug("The invalid gossip contribution and proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "gossip_reject"); } throw e; }); @@ -290,12 +271,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee; } catch (e) { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "syncCommittee", - ssz.altair.SyncCommitteeMessage.serialize(syncCommittee), - toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(syncCommittee)) - ); - logger.debug("The invalid gossip sync committee was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject"); } throw e; } diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index 229857dba231..91926a6983fa 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -173,8 +173,12 @@ export class MockBeaconChain implements IBeaconChain { return false; } - persistInvalidSszObject(): string | null { - return null; + persistInvalidSszObject(): void { + return; + } + + persistInvalidSszValue(): void { + return; } }