Skip to content

Commit

Permalink
Persist invalid SSZ objects only if enabled
Browse files Browse the repository at this point in the history
  • Loading branch information
@dapplion
dapplion committed Feb 26, 2022
1 parent 060918e commit 66473e2
Show file tree
Hide file tree
Showing 10 changed files with 68 additions and 105 deletions.
2 changes: 1 addition & 1 deletion packages/cli/src/cmds/beacon/options.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ export const beaconPathsOptions: ICliCommandOptions<IBeaconPaths> = {
},

persistInvalidSszObjectsDir: {
description: "Directory to persist invalid ssz objects",
description: "Enable and specify a directory to persist invalid ssz objects",
defaultDescription: defaultBeaconPaths.persistInvalidSszObjectsDir,
hidden: true,
type: "string",
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/options/beaconNodeOptions/chain.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ export interface IChainArgs {
"chain.useSingleThreadVerifier": boolean;
"chain.disableBlsBatchVerify": boolean;
"chain.persistInvalidSszObjects": boolean;
"chain.proposerBoostEnabled": boolean;
"safe-slots-to-import-optimistically": number;
// this is defined as part of IBeaconPaths
// "chain.persistInvalidSszObjectsDir": string;
"chain.proposerBoostEnabled": boolean;
"safe-slots-to-import-optimistically": number;
}

export function parseArgs(args: IChainArgs): IBeaconNodeOptions["chain"] {
Expand Down
15 changes: 2 additions & 13 deletions packages/lodestar/src/api/impl/beacon/pool/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import {validateGossipVoluntaryExit} from "../../../../chain/validation/voluntar
import {validateSyncCommitteeSigOnly} from "../../../../chain/validation/syncCommittee";
import {ApiModules} from "../../types";
import {OpSource} from "../../../../metrics/validatorMonitor";
import {toHexString} from "@chainsafe/ssz";
import {AttestationError, GossipAction, SyncCommitteeError} from "../../../../chain/errors";

export function getBeaconPoolApi({
Expand Down Expand Up @@ -65,12 +64,7 @@ export function getBeaconPoolApi({
e as Error
);
if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"attestation",
ssz.phase0.Attestation.serialize(attestation),
toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation))
);
logger.debug("Submitted invalid attestation was written to", archivedPath);
chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "api_reject");
}
}
})
Expand Down Expand Up @@ -153,12 +147,7 @@ export function getBeaconPoolApi({
e as Error
);
if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"syncCommittee",
ssz.altair.SyncCommitteeMessage.serialize(signature),
toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(signature))
);
logger.debug("The submitted sync committee message was written to", archivedPath);
chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, signature, "api_reject");
}
}
})
Expand Down
14 changes: 2 additions & 12 deletions packages/lodestar/src/api/impl/validator/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -467,12 +467,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
e as Error
);
if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"signedAggregatedAndProof",
ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof),
toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof))
);
logger.debug("The submitted signed aggregate and proof was written to", archivedPath);
chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "api_reject");
}
}
})
Expand Down Expand Up @@ -518,12 +513,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}:
e as Error
);
if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"contributionAndProof",
ssz.altair.SignedContributionAndProof.serialize(contributionAndProof),
toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof))
);
logger.debug("The submitted contribution adn proof was written to", archivedPath);
chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "api_reject");
}
}
})
Expand Down
46 changes: 33 additions & 13 deletions packages/lodestar/src/chain/chain.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@
*/

import fs from "node:fs";
import path from "node:path";
import {CachedBeaconStateAllForks, computeStartSlotAtEpoch} from "@chainsafe/lodestar-beacon-state-transition";
import {IBeaconConfig} from "@chainsafe/lodestar-config";
import {IForkChoice} from "@chainsafe/lodestar-fork-choice";
import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodestar-types";
import {ILogger} from "@chainsafe/lodestar-utils";
import {fromHexString, TreeBacked} from "@chainsafe/ssz";
import {ILogger, toHex} from "@chainsafe/lodestar-utils";
import {fromHexString, TreeBacked, TreeView, Type} from "@chainsafe/ssz";
import {AbortController} from "@chainsafe/abort-controller";
import {GENESIS_EPOCH, ZERO_HASH} from "../constants";
import {IBeaconDb} from "../db";
Expand Down Expand Up @@ -44,6 +45,7 @@ import {IEth1ForBlockProduction} from "../eth1";
import {IExecutionEngine} from "../executionEngine";
import {PrecomputeNextEpochTransitionScheduler} from "./precomputeNextEpochTransition";
import {ReprocessController} from "./reprocess";
import {CompositeTypeAny} from "@chainsafe/ssz/lib/type/composite";

export class BeaconChain implements IBeaconChain {
readonly genesisTime: Number64;
Expand Down Expand Up @@ -272,23 +274,41 @@ export class BeaconChain implements IBeaconChain {
return this.reprocessController.waitForBlockOfAttestation(slot, root);
}

persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix = ""): string | null {
persistInvalidSszValue<T>(type: Type<T>, sszObject: T, suffix?: string): void {
if (this.opts.persistInvalidSszObjects) {
this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix);
}
}

persistInvalidSszView(view: TreeView<CompositeTypeAny>, suffix?: string): void {
if (this.opts.persistInvalidSszObjects) {
this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix);
}
}

private persistInvalidSszObject(typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string): void {
if (!this.opts.persistInvalidSszObjects) {
return;
}

const now = new Date();
// yyyy-MM-dd
const date = now.toISOString().split("T")[0];
const dateStr = now.toISOString().split("T")[0];

// by default store to lodestar_archive of current dir
const byDate = this.opts.persistInvalidSszObjectsDir
? `${this.opts.persistInvalidSszObjectsDir}/${date}`
: `invalidSszObjects/${date}`;
if (!fs.existsSync(byDate)) {
fs.mkdirSync(byDate, {recursive: true});
const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr);
const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`);

if (!fs.existsSync(dirpath)) {
fs.mkdirSync(dirpath, {recursive: true});
}
const fileName = `${byDate}/${type}_${suffix}.ssz`;

// as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes
// remove date suffixes in file name, and check duplicate to avoid redundant persistence
if (!fs.existsSync(fileName)) {
fs.writeFileSync(fileName, bytes);
if (!fs.existsSync(filepath)) {
fs.writeFileSync(filepath, bytes);
}
return fileName;

this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath});
}
}
36 changes: 9 additions & 27 deletions packages/lodestar/src/chain/eventHandlers.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,37 +212,19 @@ export async function onErrorBlock(this: BeaconChain, err: BlockError): Promise<
const {signedBlock} = err;
const blockSlot = signedBlock.message.slot;
const {state} = err.type;
const blockPath = this.persistInvalidSszObject(
"signedBlock",
this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock),
`${blockSlot}_invalid_signature`
);
const statePath = this.persistInvalidSszObject("state", state.serialize(), `${state.slot}_invalid_signature`);
this.logger.debug("Invalid signature block and state were written to disc", {blockPath, statePath});
const forkTypes = this.config.getForkTypes(blockSlot);
this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
this.persistInvalidSszView(state, `${state.slot}_invalid_signature`);
} else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
const {signedBlock} = err;
const blockSlot = signedBlock.message.slot;
const {preState, postState} = err.type;
const forkTypes = this.config.getForkTypes(blockSlot);
const invalidRoot = toHexString(postState.hashTreeRoot());
const blockPath = this.persistInvalidSszObject(
"signedBlock",
this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock),
`${blockSlot}_invalid_state_root_${invalidRoot}`
);
const preStatePath = this.persistInvalidSszObject(
"state",
preState.serialize(),
`${blockSlot}_invalid_state_root_preState_${invalidRoot}`
);
const postStatePath = this.persistInvalidSszObject(
"state",
postState.serialize(),
`${blockSlot}_invalid_state_root_postState_${invalidRoot}`
);
this.logger.debug("Invalid state root block and states were written to disc", {
blockPath,
preStatePath,
postStatePath,
});

const suffix = `slot_${blockSlot}_invalid_state_root_${invalidRoot}`;
this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, suffix);
this.persistInvalidSszView(preState, `${suffix}_preState`);
this.persistInvalidSszView(postState, `${suffix}_postState`);
}
}
6 changes: 5 additions & 1 deletion packages/lodestar/src/chain/interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ import {allForks, Number64, Root, phase0, Slot, RootHex} from "@chainsafe/lodest
import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition";
import {IForkChoice} from "@chainsafe/lodestar-fork-choice";
import {IBeaconConfig} from "@chainsafe/lodestar-config";
import {TreeView, Type} from "@chainsafe/ssz";
import {CompositeTypeAny} from "@chainsafe/ssz/lib/type/composite";

import {IEth1ForBlockProduction} from "../eth1";
import {IExecutionEngine} from "../executionEngine";
Expand Down Expand Up @@ -97,7 +99,9 @@ export interface IBeaconChain {
waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise<boolean>;

/** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */
persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix: string): string | null;
persistInvalidSszValue<T>(type: Type<T>, sszObject: T | Uint8Array, suffix?: string): void;
/** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */
persistInvalidSszView(view: TreeView<CompositeTypeAny>, suffix?: string): void;
}

export type SSZObjectType =
Expand Down
4 changes: 1 addition & 3 deletions packages/lodestar/src/chain/options.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ export type IChainOptions = BlockProcessOpts &
ForkChoiceOpts & {
useSingleThreadVerifier?: boolean;
persistInvalidSszObjects?: boolean;
persistInvalidSszObjectsDir: string;
persistInvalidSszObjectsDir?: string;
};

export type BlockProcessOpts = {
Expand All @@ -24,8 +24,6 @@ export type BlockProcessOpts = {
export const defaultChainOptions: IChainOptions = {
useSingleThreadVerifier: false,
disableBlsBatchVerify: false,
persistInvalidSszObjects: true,
persistInvalidSszObjectsDir: "",
proposerBoostEnabled: false,
safeSlotsToImportOptimistically: SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY,
};
38 changes: 7 additions & 31 deletions packages/lodestar/src/network/gossip/handlers/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
return {
[GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => {
const slot = signedBlock.message.slot;
const blockHex = prettyBytes(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message));
const forkTypes = config.getForkTypes(slot);
const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
logger.verbose("Received gossip block", {
slot: slot,
root: blockHex,
Expand All @@ -94,12 +95,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
}

if (e instanceof BlockGossipError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"signedBlock",
config.getForkTypes(slot).SignedBeaconBlock.serialize(signedBlock),
`gossip_slot_${slot}`
);
logger.debug("The invalid gossip block was written to", archivedPath);
chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
}

throw e;
Expand Down Expand Up @@ -143,12 +139,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
validationResult = await validateGossipAggregateAndProofRetryUnknownRoot(chain, signedAggregateAndProof);
} catch (e) {
if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"signedAggregatedAndProof",
ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof),
toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof))
);
logger.debug("The invalid gossip aggregate and proof was written to", archivedPath, e);
chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject");
}
throw e;
}
Expand Down Expand Up @@ -188,12 +179,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
validationResult = await validateGossipAttestationRetryUnknownRoot(chain, attestation, subnet);
} catch (e) {
if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"attestation",
ssz.phase0.Attestation.serialize(attestation),
toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation))
);
logger.debug("The invalid gossip attestation was written to", archivedPath);
chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "gossip_reject");
}
throw e;
}
Expand Down Expand Up @@ -265,12 +251,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
contributionAndProof
).catch((e) => {
if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"contributionAndProof",
ssz.altair.SignedContributionAndProof.serialize(contributionAndProof),
toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof))
);
logger.debug("The invalid gossip contribution and proof was written to", archivedPath);
chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "gossip_reject");
}
throw e;
});
Expand All @@ -290,12 +271,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee;
} catch (e) {
if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
const archivedPath = chain.persistInvalidSszObject(
"syncCommittee",
ssz.altair.SyncCommitteeMessage.serialize(syncCommittee),
toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(syncCommittee))
);
logger.debug("The invalid gossip sync committee was written to", archivedPath);
chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject");
}
throw e;
}
Expand Down
8 changes: 6 additions & 2 deletions packages/lodestar/test/utils/mocks/chain/chain.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,8 +173,12 @@ export class MockBeaconChain implements IBeaconChain {
return false;
}

persistInvalidSszObject(): string | null {
return null;
persistInvalidSszObject(): void {
return;
}

persistInvalidSszValue(): void {
return;
}
}

Expand Down

0 comments on commit 66473e2

Please sign in to comment.