From c3f4669e64722239ba52b582065a03001d7c787a Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Sun, 20 Feb 2022 21:24:12 +0530 Subject: [PATCH 1/2] Persist invalid SSZ objects only if enabled --- packages/cli/src/cmds/beacon/options.ts | 2 +- .../src/api/impl/beacon/pool/index.ts | 15 +----- .../lodestar/src/api/impl/validator/index.ts | 16 ++----- packages/lodestar/src/chain/chain.ts | 47 +++++++++++++------ packages/lodestar/src/chain/eventHandlers.ts | 36 ++++---------- packages/lodestar/src/chain/interface.ts | 8 ++-- packages/lodestar/src/chain/options.ts | 2 - .../src/network/gossip/handlers/index.ts | 38 +++------------ .../lodestar/test/utils/mocks/chain/chain.ts | 12 +++-- 9 files changed, 69 insertions(+), 107 deletions(-) diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index 3194831ce546..54c83ddf8e05 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -98,7 +98,7 @@ export const beaconPathsOptions: ICliCommandOptions = { }, persistInvalidSszObjectsDir: { - description: "Directory to persist invalid ssz objects", + description: "Enable and specify a directory to persist invalid ssz objects", defaultDescription: defaultBeaconPaths.persistInvalidSszObjectsDir, hidden: true, type: "string", diff --git a/packages/lodestar/src/api/impl/beacon/pool/index.ts b/packages/lodestar/src/api/impl/beacon/pool/index.ts index e5f81e9dd9d3..93bee203c164 100644 --- a/packages/lodestar/src/api/impl/beacon/pool/index.ts +++ b/packages/lodestar/src/api/impl/beacon/pool/index.ts @@ -1,7 +1,6 @@ import {routes} from "@chainsafe/lodestar-api"; import {Epoch, ssz} from "@chainsafe/lodestar-types"; import {SYNC_COMMITTEE_SUBNET_SIZE} from "@chainsafe/lodestar-params"; -import {toHexString} from "@chainsafe/ssz"; import {validateGossipAttestation} from "../../../../chain/validation/index.js"; import {validateGossipAttesterSlashing} from "../../../../chain/validation/attesterSlashing.js"; import {validateGossipProposerSlashing} from "../../../../chain/validation/proposerSlashing.js"; @@ -60,12 +59,7 @@ export function getBeaconPoolApi({ e as Error ); if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "attestation", - ssz.phase0.Attestation.serialize(attestation), - toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation)) - ); - logger.debug("Submitted invalid attestation was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "api_reject"); } } }) @@ -148,12 +142,7 @@ export function getBeaconPoolApi({ e as Error ); if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "syncCommittee", - ssz.altair.SyncCommitteeMessage.serialize(signature), - toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(signature)) - ); - logger.debug("The submitted sync committee message was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, signature, "api_reject"); } } }) diff --git a/packages/lodestar/src/api/impl/validator/index.ts b/packages/lodestar/src/api/impl/validator/index.ts index 2aad7034ac49..d5ab378e9b90 100644 --- a/packages/lodestar/src/api/impl/validator/index.ts +++ b/packages/lodestar/src/api/impl/validator/index.ts @@ -16,7 +16,7 @@ import { import {Root, Slot, ValidatorIndex, ssz} from "@chainsafe/lodestar-types"; import {ExecutionStatus} from "@chainsafe/lodestar-fork-choice"; -import {fromHexString, toHexString} from "@chainsafe/ssz"; +import {fromHexString} from "@chainsafe/ssz"; import {assembleBlock} from "../../../chain/factory/block/index.js"; import {AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError} from "../../../chain/errors/index.js"; import {validateGossipAggregateAndProof} from "../../../chain/validation/index.js"; @@ -466,12 +466,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: e as Error ); if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedAggregatedAndProof", - ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof), - toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof)) - ); - logger.debug("The submitted signed aggregate and proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "api_reject"); } } }) @@ -517,12 +512,7 @@ export function getValidatorApi({chain, config, logger, metrics, network, sync}: e as Error ); if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "contributionAndProof", - ssz.altair.SignedContributionAndProof.serialize(contributionAndProof), - toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof)) - ); - logger.debug("The submitted contribution adn proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "api_reject"); } } }) diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index 399f05a832c7..e2661c4b4481 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -3,6 +3,7 @@ */ import fs from "node:fs"; +import path from "node:path"; import { BeaconStateAllForks, CachedBeaconStateAllForks, @@ -14,8 +15,8 @@ import { import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch} from "@chainsafe/lodestar-types"; -import {ILogger} from "@chainsafe/lodestar-utils"; -import {fromHexString} from "@chainsafe/ssz"; +import {ILogger, toHex} from "@chainsafe/lodestar-utils"; +import {CompositeTypeAny, fromHexString, TreeView, Type} from "@chainsafe/ssz"; import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js"; import {IBeaconDb} from "../db/index.js"; import {IMetrics} from "../metrics/index.js"; @@ -26,7 +27,7 @@ import {BlockProcessor, PartiallyVerifiedBlockFlags} from "./blocks/index.js"; import {IBeaconClock, LocalClock} from "./clock/index.js"; import {ChainEventEmitter} from "./emitter.js"; import {handleChainEvents} from "./eventHandlers.js"; -import {IBeaconChain, SSZObjectType, ProposerPreparationData} from "./interface.js"; +import {IBeaconChain, ProposerPreparationData} from "./interface.js"; import {IChainOptions} from "./options.js"; import {IStateRegenerator, QueuedStateRegenerator, RegenCaller} from "./regen/index.js"; import {initializeForkChoice} from "./forkChoice/index.js"; @@ -302,24 +303,42 @@ export class BeaconChain implements IBeaconChain { return this.reprocessController.waitForBlockOfAttestation(slot, root); } - persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix = ""): string | null { + persistInvalidSszValue(type: Type, sszObject: T, suffix?: string): void { + if (this.opts.persistInvalidSszObjects) { + this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); + } + } + + persistInvalidSszView(view: TreeView, suffix?: string): void { + if (this.opts.persistInvalidSszObjects) { + this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); + } + } + + private persistInvalidSszObject(typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string): void { + if (!this.opts.persistInvalidSszObjects) { + return; + } + const now = new Date(); // yyyy-MM-dd - const date = now.toISOString().split("T")[0]; + const dateStr = now.toISOString().split("T")[0]; + // by default store to lodestar_archive of current dir - const byDate = this.opts.persistInvalidSszObjectsDir - ? `${this.opts.persistInvalidSszObjectsDir}/${date}` - : `invalidSszObjects/${date}`; - if (!fs.existsSync(byDate)) { - fs.mkdirSync(byDate, {recursive: true}); + const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr); + const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`); + + if (!fs.existsSync(dirpath)) { + fs.mkdirSync(dirpath, {recursive: true}); } - const fileName = `${byDate}/${type}_${suffix}.ssz`; + // as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes // remove date suffixes in file name, and check duplicate to avoid redundant persistence - if (!fs.existsSync(fileName)) { - fs.writeFileSync(fileName, bytes); + if (!fs.existsSync(filepath)) { + fs.writeFileSync(filepath, bytes); } - return fileName; + + this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath}); } async updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise { diff --git a/packages/lodestar/src/chain/eventHandlers.ts b/packages/lodestar/src/chain/eventHandlers.ts index 984ab14e1295..ee3f13e43b07 100644 --- a/packages/lodestar/src/chain/eventHandlers.ts +++ b/packages/lodestar/src/chain/eventHandlers.ts @@ -231,37 +231,19 @@ export async function onErrorBlock(this: BeaconChain, err: BlockError): Promise< const {signedBlock} = err; const blockSlot = signedBlock.message.slot; const {state} = err.type; - const blockPath = this.persistInvalidSszObject( - "signedBlock", - this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock), - `${blockSlot}_invalid_signature` - ); - const statePath = this.persistInvalidSszObject("state", state.serialize(), `${state.slot}_invalid_signature`); - this.logger.debug("Invalid signature block and state were written to disc", {blockPath, statePath}); + const forkTypes = this.config.getForkTypes(blockSlot); + this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`); + this.persistInvalidSszView(state, `${state.slot}_invalid_signature`); } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) { const {signedBlock} = err; const blockSlot = signedBlock.message.slot; const {preState, postState} = err.type; + const forkTypes = this.config.getForkTypes(blockSlot); const invalidRoot = toHexString(postState.hashTreeRoot()); - const blockPath = this.persistInvalidSszObject( - "signedBlock", - this.config.getForkTypes(blockSlot).SignedBeaconBlock.serialize(signedBlock), - `${blockSlot}_invalid_state_root_${invalidRoot}` - ); - const preStatePath = this.persistInvalidSszObject( - "state", - preState.serialize(), - `${blockSlot}_invalid_state_root_preState_${invalidRoot}` - ); - const postStatePath = this.persistInvalidSszObject( - "state", - postState.serialize(), - `${blockSlot}_invalid_state_root_postState_${invalidRoot}` - ); - this.logger.debug("Invalid state root block and states were written to disc", { - blockPath, - preStatePath, - postStatePath, - }); + + const suffix = `slot_${blockSlot}_invalid_state_root_${invalidRoot}`; + this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, suffix); + this.persistInvalidSszView(preState, `${suffix}_preState`); + this.persistInvalidSszView(postState, `${suffix}_postState`); } } diff --git a/packages/lodestar/src/chain/interface.ts b/packages/lodestar/src/chain/interface.ts index b7746b68c50c..61b345c32fb0 100644 --- a/packages/lodestar/src/chain/interface.ts +++ b/packages/lodestar/src/chain/interface.ts @@ -2,6 +2,7 @@ import {allForks, UintNum64, Root, phase0, Slot, RootHex, Epoch} from "@chainsaf import {CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; import {IForkChoice} from "@chainsafe/lodestar-fork-choice"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; +import {CompositeTypeAny, TreeView, Type} from "@chainsafe/ssz"; import {IEth1ForBlockProduction} from "../eth1/index.js"; import {IExecutionEngine} from "../executionEngine/index.js"; @@ -102,10 +103,11 @@ export interface IBeaconChain { waitForBlockOfAttestation(slot: Slot, root: RootHex): Promise; - /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ - persistInvalidSszObject(type: SSZObjectType, bytes: Uint8Array, suffix: string): string | null; - updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise; + + persistInvalidSszValue(type: Type, sszObject: T | Uint8Array, suffix?: string): void; + /** Persist bad items to persistInvalidSszObjectsDir dir, for example invalid state, attestations etc. */ + persistInvalidSszView(view: TreeView, suffix?: string): void; } export type SSZObjectType = diff --git a/packages/lodestar/src/chain/options.ts b/packages/lodestar/src/chain/options.ts index a0a76fb735cc..5af30b5b2331 100644 --- a/packages/lodestar/src/chain/options.ts +++ b/packages/lodestar/src/chain/options.ts @@ -30,8 +30,6 @@ export const defaultChainOptions: IChainOptions = { blsVerifyAllMainThread: false, blsVerifyAllMultiThread: false, disableBlsBatchVerify: false, - persistInvalidSszObjects: true, - persistInvalidSszObjectsDir: "", proposerBoostEnabled: true, safeSlotsToImportOptimistically: SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY, defaultFeeRecipient: defaultDefaultFeeRecipient, diff --git a/packages/lodestar/src/network/gossip/handlers/index.ts b/packages/lodestar/src/network/gossip/handlers/index.ts index a79943025443..b218dee2c17f 100644 --- a/packages/lodestar/src/network/gossip/handlers/index.ts +++ b/packages/lodestar/src/network/gossip/handlers/index.ts @@ -75,7 +75,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH return { [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { const slot = signedBlock.message.slot; - const blockHex = prettyBytes(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message)); + const forkTypes = config.getForkTypes(slot); + const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); logger.verbose("Received gossip block", { slot: slot, @@ -96,12 +97,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } if (e instanceof BlockGossipError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedBlock", - config.getForkTypes(slot).SignedBeaconBlock.serialize(signedBlock), - `gossip_slot_${slot}` - ); - logger.debug("The invalid gossip block was written to", archivedPath); + chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`); } throw e; @@ -151,12 +147,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH validationResult = await validateGossipAggregateAndProofRetryUnknownRoot(chain, signedAggregateAndProof); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "signedAggregatedAndProof", - ssz.phase0.SignedAggregateAndProof.serialize(signedAggregateAndProof), - toHexString(ssz.phase0.SignedAggregateAndProof.hashTreeRoot(signedAggregateAndProof)) - ); - logger.debug("The invalid gossip aggregate and proof was written to", archivedPath, e); + chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "gossip_reject"); } throw e; } @@ -191,12 +182,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH validationResult = await validateGossipAttestationRetryUnknownRoot(chain, attestation, subnet); } catch (e) { if (e instanceof AttestationError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "attestation", - ssz.phase0.Attestation.serialize(attestation), - toHexString(ssz.phase0.Attestation.hashTreeRoot(attestation)) - ); - logger.debug("The invalid gossip attestation was written to", archivedPath); + chain.persistInvalidSszValue(ssz.phase0.Attestation, attestation, "gossip_reject"); } throw e; } @@ -268,12 +254,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH contributionAndProof ).catch((e) => { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "contributionAndProof", - ssz.altair.SignedContributionAndProof.serialize(contributionAndProof), - toHexString(ssz.altair.SignedContributionAndProof.hashTreeRoot(contributionAndProof)) - ); - logger.debug("The invalid gossip contribution and proof was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "gossip_reject"); } throw e; }); @@ -293,12 +274,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH indexInSubcommittee = (await validateGossipSyncCommittee(chain, syncCommittee, subnet)).indexInSubcommittee; } catch (e) { if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) { - const archivedPath = chain.persistInvalidSszObject( - "syncCommittee", - ssz.altair.SyncCommitteeMessage.serialize(syncCommittee), - toHexString(ssz.altair.SyncCommitteeMessage.hashTreeRoot(syncCommittee)) - ); - logger.debug("The invalid gossip sync committee was written to", archivedPath); + chain.persistInvalidSszValue(ssz.altair.SyncCommitteeMessage, syncCommittee, "gossip_reject"); } throw e; } diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index 7ba57044b911..e901a0ed4e58 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -1,7 +1,7 @@ import sinon from "sinon"; import {AbortController} from "@chainsafe/abort-controller"; -import {toHexString} from "@chainsafe/ssz"; +import {CompositeTypeAny, toHexString, TreeView} from "@chainsafe/ssz"; import {allForks, UintNum64, Root, Slot, ssz, Uint16, UintBn64} from "@chainsafe/lodestar-types"; import {IBeaconConfig} from "@chainsafe/lodestar-config"; import {BeaconStateAllForks, CachedBeaconStateAllForks} from "@chainsafe/lodestar-beacon-state-transition"; @@ -132,6 +132,8 @@ export class MockBeaconChain implements IBeaconChain { this.reprocessController = new ReprocessController(null); } + persistInvalidSszView(view: TreeView, suffix?: string): void {} + getHeadState(): CachedBeaconStateAllForks { return createCachedBeaconStateTest(this.state, this.config); } @@ -179,8 +181,12 @@ export class MockBeaconChain implements IBeaconChain { return false; } - persistInvalidSszObject(): string | null { - return null; + persistInvalidSszObject(): void { + return; + } + + persistInvalidSszValue(): void { + return; } async updateBeaconProposerData(): Promise {} From a607841816e51f0e30284bb1ad214e7b615a2b8d Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 27 May 2022 11:52:31 +0700 Subject: [PATCH 2/2] Switch to fs async api --- packages/lodestar/src/chain/chain.ts | 21 ++++---- packages/lodestar/src/util/file.ts | 30 +++++++---- packages/lodestar/test/unit/util/file.test.ts | 52 ++++++++++++++++--- .../lodestar/test/utils/mocks/chain/chain.ts | 2 +- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index e2661c4b4481..408e5ac3e8df 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -2,7 +2,6 @@ * @module chain */ -import fs from "node:fs"; import path from "node:path"; import { BeaconStateAllForks, @@ -22,6 +21,7 @@ import {IBeaconDb} from "../db/index.js"; import {IMetrics} from "../metrics/index.js"; import {IEth1ForBlockProduction} from "../eth1/index.js"; import {IExecutionEngine} from "../executionEngine/index.js"; +import {ensureDir, writeIfNotExist} from "../util/file.js"; import {CheckpointStateCache, StateContextCache} from "./stateCache/index.js"; import {BlockProcessor, PartiallyVerifiedBlockFlags} from "./blocks/index.js"; import {IBeaconClock, LocalClock} from "./clock/index.js"; @@ -305,17 +305,22 @@ export class BeaconChain implements IBeaconChain { persistInvalidSszValue(type: Type, sszObject: T, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); + void this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); } } persistInvalidSszView(view: TreeView, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); + void this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); } } - private persistInvalidSszObject(typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string): void { + private async persistInvalidSszObject( + typeName: string, + bytes: Uint8Array, + root: Uint8Array, + suffix?: string + ): Promise { if (!this.opts.persistInvalidSszObjects) { return; } @@ -328,15 +333,11 @@ export class BeaconChain implements IBeaconChain { const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr); const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`); - if (!fs.existsSync(dirpath)) { - fs.mkdirSync(dirpath, {recursive: true}); - } + await ensureDir(dirpath); // as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes // remove date suffixes in file name, and check duplicate to avoid redundant persistence - if (!fs.existsSync(filepath)) { - fs.writeFileSync(filepath, bytes); - } + await writeIfNotExist(filepath, bytes); this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath}); } diff --git a/packages/lodestar/src/util/file.ts b/packages/lodestar/src/util/file.ts index 11280b42520b..d4ff3ca4f98b 100644 --- a/packages/lodestar/src/util/file.ts +++ b/packages/lodestar/src/util/file.ts @@ -4,19 +4,29 @@ import fs from "node:fs"; import path from "node:path"; +import {promisify} from "node:util"; -/** - * Recursively ensures directory exists by creating any missing directories - * @param {string} filePath - */ -export function ensureDirectoryExistence(filePath: string): boolean { - const dirname = path.dirname(filePath); - if (fs.existsSync(dirname)) { +/** Ensure a directory exists */ +export async function ensureDir(path: string): Promise { + try { + await promisify(fs.stat)(path); + } catch (_) { + // not exists + await promisify(fs.mkdir)(path, {recursive: true}); + } +} + +/** Write data to a file if it does not exist */ +export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise { + try { + await promisify(fs.stat)(filepath); + return false; + // file exists, do nothing + } catch (_) { + // not exists + await promisify(fs.writeFile)(filepath, bytes); return true; } - ensureDirectoryExistence(dirname); - fs.mkdirSync(dirname); - return true; } export function rmDir(dir: string): void { diff --git a/packages/lodestar/test/unit/util/file.test.ts b/packages/lodestar/test/unit/util/file.test.ts index 5010aae3ddee..9c89c2edb9e8 100644 --- a/packages/lodestar/test/unit/util/file.test.ts +++ b/packages/lodestar/test/unit/util/file.test.ts @@ -1,14 +1,50 @@ -import {assert} from "chai"; -import {ensureDirectoryExistence} from "../../../src/util/file.js"; +import fs from "node:fs"; +import path from "node:path"; +import {expect} from "chai"; +import {ensureDir, writeIfNotExist} from "../../../src/util/file.js"; -describe("util/file", function () { - const testFilePath = "keys/toml/test_config.toml"; +describe("file util", function () { + this.timeout(3000); + const dirPath = path.join(".", "keys/toml/test_config.toml"); - it("should create directory needed for file writes", () => { - assert.isTrue(ensureDirectoryExistence(testFilePath)); + describe("ensureDir", function () { + it("create dir if not exists", async () => { + expect(fs.existsSync(dirPath), `${dirPath} should not exist`).to.be.false; + await ensureDir(dirPath); + expect(fs.existsSync(dirPath), `${dirPath} should exist`).to.be.true; + fs.rmdirSync(dirPath); + }); }); - it("should return true for existing directory", () => { - assert.isTrue(ensureDirectoryExistence("src")); + describe("writeIfNotExist", function () { + const filePath = path.join(dirPath, "test.txt"); + const data = new Uint8Array([0, 1, 2]); + before(async () => { + await ensureDir(dirPath); + }); + + after(() => { + fs.rmdirSync(dirPath); + }); + + it("write to a non-existed file", async () => { + expect(fs.existsSync(filePath)).to.be.false; + expect(await writeIfNotExist(filePath, data)).to.be.true; + const bytes = fs.readFileSync(filePath); + expect(new Uint8Array(bytes)).to.be.deep.equals(data); + + // clean up + fs.rmSync(filePath); + }); + + it("write to an existing file", async () => { + fs.writeFileSync(filePath, new Uint8Array([3, 4])); + expect(await writeIfNotExist(filePath, data)).to.be.false; + const bytes = fs.readFileSync(filePath); + expect(new Uint8Array(bytes)).not.to.be.deep.equals(data); + + // clean up + fs.rmSync(filePath); + }); }); }); diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index e901a0ed4e58..821199ac70a8 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -132,7 +132,7 @@ export class MockBeaconChain implements IBeaconChain { this.reprocessController = new ReprocessController(null); } - persistInvalidSszView(view: TreeView, suffix?: string): void {} + persistInvalidSszView(_: TreeView): void {} getHeadState(): CachedBeaconStateAllForks { return createCachedBeaconStateTest(this.state, this.config);