From a607841816e51f0e30284bb1ad214e7b615a2b8d Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 27 May 2022 11:52:31 +0700 Subject: [PATCH] Switch to fs async api --- packages/lodestar/src/chain/chain.ts | 21 ++++---- packages/lodestar/src/util/file.ts | 30 +++++++---- packages/lodestar/test/unit/util/file.test.ts | 52 ++++++++++++++++--- .../lodestar/test/utils/mocks/chain/chain.ts | 2 +- 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/packages/lodestar/src/chain/chain.ts b/packages/lodestar/src/chain/chain.ts index e2661c4b4481..408e5ac3e8df 100644 --- a/packages/lodestar/src/chain/chain.ts +++ b/packages/lodestar/src/chain/chain.ts @@ -2,7 +2,6 @@ * @module chain */ -import fs from "node:fs"; import path from "node:path"; import { BeaconStateAllForks, @@ -22,6 +21,7 @@ import {IBeaconDb} from "../db/index.js"; import {IMetrics} from "../metrics/index.js"; import {IEth1ForBlockProduction} from "../eth1/index.js"; import {IExecutionEngine} from "../executionEngine/index.js"; +import {ensureDir, writeIfNotExist} from "../util/file.js"; import {CheckpointStateCache, StateContextCache} from "./stateCache/index.js"; import {BlockProcessor, PartiallyVerifiedBlockFlags} from "./blocks/index.js"; import {IBeaconClock, LocalClock} from "./clock/index.js"; @@ -305,17 +305,22 @@ export class BeaconChain implements IBeaconChain { persistInvalidSszValue(type: Type, sszObject: T, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); + void this.persistInvalidSszObject(type.typeName, type.serialize(sszObject), type.hashTreeRoot(sszObject), suffix); } } persistInvalidSszView(view: TreeView, suffix?: string): void { if (this.opts.persistInvalidSszObjects) { - this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); + void this.persistInvalidSszObject(view.type.typeName, view.serialize(), view.hashTreeRoot(), suffix); } } - private persistInvalidSszObject(typeName: string, bytes: Uint8Array, root: Uint8Array, suffix?: string): void { + private async persistInvalidSszObject( + typeName: string, + bytes: Uint8Array, + root: Uint8Array, + suffix?: string + ): Promise { if (!this.opts.persistInvalidSszObjects) { return; } @@ -328,15 +333,11 @@ export class BeaconChain implements IBeaconChain { const dirpath = path.join(this.opts.persistInvalidSszObjectsDir ?? "invalid_ssz_objects", dateStr); const filepath = path.join(dirpath, `${typeName}_${toHex(root)}.ssz`); - if (!fs.existsSync(dirpath)) { - fs.mkdirSync(dirpath, {recursive: true}); - } + await ensureDir(dirpath); // as of Feb 17 2022 there are a lot of duplicate files stored with different date suffixes // remove date suffixes in file name, and check duplicate to avoid redundant persistence - if (!fs.existsSync(filepath)) { - fs.writeFileSync(filepath, bytes); - } + await writeIfNotExist(filepath, bytes); this.logger.debug("Persisted invalid ssz object", {id: suffix, filepath}); } diff --git a/packages/lodestar/src/util/file.ts b/packages/lodestar/src/util/file.ts index 11280b42520b..d4ff3ca4f98b 100644 --- a/packages/lodestar/src/util/file.ts +++ b/packages/lodestar/src/util/file.ts @@ -4,19 +4,29 @@ import fs from "node:fs"; import path from "node:path"; +import {promisify} from "node:util"; -/** - * Recursively ensures directory exists by creating any missing directories - * @param {string} filePath - */ -export function ensureDirectoryExistence(filePath: string): boolean { - const dirname = path.dirname(filePath); - if (fs.existsSync(dirname)) { +/** Ensure a directory exists */ +export async function ensureDir(path: string): Promise { + try { + await promisify(fs.stat)(path); + } catch (_) { + // not exists + await promisify(fs.mkdir)(path, {recursive: true}); + } +} + +/** Write data to a file if it does not exist */ +export async function writeIfNotExist(filepath: string, bytes: Uint8Array): Promise { + try { + await promisify(fs.stat)(filepath); + return false; + // file exists, do nothing + } catch (_) { + // not exists + await promisify(fs.writeFile)(filepath, bytes); return true; } - ensureDirectoryExistence(dirname); - fs.mkdirSync(dirname); - return true; } export function rmDir(dir: string): void { diff --git a/packages/lodestar/test/unit/util/file.test.ts b/packages/lodestar/test/unit/util/file.test.ts index 5010aae3ddee..9c89c2edb9e8 100644 --- a/packages/lodestar/test/unit/util/file.test.ts +++ b/packages/lodestar/test/unit/util/file.test.ts @@ -1,14 +1,50 @@ -import {assert} from "chai"; -import {ensureDirectoryExistence} from "../../../src/util/file.js"; +import fs from "node:fs"; +import path from "node:path"; +import {expect} from "chai"; +import {ensureDir, writeIfNotExist} from "../../../src/util/file.js"; -describe("util/file", function () { - const testFilePath = "keys/toml/test_config.toml"; +describe("file util", function () { + this.timeout(3000); + const dirPath = path.join(".", "keys/toml/test_config.toml"); - it("should create directory needed for file writes", () => { - assert.isTrue(ensureDirectoryExistence(testFilePath)); + describe("ensureDir", function () { + it("create dir if not exists", async () => { + expect(fs.existsSync(dirPath), `${dirPath} should not exist`).to.be.false; + await ensureDir(dirPath); + expect(fs.existsSync(dirPath), `${dirPath} should exist`).to.be.true; + fs.rmdirSync(dirPath); + }); }); - it("should return true for existing directory", () => { - assert.isTrue(ensureDirectoryExistence("src")); + describe("writeIfNotExist", function () { + const filePath = path.join(dirPath, "test.txt"); + const data = new Uint8Array([0, 1, 2]); + before(async () => { + await ensureDir(dirPath); + }); + + after(() => { + fs.rmdirSync(dirPath); + }); + + it("write to a non-existed file", async () => { + expect(fs.existsSync(filePath)).to.be.false; + expect(await writeIfNotExist(filePath, data)).to.be.true; + const bytes = fs.readFileSync(filePath); + expect(new Uint8Array(bytes)).to.be.deep.equals(data); + + // clean up + fs.rmSync(filePath); + }); + + it("write to an existing file", async () => { + fs.writeFileSync(filePath, new Uint8Array([3, 4])); + expect(await writeIfNotExist(filePath, data)).to.be.false; + const bytes = fs.readFileSync(filePath); + expect(new Uint8Array(bytes)).not.to.be.deep.equals(data); + + // clean up + fs.rmSync(filePath); + }); }); }); diff --git a/packages/lodestar/test/utils/mocks/chain/chain.ts b/packages/lodestar/test/utils/mocks/chain/chain.ts index e901a0ed4e58..821199ac70a8 100644 --- a/packages/lodestar/test/utils/mocks/chain/chain.ts +++ b/packages/lodestar/test/utils/mocks/chain/chain.ts @@ -132,7 +132,7 @@ export class MockBeaconChain implements IBeaconChain { this.reprocessController = new ReprocessController(null); } - persistInvalidSszView(view: TreeView, suffix?: string): void {} + persistInvalidSszView(_: TreeView): void {} getHeadState(): CachedBeaconStateAllForks { return createCachedBeaconStateTest(this.state, this.config);