From 2d3e8dc9ac3b65c3128e1cab60d0889511e89073 Mon Sep 17 00:00:00 2001 From: Troy Kessler <43882936+troykessler@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:18:48 +0200 Subject: [PATCH] Add size limit for debug folder (#128) * chore: started implementation of debug-max-size * chore: added debug size limit check * chore: add kysor support for debug-max-size --- common/protocol/src/index.ts | 7 +++++ .../src/methods/helpers/archiveDebugBundle.ts | 14 ++++++++- common/protocol/src/utils/helpers.ts | 29 +++++++++++++++++++ tools/kysor/src/commands/valaccounts.ts | 6 ++++ tools/kysor/src/kysor.ts | 5 ++++ tools/kysor/src/types/interfaces.ts | 1 + 6 files changed, 61 insertions(+), 1 deletion(-) diff --git a/common/protocol/src/index.ts b/common/protocol/src/index.ts index 18de8d74..160c27fb 100644 --- a/common/protocol/src/index.ts +++ b/common/protocol/src/index.ts @@ -95,6 +95,7 @@ export class Validator { protected requestBackoff!: number; protected cache!: string; protected debug!: boolean; + protected debugMaxSize!: number; protected metrics!: boolean; protected metricsPort!: number; protected home!: string; @@ -254,6 +255,11 @@ export class Validator { "jsonfile" ) .option("--debug", "Run the validator node in debug mode") + .option( + "--debug-max-size ", + "The maximum size of the debug folder in bytes. Use zero to disable [default = 10737418240 (10GB)]", + "10737418240" + ) .option( "--metrics", "Start a prometheus metrics server on http://localhost:8080/metrics" @@ -301,6 +307,7 @@ export class Validator { this.requestBackoff = parseInt(options.requestBackoff); this.cache = options.cache; this.debug = options.debug; + this.debugMaxSize = parseInt(options.debugMaxSize); this.metrics = options.metrics; this.metricsPort = parseInt(options.metricsPort); this.home = options.home; diff --git a/common/protocol/src/methods/helpers/archiveDebugBundle.ts b/common/protocol/src/methods/helpers/archiveDebugBundle.ts index 6228dfbc..dd349455 100644 --- a/common/protocol/src/methods/helpers/archiveDebugBundle.ts +++ b/common/protocol/src/methods/helpers/archiveDebugBundle.ts @@ -1,6 +1,6 @@ import { existsSync, mkdirSync, createWriteStream, readFileSync } from "fs"; import path from "path"; -import { DataItem, standardizeError, Validator } from "../.."; +import { DataItem, dirSize, standardizeError, Validator } from "../.."; import JSZip from "jszip"; import { VoteType } from "@kyvejs/types/client/kyve/bundles/v1beta1/tx"; @@ -31,6 +31,18 @@ export function archiveDebugBundle( mkdirSync(path.join(this.home, `debug`), { recursive: true }); } + // if size of debug folder exceeds debug max limit we don't store an archive + if (this.debugMaxSize > 0) { + const debugDirSize = dirSize(path.join(this.home, `debug`)); + + if (debugDirSize >= this.debugMaxSize) { + this.logger.warn( + `Skipped saving debug information, debug dir exceeded max size of ${this.debugMaxSize} with ${debugDirSize} bytes` + ); + return; + } + } + const zip = new JSZip(); // save metadata which includes vote reasons and args diff --git a/common/protocol/src/utils/helpers.ts b/common/protocol/src/utils/helpers.ts index d6407e2e..e87cca05 100644 --- a/common/protocol/src/utils/helpers.ts +++ b/common/protocol/src/utils/helpers.ts @@ -1,5 +1,7 @@ import { BigNumber } from "bignumber.js"; import crypto from "crypto"; +import { statSync, readdirSync } from "fs"; +import { join } from "path"; import { DataItem } from ".."; @@ -172,3 +174,30 @@ export async function callWithBackoffStrategy( })().catch((err) => reject(err)); }); } + +/** + * Get recursively the size of a dir including all children + * + * @method dirSize + * @param {onErrorRetryerType} dir path of the dir + * @return {number} returns the size in bytes + */ +export const dirSize = (dir: string): number => { + const files = readdirSync(dir, { withFileTypes: true }); + + const paths = files.map((file) => { + const path = join(dir, file.name); + + if (file.isDirectory()) return dirSize(path); + + if (file.isFile()) { + const { size } = statSync(path); + + return size; + } + + return 0; + }); + + return paths.flat(Infinity).reduce((i, size) => i + size, 0); +}; diff --git a/tools/kysor/src/commands/valaccounts.ts b/tools/kysor/src/commands/valaccounts.ts index 44347224..6e022474 100644 --- a/tools/kysor/src/commands/valaccounts.ts +++ b/tools/kysor/src/commands/valaccounts.ts @@ -51,6 +51,11 @@ valaccounts "Specify the port of the metrics server. Only considered if '--metrics' is set [default = 8080]", "8080" ) + .option( + "--debug-max-size ", + "Specify the max size in bytes for the debug folder. [default = 10737418240 (10GB)]", + "10737418240" + ) .option("--recover", "Create a valaccount by importing an existing mnemonic") .action(async (options) => { try { @@ -133,6 +138,7 @@ valaccounts cache: options.cache, metrics: options.metrics, metricsPort: options.metricsPort, + debugMaxSize: options.debugMaxSize, }; fs.writeFileSync( diff --git a/tools/kysor/src/kysor.ts b/tools/kysor/src/kysor.ts index e3a04124..db3956d6 100644 --- a/tools/kysor/src/kysor.ts +++ b/tools/kysor/src/kysor.ts @@ -424,6 +424,11 @@ export const run = async (options: any) => { args.push(`${valaccount.metricsPort}`); } + if (valaccount.debugMaxSize) { + args.push("--debug-max-size"); + args.push(`${valaccount.debugMaxSize}`); + } + logger.info("Starting process ..."); console.log("\n"); diff --git a/tools/kysor/src/types/interfaces.ts b/tools/kysor/src/types/interfaces.ts index be864b1d..037c0571 100644 --- a/tools/kysor/src/types/interfaces.ts +++ b/tools/kysor/src/types/interfaces.ts @@ -18,4 +18,5 @@ export interface IValaccountConfig { cache: string; metrics: boolean; metricsPort: string; + debugMaxSize: number; }