From 9500cc3475651ebeebf8be58dd6414a3e248a9e9 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 14 Aug 2024 10:54:39 +0700 Subject: [PATCH 1/5] fix: use shared Buffer for toMemoryEfficientHexStr() --- packages/state-transition/src/cache/pubkeyCache.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index b2b45ca09d25..0f7e9101c7d2 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -5,6 +5,11 @@ export type Index2PubkeyCache = PublicKey[]; type PubkeyHex = string; +/** + * BLSPubkey is of type Bytes48, we can use a single buffer to compute hex for all pubkeys + */ +const pubkeyBuf = Buffer.alloc(48); + /** * toHexString() creates hex strings via string concatenation, which are very memory inefficient. * Memory benchmarks show that Buffer.toString("hex") produces strings with 10x less memory. @@ -21,7 +26,8 @@ function toMemoryEfficientHexStr(hex: Uint8Array | string): string { return hex; } - return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); + pubkeyBuf.set(hex); + return pubkeyBuf.toString("hex"); } export class PubkeyIndexMap { From 351013e871242f4f052f01d4d7f41bf023426b01 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 14 Aug 2024 11:56:29 +0700 Subject: [PATCH 2/5] fix: handle pubkey < 48 bytes in unit test --- packages/state-transition/src/cache/pubkeyCache.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 0f7e9101c7d2..5df9a35eafa6 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -4,11 +4,12 @@ import {ValidatorIndex, phase0} from "@lodestar/types"; export type Index2PubkeyCache = PublicKey[]; type PubkeyHex = string; +const PUBKEY_LENGTH = 48; /** * BLSPubkey is of type Bytes48, we can use a single buffer to compute hex for all pubkeys */ -const pubkeyBuf = Buffer.alloc(48); +const pubkeyBuf = Buffer.alloc(PUBKEY_LENGTH); /** * toHexString() creates hex strings via string concatenation, which are very memory inefficient. @@ -26,8 +27,13 @@ function toMemoryEfficientHexStr(hex: Uint8Array | string): string { return hex; } - pubkeyBuf.set(hex); - return pubkeyBuf.toString("hex"); + if (hex.length === PUBKEY_LENGTH) { + pubkeyBuf.set(hex); + return pubkeyBuf.toString("hex"); + } else { + // only happens in unit tests + return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); + } } export class PubkeyIndexMap { From 93e2c66c4d93b5a145bcaf183a0f28982508046b Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 14 Aug 2024 15:44:06 +0700 Subject: [PATCH 3/5] feat: store pubkey as base64 --- packages/beacon-node/test/memory/bytesHex.ts | 17 ++++--- .../state-transition/src/cache/pubkeyCache.ts | 44 ++++++++++++------- .../test/perf/cache/pubkeyCache.test.ts | 43 ++++++++++++++++++ .../test/unit/cache/pubkeyCache.test.ts | 31 +++++++++++++ 4 files changed, 112 insertions(+), 23 deletions(-) create mode 100644 packages/state-transition/test/perf/cache/pubkeyCache.test.ts create mode 100644 packages/state-transition/test/unit/cache/pubkeyCache.test.ts diff --git a/packages/beacon-node/test/memory/bytesHex.ts b/packages/beacon-node/test/memory/bytesHex.ts index 44115d345b2c..0edae316d091 100644 --- a/packages/beacon-node/test/memory/bytesHex.ts +++ b/packages/beacon-node/test/memory/bytesHex.ts @@ -2,13 +2,14 @@ import crypto from "node:crypto"; import {toHexString} from "@chainsafe/ssz"; import {testRunnerMemory} from "./testRunnerMemory.js"; -// Results in Linux Dec 2021 +// Results in Mac M1 Aug 2024 // -// Bytes32 toHexString() - 902.8 bytes / instance -// Bytes32 Buffer.toString(hex) - 86.9 bytes / instance -// Bytes32 Buffer.toString(hex) from Uint8Array - 87.6 bytes / instance -// Bytes32 Buffer.toString(hex) + 0x - 121.7 bytes / instance -// Bytes32 randomBytes32Template() - 924.7 bytes / instance +// Bytes32 toHexString() - 903.4 bytes / instance +// Bytes32 Buffer.toString(hex) - 90.2 bytes / instance +// Bytes32 Buffer.toString(hex) from Uint8Array - 89.1 bytes / instance +// Bytes32 Buffer.toString(base64) from Uint8Array - 72.0 bytes / instance +// Bytes32 Buffer.toString(hex) + 0x - 119.7 bytes / instance +// Bytes32 randomBytes32Template() - 924.9 bytes / instance testRunnerMemoryBpi([ { @@ -23,6 +24,10 @@ testRunnerMemoryBpi([ id: "Bytes32 Buffer.toString(hex) from Uint8Array", getInstance: () => Buffer.from(randomBytesUint8Array(32)).toString("hex"), }, + { + id: "Bytes32 Buffer.toString(base64) from Uint8Array", + getInstance: () => Buffer.from(randomBytesUint8Array(32)).toString("base64"), + }, { id: "Bytes32 Buffer.toString(hex) + 0x", getInstance: () => "0x" + crypto.randomBytes(32).toString("hex"), diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index 5df9a35eafa6..a4fe45643c92 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -4,41 +4,36 @@ import {ValidatorIndex, phase0} from "@lodestar/types"; export type Index2PubkeyCache = PublicKey[]; type PubkeyHex = string; -const PUBKEY_LENGTH = 48; +type PubkeyBase64 = string; +const PUBKEY_BYTE_LENGTH = 48; +const PUBKEY_HEX_CHAR_LENGTH = 96; /** * BLSPubkey is of type Bytes48, we can use a single buffer to compute hex for all pubkeys */ -const pubkeyBuf = Buffer.alloc(PUBKEY_LENGTH); +const pubkeyBuf = Buffer.alloc(PUBKEY_BYTE_LENGTH); /** * toHexString() creates hex strings via string concatenation, which are very memory inefficient. * Memory benchmarks show that Buffer.toString("hex") produces strings with 10x less memory. * - * Does not prefix to save memory, thus the prefix is removed from an already string representation. + * Aug 2024: using base64 is 33% more memory efficient than hex * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(hex: Uint8Array | string): string { - if (typeof hex === "string") { - if (hex.startsWith("0x")) { - hex = hex.slice(2); - } - return hex; - } - - if (hex.length === PUBKEY_LENGTH) { - pubkeyBuf.set(hex); - return pubkeyBuf.toString("hex"); +function toMemoryEfficientHexStr(pubkey: Uint8Array): PubkeyBase64 { + if (pubkey.length === PUBKEY_BYTE_LENGTH) { + pubkeyBuf.set(pubkey); + return pubkeyBuf.toString("base64"); } else { // only happens in unit tests - return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); + return Buffer.from(pubkey.buffer, pubkey.byteOffset, pubkey.byteLength).toString("base64"); } } export class PubkeyIndexMap { - // We don't really need the full pubkey. We could just use the first 20 bytes like an Ethereum address - readonly map = new Map(); + // TODO: We don't really need the full pubkey. We could just use the first 20 bytes like an Ethereum address + readonly map = new Map(); get size(): number { return this.map.size; @@ -48,6 +43,21 @@ export class PubkeyIndexMap { * Must support reading with string for API support where pubkeys are already strings */ get(key: Uint8Array | PubkeyHex): ValidatorIndex | undefined { + if (typeof key === "string") { + if (key.startsWith("0x")) { + key = key.slice(2); + } + if (key.length === PUBKEY_HEX_CHAR_LENGTH) { + // we don't receive api requests frequently, so the below conversion to Buffer then base64 should not be an issue + pubkeyBuf.write(key, "hex"); + return this.map.get(toMemoryEfficientHexStr(pubkeyBuf)); + } else { + // base64 is only for internal use, don't support it + return undefined; + } + } + + // Uint8Array return this.map.get(toMemoryEfficientHexStr(key)); } diff --git a/packages/state-transition/test/perf/cache/pubkeyCache.test.ts b/packages/state-transition/test/perf/cache/pubkeyCache.test.ts new file mode 100644 index 000000000000..a5703d733f94 --- /dev/null +++ b/packages/state-transition/test/perf/cache/pubkeyCache.test.ts @@ -0,0 +1,43 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; + +/** + * Using base64 is a little faster than hex, and it's more memory efficient + * This is on Mac M1 Aug 2024 + * PubkeyCache + * ✔ Public key to string using hex, shared Buffer instance 8466684 ops/s 118.1100 ns/op - 168390 runs 20.0 s + * ✔ Public key to string using hex, separate Buffer instance 5850109 ops/s 170.9370 ns/op - 116277 runs 20.1 s + * ✔ Public key to string using base64, shared Buffer instance 1.031183e+7 ops/s 96.97600 ns/op - 204949 runs 20.0 s + * ✔ Public key to string using base64, separate Buffer instance 6180011 ops/s 161.8120 ns/op - 122556 runs 20.0 s + */ +describe("PubkeyCache", () => { + setBenchOpts({ + minMs: 20_000, + }); + + const pubkeyBuf = Buffer.alloc(48); + const runsFactor = 1000; + const pubkey = Uint8Array.from(Array.from({length: 48}, (_, i) => i)); + + for (const encoding of ["hex" as const, "base64" as const]) { + itBench({ + id: `Public key to string using ${encoding}, shared Buffer instance`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + pubkeyBuf.set(pubkey); + pubkeyBuf.toString(encoding); + } + }, + runsFactor, + }); + + itBench({ + id: `Public key to string using ${encoding}, separate Buffer instance`, + fn: () => { + for (let i = 0; i < runsFactor; i++) { + Buffer.from(pubkey.buffer, pubkey.byteOffset, pubkey.byteLength).toString(encoding); + } + }, + runsFactor, + }); + } +}); diff --git a/packages/state-transition/test/unit/cache/pubkeyCache.test.ts b/packages/state-transition/test/unit/cache/pubkeyCache.test.ts new file mode 100644 index 000000000000..b651640183ec --- /dev/null +++ b/packages/state-transition/test/unit/cache/pubkeyCache.test.ts @@ -0,0 +1,31 @@ +import crypto from "node:crypto"; +import {describe, beforeEach, it, expect} from "vitest"; +import {PubkeyIndexMap} from "../../../src/cache/pubkeyCache.js"; + +describe("PubkeyIndexMap", () => { + let pubkeyIndexMap: PubkeyIndexMap; + + beforeEach(() => { + pubkeyIndexMap = new PubkeyIndexMap(); + }); + + it("should get Uin8Array key", () => { + const key = crypto.randomBytes(48); + pubkeyIndexMap.set(key, 1); + expect(pubkeyIndexMap.get(key)).toEqual(1); + }); + + it("should get hex key", () => { + const key = crypto.randomBytes(48); + pubkeyIndexMap.set(key, 1); + const hexKey = key.toString("hex"); + expect(pubkeyIndexMap.get(hexKey)).toEqual(1); + expect(pubkeyIndexMap.get("0x" + hexKey)).toEqual(1); + }); + + it("should not get base64 key", () => { + const key = crypto.randomBytes(48); + pubkeyIndexMap.set(key, 1); + expect(pubkeyIndexMap.get(key.toString("base64"))).toBeUndefined(); + }); +}); From 2f0b61dde4d5f150b591ef4462759ac4ac38283a Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 15 Aug 2024 15:43:16 +0700 Subject: [PATCH 4/5] chore: memory test for pubkey cache --- .../test/memory/effectiveBalanceIncrements.ts | 20 +- .../test/memory/pubkeyCache.ts | 19 ++ .../test/memory/testRunnerMemory.ts | 237 ++++++++++++++++++ 3 files changed, 257 insertions(+), 19 deletions(-) create mode 100644 packages/state-transition/test/memory/pubkeyCache.ts create mode 100644 packages/state-transition/test/memory/testRunnerMemory.ts diff --git a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts b/packages/state-transition/test/memory/effectiveBalanceIncrements.ts index f1c603b85657..5706affeabdf 100644 --- a/packages/state-transition/test/memory/effectiveBalanceIncrements.ts +++ b/packages/state-transition/test/memory/effectiveBalanceIncrements.ts @@ -1,6 +1,6 @@ import {MutableVector} from "@chainsafe/persistent-ts"; -import {testRunnerMemory} from "@lodestar/beacon-node/test/memory/testRunnerMemory"; import {newZeroedArray} from "../../src/index.js"; +import {testRunnerMemoryBpi} from "./testRunnerMemory.js"; // Results in Linux Feb 2022 // @@ -42,21 +42,3 @@ testRunnerMemoryBpi([ }, }, ]); - -/** - * Test bytes per instance in different representations of raw binary data - */ -function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { - const longestId = Math.max(...testCases.map(({id}) => id.length)); - - for (const {id, getInstance} of testCases) { - const bpi = testRunnerMemory({ - getInstance, - convergeFactor: 1 / 100, - sampleEvery: 5, - }); - - // eslint-disable-next-line no-console - console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); - } -} diff --git a/packages/state-transition/test/memory/pubkeyCache.ts b/packages/state-transition/test/memory/pubkeyCache.ts new file mode 100644 index 000000000000..a8a00bb3e2cd --- /dev/null +++ b/packages/state-transition/test/memory/pubkeyCache.ts @@ -0,0 +1,19 @@ +import crypto from "node:crypto"; +import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; +import {testRunnerMemoryBpi} from "./testRunnerMemory.js"; + +const vcArr = [500_000, 2_000_000]; + +// Results in Mac M1 Aug 2024 using `node --expose-gc --loader=ts-node/esm pubkeyCache.ts` +// PubkeyIndexMap PubkeyIndexMap 500000 - 54672689.8 bytes / instance +// PubkeyIndexMap PubkeyIndexMap 2000000 - 218719267.4 bytes / instance +testRunnerMemoryBpi(vcArr.map((vc) => ({ + id: `PubkeyIndexMap PubkeyIndexMap ${vc}`, + getInstance: () => { + const pubkeyCache = new PubkeyIndexMap(); + for (let i = 0; i < vc; i++) { + pubkeyCache.set(crypto.randomBytes(48), i); + } + return pubkeyCache; + }, +}))); diff --git a/packages/state-transition/test/memory/testRunnerMemory.ts b/packages/state-transition/test/memory/testRunnerMemory.ts new file mode 100644 index 000000000000..690c6d73c965 --- /dev/null +++ b/packages/state-transition/test/memory/testRunnerMemory.ts @@ -0,0 +1,237 @@ +export type TestRunnerMemoryOpts = { + getInstance: (i: number) => T; + sampleEvery?: number; + maxRssBytes?: number; + maxInstances?: number; + computeUsedMemory?: (memoryUsage: NodeJS.MemoryUsage) => number; + logEachSample?: boolean; + convergeFactor?: number; +}; + +if (global.gc === undefined) { + throw Error("Must enable global.gc"); +} + +/** + * Test bytes per instance in different representations of raw binary data + */ +export function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { + const longestId = Math.max(...testCases.map(({id}) => id.length)); + + for (const {id, getInstance} of testCases) { + const bpi = testRunnerMemory({ + getInstance, + convergeFactor: 1 / 100, + sampleEvery: 5, + }); + + // eslint-disable-next-line no-console + console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); + } +} + +export async function testRunnerMemoryGc(opts: TestRunnerMemoryOpts): Promise { + const { + getInstance, + /** + * How to compute the total memory usage. + * Defaults to `heapUsed + external`. + * https://nodejs.org/api/process.html#processmemoryusage + */ + computeUsedMemory = (memoryUsage) => memoryUsage.heapUsed + memoryUsage.external, + } = opts; + + const rounds = 10; + const instancesPerRound = 1000; + const xs: number[] = []; + const usedMemoryArr: number[] = []; + + for (let n = 0; n < rounds; n++) { + global.gc?.(); + global.gc?.(); + await new Promise((r) => setTimeout(r, 100)); + global.gc?.(); + global.gc?.(); + + const totalUsedMemoryPrev = computeUsedMemory(process.memoryUsage()); + + const refs: T[] = []; + for (let i = 0; i < instancesPerRound; i++) { + refs.push(getInstance(i)); + } + + global.gc?.(); + global.gc?.(); + await new Promise((r) => setTimeout(r, 100)); + global.gc?.(); + global.gc?.(); + + const totalUsedMemory = computeUsedMemory(process.memoryUsage()); + + const totalUsedMemoryDiff = totalUsedMemory - totalUsedMemoryPrev; + refs.push(null as any); + + xs.push(n); + usedMemoryArr.push(totalUsedMemoryDiff); + + const usedMemoryReg = linearRegression(xs, usedMemoryArr); + // eslint-disable-next-line no-console + console.log("totalUsedMemoryDiff", totalUsedMemoryDiff, usedMemoryReg); + } +} + +export function testRunnerMemory(opts: TestRunnerMemoryOpts): number { + const { + getInstance, + /** + * Sample memory usage every `sampleEvery` instances + */ + sampleEvery = 1000, + /** + * Stop when `process.memoryUsage().rss > maxRssBytes`. + */ + maxRssBytes = 2e9, + /** + * Stop after creating `maxInstances` instances. + */ + maxInstances = Infinity, + /** + * How to compute the total memory usage. + * Defaults to `heapUsed + external`. + * https://nodejs.org/api/process.html#processmemoryusage + */ + computeUsedMemory = (memoryUsage) => memoryUsage.heapUsed + memoryUsage.external, + logEachSample, + convergeFactor = 0.2 / 100, // 0.2% + } = opts; + + const refs: T[] = []; + const xs: number[] = []; + const usedMemoryArr: number[] = []; + + let prevM0 = 0; + let prevM1 = 0; + + for (let i = 0; i < maxInstances; i++) { + refs.push(getInstance(i)); + + // Stores 5 floating point numbers every 5000 pushes to refs. + // The added memory should be negligible against refs, and linearRegression + // local vars will get garbage collected and won't show up in the .m result + + if (i % sampleEvery === 0) { + global.gc?.(); + global.gc?.(); + + const memoryUsage = process.memoryUsage(); + const usedMemory = computeUsedMemory(memoryUsage); + + xs.push(i); + usedMemoryArr.push(usedMemory); + + if (usedMemoryArr.length > 1) { + // When is a good time to stop a benchmark? A naive answer is after N milliseconds or M runs. + // This code aims to stop the benchmark when the average fn run time has converged at a value + // within a given convergence factor. To prevent doing expensive math to often for fast fn, + // it only takes samples every `sampleEveryMs`. It stores two past values to be able to compute + // a very rough linear and quadratic convergence. + const m = linearRegression(xs, usedMemoryArr).m; + + // Compute convergence (1st order + 2nd order) + const a = prevM0; + const b = prevM1; + const c = m; + + // Approx linear convergence + const convergence1 = Math.abs(c - a); + // Approx quadratic convergence + const convergence2 = Math.abs(b - (a + c) / 2); + // Take the greater of both to enforce linear and quadratic are below convergeFactor + const convergence = Math.max(convergence1, convergence2) / a; + + // Okay to stop + has converged, stop now + if (convergence < convergeFactor) { + return m; + } + + if (logEachSample) { + // eslint-disable-next-line no-console + console.log(i, memoryUsage.rss / maxRssBytes, {m}); + } + + prevM0 = prevM1; + prevM1 = m; + } + } + } + + return linearRegression(xs, usedMemoryArr).m; +} + +/** + * From https://github.com/simple-statistics/simple-statistics/blob/d0d177baf74976a2421638bce98ab028c5afb537/src/linear_regression.js + * + * [Simple linear regression](http://en.wikipedia.org/wiki/Simple_linear_regression) + * is a simple way to find a fitted line between a set of coordinates. + * This algorithm finds the slope and y-intercept of a regression line + * using the least sum of squares. + * + * @param data an array of two-element of arrays, + * like `[[0, 1], [2, 3]]` + * @returns object containing slope and intersect of regression line + * @example + * linearRegression([[0, 0], [1, 1]]); // => { m: 1, b: 0 } + */ +export function linearRegression(xs: number[], ys: number[]): {m: number; b: number} { + let m: number, b: number; + + // Store data length in a local variable to reduce + // repeated object property lookups + const dataLength = xs.length; + + //if there's only one point, arbitrarily choose a slope of 0 + //and a y-intercept of whatever the y of the initial point is + if (dataLength === 1) { + m = 0; + b = ys[0]; + } else { + // Initialize our sums and scope the `m` and `b` + // variables that define the line. + let sumX = 0, + sumY = 0, + sumXX = 0, + sumXY = 0; + + // Use local variables to grab point values + // with minimal object property lookups + let x: number, y: number; + + // Gather the sum of all x values, the sum of all + // y values, and the sum of x^2 and (x*y) for each + // value. + // + // In math notation, these would be SS_x, SS_y, SS_xx, and SS_xy + for (let i = 0; i < dataLength; i++) { + x = xs[i]; + y = ys[i]; + + sumX += x; + sumY += y; + + sumXX += x * x; + sumXY += x * y; + } + + // `m` is the slope of the regression line + m = (dataLength * sumXY - sumX * sumY) / (dataLength * sumXX - sumX * sumX); + + // `b` is the y-intercept of the line. + b = sumY / dataLength - (m * sumX) / dataLength; + } + + // Return both values as an object. + return { + m: m, + b: b, + }; +} From 02aa7b822530220cc38d1b6bf0516a08949270ed Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Thu, 15 Aug 2024 15:47:52 +0700 Subject: [PATCH 5/5] fix: refactor to toMemoryEfficientString() --- .../state-transition/src/cache/pubkeyCache.ts | 8 +++---- .../test/memory/pubkeyCache.ts | 22 ++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index a4fe45643c92..a64f47e8048e 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -21,7 +21,7 @@ const pubkeyBuf = Buffer.alloc(PUBKEY_BYTE_LENGTH); * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(pubkey: Uint8Array): PubkeyBase64 { +function toMemoryEfficientString(pubkey: Uint8Array): PubkeyBase64 { if (pubkey.length === PUBKEY_BYTE_LENGTH) { pubkeyBuf.set(pubkey); return pubkeyBuf.toString("base64"); @@ -50,7 +50,7 @@ export class PubkeyIndexMap { if (key.length === PUBKEY_HEX_CHAR_LENGTH) { // we don't receive api requests frequently, so the below conversion to Buffer then base64 should not be an issue pubkeyBuf.write(key, "hex"); - return this.map.get(toMemoryEfficientHexStr(pubkeyBuf)); + return this.map.get(toMemoryEfficientString(pubkeyBuf)); } else { // base64 is only for internal use, don't support it return undefined; @@ -58,11 +58,11 @@ export class PubkeyIndexMap { } // Uint8Array - return this.map.get(toMemoryEfficientHexStr(key)); + return this.map.get(toMemoryEfficientString(key)); } set(key: Uint8Array, value: ValidatorIndex): void { - this.map.set(toMemoryEfficientHexStr(key), value); + this.map.set(toMemoryEfficientString(key), value); } } diff --git a/packages/state-transition/test/memory/pubkeyCache.ts b/packages/state-transition/test/memory/pubkeyCache.ts index a8a00bb3e2cd..290765aca6fb 100644 --- a/packages/state-transition/test/memory/pubkeyCache.ts +++ b/packages/state-transition/test/memory/pubkeyCache.ts @@ -7,13 +7,15 @@ const vcArr = [500_000, 2_000_000]; // Results in Mac M1 Aug 2024 using `node --expose-gc --loader=ts-node/esm pubkeyCache.ts` // PubkeyIndexMap PubkeyIndexMap 500000 - 54672689.8 bytes / instance // PubkeyIndexMap PubkeyIndexMap 2000000 - 218719267.4 bytes / instance -testRunnerMemoryBpi(vcArr.map((vc) => ({ - id: `PubkeyIndexMap PubkeyIndexMap ${vc}`, - getInstance: () => { - const pubkeyCache = new PubkeyIndexMap(); - for (let i = 0; i < vc; i++) { - pubkeyCache.set(crypto.randomBytes(48), i); - } - return pubkeyCache; - }, -}))); +testRunnerMemoryBpi( + vcArr.map((vc) => ({ + id: `PubkeyIndexMap PubkeyIndexMap ${vc}`, + getInstance: () => { + const pubkeyCache = new PubkeyIndexMap(); + for (let i = 0; i < vc; i++) { + pubkeyCache.set(crypto.randomBytes(48), i); + } + return pubkeyCache; + }, + })) +);