From 0aa744599aeb682a2a234d58f044955569f89dec Mon Sep 17 00:00:00 2001 From: Gabriel Rocheleau Date: Sat, 17 Aug 2024 07:11:09 -0400 Subject: [PATCH] utils: refactor trie and verkle utils (#3600) * verkle: remove bytes utils * util: add matching bytes length util from verkle * util: refactor PrioritizedTaskExecutor * verkle: remove extra import --- package-lock.json | 40 ------------- packages/trie/src/util/index.ts | 1 - packages/trie/src/util/walkController.ts | 4 +- packages/util/src/bytes.ts | 23 ++++++++ packages/util/src/index.ts | 1 + packages/{trie/src/util => util/src}/tasks.ts | 0 packages/util/test/bytes.spec.ts | 45 ++++++++++++++ .../test/util => util/test}/tasks.spec.ts | 2 +- packages/verkle/src/index.ts | 1 - packages/verkle/src/util/bytes.ts | 22 ------- packages/verkle/src/util/index.ts | 2 - packages/verkle/src/util/tasks.ts | 59 ------------------- packages/verkle/src/verkleTree.ts | 2 +- packages/verkle/test/util/bytes.spec.ts | 47 --------------- packages/verkle/test/verkle.spec.ts | 3 +- 15 files changed, 74 insertions(+), 178 deletions(-) rename packages/{trie/src/util => util/src}/tasks.ts (100%) rename packages/{trie/test/util => util/test}/tasks.spec.ts (91%) delete mode 100644 packages/verkle/src/util/bytes.ts delete mode 100644 packages/verkle/src/util/index.ts delete mode 100644 packages/verkle/src/util/tasks.ts delete mode 100644 packages/verkle/test/util/bytes.spec.ts diff --git a/package-lock.json b/package-lock.json index df53374b3e..9b04819001 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1267,22 +1267,6 @@ "p-defer": "^4.0.0" } }, - "node_modules/@lmdb/lmdb-darwin-arm64": { - "dev": true, - "optional": true - }, - "node_modules/@lmdb/lmdb-darwin-x64": { - "dev": true, - "optional": true - }, - "node_modules/@lmdb/lmdb-linux-arm": { - "dev": true, - "optional": true - }, - "node_modules/@lmdb/lmdb-linux-arm64": { - "dev": true, - "optional": true - }, "node_modules/@lmdb/lmdb-linux-x64": { "version": "2.9.4", "cpu": [ @@ -1295,26 +1279,6 @@ "linux" ] }, - "node_modules/@lmdb/lmdb-win32-x64": { - "dev": true, - "optional": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": { - "dev": true, - "optional": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "dev": true, - "optional": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "dev": true, - "optional": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "dev": true, - "optional": true - }, "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { "version": "3.0.3", "cpu": [ @@ -1327,10 +1291,6 @@ "linux" ] }, - "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "dev": true, - "optional": true - }, "node_modules/@multiformats/dns": { "version": "1.0.6", "license": "Apache-2.0 OR MIT", diff --git a/packages/trie/src/util/index.ts b/packages/trie/src/util/index.ts index d86ae67d7d..14bcecbdab 100644 --- a/packages/trie/src/util/index.ts +++ b/packages/trie/src/util/index.ts @@ -1,4 +1,3 @@ export * from './encoding.js' export * from './genesisState.js' -export * from './tasks.js' export * from './walkController.js' diff --git a/packages/trie/src/util/walkController.ts b/packages/trie/src/util/walkController.ts index 6c35761308..0339f3fafa 100644 --- a/packages/trie/src/util/walkController.ts +++ b/packages/trie/src/util/walkController.ts @@ -1,6 +1,6 @@ -import { BranchNode, ExtensionNode, LeafNode } from '../node/index.js' +import { PrioritizedTaskExecutor } from '@ethereumjs/util' -import { PrioritizedTaskExecutor } from './tasks.js' +import { BranchNode, ExtensionNode, LeafNode } from '../node/index.js' import type { Trie } from '../trie.js' import type { FoundNodeFunction, Nibbles, TrieNode } from '../types.js' diff --git a/packages/util/src/bytes.ts b/packages/util/src/bytes.ts index 7226c5b4b5..543540323a 100644 --- a/packages/util/src/bytes.ts +++ b/packages/util/src/bytes.ts @@ -551,3 +551,26 @@ export { bytesToUtf8, equalsBytes, utf8ToBytes } from 'ethereum-cryptography/uti export function hexToBigInt(input: PrefixedHexString): bigint { return bytesToBigInt(hexToBytes(isHexString(input) ? input : `0x${input}`)) } + +/** + * Compares two byte arrays and returns the count of consecutively matching items from the start. + * + * @function + * @param {Uint8Array} bytes1 - The first Uint8Array to compare. + * @param {Uint8Array} bytes2 - The second Uint8Array to compare. + * @returns {number} The count of consecutively matching items from the start. + */ +export function matchingBytesLength(bytes1: Uint8Array, bytes2: Uint8Array): number { + let count = 0 + const minLength = Math.min(bytes1.length, bytes2.length) + + for (let i = 0; i < minLength; i++) { + if (bytes1[i] === bytes2[i]) { + count++ + } else { + // Break early if a mismatch is found + break + } + } + return count +} diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index bc3e1462cf..f1e333bd48 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -65,4 +65,5 @@ export * from './lock.js' export * from './mapDB.js' export * from './provider.js' export * from './requests.js' +export * from './tasks.js' export * from './verkle.js' diff --git a/packages/trie/src/util/tasks.ts b/packages/util/src/tasks.ts similarity index 100% rename from packages/trie/src/util/tasks.ts rename to packages/util/src/tasks.ts diff --git a/packages/util/test/bytes.spec.ts b/packages/util/test/bytes.spec.ts index cf031f98af..8f41e56dea 100644 --- a/packages/util/test/bytes.spec.ts +++ b/packages/util/test/bytes.spec.ts @@ -17,6 +17,7 @@ import { intToHex, intToUnpaddedBytes, isZeroAddress, + matchingBytesLength, setLengthLeft, setLengthRight, short, @@ -482,3 +483,47 @@ describe('unprefixedHexToBytes', () => { assert.deepEqual(converted, new Uint8Array([17])) }) }) + +describe('matchingBytesLength', () => { + it('should return 0 when both arrays are empty', () => { + const bytes1 = new Uint8Array([]) + const bytes2 = new Uint8Array([]) + assert.equal(matchingBytesLength(bytes1, bytes2), 0) + }) + + it('should return 0 when one of the arrays is empty', () => { + const bytes1 = new Uint8Array([1, 2, 3]) + const bytes2 = new Uint8Array([]) + assert.equal(matchingBytesLength(bytes1, bytes2), 0) + }) + + it('should return 0 when arrays have no matching elements', () => { + const bytes1 = new Uint8Array([1, 2, 3]) + const bytes2 = new Uint8Array([4, 5, 6]) + assert.equal(matchingBytesLength(bytes1, bytes2), 0) + }) + + it('should handle arrays with same elements but different lengths', () => { + const bytes1 = new Uint8Array([1, 2, 3]) + const bytes2 = new Uint8Array([1, 2, 3, 4]) + assert.equal(matchingBytesLength(bytes1, bytes2), 3) + }) + + it('should handle arrays with matching elements at end', () => { + const bytes1 = new Uint8Array([1, 2, 3]) + const bytes2 = new Uint8Array([0, 1, 2, 3]) + assert.equal(matchingBytesLength(bytes1, bytes2), 0) + }) + + it('should handle arrays with matching elements at start', () => { + const bytes1 = new Uint8Array([1, 2, 3]) + const bytes2 = new Uint8Array([1, 2, 3, 4, 5]) + assert.equal(matchingBytesLength(bytes1, bytes2), 3) + }) + + it('should handle arrays with large number of elements', () => { + const bytes1 = new Uint8Array(Array.from({ length: 1000000 }, (_, i) => i)) + const bytes2 = new Uint8Array(Array.from({ length: 1000000 }, (_, i) => i)) + assert.equal(matchingBytesLength(bytes1, bytes2), 1000000) + }) +}) diff --git a/packages/trie/test/util/tasks.spec.ts b/packages/util/test/tasks.spec.ts similarity index 91% rename from packages/trie/test/util/tasks.spec.ts rename to packages/util/test/tasks.spec.ts index 0ef1cea504..6a56326ae6 100644 --- a/packages/trie/test/util/tasks.spec.ts +++ b/packages/util/test/tasks.spec.ts @@ -1,6 +1,6 @@ import { assert, describe, it } from 'vitest' -import { PrioritizedTaskExecutor } from '../../src/index.js' +import { PrioritizedTaskExecutor } from '../src/index.js' const taskExecutor = new PrioritizedTaskExecutor(2) diff --git a/packages/verkle/src/index.ts b/packages/verkle/src/index.ts index d7fde388d7..af73aaea16 100644 --- a/packages/verkle/src/index.ts +++ b/packages/verkle/src/index.ts @@ -1,5 +1,4 @@ export * from './db/index.js' export * from './node/index.js' export * from './types.js' -export * from './util/index.js' export * from './verkleTree.js' diff --git a/packages/verkle/src/util/bytes.ts b/packages/verkle/src/util/bytes.ts deleted file mode 100644 index 4fb0c236ac..0000000000 --- a/packages/verkle/src/util/bytes.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Compares two byte arrays and returns the count of consecutively matching items from the start. - * - * @function - * @param {Uint8Array} bytes1 - The first Uint8Array to compare. - * @param {Uint8Array} bytes2 - The second Uint8Array to compare. - * @returns {number} The count of consecutively matching items from the start. - */ -export function matchingBytesLength(bytes1: Uint8Array, bytes2: Uint8Array): number { - let count = 0 - const minLength = Math.min(bytes1.length, bytes2.length) - - for (let i = 0; i < minLength; i++) { - if (bytes1[i] === bytes2[i]) { - count++ - } else { - // Break early if a mismatch is found - break - } - } - return count -} diff --git a/packages/verkle/src/util/index.ts b/packages/verkle/src/util/index.ts deleted file mode 100644 index 2da7bbb112..0000000000 --- a/packages/verkle/src/util/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './bytes.js' -export * from './tasks.js' diff --git a/packages/verkle/src/util/tasks.ts b/packages/verkle/src/util/tasks.ts deleted file mode 100644 index 26dc56a96a..0000000000 --- a/packages/verkle/src/util/tasks.ts +++ /dev/null @@ -1,59 +0,0 @@ -interface Task { - priority: number - fn: Function -} - -export class PrioritizedTaskExecutor { - /** The maximum size of the pool */ - private maxPoolSize: number - /** The current size of the pool */ - private currentPoolSize: number - /** The task queue */ - private queue: Task[] - - /** - * Executes tasks up to maxPoolSize at a time, other items are put in a priority queue. - * @class PrioritizedTaskExecutor - * @private - * @param maxPoolSize The maximum size of the pool - */ - constructor(maxPoolSize: number) { - this.maxPoolSize = maxPoolSize - this.currentPoolSize = 0 - this.queue = [] - } - - /** - * Executes the task or queues it if no spots are available. - * When a task is added, check if there are spots left in the pool. - * If a spot is available, claim that spot and give back the spot once the asynchronous task has been resolved. - * When no spots are available, add the task to the task queue. The task will be executed at some point when another task has been resolved. - * @private - * @param priority The priority of the task - * @param fn The function that accepts the callback, which must be called upon the task completion. - */ - executeOrQueue(priority: number, fn: Function) { - if (this.currentPoolSize < this.maxPoolSize) { - this.currentPoolSize++ - fn(() => { - this.currentPoolSize-- - if (this.queue.length > 0) { - this.queue.sort((a, b) => b.priority - a.priority) - const item = this.queue.shift() - this.executeOrQueue(item!.priority, item!.fn) - } - }) - } else { - this.queue.push({ priority, fn }) - } - } - - /** - * Checks if the taskExecutor is finished. - * @private - * @returns Returns `true` if the taskExecutor is finished, otherwise returns `false`. - */ - finished(): boolean { - return this.currentPoolSize === 0 - } -} diff --git a/packages/verkle/src/verkleTree.ts b/packages/verkle/src/verkleTree.ts index 186999852b..47c768c7f8 100644 --- a/packages/verkle/src/verkleTree.ts +++ b/packages/verkle/src/verkleTree.ts @@ -6,6 +6,7 @@ import { bytesToHex, equalsBytes, intToHex, + matchingBytesLength, zeros, } from '@ethereumjs/util' import debug from 'debug' @@ -22,7 +23,6 @@ import { type VerkleTreeOpts, type VerkleTreeOptsWithDefaults, } from './types.js' -import { matchingBytesLength } from './util/index.js' import type { DB, PutBatch, VerkleCrypto } from '@ethereumjs/util' import type { Debugger } from 'debug' diff --git a/packages/verkle/test/util/bytes.spec.ts b/packages/verkle/test/util/bytes.spec.ts deleted file mode 100644 index 850a2bdb14..0000000000 --- a/packages/verkle/test/util/bytes.spec.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { assert, describe, it } from 'vitest' - -import { matchingBytesLength } from '../../src/util/bytes.js' - -describe('matchingBytesLength', () => { - it('should return 0 when both arrays are empty', () => { - const bytes1 = new Uint8Array([]) - const bytes2 = new Uint8Array([]) - assert.equal(matchingBytesLength(bytes1, bytes2), 0) - }) - - it('should return 0 when one of the arrays is empty', () => { - const bytes1 = new Uint8Array([1, 2, 3]) - const bytes2 = new Uint8Array([]) - assert.equal(matchingBytesLength(bytes1, bytes2), 0) - }) - - it('should return 0 when arrays have no matching elements', () => { - const bytes1 = new Uint8Array([1, 2, 3]) - const bytes2 = new Uint8Array([4, 5, 6]) - assert.equal(matchingBytesLength(bytes1, bytes2), 0) - }) - - it('should handle arrays with same elements but different lengths', () => { - const bytes1 = new Uint8Array([1, 2, 3]) - const bytes2 = new Uint8Array([1, 2, 3, 4]) - assert.equal(matchingBytesLength(bytes1, bytes2), 3) - }) - - it('should handle arrays with matching elements at end', () => { - const bytes1 = new Uint8Array([1, 2, 3]) - const bytes2 = new Uint8Array([0, 1, 2, 3]) - assert.equal(matchingBytesLength(bytes1, bytes2), 0) - }) - - it('should handle arrays with matching elements at start', () => { - const bytes1 = new Uint8Array([1, 2, 3]) - const bytes2 = new Uint8Array([1, 2, 3, 4, 5]) - assert.equal(matchingBytesLength(bytes1, bytes2), 3) - }) - - it('should handle arrays with large number of elements', () => { - const bytes1 = new Uint8Array(Array.from({ length: 1000000 }, (_, i) => i)) - const bytes2 = new Uint8Array(Array.from({ length: 1000000 }, (_, i) => i)) - assert.equal(matchingBytesLength(bytes1, bytes2), 1000000) - }) -}) diff --git a/packages/verkle/test/verkle.spec.ts b/packages/verkle/test/verkle.spec.ts index c023ec604f..17c11b5f69 100644 --- a/packages/verkle/test/verkle.spec.ts +++ b/packages/verkle/test/verkle.spec.ts @@ -1,4 +1,4 @@ -import { MapDB, equalsBytes, hexToBytes } from '@ethereumjs/util' +import { MapDB, equalsBytes, hexToBytes, matchingBytesLength } from '@ethereumjs/util' import { loadVerkleCrypto } from 'verkle-cryptography-wasm' import { assert, beforeAll, describe, it } from 'vitest' @@ -8,7 +8,6 @@ import { VerkleLeafNodeValue, VerkleNodeType, decodeNode, - matchingBytesLength, } from '../src/index.js' import { VerkleTree } from '../src/verkleTree.js'