Merge branch 'util-internal-tests-and-fix' of github.com:ethereumjs/e…

…thereumjs-monorepo into util-internal-tests-and-fix

packages/blockchain/src/blockchain.ts

@@ -312,13 +312,31 @@ export class Blockchain implements BlockchainInterface {
    */
   async getIteratorHead(name = 'vm'): Promise<Block> {
     return this.runWithLock<Block>(async () => {
-      // if the head is not found return the genesis hash
-      const hash = this._heads[name] ?? this.genesisBlock.hash()
-      const block = await this.getBlock(hash)
-      return block
+      return (await this.getHead(name, false))!
     })
   }
 
+  /**
+   * This method differs from `getIteratorHead`. If the head is not found, it returns `undefined`.
+   * @param name - Optional name of the iterator head (default: 'vm')
+   * @returns
+   */
+  async getIteratorHeadSafe(name = 'vm'): Promise<Block | undefined> {
+    return this.runWithLock<Block | undefined>(async () => {
+      return this.getHead(name, true)
+    })
+  }
+
+  private async getHead(name: string, returnUndefinedIfNotSet: boolean = false) {
+    const headHash = this._heads[name]
+    if (headHash === undefined && returnUndefinedIfNotSet) {
+      return undefined
+    }
+    const hash = this._heads[name] ?? this.genesisBlock.hash()
+    const block = await this.getBlock(hash)
+    return block
+  }
+
   /**
    * Returns the latest header in the canonical chain.
    */
@@ -450,7 +468,13 @@ export class Blockchain implements BlockchainInterface {
 
         // we cannot overwrite the Genesis block after initializing the Blockchain
         if (isGenesis) {
-          throw new Error('Cannot put a genesis block: create a new Blockchain')
+          if (equalsBytes(this.genesisBlock.hash(), block.hash())) {
+            // Try to re-put the exisiting genesis block, accept this
+            return
+          }
+          throw new Error(
+            'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain'
+          )
         }
 
         const { header } = block

packages/blockchain/test/index.spec.ts

@@ -844,7 +844,7 @@ describe('initialization tests', () => {
     } catch (e: any) {
       assert.equal(
         e.message,
-        'Cannot put a genesis block: create a new Blockchain',
+        'Cannot put a different genesis block than current blockchain genesis: create a new Blockchain',
         'putting a genesis block did throw (otherGenesisBlock not found in chain)'
       )
     }

packages/client/src/blockchain/chain.ts

@@ -294,17 +294,17 @@ export class Chain {
       height: BIGINT_0,
     }
 
+    blocks.latest = await this.getCanonicalHeadBlock()
+    blocks.finalized = (await this.getCanonicalFinalizedBlock()) ?? null
+    blocks.safe = (await this.getCanonicalSafeBlock()) ?? null
+    blocks.vm = await this.getCanonicalVmHead()
+
     headers.latest = await this.getCanonicalHeadHeader()
     // finalized and safe are always blocks since they have to have valid execution
     // before they can be saved in chain
-    headers.finalized = (await this.getCanonicalFinalizedBlock()).header
-    headers.safe = (await this.getCanonicalSafeBlock()).header
-    headers.vm = (await this.getCanonicalVmHead()).header
-
-    blocks.latest = await this.getCanonicalHeadBlock()
-    blocks.finalized = await this.getCanonicalFinalizedBlock()
-    blocks.safe = await this.getCanonicalSafeBlock()
-    blocks.vm = await this.getCanonicalVmHead()
+    headers.finalized = blocks.finalized?.header ?? null
+    headers.safe = blocks.safe?.header ?? null
+    headers.vm = blocks.vm.header
 
     headers.height = headers.latest.number
     blocks.height = blocks.latest.header.number
@@ -513,17 +513,17 @@ export class Chain {
   /**
    * Gets the latest block in the canonical chain
    */
-  async getCanonicalSafeBlock(): Promise<Block> {
+  async getCanonicalSafeBlock(): Promise<Block | undefined> {
     if (!this.opened) throw new Error('Chain closed')
-    return this.blockchain.getIteratorHead('safe')
+    return this.blockchain.getIteratorHeadSafe('safe')
   }
 
   /**
    * Gets the latest block in the canonical chain
    */
-  async getCanonicalFinalizedBlock(): Promise<Block> {
+  async getCanonicalFinalizedBlock(): Promise<Block | undefined> {
     if (!this.opened) throw new Error('Chain closed')
-    return this.blockchain.getIteratorHead('finalized')
+    return this.blockchain.getIteratorHeadSafe('finalized')
   }
 
   /**

packages/client/src/execution/vmexecution.ts

@@ -37,10 +37,14 @@ export class VMExecution extends Execution {
   private MAX_TOLERATED_BLOCK_TIME = 12
 
   /**
-   * Display state cache stats every num blocks
+   * Interval for client execution stats output (in ms)
+   * for debug log level
+   *
    */
-  private STATS_NUM_BLOCKS = 5000
-  private statsCount = 0
+  private STATS_INTERVAL = 1000 * 90 // 90 seconds
+
+  private _statsInterval: NodeJS.Timeout | undefined /* global NodeJS */
+  private _statsVm: VM | undefined
 
   /**
    * Create new VM execution module
@@ -229,12 +233,12 @@ export class VMExecution extends Execution {
   ): Promise<void> {
     return this.runWithLock<void>(async () => {
       const vmHeadBlock = blocks[blocks.length - 1]
-      const chainPointers: [string, Block | null][] = [
+      const chainPointers: [string, Block][] = [
         ['vmHeadBlock', vmHeadBlock],
         // if safeBlock is not provided, the current safeBlock of chain should be used
         // which is genesisBlock if it has never been set for e.g.
-        ['safeBlock', safeBlock ?? this.chain.blocks.safe],
-        ['finalizedBlock', finalizedBlock ?? this.chain.blocks.finalized],
+        ['safeBlock', safeBlock ?? this.chain.blocks.safe ?? this.chain.genesis],
+        ['finalizedBlock', finalizedBlock ?? this.chain.blocks.finalized ?? this.chain.genesis],
       ]
 
       let isSortedDesc = true
@@ -258,7 +262,7 @@ export class VMExecution extends Execution {
 
       if (isSortedDesc === false) {
         throw Error(
-          `headBlock=${vmHeadBlock?.header.number} should be >= safeBlock=${safeBlock?.header.number} should be >= finalizedBlock=${finalizedBlock?.header.number}`
+          `headBlock=${chainPointers[0][1].header.number} should be >= safeBlock=${chainPointers[1][1]?.header.number} should be >= finalizedBlock=${chainPointers[2][1]?.header.number}`
         )
       }
       // skip emitting the chain update event as we will manually do it
@@ -411,8 +415,8 @@ export class VMExecution extends Execution {
                     throw Error('Execution stopped')
                   }
 
+                  this._statsVm = this.vm
                   const beforeTS = Date.now()
-                  this.stats(this.vm)
                   const result = await this.vm.runBlock({
                     block,
                     root: parentState,
@@ -593,6 +597,12 @@ export class VMExecution extends Execution {
    * Start execution
    */
   async start(): Promise<boolean> {
+    this._statsInterval = setInterval(
+      // eslint-disable-next-line @typescript-eslint/await-thenable
+      await this.stats.bind(this),
+      this.STATS_INTERVAL
+    )
+
     const { blockchain } = this.vm
     if (this.running || !this.started) {
       return false
@@ -627,6 +637,7 @@ export class VMExecution extends Execution {
    * Stop VM execution. Returns a promise that resolves once its stopped.
    */
   async stop(): Promise<boolean> {
+    clearInterval(this._statsInterval)
     // Stop with the lock to be concurrency safe and flip started flag so that
     // vmPromise can resolve early
     await this.runWithLock<void>(async () => {
@@ -676,10 +687,11 @@ export class VMExecution extends Execution {
       })
 
       if (txHashes.length === 0) {
+        this._statsVm = vm
+
         // we are skipping header validation because the block has been picked from the
         // blockchain and header should have already been validated while putBlock
         const beforeTS = Date.now()
-        this.stats(vm)
         const res = await vm.runBlock({
           block,
           root,
@@ -722,10 +734,9 @@ export class VMExecution extends Execution {
     }
   }
 
-  stats(vm: VM) {
-    this.statsCount += 1
-    if (this.statsCount === this.STATS_NUM_BLOCKS) {
-      const sm = vm.stateManager as any
+  stats() {
+    if (this._statsVm !== undefined) {
+      const sm = this._statsVm.stateManager as any
       const disactivatedStats = { size: 0, reads: 0, hits: 0, writes: 0 }
       let stats
       // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
@@ -748,7 +759,6 @@ export class VMExecution extends Execution {
         `Trie cache stats size=${tStats.size} reads=${tStats.cache.reads} hits=${tStats.cache.hits} ` +
           `writes=${tStats.cache.writes} readsDB=${tStats.db.reads} hitsDB=${tStats.db.hits} writesDB=${tStats.db.writes}`
       )
-      this.statsCount = 0
     }
   }
 }

packages/client/src/miner/pendingBlock.ts

@@ -1,6 +1,7 @@
 import { Hardfork } from '@ethereumjs/common'
 import { BlobEIP4844Transaction } from '@ethereumjs/tx'
 import {
+  Address,
   BIGINT_1,
   BIGINT_2,
   TypeOutput,
@@ -130,6 +131,22 @@ export class PendingBlock {
       toType(parentBeaconBlockRoot!, TypeOutput.Uint8Array) ?? zeros(32)
     const coinbaseBuf = toType(coinbase ?? zeros(20), TypeOutput.Uint8Array)
 
+    let withdrawalsBuf = zeros(0)
+
+    if (withdrawals !== undefined) {
+      const withdrawalsBufTemp: Uint8Array[] = []
+      for (const withdrawal of withdrawals) {
+        const indexBuf = bigIntToUnpaddedBytes(toType(withdrawal.index ?? 0, TypeOutput.BigInt))
+        const validatorIndex = bigIntToUnpaddedBytes(
+          toType(withdrawal.validatorIndex ?? 0, TypeOutput.BigInt)
+        )
+        const address = toType(withdrawal.address ?? Address.zero(), TypeOutput.Uint8Array)
+        const amount = bigIntToUnpaddedBytes(toType(withdrawal.amount ?? 0, TypeOutput.BigInt))
+        withdrawalsBufTemp.push(concatBytes(indexBuf, validatorIndex, address, amount))
+      }
+      withdrawalsBuf = concatBytes(...withdrawalsBufTemp)
+    }
+
     const payloadIdBytes = toBytes(
       keccak256(
         concatBytes(
@@ -138,7 +155,8 @@ export class PendingBlock {
           timestampBuf,
           gasLimitBuf,
           parentBeaconBlockRootBuf,
-          coinbaseBuf
+          coinbaseBuf,
+          withdrawalsBuf
         )
       ).subarray(0, 8)
     )

packages/client/src/rpc/error-code.ts

@@ -19,3 +19,6 @@ export const validEngineCodes = [
   UNSUPPORTED_FORK,
   UNKNOWN_PAYLOAD,
 ]
+
+// Errors for the ETH protocol
+export const INVALID_BLOCK = -39001

packages/client/src/rpc/helpers.ts

@@ -1,6 +1,6 @@
 import { BIGINT_0, bigIntToHex, bytesToHex, intToHex } from '@ethereumjs/util'
 
-import { INTERNAL_ERROR, INVALID_PARAMS } from './error-code'
+import { INTERNAL_ERROR, INVALID_BLOCK, INVALID_PARAMS } from './error-code'
 
 import type { Chain } from '../blockchain'
 import type { Block } from '@ethereumjs/block'
@@ -73,6 +73,7 @@ export const getBlockByOption = async (blockOpt: string, chain: Chain) => {
   }
 
   let block: Block
+  let tempBlock: Block | undefined // Used in `safe` and `finalized` blocks
   const latest = chain.blocks.latest ?? (await chain.getCanonicalHeadBlock())
 
   switch (blockOpt) {
@@ -83,10 +84,24 @@ export const getBlockByOption = async (blockOpt: string, chain: Chain) => {
       block = latest
       break
     case 'safe':
-      block = chain.blocks.safe ?? (await chain.getCanonicalSafeBlock())
+      tempBlock = chain.blocks.safe ?? (await chain.getCanonicalSafeBlock())
+      if (tempBlock === null || tempBlock === undefined) {
+        throw {
+          message: 'Unknown block',
+          code: INVALID_BLOCK,
+        }
+      }
+      block = tempBlock
       break
     case 'finalized':
-      block = chain.blocks.finalized ?? (await chain.getCanonicalFinalizedBlock())
+      tempBlock = chain.blocks.finalized ?? (await chain.getCanonicalFinalizedBlock())
+      if (tempBlock === null || tempBlock === undefined) {
+        throw {
+          message: 'Unknown block',
+          code: INVALID_BLOCK,
+        }
+      }
+      block = tempBlock
       break
     default: {
       const blockNumber = BigInt(blockOpt)

packages/client/src/rpc/modules/engine.ts

@@ -276,6 +276,11 @@ const recursivelyFindParents = async (
     )
     parentBlocks.push(block)
 
+    if (block.isGenesis()) {
+      // In case we hit the genesis block we should stop finding additional parents
+      break
+    }
+
     // throw error if lookups have exceeded maxDepth
     if (parentBlocks.length > maxDepth) {
       throw Error(`recursivelyFindParents lookups deeper than maxDepth=${maxDepth}`)
@@ -998,6 +1003,7 @@ export class Engine {
     const newPayloadRes = await this.newPayload(params)
     if (newPayloadRes.status === Status.INVALID_BLOCK_HASH) {
       newPayloadRes.status = Status.INVALID
+      newPayloadRes.latestValidHash = null
     }
     return newPayloadRes
   }
@@ -1015,6 +1021,7 @@ export class Engine {
     const newPayloadRes = await this.newPayload(params)
     if (newPayloadRes.status === Status.INVALID_BLOCK_HASH) {
       newPayloadRes.status = Status.INVALID
+      newPayloadRes.latestValidHash = null
     }
     return newPayloadRes
   }
@@ -1225,7 +1232,13 @@ export class Engine {
         }
       }
       this.service.txPool.removeNewBlockTxs(blocks)
-    } else {
+
+      const isPrevSynced = this.chain.config.synchronized
+      this.config.updateSynchronizedState(headBlock.header)
+      if (!isPrevSynced && this.chain.config.synchronized) {
+        this.service.txPool.checkRunState()
+      }
+    } else if (!headBlock.isGenesis()) {
       // even if the vmHead is same still validations need to be done regarding the correctness
       // of the sequence and canonical-ity
       try {

packages/client/src/service/service.ts

@@ -57,7 +57,7 @@ export class Service {
    *
    * (for info there will be somewhat reduced output)
    */
-  private STATS_INTERVAL = 20000
+  private STATS_INTERVAL = 1000 * 20 // 20 seconds
 
   /**
    * Shutdown the client when memory threshold is reached (in percent)

packages/client/test/rpc/engine/forkchoiceUpdatedV1.spec.ts

@@ -280,7 +280,7 @@ describe(method, () => {
   })
 
   it('latest block after reorg', async () => {
-    const { server } = await setupChain(genesisJSON, 'post-merge', { engine: true })
+    const { server, blockchain } = await setupChain(genesisJSON, 'post-merge', { engine: true })
     let req = params(method, [validForkChoiceState])
     let expectRes = (res: any) => {
       assert.equal(res.body.result.payloadStatus.status, 'VALID')
@@ -294,6 +294,7 @@ describe(method, () => {
         ...validForkChoiceState,
         headBlockHash: blocks[2].blockHash,
         safeBlockHash: blocks[0].blockHash,
+        finalizedBlockHash: bytesToHex(blockchain.genesisBlock.hash()),
       },
     ])
     expectRes = (res: any) => {

packages/common/src/common.ts

@@ -384,7 +384,7 @@ export class Common {
       if (mergeIndex >= 0 && td !== undefined && td !== null) {
         if (hfIndex >= mergeIndex && BigInt(hfs[mergeIndex].ttd!) > td) {
           throw Error('Maximum HF determined by total difficulty is lower than the block number HF')
-        } else if (hfIndex < mergeIndex && BigInt(hfs[mergeIndex].ttd!) <= td) {
+        } else if (hfIndex < mergeIndex && BigInt(hfs[mergeIndex].ttd!) < td) {
           throw Error('HF determined by block number is lower than the minimum total difficulty HF')
         }
       }

packages/common/test/mergePOS.spec.ts

@@ -128,7 +128,7 @@ describe('[Common]: Merge/POS specific logic', () => {
       assert.ok(e.message.includes(eMsg), msg)
     }
     try {
-      c.setHardforkBy({ blockNumber: 14n, td: 5000n })
+      c.setHardforkBy({ blockNumber: 14n, td: 5001n })
       assert.fail(`should have thrown td > ttd validation error`)
     } catch (e: any) {
       msg = 'block number < last HF block number set, TD set and higher (should throw)'