Wiring ForkedChainRef to other components

- Disable majority of hive simulators - Only enable pyspec_sim for the moment - The pyspec_sim is using a smaller RPC service wired to ForkedChainRef - The RPC service will gradually grow
status-im · Jun 28, 2024 · af99a18 · af99a18
1 parent 44deff9
commit af99a18
Show file tree

Hide file tree

Showing 26 changed files with 269 additions and 119 deletions.
diff --git a/hive_integration/nodocker/build_sims.sh b/hive_integration/nodocker/build_sims.sh
@@ -19,17 +19,18 @@ ENV_SCRIPT="vendor/nimbus-build-system/scripts/env.sh"
 # nimbus_db_backend:none -> we only use memory db in simulators
 NIM_FLAGS="c -d:release"
 
-${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/engine/engine_sim
-${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/consensus/consensus_sim
-${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/graphql/graphql_sim
-${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/rpc/rpc_sim
+# ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/engine/engine_sim
+# ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/consensus/consensus_sim
+# ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/graphql/graphql_sim
+# ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/rpc/rpc_sim
 ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/pyspec/pyspec_sim
 
-${SIM_DIR}/engine/engine_sim
-${SIM_DIR}/consensus/consensus_sim
-${SIM_DIR}/graphql/graphql_sim
-${SIM_DIR}/rpc/rpc_sim
+# ${SIM_DIR}/engine/engine_sim
+# ${SIM_DIR}/consensus/consensus_sim
+# ${SIM_DIR}/graphql/graphql_sim
+# ${SIM_DIR}/rpc/rpc_sim
 ${SIM_DIR}/pyspec/pyspec_sim
 
 echo "## ${1}" > simulators.md
-cat engine.md consensus.md graphql.md rpc.md pyspec.md >> simulators.md
+# cat engine.md consensus.md graphql.md rpc.md pyspec.md >> simulators.md
+cat pyspec.md >> simulators.md
diff --git a/hive_integration/nodocker/pyspec/pyspec_sim.nim b/hive_integration/nodocker/pyspec/pyspec_sim.nim
@@ -116,8 +116,7 @@ proc validatePostState(node: JsonNode, t: TestEnv): bool =
 
 proc runTest(node: JsonNode, network: string): TestStatus =
   let conf = getChainConfig(network)
-  var t = TestEnv(conf: makeTestConfig())
-  t.setupELClient(conf, node)
+  var env = setupELClient(conf, node)
 
   let blks = node["blocks"]
   var
@@ -143,7 +142,7 @@ proc runTest(node: JsonNode, network: string): TestStatus =
 
     latestVersion = payload.payload.version
 
-    let res = t.rpcClient.newPayload(payload.payload, payload.beaconRoot)
+    let res = env.rpcClient.newPayload(payload.payload, payload.beaconRoot)
     if res.isErr:
       result = TestStatus.Failed
       echo "unable to send block ",
@@ -164,22 +163,22 @@ proc runTest(node: JsonNode, network: string): TestStatus =
         echo pStatus.validationError.get
       break
 
-  block:
+  block blockOne:
     # only update head of beacon chain if valid response occurred
     if latestValidHash != common.Hash256():
       # update with latest valid response
       let fcState = ForkchoiceStateV1(headBlockHash: BlockHash latestValidHash.data)
-      let res = t.rpcClient.forkchoiceUpdated(latestVersion, fcState)
+      let res = env.rpcClient.forkchoiceUpdated(latestVersion, fcState)
       if res.isErr:
         result = TestStatus.Failed
         echo "unable to update head of beacon chain: ", res.error
-        break
+        break blockOne
 
-    if not validatePostState(node, t):
+    if not validatePostState(node, env):
       result = TestStatus.Failed
-      break
+      break blockOne
 
-  t.stopELClient()
+  env.stopELClient()
 
 const
   skipName = [

diff --git a/hive_integration/nodocker/pyspec/test_env.nim b/hive_integration/nodocker/pyspec/test_env.nim
@@ -10,8 +10,6 @@
 
 import
   std/[json],
-  eth/p2p as eth_p2p,
-  eth/trie/trie_defs,
   stew/[byteutils],
   json_rpc/[rpcserver, rpcclient],
   ../../../nimbus/[
@@ -22,62 +20,57 @@ import
     core/chain,
     core/tx_pool,
     rpc,
-    sync/protocol,
     beacon/beacon_engine,
     common
   ],
-  ../../../tests/test_helpers,
   ../../../tools/evmstate/helpers
 
 type
   TestEnv* = ref object
-    conf*: NimbusConf
-    ctx: EthContext
-    ethNode: EthereumNode
-    com: CommonRef
-    chainRef: ChainRef
-    rpcServer: RpcHttpServer
+    chain     : ForkedChainRef
+    rpcServer : RpcHttpServer
     rpcClient*: RpcHttpClient
 
 proc genesisHeader(node: JsonNode): BlockHeader =
   let genesisRLP = hexToSeqByte(node["genesisRLP"].getStr)
   rlp.decode(genesisRLP, EthBlock).header
 
-proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
-  let memDB = newCoreDbRef DefaultDbMemory
-  t.ctx  = newEthContext()
-  t.ethNode = setupEthNode(t.conf, t.ctx, eth)
-  t.com = CommonRef.new(
-      memDB,
-      conf
-    )
-  t.chainRef = newChain(t.com, extraValidation = true)
+proc setupELClient*(conf: ChainConfig, node: JsonNode): TestEnv =
   let
-    stateDB = LedgerRef.init(memDB, emptyRlpHash)
+    memDB = newCoreDbRef DefaultDbMemory
     genesisHeader = node.genesisHeader
+    com = CommonRef.new(memDB, conf)
+    stateDB = LedgerRef.init(memDB, EMPTY_ROOT_HASH)
+    chain = newForkedChain(com, genesisHeader)
 
   setupStateDB(node["pre"], stateDB)
   stateDB.persist()
-
   doAssert stateDB.rootHash == genesisHeader.stateRoot
 
-  doAssert t.com.db.persistHeader(genesisHeader,
-    t.com.consensus == ConsensusType.POS)
-  doAssert(t.com.db.getCanonicalHead().blockHash == genesisHeader.blockHash)
+  doAssert com.db.persistHeader(genesisHeader,
+              com.consensus == ConsensusType.POS)
+  doAssert(com.db.getCanonicalHead().blockHash ==
+              genesisHeader.blockHash)
 
-  let txPool  = TxPoolRef.new(t.com)
-  t.rpcServer = newRpcHttpServer(["127.0.0.1:8545"])
+  let
+    txPool  = TxPoolRef.new(com)
+    beaconEngine = BeaconEngineRef.new(txPool, chain)
+    serverApi = newServerAPI(chain)
+    rpcServer = newRpcHttpServer(["127.0.0.1:0"])
+    rpcClient = newRpcHttpClient()
 
-  let beaconEngine = BeaconEngineRef.new(txPool, t.chainRef)
-  let oracle = Oracle.new(t.com)
-  setupEthRpc(t.ethNode, t.ctx, t.com, txPool, oracle, t.rpcServer)
-  setupEngineAPI(beaconEngine, t.rpcServer)
+  setupServerAPI(serverApi, rpcServer)
+  setupEngineAPI(beaconEngine, rpcServer)
 
-  t.rpcServer.start()
+  rpcServer.start()
+  waitFor rpcClient.connect("127.0.0.1", rpcServer.localAddress[0].port, false)
 
-  t.rpcClient = newRpcHttpClient()
-  waitFor t.rpcClient.connect("127.0.0.1", 8545.Port, false)
+  TestEnv(
+    chain: chain,
+    rpcServer: rpcServer,
+    rpcClient: rpcClient,
+  )
 
-proc stopELClient*(t: TestEnv) =
-  waitFor t.rpcClient.close()
-  waitFor t.rpcServer.closeWait()
+proc stopELClient*(env: TestEnv) =
+  waitFor env.rpcClient.close()
+  waitFor env.rpcServer.closeWait()
diff --git a/nimbus/beacon/api_handler/api_forkchoice.nim b/nimbus/beacon/api_handler/api_forkchoice.nim
@@ -161,10 +161,7 @@ proc forkchoiceUpdated*(ben: BeaconEngineRef,
       blockHash=blockHash.short,
       blockNumber=header.number
     return validFCU(Opt.none(PayloadID), blockHash)
-
-  chain.setCanonical(header).isOkOr:
-    return invalidFCU(error, com, header)
-
+
   # If the beacon client also advertised a finalized block, mark the local
   # chain final and completely in PoS mode.
   let finalizedBlockHash = ethHash update.finalizedBlockHash
@@ -212,6 +209,9 @@ proc forkchoiceUpdated*(ben: BeaconEngineRef,
       raise invalidForkChoiceState("safe head not canonical")
     db.safeHeaderHash(safeBlockHash)
 
+  chain.forkChoice(finalizedBlockHash, blockHash).isOkOr:
+    return invalidFCU(error, com, header)
+
   # If payload generation was requested, create a new block to be potentially
   # sealed by the beacon client. The payload will be requested later, and we
   # might replace it arbitrarilly many times in between.

diff --git a/nimbus/beacon/api_handler/api_newpayload.nim b/nimbus/beacon/api_handler/api_newpayload.nim
@@ -177,7 +177,7 @@ proc newPayload*(ben: BeaconEngineRef,
     if api.eth.SyncMode() != downloader.FullSync:
       return api.delayPayloadImport(header)
 
-  if not db.haveBlockAndState(header.parentHash):
+  if not ben.chain.haveBlockAndState(header.parentHash):
     ben.put(blockHash, header)
     warn "State not available, ignoring new payload",
       hash   = blockHash,
@@ -187,7 +187,7 @@ proc newPayload*(ben: BeaconEngineRef,
 
   trace "Inserting block without sethead",
     hash = blockHash, number = header.number
-  let vres = ben.chain.insertBlockWithoutSetHead(blk)
+  let vres = ben.chain.importBlock(blk)
   if vres.isErr:
     ben.setInvalidAncestor(header, blockHash)
     let blockHash = latestValidHash(db, parent, ttd)

diff --git a/nimbus/beacon/beacon_engine.nim b/nimbus/beacon/beacon_engine.nim
@@ -29,7 +29,7 @@ type
     txPool: TxPoolRef
     merge : MergeTracker
     queue : PayloadQueue
-    chain : ChainRef
+    chain : ForkedChainRef
 
     # The forkchoice update and new payload method require us to return the
     # latest valid hash in an invalid chain. To support that return, we need
@@ -98,7 +98,7 @@ proc setInvalidAncestor(ben: BeaconEngineRef,
 
 proc new*(_: type BeaconEngineRef,
           txPool: TxPoolRef,
-          chain: ChainRef): BeaconEngineRef =
+          chain: ForkedChainRef): BeaconEngineRef =
   let ben = BeaconEngineRef(
     txPool: txPool,
     merge : MergeTracker.init(txPool.com.db),
@@ -154,7 +154,7 @@ proc put*(ben: BeaconEngineRef, id: PayloadID,
 func com*(ben: BeaconEngineRef): CommonRef =
   ben.txPool.com
 
-func chain*(ben: BeaconEngineRef): ChainRef =
+func chain*(ben: BeaconEngineRef): ForkedChainRef =
   ben.chain
 
 func ttdReached*(ben: BeaconEngineRef): bool =
@@ -215,9 +215,8 @@ proc generatePayload*(ben: BeaconEngineRef,
   wrapException:
     let
       xp  = ben.txPool
-      db  = xp.com.db
       pos = xp.com.pos
-      headBlock = db.getCanonicalHead()
+      headBlock = ben.chain.latestHeader
 
     pos.prevRandao   = ethHash attrs.prevRandao
     pos.timestamp    = ethTime attrs.timestamp

diff --git a/nimbus/core/chain.nim b/nimbus/core/chain.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2018 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
 #    http://www.apache.org/licenses/LICENSE-2.0)
@@ -9,10 +9,11 @@
 # according to those terms.
 
 import
-  ./chain/[chain_desc, persist_blocks]
+  ./chain/[chain_desc, persist_blocks, forked_chain]
 
 export
   chain_desc,
-  persist_blocks
+  persist_blocks,
+  forked_chain
 
 # End