Core db disable legacy api n remove distinct tries (#2299)

* CoreDb: Remove crufty second/off-site KVT why: Was used to allow late `Clique` to store directly to disk * CoreDb: Remove prune flag related functionality why: Is completely legacy stuff * CoreDb: Remove dependence on legacy API (tests unsupported yet) why: Does not fully support Aristo * Re-factoring `state_db` using new API details: Only minimum changes needed to compile `nimbus` * Update tests and aux modules * Turn off legacy API and remove `distinct_tries` comment: The legacy API has now cruft status, will be removed soon * Fix copyright years * Update rpc for verified proxy --------- Co-authored-by: Jacek Sieka <[email protected]>
status-im · Jun 5, 2024 · e9eae4d · e9eae4d
1 parent effed1a
commit e9eae4d
Show file tree

Hide file tree

Showing 43 changed files with 638 additions and 838 deletions.
diff --git a/hive_integration/nodocker/engine/node.nim b/hive_integration/nodocker/engine/node.nim
@@ -44,7 +44,7 @@ proc processBlock(
   ## implementations (but can be savely removed, as well.)
   ## variant of `processBlock()` where the `header` argument is explicitely set.
 
-  var dbTx = vmState.com.db.beginTransaction()
+  var dbTx = vmState.com.db.newTransaction()
   defer: dbTx.dispose()
 
   if vmState.com.daoForkSupport and
@@ -76,12 +76,7 @@ proc processBlock(
     let clearEmptyAccount = vmState.determineFork >= FkSpurious
     db.persist(clearEmptyAccount)
 
-  # `applyDeletes = false`
-  # If the trie pruning activated, each of the block will have its own state
-  # trie keep intact, rather than destroyed by trie pruning. But the current
-  # block will still get a pruned trie. If trie pruning deactivated,
-  # `applyDeletes` have no effects.
-  dbTx.commit(applyDeletes = false)
+  dbTx.commit()
 
   ValidationResult.OK
 
@@ -103,7 +98,7 @@ proc getVmState(c: ChainRef, header: BlockHeader):
 proc setBlock*(c: ChainRef; header: BlockHeader;
                   body: BlockBody): ValidationResult
                           {.inline, raises: [CatchableError].} =
-  let dbTx = c.db.beginTransaction()
+  let dbTx = c.db.newTransaction()
   defer: dbTx.dispose()
 
   c.com.hardForkTransition(header)

diff --git a/nimbus/beacon/merge_tracker.nim b/nimbus/beacon/merge_tracker.nim
@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022-2023 Status Research & Development GmbH
+# Copyright (c) 2022-2024 Status Research & Development GmbH
 # Licensed under either of
 #  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
 #  * MIT license ([LICENSE-MIT](LICENSE-MIT))
@@ -12,6 +12,7 @@
 import
   chronicles,
   eth/rlp,
+  results,
   ../db/[core_db, storage_types]
 
 type
@@ -37,10 +38,12 @@ type
 # ------------------------------------------------------------------------------
 
 proc writeStatus(db: CoreDbRef, status: TransitionStatus) =
-  db.kvt.put(transitionStatusKey().toOpenArray(), rlp.encode(status))
+  db.newKvt.put(transitionStatusKey().toOpenArray(), rlp.encode(status)).isOkOr:
+    raiseAssert "writeStatus(): put() failed " & $$error
 
 proc readStatus(db: CoreDbRef): TransitionStatus =
-  var bytes = db.kvt.get(transitionStatusKey().toOpenArray())
+  var bytes = db.newKvt.get(transitionStatusKey().toOpenArray()).valueOr:
+    EmptyBlob
   if bytes.len > 0:
     try:
       result = rlp.decode(bytes, typeof result)

diff --git a/nimbus/common/common.nim b/nimbus/common/common.nim
@@ -367,7 +367,9 @@ proc consensus*(com: CommonRef, header: BlockHeader): ConsensusType
 
 proc initializeEmptyDb*(com: CommonRef)
     {.gcsafe, raises: [CatchableError].} =
-  let kvt = com.db.kvt()
+  let kvt = com.db.newKvt()
+  proc contains(kvt: CoreDxKvtRef; key: openArray[byte]): bool =
+    kvt.hasKey(key).expect "valid bool"
   if canonicalHeadHashKey().toOpenArray notin kvt:
     info "Writing genesis to DB"
     doAssert(com.genesisHeader.blockNumber.isZero,

diff --git a/nimbus/core/chain/persist_blocks.nim b/nimbus/core/chain/persist_blocks.nim
@@ -23,7 +23,7 @@ import
 
 when not defined(release):
   import
-    ../../tracer,
+    #../../tracer,
     ../../utils/utils
 
 export results
@@ -80,7 +80,7 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
                        bodies: openArray[BlockBody],
                        flags: PersistBlockFlags = {}): Result[PersistStats, string]
                          {.raises: [CatchableError] .} =
-  let dbTx = c.db.beginTransaction()
+  let dbTx = c.db.newTransaction()
   defer: dbTx.dispose()
 
   c.com.hardForkTransition(headers[0])
@@ -95,10 +95,10 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
   for i in 0 ..< headers.len:
     let (header, body) = (headers[i], bodies[i])
 
-    # This transaction keeps the current state open for inspection
-    # if an error occurs (as needed for `Aristo`.).
-    let lapTx = c.db.beginTransaction()
-    defer: lapTx.dispose()
+    # # This transaction keeps the current state open for inspection
+    # # if an error occurs (as needed for `Aristo`.).
+    # let lapTx = c.db.newTransaction()
+    # defer: lapTx.dispose()
 
     c.com.hardForkTransition(header)
 
@@ -125,17 +125,17 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
                          else:
                            ValidationResult.OK
 
-    when defined(nimbusDumpDebuggingMetaData):
-      if validationResult == ValidationResult.Error and
-         body.transactions.calcTxRoot == header.txRoot:
-        vmState.dumpDebuggingMetaData(header, body)
-        warn "Validation error. Debugging metadata dumped."
+    # when defined(nimbusDumpDebuggingMetaData):
+    #   if validationResult == ValidationResult.Error and
+    #      body.transactions.calcTxRoot == header.txRoot:
+    #     vmState.dumpDebuggingMetaData(header, body)
+    #     warn "Validation error. Debugging metadata dumped."
 
     if validationResult != ValidationResult.OK:
       return err("Failed to validate block")
 
     if c.generateWitness:
-      let dbTx = c.db.beginTransaction()
+      let dbTx = c.db.newTransaction()
       defer: dbTx.dispose()
 
       let
@@ -167,7 +167,7 @@ proc persistBlocksImpl(c: ChainRef; headers: openArray[BlockHeader];
     c.com.syncCurrent = header.blockNumber
 
     # Done with this block
-    lapTx.commit()
+    # lapTx.commit()
 
     txs += body.transactions.len
 

diff --git a/nimbus/core/executor/process_block.nim b/nimbus/core/executor/process_block.nim
@@ -164,7 +164,7 @@ proc processBlock*(
   ## implementations (but can be savely removed, as well.)
   ## variant of `processBlock()` where the `header` argument is explicitely set.
 
-  var dbTx = vmState.com.db.beginTransaction()
+  var dbTx = vmState.com.db.newTransaction()
   defer: dbTx.dispose()
 
   if not vmState.procBlkPreamble(header, body):
@@ -177,12 +177,7 @@ proc processBlock*(
   if not vmState.procBlkEpilogue(header, body):
     return ValidationResult.Error
 
-  # `applyDeletes = false`
-  # If the trie pruning activated, each of the block will have its own state
-  # trie keep intact, rather than destroyed by trie pruning. But the current
-  # block will still get a pruned trie. If trie pruning deactivated,
-  # `applyDeletes` have no effects.
-  dbTx.commit(applyDeletes = false)
+  dbTx.commit()
 
   ValidationResult.OK
 

diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim
@@ -36,7 +36,7 @@ type
 
   TxPackerStateRef = ref object
     xp: TxPoolRef
-    tr: CoreDbMptRef
+    tr: CoreDxMptRef
     cleanState: bool
     balance: UInt256
     blobGasUsed: uint64
@@ -139,7 +139,8 @@ proc runTxCommit(pst: TxPackerStateRef; item: TxItemRef; gasBurned: GasInt)
     pst.blobGasUsed += item.tx.getTotalBlobGas
 
   # Update txRoot
-  pst.tr.put(rlp.encode(inx), rlp.encode(item.tx))
+  pst.tr.merge(rlp.encode(inx), rlp.encode(item.tx)).isOkOr:
+    raiseAssert "runTxCommit(): merge failed, " & $$error
 
   # Add the item to the `packed` bucket. This implicitely increases the
   # receipts index `inx` at the next visit of this function.
@@ -173,7 +174,7 @@ proc vmExecInit(xp: TxPoolRef): Result[TxPackerStateRef, string]
 
   let packer = TxPackerStateRef( # return value
     xp: xp,
-    tr: AristoDbMemory.newCoreDbRef().mptPrune,
+    tr: AristoDbMemory.newCoreDbRef().ctx.getMpt CtGeneric,
     balance: xp.chain.vmState.readOnlyStateDB.getBalance(xp.chain.feeRecipient),
     numBlobPerBlock: 0,
   )
@@ -257,7 +258,8 @@ proc vmExecCommit(pst: TxPackerStateRef)
   vmState.receipts.setLen(nItems)
 
   xp.chain.receipts = vmState.receipts
-  xp.chain.txRoot = pst.tr.rootHash
+  xp.chain.txRoot = pst.tr.getColumn.state.valueOr:
+    raiseAssert "vmExecCommit(): state() failed " & $$error
   xp.chain.stateRoot = vmState.stateDB.rootHash
 
   if vmState.com.forkGTE(Cancun):
@@ -281,7 +283,7 @@ proc packerVmExec*(xp: TxPoolRef): Result[void, string] {.gcsafe,raises: [Catcha
   ## Rebuild `packed` bucket by selection items from the `staged` bucket
   ## after executing them in the VM.
   let db = xp.chain.com.db
-  let dbTx = db.beginTransaction
+  let dbTx = db.newTransaction
   defer: dbTx.dispose()
 
   var pst = xp.vmExecInit.valueOr:

diff --git a/nimbus/db/core_db/backend/aristo_db.nim b/nimbus/db/core_db/backend/aristo_db.nim
@@ -71,29 +71,31 @@ proc txMethods(
     levelFn: proc(): int =
       aTx.level,
 
-    commitFn: proc(ignore: bool): CoreDbRc[void] =
+    commitFn: proc() =
       const info = "commitFn()"
-      ? adbApi.commit(aTx).toVoidRc(adbBase, info)
-      ? kdbApi.commit(kTx).toVoidRc(kdbBase, info)
-      ok(),
+      adbApi.commit(aTx).isOkOr:
+        raiseAssert info & ": " & $error
+      kdbApi.commit(kTx).isOkOr:
+        raiseAssert info & ": " & $error
+      discard,
 
-    rollbackFn: proc(): CoreDbRc[void] =
+    rollbackFn: proc() =
       const info = "rollbackFn()"
-      ? adbApi.rollback(aTx).toVoidRc(adbBase, info)
-      ? kdbApi.rollback(kTx).toVoidRc(kdbBase, info)
-      ok(),
+      adbApi.rollback(aTx).isOkOr:
+        raiseAssert info & ": " & $error
+      kdbApi.rollback(kTx).isOkOr:
+        raiseAssert info & ": " & $error
+      discard,
 
-    disposeFn: proc(): CoreDbRc[void] =
+    disposeFn: proc() =
       const info =  "disposeFn()"
-      if adbApi.isTop(aTx): ? adbApi.rollback(aTx).toVoidRc(adbBase, info)
-      if kdbApi.isTop(kTx): ? kdbApi.rollback(kTx).toVoidRc(kdbBase, info)
-      ok(),
-
-    safeDisposeFn: proc(): CoreDbRc[void] =
-      const info =  "safeDisposeFn()"
-      if adbApi.isTop(aTx): ? adbApi.rollback(aTx).toVoidRc(adbBase, info)
-      if kdbApi.isTop(kTx): ? kdbApi.rollback(kTx).toVoidRc(kdbBase, info)
-      ok())
+      if adbApi.isTop(aTx):
+        adbApi.rollback(aTx).isOkOr:
+          raiseAssert info & ": " & $error
+      if kdbApi.isTop(kTx):
+        kdbApi.rollback(kTx).isOkOr:
+          raiseAssert info & ": " & $error
+      discard)
 
 proc cptMethods(
     tracer: AristoTracerRef;
@@ -158,8 +160,8 @@ proc baseMethods(db: AristoCoreDbRef): CoreDbBaseFns =
     errorPrintFn: proc(e: CoreDbErrorRef): string =
       e.errorPrint(),
 
-    newKvtFn: proc(offSite: bool): CoreDbRc[CoreDxKvtRef] =
-      kBase.newKvtHandler(offSite, "newKvtFn()"),
+    newKvtFn: proc(): CoreDbRc[CoreDxKvtRef] =
+      kBase.newKvtHandler("newKvtFn()"),
 
     newCtxFn: proc(): CoreDbCtxRef =
       aBase.ctx,
@@ -170,11 +172,13 @@ proc baseMethods(db: AristoCoreDbRef): CoreDbBaseFns =
     swapCtxFn: proc(ctx: CoreDbCtxRef): CoreDbCtxRef =
       aBase.swapCtx(ctx),
 
-    beginFn: proc(): CoreDbRc[CoreDxTxRef] =
+    beginFn: proc(): CoreDxTxRef =
       const info = "beginFn()"
-      let dsc = CoreDxTxRef(
-        methods: db.txMethods(? aBase.txBegin info, ? kBase.txBegin info))
-      ok(db.bless dsc),
+      let
+        aTx = aBase.txBegin info
+        kTx = kBase.txBegin info
+        dsc = CoreDxTxRef(methods: db.txMethods(aTx, kTx))
+      db.bless(dsc),
 
     newCaptureFn: proc(flags: set[CoreDbCaptFlags]): CoreDbRc[CoreDxCaptRef] =
       ok(db.bless flags.tracerSetup()),

diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim
@@ -299,10 +299,7 @@ proc mptMethods(cMpt: AristoCoreDxMptRef): CoreDbMptFns =
       mptHasPath(k),
 
     getColFn: proc(): CoreDbColRef =
-      mptColFn(),
-
-    isPruningFn: proc(): bool =
-      true)
+      mptColFn())
 
 # ------------------------------------------------------------------------------
 # Private account call back functions
@@ -419,10 +416,7 @@ proc accMethods(cAcc: AristoCoreDxAccRef): CoreDbAccFns =
       accHasPath(address),
 
     getColFn: proc(): CoreDbColRef =
-      getColFn(),
-
-    isPruningFn: proc(): bool =
-      true)
+      getColFn())
 
 # ------------------------------------------------------------------------------
 # Private context call back functions
@@ -543,10 +537,10 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns =
           ): CoreDbRc[CoreDbColRef] =
       ctxNewCol(col, colState, address),
 
-    getMptFn: proc(col: CoreDbColRef; prune: bool): CoreDbRc[CoreDxMptRef] =
+    getMptFn: proc(col: CoreDbColRef): CoreDbRc[CoreDxMptRef] =
       ctxGetMpt(col),
 
-    getAccFn: proc(col: CoreDbColRef; prune: bool): CoreDbRc[CoreDxAccRef] =
+    getAccFn: proc(col: CoreDbColRef): CoreDbRc[CoreDxAccRef] =
       ctxGetAcc(col),
 
     forgetFn: proc() =
@@ -606,8 +600,11 @@ func txTop*(
 proc txBegin*(
     base: AristoBaseRef;
     info: static[string];
-      ): CoreDbRc[AristoTxRef] =
-  base.api.txBegin(base.ctx.mpt).toRc(base, info)
+      ): AristoTxRef =
+  let rc = base.api.txBegin(base.ctx.mpt)
+  if rc.isErr:
+    raiseAssert info & ": " & $rc.error
+  rc.value
 
 proc getLevel*(base: AristoBaseRef): int =
   base.api.level(base.ctx.mpt)