From 1064b43aa9ac6b8e50d267ee9df656a77aafe206 Mon Sep 17 00:00:00 2001 From: Jordan Hrycaj Date: Thu, 20 Jun 2024 19:07:17 +0100 Subject: [PATCH] no really tested yet ... --- nimbus/core/tx_pool/tx_tasks/tx_packer.nim | 4 +- nimbus/db/aristo/aristo_api.nim | 17 ++ nimbus/db/aristo/aristo_fetch.nim | 41 +-- .../backend/aristo_db/handlers_aristo.nim | 251 ++++-------------- nimbus/db/core_db/base.nim | 155 +++-------- nimbus/db/core_db/base/api_tracking.nim | 18 +- nimbus/db/core_db/base/base_desc.nim | 20 +- nimbus/db/core_db/base/validate.nim | 4 +- nimbus/db/core_db/core_apps.nim | 114 ++++---- tests/test_ledger.nim | 8 +- 10 files changed, 195 insertions(+), 437 deletions(-) diff --git a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim index 43d5e56981..73f49c52dc 100644 --- a/nimbus/core/tx_pool/tx_tasks/tx_packer.nim +++ b/nimbus/core/tx_pool/tx_tasks/tx_packer.nim @@ -171,7 +171,7 @@ proc vmExecInit(xp: TxPoolRef): Result[TxPackerStateRef, string] let packer = TxPackerStateRef( # return value xp: xp, - tr: AristoDbMemory.newCoreDbRef().ctx.getMpt CtGeneric, + tr: AristoDbMemory.newCoreDbRef().ctx.getColumn(CtGeneric, clearData=true), balance: xp.chain.vmState.readOnlyStateDB.getBalance(xp.chain.feeRecipient), numBlobPerBlock: 0, ) @@ -255,7 +255,7 @@ proc vmExecCommit(pst: TxPackerStateRef) vmState.receipts.setLen(nItems) xp.chain.receipts = vmState.receipts - xp.chain.txRoot = pst.tr.getColumn.state.valueOr: + xp.chain.txRoot = pst.tr.state.valueOr: raiseAssert "vmExecCommit(): state() failed " & $$error xp.chain.stateRoot = vmState.stateDB.rootHash diff --git a/nimbus/db/aristo/aristo_api.nim b/nimbus/db/aristo/aristo_api.nim index 6070fd1923..1f7e042660 100644 --- a/nimbus/db/aristo/aristo_api.nim +++ b/nimbus/db/aristo/aristo_api.nim @@ -130,6 +130,13 @@ type ## For a generic sub-tree starting at `root`, fetch the data record ## indexed by `path`. + AristoApiFetchGenericStateFn* = + proc(db: AristoDbRef; + root: VertexID; + ): Result[Hash256,AristoError] + {.noRaise.} + ## Fetch the Merkle hash of the argument `root`. + AristoApiFetchStorageDataFn* = proc(db: AristoDbRef; path: openArray[byte]; @@ -434,6 +441,7 @@ type fetchAccountPayload*: AristoApiFetchAccountPayloadFn fetchAccountState*: AristoApiFetchAccountStateFn fetchGenericData*: AristoApiFetchGenericDataFn + fetchGenericState*: AristoApiFetchGenericStateFn fetchStorageData*: AristoApiFetchStorageDataFn fetchStorageState*: AristoApiFetchStorageStateFn @@ -482,6 +490,7 @@ type AristoApiProfFetchAccountPayloadFn = "fetchAccountPayload" AristoApiProfFetchAccountStateFn = "fetchAccountState" AristoApiProfFetchGenericDataFn = "fetchGenericData" + AristoApiProfFetchGenericStateFn = "fetchGenericState" AristoApiProfFetchStorageDataFn = "fetchStorageData" AristoApiProfFetchStorageStateFn = "fetchStorageState" @@ -547,6 +556,7 @@ when AutoValidateApiHooks: doAssert not api.fetchAccountPayload.isNil doAssert not api.fetchAccountState.isNil doAssert not api.fetchGenericData.isNil + doAssert not api.fetchGenericState.isNil doAssert not api.fetchStorageData.isNil doAssert not api.fetchStorageState.isNil @@ -616,6 +626,7 @@ func init*(api: var AristoApiObj) = api.fetchAccountPayload = fetchAccountPayload api.fetchAccountState = fetchAccountState api.fetchGenericData = fetchGenericData + api.fetchGenericState = fetchGenericState api.fetchStorageData = fetchStorageData api.fetchStorageState = fetchStorageState @@ -667,6 +678,7 @@ func dup*(api: AristoApiRef): AristoApiRef = fetchAccountPayload: api.fetchAccountPayload, fetchAccountState: api.fetchAccountState, fetchGenericData: api.fetchGenericData, + fetchGenericState: api.fetchGenericState, fetchStorageData: api.fetchStorageData, fetchStorageState: api.fetchStorageState, @@ -777,6 +789,11 @@ func init*( AristoApiProfFetchGenericDataFn.profileRunner: result = api.fetchGenericData(a, b, c) + profApi.fetchGenericState = + proc(a: AristoDbRef; b: VertexID;): auto = + AristoApiProfFetchGenericStateFn.profileRunner: + result = api.fetchGenericState(a, b) + profApi.fetchStorageData = proc(a: AristoDbRef; b: openArray[byte]; c: PathID;): auto = AristoApiProfFetchStorageDataFn.profileRunner: diff --git a/nimbus/db/aristo/aristo_fetch.nim b/nimbus/db/aristo/aristo_fetch.nim index 60a208de84..c41dbd198d 100644 --- a/nimbus/db/aristo/aristo_fetch.nim +++ b/nimbus/db/aristo/aristo_fetch.nim @@ -66,6 +66,17 @@ proc retrieveStoID( ok stoID +proc retrieveMerkleHash( + db: AristoDbRef; + root: VertexID; + ): Result[Hash256,AristoError] = + let key = db.getKeyRc(root).valueOr: + if error == GetKeyNotFound: + return ok(EMPTY_ROOT_HASH) # empty sub-tree + return err(error) + ok key.to(Hash256) + + proc hasPayload( db: AristoDbRef; root: VertexID; @@ -107,11 +118,7 @@ proc fetchAccountState*( db: AristoDbRef; ): Result[Hash256,AristoError] = ## Fetch the Merkle hash of the account root. - let key = db.getKeyRc(VertexID 1).valueOr: - if error == GetKeyNotFound: - return ok(EMPTY_ROOT_HASH) # empty sub-tree - return err(error) - ok key.to(Hash256) + db.retrieveMerkleHash VertexID(1) proc hasPathAccount*( db: AristoDbRef; @@ -136,6 +143,13 @@ proc fetchGenericData*( assert pyl.pType == RawData # debugging only ok pyl.rawBlob +proc fetchGenericState*( + db: AristoDbRef; + root: VertexID; + ): Result[Hash256,AristoError] = + ## Fetch the Merkle hash of the argument `root`. + db.retrieveMerkleHash root + proc hasPathGeneric*( db: AristoDbRef; root: VertexID; @@ -165,18 +179,11 @@ proc fetchStorageState*( accPath: PathID; ): Result[Hash256,AristoError] = ## Fetch the Merkle hash of the storage root related to `accPath`. - let - stoID = db.retrieveStoID(accPath).valueOr: - if error == FetchPathNotFound: - return ok(EMPTY_ROOT_HASH) # no sub-tree - return err(error) - - key = db.getKeyRc(stoID).valueOr: - if error == GetKeyNotFound: - return ok(EMPTY_ROOT_HASH) # empty or no sub-tree - return err(error) - - ok key.to(Hash256) + let stoID = db.retrieveStoID(accPath).valueOr: + if error == FetchPathNotFound: + return ok(EMPTY_ROOT_HASH) # no sub-tree + return err(error) + db.retrieveMerkleHash stoID proc hasPathStorage*( db: AristoDbRef; diff --git a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim index 956437d62f..b5326e0867 100644 --- a/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim +++ b/nimbus/db/core_db/backend/aristo_db/handlers_aristo.nim @@ -37,18 +37,11 @@ type AristoCoreDbMptRef = ref object of CoreDbMptRef base: AristoBaseRef ## Local base descriptor mptRoot: VertexID ## State root, may be zero unless account - accPath: PathID ## Needed for storage tree/columns - address: EthAddress ## For storage tree debugging AristoColRef* = ref object of CoreDbColRef ## Vertex ID wrapper, optionally with *MPT* context base: AristoBaseRef - case colType: CoreDbColType ## Current column type - of CtStorage: - stoRoot: VertexID ## State root, may be zero if unknown - stoAddr: EthAddress ## Associated storage account address - else: - reset: bool ## Internal delete request + stoRoot: VertexID ## State root, may be zero if unknown AristoCoreDbMptBE* = ref object of CoreDbMptBackendRef adb*: AristoDbRef @@ -58,17 +51,11 @@ type const VoidVID = VertexID(0) - # StorageVID = VertexID(CtStorage) -- currently unused - AccountsVID = VertexID(CtAccounts) - GenericVID = VertexID(CtGeneric) logScope: topics = "aristo-hdl" static: - doAssert CtStorage.ord == 0 - doAssert CtAccounts.ord == 1 - doAssert low(CoreDbColType).ord == 0 doAssert high(CoreDbColType).ord < LEAST_FREE_VID # ------------------------------------------------------------------------------ @@ -80,10 +67,7 @@ func isValid(col: CoreDbColRef): bool = func to(col: CoreDbColRef; T: type VertexID): T = if col.isValid: - let col = AristoColRef(col) - if col.colType == CtStorage: - return col.stoRoot - return VertexID(col.colType) + return AristoColRef(col).stoRoot func to(eAddr: EthAddress; T: type PathID): T = HashKey.fromBytes(eAddr.keccakHash.data).value.to(T) @@ -110,9 +94,7 @@ func toCoreDbAccount( if acc.storageID.isValid: result.storage = db.bless AristoColRef( base: cAcc.base, - colType: CtStorage, - stoRoot: acc.storageID, - stoAddr: address) + stoRoot: acc.storageID) func toPayloadRef(acc: CoreDbAccount): PayloadRef = PayloadRef( @@ -199,103 +181,49 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns = proc mptBackend(): CoreDbMptBackendRef = db.bless AristoCoreDbMptBE(adb: mpt) - proc mptColFn(): CoreDbColRef = - if cMpt.mptRoot.distinctBase < LEAST_FREE_VID: - return db.bless(AristoColRef( - base: base, - colType: CoreDbColType(cMpt.mptRoot))) - - assert cMpt.accPath.isValid # debug mode only - if cMpt.mptRoot.isValid: - # The mpt might have become empty - let - key = cMpt.address.keccakHash.data - acc = api.fetchAccountPayload(mpt, key).valueOr: - raiseAssert "mptColFn(): " & $error - - # Update by accounts data - cMpt.mptRoot = acc.storageID - - db.bless AristoColRef( - base: base, - colType: CtStorage, - stoRoot: cMpt.mptRoot, - stoAddr: cMpt.address) - proc mptFetch(key: openArray[byte]): CoreDbRc[Blob] = const info = "fetchFn()" - - let rc = block: - if cMpt.accPath.isValid: - api.fetchStorageData(mpt, key, cMpt.accPath) - elif cMpt.mptRoot.isValid: - api.fetchGenericData(mpt, cMpt.mptRoot, key) - else: - # Some pathological behaviour observed with storage column due to lazy - # update. The `fetchXxxPayload()` does not now about this and would - # complain an error different from `FetchPathNotFound`. - return err(MptRootMissing.toError(base, info, MptNotFound)) - - # let rc = api.fetchPayload(mpt, rootVID, key) - if rc.isOk: - ok rc.value - elif rc.error != FetchPathNotFound: - err(rc.error.toError(base, info)) - else: - err(rc.error.toError(base, info, MptNotFound)) + let data = api.fetchGenericData(mpt, cMpt.mptRoot, key).valueOr: + if error == FetchPathNotFound: + return err(error.toError(base, info, MptNotFound)) + return err(error.toError(base, info)) + ok(data) proc mptMerge(k: openArray[byte]; v: openArray[byte]): CoreDbRc[void] = const info = "mergeFn()" - - if cMpt.accPath.isValid: - let rc = api.mergeStorageData(mpt, k, v, cMpt.accPath) - if rc.isErr: - return err(rc.error.toError(base, info)) - if rc.value.isValid: - cMpt.mptRoot = rc.value - else: - let rc = api.mergeGenericData(mpt, cMpt.mptRoot, k, v) - if rc.isErr: - return err(rc.error.toError(base, info)) - + api.mergeGenericData(mpt, cMpt.mptRoot, k, v).isOkOr: + return err(error.toError(base, info)) ok() proc mptDelete(key: openArray[byte]): CoreDbRc[void] = const info = "deleteFn()" - - let rc = block: - if cMpt.accPath.isValid: - api.deleteStorageData(mpt, key, cMpt.accPath) - else: - api.deleteGenericData(mpt, cMpt.mptRoot, key) - - if rc.isErr: - if rc.error == DelPathNotFound: - return err(rc.error.toError(base, info, MptNotFound)) - if rc.error == DelStoRootMissing: - # This is insane but legit. A storage column was announced for an - # account but no data have been added, yet. - return ok() - return err(rc.error.toError(base, info)) - - if rc.value: - # Column has become empty - cMpt.mptRoot = VoidVID - + api.deleteGenericData(mpt, cMpt.mptRoot, key).isOkOr: + if error == DelPathNotFound: + return err(error.toError(base, info, MptNotFound)) + return err(error.toError(base, info)) ok() proc mptHasPath(key: openArray[byte]): CoreDbRc[bool] = const info = "hasPathFn()" + let yn = api.hasPathGeneric(mpt, cMpt.mptRoot, key).valueOr: + return err(error.toError(base, info)) + ok(yn) - let rc = block: - if cMpt.accPath.isValid: - api.hasPathStorage(mpt, key, cMpt.accPath) - else: - api.hasPathGeneric(mpt, cMpt.mptRoot, key) + proc mptState(updateOk: bool): CoreDbRc[Hash256] = + const info = "mptState()" - if rc.isErr: + let rc = api.fetchGenericState(mpt, cMpt.mptRoot) + if rc.isOk: + return ok(rc.value) + elif not updateOk and rc.error != GetKeyUpdateNeeded: return err(rc.error.toError(base, info)) - ok(rc.value) + + # FIXME: `hashify()` should probably throw an assert on failure + ? api.hashify(mpt).toVoidRc(base, info, HashNotAvailable) + + let state = api.fetchGenericState(mpt, cMpt.mptRoot).valueOr: + raiseAssert info & ": " & $error + ok(state) CoreDbMptFns( @@ -312,10 +240,7 @@ proc mptMethods(cMpt: AristoCoreDbMptRef): CoreDbMptFns = mptMerge(k, v), hasPathFn: proc(k: openArray[byte]): CoreDbRc[bool] = - mptHasPath(k), - - getColFn: proc(): CoreDbColRef = - mptColFn()) + mptHasPath(k)) # ------------------------------------------------------------------------------ # Private account call back functions @@ -496,91 +421,19 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns = api = base.api # Ditto mpt = cCtx.mpt # Ditto - proc ctxNewCol( - colType: CoreDbColType; - colState: Hash256; - address: Opt[EthAddress]; - ): CoreDbRc[CoreDbColRef] = - const info = "ctx/newColFn()" - - let col = AristoColRef( - base: base, - colType: colType) - - if colType == CtStorage: - if address.isNone: - let error = aristo.UtilsAccPathMissing - return err(error.toError(base, info, AccAddrMissing)) - col.stoAddr = address.unsafeGet - - if not colState.isValid: - return ok(db.bless col) - - # Reset some non-dynamic col when instantiating. It emulates the behaviour - # of a new empty MPT on the legacy database. - col.reset = colType.resetCol() - - # Update hashes in order to verify the column state. - ? api.hashify(mpt).toVoidRc(base, info, HashNotAvailable) - - # Assure that hash is available as state for the main/accounts column - let rc = api.getKeyRc(mpt, VertexID colType) - if rc.isErr: - doAssert rc.error == GetKeyNotFound - elif rc.value == colState.to(HashKey): - return ok(db.bless col) - err(aristo.GenericError.toError(base, info, RootNotFound)) - - - proc ctxGetMpt(col: CoreDbColRef): CoreDbRc[CoreDbMptRef] = - const - info = "ctx/getMptFn()" - let - col = AristoColRef(col) - var - reset = false - newMpt: AristoCoreDbMptRef - if not col.isValid: - reset = true - newMpt = AristoCoreDbMptRef( - mptRoot: GenericVID, - accPath: VOID_PATH_ID) - - elif col.colType == CtStorage: - newMpt = AristoCoreDbMptRef( - mptRoot: col.stoRoot, - accPath: col.stoAddr.to(PathID), - address: col.stoAddr) - if col.stoRoot.isValid: - if col.stoRoot.distinctBase < LEAST_FREE_VID: - let error = (col.stoRoot,MptRootUnacceptable) - return err(error.toError(base, info, RootUnacceptable)) - # Verify path if there is a particular storge root VID - let rc = api.hikeUp(newMpt.accPath.to(NibblesSeq), AccountsVID, mpt) - if rc.isErr: - return err(rc.error[1].toError(base, info, AccNotFound)) - else: - reset = col.colType.resetCol() - newMpt = AristoCoreDbMptRef( - mptRoot: VertexID(col.colType), - accPath: VOID_PATH_ID) - - # Reset column. This a emulates the behaviour of a new empty MPT on the - # legacy database. - if reset: - let rc = api.deleteGenericTree(mpt, newMpt.mptRoot) - if rc.isErr: - return err(rc.error.toError(base, info, AutoFlushFailed)) - col.reset = false - - newMpt.base = base - newMpt.methods = newMpt.mptMethods() - ok(db.bless newMpt) + proc ctxGetColumn(colType: CoreDbColType; clearData: bool): CoreDbMptRef = + const info = "getColumnFn()" + let cMpt = AristoCoreDbMptRef(base: base, mptRoot: VertexID(colType)) + cMpt.methods = cMpt.mptMethods() + if clearData: + api.deleteGenericTree(mpt, VertexID(colType)).isOkOr: + raiseAssert info & " clearing up failed: " & $error + db.bless cMpt proc ctxGetAccounts(): CoreDbAccRef = - let acc = AristoCoreDbAccRef(base: base) - acc.methods = acc.accMethods() - db.bless acc + let cAcc = AristoCoreDbAccRef(base: base) + cAcc.methods = cAcc.accMethods() + db.bless cAcc proc ctxForget() = api.forget(mpt).isOkOr: @@ -588,15 +441,8 @@ proc ctxMethods(cCtx: AristoCoreDbCtxRef): CoreDbCtxFns = CoreDbCtxFns( - newColFn: proc( - col: CoreDbColType; - colState: Hash256; - address: Opt[EthAddress]; - ): CoreDbRc[CoreDbColRef] = - ctxNewCol(col, colState, address), - - getMptFn: proc(col: CoreDbColRef): CoreDbRc[CoreDbMptRef] = - ctxGetMpt(col), + getColumnFn: proc(colType: CoreDbColType; clearData: bool): CoreDbMptRef = + ctxGetColumn(colType, clearData), getAccountsFn: proc(): CoreDbAccRef = ctxGetAccounts(), @@ -683,15 +529,8 @@ proc colPrint*( let col = AristoColRef(col) root = col.to(VertexID) - result = "(" & $col.colType & "," - # Do vertex ID and address/hash - if col.colType == CtStorage: - result &= col.stoRoot.toStr - if col.stoAddr != EthAddress.default: - result &= ",%" & $col.stoAddr.toHex - else: - result &= VertexID(col.colType).toStr + result = "(CtGeneric," # Do the Merkle hash key if not root.isValid: diff --git a/nimbus/db/core_db/base.nim b/nimbus/db/core_db/base.nim index 96b2312034..66819f7069 100644 --- a/nimbus/db/core_db/base.nim +++ b/nimbus/db/core_db/base.nim @@ -299,18 +299,18 @@ proc ctx*(db: CoreDbRef): CoreDbCtxRef = result = db.methods.newCtxFn() db.ifTrackNewApi: debug newApiTxt, api, elapsed -proc ctxFromTx*( - db: CoreDbRef; - colState: Hash256; - colType = CtAccounts; - ): CoreDbRc[CoreDbCtxRef] = - ## Create new context derived from matching transaction of the currently - ## active column context. For the legacy backend, this function always - ## returns the currently active context (i.e. the same as `db.ctx()`.) - ## - db.setTrackNewApi BaseNewCtxFromTxFn - result = db.methods.newCtxFromTxFn(colState, colType) - db.ifTrackNewApi: debug newApiTxt, api, elapsed, result +#proc ctxFromTx*( +# db: CoreDbRef; +# colState: Hash256; +# colType = CtAccounts; +# ): CoreDbRc[CoreDbCtxRef] = +# ## Create new context derived from matching transaction of the currently +# ## active column context. For the legacy backend, this function always +# ## returns the currently active context (i.e. the same as `db.ctx()`.) +# ## +# db.setTrackNewApi BaseNewCtxFromTxFn +# result = db.methods.newCtxFromTxFn(colState, colType) +# db.ifTrackNewApi: debug newApiTxt, api, elapsed, result proc swapCtx*(db: CoreDbRef; ctx: CoreDbCtxRef): CoreDbCtxRef = ## Activate argument context `ctx` and return the previously active column @@ -338,73 +338,6 @@ proc forget*(ctx: CoreDbCtxRef) = # Public Merkle Patricia Tree sub-trie abstaction management # ------------------------------------------------------------------------------ -proc newColumn*( - ctx: CoreDbCtxRef; - colType: CoreDbColType; - colState: Hash256; - address = Opt.none(EthAddress); - ): CoreDbRc[CoreDbColRef] = - ## Retrieve a new column descriptor. - ## - ## The database is can be viewed as a matrix of rows and columns, potenially - ## with values at their intersection. A row is identified by a lookup key - ## and a column is identified by a state hash. - ## - ## Additionally, any column has a column type attribute given as `colType` - ## argument. Only storage columns also have an address attribute which must - ## be passed as argument `address` when the `colType` argument is `CtStorage`. - ## - ## If the state hash argument `colState` is passed as `EMPTY_ROOT_HASH`, this - ## function always succeeds. The result is the equivalent of a potential - ## column be incarnated later. If the column type is different from - ## `CtStorage` and `CtAccounts`, then the returned column descriptor will be - ## flagged to reset all column data when incarnated as MPT (see `newMpt()`.). - ## - ## Otherwise, the function will fail unless a column with the corresponding - ## argument `colState` identifier exists and can be found on the database. - ## Note that on a single state database like `Aristo`, the requested column - ## might exist but is buried in some history journal (which needs an extra - ## effort to unwrap.) - ## - ## This function is intended to open a column on the database as in: - ## :: - ## proc openAccountLedger(db: CoreDbRef, colState: Hash256): CoreDbMptRef = - ## let col = db.ctx.newColumn(CtAccounts, colState).valueOr: - ## # some error handling - ## return - ## db.getAcc col - ## - ctx.setTrackNewApi CtxNewColFn - result = ctx.methods.newColFn(colType, colState, address) - ctx.ifTrackNewApi: - debug newApiTxt, api, elapsed, colType, colState, address, result - -proc newColumn*( - ctx: CoreDbCtxRef; - colState: Hash256; - address: EthAddress; - ): CoreDbRc[CoreDbColRef] = - ## Shortcut for `ctx.newColumn(CtStorage,colState,some(address))`. - ## - ctx.setTrackNewApi CtxNewColFn - result = ctx.methods.newColFn(CtStorage, colState, Opt.some(address)) - ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, colState, address, result - -proc newColumn*( - ctx: CoreDbCtxRef; - address: EthAddress; - ): CoreDbColRef = - ## Shortcut for `ctx.newColumn(EMPTY_ROOT_HASH,address).value`. The function - ## will throw an exception on error. So the result will always be a valid - ## descriptor. - ## - ctx.setTrackNewApi CtxNewColFn - result = ctx.methods.newColFn( - CtStorage, EMPTY_ROOT_HASH, Opt.some(address)).valueOr: - raiseAssert error.prettyText() - ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, address, result - - proc `$$`*(col: CoreDbColRef): string = ## Pretty print the column descriptor. Note that this directive may have side ## effects as it calls a backend function. @@ -452,54 +385,19 @@ proc stateEmptyOrVoid*(col: CoreDbColRef): bool = col.stateEmpty.valueOr: true # ------------------------------------------------------------------------------ -# Public Merkle Patricia Tree, hexary trie constructors +# Public functions for generic columns # ------------------------------------------------------------------------------ -proc getMpt*( - ctx: CoreDbCtxRef; - col: CoreDbColRef; - ): CoreDbRc[CoreDbMptRef] = - ## Get an MPT sub-trie view. - ## - ## If the `col` argument descriptor was created for an `EMPTY_ROOT_HASH` - ## column state of type different form `CtStorage` or `CtAccounts`, all - ## column will be flushed. There is no need to hold the `col` argument for - ## later use. It can always be rerieved for this particular MPT using the - ## function `getColumn()`. - ## - ctx.setTrackNewApi CtxGetMptFn - result = ctx.methods.getMptFn col - ctx.ifTrackNewApi: debug newApiTxt, api, elapsed, col, result - -proc getMpt*( +proc getColumn*( ctx: CoreDbCtxRef; colType: CoreDbColType; - address = Opt.none(EthAddress); + clearData = false; ): CoreDbMptRef = - ## Shortcut for `getMpt(col)` where the `col` argument is - ## `db.getColumn(colType,EMPTY_ROOT_HASH).value`. This function will always - ## return a non-nil descriptor or throw an exception. + ## ... ## - ctx.setTrackNewApi CtxGetMptFn - let col = ctx.methods.newColFn(colType, EMPTY_ROOT_HASH, address).value - result = ctx.methods.getMptFn(col).valueOr: - raiseAssert error.prettyText() - ctx.ifTrackNewApi: debug newApiTxt, api, colType, elapsed - -# ------------------------------------------------------------------------------ -# Public common methods for all hexary trie databases (`mpt`, or `acc`) -# ------------------------------------------------------------------------------ - -proc getColumn*(mpt: CoreDbMptRef): CoreDbColRef = - ## Variant of `getColumn()` - ## - mpt.setTrackNewApi MptGetColFn - result = mpt.methods.getColFn() - mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, result - -# ------------------------------------------------------------------------------ -# Public generic hexary trie database methods -# ------------------------------------------------------------------------------ + ctx.setTrackNewApi CtxGetColumnFn + result = ctx.methods.getColumnFn(colType, clearData) + ctx.ifTrackNewApi: debug newApiTxt, api, colType, clearData, elapsed proc fetch*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[Blob] = ## Fetch data from the argument `mpt`. The function always returns a @@ -551,6 +449,17 @@ proc hasPath*(mpt: CoreDbMptRef; key: openArray[byte]): CoreDbRc[bool] = let col = mpt.methods.getColFn() debug newApiTxt, api, elapsed, col, key=key.toStr, result +proc state*(mpt: CoreDbMptRef; updateOk = false): CoreDbRc[Hash256] = + ## This function retrieves the Merkle state hash of the argument + ## database column (if acvailable.) + ## + ## If the argument `updateOk` is set `true`, the Merkle hashes of the + ## database will be updated first (if needed, at all). + ## + mpt.setTrackNewApi MptStateFn + result = mpt.methods.stateFn updateOk + mpt.ifTrackNewApi: debug newApiTxt, api, elapsed, updateOK, result + # ------------------------------------------------------------------------------ # Public methods for accounts # ------------------------------------------------------------------------------ @@ -606,8 +515,8 @@ proc hasPath*(acc: CoreDbAccRef; eAddr: EthAddress): CoreDbRc[bool] = acc.ifTrackNewApi: debug newApiTxt, api, elapsed, eAddr, result proc state*(acc: CoreDbAccRef; updateOk = false): CoreDbRc[Hash256] = - ## Getter (well, sort of). It retrieves the account column Merkle state - ## hash if acvailable. + ## This function retrieves the Merkle state hash of the accounts + ## column (if acvailable.) ## ## If the argument `updateOk` is set `true`, the Merkle hashes of the ## database will be updated first (if needed, at all). diff --git a/nimbus/db/core_db/base/api_tracking.nim b/nimbus/db/core_db/base/api_tracking.nim index 1c0cfaf09d..bf0e1e3dec 100644 --- a/nimbus/db/core_db/base/api_tracking.nim +++ b/nimbus/db/core_db/base/api_tracking.nim @@ -66,29 +66,29 @@ type CtxForgetFn = "ctx/forget" CtxGetAccountsFn = "getAccounts" - CtxGetMptFn = "ctx/getMpt" + CtxGetColumnFn = "getColumn" CtxNewColFn = "ctx/newColumn" ErrorPrintFn = "$$" EthAccRecastFn = "recast" - KvtDelFn = "kvt/del" - KvtForgetFn = "kvt/forget" - KvtGetFn = "kvt/get" - KvtGetOrEmptyFn = "kvt/getOrEmpty" - KvtHasKeyFn = "kvt/hasKey" - KvtPairsIt = "kvt/pairs" - KvtPutFn = "kvt/put" + KvtDelFn = "del" + KvtForgetFn = "forget" + KvtGetFn = "get" + KvtGetOrEmptyFn = "getOrEmpty" + KvtHasKeyFn = "hasKey" + KvtPairsIt = "pairs" + KvtPutFn = "put" MptDeleteFn = "mpt/delete" MptFetchFn = "mpt/fetch" MptFetchOrEmptyFn = "mpt/fetchOrEmpty" MptForgetFn = "mpt/forget" - MptGetColFn = "mpt/getColumn" MptHasPathFn = "mpt/hasPath" MptMergeFn = "mpt/merge" MptPairsIt = "mpt/pairs" MptReplicateIt = "mpt/replicate" + MptStateFn = "mpt/state" TxCommitFn = "commit" TxDisposeFn = "dispose" diff --git a/nimbus/db/core_db/base/base_desc.nim b/nimbus/db/core_db/base/base_desc.nim index 7a69a0a848..2c7c3c9e65 100644 --- a/nimbus/db/core_db/base/base_desc.nim +++ b/nimbus/db/core_db/base/base_desc.nim @@ -76,9 +76,7 @@ type TxPending CoreDbColType* = enum - CtStorage = 0 - CtAccounts - CtGeneric + CtGeneric = 2 # columns smaller than 2 are not provided CtReceipts CtTxs CtWithdrawals @@ -161,23 +159,23 @@ type # -------------------------------------------------- CoreDbCtxFromTxFn* = proc(root: Hash256; kind: CoreDbColType): CoreDbRc[CoreDbCtxRef] {.noRaise.} - CoreDbCtxNewColFn* = proc( + CoreDbCtxNewColFn* = proc( # <--- deprecated colType: CoreDbColType; colState: Hash256; address: Opt[EthAddress]; ): CoreDbRc[CoreDbColRef] {.noRaise.} - CoreDbCtxGetMptFn* = proc( - root: CoreDbColRef): CoreDbRc[CoreDbMptRef] {.noRaise.} + CoreDbCtxGetColumnFn* = proc( + colType: CoreDbColType; clearData: bool): CoreDbMptRef {.noRaise.} CoreDbCtxGetAccountsFn* = proc(): CoreDbAccRef {.noRaise.} CoreDbCtxForgetFn* = proc() {.noRaise.} CoreDbCtxFns* = object ## Methods for context maniulation - newColFn*: CoreDbCtxNewColFn - getMptFn*: CoreDbCtxGetMptFn + #newColFn*: CoreDbCtxNewColFn # <--- deprecated + getColumnFn*: CoreDbCtxGetColumnFn getAccountsFn*: CoreDbCtxGetAccountsFn forgetFn*: CoreDbCtxForgetFn # -------------------------------------------------- - # Sub-descriptor: generic Mpt/hexary trie methods + # Sub-descriptor: generic Mpt methods # -------------------------------------------------- CoreDbMptBackendFn* = proc(): CoreDbMptBackendRef {.noRaise.} CoreDbMptFetchFn* = @@ -191,8 +189,8 @@ type CoreDbMptMergeAccountFn* = proc(k: openArray[byte]; v: CoreDbAccount): CoreDbRc[void] {.noRaise.} CoreDbMptHasPathFn* = proc(k: openArray[byte]): CoreDbRc[bool] {.noRaise.} - CoreDbMptGetColFn* = proc(): CoreDbColRef {.noRaise.} CoreDbMptForgetFn* = proc(): CoreDbRc[void] {.noRaise.} + CoreDbMptStateFn* = proc(updateOk: bool): CoreDbRc[Hash256] {.noRaise.} CoreDbMptFns* = object ## Methods for trie objects @@ -201,7 +199,7 @@ type deleteFn*: CoreDbMptDeleteFn mergeFn*: CoreDbMptMergeFn hasPathFn*: CoreDbMptHasPathFn - getColFn*: CoreDbMptGetColFn + stateFn*: CoreDbMptStateFn # ---------------------------------------------------- diff --git a/nimbus/db/core_db/base/validate.nim b/nimbus/db/core_db/base/validate.nim index 6731cf0cd3..56f5f25bb2 100644 --- a/nimbus/db/core_db/base/validate.nim +++ b/nimbus/db/core_db/base/validate.nim @@ -51,9 +51,8 @@ proc validateMethodsDesc(kvt: CoreDbKvtFns) = doAssert not kvt.forgetFn.isNil proc validateMethodsDesc(ctx: CoreDbCtxFns) = - doAssert not ctx.newColFn.isNil - doAssert not ctx.getMptFn.isNil doAssert not ctx.getAccountsFn.isNil + doAssert not ctx.getColumnFn.isNil doAssert not ctx.forgetFn.isNil proc validateMethodsDesc(fns: CoreDbMptFns) = @@ -62,7 +61,6 @@ proc validateMethodsDesc(fns: CoreDbMptFns) = doAssert not fns.deleteFn.isNil doAssert not fns.mergeFn.isNil doAssert not fns.hasPathFn.isNil - doAssert not fns.getColFn.isNil proc validateMethodsDesc(fns: CoreDbAccFns) = doAssert not fns.backendFn.isNil diff --git a/nimbus/db/core_db/core_apps.nim b/nimbus/db/core_db/core_apps.nim index b66c47d130..33770e200d 100644 --- a/nimbus/db/core_db/core_apps.nim +++ b/nimbus/db/core_db/core_apps.nim @@ -116,26 +116,23 @@ iterator getBlockTransactionData*( db: CoreDbRef; transactionRoot: Hash256; ): Blob = + const info = "getBlockTransactionData()" block body: if transactionRoot == EMPTY_ROOT_HASH: break body - let - ctx = db.ctx - col = ctx.newColumn(CtTxs, transactionRoot).valueOr: - warn logTxt "getBlockTransactionData()", - transactionRoot, action="newColumn()", `error`=($$error) - break body - transactionDb = ctx.getMpt(col).valueOr: - warn logTxt "getBlockTransactionData()", transactionRoot, - action="newMpt()", col=($$col), error=($$error) - break body + transactionDb = db.ctx.getColumn CtTxs + state = transactionDb.state(updateOk=true).valueOr: + raiseAssert info & ": " & $$error + if state != transactionRoot: + warn logTxt info, transactionRoot, state, error="state mismatch" + break body var transactionIdx = 0'u64 while true: let transactionKey = rlp.encode(transactionIdx) let data = transactionDb.fetch(transactionKey).valueOr: if error.error != MptNotFound: - warn logTxt "getBlockTransactionData()", transactionRoot, + warn logTxt info, transactionRoot, transactionKey, action="fetch()", error=($$error) break body yield data @@ -165,20 +162,17 @@ iterator getWithdrawalsData*( db: CoreDbRef; withdrawalsRoot: Hash256; ): Blob = + const info = "getWithdrawalsData()" block body: if withdrawalsRoot == EMPTY_ROOT_HASH: break body - let - ctx = db.ctx - col = ctx.newColumn(CtWithdrawals, withdrawalsRoot).valueOr: - warn logTxt "getWithdrawalsData()", - withdrawalsRoot, action="newColumn()", error=($$error) - break body - wddb = ctx.getMpt(col).valueOr: - warn logTxt "getWithdrawalsData()", - withdrawalsRoot, action="newMpt()", col=($$col), error=($$error) - break body + wddb = db.ctx.getColumn CtWithdrawals + state = wddb.state(updateOk=true).valueOr: + raiseAssert info & ": " & $$error + if state != withdrawalsRoot: + warn logTxt info, withdrawalsRoot, state, error="state mismatch" + break body var idx = 0 while true: let wdKey = rlp.encode(idx.uint) @@ -196,20 +190,17 @@ iterator getReceipts*( receiptsRoot: Hash256; ): Receipt {.gcsafe, raises: [RlpError].} = + const info = "getReceipts()" block body: if receiptsRoot == EMPTY_ROOT_HASH: break body - let - ctx = db.ctx - col = ctx.newColumn(CtReceipts, receiptsRoot).valueOr: - warn logTxt "getWithdrawalsData()", - receiptsRoot, action="newColumn()", error=($$error) - break body - receiptDb = ctx.getMpt(col).valueOr: - warn logTxt "getWithdrawalsData()", - receiptsRoot, action="getMpt()", col=($$col), error=($$error) - break body + receiptDb = db.ctx.getColumn CtReceipts + state = receiptDb.state(updateOk=true).valueOr: + raiseAssert info & ": " & $$error + if state != receiptsRoot: + warn logTxt info, receiptsRoot, state, error="state mismatch" + break body var receiptIdx = 0 while true: let receiptKey = rlp.encode(receiptIdx.uint) @@ -347,15 +338,16 @@ proc getSavedStateBlockNumber*( ## the `relax` argument can be set `true` so this function also returns ## zero if the state consistency check fails. ## + const info = "getSavedStateBlockNumber(): " var header: BlockHeader - let st = db.ctx.getMpt(CtGeneric).backend.toAristoSavedStateBlockNumber() + let st = db.ctx.getColumn(CtGeneric).backend.toAristoSavedStateBlockNumber() if db.getBlockHeader(st.blockNumber, header): - discard db.ctx.newColumn(CtAccounts,header.stateRoot).valueOr: - if relax: - return - raiseAssert "getSavedStateBlockNumber(): state mismatch at " & - "#" & $st.blockNumber - return st.blockNumber + let state = db.ctx.getAccounts.state.valueOr: + raiseAssert info & $$error + if state == header.stateRoot: + return st.blockNumber + if not relax: + raiseAssert info & ": state mismatch at " & "#" & $st.blockNumber proc getBlockHeader*( db: CoreDbRef; @@ -548,7 +540,7 @@ proc persistTransactions*( return let - mpt = db.ctx.getMpt(CtTxs) + mpt = db.ctx.getColumn(CtTxs, clearData=true) kvt = db.newKvt() for idx, tx in transactions: @@ -592,23 +584,23 @@ proc getTransaction*( const info = "getTransaction()" let - ctx = db.ctx - col = ctx.newColumn(CtTxs, txRoot).valueOr: - warn logTxt info, txRoot, action="newColumn()", error=($$error) - return false - mpt = ctx.getMpt(col).valueOr: - warn logTxt info, - txRoot, action="newMpt()", col=($$col), error=($$error) + clearOk = txRoot == EMPTY_ROOT_HASH + mpt = db.ctx.getColumn(CtTxs, clearData=clearOk) + if not clearOk: + let state = mpt.state(updateOk=true).valueOr: + raiseAssert info & ": " & $$error + if state != txRoot: + warn logTxt info, txRoot, state, error="state mismatch" return false + let txData = mpt.fetch(rlp.encode(txIndex)).valueOr: if error.error != MptNotFound: - warn logTxt info, txIndex, action="fetch()", error=($$error) + warn logTxt info, txIndex, error=($$error) return false try: res = rlp.decode(txData, Transaction) - except RlpError as exc: - warn logTxt info, - txRoot, action="rlp.decode()", col=($$col), error=exc.msg + except RlpError as e: + warn logTxt info, txRoot, action="rlp.decode()", name=($e.name), msg=e.msg return false true @@ -619,13 +611,13 @@ proc getTransactionCount*( const info = "getTransactionCount()" let - ctx = db.ctx - col = ctx.newColumn(CtTxs, txRoot).valueOr: - warn logTxt info, txRoot, action="newColumn()", error=($$error) - return 0 - mpt = ctx.getMpt(col).valueOr: - warn logTxt info, txRoot, - action="newMpt()", col=($$col), error=($$error) + clearOk = txRoot == EMPTY_ROOT_HASH + mpt = db.ctx.getColumn(CtTxs, clearData=clearOk) + if not clearOk: + let state = mpt.state(updateOk=true).valueOr: + raiseAssert info & ": " & $$error + if state != txRoot: + warn logTxt info, txRoot, state, error="state mismatch" return 0 var txCount = 0 while true: @@ -676,11 +668,10 @@ proc persistWithdrawals*( const info = "persistWithdrawals()" if withdrawals.len == 0: return - - let mpt = db.ctx.getMpt(CtWithdrawals) + let mpt = db.ctx.getColumn(CtWithdrawals, clearData=true) for idx, wd in withdrawals: mpt.merge(rlp.encode(idx.uint), rlp.encode(wd)).isOkOr: - warn logTxt info, idx, action="merge()", error=($$error) + warn logTxt info, idx, error=($$error) return proc getWithdrawals*( @@ -847,8 +838,7 @@ proc persistReceipts*( const info = "persistReceipts()" if receipts.len == 0: return - - let mpt = db.ctx.getMpt(CtReceipts) + let mpt = db.ctx.getColumn(CtReceipts, clearData=true) for idx, rec in receipts: mpt.merge(rlp.encode(idx.uint), rlp.encode(rec)).isOkOr: warn logTxt info, idx, action="merge()", error=($$error) diff --git a/tests/test_ledger.nim b/tests/test_ledger.nim index 97e600bb24..c3f43d73bf 100644 --- a/tests/test_ledger.nim +++ b/tests/test_ledger.nim @@ -181,24 +181,24 @@ proc runTrial3(env: TestEnv, ledger: LedgerRef; inx: int; rollback: bool) = ## Run three blocks, the second one optionally with *rollback*. let eAddr = env.txs[inx].getRecipient - block: + block body1: let accTx = ledger.beginSavepoint ledger.modBalance(eAddr) ledger.commit(accTx) ledger.persist() - block: + block body2: let accTx = ledger.beginSavepoint ledger.modBalance(eAddr) if rollback: ledger.rollback(accTx) - break + break body2 ledger.commit(accTx) ledger.persist() - block: + block body3: let accTx = ledger.beginSavepoint ledger.modBalance(eAddr) ledger.commit(accTx)