Merge branch 'master' into lru-metrics

nimbus/db/aristo/aristo_blobify.nim

@@ -203,12 +203,12 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
         if vtx.ePfx.len > 0:
           vtx.ePfx.toHexPrefix(isleaf = false)
         else:
-          @[]
+          default(HexPrefixBuf)
       psLen = pSegm.len.byte
     if 33 < psLen:
       return err(BlobifyExtPathOverflow)
 
-    data &= pSegm
+    data &= pSegm.data()
     data &= lens.toBytesBE
     data &= [0x80u8 or psLen]
 
@@ -219,16 +219,16 @@ proc blobifyTo*(vtx: VertexRef; data: var Blob): Result[void,AristoError] =
     if psLen == 0 or 33 < psLen:
       return err(BlobifyLeafPathOverflow)
     vtx.lData.blobifyTo(data)
-    data &= pSegm
+    data &= pSegm.data()
     data &= [0xC0u8 or psLen]
 
   ok()
 
-proc blobify*(vtx: VertexRef): Result[Blob, AristoError] =
+proc blobify*(vtx: VertexRef): Blob =
   ## Variant of `blobify()`
-  var data: Blob
-  ? vtx.blobifyTo data
-  ok(move(data))
+  result = newSeqOfCap[byte](128)
+  if vtx.blobifyTo(result).isErr:
+    result.setLen(0) # blobify only fails on invalid verticies
 
 proc blobifyTo*(lSst: SavedState; data: var Blob): Result[void,AristoError] =
   ## Serialise a last saved state record
@@ -246,45 +246,48 @@ proc blobify*(lSst: SavedState): Result[Blob,AristoError] =
 # -------------
 proc deblobify(
     data: openArray[byte];
-    T: type LeafPayload;
-      ): Result[LeafPayload,AristoError] =
+    pyl: var LeafPayload;
+      ): Result[void,AristoError] =
   if data.len == 0:
-    return ok LeafPayload(pType: RawData)
+    pyl = LeafPayload(pType: RawData)
+    return ok()
 
   let mask = data[^1]
   if (mask and 0x10) > 0: # unstructured payload
-    return ok LeafPayload(pType: RawData, rawBlob: data[0 .. ^2])
+    pyl = LeafPayload(pType: RawData, rawBlob: data[0 .. ^2])
+    return ok()
 
   if (mask and 0x20) > 0: # Slot storage data
-    return ok LeafPayload(
+    pyl = LeafPayload(
       pType: StoData,
       stoData: ?deblobify(data.toOpenArray(0, data.len - 2), UInt256))
+    return ok()
 
+  pyl = LeafPayload(pType: AccountData)
   var
-    pAcc = LeafPayload(pType: AccountData)
     start = 0
     lens = uint16.fromBytesBE(data.toOpenArray(data.len - 3, data.len - 2))
 
   if (mask and 0x01) > 0:
     let len = lens and 0b111
-    pAcc.account.nonce = ? load64(data, start, int(len + 1))
+    pyl.account.nonce = ? load64(data, start, int(len + 1))
 
   if (mask and 0x02) > 0:
     let len = (lens shr 3) and 0b11111
-    pAcc.account.balance = ? load256(data, start, int(len + 1))
+    pyl.account.balance = ? load256(data, start, int(len + 1))
 
   if (mask and 0x04) > 0:
     let len = (lens shr 8) and 0b111
-    pAcc.stoID = (true, VertexID(? load64(data, start, int(len + 1))))
+    pyl.stoID = (true, VertexID(? load64(data, start, int(len + 1))))
 
   if (mask and 0x08) > 0:
     if data.len() < start + 32:
       return err(DeblobCodeLenUnsupported)
-    discard pAcc.account.codeHash.data.copyFrom(data.toOpenArray(start, start + 31))
+    discard pyl.account.codeHash.data.copyFrom(data.toOpenArray(start, start + 31))
   else:
-    pAcc.account.codeHash = EMPTY_CODE_HASH
+    pyl.account.codeHash = EMPTY_CODE_HASH
 
-  ok(pAcc)
+  ok()
 
 proc deblobify*(
     record: openArray[byte];
@@ -336,11 +339,12 @@ proc deblobify*(
       NibblesBuf.fromHexPrefix record.toOpenArray(pLen, rLen-1)
     if not isLeaf:
       return err(DeblobLeafGotExtPrefix)
-    let pyl = ? record.toOpenArray(0, pLen - 1).deblobify(LeafPayload)
-    VertexRef(
+    let vtx = VertexRef(
       vType: Leaf,
-      lPfx:  pathSegment,
-      lData: pyl)
+      lPfx:  pathSegment)
+
+    ? record.toOpenArray(0, pLen - 1).deblobify(vtx.lData)
+    vtx
 
   else:
     return err(DeblobUnknown)

nimbus/db/aristo/aristo_compute.nim

@@ -68,7 +68,7 @@ proc computeKeyImpl(
   case vtx.vType:
   of Leaf:
     writer.startList(2)
-    writer.append(vtx.lPfx.toHexPrefix(isLeaf = true))
+    writer.append(vtx.lPfx.toHexPrefix(isLeaf = true).data())
 
     case vtx.lData.pType
     of AccountData:
@@ -111,7 +111,7 @@ proc computeKeyImpl(
       writeBranch(bwriter)
 
       writer.startList(2)
-      writer.append(vtx.ePfx.toHexPrefix(isleaf = false))
+      writer.append(vtx.ePfx.toHexPrefix(isleaf = false).data())
       writer.append(bwriter.finish().digestTo(HashKey))
     else:
       writeBranch(writer)

nimbus/db/aristo/aristo_delete/delete_subtree.nim

@@ -12,21 +12,41 @@
 
 import
   eth/common,
-  results,
   ".."/[aristo_desc, aristo_get, aristo_layers],
   ./delete_helpers
 
 # ------------------------------------------------------------------------------
 # Private heplers
 # ------------------------------------------------------------------------------
 
-proc collectStoTreeLazily(
-  db: AristoDbRef;                     # Database, top layer
-  rvid: RootedVertexID;                # Root vertex
-  accPath: Hash256;                    # Accounts cache designator
-  stoPath: NibblesBuf;                 # Current storage path
+proc delSubTreeNow(
+    db: AristoDbRef;
+    rvid: RootedVertexID;
+      ): Result[void,AristoError] =
+  ## Delete sub-tree now
+  let (vtx, _) = db.getVtxRc(rvid).valueOr:
+    if error == GetVtxNotFound:
+      return ok()
+    return err(error)
+
+  if vtx.vType == Branch:
+    for n in 0..15:
+      if vtx.bVid[n].isValid:
+        ? db.delSubTreeNow((rvid.root,vtx.bVid[n]))
+
+  db.disposeOfVtx(rvid)
+
+  ok()
+
+
+proc delStoTreeNow(
+  db: AristoDbRef;                   # Database, top layer
+  rvid: RootedVertexID;              # Root vertex
+  accPath: Hash256;                  # Accounts cache designator
+  stoPath: NibblesBuf;               # Current storage path
     ): Result[void,AristoError] =
-  ## Collect vertex/vid and delete cache entries.
+  ## Implementation of *delete* sub-trie.
+
   let (vtx, _) = db.getVtxRc(rvid).valueOr:
     if error == GetVtxNotFound:
       return ok()
@@ -36,80 +56,36 @@ proc collectStoTreeLazily(
   of Branch:
     for i in 0..15:
       if vtx.bVid[i].isValid:
-        ? db.collectStoTreeLazily(
+        ? db.delStoTreeNow(
           (rvid.root, vtx.bVid[i]), accPath,
           stoPath & vtx.ePfx & NibblesBuf.nibble(byte i))
 
   of Leaf:
     let stoPath = Hash256(data: (stoPath & vtx.lPfx).getBytes())
     db.layersPutStoLeaf(AccountKey.mixUp(accPath, stoPath), nil)
 
-  # There is no useful approach avoiding to walk the whole tree for updating
-  # the storage data access cache.
-  #
-  # The alternative of stopping here and clearing the whole cache did degrade
-  # performance significantly in some tests on mainnet when importing `era1`.
-  #
-  # The cache it was seen
-  # * filled up to maximum size most of the time
-  # * at the same time having no `stoPath` hit at all (so there was nothing
-  #   to be cleared.)
-  #
-  ok()
-
-
-proc disposeOfSubTree(
-    db: AristoDbRef;                   # Database, top layer
-    rvid: RootedVertexID;              # Root vertex
-      ) =
-  ## Evaluate results from `collectSubTreeLazyImpl()` or ftom
-  ## `collectStoTreeLazyImpl)`.
-  ##
-  let vtx = db.getVtxRc(rvid).value[0]
-  if vtx.vType == Branch:
-    for n in 0..15:
-      if vtx.bVid[n].isValid:
-        db.top.delTree.add (rvid.root,vtx.bVid[n])
-
-  # Delete top of tree now.
   db.disposeOfVtx(rvid)
 
+  ok()
+
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
 
 proc delSubTreeImpl*(
-    db: AristoDbRef;                   # Database, top layer
-    root: VertexID;                    # Root vertex
+    db: AristoDbRef;
+    root: VertexID;
       ): Result[void,AristoError] =
-  ## Delete all the `subRoots`if there are a few, only. Otherwise
-  ## mark it for deleting later.
-  discard db.getVtxRc((root,root)).valueOr:
-    if error == GetVtxNotFound:
-      return ok()
-    return err(error)
-
-  db.disposeOfSubTree((root,root))
-
-  ok()
+  db.delSubTreeNow (root,root)
 
 
 proc delStoTreeImpl*(
     db: AristoDbRef;                   # Database, top layer
     rvid: RootedVertexID;              # Root vertex
     accPath: Hash256;
       ): Result[void,AristoError] =
-  ## Collect vertex/vid and cache entry.
-  discard db.getVtxRc(rvid).valueOr:
-    if error == GetVtxNotFound:
-      return ok()
-    return err(error)
-
-  ? db.collectStoTreeLazily(rvid, accPath, NibblesBuf())
-
-  db.disposeOfSubTree(rvid)
-
-  ok()
+  ## Implementation of *delete* sub-trie.
+  db.delStoTreeNow(rvid, accPath, NibblesBuf())
 
 # ------------------------------------------------------------------------------
 # End

nimbus/db/aristo/aristo_delta.nim

@@ -24,26 +24,6 @@ import
 logScope:
   topics = "aristo-delta"
 
-# ------------------------------------------------------------------------------
-# Private functions
-# ------------------------------------------------------------------------------
-
-proc toStr(rvid: RootedVertexID): string =
-  "$" & rvid.root.uint64.toHex & ":" & rvid.vid.uint64.toHex
-
-proc delSubTree(db: AristoDbRef; writer: PutHdlRef; rvid: RootedVertexID) =
-  ## Collect subtrees marked for deletion
-  let (vtx,_) = db.getVtxRc(rvid).valueOr:
-    notice "Descending for deletion stopped", rvid=(rvid.toStr), error
-    return
-  for vid in vtx.subVids:
-    db.delSubTree(writer, (rvid.root, vid))
-  db.backend.putVtxFn(writer, rvid, VertexRef(nil))
-  db.backend.putKeyFn(writer, rvid, VOID_HASH_KEY)
-  # Make sure the `rvid` is not mentioned here, anymore for further update.
-  db.balancer.sTab.del rvid
-  db.balancer.kMap.del rvid
-
 # ------------------------------------------------------------------------------
 # Public functions, save to backend
 # ------------------------------------------------------------------------------
@@ -109,12 +89,6 @@ proc deltaPersistent*(
 
   # Store structural single trie entries
   let writeBatch = ? be.putBegFn()
-  # This one must come first in order to avoid duplicate `sTree[]` or
-  # `kMap[]` instructions, in the worst case overwiting previously deleted
-  # entries.
-  for rvid in db.balancer.delTree:
-    db.delSubTree(writeBatch, rvid)
-  # Now the standard `sTree[]` and `kMap[]` instructions.
   for rvid, vtx in db.balancer.sTab:
     be.putVtxFn(writeBatch, rvid, vtx)
   for rvid, key in db.balancer.kMap:

nimbus/db/aristo/aristo_delta/delta_merge.nim

@@ -48,7 +48,6 @@ proc deltaMerge*(
     result = LayerRef(
       sTab:      lower.sTab, # shallow copy (entries will not be modified)
       kMap:      lower.kMap,
-      delTree:   lower.delTree,
       accLeaves: lower.accLeaves,
       stoLeaves: lower.stoLeaves,
       vTop:      upper.vTop)
@@ -68,9 +67,6 @@ proc deltaMerge*(
       if not upper.kMap.hasKey(rvid):
         upper.kMap[rvid] = key
 
-    for rvid in lower.delTree:
-      upper.delTree.add rvid
-
     for (accPath,leafVtx) in lower.accLeaves.pairs:
       if not upper.accLeaves.hasKey(accPath):
         upper.accLeaves[accPath] = leafVtx

nimbus/db/aristo/aristo_delta/delta_reverse.nim

@@ -97,10 +97,6 @@ proc revFilter*(
     else:
       return err((rvid.vid,rc.error))
 
-  # Reverse changes for `delTree[]` list.
-  for rvid in filter.delTree:
-    ? db.revSubTree(rev, rvid)
-
   ok(rev)
 
 # ------------------------------------------------------------------------------