Remove vid (re)use more thoroughly

nimbus/db/aristo/aristo_blobify.nim

@@ -200,12 +200,11 @@ proc blobifyTo*(filter: FilterRef; data: var Blob): Result[void,AristoError] =
   data &= filter.src.data
   data &= filter.trg.data
 
-  data &= filter.vGen.len.uint32.toBytesBE
+  data &= 1.uint32.toBytesBE # Legacy freelist support
   data &= default(array[4, byte]) # place holder
 
   # Store vertex ID generator state
-  for w in filter.vGen:
-    data &= w.uint64.toBytesBE
+  data &= filter.vGen.uint64.toBytesBE
 
   var
     n = 0
@@ -463,7 +462,7 @@ proc deblobify*(
     return err(DeblobWrongType)
   (addr lSst.src.data[0]).copyMem(unsafeAddr data[0], 32)
   (addr lSst.trg.data[0]).copyMem(unsafeAddr data[32], 32)
-  lSst.serial = uint64.fromBytesBE data[64..72]
+  lSst.serial = uint64.fromBytesBE data.toOpenArray(64, 72)
   ok()
 
 proc deblobify*(
@@ -476,7 +475,7 @@ proc deblobify*(
   ok move(lSst)
 
 
-proc deblobify*(data: Blob; filter: var FilterRef): Result[void,AristoError] =
+proc deblobify*(data: openArray[byte]; filter: var FilterRef): Result[void,AristoError] =
   ## De-serialise an Aristo DB filter object
   if data.len < 80: # minumum length 80 for an empty filter
     return err(DeblobFilterTooShort)
@@ -503,7 +502,8 @@ proc deblobify*(data: Blob; filter: var FilterRef): Result[void,AristoError] =
     return err(DeblobFilterGenTooShort)
   for n in 0 ..< nVids:
     let w = 80 + n * 8
-    f.vGen.add (uint64.fromBytesBE data.toOpenArray(int w, int w+7)).VertexID
+    # Legacy freelist support: use the last item (new version will store at most one)
+    f.vGen = (uint64.fromBytesBE data.toOpenArray(int w, int w+7)).VertexID
 
   var offs = nTrplStart
   for n in 0 ..< nTriplets:
@@ -552,7 +552,7 @@ proc deblobify*(data: Blob; T: type FilterRef): Result[T,AristoError] =
   ok filter
 
 proc deblobify*(
-    data: Blob;
+    data: openArray[byte];
     vFqs: var seq[(QueueID,QueueID)];
       ): Result[void,AristoError] =
   ## De-serialise the data record encoded with `blobify()` into a filter queue
@@ -573,7 +573,7 @@ proc deblobify*(
   ok()
 
 proc deblobify*(
-    data: Blob;
+    data: openArray[byte];
     T: type seq[(QueueID,QueueID)];
       ): Result[T,AristoError] =
   ## Variant of `deblobify()` for deserialising the vertex ID generator state

nimbus/db/aristo/aristo_check/check_be.nim

@@ -11,16 +11,13 @@
 {.push raises: [].}
 
 import
-  std/[algorithm, sequtils, sets, tables, typetraits],
+  std/[algorithm, sets, tables, typetraits],
   eth/[common, trie/nibbles],
   stew/interval_set,
   ../../aristo,
   ../aristo_walk/persistent,
   ".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise]
 
-const
-  Vid2 = @[VertexID(LEAST_FREE_VID)].toHashSet
-
 # ------------------------------------------------------------------------------
 # Private helper
 # ------------------------------------------------------------------------------
@@ -76,17 +73,6 @@ proc toNodeBE(
       return ok node
     return err(vid)
 
-proc vidReorgAlways(vGen: seq[VertexID]): seq[VertexID] =
-  ## See `vidReorg()`, this one always sorts and optimises
-  ##
-  if 1 < vGen.len:
-    let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it))
-    for n in 0 .. lst.len-2:
-      if lst[n].uint64 != lst[n+1].uint64 + 1:
-        return lst[n+1 .. lst.len-1] & @[lst[n]]
-    return @[lst[^1]]
-  vGen
-
 # ------------------------------------------------------------------------------
 # Public functions
 # ------------------------------------------------------------------------------
@@ -139,25 +125,6 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
         return err((vid,CheckBeKeyMismatch))
     discard vids.reduce Interval[VertexID,uint64].new(vid,vid)
 
-  # Compare calculated state against database state
-  block:
-    # Extract vertex ID generator state
-    let vGen = block:
-      let rc = db.getIdgBE()
-      if rc.isOk:
-        rc.value.vidReorgAlways.toHashSet
-      elif rc.error == GetIdgNotFound:
-        EmptyVidSeq.toHashSet
-      else:
-        return err((VertexID(0),rc.error))
-    let
-      vGenExpected = vids.to(HashSet[VertexID])
-      delta = vGenExpected -+- vGen # symmetric difference
-    if 0 < delta.len:
-      # Exclude fringe case when there is a single root vertex only
-      if vGenExpected != Vid2 or 0 < vGen.len:
-        return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen))
-
   # Check top layer cache against backend
   if cache:
     let checkKeysOk = block:
@@ -222,23 +189,6 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef](
       if expected != key:
         return err((vid,CheckBeCacheKeyMismatch))
 
-    # Check vGen
-    let
-      vGen = db.vGen.vidReorgAlways.toHashSet
-      vGenExpected = vids.to(HashSet[VertexID])
-      delta = vGenExpected -+- vGen # symmetric difference
-    if 0 < delta.len:
-      if vGen == Vid2 and vGenExpected.len == 0:
-        # Fringe case when the database is empty
-        discard
-      elif vGen.len == 0 and vGenExpected == Vid2:
-        # Fringe case when there is a single root vertex only
-        discard
-      else:
-        let delta = delta.toSeq
-        if delta.len != 1 or delta[0] != VertexID(1) or VertexID(1) in vGen:
-          return err((delta.sorted[^1],CheckBeCacheGarbledVGen))
-
   ok()
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_check/check_top.nim

@@ -92,8 +92,7 @@ proc checkTopCommon*(
   let
     kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len
     kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount
-    vGen = db.vGen.toHashSet
-    vGenMax = if vGen.len == 0: VertexID(0) else: db.vGen[^1]
+    vGenMax = db.vGen
   var
     stoRoots: HashSet[VertexID]
 
@@ -108,7 +107,7 @@ proc checkTopCommon*(
           if stoVid.isValid:
             if stoVid in stoRoots:
               return err((stoVid,CheckAnyVidSharedStorageRoot))
-            if vGenMax.isValid and (vGenMax < stoVid or stoVid in vGen):
+            if vGenMax.isValid and (vGenMax < stoVid):
               return err((stoVid,CheckAnyVidDeadStorageRoot))
             stoRoots.incl stoVid
       of Branch:

nimbus/db/aristo/aristo_debug.nim

@@ -433,8 +433,7 @@ proc ppFilter(
   result &= pfx & "fid=" & fl.fid.ppFid
   result &= pfx & "src=" & fl.src.to(HashKey).ppKey(db)
   result &= pfx & "trg=" & fl.trg.to(HashKey).ppKey(db)
-  result &= pfx & "vGen" & pfx1 & "[" &
-    fl.vGen.mapIt(it.ppVid).join(",") & "]"
+  result &= pfx & "vGen" & pfx1 & "[" & fl.vGen.ppVid & "]"
   result &= pfx & "sTab" & pfx1 & "{"
   for n,vid in fl.sTab.sortedKeys:
     let vtx = fl.sTab.getOrVoid vid
@@ -457,11 +456,10 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string =
   var (dump,dataOk) = ("",false)
   dump &= pfx & "vGen"
   block:
-    let q = be.getIdgFn().get(otherwise = EmptyVidSeq)
-    dump &= "(" & $q.len & ")"
-    if 0 < q.len:
-      dataOk = true
-      dump &= pfx1 & q.ppVidList()
+    let q = be.getIdgFn().get(otherwise = VertexID(0))
+    dump &= "(1)"
+    dataOk = true
+    dump &= pfx1 & q.ppVid()
   block:
     dump &= pfx & "sTab"
     var (n, data) = (0, "")
@@ -530,9 +528,8 @@ proc ppLayer(
       result &= "<layer>".doPrefix(false)
     if vGenOk:
       let
-        tLen = layer.final.vGen.len
-        info = "vGen(" & $tLen & ")"
-      result &= info.doPrefix(0 < tLen) & layer.final.vGen.ppVidList
+        info = "vGen(1)"
+      result &= info.doPrefix(true) & layer.final.vGen.ppVid
     if sTabOk:
       let
         tLen = layer.delta.sTab.len

nimbus/db/aristo/aristo_delete.nim

@@ -300,13 +300,6 @@ proc delSubTreeImpl(
     db.layersPutVtx(VertexID(1), wp.vid, leaf)
     db.layersResKey(VertexID(1), wp.vid)
 
-  # Squeeze list of recycled vertex IDs
-  # TODO this causes a reallocation of vGen which slows down subsequent
-  #      additions to the list because the sequence must grow which entails a
-  #      full copy in addition to this reorg itself - around block 2.5M this
-  #      causes significant slowdown as the vid list is >1M entries long
-  #      See also EIP-161 which is why there are so many deletions
-  # db.top.final.vGen = db.vGen.vidReorg()
   ok()
 
 
@@ -398,13 +391,6 @@ proc deleteImpl(
     db.layersPutVtx(VertexID(1), wp.vid, leaf)
     db.layersResKey(VertexID(1), wp.vid)
 
-  # Squeeze list of recycled vertex IDs
-  # TODO this causes a reallocation of vGen which slows down subsequent
-  #      additions to the list because the sequence must grow which entails a
-  #      full copy in addition to this reorg itself - around block 2.5M this
-  #      causes significant slowdown as the vid list is >1M entries long
-  #      See also EIP-161 which is why there are so many deletions```
-  # db.top.final.vGen = db.vGen.vidReorg()
   ok(emptySubTreeOk)
 
 # ------------------------------------------------------------------------------

nimbus/db/aristo/aristo_desc/desc_backend.nim

@@ -35,7 +35,7 @@ type
         ## Generic backend database retrieval function for a filter record.
 
   GetIdgFn* =
-    proc(): Result[seq[VertexID],AristoError] {.gcsafe, raises: [].}
+    proc(): Result[VertexID,AristoError] {.gcsafe, raises: [].}
       ## Generic backend database retrieval function for a the ID generator
       ## `Aristo DB` state record.
 
@@ -79,7 +79,7 @@ type
         ## Generic backend database storage function for filter records.
 
   PutIdgFn* =
-    proc(hdl: PutHdlRef; vs: openArray[VertexID])
+    proc(hdl: PutHdlRef; vid: VertexID)
       {.gcsafe, raises: [].}
         ## Generic backend database ID generator state storage function. This
         ## function replaces the current generator state.

nimbus/db/aristo/aristo_desc/desc_structural.nim

@@ -85,7 +85,7 @@ type
     trg*: Hash256                    ## Resulting state root (i.e. `kMap[1]`)
     sTab*: Table[VertexID,VertexRef] ## Filter structural vertex table
     kMap*: Table[VertexID,HashKey]   ## Filter Merkle hash key mapping
-    vGen*: seq[VertexID]             ## Filter unique vertex ID generator
+    vGen*: VertexID                  ## Filter unique vertex ID generator
 
   LayerDeltaRef* = ref object
     ## Delta layers are stacked implying a tables hierarchy. Table entries on
@@ -123,7 +123,7 @@ type
     ##
     pPrf*: HashSet[VertexID]         ## Locked vertices (proof nodes)
     fRpp*: Table[HashKey,VertexID]   ## Key lookup for `pPrf[]` (proof nodes)
-    vGen*: seq[VertexID]             ## Recycling state for vertex IDs
+    vGen*: VertexID                  ## The first free VertexID
     dirty*: HashSet[VertexID]        ## Start nodes to re-hashiy from
 
   LayerRef* = ref LayerObj

nimbus/db/aristo/aristo_get.nim

@@ -24,7 +24,7 @@ import
 
 proc getIdgUbe*(
     db: AristoDbRef;
-      ): Result[seq[VertexID],AristoError] =
+      ): Result[VertexID,AristoError] =
   ## Get the ID generator state from the unfiltered backened if available.
   let be = db.backend
   if not be.isNil:
@@ -83,7 +83,7 @@ proc getFilUbe*(
 
 proc getIdgBE*(
     db: AristoDbRef;
-      ): Result[seq[VertexID],AristoError] =
+      ): Result[VertexID,AristoError] =
   ## Get the ID generator state the `backened` layer if available.
   if not db.roFilter.isNil:
     return ok(db.roFilter.vGen)

nimbus/db/aristo/aristo_init/memory_db.nim

@@ -46,7 +46,7 @@ type
     sTab: Table[VertexID,Blob]       ## Structural vertex table making up a trie
     kMap: Table[VertexID,HashKey]    ## Merkle hash key mapping
     rFil: Table[QueueID,Blob]        ## Backend journal filters
-    vGen: Option[seq[VertexID]]      ## ID generator state
+    vGen: Option[VertexID]           ## ID generator state
     lSst: Option[SavedState]         ## Last saved state
     vFqs: Option[seq[(QueueID,QueueID)]]
     noFq: bool                       ## No filter queues available
@@ -59,7 +59,7 @@ type
     sTab: Table[VertexID,Blob]
     kMap: Table[VertexID,HashKey]
     rFil: Table[QueueID,Blob]
-    vGen: Option[seq[VertexID]]
+    vGen: Option[VertexID]
     lSst: Option[SavedState]
     vFqs: Option[seq[(QueueID,QueueID)]]
 
@@ -128,7 +128,7 @@ proc getFilFn(db: MemBackendRef): GetFilFn =
 
 proc getIdgFn(db: MemBackendRef): GetIdgFn =
   result =
-    proc(): Result[seq[VertexID],AristoError]=
+    proc(): Result[VertexID,AristoError]=
       if db.mdb.vGen.isSome:
         return ok db.mdb.vGen.unsafeGet
       err(GetIdgNotFound)
@@ -216,10 +216,10 @@ proc putFilFn(db: MemBackendRef): PutFilFn =
 
 proc putIdgFn(db: MemBackendRef): PutIdgFn =
   result =
-    proc(hdl: PutHdlRef; vs: openArray[VertexID])  =
+    proc(hdl: PutHdlRef; vid: VertexID)  =
       let hdl = hdl.getSession db
       if hdl.error.isNil:
-        hdl.vGen = some(vs.toSeq)
+        hdl.vGen = some(vid)
 
 proc putLstFn(db: MemBackendRef): PutLstFn =
   result =
@@ -282,11 +282,7 @@ proc putEndFn(db: MemBackendRef): PutEndFn =
           db.mdb.rFil.del qid
 
       if hdl.vGen.isSome:
-        let vGen = hdl.vGen.unsafeGet
-        if vGen.len == 0:
-          db.mdb.vGen = none(seq[VertexID])
-        else:
-          db.mdb.vGen = some(vGen)
+        db.mdb.vGen = hdl.vGen
 
       if hdl.lSst.isSome:
         db.mdb.lSst = hdl.lSst
@@ -406,7 +402,7 @@ iterator walk*(
   ## Non-decodable entries are stepped over while the counter `n` of the
   ## yield record is still incremented.
   if be.mdb.vGen.isSome:
-    yield(AdmPfx, AdmTabIdIdg.uint64, be.mdb.vGen.unsafeGet.blobify)
+    yield(AdmPfx, AdmTabIdIdg.uint64, [be.mdb.vGen.unsafeGet].blobify)
 
   if not be.mdb.noFq:
     if be.mdb.vFqs.isSome:

nimbus/db/aristo/aristo_init/rocks_db.nim

@@ -133,7 +133,7 @@ proc getFilFn(db: RdbBackendRef): GetFilFn =
 
 proc getIdgFn(db: RdbBackendRef): GetIdgFn =
   result =
-    proc(): Result[seq[VertexID],AristoError]=
+    proc(): Result[VertexID,AristoError]=
 
       # Fetch serialised data record.
       let data = db.rdb.getByPfx(AdmPfx, AdmTabIdIdg.uint64).valueOr:
@@ -143,17 +143,16 @@ proc getIdgFn(db: RdbBackendRef): GetIdgFn =
 
       # Decode data record
       if data.len == 0:
-        let w = EmptyVidSeq   # Must be `let`
-        return ok w           # Compiler error with `ok(EmptyVidSeq)`
+        return ok(VertexID(0))
 
       # Decode data record
-      # TODO vid reuse disabled, implementation too slow since list could have
-      #      millions of entries
-      data.deblobify(seq[VertexID]).map(proc (v: seq[VertexID]): seq[VertexID] =
-        if v.len > 1:
-          @[v[^1]]
+      # TODO Database uses a sequence here because earlier versions of the code
+      #      supported maintaining a list of freed vids
+      data.deblobify(seq[VertexID]).map(proc (v: seq[VertexID]): VertexID =
+        if v.len >= 1:
+          v[^1]
         else:
-          v
+          VertexID(0)
       )
 
 proc getLstFn(db: RdbBackendRef): GetLstFn =
@@ -291,10 +290,10 @@ proc putFilFn(db: RdbBackendRef): PutFilFn =
 
 proc putIdgFn(db: RdbBackendRef): PutIdgFn =
   result =
-    proc(hdl: PutHdlRef; vs: openArray[VertexID])  =
+    proc(hdl: PutHdlRef; vid: VertexID)  =
       let hdl = hdl.getSession db
       if hdl.error.isNil:
-        let data = if 0 < vs.len: vs.blobify else: EmptyBlob
+        let data = if vid.isValid(): [vid].blobify else: EmptyBlob
         db.rdb.putByPfx(AdmPfx, @[(AdmTabIdIdg.uint64, data)]).isOkOr:
           hdl.error = TypedPutHdlErrRef(
             pfx:  AdmPfx,