From f7ac2b0e75e63e65c179aebb640fb950cf47bcca Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Mon, 3 Jun 2024 11:02:28 +0200 Subject: [PATCH] Remove vid (re)use more thoroughly --- nimbus/db/aristo/aristo_blobify.nim | 16 ++-- nimbus/db/aristo/aristo_check/check_be.nim | 52 +------------ nimbus/db/aristo/aristo_check/check_top.nim | 5 +- nimbus/db/aristo/aristo_debug.nim | 17 ++-- nimbus/db/aristo/aristo_delete.nim | 14 ---- nimbus/db/aristo/aristo_desc/desc_backend.nim | 4 +- .../db/aristo/aristo_desc/desc_structural.nim | 4 +- nimbus/db/aristo/aristo_get.nim | 4 +- nimbus/db/aristo/aristo_init/memory_db.nim | 18 ++--- nimbus/db/aristo/aristo_init/rocks_db.nim | 21 +++-- nimbus/db/aristo/aristo_journal.nim | 2 +- nimbus/db/aristo/aristo_layers.nim | 2 +- nimbus/db/aristo/aristo_vid.nim | 77 ++++-------------- tests/test_aristo.nim | 4 - tests/test_aristo/test_backend.nim | 3 +- tests/test_aristo/test_misc.nim | 78 ------------------- 16 files changed, 60 insertions(+), 261 deletions(-) diff --git a/nimbus/db/aristo/aristo_blobify.nim b/nimbus/db/aristo/aristo_blobify.nim index edf94ad31..11a182516 100644 --- a/nimbus/db/aristo/aristo_blobify.nim +++ b/nimbus/db/aristo/aristo_blobify.nim @@ -200,12 +200,11 @@ proc blobifyTo*(filter: FilterRef; data: var Blob): Result[void,AristoError] = data &= filter.src.data data &= filter.trg.data - data &= filter.vGen.len.uint32.toBytesBE + data &= 1.uint32.toBytesBE # Legacy freelist support data &= default(array[4, byte]) # place holder # Store vertex ID generator state - for w in filter.vGen: - data &= w.uint64.toBytesBE + data &= filter.vGen.uint64.toBytesBE var n = 0 @@ -463,7 +462,7 @@ proc deblobify*( return err(DeblobWrongType) (addr lSst.src.data[0]).copyMem(unsafeAddr data[0], 32) (addr lSst.trg.data[0]).copyMem(unsafeAddr data[32], 32) - lSst.serial = uint64.fromBytesBE data[64..72] + lSst.serial = uint64.fromBytesBE data.toOpenArray(64, 72) ok() proc deblobify*( @@ -476,7 +475,7 @@ proc deblobify*( ok move(lSst) -proc deblobify*(data: Blob; filter: var FilterRef): Result[void,AristoError] = +proc deblobify*(data: openArray[byte]; filter: var FilterRef): Result[void,AristoError] = ## De-serialise an Aristo DB filter object if data.len < 80: # minumum length 80 for an empty filter return err(DeblobFilterTooShort) @@ -503,7 +502,8 @@ proc deblobify*(data: Blob; filter: var FilterRef): Result[void,AristoError] = return err(DeblobFilterGenTooShort) for n in 0 ..< nVids: let w = 80 + n * 8 - f.vGen.add (uint64.fromBytesBE data.toOpenArray(int w, int w+7)).VertexID + # Legacy freelist support: use the last item (new version will store at most one) + f.vGen = (uint64.fromBytesBE data.toOpenArray(int w, int w+7)).VertexID var offs = nTrplStart for n in 0 ..< nTriplets: @@ -552,7 +552,7 @@ proc deblobify*(data: Blob; T: type FilterRef): Result[T,AristoError] = ok filter proc deblobify*( - data: Blob; + data: openArray[byte]; vFqs: var seq[(QueueID,QueueID)]; ): Result[void,AristoError] = ## De-serialise the data record encoded with `blobify()` into a filter queue @@ -573,7 +573,7 @@ proc deblobify*( ok() proc deblobify*( - data: Blob; + data: openArray[byte]; T: type seq[(QueueID,QueueID)]; ): Result[T,AristoError] = ## Variant of `deblobify()` for deserialising the vertex ID generator state diff --git a/nimbus/db/aristo/aristo_check/check_be.nim b/nimbus/db/aristo/aristo_check/check_be.nim index 7a9e30dc2..2f14b6dd7 100644 --- a/nimbus/db/aristo/aristo_check/check_be.nim +++ b/nimbus/db/aristo/aristo_check/check_be.nim @@ -11,16 +11,13 @@ {.push raises: [].} import - std/[algorithm, sequtils, sets, tables, typetraits], + std/[algorithm, sets, tables, typetraits], eth/[common, trie/nibbles], stew/interval_set, ../../aristo, ../aristo_walk/persistent, ".."/[aristo_desc, aristo_get, aristo_layers, aristo_serialise] -const - Vid2 = @[VertexID(LEAST_FREE_VID)].toHashSet - # ------------------------------------------------------------------------------ # Private helper # ------------------------------------------------------------------------------ @@ -76,17 +73,6 @@ proc toNodeBE( return ok node return err(vid) -proc vidReorgAlways(vGen: seq[VertexID]): seq[VertexID] = - ## See `vidReorg()`, this one always sorts and optimises - ## - if 1 < vGen.len: - let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it)) - for n in 0 .. lst.len-2: - if lst[n].uint64 != lst[n+1].uint64 + 1: - return lst[n+1 .. lst.len-1] & @[lst[n]] - return @[lst[^1]] - vGen - # ------------------------------------------------------------------------------ # Public functions # ------------------------------------------------------------------------------ @@ -139,25 +125,6 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( return err((vid,CheckBeKeyMismatch)) discard vids.reduce Interval[VertexID,uint64].new(vid,vid) - # Compare calculated state against database state - block: - # Extract vertex ID generator state - let vGen = block: - let rc = db.getIdgBE() - if rc.isOk: - rc.value.vidReorgAlways.toHashSet - elif rc.error == GetIdgNotFound: - EmptyVidSeq.toHashSet - else: - return err((VertexID(0),rc.error)) - let - vGenExpected = vids.to(HashSet[VertexID]) - delta = vGenExpected -+- vGen # symmetric difference - if 0 < delta.len: - # Exclude fringe case when there is a single root vertex only - if vGenExpected != Vid2 or 0 < vGen.len: - return err((delta.toSeq.sorted[^1],CheckBeGarbledVGen)) - # Check top layer cache against backend if cache: let checkKeysOk = block: @@ -222,23 +189,6 @@ proc checkBE*[T: RdbBackendRef|MemBackendRef|VoidBackendRef]( if expected != key: return err((vid,CheckBeCacheKeyMismatch)) - # Check vGen - let - vGen = db.vGen.vidReorgAlways.toHashSet - vGenExpected = vids.to(HashSet[VertexID]) - delta = vGenExpected -+- vGen # symmetric difference - if 0 < delta.len: - if vGen == Vid2 and vGenExpected.len == 0: - # Fringe case when the database is empty - discard - elif vGen.len == 0 and vGenExpected == Vid2: - # Fringe case when there is a single root vertex only - discard - else: - let delta = delta.toSeq - if delta.len != 1 or delta[0] != VertexID(1) or VertexID(1) in vGen: - return err((delta.sorted[^1],CheckBeCacheGarbledVGen)) - ok() # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_check/check_top.nim b/nimbus/db/aristo/aristo_check/check_top.nim index ce41558b5..405dd682e 100644 --- a/nimbus/db/aristo/aristo_check/check_top.nim +++ b/nimbus/db/aristo/aristo_check/check_top.nim @@ -92,8 +92,7 @@ proc checkTopCommon*( let kMapCount = db.layersWalkKey.toSeq.mapIt(it[1]).filterIt(it.isValid).len kMapNilCount = db.layersWalkKey.toSeq.len - kMapCount - vGen = db.vGen.toHashSet - vGenMax = if vGen.len == 0: VertexID(0) else: db.vGen[^1] + vGenMax = db.vGen var stoRoots: HashSet[VertexID] @@ -108,7 +107,7 @@ proc checkTopCommon*( if stoVid.isValid: if stoVid in stoRoots: return err((stoVid,CheckAnyVidSharedStorageRoot)) - if vGenMax.isValid and (vGenMax < stoVid or stoVid in vGen): + if vGenMax.isValid and (vGenMax < stoVid): return err((stoVid,CheckAnyVidDeadStorageRoot)) stoRoots.incl stoVid of Branch: diff --git a/nimbus/db/aristo/aristo_debug.nim b/nimbus/db/aristo/aristo_debug.nim index cc67a306e..380a039df 100644 --- a/nimbus/db/aristo/aristo_debug.nim +++ b/nimbus/db/aristo/aristo_debug.nim @@ -433,8 +433,7 @@ proc ppFilter( result &= pfx & "fid=" & fl.fid.ppFid result &= pfx & "src=" & fl.src.to(HashKey).ppKey(db) result &= pfx & "trg=" & fl.trg.to(HashKey).ppKey(db) - result &= pfx & "vGen" & pfx1 & "[" & - fl.vGen.mapIt(it.ppVid).join(",") & "]" + result &= pfx & "vGen" & pfx1 & "[" & fl.vGen.ppVid & "]" result &= pfx & "sTab" & pfx1 & "{" for n,vid in fl.sTab.sortedKeys: let vtx = fl.sTab.getOrVoid vid @@ -457,11 +456,10 @@ proc ppBe[T](be: T; db: AristoDbRef; limit: int; indent: int): string = var (dump,dataOk) = ("",false) dump &= pfx & "vGen" block: - let q = be.getIdgFn().get(otherwise = EmptyVidSeq) - dump &= "(" & $q.len & ")" - if 0 < q.len: - dataOk = true - dump &= pfx1 & q.ppVidList() + let q = be.getIdgFn().get(otherwise = VertexID(0)) + dump &= "(1)" + dataOk = true + dump &= pfx1 & q.ppVid() block: dump &= pfx & "sTab" var (n, data) = (0, "") @@ -530,9 +528,8 @@ proc ppLayer( result &= "".doPrefix(false) if vGenOk: let - tLen = layer.final.vGen.len - info = "vGen(" & $tLen & ")" - result &= info.doPrefix(0 < tLen) & layer.final.vGen.ppVidList + info = "vGen(1)" + result &= info.doPrefix(true) & layer.final.vGen.ppVid if sTabOk: let tLen = layer.delta.sTab.len diff --git a/nimbus/db/aristo/aristo_delete.nim b/nimbus/db/aristo/aristo_delete.nim index 6a59c6ba9..abb62fa33 100644 --- a/nimbus/db/aristo/aristo_delete.nim +++ b/nimbus/db/aristo/aristo_delete.nim @@ -300,13 +300,6 @@ proc delSubTreeImpl( db.layersPutVtx(VertexID(1), wp.vid, leaf) db.layersResKey(VertexID(1), wp.vid) - # Squeeze list of recycled vertex IDs - # TODO this causes a reallocation of vGen which slows down subsequent - # additions to the list because the sequence must grow which entails a - # full copy in addition to this reorg itself - around block 2.5M this - # causes significant slowdown as the vid list is >1M entries long - # See also EIP-161 which is why there are so many deletions - # db.top.final.vGen = db.vGen.vidReorg() ok() @@ -398,13 +391,6 @@ proc deleteImpl( db.layersPutVtx(VertexID(1), wp.vid, leaf) db.layersResKey(VertexID(1), wp.vid) - # Squeeze list of recycled vertex IDs - # TODO this causes a reallocation of vGen which slows down subsequent - # additions to the list because the sequence must grow which entails a - # full copy in addition to this reorg itself - around block 2.5M this - # causes significant slowdown as the vid list is >1M entries long - # See also EIP-161 which is why there are so many deletions``` - # db.top.final.vGen = db.vGen.vidReorg() ok(emptySubTreeOk) # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_desc/desc_backend.nim b/nimbus/db/aristo/aristo_desc/desc_backend.nim index f0cf371ba..763d0b69b 100644 --- a/nimbus/db/aristo/aristo_desc/desc_backend.nim +++ b/nimbus/db/aristo/aristo_desc/desc_backend.nim @@ -35,7 +35,7 @@ type ## Generic backend database retrieval function for a filter record. GetIdgFn* = - proc(): Result[seq[VertexID],AristoError] {.gcsafe, raises: [].} + proc(): Result[VertexID,AristoError] {.gcsafe, raises: [].} ## Generic backend database retrieval function for a the ID generator ## `Aristo DB` state record. @@ -79,7 +79,7 @@ type ## Generic backend database storage function for filter records. PutIdgFn* = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) + proc(hdl: PutHdlRef; vid: VertexID) {.gcsafe, raises: [].} ## Generic backend database ID generator state storage function. This ## function replaces the current generator state. diff --git a/nimbus/db/aristo/aristo_desc/desc_structural.nim b/nimbus/db/aristo/aristo_desc/desc_structural.nim index 19363e002..d15b1942c 100644 --- a/nimbus/db/aristo/aristo_desc/desc_structural.nim +++ b/nimbus/db/aristo/aristo_desc/desc_structural.nim @@ -85,7 +85,7 @@ type trg*: Hash256 ## Resulting state root (i.e. `kMap[1]`) sTab*: Table[VertexID,VertexRef] ## Filter structural vertex table kMap*: Table[VertexID,HashKey] ## Filter Merkle hash key mapping - vGen*: seq[VertexID] ## Filter unique vertex ID generator + vGen*: VertexID ## Filter unique vertex ID generator LayerDeltaRef* = ref object ## Delta layers are stacked implying a tables hierarchy. Table entries on @@ -123,7 +123,7 @@ type ## pPrf*: HashSet[VertexID] ## Locked vertices (proof nodes) fRpp*: Table[HashKey,VertexID] ## Key lookup for `pPrf[]` (proof nodes) - vGen*: seq[VertexID] ## Recycling state for vertex IDs + vGen*: VertexID ## The first free VertexID dirty*: HashSet[VertexID] ## Start nodes to re-hashiy from LayerRef* = ref LayerObj diff --git a/nimbus/db/aristo/aristo_get.nim b/nimbus/db/aristo/aristo_get.nim index 52949017c..59c6ba15a 100644 --- a/nimbus/db/aristo/aristo_get.nim +++ b/nimbus/db/aristo/aristo_get.nim @@ -24,7 +24,7 @@ import proc getIdgUbe*( db: AristoDbRef; - ): Result[seq[VertexID],AristoError] = + ): Result[VertexID,AristoError] = ## Get the ID generator state from the unfiltered backened if available. let be = db.backend if not be.isNil: @@ -83,7 +83,7 @@ proc getFilUbe*( proc getIdgBE*( db: AristoDbRef; - ): Result[seq[VertexID],AristoError] = + ): Result[VertexID,AristoError] = ## Get the ID generator state the `backened` layer if available. if not db.roFilter.isNil: return ok(db.roFilter.vGen) diff --git a/nimbus/db/aristo/aristo_init/memory_db.nim b/nimbus/db/aristo/aristo_init/memory_db.nim index 8abe41388..34fe6fbb1 100644 --- a/nimbus/db/aristo/aristo_init/memory_db.nim +++ b/nimbus/db/aristo/aristo_init/memory_db.nim @@ -46,7 +46,7 @@ type sTab: Table[VertexID,Blob] ## Structural vertex table making up a trie kMap: Table[VertexID,HashKey] ## Merkle hash key mapping rFil: Table[QueueID,Blob] ## Backend journal filters - vGen: Option[seq[VertexID]] ## ID generator state + vGen: Option[VertexID] ## ID generator state lSst: Option[SavedState] ## Last saved state vFqs: Option[seq[(QueueID,QueueID)]] noFq: bool ## No filter queues available @@ -59,7 +59,7 @@ type sTab: Table[VertexID,Blob] kMap: Table[VertexID,HashKey] rFil: Table[QueueID,Blob] - vGen: Option[seq[VertexID]] + vGen: Option[VertexID] lSst: Option[SavedState] vFqs: Option[seq[(QueueID,QueueID)]] @@ -128,7 +128,7 @@ proc getFilFn(db: MemBackendRef): GetFilFn = proc getIdgFn(db: MemBackendRef): GetIdgFn = result = - proc(): Result[seq[VertexID],AristoError]= + proc(): Result[VertexID,AristoError]= if db.mdb.vGen.isSome: return ok db.mdb.vGen.unsafeGet err(GetIdgNotFound) @@ -216,10 +216,10 @@ proc putFilFn(db: MemBackendRef): PutFilFn = proc putIdgFn(db: MemBackendRef): PutIdgFn = result = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) = + proc(hdl: PutHdlRef; vid: VertexID) = let hdl = hdl.getSession db if hdl.error.isNil: - hdl.vGen = some(vs.toSeq) + hdl.vGen = some(vid) proc putLstFn(db: MemBackendRef): PutLstFn = result = @@ -282,11 +282,7 @@ proc putEndFn(db: MemBackendRef): PutEndFn = db.mdb.rFil.del qid if hdl.vGen.isSome: - let vGen = hdl.vGen.unsafeGet - if vGen.len == 0: - db.mdb.vGen = none(seq[VertexID]) - else: - db.mdb.vGen = some(vGen) + db.mdb.vGen = hdl.vGen if hdl.lSst.isSome: db.mdb.lSst = hdl.lSst @@ -406,7 +402,7 @@ iterator walk*( ## Non-decodable entries are stepped over while the counter `n` of the ## yield record is still incremented. if be.mdb.vGen.isSome: - yield(AdmPfx, AdmTabIdIdg.uint64, be.mdb.vGen.unsafeGet.blobify) + yield(AdmPfx, AdmTabIdIdg.uint64, [be.mdb.vGen.unsafeGet].blobify) if not be.mdb.noFq: if be.mdb.vFqs.isSome: diff --git a/nimbus/db/aristo/aristo_init/rocks_db.nim b/nimbus/db/aristo/aristo_init/rocks_db.nim index c1f69dcc5..6bb8da670 100644 --- a/nimbus/db/aristo/aristo_init/rocks_db.nim +++ b/nimbus/db/aristo/aristo_init/rocks_db.nim @@ -133,7 +133,7 @@ proc getFilFn(db: RdbBackendRef): GetFilFn = proc getIdgFn(db: RdbBackendRef): GetIdgFn = result = - proc(): Result[seq[VertexID],AristoError]= + proc(): Result[VertexID,AristoError]= # Fetch serialised data record. let data = db.rdb.getByPfx(AdmPfx, AdmTabIdIdg.uint64).valueOr: @@ -143,17 +143,16 @@ proc getIdgFn(db: RdbBackendRef): GetIdgFn = # Decode data record if data.len == 0: - let w = EmptyVidSeq # Must be `let` - return ok w # Compiler error with `ok(EmptyVidSeq)` + return ok(VertexID(0)) # Decode data record - # TODO vid reuse disabled, implementation too slow since list could have - # millions of entries - data.deblobify(seq[VertexID]).map(proc (v: seq[VertexID]): seq[VertexID] = - if v.len > 1: - @[v[^1]] + # TODO Database uses a sequence here because earlier versions of the code + # supported maintaining a list of freed vids + data.deblobify(seq[VertexID]).map(proc (v: seq[VertexID]): VertexID = + if v.len >= 1: + v[^1] else: - v + VertexID(0) ) proc getLstFn(db: RdbBackendRef): GetLstFn = @@ -291,10 +290,10 @@ proc putFilFn(db: RdbBackendRef): PutFilFn = proc putIdgFn(db: RdbBackendRef): PutIdgFn = result = - proc(hdl: PutHdlRef; vs: openArray[VertexID]) = + proc(hdl: PutHdlRef; vid: VertexID) = let hdl = hdl.getSession db if hdl.error.isNil: - let data = if 0 < vs.len: vs.blobify else: EmptyBlob + let data = if vid.isValid(): [vid].blobify else: EmptyBlob db.rdb.putByPfx(AdmPfx, @[(AdmTabIdIdg.uint64, data)]).isOkOr: hdl.error = TypedPutHdlErrRef( pfx: AdmPfx, diff --git a/nimbus/db/aristo/aristo_journal.nim b/nimbus/db/aristo/aristo_journal.nim index c8bee6baf..f43b887af 100644 --- a/nimbus/db/aristo/aristo_journal.nim +++ b/nimbus/db/aristo/aristo_journal.nim @@ -58,7 +58,7 @@ proc journalFwdFilter*( src: srcRoot, sTab: layer.delta.sTab, kMap: layer.delta.kMap, - vGen: layer.final.vGen.vidReorg, # Compact recycled IDs + vGen: layer.final.vGen, trg: trgRoot) # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_layers.nim b/nimbus/db/aristo/aristo_layers.nim index 5810da3e0..bf6e6a889 100644 --- a/nimbus/db/aristo/aristo_layers.nim +++ b/nimbus/db/aristo/aristo_layers.nim @@ -35,7 +35,7 @@ func dirty*(db: AristoDbRef): lent HashSet[VertexID] = func pPrf*(db: AristoDbRef): lent HashSet[VertexID] = db.top.final.pPrf -func vGen*(db: AristoDbRef): lent seq[VertexID] = +func vGen*(db: AristoDbRef): VertexID = db.top.final.vGen # ------------------------------------------------------------------------------ diff --git a/nimbus/db/aristo/aristo_vid.nim b/nimbus/db/aristo/aristo_vid.nim index adba964cb..55daecf0e 100644 --- a/nimbus/db/aristo/aristo_vid.nim +++ b/nimbus/db/aristo/aristo_vid.nim @@ -14,7 +14,7 @@ {.push raises: [].} import - std/[algorithm, sequtils, typetraits], + std/[typetraits], "."/[aristo_desc, aristo_layers] # ------------------------------------------------------------------------------ @@ -26,76 +26,31 @@ proc vidFetch*(db: AristoDbRef; pristine = false): VertexID = ## list where the top entry *ID* has the property that any other *ID* larger ## is also not used on the database. ## - ## The function prefers to return recycled vertex *ID*s if there are any. - ## When the argument `pristine` is set `true`, the function guarantees to - ## return a non-recycled, brand new vertex *ID* which is the preferred mode - ## when creating leaf vertices. + ## In most cases, this function will return a vid that's never been used, + ## rolled back transactions and some disposals may however result in reuse. ## - if db.vGen.len == 0: + ## Earlier versions of this code maintained a free-list for recycling - this + ## has been removed as maintenance of the free-list caused significant + ## overhead due to implementation issues - a new implementation should be + ## evaluated also from a database key perspective (since there may be + ## advantages to using monotonically increasing keys) + ## + if db.vGen.uint64 < LEAST_FREE_VID: # Note that `VertexID(1)` is the root of the main trie - db.top.final.vGen = @[VertexID(LEAST_FREE_VID+1)] + db.top.final.vGen = VertexID(LEAST_FREE_VID + 1) result = VertexID(LEAST_FREE_VID) - elif db.vGen.len == 1 or pristine: - result = db.vGen[^1] - db.top.final.vGen[^1] = result + 1 else: - result = db.vGen[^2] - db.top.final.vGen[^2] = db.top.final.vGen[^1] - db.top.final.vGen.setLen(db.vGen.len-1) - doAssert LEAST_FREE_VID <= result.distinctBase - + result = db.top.final.vGen + db.top.final.vGen = VertexID(db.top.final.vGen.uint64 + 1) proc vidDispose*(db: AristoDbRef; vid: VertexID) = ## Recycle the argument `vtxID` which is useful after deleting entries from ## the vertex table to prevent the `VertexID` type key values small. ## if LEAST_FREE_VID <= vid.distinctBase: - if db.vGen.len == 0: - db.top.final.vGen = @[vid] - else: - let topID = db.vGen[^1] - # Only store smaller numbers: all numberts larger than `topID` - # are free numbers - # TODO vid reuse disabled, implementation too slow since list could grow - # to millions of entries - # if vid < topID: - # db.top.final.vGen[^1] = vid - # db.top.final.vGen.add topID - if vid == topID - 1: # no gap - can recycle - db.top.final.vGen[^1] = vid - -proc vidReorg*(vGen: seq[VertexID]): seq[VertexID] = - ## Return a compacted version of the argument vertex ID generator state - ## `vGen`. The function removes redundant items from the recycle queue and - ## orders it in a way so that smaller `VertexID` numbers are re-used first. - ## - # Apply heuristic test to avoid unnecessary sorting - var reOrgOk = false - if 2 < vGen.len and vGen[0] < vGen[^2]: - if vGen.len < 10: - reOrgOk = true - elif vGen[0] < vGen[1] and vGen[^3] < vGen[^2]: - reOrgOk = true - - if reOrgOk: - let lst = vGen.mapIt(uint64(it)).sorted(Descending).mapIt(VertexID(it)) - for n in 0 .. lst.len-2: - if lst[n].uint64 != lst[n+1].uint64 + 1: - # All elements of the sequence `lst[0]`..`lst[n]` are in decreasing - # order with distance 1. Only the smallest item is needed and the - # rest can be removed (as long as distance is 1.) - # - # Example: - # 7, 6, 5, 3.. => 5, 3.. => @[3..] & @[5] - # ^ - # | - # n - # - return lst[n+1 .. lst.len-1] & @[lst[n]] - # Entries decrease continuously - return @[lst[^1]] - - vGen + if vid.uint64 == (db.vGen.uint64 + 1): + # We can recycle the VertexID as long as there is no gap + db.top.final.vGen = vid # ------------------------------------------------------------------------------ # End diff --git a/tests/test_aristo.nim b/tests/test_aristo.nim index 1028137cf..8433a6e54 100644 --- a/tests/test_aristo.nim +++ b/tests/test_aristo.nim @@ -79,10 +79,6 @@ proc miscRunner( let (lyo,qidSampleSize) = layout suite "Aristo: Miscellaneous tests": - - test "VertexID recyling lists": - check noisy.testVidRecycleLists() - test &"Low level cascaded fifos API (sample size: {qidSampleSize})": check noisy.testQidScheduler(layout = lyo, sampleSize = qidSampleSize) diff --git a/tests/test_aristo/test_backend.nim b/tests/test_aristo/test_backend.nim index 36519907c..88740ce13 100644 --- a/tests/test_aristo/test_backend.nim +++ b/tests/test_aristo/test_backend.nim @@ -50,8 +50,7 @@ func hash(filter: FilterRef): Hash = h = h !& filter.src.hash h = h !& filter.trg.hash - for w in filter.vGen.vidReorg: - h = h !& w.uint64.hash + h = h !& filter.vGen.uint64.hash for w in filter.sTab.keys.toSeq.mapIt(it.uint64).sorted.mapIt(it.VertexID): let data = filter.sTab.getOrVoid(w).blobify.get(otherwise = EmptyBlob) diff --git a/tests/test_aristo/test_misc.nim b/tests/test_aristo/test_misc.nim index 0d2f830f2..c1b405904 100644 --- a/tests/test_aristo/test_misc.nim +++ b/tests/test_aristo/test_misc.nim @@ -256,84 +256,6 @@ proc validate(db: QTabRef; scd: QidSchedRef; serial: int; relax: bool): bool = # Public test function # ------------------------------------------------------------------------------ -proc testVidRecycleLists*(noisy = true; seed = 42): bool = - ## Transcode VID lists held in `AristoDb` descriptor - ## - var td = TesterDesc.init seed - let db = AristoDbRef.init() - - # Add some randum numbers - block: - let first = td.vidRand() - db.vidDispose first - - var - expectedVids = 1 - count = 1 - # Feed some numbers used and some discaded - while expectedVids < 5 or count < 5 + expectedVids: - count.inc - let vid = td.vidRand() - expectedVids += (vid < first).ord - db.vidDispose vid - - xCheck db.vGen.len == expectedVids: - noisy.say "***", "vids=", db.vGen.len, " discarded=", count-expectedVids - - # Serialise/deserialise - block: - let dbBlob = db.vGen.blobify - - # Deserialise - let - db1 = AristoDbRef.init() - rc = dbBlob.deblobify seq[VertexID] - xCheckRc rc.error == 0 - db1.top.final.vGen = rc.value - - xCheck db.vGen == db1.vGen - - # Make sure that recycled numbers are fetched first - let topVid = db.vGen[^1] - while 1 < db.vGen.len: - let w = db.vidFetch() - xCheck w < topVid - xCheck db.vGen.len == 1 and db.vGen[0] == topVid - - # Get some consecutive vertex IDs - for n in 0 .. 5: - let w = db.vidFetch() - xCheck w == topVid + n - xCheck db.vGen.len == 1 - - # Repeat last test after clearing the cache - db.top.final.vGen.setLen(0) - for n in 0 .. 5: - let w = db.vidFetch() - xCheck w == VertexID(LEAST_FREE_VID) + n # VertexID(1) is default root ID - xCheck db.vGen.len == 1 - - # Recycling and re-org tests - func toVQ(a: seq[int]): seq[VertexID] = a.mapIt(VertexID(LEAST_FREE_VID+it)) - - # Heuristic prevents from re-org - xCheck @[8, 7, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 3, 4, 5, 9] .toVQ - xCheck @[8, 7, 6, 3, 4, 5, 9] .toVQ.vidReorg == @[8, 7, 6, 3, 4, 5, 9].toVQ - xCheck @[5, 4, 3, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - xCheck @[5] .toVQ.vidReorg == @[5] .toVQ - xCheck @[3, 5] .toVQ.vidReorg == @[3, 5] .toVQ - xCheck @[4, 5] .toVQ.vidReorg == @[4, 5] .toVQ - - # performing re-org - xCheck @[5, 7, 3, 4, 8, 9] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - xCheck @[5, 7, 6, 3, 4, 8, 9] .toVQ.vidReorg == @[3] .toVQ - xCheck @[3, 4, 5, 7] .toVQ.vidReorg == @[5, 4, 3, 7] .toVQ - - xCheck newSeq[VertexID](0).vidReorg().len == 0 - - true - - proc testQidScheduler*( noisy = true; layout = LyoSamples[0][0];