Merge pull request #859 from SmithSamuelM/fix_del_ddos

Extensive changes to the delegation validation and escrow logic. Extensive refactoring of the classes in subing.py

src/keri/app/cli/commands/escrow.py

@@ -56,8 +56,8 @@ def escrows(tymth, tock=0.0, **opts):
                 oots = list()
                 key = ekey = b''  # both start same. when not same means escrows found
                 while True:
-                    for ekey, edig in hby.db.getOoeItemsNextIter(key=key):
-                        pre, sn = dbing.splitKeySN(ekey)  # get pre and sn from escrow item
+                    for ekey, edig in hby.db.getOoeItemIter(key=key):
+                        pre, sn = dbing.splitSnKey(ekey)  # get pre and sn from escrow item
 
                         try:
                             oots.append(eventing.loadEvent(hby.db, pre, edig))
@@ -74,8 +74,8 @@ def escrows(tymth, tock=0.0, **opts):
                 pwes = list()
                 key = ekey = b''  # both start same. when not same means escrows found
                 while True:  # break when done
-                    for ekey, edig in hby.db.getPweItemsNextIter(key=key):
-                        pre, sn = dbing.splitKeySN(ekey)  # get pre and sn from escrow item
+                    for ekey, edig in hby.db.getPweItemIter(key=key):
+                        pre, sn = dbing.splitSnKey(ekey)  # get pre and sn from escrow item
 
                         try:
                             pwes.append(eventing.loadEvent(hby.db, pre, edig))
@@ -92,8 +92,8 @@ def escrows(tymth, tock=0.0, **opts):
                 pses = list()
                 key = ekey = b''  # both start same. when not same means escrows found
                 while True:  # break when done
-                    for ekey, edig in hby.db.getPseItemsNextIter(key=key):
-                        pre, sn = dbing.splitKeySN(ekey)  # get pre and sn from escrow item
+                    for ekey, edig in hby.db.getPseItemIter(key=key):
+                        pre, sn = dbing.splitSnKey(ekey)  # get pre and sn from escrow item
 
                         try:
                             pses.append(eventing.loadEvent(hby.db, pre, edig))
@@ -110,8 +110,8 @@ def escrows(tymth, tock=0.0, **opts):
                 ldes = list()
                 key = ekey = b''  # both start same. when not same means escrows found
                 while True:  # break when done
-                    for ekey, edig in hby.db.getLdeItemsNextIter(key=key):
-                        pre, sn = dbing.splitKeySN(ekey)  # get pre and sn from escrow item
+                    for ekey, edig in hby.db.getLdeItemIter(key=key):
+                        pre, sn = dbing.splitSnKey(ekey)  # get pre and sn from escrow item
 
                         try:
                             ldes.append(eventing.loadEvent(hby.db, pre, edig))

src/keri/app/cli/commands/ipex/grant.py

@@ -111,8 +111,8 @@ def grantDo(self, tymth, tock=0.0):
         iserder = serdering.SerderKERI(raw=bytes(iss))
         seqner = coring.Seqner(sn=iserder.sn)
 
-        serder = self.hby.db.findAnchoringSealEvent(creder.sad['i'],
-                                                    seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said))
+        serder = self.hby.db.fetchAllSealingEventByEventSeal(creder.sad['i'],
+                        seal=dict(i=iserder.pre, s=seqner.snh, d=iserder.said))
         anc = self.hby.db.cloneEvtMsg(pre=serder.pre, fn=0, dig=serder.said)
 
         exn, atc = protocoling.ipexGrantExn(hab=self.hab, recp=recp, message=self.message, acdc=acdc, iss=iss, anc=anc,

src/keri/app/cli/commands/migrate.py

@@ -148,7 +148,7 @@ def migrateKeys(db):
     digs = subing.CatCesrIoSetSuber(db=db, subkey="digs.",
                                     klas=(coring.Prefixer, coring.Seqner))
 
-    for pre, fn, dig in db.getFelItemAllPreIter(key=b''):
+    for pre, fn, dig in db.getFelItemAllPreIter():
         dgkey = dbing.dgKey(pre, dig)  # get message
         if not (raw := db.getEvt(key=dgkey)):
             raise kering.MissingEntryError("Missing event for dig={}.".format(dig))

src/keri/app/connecting.py

@@ -201,7 +201,7 @@ def setImg(self, pre, typ, stream):
             stream (file): file-like stream of image data
 
         """
-        self.hby.db.delTopVal(db=self.hby.db.imgs, key=pre.encode("utf-8"))
+        self.hby.db.delTopVal(db=self.hby.db.imgs, top=pre.encode("utf-8"))
 
         key = f"{pre}.content-type".encode("utf-8")
         self.hby.db.setVal(db=self.hby.db.imgs, key=key, val=typ.encode("utf-8"))

src/keri/app/delegating.py

@@ -138,7 +138,7 @@ def processUnanchoredEscrow(self):
             dkever = self.hby.kevers[kever.delpre]
 
             seal = dict(i=serder.pre, s=serder.snh, d=serder.said)
-            if dserder := self.hby.db.findAnchoringSealEvent(dkever.prefixer.qb64, seal=seal):
+            if dserder := self.hby.db.fetchAllSealingEventByEventSeal(dkever.prefixer.qb64, seal=seal):
                 seqner = coring.Seqner(sn=dserder.sn)
                 couple = seqner.qb64b + dserder.saidb
                 dgkey = dbing.dgKey(kever.prefixer.qb64b, kever.serder.saidb)

src/keri/app/grouping.py

@@ -171,7 +171,8 @@ def processDelegateEscrow(self):
                     self.hby.db.cgms.put(keys=(pre, seqner.qb64), val=saider)
 
             else:  # Not witnesser, we need to look for the anchor and then wait for receipts
-                if serder := self.hby.db.findAnchoringSealEvent(kever.delpre, seal=anchor):
+                if serder := self.hby.db.fetchAllSealingEventByEventSeal(kever.delpre,
+                                                                         seal=anchor):
                     aseq = coring.Seqner(sn=serder.sn)
                     couple = aseq.qb64b + serder.saidb
                     dgkey = dbing.dgKey(pre, saider.qb64b)

src/keri/app/habbing.py

@@ -1538,7 +1538,7 @@ def replay(self, pre=None, fn=0):
 
         return msgs
 
-    def replayAll(self, key=b''):
+    def replayAll(self):
         """
         Returns replay of FEL first seen event log for all pre starting at key
 
@@ -1547,7 +1547,7 @@ def replayAll(self, key=b''):
 
         """
         msgs = bytearray()
-        for msg in self.db.cloneAllPreIter(key=key):
+        for msg in self.db.cloneAllPreIter():
             msgs.extend(msg)
         return msgs
 
@@ -2284,6 +2284,9 @@ def make(self, *, secrecies=None, iridx=0, code=coring.MtrDex.Blake3_256, dcode=
         # may want db method that updates .habs. and .prefixes together
         habord = basing.HabitatRecord(hid=self.pre, name=self.name, domain=self.ns)
 
+        # must add self.pre to self.prefixes before calling processEvent so that
+        # Kever.locallyOwned or Kever.locallyDelegated or Kever.locallyWitnessed
+        # evaluates correctly when processing own inception event.
         if not hidden:
             self.save(habord)
             self.prefixes.add(self.pre)

src/keri/app/notifying.py

@@ -184,7 +184,8 @@ def getItemIter(self, keys: Union[str, Iterable] = b""):
             each entry in db
 
         """
-        for key, val in self.db.getAllItemIter(db=self.sdb, key=self._tokey(keys), split=False):
+        for key, val in self.db.getTopItemIter(db=self.sdb,
+                                               top=self._tokey(keys)):
             yield self._tokeys(key), self.klas(raw=bytes(val))
 
     def cntAll(self):

src/keri/app/querying.py

@@ -141,7 +141,7 @@ def recur(self, tyme, deeds=None):
             return False
 
         kever = self.hab.kevers[self.pre]
-        if self.hby.db.findAnchoringSealEvent(self.pre, seal=self.anchor):
+        if self.hby.db.fetchAllSealingEventByEventSeal(self.pre, seal=self.anchor):
             self.remove([self.witq])
             return True
 

src/keri/app/storing.py

@@ -34,6 +34,17 @@ def __init__(self, name="mbx", headDirPath=None, reopen=True, **kwa):
             perm:
             reopen:
             kwa:
+
+        Mailboxer uses two dbs for mailbox messages these are .tpcs and .msgs.
+        The message index is in .tpcs (topics).
+            Each .tpcs index key consists of topic.on where topic is bytes
+                identifier or prefix/topic for message and on is serialized
+                ordinal number to orders the appearance of a topic message.
+            Eash .tpcs val is the digest of the message.
+        The message itself is stored in .msgs where the key is the msg digest
+            and the value is the serialized messag itself.
+        Multiple messages can share the same topic but with a different ordinal.
+
         """
         self.tpcs = None
         self.msgs = None
@@ -47,88 +58,102 @@ def reopen(self, **kwa):
         :return:
         """
         super(Mailboxer, self).reopen(**kwa)
-
-        self.tpcs = self.env.open_db(key=b'tpcs.', dupsort=True)
+        self.tpcs = subing.OnSuber(db=self, subkey='tpcs.')
         self.msgs = subing.Suber(db=self, subkey='msgs.')  # key states
 
         return self.env
 
-    def delTopic(self, key):
-        """
-        Use snKey()
-        Deletes value at key.
-        Returns True If key exists in database Else False
+    def delTopic(self, key, on=0):
+        """Removes topic index from .tpcs without deleting message from .msgs
+
+        Returns:
+            result (boo): True if full key consisting of key and serialized on
+                             exists in database so removed
+                          False otherwise (not removed)
         """
-        return self.delIoSetVals(self.tpcs, key)
+        return self.tpcs.remOn(keys=key, on=on)
 
     def appendToTopic(self, topic, val):
-        """
-        Return first seen order number int, fn, of appended entry.
-        Computes fn as next fn after last entry.
+        """Appends val to end of db entries with same topic but with on
+        incremented by 1 relative to last preexisting entry at topic.
 
-        Append val to end of db entries with same topic but with fn incremented by
-        1 relative to last preexisting entry at pre.
+        Returns:
+            on (int): order number int, on, of appended entry.
+                      Computes on as next on after last entry.
 
         Parameters:
-            topic is bytes identifier prefix/topic for message
-            val is event digest
+            topic (bytes):  topic identifier for message
+            val (bytes): msg digest
         """
-        return self.appendIoSetVal(db=self.tpcs, key=topic, val=val)
+        return self.tpcs.appendOn(key=topic, val=val)
+
 
     def getTopicMsgs(self, topic, fn=0):
         """
         Returns:
-             ioset (oset): the insertion ordered set of values at same apparent
-             effective key.
-             Uses hidden ordinal key suffix for insertion ordering.
-             The suffix is appended and stripped transparently.
+            msgs (Iterable[bytes]): belonging to topic indices with same topic but all
+                on >= fn i.e. all topic.on beginning with fn
 
          Parameters:
-             topic (Option(bytes|str)): Apparent effective key
-             fn (int) starting index
+             topic (Option(bytes|str)): key prefix combined with serialized on
+                    to form full actual key.  When key is empty then retrieves
+                    whole database.
+             fn (int): starting index ordinal number used with onKey(pre,on)
+                    to form key at at which to initiate retrieval
         """
-        if hasattr(topic, "encode"):
-            topic = topic.encode("utf-8")
-
-        digs = self.getIoSetVals(db=self.tpcs, key=topic, ion=fn)
         msgs = []
-        for dig in digs:
+        for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn):
             if msg := self.msgs.get(keys=dig):
-                msgs.append(msg.encode("utf-8"))
+                msgs.append(msg.encode())  # want bytes not str
         return msgs
 
+
     def storeMsg(self, topic, msg):
         """
-        Add exn event to mailbox of dest identifier
+        Add exn event to mailbox topic and on that is 1 greater than last msg
+        at topic.
+
+        Returns:
+            result (bool): True if msg successfully stored and indexed at topic
+                           False otherwise
 
         Parameters:
-            msg (bytes):
-            topic (qb64b):
+            topic (str | bytes):  topic (Option(bytes|str)): key prefix combined
+                with serialized on to form full actual key.
+            msg (bytes): serialized message
 
         """
-        if hasattr(topic, "encode"):
-            topic = topic.encode("utf-8")
-
         if hasattr(msg, "encode"):
             msg = msg.encode("utf-8")
 
         digb = coring.Diger(ser=msg, code=MtrDex.Blake3_256).qb64b
-        self.appendToTopic(topic=topic, val=digb)
+        on = self.tpcs.appendOn(keys=topic, val=digb)
         return self.msgs.pin(keys=digb, val=msg)
 
+
     def cloneTopicIter(self, topic, fn=0):
         """
-        Returns iterator of first seen exn messages with attachments for the
-        identifier prefix pre starting at first seen order number, fn.
+        Returns:
+            triple (Iterator[(on, topic, msg): iterator of messages at topic
+            beginning with ordinal fn.
 
-        """
-        if hasattr(topic, 'encode'):
-            topic = topic.encode("utf-8")
+        topic (Option(bytes|str)): key prefix combined with serialized on
+                    to form full actual key.  When key is empty then retrieves
+                    whole database.
+             fn (int): starting index ordinal number used with onKey(pre,on)
+                    to form key at at which to initiate retrieval
 
-        for (key, dig) in self.getIoSetItemsIter(self.tpcs, key=topic, ion=fn):
-            topic, ion = dbing.unsuffix(key)
+
+
+        ToDo looks like misuse of IoSet this should not be IoSet but simply
+        Ordinal Numbered db.  since should not be using hidden ion has not
+        hidden.
+
+        """
+        for keys, on, dig in self.tpcs.getOnItemIter(keys=topic, on=fn):
             if msg := self.msgs.get(keys=dig):
-                yield ion, topic, msg.encode("utf-8")
+                yield (on, topic, msg.encode("utf-8"))
+
 
 
 class Respondant(doing.DoDoer):