New and updated kli commands for witnesss, watchers and mailboxes. De…

…legation fixes (#805)

* New kli commands for listing witnesss, watchers and mailboxes.

Update to kli command `kli watcher add` to use new BADA-RUN protected data structures for tracking AIDs being observed by a watcher on behalf of a controller

New kli command to perform key event adjudication across the set of watchers watching an AID.  This command will retrieve updated key state if a threshold satisfying number watchers response with non-duplicitous key state change.

Addition to rotate and delegate confirm to all the specification of witness auth code time stamps.

Update to delegation processing to propagate the delegator anchor event to witnesses after delegation approval

New databases to track observed AIDs (observed by watchers) and delegation propogation escrow.

Bug fix in `kli mailbox debug` to account for empty local state.

Signed-off-by: pfeairheller <[email protected]>

* Factoring several classes and methods into a watching package including a new Adjudicator class that can be used outside of the KERIpy command line.

Signed-off-by: pfeairheller <[email protected]>

* Added tests for Adjudicator and diffState

Signed-off-by: pfeairheller <[email protected]>

---------

Signed-off-by: pfeairheller <[email protected]>

src/keri/app/agenting.py

@@ -581,6 +581,7 @@ def __init__(self, hby, msgs=None, cues=None, **kwa):
 
         """
         self.hby = hby
+        self.posted = 0
         self.msgs = msgs if msgs is not None else decking.Deck()
         self.cues = cues if cues is not None else decking.Deck()
         super(WitnessPublisher, self).__init__(doers=[doing.doify(self.sendDo)], **kwa)
@@ -599,6 +600,7 @@ def sendDo(self, tymth=None, tock=0.0, **opts):
         while True:
             while self.msgs:
                 evt = self.msgs.popleft()
+                self.posted += 1
                 pre = evt["pre"]
                 msg = evt["msg"]
 
@@ -642,6 +644,10 @@ def sent(self, said):
 
         return False
 
+    @property
+    def idle(self):
+        return len(self.msgs) == 0 and self.posted == len(self.cues)
+
 
 class TCPMessenger(doing.DoDoer):
     """ Send events to witnesses for receipting using TCP direct connection

src/keri/app/cli/commands/delegate/confirm.py

@@ -37,6 +37,7 @@
                     action='store_true')
 parser.add_argument('--code', help='<Witness AID>:<code> formatted witness auth codes.  Can appear multiple times',
                     default=[], action="append", required=False)
+parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False)
 
 
 def confirm(args):
@@ -54,16 +55,18 @@ def confirm(args):
     auto = args.auto
     authenticate = args.authenticate
     codes = args.code
+    codeTime = args.code_time
 
     confirmDoer = ConfirmDoer(name=name, base=base, alias=alias, bran=bran, interact=interact, auto=auto,
-                              authenticate=authenticate, codes=codes)
+                              authenticate=authenticate, codes=codes, codeTime=codeTime)
 
     doers = [confirmDoer]
     return doers
 
 
 class ConfirmDoer(doing.DoDoer):
-    def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None):
+    def __init__(self, name, base, alias, bran, interact=False, auto=False, authenticate=False, codes=None,
+                 codeTime=None):
         hby = existing.setupHby(name=name, base=base, bran=bran)
         self.hbyDoer = habbing.HaberyDoer(habery=hby)  # setup doer
         self.witq = agenting.WitnessInquisitor(hby=hby)
@@ -73,6 +76,7 @@ def __init__(self, name, base, alias, bran, interact=False, auto=False, authenti
         self.mux = grouping.Multiplexor(hby=hby, notifier=self.notifier)
         self.authenticate = authenticate
         self.codes = codes if codes is not None else []
+        self.codeTime = codeTime
 
         exc = exchanging.Exchanger(hby=hby, handlers=[])
         delegating.loadHandlers(hby=hby, exc=exc, notifier=self.notifier)
@@ -185,9 +189,11 @@ def confirmDo(self, tymth, tock=0.0):
 
                         auths = {}
                         if self.authenticate:
+                            codeTime = helping.fromIso8601(
+                                self.codeTime) if self.codeTime is not None else helping.nowIso8601()
                             for arg in self.codes:
                                 (wit, code) = arg.split(":")
-                                auths[wit] = f"{code}#{helping.nowIso8601()}"
+                                auths[wit] = f"{code}#{codeTime}"
 
                             for wit in hab.kever.wits:
                                 if wit in auths:

src/keri/app/cli/commands/local/watch.py

@@ -7,14 +7,13 @@
 import random
 import sys
 import time
-from collections import namedtuple
 
 from hio import help
 from hio.base import doing
 from keri.app import agenting, indirecting, habbing, forwarding
 from keri.app.cli.common import existing, terming
 from keri.app.habbing import GroupHab
-from keri.core import coring
+from keri.app.watching import States, diffState
 
 logger = help.ogler.getLogger()
 
@@ -31,17 +30,6 @@
 parser.add_argument('--aeid', help='qualified base64 of non-transferable identifier prefix for  authentication '
                                    'and encryption of secrets in keystore', default=None)
 
-Stateage = namedtuple("Stateage", 'even ahead behind duplicitous')
-
-States = Stateage(even="even", ahead="ahead", behind="behind", duplicitous="duplicitous")
-
-
-class WitnessState:
-    wit: str
-    state: Stateage
-    sn: int
-    dig: str
-
 
 def watch(args):
     name = args.name
@@ -135,7 +123,7 @@ def watchDo(self, tymth, tock=0.0, **opts):
                 mystate = hab.kever.state()
                 witstate = hab.db.ksns.get((saider.qb64,))
 
-                states.append(self.diffState(wit, mystate, witstate))
+                states.append(diffState(wit, mystate, witstate))
 
             # First check for any duplicity, if so get out of here
             dups = [state for state in states if state.state == States.duplicitous]
@@ -213,31 +201,3 @@ def cueDo(self, tymth, tock=0.0, **opts):
 
                 yield self.tock
             yield self.tock
-
-    @staticmethod
-    def diffState(wit, preksn, witksn):
-
-        witstate = WitnessState()
-        witstate.wit = wit
-        mysn = int(preksn.s, 16)
-        mydig = preksn.d
-        witstate.sn = int(witksn.f, 16)
-        witstate.dig = witksn.d
-
-        # At the same sequence number, check the DIGs
-        if mysn == witstate.sn:
-            if mydig == witstate.dig:
-                witstate.state = States.even
-            else:
-                witstate.state = States.duplicitous
-
-        # This witness is behind and will need to be caught up.
-        elif mysn > witstate.sn:
-            witstate.state = States.behind
-
-        # mysn < witstate.sn - We are behind this witness (multisig or restore situation).
-        # Must ensure that controller approves this event or a recovery rotation is needed
-        else:
-            witstate.state = States.ahead
-
-        return witstate

src/keri/app/cli/commands/mailbox/debug.py

@@ -85,7 +85,7 @@ def readDo(self, tymth, tock=0.0):
 
         hab = self.hby.habByName(name=self.alias)
         topics = {"/receipt": 0, "/replay": 0, "/multisig": 0, "/credential": 0, "/delegate": 0, "/challenge": 0,
-                  "/oobi": 0}
+                  "/oobi": 0, "/reply": 0}
         try:
             client, clientDoer = agenting.httpClient(hab, self.witness)
         except kering.MissingEntryError as e:
@@ -95,8 +95,11 @@ def readDo(self, tymth, tock=0.0):
 
         print("Local Index per Topic")
         witrec = hab.db.tops.get((hab.pre, self.witness))
-        for topic in witrec.topics:
-            print(f"   Topic {topic}:   {witrec.topics[topic]}")
+        if witrec:
+            for topic in witrec.topics:
+                print(f"   Topic {topic}:   {witrec.topics[topic]}")
+        else:
+            print("\tNo local index")
         print()
 
         q = dict(pre=hab.pre, topics=topics)
@@ -107,7 +110,7 @@ def readDo(self, tymth, tock=0.0):
 
         httping.createCESRRequest(msg, client, dest=self.witness)
 
-        while client.requests:
+        while client.requests or (not client.events and not client.requests):
             yield self.tock
 
         yield 1.0

src/keri/app/cli/commands/mailbox/list.py

@@ -0,0 +1,68 @@
+# -*- encoding: utf-8 -*-
+"""
+KERI
+keri.kli.commands module
+
+"""
+import argparse
+
+from hio import help
+from hio.base import doing
+
+from keri.app import connecting
+from keri.app.cli.common import existing
+from keri.kering import ConfigurationError, Roles
+
+logger = help.ogler.getLogger()
+
+parser = argparse.ArgumentParser(description='List current mailboxes')
+parser.set_defaults(handler=lambda args: handle(args),
+                    transferable=True)
+parser.add_argument('--name', '-n', help='keystore name and file location of KERI keystore', required=True)
+parser.add_argument('--alias', '-a', help='human readable alias for the identifier to whom the credential was issued',
+                    required=True)
+parser.add_argument('--base', '-b', help='additional optional prefix to file location of KERI keystore',
+                    required=False, default="")
+parser.add_argument('--passcode', '-p', help='22 character encryption passcode for keystore (is not saved)',
+                    dest="bran", default=None)  # passcode => bran
+
+
+def handle(args):
+    """ Command line handler for adding an aid to a watcher's list of AIds to watch
+
+    Parameters:
+        args(Namespace): parsed command line arguments
+
+    """
+
+    kwa = dict(args=args)
+    return [doing.doify(listMailboxes, **kwa)]
+
+
+def listMailboxes(tymth, tock=0.0, **opts):
+    """ Command line status handler
+
+    """
+    _ = (yield tock)
+    args = opts["args"]
+    name = args.name
+    alias = args.alias
+    base = args.base
+    bran = args.bran
+
+    try:
+        with existing.existingHby(name=name, base=base, bran=bran) as hby:
+            org = connecting.Organizer(hby=hby)
+            if alias is None:
+                alias = existing.aliasInput(hby)
+
+            hab = hby.habByName(alias)
+
+            for (aid, role, eid), ender in hab.db.ends.getItemIter(keys=(hab.pre, Roles.mailbox)):
+                if ender.allowed:
+                    contact = org.get(eid)
+                    print(f"{contact['alias']}: {eid}")
+
+    except ConfigurationError as e:
+        print(f"identifier prefix for {name} does not exist, incept must be run first", )
+        return -1

src/keri/app/cli/commands/rotate.py

@@ -31,6 +31,7 @@
                     action='store_true')
 parser.add_argument('--code', help='<Witness AID>:<code> formatted witness auth codes.  Can appear multiple times',
                     default=[], action="append", required=False)
+parser.add_argument('--code-time', help='Time the witness codes were captured.', default=None, required=False)
 
 parser.add_argument("--proxy", help="alias for delegation communication proxy", default="")
 
@@ -66,7 +67,8 @@ def rotate(args):
                          cuts=opts.witsCut, adds=opts.witsAdd,
                          isith=opts.isith, nsith=opts.nsith,
                          count=opts.ncount, toad=opts.toad,
-                         data=opts.data, proxy=args.proxy, authenticate=args.authenticate, codes=args.code)
+                         data=opts.data, proxy=args.proxy, authenticate=args.authenticate,
+                         codes=args.code, codeTime=args.code_time)
 
     doers = [rotDoer]
 
@@ -122,7 +124,7 @@ class RotateDoer(doing.DoDoer):
 
     def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=None, count=None,
                  toad=None, wits=None, cuts=None, adds=None, data: list = None, proxy=None, authenticate=False,
-                 codes=None):
+                 codes=None, codeTime=None):
         """
         Returns DoDoer with all registered Doers needed to perform rotation.
 
@@ -149,6 +151,7 @@ def __init__(self, name, base, bran, alias, endpoint=False, isith=None, nsith=No
         self.proxy = proxy
         self.authenticate = authenticate
         self.codes = codes if codes is not None else []
+        self.codeTime = codeTime
 
         self.wits = wits if wits is not None else []
         self.cuts = cuts if cuts is not None else []
@@ -198,9 +201,10 @@ def rotateDo(self, tymth, tock=0.0):
 
         auths = {}
         if self.authenticate:
+            codeTime = helping.fromIso8601(self.codeTime) if self.codeTime is not None else helping.nowIso8601()
             for arg in self.codes:
                 (wit, code) = arg.split(":")
-                auths[wit] = f"{code}#{helping.nowIso8601()}"
+                auths[wit] = f"{code}#{codeTime}"
 
             for wit in hab.kever.wits:
                 if wit in auths:

src/keri/app/cli/commands/watcher/add.py

@@ -11,7 +11,8 @@
 
 from keri.app import connecting, habbing, forwarding
 from keri.app.cli.common import existing
-from keri.core import eventing, serdering
+from keri.core import serdering
+from keri.kering import Roles
 
 logger = help.ogler.getLogger()
 
@@ -91,7 +92,7 @@ def __init__(self, name, alias, base, bran, watcher, watched):
         super(AddDoer, self).__init__(doers=doers)
 
     def addDo(self, tymth, tock=0.0):
-        """ Grant credential by creating /ipex/grant exn message
+        """ Add an AID to a watcher's list of AIDs to watch
 
         Parameters:
             tymth (function): injected function wrapper closure returned by .tymen() of
@@ -109,17 +110,28 @@ def addDo(self, tymth, tock=0.0):
         if isinstance(self.hab, habbing.GroupHab):
             raise ValueError("watchers for multisig AIDs not currently supported")
 
+        ender = self.hab.db.ends.get(keys=(self.hab.pre, Roles.watcher, self.watcher))
+        if not ender or not ender.allowed:
+            msg = self.hab.reply(route="/end/role/add",
+                                 data=dict(cid=self.hab.pre, role=Roles.watcher, eid=self.watcher))
+            self.hab.psr.parseOne(ims=msg)
+
         postman = forwarding.StreamPoster(hby=self.hby, hab=self.hab, recp=self.watcher, topic="reply")
+        for msg in self.hab.db.cloneDelegation(self.hab.kever):
+            serder = serdering.SerderKERI(raw=msg)
+            postman.send(serder=serder, attachment=msg[serder.size:])
+
         for msg in self.hab.db.clonePreIter(pre=self.hab.pre):
             serder = serdering.SerderKERI(raw=msg)
             postman.send(serder=serder, attachment=msg[serder.size:])
 
         data = dict(cid=self.hab.pre,
-                    wid=self.watched,
+                    oid=self.watched,
                     oobi=self.oobi)
 
-        route = "/watcher/aid/add"
+        route = f"/watcher/{self.watcher}/add"
         msg = self.hab.reply(route=route, data=data)
+        self.hab.psr.parseOne(ims=bytes(msg))
         rpy = serdering.SerderKERI(raw=msg)
         postman.send(serder=rpy, attachment=msg[rpy.size:])