From df498f53eff129f0f205c38fdc3e4cd87bfc7c6e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 1 Feb 2025 13:34:09 -0700 Subject: [PATCH 01/33] clean up doc strings --- src/hio/core/udp/udping.py | 48 ++++++++++++++++++++++++++++++-------- src/hio/core/uxd/uxding.py | 14 +++++------ 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index a9b1937..496b2bf 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -30,7 +30,7 @@ def openPeer(cls=None, **kwa): When used in with statement block, calls .close() on exit of with block Parameters: - cls is Class instance of subclass instance + cls (Class): instance of subclass instance Usage: with openPeer() as peer0: @@ -59,6 +59,34 @@ def openPeer(cls=None, **kwa): class Peer(tyming.Tymee): """Class to manage non blocking I/O on UDP socket. SubClass of Tymee to enable support for retry tymers as UDP is unreliable. + + Class Attributes: + Tymeout (float): default timeout for retry tymer(s) if any + + Inherited Properties: + .tyme is float relative cycle time of associated Tymist .tyme obtained + via injected .tymth function wrapper closure. + .tymth is function wrapper closure returned by Tymist .tymeth() method. + When .tymth is called it returns associated Tymist .tyme. + .tymth provides injected dependency on Tymist tyme base. + + Attributes: + tymeout (float): timeout for retry tymer(s) if any + + ha (tuple): host address of form (host,port) of type (str, int) of this + peer's socket address. + bs (int): buffer size + wl (WireLog): instance ref for debug logging of over the wire tx and rx + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + ls (socket.socket): local socket of this Peer + opened (bool): True local socket is created and opened. False otherwise + + Properties: + host (str): element of .ha duple + port (int): element of .ha duple + + """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout @@ -73,14 +101,14 @@ def __init__(self, *, **kwa): """ Initialization method for instance. - - ha = host address duple (host, port) - host = '' equivalant to any interface on host - port = socket port - bs = buffer size - path = path to log file directory - wl = WireLog instance ref for debug logging or over the wire tx and rx - bcast = Flag if True enables sending to broadcast addresses on socket + tymeout (float): default for retry tymer if any + ha (tuple): local socket (host, port) address duple of type (str, int) + host (str): address where '' means any interface on host + port (int): socket port + bs (int): buffer size + wl (WireLog): instance to log over the wire tx and rx + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise """ super(Peer, self).__init__(**kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout @@ -273,7 +301,7 @@ def __init__(self, peer, **kwa): Initialize instance. Parameters: - peer is UDP Peer instance + peer (Peer): UDP instance """ super(PeerDoer, self).__init__(**kwa) self.peer = peer diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 1cd7793..f03f6bf 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -27,17 +27,17 @@ def openPeer(cls=None, name="test", temp=True, reopen=True, clear=True, When used in with statement block, calls .close() on exit of with block Parameters: - cls is Class instance of subclass instance - name is str name of Peer instance path part so can have multiple Peers + cls (Class): instance of subclass instance + name (str): path part so can have multiple Peers at different paths that each use different dirs or files - temp is Boolean, True means open in temporary directory, clear on close - Otherwise open in persistent directory, do not clear on close + temp (bool): True means open in temporary directory, clear on close + Otherwise open in persistent directory, do not clear on close reopen (bool): True (re)open with this init - False not (re)open with this init but later (default) + False not (re)open with this init but later (default) clear (bool): True means remove directory upon close when reopening - False means do not remove directory upon close when reopening + False means do not remove directory upon close when reopening filed (bool): True means .path is file path not directory path - False means .path is directiory path not file path + False means .path is directiory path not file path extensioned (bool): When not filed: True means ensure .path ends with fext False means do not ensure .path ends with fext From b167ff959be5ed60582906ef490f2aca1714627a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 12 Feb 2025 14:12:04 -0700 Subject: [PATCH 02/33] added Namer mixin class for mapping and inverse mapping in memory between name and address added Memoir prelinimary --- src/hio/base/filing.py | 3 +- src/hio/core/memoing.py | 422 +++++++++++++++++++++++++++++++++++++ src/hio/core/udp/udping.py | 37 ++-- src/hio/core/uxd/uxding.py | 6 +- src/hio/help/__init__.py | 2 + src/hio/help/decking.py | 2 +- src/hio/help/helping.py | 8 +- src/hio/help/naming.py | 269 +++++++++++++++++++++++ src/hio/hioing.py | 8 + tests/core/udp/test_udp.py | 10 +- tests/help/test_naming.py | 110 ++++++++++ 11 files changed, 851 insertions(+), 26 deletions(-) create mode 100644 src/hio/core/memoing.py create mode 100644 src/hio/help/naming.py create mode 100644 tests/help/test_naming.py diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index b58c9c6..c9841ac 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -132,7 +132,8 @@ def __init__(self, *, name='main', base="", temp=False, headDirPath=None, """Setup directory of file at .path Parameters: - name (str): directory path name differentiator directory/file + name (str): Unique identifier of file. Unique directory path name + differentiator directory/file When system employs more than one keri installation, name allows differentiating each instance by name base (str): optional directory path segment inserted before name diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py new file mode 100644 index 0000000..fbba9b5 --- /dev/null +++ b/src/hio/core/memoing.py @@ -0,0 +1,422 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.memoing Module + +Mixin Base Classes that add support for MemoGrams to datagram based transports. + +Provide mixin classes. +Takes of advantage of multiple inheritance to enable mixtures of behaviors +with minimal code duplication (more DRY). + +New style python classes use the C3 linearization algorithm. Multiple inheritance +forms a directed acyclic graph called a diamond graph. This graph is linarized +into the method resolution order. +Use class.mro() or class.__mro__ + +(see https://www.geeksforgeeks.org/method-resolution-order-in-python-inheritance/) +Basically: +* children always precede their parents +* immediate parent classes of a child are visited in the order listed in the +child class statement. +* a super class is visited only after all sub classes have been visited +* linearized graph is monotonic (a class is only visted once) + +""" +from collections import deque + +from .. import hioing + + + +class MemoirBase(hioing.Mixin): + """ + Base Memoir object. + Base class for memogram support + Provides common methods for subclasses + Do not instantiate but use a subclass + """ + + def __init__(self, + version=None, + remotes=None, + rxbs=None, + rxPkts=None, + rxMsgs=None, + txbs=None, + txPkts=None, + txMsgs=None, + remotes=None, + **kwa + ): + """ + Setup instance + + Inherited Parameters: + + + Parameters: + version is version tuple or string for this memoir instance + remotes is odict of of remote (name, addr) pairs to bulk initialize + the mappings between remote addresses and remote names using the + methods .addrByName and .nameByAddr + rxbs is bytearray buffer to hold rx data stream if any + rxPkts is deque to hold received packet if any + rxMsgs is deque to hold received msgs if any + txbs is bytearray buffer to hold rx data stream if any + txPkts is deque to hold packet to be transmitted if any + txMsgs is deque to hold messages to be transmitted if any + + Inherited Attributes: + + Attributes: + .version is version tuple or string for this stack + .remotes is odict to hold remotes keyed by uid if any + .rxbs is bytearray buffer to hold input data stream + .rxPkts is deque to hold received packets + .rxMsgs is to hold received msgs + .txbs is bytearray buffer to hold rx data stream if any + .txPkts is deque to hold packets to be transmitted + .txMsgs is deque to hold messages to be transmitted + + Inherited Properties: + + Properties: + + + """ + super(MemoirBase, self).__init__(**kwa) + self._remoteNames = dict() + self._remoteAddrs = dict() + + self.version = version + + for name, addr in remotes.items(): + self.addEntry(name=name, addr=addr) + + + self.rxbs = rxbs if rxbs is not None else bytearray() + self.rxPkts = rxPkts if rxPkts is not None else deque() + self.rxMsgs = rxMsgs if rxMsgs is not None else deque() + self.txbs = txbs if txbs is not None else bytearray() + self.txPkts = txPkts if txPkts is not None else deque() + self.txMsgs = txMsgs if txMsgs is not None else deque() + + + def addrByName(self, name): + """ + tymth property getter, get ._tymth + returns own copy of tymist.tynth function wrapper closure for subsequent + injection into related objects that want to be on same tymist tyme base. + """ + return self._tymth + + + + + def clearTxbs(self): + """ + Clear .txbs + """ + del self.txbs[:] # self.txbs.clear() not supported before python 3.3 + + def _serviceOneTxPkt(self): + """ + Service one (packet, ha) duple on .txPkts deque + Packet is assumed to be packed already in .packed + Override in subclass + """ + pkt = None + if not self.txbs: # everything sent last time + pkt, ha = self.txPkts.popleft() + self.txbs.extend(pkt.packed) + + try: + count = self.send(self.txbs, ha) + except Exception as ex: + raise + + + + if count < len(self.txbs): # delete sent portion + del self.txbs[:count] + return False # partially blocked try again later + + self.clearTxbs() + return True # not blocked + + + def serviceTxPkts(self): + """ + Service the .txPkts deque to send packets through server + Override in subclass + """ + while self.opened and self.txPkts: + if not self._serviceOneTxPkt(): + break # blocked try again later + + def serviceTxPktsOnce(self): + ''' + Service .txPkts deque once (one pkt) + ''' + if self.opened and self.txPkts: + self._serviceOneTxPkt() + + + def transmit(self, pkt, ha=None): + """ + Pack and Append (pkt, ha) duple to .txPkts deque + Override in subclass + """ + try: + pkt.pack() + except ValueError as ex: + emsg = "{}: Error packing pkt.\n{}\n{}\n".format(self.name, pkt, ex) + console.terse(emsg) + + else: + self.txPkts.append((pkt, ha)) + + + def packetize(self, msg, remote=None): + """ + Returns packed packet created from msg destined for remote + Remote provided if attributes needed to fill in packet + Override in subclass + """ + packet = packeting.Packet(stack=self, packed=msg.encode('ascii')) + try: + packet.pack() + except ValueError as ex: + emsg = "{}: Error packing msg.\n{}\n{}\n".format(self.name, msg, ex) + console.terse(emsg) + + return None + return packet + + + def _serviceOneTxMsg(self): + """ + Handle one (message, remote) duple from .txMsgs deque + Assumes there is a duple on the deque + Appends (packet, ha) duple to txPkts deque + """ + msg, remote = self.txMsgs.popleft() # duple (msg, destination uid + console.verbose("{0} sending to {1}\n{2}\n".format(self.name, + remote.name, + msg)) + packet = self.packetize(msg, remote) + if packet is not None: + self.txPkts.append((packet, remote.ha)) + + + def serviceTxMsgs(self): + """ + Service .txMsgs deque of outgoing messages + """ + while self.txMsgs: + self._serviceOneTxMsg() + + def serviceTxMsgOnce(self): + """ + Service .txMsgs deque once (one msg) + """ + if self.txMsgs: + self._serviceOneTxMsg() + + def message(self, msg, remote=None): + """ + Append (msg, remote) duple to .txMsgs deque + If destination remote not given + Then use zeroth remote If any otherwise Raise exception + Override in subclass + """ + if remote is None: + if not self.remotes: + emsg = "No remote to send to.\n" + console.terse(emsg) + + return + remote = self.remotes.values()[0] + self.txMsgs.append((msg, remote)) + + def serviceAllTx(self): + ''' + Service: + txMsgs queue + txes queue to server send + ''' + self.serviceTxMsgs() + self.serviceTxPkts() + + def serviceAllTxOnce(self): + """ + Service the transmit side of the stack once (one transmission) + """ + self.serviceTxMsgOnce() + self.serviceTxPktsOnce() + + def clearRxbs(self): + """ + Clear .rxbs + """ + del self.rxbs[:] # self.rxbs.clear() not supported before python 3.3 + + def parserize(self, raw): + """ + Returns packet parsed from raw data + Override in subclass + """ + packet = packeting.Packet(stack=self) + try: + packet.parse(raw=raw) + except ValueError as ex: + emsg = "{}: Error parsing raw.\n{}\n{}\n".format(self.name, raw, ex) + console.terse(emsg) + self.incStat("pkt_parse_error") + return None + return packet + + def _serviceOneReceived(self): + """ + Service one received raw packet data or chunk from .handler + assumes that there is a .handler + Override in subclass + """ + while True: # keep receiving until empty + try: + raw = self.receive() + except Exception as ex: + raise + + if not raw: + return False # no received data + self.rxbs.extend(raw) + + packet = self.parserize(self.rxbs[:]) + + if packet is not None: # queue packet + console.profuse("{0}: received\n 0x{1}\n".format(self.name, + hexlify(self.rxbs[:packet.size]).decode('ascii'))) + del self.rxbs[:packet.size] + self.rxPkts.append(packet) + return True # received data + + def serviceReceives(self): + """ + Retrieve from server all received and queue up + """ + while self.opened: + if not self._serviceOneReceived(): + break + + def serviceReceivesOnce(self): + """ + Service receives once (one reception) and queue up + """ + if self.opened: + self._serviceOneReceived() + + def messagize(self, pkt, ha): + """ + Returns duple of (message, remote) converted from rx packet pkt and ha + Override in subclass + """ + msg = pkt.packed.decode("ascii") + try: + remote = self.haRemotes[ha] + except KeyError as ex: + console.verbose(("{0}: Dropping packet received from unknown remote " + "ha '{1}'.\n{2}\n".format(self.name, ha, pkt.packed))) + return (None, None) + return (msg, remote) + + + + def _serviceOneRxPkt(self): + """ + Service pkt from .rxPkts deque + Assumes that there is a message on the .rxes deque + """ + pkt = self.rxPkts.popleft() + console.verbose("{0} received packet\n{1}\n".format(self.name, pkt.show())) + self.incStat("pkt_received") + message = self.messagize(pkt) + if message is not None: + self.rxMsgs.append(message) + + def serviceRxPkts(self): + """ + Process all duples in .rxPkts deque + """ + while self.rxPkts: + self._serviceOneRxPkt() + + def serviceRxPktsOnce(self): + """ + Service .rxPkts deque once (one pkt) + """ + if self.rxPkts: + self._serviceOneRxPkt() + + def _serviceOneRxMsg(self): + """ + Service one duple from .rxMsgs deque + """ + msg = self.rxMsgs.popleft() + + + def serviceRxMsgs(self): + """ + Service .rxMsgs deque of duples + """ + while self.rxMsgs: + self._serviceOneRxMsg() + + def serviceRxMsgsOnce(self): + """ + Service .rxMsgs deque once (one msg) + """ + if self.rxMsgs: + self._serviceOneRxMsg() + + + def serviceAllRx(self): + """ + Service receive side of stack + """ + self.serviceReceives() + self.serviceRxPkts() + self.serviceRxMsgs() + self.serviceTimers() + + def serviceAllRxOnce(self): + """ + Service receive side of stack once (one reception) + """ + self.serviceReceivesOnce() + self.serviceRxPktsOnce() + self.serviceRxMsgsOnce() + self.serviceTimers() + + def serviceAll(self): + """ + Service all Rx and Tx + """ + self.serviceAllRx() + self.serviceAllTx() + + def serviceLocal(self): + """ + Service the local Peer's receive and transmit queues + """ + self.serviceReceives() + self.serviceTxPkts() + + + + + + + + + diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 496b2bf..616fab8 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -24,14 +24,15 @@ @contextmanager -def openPeer(cls=None, **kwa): +def openPeer(cls=None, name="test", **kwa): """ Wrapper to create and open UDP Peer instances When used in with statement block, calls .close() on exit of with block Parameters: cls (Class): instance of subclass instance - + name (str): unique identifer of peer. Enables management of Peer sockets + by name. Usage: with openPeer() as peer0: peer0.receive() @@ -45,7 +46,7 @@ def openPeer(cls=None, **kwa): if cls is None: cls = Peer try: - peer = cls(**kwa) + peer = cls(name=name, **kwa) peer.reopen() yield peer @@ -71,8 +72,8 @@ class Peer(tyming.Tymee): .tymth provides injected dependency on Tymist tyme base. Attributes: + name (str): unique identifier of peer for managment purposes tymeout (float): timeout for retry tymer(s) if any - ha (tuple): host address of form (host,port) of type (str, int) of this peer's socket address. bs (int): buffer size @@ -91,6 +92,7 @@ class Peer(tyming.Tymee): Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout def __init__(self, *, + name='main', tymeout=None, ha=None, host='', @@ -101,16 +103,19 @@ def __init__(self, *, **kwa): """ Initialization method for instance. - tymeout (float): default for retry tymer if any - ha (tuple): local socket (host, port) address duple of type (str, int) - host (str): address where '' means any interface on host - port (int): socket port - bs (int): buffer size - wl (WireLog): instance to log over the wire tx and rx - bcast (bool): True enables sending to broadcast addresses from local socket - False otherwise + Parameters: + name (str): unique identifier of peer for managment purposes + tymeout (float): default for retry tymer if any + ha (tuple): local socket (host, port) address duple of type (str, int) + host (str): address where '' means any interface on host + port (int): socket port + bs (int): buffer size + wl (WireLog): instance to log over the wire tx and rx + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise """ super(Peer, self).__init__(**kwa) + self.name = name self.tymeout = tymeout if tymeout is not None else self.Tymeout #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer @@ -267,16 +272,16 @@ def send(self, data, da): da is destination address tuple (destHost, destPort) """ try: - result = self.ls.sendto(data, da) #result is number of bytes sent + cnt = self.ls.sendto(data, da) # count == int number of bytes sent except OSError as ex: logger.error("Error send UDP from %s to %s.\n %s\n", self.ha, da, ex) - result = 0 + cnt = 0 raise if self.wl: # log over the wire send - self.wl.writeTx(data[:result], who=da) + self.wl.writeTx(data[:cnt], who=da) - return result + return cnt def service(self): diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index f03f6bf..a465a50 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -28,8 +28,8 @@ def openPeer(cls=None, name="test", temp=True, reopen=True, clear=True, Parameters: cls (Class): instance of subclass instance - name (str): path part so can have multiple Peers - at different paths that each use different dirs or files + name (str): unique identifer of peer. Unique path part so can have many + Peers each at different paths that each use different dirs or files temp (bool): True means open in temporary directory, clear on close Otherwise open in persistent directory, do not clear on close reopen (bool): True (re)open with this init @@ -315,7 +315,7 @@ def send(self, data, dst): dst (str): uxd destination path """ try: - cnt = self.ls.sendto(data, dst) # count is int number of bytes sent + cnt = self.ls.sendto(data, dst) # count == int number of bytes sent except OSError as ex: logger.error("Error send UXD from %s to %s.\n %s\n", self.path, dst, ex) cnt = 0 diff --git a/src/hio/help/__init__.py b/src/hio/help/__init__.py index 722769e..1dfda52 100644 --- a/src/hio/help/__init__.py +++ b/src/hio/help/__init__.py @@ -17,4 +17,6 @@ from .decking import Deck from .hicting import Hict, Mict from .timing import Timer, MonoTimer, TimerError, RetroTimerError +from .naming import Namer +from .helping import isNonStringIterable, isNonStringSequence, isIterator diff --git a/src/hio/help/decking.py b/src/hio/help/decking.py index 7714868..a364801 100644 --- a/src/hio/help/decking.py +++ b/src/hio/help/decking.py @@ -13,7 +13,7 @@ class Deck(deque): Extends deque to support deque access convenience methods .push and .pull to remove confusion about which side of the deque to use (left or right). - Extends deque with .push an .pull methods to support a different pattern for + Extends deque with .push and .pull methods to support a different pattern for access. .push does not allow a value of None to be added to the Deck. This enables retrieval with .pull(emptive=True) which returns None when empty instead of raising IndexError. This allows use of the walrus operator on diff --git a/src/hio/help/helping.py b/src/hio/help/helping.py index d947f26..05d3ea5 100644 --- a/src/hio/help/helping.py +++ b/src/hio/help/helping.py @@ -292,7 +292,7 @@ def __subclasshook__(cls, C): return NotImplemented -def nonStringIterable(obj): +def isNonStringIterable(obj): """ Returns: (bool) True if obj is non-string iterable, False otherwise @@ -301,13 +301,17 @@ def nonStringIterable(obj): """ return (not isinstance(obj, (str, bytes)) and isinstance(obj, Iterable)) +nonStringIterable = isNonStringIterable # legacy interface -def nonStringSequence(obj): + +def isNonStringSequence(obj): """ Returns: (bool) True if obj is non-string sequence, False otherwise """ return (not isinstance(obj, (str, bytes)) and isinstance(obj, Sequence) ) +nonStringSequence = isNonStringSequence # legacy interface + def isIterator(obj): """ diff --git a/src/hio/help/naming.py b/src/hio/help/naming.py new file mode 100644 index 0000000..eee7342 --- /dev/null +++ b/src/hio/help/naming.py @@ -0,0 +1,269 @@ +# -*- encoding: utf-8 -*- +""" +hio.help.naming Module + +Mixin Base Class for one-to-one mapping with inverse: name-to-addr and addr-to-name +that add support for MemoGrams to datagram based transports. + +Provide mixin classes. +Takes of advantage of multiple inheritance to enable mixtures of behaviors +with minimal code duplication (more DRY). + +New style python classes use the C3 linearization algorithm. Multiple inheritance +forms a directed acyclic graph called a diamond graph. This graph is linarized +into the method resolution order. +Use class.mro() or class.__mro__ + +(see https://www.geeksforgeeks.org/method-resolution-order-in-python-inheritance/) +Basically: +* children always precede their parents +* immediate parent classes of a child are visited in the order listed in the +child class statement. +* a super class is visited only after all sub classes have been visited +* linearized graph is monotonic (a class is only visted once) + +""" + + +from .. import hioing +from . import helping + +class Namer(hioing.Mixin): + """ + Namer mixin class for adding support for mappings between names and addresses. + May be used to provide in-memory lookup of mapping and its inverse of + address from name and name from address. + + """ + + def __init__(self, entries=None, **kwa): + """ + Setup instance + + Inherited Parameters: + + Parameters: + entries (dict | interable): dict or iterable of (name, addr) duples + to bulk initialize the mappings between addresses and names + + + Attributes: + + Hidden: + _addrByName (dict): mapping between (name, address) pairs, these + must be one-to-one so that inverse is also one-to-one + _nameByAddr (dict): mapping between (address, name) pairs, these + must be one-to-one so that inverse is also one-to-one + + + + """ + self._addrByName = dict() + self._nameByAddr = dict() + + if helping.isNonStringIterable(entries): + items = entries + else: + items = entries.items() if entries else [] + + for name, addr in items: + self.addEntry(name=name, addr=addr) + + super(Namer, self).__init__(**kwa) + + + @property + def addrByName(self): + """Property that returns copy of ._addrByName + """ + return dict(self._addrByName) + + @property + def nameByAddr(self): + """Property that returns copy of ._nameByAddr + """ + return dict(self._nameByAddr) + + + def getAddr(self, name): + """Returns addr for given name or None if non-existant + + Parameters: + name (str): name + """ + return self._addrByName.get(name) + + + def getName(self, addr): + """Returns name for given addr or None if non-existant + + Parameters: + name (str): address + """ + return self._nameByAddr.get(addr) + + + def addEntry(self, name, addr): + """Add mapping and inverse mapping entries for + name to addr and addr to name + All mappings must be one-to-one + + Returns: + result (bool): True if successfully added new entry. + False If matching entry already exists. + Otherwise raises NamerError if partial matching + entry with either name or addr + + Parameters: + name (str): name of entry + addr (str): address of entry + + Raise error if preexistant + """ + if not name or not addr: + raise hioing.NamerError(f"Attempt to add incomplete or empty entry " + f"({name=}, {addr=}).") + if name in self._addrByName: + if addr == self._addrByName[name]: + return False # already existing matching entry + else: + raise hioing.NamerError(f"Attempt to add conflicting entry " + f"({name=}, {addr=}).") + + if addr in self._nameByAddr: + if name == self._nameByAddr[addr]: + return False # already existing matching entry + else: + raise hioing.NamerError(f"Attempt to add conflicting entry " + f"({name=}, {addr=}).") + + self._addrByName[name] = addr + self._nameByAddr[addr] = name + + return True # added new entry + + + def remEntry(self, name=None, addr=None): + """Delete both name to addr and addr to name mapping entries. + When an entry is found for either name or addr or both. + When both provided must be matching one-to-one entries. + All mappings must be one-to-one + + Returns: + result (bool): True if matching entry successfully deleted + False if no matching entry, nothing deleted + + Parameters: + name (str | None): name of remote to delete or None if delete by addr + addr (str | None): addr of remote to delete or None if delete by name + + When both name and addr provided deletes by name + + Raise error when at least one of name or addr is not None and no mappings + exist. + """ + + if name: + if name not in self._addrByName: # no entry for name + return False + + if not addr: # addr not provided so look it up + addr = self._addrByName[name] + + if addr != self._addrByName[name]: # mismatch do nothing + return False + + del self._addrByName[name] + del self._nameByAddr[addr] + return True + + if addr: + if addr not in self._nameByAddr: # no entry for addr + return False + + if not name: # no name so look it up + name = self._nameByAddr[addr] + + if name != self._nameByAddr[addr]: # mismatch do nothing + return False + + del self._addrByName[name] + del self._nameByAddr[addr] + return True + + return False # nothing deleted neither name nor addr provided + + + def updateAddrAtName(self, *, name=None, addr=None): + """Updates existing name to addr mapping with new addr. Replaces + inverse mapping of addr to name. + All mappings must be one-to-one + + Returns: + result (bool): True If successfully updated existing entries. + False If matching entry already exists, no change. + Otherwise raises NamerError + + Parameters: + name (str): name of entry + addr (str): address of entry + + Raise error if preexistant + """ + if not name or not addr: + raise hioing.NamerError(f"Attempt to update with incomplete " + f"or empty entry ({name=}, {addr=}).") + + if name not in self._addrByName: + return False + + if addr == self._addrByName[name]: + return False # no change to entries + + if addr in self._nameByAddr: + raise hioing.NamerError(f"Conflicting entry for {addr=}.") + + oldAddr = self._addrByName[name] + self._addrByName[name] = addr + del self._nameByAddr[oldAddr] + self._nameByAddr[addr] = name + + return True # updated entries + + + def updateNameAtAddr(self, *, addr=None, name=None): + """Updates existing addr to name mapping with new name. Replaces + inverse mapping of name to addr. + All mappings must be one-to-one + + Returns: + result (bool): True If successfully updated existing entries. + False If matching entry already exists, no change. + Otherwise raises NamerError + + Parameters: + addr (str): address of entry + name (str): name of entry + + Raise error if preexistant + """ + if not name or not addr: + raise hioing.NamerError(f"Attempt to update with incomplete entry " + f"({name=}, {addr=}).") + + if addr not in self._nameByAddr: + return False + + if name == self._nameByAddr[addr]: + return False # no change to entries + + if name in self._addrByName: + raise hioing.NamerError(f"Conflicting entry for {name=}.") + + oldName = self._nameByAddr[addr] + self._nameByAddr[addr] = name + del self._addrByName[oldName] + self._addrByName[name] = addr + + return True # updated entries + diff --git a/src/hio/hioing.py b/src/hio/hioing.py index 1bcb4fd..bb47a05 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -58,6 +58,14 @@ class FilerError(HioError): raise FilerError("error message") """ +class NamerError(HioError): + """ + Error using or configuring Remoter + + Usage: + raise NamerError("error message") + """ + class Mixin(): """ diff --git a/tests/core/udp/test_udp.py b/tests/core/udp/test_udp.py index 30fe151..7f8fbbc 100644 --- a/tests/core/udp/test_udp.py +++ b/tests/core/udp/test_udp.py @@ -22,12 +22,14 @@ def test_udp_basic(): alpha = udping.Peer(port = 6101, wl=wl) # any interface on port 6101 assert not alpha.opened + assert alpha.name == 'main' # default assert alpha.reopen() assert alpha.opened assert alpha.ha == ('0.0.0.0', 6101) - beta = udping.Peer(port = 6102, wl=wl) # any interface on port 6102 + beta = udping.Peer(name='beta', port = 6102, wl=wl) # any interface on port 6102 assert not beta.opened + assert beta.name == 'beta' assert beta.reopen() assert beta.opened assert beta.ha == ('0.0.0.0', 6102) @@ -93,13 +95,15 @@ def test_open_peer(): """ tymist = tyming.Tymist() with (wiring.openWL(samed=True, filed=True) as wl, - udping.openPeer(port = 6101, wl=wl) as alpha, # any interface on port 6101 - udping.openPeer(port = 6102, wl=wl) as beta): # any interface on port 6102 + udping.openPeer(name='alpha', port = 6101, wl=wl) as alpha, # any interface on port 6101 + udping.openPeer(name='beta', port = 6102, wl=wl) as beta): # any interface on port 6102 assert alpha.opened + assert alpha.name == 'alpha' assert alpha.ha == ('0.0.0.0', 6101) assert beta.opened + assert beta.name == 'beta' assert beta.ha == ('0.0.0.0', 6102) msgOut = b"alpha sends to beta" diff --git a/tests/help/test_naming.py b/tests/help/test_naming.py new file mode 100644 index 0000000..97a8267 --- /dev/null +++ b/tests/help/test_naming.py @@ -0,0 +1,110 @@ +# -*- encoding: utf-8 -*- +""" +tests.help.test_naming module + +""" +import pytest + +from hio import hioing +from hio.help import Namer + +def test_namer(): + """ + Test Namer class + """ + namer = Namer() + assert not namer.addrByName + assert not namer.nameByAddr + + assert namer.addEntry("alpha", "/path/to/alpha") + assert namer.addEntry("beta", "/path/to/beta") + assert namer.addEntry("gamma", "/path/to/gamma") + assert not namer.addEntry("gamma", "/path/to/gamma") # already added + + assert namer.addrByName == {'alpha': '/path/to/alpha', + 'beta': '/path/to/beta', + 'gamma': '/path/to/gamma'} + + assert namer.nameByAddr == {'/path/to/alpha': 'alpha', + '/path/to/beta': 'beta', + '/path/to/gamma': 'gamma'} + + with pytest.raises(hioing.NamerError): + namer.addEntry("", "") + + with pytest.raises(hioing.NamerError): + namer.addEntry("delta", "") + + with pytest.raises(hioing.NamerError): + namer.addEntry("", "/path/to/delta") + + with pytest.raises(hioing.NamerError): + namer.addEntry("delta", "/path/to/alpha") + + with pytest.raises(hioing.NamerError): + namer.addEntry("alpha", "/path/to/delta") + + assert namer.addEntry("delta", "/path/to/delta") + + assert namer.addrByName == {'alpha': '/path/to/alpha', + 'beta': '/path/to/beta', + 'gamma': '/path/to/gamma', + 'delta': '/path/to/delta'} + + + assert namer.nameByAddr == {'/path/to/alpha': 'alpha', + '/path/to/beta': 'beta', + '/path/to/gamma': 'gamma', + '/path/to/delta': 'delta'} + + assert not namer.remEntry(name="delta", addr="/path/to/alpha") + + assert namer.remEntry(name="delta", addr="/path/to/delta") + assert not namer.remEntry(name="delta", addr="/path/to/delta") + assert not namer.remEntry(name="delta") + assert not namer.remEntry(addr="/path/to/delta") + + assert namer.remEntry(name="gamma") + assert namer.remEntry(addr="/path/to/beta") + + assert namer.addrByName == {'alpha': '/path/to/alpha'} + + assert namer.nameByAddr == {'/path/to/alpha': 'alpha'} + + assert namer.addEntry("beta", "/path/to/beta") + assert namer.addEntry("gamma", "/path/to/gamma") + assert namer.addEntry("delta", "/path/to/delta") + + assert not namer.updateAddrAtName(name='alpha', addr="/path/to/alpha") + assert namer.updateAddrAtName(name='alpha', addr="/alt/path/to/alpha") + assert not namer.updateNameAtAddr(addr="/path/to/alpha", name="alphaneo") + assert not namer.updateAddrAtName(name='alphaneo', addr="/path/to/alpha") + with pytest.raises(hioing.NamerError): + assert namer.updateAddrAtName(name='alpha', addr="/path/to/beta") + + assert not namer.updateNameAtAddr(addr="/path/to/beta", name='beta') + assert namer.updateNameAtAddr(addr="/path/to/beta", name='betaneo') + assert not namer.updateAddrAtName(name="beta", addr="/alt/path/to/beta") + assert not namer.updateNameAtAddr(addr="/alt/path/to/beta", name='beta') + with pytest.raises(hioing.NamerError): + assert namer.updateNameAtAddr(addr="/path/to/beta", name='delta') + + + assert namer.addrByName == {'alpha': '/alt/path/to/alpha', + 'gamma': '/path/to/gamma', + 'delta': '/path/to/delta', + 'betaneo': '/path/to/beta'} + + + assert namer.nameByAddr == {'/path/to/beta': 'betaneo', + '/path/to/gamma': 'gamma', + '/path/to/delta': 'delta', + '/alt/path/to/alpha': 'alpha'} + + + """End Test""" + + +if __name__ == "__main__": + test_namer() + From e0c448782108a4870e42061ff607362773c1ae6d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 12 Feb 2025 17:39:23 -0700 Subject: [PATCH 03/33] fix tests --- src/hio/help/naming.py | 8 ++++---- tests/help/test_naming.py | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/hio/help/naming.py b/src/hio/help/naming.py index eee7342..2c845ab 100644 --- a/src/hio/help/naming.py +++ b/src/hio/help/naming.py @@ -194,8 +194,8 @@ def remEntry(self, name=None, addr=None): return False # nothing deleted neither name nor addr provided - def updateAddrAtName(self, *, name=None, addr=None): - """Updates existing name to addr mapping with new addr. Replaces + def changeAddrAtName(self, *, name=None, addr=None): + """Changes existing name to addr mapping to new addr. Replaces inverse mapping of addr to name. All mappings must be one-to-one @@ -231,8 +231,8 @@ def updateAddrAtName(self, *, name=None, addr=None): return True # updated entries - def updateNameAtAddr(self, *, addr=None, name=None): - """Updates existing addr to name mapping with new name. Replaces + def changeNameAtAddr(self, *, addr=None, name=None): + """Changes existing addr to name mapping to new name. Replaces inverse mapping of name to addr. All mappings must be one-to-one diff --git a/tests/help/test_naming.py b/tests/help/test_naming.py index 97a8267..bd7384d 100644 --- a/tests/help/test_naming.py +++ b/tests/help/test_naming.py @@ -75,19 +75,19 @@ def test_namer(): assert namer.addEntry("gamma", "/path/to/gamma") assert namer.addEntry("delta", "/path/to/delta") - assert not namer.updateAddrAtName(name='alpha', addr="/path/to/alpha") - assert namer.updateAddrAtName(name='alpha', addr="/alt/path/to/alpha") - assert not namer.updateNameAtAddr(addr="/path/to/alpha", name="alphaneo") - assert not namer.updateAddrAtName(name='alphaneo', addr="/path/to/alpha") + assert not namer.changeAddrAtName(name='alpha', addr="/path/to/alpha") + assert namer.changeAddrAtName(name='alpha', addr="/alt/path/to/alpha") + assert not namer.changeNameAtAddr(addr="/path/to/alpha", name="alphaneo") + assert not namer.changeAddrAtName(name='alphaneo', addr="/path/to/alpha") with pytest.raises(hioing.NamerError): - assert namer.updateAddrAtName(name='alpha', addr="/path/to/beta") + assert namer.changeAddrAtName(name='alpha', addr="/path/to/beta") - assert not namer.updateNameAtAddr(addr="/path/to/beta", name='beta') - assert namer.updateNameAtAddr(addr="/path/to/beta", name='betaneo') - assert not namer.updateAddrAtName(name="beta", addr="/alt/path/to/beta") - assert not namer.updateNameAtAddr(addr="/alt/path/to/beta", name='beta') + assert not namer.changeNameAtAddr(addr="/path/to/beta", name='beta') + assert namer.changeNameAtAddr(addr="/path/to/beta", name='betaneo') + assert not namer.changeAddrAtName(name="beta", addr="/alt/path/to/beta") + assert not namer.changeNameAtAddr(addr="/alt/path/to/beta", name='beta') with pytest.raises(hioing.NamerError): - assert namer.updateNameAtAddr(addr="/path/to/beta", name='delta') + assert namer.changeNameAtAddr(addr="/path/to/beta", name='delta') assert namer.addrByName == {'alpha': '/alt/path/to/alpha', From 45472b84458af53f1af150ec5f2c613e7df9d312 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 12 Feb 2025 17:44:38 -0700 Subject: [PATCH 04/33] added clearEntries --- src/hio/core/memoing.py | 23 ----------------------- src/hio/help/naming.py | 10 +++++++++- tests/help/test_naming.py | 5 +++++ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index fbba9b5..a3235c2 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -38,14 +38,12 @@ class MemoirBase(hioing.Mixin): def __init__(self, version=None, - remotes=None, rxbs=None, rxPkts=None, rxMsgs=None, txbs=None, txPkts=None, txMsgs=None, - remotes=None, **kwa ): """ @@ -56,9 +54,6 @@ def __init__(self, Parameters: version is version tuple or string for this memoir instance - remotes is odict of of remote (name, addr) pairs to bulk initialize - the mappings between remote addresses and remote names using the - methods .addrByName and .nameByAddr rxbs is bytearray buffer to hold rx data stream if any rxPkts is deque to hold received packet if any rxMsgs is deque to hold received msgs if any @@ -70,7 +65,6 @@ def __init__(self, Attributes: .version is version tuple or string for this stack - .remotes is odict to hold remotes keyed by uid if any .rxbs is bytearray buffer to hold input data stream .rxPkts is deque to hold received packets .rxMsgs is to hold received msgs @@ -85,15 +79,9 @@ def __init__(self, """ super(MemoirBase, self).__init__(**kwa) - self._remoteNames = dict() - self._remoteAddrs = dict() self.version = version - for name, addr in remotes.items(): - self.addEntry(name=name, addr=addr) - - self.rxbs = rxbs if rxbs is not None else bytearray() self.rxPkts = rxPkts if rxPkts is not None else deque() self.rxMsgs = rxMsgs if rxMsgs is not None else deque() @@ -102,17 +90,6 @@ def __init__(self, self.txMsgs = txMsgs if txMsgs is not None else deque() - def addrByName(self, name): - """ - tymth property getter, get ._tymth - returns own copy of tymist.tynth function wrapper closure for subsequent - injection into related objects that want to be on same tymist tyme base. - """ - return self._tymth - - - - def clearTxbs(self): """ Clear .txbs diff --git a/src/hio/help/naming.py b/src/hio/help/naming.py index 2c845ab..ed70ff7 100644 --- a/src/hio/help/naming.py +++ b/src/hio/help/naming.py @@ -98,7 +98,7 @@ def getName(self, addr): """Returns name for given addr or None if non-existant Parameters: - name (str): address + addr (str): address """ return self._nameByAddr.get(addr) @@ -194,6 +194,14 @@ def remEntry(self, name=None, addr=None): return False # nothing deleted neither name nor addr provided + def clearEntries(self): + """Clears all entries + + """ + self._addrByName = dict() + self._nameByAddr = dict() + + def changeAddrAtName(self, *, name=None, addr=None): """Changes existing name to addr mapping to new addr. Replaces inverse mapping of addr to name. diff --git a/tests/help/test_naming.py b/tests/help/test_naming.py index bd7384d..30cb333 100644 --- a/tests/help/test_naming.py +++ b/tests/help/test_naming.py @@ -101,6 +101,11 @@ def test_namer(): '/path/to/delta': 'delta', '/alt/path/to/alpha': 'alpha'} + namer.clearEntries() + assert not namer.addrByName + assert not namer.nameByAddr + + """End Test""" From b01235f215d1fc8f5c9afa9003e4deab75921dda Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 13 Feb 2025 18:51:16 -0700 Subject: [PATCH 05/33] added check for path length to uxd Peer some prelim work on Memoir class for memograms --- src/hio/core/memoing.py | 532 ++++++++++++++++++++++++++----------- src/hio/core/uxd/uxding.py | 20 +- src/hio/hioing.py | 6 + tests/core/uxd/test_uxd.py | 14 + 4 files changed, 417 insertions(+), 155 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index a3235c2..c20b953 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -21,237 +21,432 @@ * a super class is visited only after all sub classes have been visited * linearized graph is monotonic (a class is only visted once) +Datagram IP Fragmentation +https://packetpushers.net/blog/ip-fragmentation-in-detail/ +Only the first fragment contains the header that is needed to defragment at +the other end. + +https://stackoverflow.com/questions/24045424/when-acting-on-a-udp-socket-what-can-cause-sendto-to-send-fewer-bytes-than-re +Answer is that UDP sendto always sends all the bytes but does not make a distinction +between nonblocking and blocking calls to sendto. +Other documentation seems to indicate that sendto when non-blocking may not +send all the bytes. +https://learn.microsoft.com/en-us/dotnet/api/system.net.sockets.socket.sendto?view=net-9.0 + +http://esp32.io/viewtopic.php?t=2378 +For UDP, sendto() will try to allocate a packet buffer to hold the full +UDP packet of the size requested (up to 64KB) and will return with errno=ENOMEM +if this fails. Otherwise, provided FD_ISSET(sock, &write_set) is set then the +sendto() will succeed. But you should check for this failure. +Generally if the ESP32 is not low on free memory then you won't see ENOMEM here. + +Every call to send must send less that sf::UdpSocket::MaxDatagramSize bytes -- +which is a little less than 2^16 (65536) bytes. + +https://people.computing.clemson.edu/~westall/853/notes/udpsend.pdf +However, the existence of the UDP and IP +headers should also limit. + +Maximum IPV4 header size is 60 bytes and UDP header is 8 bytes so maximum UDP +datagram payload size is 65535 - 60 - 8 = 65467 + +With ipv6 the IP headers are done differently so maximum payload size is +65,535 - 8 = 65527 + +https://en.wikipedia.org/wiki/Maximum_transmission_unit +https://stackoverflow.com/questions/1098897/what-is-the-largest-safe-udp-packet-size-on-the-internet +The maximum safe UDP payload is 508 bytes. This is a packet size of 576 +(the "minimum maximum reassembly buffer size"), minus the maximum 60-byte +IP header and the 8-byte UDP header. As others have mentioned, additional +protocol headers could be added in some circumstances. A more conservative +value of around 300-400 bytes may be preferred instead. + +Any UDP payload this size or smaller is guaranteed to be deliverable over IP +(though not guaranteed to be delivered). Anything larger is allowed to be +outright dropped by any router for any reason. + +Except on an IPv6-only route, +where the maximum payload is 1,212 bytes. + +Unix Domain Socket: +https://unix.stackexchange.com/questions/424380/what-values-may-linux-use-for-the-default-unix-socket-buffer-size + +Path cannot exceed 108 bytes +The SO_SNDBUF socket option does have an effect for UNIX domain sockets, +but the SO_RCVBUF option does not. +For datagram sockets, the SO_SNDBUF value imposes an upper limit on the size +of outgoing datagrams. +This limit is calculated as the doubled (see socket(7)) option value less 32 bytes used for overhead. + + +SO_SNDBUF + Sets or gets the maximum socket send buffer in bytes. The kernel doubles this value (to allow + space for bookkeeping overhead) when it is set using setsockopt(2), and this doubled value is + returned by getsockopt(2). The default value is set by the /proc/sys/net/core/wmem_default file + and the maximum allowed value is set by the /proc/sys/net/core/wmem_max file. The minimum + (doubled) value for this option is 2048. + +https://stackoverflow.com/questions/21856517/whats-the-practical-limit-on-the-size-of-single-packet-transmitted-over-domain + """ -from collections import deque +import socket +import errno -from .. import hioing +from collections import deque, namedtuple +from .. import hioing +# namedtuple of ints (major: int, minor: int) +Versionage = namedtuple("Versionage", "major minor") -class MemoirBase(hioing.Mixin): +class Memoir(hioing.Mixin): """ - Base Memoir object. - Base class for memogram support - Provides common methods for subclasses - Do not instantiate but use a subclass + Memoir mixin base class to adds memogram support to a transport class. + Memoir supports asynchronous memograms. Provides common methods for subclasses. + + A memogram is a higher level construct that sits on top of a datagram. + A memogram supports the segmentation and desegmentation of memos to + respect the underlying datagram size, buffer, and fragmentation behavior. + + Layering a reliable transaction protocol on top of a memogram enables reliable + asynchronous messaging over unreliable datagram transport protocols. + + When the datagram protocol is already reliable, then a memogram enables + larger memos (messages) than that natively supported by the datagram. + + Usage: + Do not instantiate directly but use as a mixin with a transport class + in order to create a new subclass that adds memogram support to the + transport class. For example MemoirUdp or MemoGramUdp or MemoirUxd or + MemoGramUXD + + Each direction of dataflow uses a triple-tiered set of buffers that respect + the constraints of non-blocking asynchronous IO with datagram transports. + + On the transmit side memos are placed in a deque (double ended queue). Each + memor is then segmented into grams (datagrams) the respect the size constraints + of the underlying datagram transport. These grams are placed in another deque. + Each gram in its deque is then places in a transmit buffer to be sent over the + transport. When using non-blocking IO, asynchronous datagram transport + protocols may have hidden buffering constraints that result in fragmentation + of the sent datagram which means the whole datagram is not sent at once via + a non-blocking send call. This means that the remainer of the datagram must + be sent later and may take multiple send calls to complete. The datagram + protocol is responsible for managing the buffering and fragmentation but + depends on the sender repeated attempts to send the reaminder of the + full datagram. This is ensured with the final tier with a raw transmit + buffer that waits until it is empty before attempting to send another + gram. + + On the receive the raw data is received into the raw data buffer. This is then + converted into a gram and queued in the gram receive deque as a seqment of + a memo. The grams in the gram recieve deque are then desegmented into a memo + and placed in the memo deque for consumption by the application or some other + higher level protocol. + + Memo segmentation information is embedded in the grams. + + + Class Attributes: + Version (Versionage): default version consisting of namedtuple of form + (major: int, minor: int) + + Attributes: + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) + rxbs (bytearray): buffer holding rx (receive) data from raw transport + txbs (bytearray): buffer holding tx (transmit) raw data to raw transport + rxgs (deque): holding rx (receive) (data) grams (segments) recieved via rxbs raw + txgs (deque): holding tx (transmit) (data) grams (segments) to be sent via txbs raw + rxms (deque): holding rx (receive) memos desegmented from rxgs grams + txms (deque): holding tx (transmit) memos to be segmented into txgs grams + + + Properties: + + """ + Version = Versionage(major=0, minor=0) # default version + def __init__(self, version=None, rxbs=None, - rxPkts=None, - rxMsgs=None, txbs=None, - txPkts=None, - txMsgs=None, + rxgs=None, + txgs=None, + rxms=None, + txms=None, **kwa ): - """ - Setup instance + """Setup instance Inherited Parameters: Parameters: - version is version tuple or string for this memoir instance - rxbs is bytearray buffer to hold rx data stream if any - rxPkts is deque to hold received packet if any - rxMsgs is deque to hold received msgs if any - txbs is bytearray buffer to hold rx data stream if any - txPkts is deque to hold packet to be transmitted if any - txMsgs is deque to hold messages to be transmitted if any + version (Versionage): version for this memoir instance consisting of + namedtuple of form (major: int, minor: int) - Inherited Attributes: + txbs (bytearray): buffer holding tx (transmit) raw data to raw transport + rxbs (bytearray): buffer holding rx (receive) data from raw transport - Attributes: - .version is version tuple or string for this stack - .rxbs is bytearray buffer to hold input data stream - .rxPkts is deque to hold received packets - .rxMsgs is to hold received msgs - .txbs is bytearray buffer to hold rx data stream if any - .txPkts is deque to hold packets to be transmitted - .txMsgs is deque to hold messages to be transmitted + txgs (deque): holding tx (transmit) (data) grams to be sent via txbs raw + rxgs (deque): holding rx (receive) (data) grams recieved via rxbs raw - Inherited Properties: + txms (deque): holding tx (transmit) memos to be segmented into txgs grams + rxms (deque): holding rx (receive) memos desegmented from rxgs grams - Properties: """ - super(MemoirBase, self).__init__(**kwa) + # initialize attributes + self.version = version if version is not None else self.Version - self.version = version - self.rxbs = rxbs if rxbs is not None else bytearray() - self.rxPkts = rxPkts if rxPkts is not None else deque() - self.rxMsgs = rxMsgs if rxMsgs is not None else deque() self.txbs = txbs if txbs is not None else bytearray() - self.txPkts = txPkts if txPkts is not None else deque() - self.txMsgs = txMsgs if txMsgs is not None else deque() + self.rxbs = rxbs if rxbs is not None else bytearray() + + self.txgs = txgs if txgs is not None else deque() + self.rxgs = rxgs if rxgs is not None else deque() + + + self.txms = txms if txms is not None else deque() + self.rxms = rxms if rxms is not None else deque() + + super(Memoir, self).__init__(**kwa) + + if not hasattr(self, "opened"): # stub so mixin works in isolation + self.opened = False # mixed with subclass should provide this. + + def send(self, txbs, dst): + """Attemps to send bytes in txbs to remote destination dst. Must be + overridden in subclass. This is a stub to define mixin interface. + + Returns: + count (int): bytes actually sent + + Parameters: + txbs (bytes | bytearray): of bytes to send + dst (str): remote destination address + """ + return 0 def clearTxbs(self): """ Clear .txbs """ - del self.txbs[:] # self.txbs.clear() not supported before python 3.3 + self.txbs.clear() + - def _serviceOneTxPkt(self): + def clearRxbs(self): """ - Service one (packet, ha) duple on .txPkts deque - Packet is assumed to be packed already in .packed - Override in subclass + Clear .rxbs """ - pkt = None - if not self.txbs: # everything sent last time - pkt, ha = self.txPkts.popleft() - self.txbs.extend(pkt.packed) + del self.rxbs.clear() - try: - count = self.send(self.txbs, ha) - except Exception as ex: - raise + def memoit(self, memo, dst): + """Append (memo, dst) duple to .txms deque + Parameters: + memo (str): to be segmented and packed into gram(s) + dst (str): address of remote destination of memo + """ + self.txms.append((memo, dst)) - if count < len(self.txbs): # delete sent portion - del self.txbs[:count] - return False # partially blocked try again later - self.clearTxbs() - return True # not blocked + def segment(self, memo): + """Segment and package up memo into grams. + This is a stub method meant to be overridden in subclass + Returns: + grams (list): packed segments of memo - def serviceTxPkts(self): - """ - Service the .txPkts deque to send packets through server - Override in subclass + Parameters: + memo (str): to be segmented into grams """ - while self.opened and self.txPkts: - if not self._serviceOneTxPkt(): - break # blocked try again later + grams = [] + return grams - def serviceTxPktsOnce(self): - ''' - Service .txPkts deque once (one pkt) - ''' - if self.opened and self.txPkts: - self._serviceOneTxPkt() + + def _serviceOneTxMemo(self): + """Service one (memo, dst) duple from .txms deque where duple is of form + (memo: str, dst: str) where: + memo is outgoing memo + dst is destination address + Calls .segment method to process the segmentation and packing as + appropriate to convert memo into gram(s). + Appends duples of (gram, dst) from grams to .txgs deque. + """ + memo, dst = self.txms.popleft() # duple (msg, destination addr) + grams = self.segment(memo) + for gram in grams: + self.txgs.append((gram, dst)) - def transmit(self, pkt, ha=None): + def serviceTxMemosOnce(self): + """Service one outgoing memo from .txms deque if any (non-greedy) """ - Pack and Append (pkt, ha) duple to .txPkts deque - Override in subclass + if self.txms: + self._serviceOneTxMemo() + + + def serviceTxMemos(self): + """Service all outgoing memos in .txms deque if any (greedy) """ - try: - pkt.pack() - except ValueError as ex: - emsg = "{}: Error packing pkt.\n{}\n{}\n".format(self.name, pkt, ex) - console.terse(emsg) + while self.txms: + self._serviceOneTxMemo() - else: - self.txPkts.append((pkt, ha)) + def gramit(self, gram, dst): + """Append (gram, dst) duple to .txgs deque - def packetize(self, msg, remote=None): + Parameters: + gram (str): gram to be sent + dst (str): address of remote destination of gram """ - Returns packed packet created from msg destined for remote - Remote provided if attributes needed to fill in packet - Override in subclass + self.txgs.append((gram, dst)) + + + def _serviceOneTxGram(self): + """Service one (gram, dst) duple from .txgs deque for each unique dst + if more than one, where duple is of form + (gram: str, dst: str) where: + gram is outgoing gram segment from associated memo + dst is destination address + + Returns: + result (bool): True means full gram sent so can send another + False means full gram not sent so try again later + + Copies gram to .txbs buffer and sends to dst keeping track of the actual + portion sent and then deletes the sent portion from .txbs leaving the + remainder. + + When there is a remainder each subsequent call of this method + will attempt to send the remainder until the the full gram has been sent. + This accounts for datagramprotocols that expect continuing attempts to + send remainder of a datagram when using nonblocking sends. + + When there is no remainder then takes a gram from .txgs and sends it. + + Internally, an empty .txbs indicates its ok to take another gram from + .txgs and start sending it. + + The return value True or False enables backpressure on greedy callers + to block waiting for current gram to be fully sent. """ - packet = packeting.Packet(stack=self, packed=msg.encode('ascii')) - try: - packet.pack() - except ValueError as ex: - emsg = "{}: Error packing msg.\n{}\n{}\n".format(self.name, msg, ex) - console.terse(emsg) + if not self.txbs: # everything sent last time + gram, dst = self.txgs.popleft() + self.txbs.extend(gram) + + count = self.send(self.txbs, dst) # assumes opened - return None - return packet + if count < len(self.txbs): # delete sent portion + del self.txbs[:count] + return False # partially blocked try send of remainder later + self.clearTxbs() + return True # fully sent so can sent another gramm, not blocked - def _serviceOneTxMsg(self): + def _serviceOneTxPkt(self, laters, blockeds): """ - Handle one (message, remote) duple from .txMsgs deque + Service one (packet, ha) duple on .txPkts deque + Packet is assumed to be packed already in .packed Assumes there is a duple on the deque - Appends (packet, ha) duple to txPkts deque + laters is deque of packed packets to try again later + blockeds is list of ha destinations that have already blocked on this pass + Override in subclass """ - msg, remote = self.txMsgs.popleft() # duple (msg, destination uid - console.verbose("{0} sending to {1}\n{2}\n".format(self.name, - remote.name, - msg)) - packet = self.packetize(msg, remote) - if packet is not None: - self.txPkts.append((packet, remote.ha)) + pkt, ha = self.txPkts.popleft() # duple = (packet, destination address) + if ha in blockeds: # already blocked on this iteration + laters.append((pkt, ha)) # keep sequential + return False # blocked - def serviceTxMsgs(self): - """ - Service .txMsgs deque of outgoing messages - """ - while self.txMsgs: - self._serviceOneTxMsg() + try: + count = self.handler.send(pkt.packed, ha) # datagram always sends all + except socket.error as ex: + # ex.args[0] is always ex.errno for better compat + if (ex.args[0] in (errno.ECONNREFUSED, + errno.ECONNRESET, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.EHOSTDOWN, + errno.ETIMEDOUT, + errno.ETIME)): # ENOBUFS ENOMEM + # problem sending such as busy with last message. save it for later + laters.append((pkt, ha)) + blockeds.append(ha) + else: + raise - def serviceTxMsgOnce(self): - """ - Service .txMsgs deque once (one msg) - """ - if self.txMsgs: - self._serviceOneTxMsg() + console.profuse("{0}: sent to {1}\n 0x{2}\n".format(self.name, + ha, + hexlify(pkt.packed).decode('ascii'))) + return True # not blocked - def message(self, msg, remote=None): + def serviceTxPkts(self): """ - Append (msg, remote) duple to .txMsgs deque - If destination remote not given - Then use zeroth remote If any otherwise Raise exception + Service the .txPcks deque to send packets through server Override in subclass """ - if remote is None: - if not self.remotes: - emsg = "No remote to send to.\n" - console.terse(emsg) + if self.handler.opened: + laters = deque() + blockeds = [] + while self.txPkts: + again = self._serviceOneTxPkt(laters, blockeds) + if not again: + break + while laters: + self.txPkts.append(laters.popleft()) - return - remote = self.remotes.values()[0] - self.txMsgs.append((msg, remote)) - - def serviceAllTx(self): + def serviceTxPktsOnce(self): ''' - Service: - txMsgs queue - txes queue to server send + Service .txPkts deque once (one pkt) ''' - self.serviceTxMsgs() - self.serviceTxPkts() + if self.handler.opened: + laters = deque() + blockeds = [] # will always be empty since only once + if self.txPkts: + self._serviceOneTxPkt(laters, blockeds) + while laters: + self.txPkts.append(laters.popleft()) - def serviceAllTxOnce(self): + def serviceTxGramsOnce(self): + """Service one outgoing gram from .txgs deque if any (non-greedy) """ - Service the transmit side of the stack once (one transmission) - """ - self.serviceTxMsgOnce() - self.serviceTxPktsOnce() + if self.opened and self.txgs: + self._serviceOneTxGram() - def clearRxbs(self): - """ - Clear .rxbs + + def serviceTxGrams(self): + """Service all outgoing grams in .txgs deque if any (greedy) until + blocked by pending transmissions on transport. """ - del self.rxbs[:] # self.rxbs.clear() not supported before python 3.3 + while self.opened and self.txgs: + if not self._serviceOneTxGram(): + break # blocked try again later + - def parserize(self, raw): + def serviceAllTxOnce(self): + """Service the transmit side once (non-greedy) one transmission. """ - Returns packet parsed from raw data - Override in subclass + self.serviceTxMemosOnce() + self.serviceTxGramsOnce() + + + def serviceAllTx(self): + """Service the transmit side until empty (greedy) multiple transmissions + until blocked by pending transmission on transport. """ - packet = packeting.Packet(stack=self) - try: - packet.parse(raw=raw) - except ValueError as ex: - emsg = "{}: Error parsing raw.\n{}\n{}\n".format(self.name, raw, ex) - console.terse(emsg) - self.incStat("pkt_parse_error") - return None - return packet + self.serviceTxMemos() + self.serviceTxGrams() + def _serviceOneReceived(self): """ @@ -278,6 +473,39 @@ def _serviceOneReceived(self): self.rxPkts.append(packet) return True # received data + def _serviceOneReceived(self): + ''' + Service one received duple (raw, ha) raw packet data or chunk from server + assumes that there is a server + Override in subclass + ''' + try: + raw, ha = self.handler.receive() # if no data the duple is (b'', None) + except socket.error as ex: + # ex.args[0] always ex.errno for compat + if (ex.args[0] == (errno.ECONNREFUSED, + errno.ECONNRESET, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.EHOSTDOWN, + errno.ETIMEDOUT, + errno.ETIME)): + return False # no received data + else: + raise + + if not raw: # no received data + return False + + packet = self.parserize(raw, ha) + if packet is not None: + console.profuse("{0}: received\n 0x{1}\n".format(self.name, + hexlify(raw).decode('ascii'))) + self.rxPkts.append((packet, ha)) # duple = ( packed, source address) + return True # received data + def serviceReceives(self): """ Retrieve from server all received and queue up diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index a465a50..6c0fc9a 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -109,6 +109,9 @@ class Peer(filing.Filer): Class Attributes: Umask (int): octal default umask permissions such as 0o022 + MaxUxdPathSize (int:) max characters in uxd file path + UxdBufSize (int): used to set buffer size for UXD datagram buffers + MaxUxdPayloadSize (int): limit datagram payload to this size Attributes: umask (int): unpermission mask for uxd file, usually octal 0o022 @@ -131,11 +134,15 @@ class Peer(filing.Filer): Mode = "r+" Fext = "uxd" Umask = 0o022 # default + MaxUxdPathSize = 108 + UxdBufSize = 64535 + MaxUxdPayloadSize = UxdBufSize - def __init__(self, *, umask=None, bs = 1024, wl=None, + def __init__(self, *, umask=None, bs = None, wl=None, reopen=False, clear=True, filed=False, extensioned=True, **kwa): """Initialization method for instance. + Inherited Parameters: reopen (bool): True (re)open with this init False not (re)open with this init but later (default) @@ -155,7 +162,7 @@ def __init__(self, *, umask=None, bs = 1024, wl=None, wl (WireLog): instance ref for debug logging of over the wire tx and rx """ self.umask = umask # only change umask if umask is not None below - self.bs = bs + self.bs = bs if bs is not None else self.UxdBufSize self.wl = wl self.ls = None # local socket of this Peer, needs to be opened/bound @@ -166,6 +173,7 @@ def __init__(self, *, umask=None, bs = 1024, wl=None, **kwa) + def actualBufSizes(self): """Returns: sizes (tuple); duple of the form (int, int) of the (tx, rx) @@ -188,6 +196,11 @@ def open(self): if socket not closed properly, binding socket gets error OSError: (48, 'Address already in use') """ + if len(self.path) > self.MaxUxdPathSize: + self.close() + raise hioing.SizeError(f"UXD path={self.path} too long, > " + f"{self.MaxUxdPathSize}.") + #create socket ss = server socket self.ls = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) @@ -196,7 +209,8 @@ def open(self): # TIME_WAIT state, without waiting for its natural timeout to expire. self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - # setup buffers + # setup buffers. With uxd sockets only the SO_SNDBUF matters, but set + # both anyway if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) < self.bs: self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.bs) if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) < self.bs: diff --git a/src/hio/hioing.py b/src/hio/hioing.py index bb47a05..a6f9f2b 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -25,6 +25,12 @@ class HioError(Exception): To use raise HioError("Error: message") """ +class SizeError(HioError): + """ + Resource size related errors + Usage: + raise SizeError("error message") + """ class ValidationError(HioError): """ diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxd.py index b59f003..6f06b21 100644 --- a/tests/core/uxd/test_uxd.py +++ b/tests/core/uxd/test_uxd.py @@ -11,6 +11,7 @@ import tempfile import shutil +from hio import hioing from hio.base import tyming, doing from hio.core import wiring from hio.core.uxd import uxding @@ -156,6 +157,7 @@ def test_open_peer(): assert alpha.opened assert alpha.path.endswith("alpha.uxd") + assert alpha.actualBufSizes() == (64535, 64535) == (alpha.UxdBufSize, alpha.UxdBufSize) assert beta.opened assert beta.path.endswith("beta.uxd") @@ -265,6 +267,17 @@ def test_open_peer(): """Done Test""" +def test_uxd_path_len(): + """ Test the uxd path length + + """ + with pytest.raises(hioing.SizeError): + name = "alpha" * 25 + assert len(name) > uxding.Peer.MaxUxdPathSize + alpha = uxding.Peer(name=name, temp=True, reopen=True) + """Done Test""" + + def test_peer_doer(): """ Test PeerDoer class @@ -304,4 +317,5 @@ def test_peer_doer(): if __name__ == "__main__": test_uxd_basic() test_open_peer() + test_uxd_path_len() test_peer_doer() From 274c77cf418d06a241dc73f28efee30e735f9420 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 13 Feb 2025 19:17:00 -0700 Subject: [PATCH 06/33] fixed typos --- src/hio/core/uxd/uxding.py | 4 ++-- tests/core/uxd/test_uxd.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 6c0fc9a..ab5aa4b 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -135,8 +135,8 @@ class Peer(filing.Filer): Fext = "uxd" Umask = 0o022 # default MaxUxdPathSize = 108 - UxdBufSize = 64535 - MaxUxdPayloadSize = UxdBufSize + UxdBufSize = 65535 # 2 ** 16 - 1 + MaxUxdGramSize = UxdBufSize def __init__(self, *, umask=None, bs = None, wl=None, reopen=False, clear=True, diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxd.py index 6f06b21..56619f0 100644 --- a/tests/core/uxd/test_uxd.py +++ b/tests/core/uxd/test_uxd.py @@ -157,7 +157,7 @@ def test_open_peer(): assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert alpha.actualBufSizes() == (64535, 64535) == (alpha.UxdBufSize, alpha.UxdBufSize) + assert alpha.actualBufSizes() == (65535, 65535) == (alpha.UxdBufSize, alpha.UxdBufSize) assert beta.opened assert beta.path.endswith("beta.uxd") From c738e699e85b49c0172d24b4db8de35fcae873a1 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 13 Feb 2025 19:27:04 -0700 Subject: [PATCH 07/33] update comments --- src/hio/core/memoing.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index c20b953..f57c962 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -78,14 +78,21 @@ of outgoing datagrams. This limit is calculated as the doubled (see socket(7)) option value less 32 bytes used for overhead. - +Doubling accounts for the overhead. SO_SNDBUF - Sets or gets the maximum socket send buffer in bytes. The kernel doubles this value (to allow - space for bookkeeping overhead) when it is set using setsockopt(2), and this doubled value is - returned by getsockopt(2). The default value is set by the /proc/sys/net/core/wmem_default file - and the maximum allowed value is set by the /proc/sys/net/core/wmem_max file. The minimum - (doubled) value for this option is 2048. - + Sets or gets the maximum socket send buffer in bytes. The + kernel doubles this value (to allow space for bookkeeping + overhead) when it is set using setsockopt(2), and this + doubled value is returned by getsockopt(2). The default + value is set by the /proc/sys/net/core/wmem_default file + and the maximum allowed value is set by the + /proc/sys/net/core/wmem_max file. The minimum (doubled) + value for this option is 2048. + +In python the getsockopt apparently does not return the doubled value but +the undoubled value. + +https://unix.stackexchange.com/questions/38043/size-of-data-that-can-be-written-to-read-from-sockets https://stackoverflow.com/questions/21856517/whats-the-practical-limit-on-the-size-of-single-packet-transmitted-over-domain """ From 9d08c6344344e3b3be366dbe8b6cf744177f7bdb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 14 Feb 2025 16:08:11 -0700 Subject: [PATCH 08/33] finished first pass through transmit side of code. --- src/hio/core/memoing.py | 280 ++++++++++++++++++++-------------------- 1 file changed, 139 insertions(+), 141 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index f57c962..f0944d9 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -131,10 +131,12 @@ class Memoir(hioing.Mixin): the constraints of non-blocking asynchronous IO with datagram transports. On the transmit side memos are placed in a deque (double ended queue). Each - memor is then segmented into grams (datagrams) the respect the size constraints - of the underlying datagram transport. These grams are placed in another deque. - Each gram in its deque is then places in a transmit buffer to be sent over the - transport. When using non-blocking IO, asynchronous datagram transport + memo is then segmented into grams (datagrams) that respect the size constraints + of the underlying datagram transport. These grams are placed in a gram deque. + Each gram in this deque is then placed in a transmit buffer dict whose key + is the destination and value is the gram. Each item is to be sent + over the transport. One item buffer per unique destination. + When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation of the sent datagram which means the whole datagram is not sent at once via a non-blocking send call. This means that the remainer of the datagram must @@ -143,7 +145,9 @@ class Memoir(hioing.Mixin): depends on the sender repeated attempts to send the reaminder of the full datagram. This is ensured with the final tier with a raw transmit buffer that waits until it is empty before attempting to send another - gram. + gram. Because sending to different destinations may fail for different reasons + such as bad addresses or bad routing each destination gets its own buffer + so that a bad destination does not block other destinations. On the receive the raw data is received into the raw data buffer. This is then converted into a gram and queued in the gram receive deque as a seqment of @@ -161,12 +165,21 @@ class Memoir(hioing.Mixin): Attributes: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - rxbs (bytearray): buffer holding rx (receive) data from raw transport - txbs (bytearray): buffer holding tx (transmit) raw data to raw transport - rxgs (deque): holding rx (receive) (data) grams (segments) recieved via rxbs raw - txgs (deque): holding tx (transmit) (data) grams (segments) to be sent via txbs raw - rxms (deque): holding rx (receive) memos desegmented from rxgs grams - txms (deque): holding tx (transmit) memos to be segmented into txgs grams + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is duple of form + (memo: str, dst: str) + txgs (dict): holding tx (transmit) (data) gram deques of grams. + Each item in dict has key=dst and value=deque of grams to be + sent via txbs of form (dst: str, grams: deque). Each entry + in deque is bytes of gram. + txbs (dict): holding tx (transmit) raw data items to transport. + Each item in dict has key=dst and value=bytearray holding + unsent portion of gram bytes of form (dst: str, gram: bytearray). + rxbs (bytearray): buffer holding rx (receive) raw bytes from transport + rxgs (deque): holding rx (receive) (data) gram tuples recieved via rxbs raw + each entry in deque is duple of form (gram: bytes, dst: str) + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is duple of form (memo: str, dst: str) Properties: @@ -178,12 +191,12 @@ class Memoir(hioing.Mixin): def __init__(self, version=None, - rxbs=None, + txms=None, + txgs=None, txbs=None, + rxbs=None, rxgs=None, - txgs=None, rxms=None, - txms=None, **kwa ): """Setup instance @@ -194,31 +207,33 @@ def __init__(self, Parameters: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - - txbs (bytearray): buffer holding tx (transmit) raw data to raw transport - rxbs (bytearray): buffer holding rx (receive) data from raw transport - - txgs (deque): holding tx (transmit) (data) grams to be sent via txbs raw - rxgs (deque): holding rx (receive) (data) grams recieved via rxbs raw - - txms (deque): holding tx (transmit) memos to be segmented into txgs grams - rxms (deque): holding rx (receive) memos desegmented from rxgs grams - + txms (deque): holding tx (transmit) memo tuples to be segmented into + txgs grams where each entry in deque is duple of form + (memo: str, dst: str) + txgs (dict): holding tx (transmit) (data) gram deques of grams. + Each item in dict has key=dst and value=deque of grams to be + sent via txbs of form (dst: str, grams: deque). Each entry + in deque is bytes of gram. + txbs (dict): holding tx (transmit) raw data items to transport. + Each item in dict has key=dst and value=bytearray holding + unsent portion of gram bytes of form (dst: str, gram: bytearray). + rxbs (bytearray): buffer holding rx (receive) raw bytes from transport + rxgs (deque): holding rx (receive) (data) gram tuples recieved via rxbs raw + each entry in deque is duple of form (gram: bytes, dst: str) + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is duple of form (memo: str, dst: str) """ # initialize attributes self.version = version if version is not None else self.Version + self.txms = txms if txms is not None else deque() + self.txgs = txgs if txgs is not None else dict() + self.txbs = txbs if txbs is not None else dict() - self.txbs = txbs if txbs is not None else bytearray() self.rxbs = rxbs if rxbs is not None else bytearray() - - self.txgs = txgs if txgs is not None else deque() self.rxgs = rxgs if rxgs is not None else deque() - - - self.txms = txms if txms is not None else deque() self.rxms = rxms if rxms is not None else deque() super(Memoir, self).__init__(**kwa) @@ -240,12 +255,6 @@ def send(self, txbs, dst): """ return 0 - def clearTxbs(self): - """ - Clear .txbs - """ - self.txbs.clear() - def clearRxbs(self): """ @@ -269,7 +278,7 @@ def segment(self, memo): This is a stub method meant to be overridden in subclass Returns: - grams (list): packed segments of memo + grams (list[bytes]): packed segments of memo, each seqment is bytes Parameters: memo (str): to be segmented into grams @@ -290,7 +299,9 @@ def _serviceOneTxMemo(self): memo, dst = self.txms.popleft() # duple (msg, destination addr) grams = self.segment(memo) for gram in grams: - self.txgs.append((gram, dst)) + if dst not in self.txgs: + self.txgs[dst] = deque() + self.txgs[dst].append(gram) def serviceTxMemosOnce(self): @@ -317,127 +328,114 @@ def gramit(self, gram, dst): self.txgs.append((gram, dst)) - def _serviceOneTxGram(self): - """Service one (gram, dst) duple from .txgs deque for each unique dst - if more than one, where duple is of form - (gram: str, dst: str) where: - gram is outgoing gram segment from associated memo - dst is destination address + def _serviceOnceTxGrams(self): + """Service one pass over .txgs dict for each unique dst in .txgs and then + services one pass over the .txbs of those outgoing grams + + Each item in.txgs is (key: dst, val: deque of grams) + where: + dst is destination address + gram is outgoing gram segment from associated memo + + Each item in.txbs is (key: dst, val: buf of gram) + where: + dst (str): destination address + buf (bytearray): outgoing gram segment or remainder of segment Returns: - result (bool): True means full gram sent so can send another - False means full gram not sent so try again later + result (bool): True means at least one destination is unblocked + so can keep sending. Unblocked at destination means + last full gram sent with another sitting in deque. + False means all destinations blocked so try again later. + + For each destination, if .txbs item is empty creates new item at destination + and copies gram to bytearray. - Copies gram to .txbs buffer and sends to dst keeping track of the actual - portion sent and then deletes the sent portion from .txbs leaving the - remainder. + Once it has serviced each item in .txgs then services each items in .txbs. + For each destination in .txbs, attempts to send bytearray to dst + keeping track of the actual portion sent and then deletes the sent + portion from item in .txbs leaving the remainder. When there is a remainder each subsequent call of this method will attempt to send the remainder until the the full gram has been sent. - This accounts for datagramprotocols that expect continuing attempts to + This accounts for datagram protocols that expect continuing attempts to send remainder of a datagram when using nonblocking sends. - When there is no remainder then takes a gram from .txgs and sends it. - - Internally, an empty .txbs indicates its ok to take another gram from - .txgs and start sending it. - - The return value True or False enables backpressure on greedy callers - to block waiting for current gram to be fully sent. - """ - if not self.txbs: # everything sent last time - gram, dst = self.txgs.popleft() - self.txbs.extend(gram) - - count = self.send(self.txbs, dst) # assumes opened - - if count < len(self.txbs): # delete sent portion - del self.txbs[:count] - return False # partially blocked try send of remainder later - - self.clearTxbs() - return True # fully sent so can sent another gramm, not blocked - - def _serviceOneTxPkt(self, laters, blockeds): - """ - Service one (packet, ha) duple on .txPkts deque - Packet is assumed to be packed already in .packed - Assumes there is a duple on the deque - laters is deque of packed packets to try again later - blockeds is list of ha destinations that have already blocked on this pass - Override in subclass - """ - pkt, ha = self.txPkts.popleft() # duple = (packet, destination address) - - if ha in blockeds: # already blocked on this iteration - laters.append((pkt, ha)) # keep sequential - return False # blocked - - try: - count = self.handler.send(pkt.packed, ha) # datagram always sends all - except socket.error as ex: - # ex.args[0] is always ex.errno for better compat - if (ex.args[0] in (errno.ECONNREFUSED, - errno.ECONNRESET, - errno.ENETRESET, - errno.ENETUNREACH, - errno.EHOSTUNREACH, - errno.ENETDOWN, - errno.EHOSTDOWN, - errno.ETIMEDOUT, - errno.ETIME)): # ENOBUFS ENOMEM - # problem sending such as busy with last message. save it for later - laters.append((pkt, ha)) - blockeds.append(ha) - else: - raise - - console.profuse("{0}: sent to {1}\n 0x{2}\n".format(self.name, - ha, - hexlify(pkt.packed).decode('ascii'))) - return True # not blocked + When there is no remainder then removes .txbs buffer at dst so that + subsequent grams are reordered with respect to dst. + + Internally, an empty .txbs at a destination indicates its ok to take + another gram from its .txgs deque if any and start sending it. + + The return value True or False enables back pressure on greedy callers + so they know when to block waiting for at least one unblocked destination + with a pending gram. + + Deleting an item from a dict at a key (since python dicts are key creation + ordered) means that the next time an item is created at that key, that + item will be last in order. In order to dynamically change the ordering + of iteration over destinations, when there are no pending grams for a + given destination we remove its dict item. This reorders the destination + as last when a new gram is created and avoids iterating over destinations + with no pending grams. + """ + # service grams by reloading buffers from grams + for dst in list(self.txgs.keys()): # list since items may be deleted in loop + # if dst then grams deque at dst must not be empty + if dst not in self.txbs: # no transmit buffer + self.txbs[dst] = bytearray(self.txgs[dst].popleft()) # new gram + if not self.txgs[dst]: # deque at dst empty so remove deque + del self.txgs[dst] # makes dst unordered until next memo at dst + + + # service buffers by attempting to send + sents = [False] * len(self.txbs) # list of sent states True unblocked False blocked + for i, dst in enumerate(list(self.txbs.keys())): # list since items may be deleted in loop + # if dst then bytearray at dst must not be empty + try: + count = self.send(self.txbs[dst], dst) # assumes .opened == True + except socket.error as ex: # OSError.errno always .args[0] for compat + if (ex.args[0] in (errno.ECONNREFUSED, + errno.ECONNRESET, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.EHOSTDOWN, + errno.ETIMEDOUT, + errno.ETIME, + errno.ENOBUFS, + errno.ENOMEM)): # problem sending try again later + count = 0 # nothing sent + else: + raise + + del self.txbs[dst][:count] # remove from buffer those bytes sent + if not self.txbs[dst]: # empty buffer so gram fully sent + if dst in self.txgs: # another gram waiting + sents[i] = True # not blocked with waiting gram + del self.txbs[dst] # remove so dst is unordered until next gram + + return any(sents) # at least one unblocked destingation with waiting gram - def serviceTxPkts(self): - """ - Service the .txPcks deque to send packets through server - Override in subclass - """ - if self.handler.opened: - laters = deque() - blockeds = [] - while self.txPkts: - again = self._serviceOneTxPkt(laters, blockeds) - if not again: - break - while laters: - self.txPkts.append(laters.popleft()) - - def serviceTxPktsOnce(self): - ''' - Service .txPkts deque once (one pkt) - ''' - if self.handler.opened: - laters = deque() - blockeds = [] # will always be empty since only once - if self.txPkts: - self._serviceOneTxPkt(laters, blockeds) - while laters: - self.txPkts.append(laters.popleft()) def serviceTxGramsOnce(self): - """Service one outgoing gram from .txgs deque if any (non-greedy) + """Service one pass (non-greedy) over all unique destinations in .txgs + dict if any for blocked destination or unblocked with pending outgoing + grams. """ if self.opened and self.txgs: - self._serviceOneTxGram() + self._serviceOnceTxGrams() def serviceTxGrams(self): - """Service all outgoing grams in .txgs deque if any (greedy) until - blocked by pending transmissions on transport. + """Service multiple passes (greedy) over all unqique destinations in + .txgs dict if any for blocked destinations or unblocked with pending + outgoing grams until there is no unblocked destination with a pending gram. """ while self.opened and self.txgs: - if not self._serviceOneTxGram(): - break # blocked try again later + if not self._serviceOnceTxGrams(): # no pending gram on any unblocked dst, + break # so try again later def serviceAllTxOnce(self): From 0be9f1b32f36b020eec55ff83316afac295be3c8 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 15 Feb 2025 16:36:08 -0700 Subject: [PATCH 09/33] first draft of receive side of Memoir done. Now write unit tests. --- src/hio/core/memoing.py | 427 ++++++++++++++++++++++------------------ 1 file changed, 231 insertions(+), 196 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index f0944d9..00006d1 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -127,9 +127,15 @@ class Memoir(hioing.Mixin): transport class. For example MemoirUdp or MemoGramUdp or MemoirUxd or MemoGramUXD - Each direction of dataflow uses a triple-tiered set of buffers that respect + Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. + On the receive side each complete datagram (gram) is put in a gram receive + deque as a segment of a memo. These deques are indexed by the sender's + source addr. The grams in the gram recieve deque are then desegmented into a memo + and placed in the memo deque for consumption by the application or some other + higher level protocol. + On the transmit side memos are placed in a deque (double ended queue). Each memo is then segmented into grams (datagrams) that respect the size constraints of the underlying datagram transport. These grams are placed in a gram deque. @@ -149,12 +155,6 @@ class Memoir(hioing.Mixin): such as bad addresses or bad routing each destination gets its own buffer so that a bad destination does not block other destinations. - On the receive the raw data is received into the raw data buffer. This is then - converted into a gram and queued in the gram receive deque as a seqment of - a memo. The grams in the gram recieve deque are then desegmented into a memo - and placed in the memo deque for consumption by the application or some other - higher level protocol. - Memo segmentation information is embedded in the grams. @@ -165,21 +165,21 @@ class Memoir(hioing.Mixin): Attributes: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) + rxgs (dict): holding rx (receive) (data) gram deques of grams. + Each item in dict has key=src and val=deque of grames received + from transport. Each item of form (src: str, gram: deque) + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is duple of form (memo: str, dst: str) txgs (dict): holding tx (transmit) (data) gram deques of grams. - Each item in dict has key=dst and value=deque of grams to be + Each item in dict has key=dst and val=deque of grams to be sent via txbs of form (dst: str, grams: deque). Each entry in deque is bytes of gram. txbs (dict): holding tx (transmit) raw data items to transport. Each item in dict has key=dst and value=bytearray holding unsent portion of gram bytes of form (dst: str, gram: bytearray). - rxbs (bytearray): buffer holding rx (receive) raw bytes from transport - rxgs (deque): holding rx (receive) (data) gram tuples recieved via rxbs raw - each entry in deque is duple of form (gram: bytes, dst: str) - rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams - each entry in deque is duple of form (memo: str, dst: str) Properties: @@ -191,12 +191,11 @@ class Memoir(hioing.Mixin): def __init__(self, version=None, + rxgs=None, + rxms=None, txms=None, txgs=None, txbs=None, - rxbs=None, - rxgs=None, - rxms=None, **kwa ): """Setup instance @@ -207,6 +206,11 @@ def __init__(self, Parameters: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) + rxgs (dict): holding rx (receive) (data) gram deques of grams. + Each item in dict has key=src and val=deque of grames received + from transport. Each item of form (src: str, gram: deque) + rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is duple of form (memo: str, dst: str) @@ -217,31 +221,215 @@ def __init__(self, txbs (dict): holding tx (transmit) raw data items to transport. Each item in dict has key=dst and value=bytearray holding unsent portion of gram bytes of form (dst: str, gram: bytearray). - rxbs (bytearray): buffer holding rx (receive) raw bytes from transport - rxgs (deque): holding rx (receive) (data) gram tuples recieved via rxbs raw - each entry in deque is duple of form (gram: bytes, dst: str) - rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams - each entry in deque is duple of form (memo: str, dst: str) """ # initialize attributes self.version = version if version is not None else self.Version + self.rxgs = rxgs if rxgs is not None else dict() + self.rxms = rxms if rxms is not None else deque() + self.txms = txms if txms is not None else deque() self.txgs = txgs if txgs is not None else dict() self.txbs = txbs if txbs is not None else dict() - self.rxbs = rxbs if rxbs is not None else bytearray() - self.rxgs = rxgs if rxgs is not None else deque() - self.rxms = rxms if rxms is not None else deque() - super(Memoir, self).__init__(**kwa) if not hasattr(self, "opened"): # stub so mixin works in isolation self.opened = False # mixed with subclass should provide this. + + + + def receive(self): + """Attemps to send bytes in txbs to remote destination dst. Must be + overridden in subclass. This is a stub to define mixin interface. + + Returns: + duple (tuple): of form (data: bytes, src: str) where data is the + bytes of received data and src is the source address. + When no data the duple is (b'', None) + """ + return (b'', None) + + + def _serviceOneReceived(self): + """Service one received duple (raw, src) raw packet data. Always returns + complete datagram. + + Returns: + result (bool): True means data received from source over transport + False means no data so try again later + + return enables greedy callers to keep calling until no + more data to receive from transport + """ + try: + gram, src = self.receive() # if no data the duple is (b'', None) + except socket.error as ex: # OSError.errno always .args[0] for compat + if (ex.args[0] == (errno.ECONNREFUSED, + errno.ECONNRESET, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.EHOSTDOWN, + errno.ETIMEDOUT, + errno.ETIME, + errno.ENOBUFS, + errno.ENOMEM)): + return False # no received data + else: + raise + + if not gram: # no received data + return False + + if src not in self.rxgs: + self.rxgs[src] = deque() + + self.rxgs[src].append(fram) + return True # received data + + + def serviceReceivesOnce(self): + """Service receives once (non-greedy) and queue up + """ + if self.opened: + self._serviceOneReceived() + + + def serviceReceives(self): + """Service all receives (greedy) and queue up + """ + while self.opened: + if not self._serviceOneReceived(): + break + + + def desegment(self, grams): + """Desegment deque of grams as segments into whole memo. If grams is + missing all the segments then returns None. + + Returns: + memo (str | None): desegmented memo or None if incomplete. + + Override in subclass + + Parameters: + grams (deque): gram segments + """ + return "" + + + def _serviceOnceRxGrams(self): + """Service one pass over .rxgs dict for each unique src in .rxgs + + Returns: + result (bool): True means at least one src has received a memo and + has writing grams so can keep desegmenting. + False means all sources waiting for more grams + so try again later. + + The return value True or False enables back pressure on greedy callers + so they know when to block waiting for at least one source with received + memo and additional grams to desegment. + + Deleting an item from a dict at a key (since python dicts are key creation + ordered) means that the next time an item is created at that key, that + item will be last in order. In order to dynamically change the ordering + of iteration over sources, when there are no received grams from a + given source we remove its dict item. This reorders the source + as last when a new gram is received and avoids iterating over sources + with no received grams. + + """ + goods = [False] * len(self.rxgs) # list by src, True memo False no-memo + # service grams to desegment + for i, src in enumerate(list(self.rxgs.keys())): # list since items may be deleted in loop + # if src then grams deque at src must not be empty + memo = self.desegment(self.rxgs[src]) + if memo is not None: # could be empty memo for some src + self.rxms.append((memo, src)) + + if not self.txgs[src]: # deque at src empty so remove deque + del self.txgs[src] # makes src unordered until next memo at src + + if memo is not None: # recieved and desegmented memo + if src in self.txgs: # more received gram(s) at src + goods[i] = True # indicate memo recieved with more received gram(s) + + return any(goods) # at least one memo from a source with more received grams + + + + + def serviceRxGramsOnce(self): + """Service one pass (non-greedy) over all unique sources in .rxgs + dict if any for received incoming grams. + """ + if self.rxgs: + self._serviceOnceRxGrams() + + + def serviceRxGrams(self): + """Service multiple passes (greedy) over all unqique sources in + .rxgs dict if any for sources with complete desegmented memos and more + incoming grams. + """ + while self.rxgs: + if not self._serviceOnceRxGrams(): + break + + + def _serviceOneRxMemo(self): + """Service one duple from .rxMsgs deque + + Returns: + duple (tuple | None): of form (memo: str, src: str) if any + otherwise None + + Override in subclass to consume + """ + return (self.rxms.popleft()) + + + def serviceRxMemosOnce(self): + """Service memos in .rxms deque once (non-greedy one memo) if any + + Override in subclass to handle result and put it somewhere + """ + if self.rxms: + result = self._serviceOneRxMemo() + + + def serviceRxMemos(self): + """Service all memos in .rxms (greedy) if any + + Override in subclass to handle result(s) and put them somewhere + """ + while self.rxms: + result = self._serviceOneRxMsg() + + + def serviceAllRxOnce(self): + """Service receive side of stack once (non-greedy) + """ + self.serviceReceivesOnce() + self.serviceRxGramsOnce() + self.serviceRxMemosOnce() + + + def serviceAllRx(self): + """Service receive side of stack (greedy) until empty or waiting for more + """ + self.serviceReceives() + self.serviceRxGrams() + self.serviceRxMemos() + + def send(self, txbs, dst): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. @@ -253,15 +441,9 @@ def send(self, txbs, dst): txbs (bytes | bytearray): of bytes to send dst (str): remote destination address """ - return 0 + return len(txbs) # all sent - def clearRxbs(self): - """ - Clear .rxbs - """ - del self.rxbs.clear() - def memoit(self, memo, dst): """Append (memo, dst) duple to .txms deque @@ -348,6 +530,10 @@ def _serviceOnceTxGrams(self): last full gram sent with another sitting in deque. False means all destinations blocked so try again later. + The return value True or False enables back pressure on greedy callers + so they know when to block waiting for at least one unblocked destination + with a pending gram. + For each destination, if .txbs item is empty creates new item at destination and copies gram to bytearray. @@ -367,10 +553,6 @@ def _serviceOnceTxGrams(self): Internally, an empty .txbs at a destination indicates its ok to take another gram from its .txgs deque if any and start sending it. - The return value True or False enables back pressure on greedy callers - so they know when to block waiting for at least one unblocked destination - with a pending gram. - Deleting an item from a dict at a key (since python dicts are key creation ordered) means that the next time an item is created at that key, that item will be last in order. In order to dynamically change the ordering @@ -389,7 +571,7 @@ def _serviceOnceTxGrams(self): # service buffers by attempting to send - sents = [False] * len(self.txbs) # list of sent states True unblocked False blocked + goods = [False] * len(self.txbs) # list by dst, True unblocked False blocked for i, dst in enumerate(list(self.txbs.keys())): # list since items may be deleted in loop # if dst then bytearray at dst must not be empty try: @@ -413,10 +595,10 @@ def _serviceOnceTxGrams(self): del self.txbs[dst][:count] # remove from buffer those bytes sent if not self.txbs[dst]: # empty buffer so gram fully sent if dst in self.txgs: # another gram waiting - sents[i] = True # not blocked with waiting gram + goods[i] = True # idicate not blocked and with waiting gram del self.txbs[dst] # remove so dst is unordered until next gram - return any(sents) # at least one unblocked destingation with waiting gram + return any(goods) # at least one unblocked destingation with waiting gram def serviceTxGramsOnce(self): @@ -453,174 +635,27 @@ def serviceAllTx(self): self.serviceTxGrams() - def _serviceOneReceived(self): - """ - Service one received raw packet data or chunk from .handler - assumes that there is a .handler - Override in subclass - """ - while True: # keep receiving until empty - try: - raw = self.receive() - except Exception as ex: - raise - - if not raw: - return False # no received data - self.rxbs.extend(raw) - - packet = self.parserize(self.rxbs[:]) - - if packet is not None: # queue packet - console.profuse("{0}: received\n 0x{1}\n".format(self.name, - hexlify(self.rxbs[:packet.size]).decode('ascii'))) - del self.rxbs[:packet.size] - self.rxPkts.append(packet) - return True # received data - - def _serviceOneReceived(self): - ''' - Service one received duple (raw, ha) raw packet data or chunk from server - assumes that there is a server - Override in subclass - ''' - try: - raw, ha = self.handler.receive() # if no data the duple is (b'', None) - except socket.error as ex: - # ex.args[0] always ex.errno for compat - if (ex.args[0] == (errno.ECONNREFUSED, - errno.ECONNRESET, - errno.ENETRESET, - errno.ENETUNREACH, - errno.EHOSTUNREACH, - errno.ENETDOWN, - errno.EHOSTDOWN, - errno.ETIMEDOUT, - errno.ETIME)): - return False # no received data - else: - raise - - if not raw: # no received data - return False - - packet = self.parserize(raw, ha) - if packet is not None: - console.profuse("{0}: received\n 0x{1}\n".format(self.name, - hexlify(raw).decode('ascii'))) - self.rxPkts.append((packet, ha)) # duple = ( packed, source address) - return True # received data - - def serviceReceives(self): - """ - Retrieve from server all received and queue up - """ - while self.opened: - if not self._serviceOneReceived(): - break - - def serviceReceivesOnce(self): - """ - Service receives once (one reception) and queue up - """ - if self.opened: - self._serviceOneReceived() - - def messagize(self, pkt, ha): - """ - Returns duple of (message, remote) converted from rx packet pkt and ha - Override in subclass - """ - msg = pkt.packed.decode("ascii") - try: - remote = self.haRemotes[ha] - except KeyError as ex: - console.verbose(("{0}: Dropping packet received from unknown remote " - "ha '{1}'.\n{2}\n".format(self.name, ha, pkt.packed))) - return (None, None) - return (msg, remote) - - - - def _serviceOneRxPkt(self): - """ - Service pkt from .rxPkts deque - Assumes that there is a message on the .rxes deque - """ - pkt = self.rxPkts.popleft() - console.verbose("{0} received packet\n{1}\n".format(self.name, pkt.show())) - self.incStat("pkt_received") - message = self.messagize(pkt) - if message is not None: - self.rxMsgs.append(message) - - def serviceRxPkts(self): - """ - Process all duples in .rxPkts deque - """ - while self.rxPkts: - self._serviceOneRxPkt() - - def serviceRxPktsOnce(self): - """ - Service .rxPkts deque once (one pkt) - """ - if self.rxPkts: - self._serviceOneRxPkt() - - def _serviceOneRxMsg(self): - """ - Service one duple from .rxMsgs deque + def serviceLocal(self): + """Service the local Peer's receive and transmit queues """ - msg = self.rxMsgs.popleft() - + self.serviceReceives() + self.serviceTxGrams() - def serviceRxMsgs(self): - """ - Service .rxMsgs deque of duples - """ - while self.rxMsgs: - self._serviceOneRxMsg() - def serviceRxMsgsOnce(self): - """ - Service .rxMsgs deque once (one msg) + def serviceAllOnce(self): + """Service all Rx and Tx Once (non-greedy) """ - if self.rxMsgs: - self._serviceOneRxMsg() + self.serviceAllRxOnce() + self.serviceAllTxOnce() - def serviceAllRx(self): - """ - Service receive side of stack - """ - self.serviceReceives() - self.serviceRxPkts() - self.serviceRxMsgs() - self.serviceTimers() - - def serviceAllRxOnce(self): - """ - Service receive side of stack once (one reception) - """ - self.serviceReceivesOnce() - self.serviceRxPktsOnce() - self.serviceRxMsgsOnce() - self.serviceTimers() - def serviceAll(self): - """ - Service all Rx and Tx + """Service all Rx and Tx (greedy) """ self.serviceAllRx() self.serviceAllTx() - def serviceLocal(self): - """ - Service the local Peer's receive and transmit queues - """ - self.serviceReceives() - self.serviceTxPkts() + From 2f0920eb3604d6bb6aec5bcb15d1ce8b8041418e Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 16 Feb 2025 09:18:26 -0700 Subject: [PATCH 10/33] added comments and some cleanup --- src/hio/core/memoing.py | 22 ++++++++++++++-------- src/hio/help/decking.py | 9 +++++++++ 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 00006d1..3662e40 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -351,7 +351,7 @@ def _serviceOnceRxGrams(self): for i, src in enumerate(list(self.rxgs.keys())): # list since items may be deleted in loop # if src then grams deque at src must not be empty memo = self.desegment(self.rxgs[src]) - if memo is not None: # could be empty memo for some src + if memo is not None: # allows for empty "" memo for some src self.rxms.append((memo, src)) if not self.txgs[src]: # deque at src empty so remove deque @@ -391,9 +391,8 @@ def _serviceOneRxMemo(self): duple (tuple | None): of form (memo: str, src: str) if any otherwise None - Override in subclass to consume """ - return (self.rxms.popleft()) + return (self.rxms.popleft()) # raises IndexError if empty deque def serviceRxMemosOnce(self): @@ -401,8 +400,11 @@ def serviceRxMemosOnce(self): Override in subclass to handle result and put it somewhere """ - if self.rxms: - result = self._serviceOneRxMemo() + try: + pass + #memo, src = self._serviceOneRxMemo() + except IndexError: + pass def serviceRxMemos(self): @@ -411,7 +413,8 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - result = self._serviceOneRxMsg() + break + # memo, src = self._serviceOneRxMsg() def serviceAllRxOnce(self): @@ -478,7 +481,7 @@ def _serviceOneTxMemo(self): appropriate to convert memo into gram(s). Appends duples of (gram, dst) from grams to .txgs deque. """ - memo, dst = self.txms.popleft() # duple (msg, destination addr) + memo, dst = self.txms.popleft() # raises IndexError if empty deque grams = self.segment(memo) for gram in grams: if dst not in self.txgs: @@ -489,8 +492,11 @@ def _serviceOneTxMemo(self): def serviceTxMemosOnce(self): """Service one outgoing memo from .txms deque if any (non-greedy) """ - if self.txms: + + try: self._serviceOneTxMemo() + except IndexError: + pass def serviceTxMemos(self): diff --git a/src/hio/help/decking.py b/src/hio/help/decking.py index a364801..e249f67 100644 --- a/src/hio/help/decking.py +++ b/src/hio/help/decking.py @@ -19,6 +19,13 @@ class Deck(deque): instead of raising IndexError. This allows use of the walrus operator on a pull to both assign and check for empty. For example: + deck.extend([False, "", []]) # falsy elements but not None + stuff = [] + if x := deck.pull(emptive=True)) is not None: + stuff.append(x) # do something with x + assert stuff == [False] + + deck.extend([False, "", []]) # falsy elements but not None stuff = [] while (x := deck.pull(emptive=True)) is not None: @@ -26,6 +33,8 @@ class Deck(deque): assert stuff == [False, "", []] assert not deck + + Local methods: .push(x) = add x if x is not None to the right side of deque (like append) .pull(x) = remove and return element from left side of deque (like popleft) From eebbdc5c61591571db48429c8c162426eea7a863 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 16 Feb 2025 13:24:41 -0700 Subject: [PATCH 11/33] refactored udp peer. Retry timers belong in appropriate Memoir superclass that supports reliable memograms. so removed as subclass of Tymee and made it a Mixin. --- src/hio/core/memoing.py | 276 +++++++++++++++++++++++++++++++++++++ src/hio/core/udp/udping.py | 58 ++------ 2 files changed, 289 insertions(+), 45 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 3662e40..ce23603 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -665,9 +665,285 @@ def serviceAll(self): +class Peer(tyming.Tymee): + """Class to manage non blocking I/O on UDP socket. + SubClass of Tymee to enable support for retry tymers as UDP is unreliable. + + Class Attributes: + Tymeout (float): default timeout for retry tymer(s) if any + + Inherited Properties: + .tyme is float relative cycle time of associated Tymist .tyme obtained + via injected .tymth function wrapper closure. + .tymth is function wrapper closure returned by Tymist .tymeth() method. + When .tymth is called it returns associated Tymist .tyme. + .tymth provides injected dependency on Tymist tyme base. + + Attributes: + name (str): unique identifier of peer for managment purposes + tymeout (float): timeout for retry tymer(s) if any + ha (tuple): host address of form (host,port) of type (str, int) of this + peer's socket address. + bs (int): buffer size + wl (WireLog): instance ref for debug logging of over the wire tx and rx + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + ls (socket.socket): local socket of this Peer + opened (bool): True local socket is created and opened. False otherwise + + Properties: + host (str): element of .ha duple + port (int): element of .ha duple + + + """ + Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout + + def __init__(self, *, + name='main', + tymeout=None, + ha=None, + host='', + port=55000, + bufsize=1024, + wl=None, + bcast=False, + **kwa): + """ + Initialization method for instance. + Parameters: + name (str): unique identifier of peer for managment purposes + tymeout (float): default for retry tymer if any + ha (tuple): local socket (host, port) address duple of type (str, int) + host (str): address where '' means any interface on host + port (int): socket port + bs (int): buffer size + wl (WireLog): instance to log over the wire tx and rx + bcast (bool): True enables sending to broadcast addresses from local socket + False otherwise + """ + super(Peer, self).__init__(**kwa) + self.name = name + self.tymeout = tymeout if tymeout is not None else self.Tymeout + #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer + + self.ha = ha or (host, port) # ha = host address duple (host, port) + host, port = self.ha + host = coring.normalizeHost(host) # ip host address + self.ha = (host, port) + + self.bs = bufsize + self.wl = wl + self.bcast = bcast + + self.ls = None # local socket for this Peer + self.opened = False + + @property + def host(self): + """ + Property that returns host in .ha duple + """ + return self.ha[0] + + + @host.setter + def host(self, value): + """ + setter for host property + """ + self.ha = (value, self.port) + + + @property + def port(self): + """ + Property that returns port in .ha duple + """ + return self.ha[1] + + + @port.setter + def port(self, value): + """ + setter for port property + """ + self.ha = (self.host, value) + + + + def wind(self, tymth): + """ + Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + Updates winds .tymer .tymth + """ + super(Peer, self).wind(tymth) + #self.tymer.wind(tymth) + + + def actualBufSizes(self): + """Returns duple of the the actual socket send and receive buffer size + (send, receive) + """ + if not self.ls: + return (0, 0) + + return (self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF), + self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) + + def open(self): + """Opens socket in non blocking mode. + + if socket not closed properly, binding socket gets error + OSError: (48, 'Address already in use') + """ + #create socket ss = server socket + self.ls = socket.socket(socket.AF_INET, + socket.SOCK_DGRAM, + socket.IPPROTO_UDP) + + if self.bcast: # needed to send broadcast, not needed to receive + self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + + # make socket address and port reusable. doesn't seem to have an effect. + # the SO_REUSEADDR flag tells the kernel to reuse a local socket in + # TIME_WAIT state, without waiting for its natural timeout to expire. + # Also use SO_REUSEPORT on linux and darwin + # https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ + + self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if platform.system() != 'Windows': + self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + # setup buffers + if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) < self.bs: + self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.bs) + if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) < self.bs: + self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.bs) + self.ls.setblocking(0) # non blocking socket + + #bind to Host Address Port + try: + self.ls.bind(self.ha) + except OSError as ex: + logger.error("Error opening UDP %s\n %s\n", self.ha, ex) + return False + + self.ha = self.ls.getsockname() # get resolved ha after bind + self.opened = True + return True + + def reopen(self): + """Idempotently open socket + """ + self.close() + return self.open() + + def close(self): + """Closes socket. + """ + if self.ls: + self.ls.close() #close socket + self.ls = None + self.opened = False + + def receive(self): + """Perform non blocking read on socket. + + Returns: + tuple of form (data, sa) + if no data then returns (b'',None) + but always returns a tuple with two elements + """ + try: + data, sa = self.ls.recvfrom(self.bs) # sa is source (host, port) + except OSError as ex: + # ex.args[0] == ex.errno for better compat + # the value of a given errno.XXXXX may be different on each os + # EAGAIN: BSD 35, Linux 11, Windows 11 + # EWOULDBLOCK: BSD 35 Linux 11 Windows 140 + if ex.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK): + return (b'', None) #receive has nothing empty string for data + else: + logger.error("Error receive on UDP %s\n %s\n", self.ha, ex) + raise #re raise exception ex + + if self.wl: # log over the wire receive + self.wl.writeRx(data, who=sa) + + return (data, sa) + + def send(self, data, da): + """Perform non blocking send on socket. + + data is string in python2 and bytes in python3 + da is destination address tuple (destHost, destPort) + """ + try: + cnt = self.ls.sendto(data, da) # count == int number of bytes sent + except OSError as ex: + logger.error("Error send UDP from %s to %s.\n %s\n", self.ha, da, ex) + cnt = 0 + raise + + if self.wl: # log over the wire send + self.wl.writeTx(data[:cnt], who=da) + + return cnt + + + def service(self): + """ + Service sends and receives + """ + + + + + +class TymeeMemoirDoer(doing.Doer): + """ + TymeeMemorir Doer for unreliable transports that require retry tymers. + + See Doer for inherited attributes, properties, and methods. + + Attributes: + .peer is underlying transport instance subclass of Memoir + + """ + + def __init__(self, peer, **kwa): + """ + Initialize instance. + + Parameters: + peer (Peer): UDP instance + """ + super(TymeeMemoirDoer, self).__init__(**kwa) + self.peer = peer + if self.tymth: + self.peer.wind(self.tymth) + + + def wind(self, tymth): + """ + Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + Updates winds .tymer .tymth + """ + super(TymeeMemoirDoer, self).wind(tymth) + self.peer.wind(tymth) + def enter(self): + """""" + self.peer.reopen() + def recur(self, tyme): + """""" + self.peer.service() + def exit(self): + """""" + self.peer.close() diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 616fab8..92fb8ba 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -9,9 +9,10 @@ import socket from contextlib import contextmanager -from ... import help -from ...base import tyming, doing +from ... import hioing from .. import coring +from ...base import tyming, doing +from ... import help logger = help.ogler.getLogger() @@ -57,23 +58,11 @@ def openPeer(cls=None, name="test", **kwa): -class Peer(tyming.Tymee): +class Peer(hioing.Mixin): """Class to manage non blocking I/O on UDP socket. - SubClass of Tymee to enable support for retry tymers as UDP is unreliable. - - Class Attributes: - Tymeout (float): default timeout for retry tymer(s) if any - - Inherited Properties: - .tyme is float relative cycle time of associated Tymist .tyme obtained - via injected .tymth function wrapper closure. - .tymth is function wrapper closure returned by Tymist .tymeth() method. - When .tymth is called it returns associated Tymist .tyme. - .tymth provides injected dependency on Tymist tyme base. Attributes: name (str): unique identifier of peer for managment purposes - tymeout (float): timeout for retry tymer(s) if any ha (tuple): host address of form (host,port) of type (str, int) of this peer's socket address. bs (int): buffer size @@ -89,11 +78,9 @@ class Peer(tyming.Tymee): """ - Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout def __init__(self, *, name='main', - tymeout=None, ha=None, host='', port=55000, @@ -105,7 +92,6 @@ def __init__(self, *, Initialization method for instance. Parameters: name (str): unique identifier of peer for managment purposes - tymeout (float): default for retry tymer if any ha (tuple): local socket (host, port) address duple of type (str, int) host (str): address where '' means any interface on host port (int): socket port @@ -116,9 +102,6 @@ def __init__(self, *, """ super(Peer, self).__init__(**kwa) self.name = name - self.tymeout = tymeout if tymeout is not None else self.Tymeout - #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer - self.ha = ha or (host, port) # ha = host address duple (host, port) host, port = self.ha host = coring.normalizeHost(host) # ip host address @@ -163,16 +146,6 @@ def port(self, value): self.ha = (self.host, value) - - def wind(self, tymth): - """ - Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. - Updates winds .tymer .tymth - """ - super(Peer, self).wind(tymth) - #self.tymer.wind(tymth) - - def actualBufSizes(self): """Returns duple of the the actual socket send and receive buffer size (send, receive) @@ -290,14 +263,19 @@ def service(self): """ + class PeerDoer(doing.Doer): """ - Basic UDP Peer Doer + Basic UXD Peer Doer + Because Unix Domain Sockets are reliable no need for retry tymer. + + To test in WingIde must configure Debug I/O to use external console + See Doer for inherited attributes, properties, and methods. See Doer for inherited attributes, properties, and methods. Attributes: - .peer is UDP Peer instance + .peer is UXD Peer instance """ @@ -306,21 +284,10 @@ def __init__(self, peer, **kwa): Initialize instance. Parameters: - peer (Peer): UDP instance + peer is UXD Peer instance """ super(PeerDoer, self).__init__(**kwa) self.peer = peer - if self.tymth: - self.peer.wind(self.tymth) - - - def wind(self, tymth): - """ - Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. - Updates winds .tymer .tymth - """ - super(PeerDoer, self).wind(tymth) - self.peer.wind(tymth) def enter(self): @@ -336,3 +303,4 @@ def recur(self, tyme): def exit(self): """""" self.peer.close() + From 64a99305445ca5acaa293211aa3d8885e954cf78 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 17 Feb 2025 14:08:36 -0700 Subject: [PATCH 12/33] refactored Memoir to Memogram. Refactored transmit side of memogram added TymeeMemogram subclass --- src/hio/core/memoing.py | 510 ++++++++++++++----------------------- src/hio/core/udp/udping.py | 42 ++- src/hio/core/uxd/uxding.py | 27 +- 3 files changed, 236 insertions(+), 343 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index ce23603..d2c343b 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -101,15 +101,20 @@ from collections import deque, namedtuple + from .. import hioing +from ...base import tyming, doing +from ... import help + +logger = help.ogler.getLogger() # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") -class Memoir(hioing.Mixin): +class Memogram(hioing.Mixin): """ - Memoir mixin base class to adds memogram support to a transport class. - Memoir supports asynchronous memograms. Provides common methods for subclasses. + Memogram mixin base class to adds memogram support to a transport class. + Memogram supports asynchronous memograms. Provides common methods for subclasses. A memogram is a higher level construct that sits on top of a datagram. A memogram supports the segmentation and desegmentation of memos to @@ -124,8 +129,7 @@ class Memoir(hioing.Mixin): Usage: Do not instantiate directly but use as a mixin with a transport class in order to create a new subclass that adds memogram support to the - transport class. For example MemoirUdp or MemoGramUdp or MemoirUxd or - MemoGramUXD + transport class. For example MemogramUdp or MemoGramUxd Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. @@ -138,10 +142,11 @@ class Memoir(hioing.Mixin): On the transmit side memos are placed in a deque (double ended queue). Each memo is then segmented into grams (datagrams) that respect the size constraints - of the underlying datagram transport. These grams are placed in a gram deque. - Each gram in this deque is then placed in a transmit buffer dict whose key - is the destination and value is the gram. Each item is to be sent - over the transport. One item buffer per unique destination. + of the underlying datagram transport. These grams are placed in the outgoing + gram deque. Each entry in this deque is a duple of form: + (gram: bytes, dst: str). Each duple is pulled off the deque and its + gram is put in bytearray for transport. + When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation of the sent datagram which means the whole datagram is not sent at once via @@ -155,7 +160,7 @@ class Memoir(hioing.Mixin): such as bad addresses or bad routing each destination gets its own buffer so that a bad destination does not block other destinations. - Memo segmentation information is embedded in the grams. + Memo segmentation/desegmentation information is embedded in the grams. Class Attributes: @@ -168,18 +173,19 @@ class Memoir(hioing.Mixin): rxgs (dict): holding rx (receive) (data) gram deques of grams. Each item in dict has key=src and val=deque of grames received from transport. Each item of form (src: str, gram: deque) - rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams - each entry in deque is duple of form (memo: str, dst: str) + rxms (deque): holding rx (receive) memo duples desegmented from rxgs grams + each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is duple of form (memo: str, dst: str) - txgs (dict): holding tx (transmit) (data) gram deques of grams. - Each item in dict has key=dst and val=deque of grams to be - sent via txbs of form (dst: str, grams: deque). Each entry - in deque is bytes of gram. - txbs (dict): holding tx (transmit) raw data items to transport. - Each item in dict has key=dst and value=bytearray holding - unsent portion of gram bytes of form (dst: str, gram: bytearray). + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when datagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) + Properties: @@ -207,20 +213,20 @@ def __init__(self, version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) rxgs (dict): holding rx (receive) (data) gram deques of grams. - Each item in dict has key=src and val=deque of grames received + Each item in dict has key=src and val=deque of grams received from transport. Each item of form (src: str, gram: deque) - rxms (deque): holding rx (receive) memo tuples desegmented from rxgs grams + rxms (deque): holding rx (receive) memo duples desegmented from rxgs grams each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into txgs grams where each entry in deque is duple of form (memo: str, dst: str) - txgs (dict): holding tx (transmit) (data) gram deques of grams. - Each item in dict has key=dst and value=deque of grams to be - sent via txbs of form (dst: str, grams: deque). Each entry - in deque is bytes of gram. - txbs (dict): holding tx (transmit) raw data items to transport. - Each item in dict has key=dst and value=bytearray holding - unsent portion of gram bytes of form (dst: str, gram: bytearray). + txgs (deque): grams to transmit, each entry is duple of form: + (gram: bytes, dst: str). + txbs (tuple): current transmisstion duple of form: + (gram: bytearray, dst: str). gram bytearray may hold untransmitted + portion when datagram is not able to be sent all at once so can + keep trying. Nothing to send indicated by (bytearray(), None) + for (gram, dst) """ @@ -231,10 +237,10 @@ def __init__(self, self.rxms = rxms if rxms is not None else deque() self.txms = txms if txms is not None else deque() - self.txgs = txgs if txgs is not None else dict() - self.txbs = txbs if txbs is not None else dict() + self.txgs = txgs if txgs is not None else deque() + self.txbs = txbs if txbs is not None else (bytearray(), None) - super(Memoir, self).__init__(**kwa) + super(Memogram, self).__init__(**kwa) if not hasattr(self, "opened"): # stub so mixin works in isolation @@ -269,20 +275,20 @@ def _serviceOneReceived(self): try: gram, src = self.receive() # if no data the duple is (b'', None) except socket.error as ex: # OSError.errno always .args[0] for compat - if (ex.args[0] == (errno.ECONNREFUSED, - errno.ECONNRESET, - errno.ENETRESET, - errno.ENETUNREACH, - errno.EHOSTUNREACH, - errno.ENETDOWN, - errno.EHOSTDOWN, - errno.ETIMEDOUT, - errno.ETIME, - errno.ENOBUFS, - errno.ENOMEM)): - return False # no received data - else: - raise + #if (ex.args[0] == (errno.ECONNREFUSED, + #errno.ECONNRESET, + #errno.ENETRESET, + #errno.ENETUNREACH, + #errno.EHOSTUNREACH, + #errno.ENETDOWN, + #errno.EHOSTDOWN, + #errno.ETIMEDOUT, + #errno.ETIME, + #errno.ENOBUFS, + #errno.ENOMEM)): + #return False # no received data + #else: + raise if not gram: # no received data return False @@ -290,7 +296,8 @@ def _serviceOneReceived(self): if src not in self.rxgs: self.rxgs[src] = deque() - self.rxgs[src].append(fram) + self.rxgs[src].append(gram) + return True # received data @@ -357,15 +364,13 @@ def _serviceOnceRxGrams(self): if not self.txgs[src]: # deque at src empty so remove deque del self.txgs[src] # makes src unordered until next memo at src - if memo is not None: # recieved and desegmented memo + if memo is not None: # received and desegmented memo if src in self.txgs: # more received gram(s) at src goods[i] = True # indicate memo recieved with more received gram(s) return any(goods) # at least one memo from a source with more received grams - - def serviceRxGramsOnce(self): """Service one pass (non-greedy) over all unique sources in .rxgs dict if any for received incoming grams. @@ -484,9 +489,7 @@ def _serviceOneTxMemo(self): memo, dst = self.txms.popleft() # raises IndexError if empty deque grams = self.segment(memo) for gram in grams: - if dst not in self.txgs: - self.txgs[dst] = deque() - self.txgs[dst].append(gram) + self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) def serviceTxMemosOnce(self): @@ -520,91 +523,75 @@ def _serviceOnceTxGrams(self): """Service one pass over .txgs dict for each unique dst in .txgs and then services one pass over the .txbs of those outgoing grams - Each item in.txgs is (key: dst, val: deque of grams) + Each entry in.txgs is duple of form: (gram: bytes, dst: str) where: - dst is destination address - gram is outgoing gram segment from associated memo + gram (bytes): is outgoing gram segment from associated memo + dst (str): is far peer destination address - Each item in.txbs is (key: dst, val: buf of gram) + .txbs is duple of form: (gram: bytearray, dst: str) where: - dst (str): destination address - buf (bytearray): outgoing gram segment or remainder of segment + gram (bytearray): holds incompletly sent gram portion if any + dst (str | None): destination or None if last completely sent Returns: - result (bool): True means at least one destination is unblocked - so can keep sending. Unblocked at destination means - last full gram sent with another sitting in deque. - False means all destinations blocked so try again later. + result (bool): True means greedy callers can keep sending since + last gram sent completely. + False means either last gram send was incomplete and + or there are no more grams in .txgs deque. In + either case greedy callers need to wait and + try again later. The return value True or False enables back pressure on greedy callers - so they know when to block waiting for at least one unblocked destination - with a pending gram. + so they know when to block waiting. - For each destination, if .txbs item is empty creates new item at destination - and copies gram to bytearray. - - Once it has serviced each item in .txgs then services each items in .txbs. - For each destination in .txbs, attempts to send bytearray to dst - keeping track of the actual portion sent and then deletes the sent - portion from item in .txbs leaving the remainder. - - When there is a remainder each subsequent call of this method + When there is a remainder in .txbs each subsequent call of this method will attempt to send the remainder until the the full gram has been sent. This accounts for datagram protocols that expect continuing attempts to send remainder of a datagram when using nonblocking sends. - When there is no remainder then removes .txbs buffer at dst so that - subsequent grams are reordered with respect to dst. + When the far side peer is unavailable the gram is dropped. This means + that unreliable transports need to have a timeour retry mechanism. - Internally, an empty .txbs at a destination indicates its ok to take - another gram from its .txgs deque if any and start sending it. + Internally, a dst of None in the .txbs duple indicates its ok to take + another gram from the .txgs deque if any and start sending it. - Deleting an item from a dict at a key (since python dicts are key creation - ordered) means that the next time an item is created at that key, that - item will be last in order. In order to dynamically change the ordering - of iteration over destinations, when there are no pending grams for a - given destination we remove its dict item. This reorders the destination - as last when a new gram is created and avoids iterating over destinations - with no pending grams. - """ - # service grams by reloading buffers from grams - for dst in list(self.txgs.keys()): # list since items may be deleted in loop - # if dst then grams deque at dst must not be empty - if dst not in self.txbs: # no transmit buffer - self.txbs[dst] = bytearray(self.txgs[dst].popleft()) # new gram - if not self.txgs[dst]: # deque at dst empty so remove deque - del self.txgs[dst] # makes dst unordered until next memo at dst - - - # service buffers by attempting to send - goods = [False] * len(self.txbs) # list by dst, True unblocked False blocked - for i, dst in enumerate(list(self.txbs.keys())): # list since items may be deleted in loop - # if dst then bytearray at dst must not be empty + """ + gram, dst = self.txbs + if dst == None: try: - count = self.send(self.txbs[dst], dst) # assumes .opened == True - except socket.error as ex: # OSError.errno always .args[0] for compat - if (ex.args[0] in (errno.ECONNREFUSED, - errno.ECONNRESET, - errno.ENETRESET, - errno.ENETUNREACH, - errno.EHOSTUNREACH, - errno.ENETDOWN, - errno.EHOSTDOWN, - errno.ETIMEDOUT, - errno.ETIME, - errno.ENOBUFS, - errno.ENOMEM)): # problem sending try again later - count = 0 # nothing sent - else: - raise - - del self.txbs[dst][:count] # remove from buffer those bytes sent - if not self.txbs[dst]: # empty buffer so gram fully sent - if dst in self.txgs: # another gram waiting - goods[i] = True # idicate not blocked and with waiting gram - del self.txbs[dst] # remove so dst is unordered until next gram - - return any(goods) # at least one unblocked destingation with waiting gram + gram, dst = self.txgs.popleft() + gram = bytearray(gram) + except IndexError: + return False # nothing more to send, return False to try later + + + try: + cnt = self.send(gram, dst) # assumes .opened == True + except socket.error as ex: # OSError.errno always .args[0] for compat + if (ex.args[0] in (errno.ECONNREFUSED, + errno.ECONNRESET, + errno.ENETRESET, + errno.ENETUNREACH, + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.EHOSTDOWN, + errno.ETIMEDOUT, + errno.ETIME)): # far peer problem + # try again later usually won't work here so we log error + # and drop gram so as to allow grams to other destinations + # to get sent. + logger.error("Error send from %s to %s\n %s\n", + self.name, dst, ex) + self.txbs = (bytearray(), None) # far peer unavailable, so drop. + else: + raise # unexpected error + + del gram[:cnt] # remove from buffer those bytes sent + if not gram: # all sent + dst = None # indicate by setting dst to None + self.txbs = (gram, dst) + + return (False if dst else True) # incomplete return False, else True def serviceTxGramsOnce(self): @@ -661,276 +648,149 @@ def serviceAll(self): self.serviceAllRx() self.serviceAllTx() + service = serviceAll # alias override peer service method +class MemogramDoer(doing.Doer): + """Memogram Doer for reliable transports that do not require retry tymers. -class Peer(tyming.Tymee): - """Class to manage non blocking I/O on UDP socket. - SubClass of Tymee to enable support for retry tymers as UDP is unreliable. - - Class Attributes: - Tymeout (float): default timeout for retry tymer(s) if any - - Inherited Properties: - .tyme is float relative cycle time of associated Tymist .tyme obtained - via injected .tymth function wrapper closure. - .tymth is function wrapper closure returned by Tymist .tymeth() method. - When .tymth is called it returns associated Tymist .tyme. - .tymth provides injected dependency on Tymist tyme base. + See Doer for inherited attributes, properties, and methods. Attributes: - name (str): unique identifier of peer for managment purposes - tymeout (float): timeout for retry tymer(s) if any - ha (tuple): host address of form (host,port) of type (str, int) of this - peer's socket address. - bs (int): buffer size - wl (WireLog): instance ref for debug logging of over the wire tx and rx - bcast (bool): True enables sending to broadcast addresses from local socket - False otherwise - ls (socket.socket): local socket of this Peer - opened (bool): True local socket is created and opened. False otherwise - - Properties: - host (str): element of .ha duple - port (int): element of .ha duple - + .peer (Memogram): underlying transport instance subclass of Memogram """ - Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout - def __init__(self, *, - name='main', - tymeout=None, - ha=None, - host='', - port=55000, - bufsize=1024, - wl=None, - bcast=False, - **kwa): - """ - Initialization method for instance. + def __init__(self, peer, **kwa): + """Initialize instance. + Parameters: - name (str): unique identifier of peer for managment purposes - tymeout (float): default for retry tymer if any - ha (tuple): local socket (host, port) address duple of type (str, int) - host (str): address where '' means any interface on host - port (int): socket port - bs (int): buffer size - wl (WireLog): instance to log over the wire tx and rx - bcast (bool): True enables sending to broadcast addresses from local socket - False otherwise - """ - super(Peer, self).__init__(**kwa) - self.name = name - self.tymeout = tymeout if tymeout is not None else self.Tymeout - #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer + peer (Peer): is Memogram Subclass instance + """ + super(MemogramDoer, self).__init__(**kwa) + self.peer = peer - self.ha = ha or (host, port) # ha = host address duple (host, port) - host, port = self.ha - host = coring.normalizeHost(host) # ip host address - self.ha = (host, port) - self.bs = bufsize - self.wl = wl - self.bcast = bcast + def enter(self): + """""" + self.peer.reopen() - self.ls = None # local socket for this Peer - self.opened = False - @property - def host(self): - """ - Property that returns host in .ha duple - """ - return self.ha[0] + def recur(self, tyme): + """""" + self.peer.service() - @host.setter - def host(self, value): - """ - setter for host property - """ - self.ha = (value, self.port) + def exit(self): + """""" + self.peer.close() - @property - def port(self): - """ - Property that returns port in .ha duple - """ - return self.ha[1] +class TymeeMemogram(tyming.Tymee): + """TymeeMemogram mixin base class to add tymer support for unreliable transports + that need retry tymers. Subclass of tyming.Tymee - @port.setter - def port(self, value): - """ - setter for port property - """ - self.ha = (self.host, value) + Inherited Class Attributes: + see superclass + Class Attributes: + Tymeout (float): default timeout for retry tymer(s) if any - def wind(self, tymth): - """ - Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. - Updates winds .tymer .tymth - """ - super(Peer, self).wind(tymth) - #self.tymer.wind(tymth) + Inherited Attributes: + see superclass + Attributes: + tymeout (float): default timeout for retry tymer(s) if any - def actualBufSizes(self): - """Returns duple of the the actual socket send and receive buffer size - (send, receive) - """ - if not self.ls: - return (0, 0) - return (self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF), - self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF)) - def open(self): - """Opens socket in non blocking mode. + """ + Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout - if socket not closed properly, binding socket gets error - OSError: (48, 'Address already in use') + def __init__(self, *, tymeout=None, **kwa): """ - #create socket ss = server socket - self.ls = socket.socket(socket.AF_INET, - socket.SOCK_DGRAM, - socket.IPPROTO_UDP) - - if self.bcast: # needed to send broadcast, not needed to receive - self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) - - # make socket address and port reusable. doesn't seem to have an effect. - # the SO_REUSEADDR flag tells the kernel to reuse a local socket in - # TIME_WAIT state, without waiting for its natural timeout to expire. - # Also use SO_REUSEPORT on linux and darwin - # https://stackoverflow.com/questions/14388706/how-do-so-reuseaddr-and-so-reuseport-differ - - self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if platform.system() != 'Windows': - self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + Initialization method for instance. + Inherited Parameters: + see superclass - # setup buffers - if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF) < self.bs: - self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.bs) - if self.ls.getsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF) < self.bs: - self.ls.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.bs) - self.ls.setblocking(0) # non blocking socket + Parameters: + tymeout (float): default for retry tymer if any - #bind to Host Address Port - try: - self.ls.bind(self.ha) - except OSError as ex: - logger.error("Error opening UDP %s\n %s\n", self.ha, ex) - return False + """ + super(TymeeMemogram, self).__init__(**kwa) + self.tymeout = tymeout if tymeout is not None else self.Tymeout + #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer - self.ha = self.ls.getsockname() # get resolved ha after bind - self.opened = True - return True - def reopen(self): - """Idempotently open socket + def wind(self, tymth): """ - self.close() - return self.open() - - def close(self): - """Closes socket. + Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + Updates winds .tymer .tymth """ - if self.ls: - self.ls.close() #close socket - self.ls = None - self.opened = False + super(TymeeMemogram, self).wind(tymth) + #self.tymer.wind(tymth) - def receive(self): - """Perform non blocking read on socket. + def serviceTymers(self): + """Service all retry tymers - Returns: - tuple of form (data, sa) - if no data then returns (b'',None) - but always returns a tuple with two elements + Stub override in subclass """ - try: - data, sa = self.ls.recvfrom(self.bs) # sa is source (host, port) - except OSError as ex: - # ex.args[0] == ex.errno for better compat - # the value of a given errno.XXXXX may be different on each os - # EAGAIN: BSD 35, Linux 11, Windows 11 - # EWOULDBLOCK: BSD 35 Linux 11 Windows 140 - if ex.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK): - return (b'', None) #receive has nothing empty string for data - else: - logger.error("Error receive on UDP %s\n %s\n", self.ha, ex) - raise #re raise exception ex - - if self.wl: # log over the wire receive - self.wl.writeRx(data, who=sa) + pass - return (data, sa) - def send(self, data, da): - """Perform non blocking send on socket. - - data is string in python2 and bytes in python3 - da is destination address tuple (destHost, destPort) + def serviceLocal(self): + """Service the local Peer's receive and transmit queues """ - try: - cnt = self.ls.sendto(data, da) # count == int number of bytes sent - except OSError as ex: - logger.error("Error send UDP from %s to %s.\n %s\n", self.ha, da, ex) - cnt = 0 - raise - - if self.wl: # log over the wire send - self.wl.writeTx(data[:cnt], who=da) - - return cnt + self.serviceReceives() + self.serviceTxGrams() - def service(self): - """ - Service sends and receives + def serviceAllOnce(self): + """Service all Rx and Tx Once (non-greedy) """ + self.serviceAllRxOnce() + self.serviceTymers() + self.serviceAllTxOnce() + def serviceAll(self): + """Service all Rx and Tx (greedy) + """ + self.serviceAllRx() + self.serviceTymers() + self.serviceAllTx() - -class TymeeMemoirDoer(doing.Doer): - """ - TymeeMemorir Doer for unreliable transports that require retry tymers. +class TymeeMemogramDoer(doing.Doer): + """TymeeMemogram Doer for unreliable transports that require retry tymers. See Doer for inherited attributes, properties, and methods. Attributes: - .peer is underlying transport instance subclass of Memoir + .peer (TymeeMemogram) is underlying transport instance subclass of TymeeMemogram """ def __init__(self, peer, **kwa): - """ - Initialize instance. + """Initialize instance. Parameters: - peer (Peer): UDP instance + peer (TymeeMemogram): subclass instance """ - super(TymeeMemoirDoer, self).__init__(**kwa) + super(TymeeMemogramDoer, self).__init__(**kwa) self.peer = peer if self.tymth: self.peer.wind(self.tymth) def wind(self, tymth): - """ - Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. + """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoirDoer, self).wind(tymth) + super(TymeeMemogramDoer, self).wind(tymth) self.peer.wind(tymth) diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 92fb8ba..2a29c6a 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -114,6 +114,7 @@ def __init__(self, *, self.ls = None # local socket for this Peer self.opened = False + @property def host(self): """ @@ -227,7 +228,8 @@ def receive(self): # the value of a given errno.XXXXX may be different on each os # EAGAIN: BSD 35, Linux 11, Windows 11 # EWOULDBLOCK: BSD 35 Linux 11 Windows 140 - if ex.args[0] in (errno.EAGAIN, errno.EWOULDBLOCK): + if (ex.args[0] in (errno.EAGAIN, + errno.EWOULDBLOCK)): return (b'', None) #receive has nothing empty string for data else: logger.error("Error receive on UDP %s\n %s\n", self.ha, ex) @@ -238,29 +240,47 @@ def receive(self): return (data, sa) - def send(self, data, da): + def send(self, data, dst): """Perform non blocking send on socket. - data is string in python2 and bytes in python3 - da is destination address tuple (destHost, destPort) + Returns: + cnt (int): count of bytes actually sent, may be less than len(data). + + Parameters: + data (bytes): payload to send + dst (str): udp destination addr duple of form (host: str, port: int) """ try: - cnt = self.ls.sendto(data, da) # count == int number of bytes sent + cnt = self.ls.sendto(data, dst) # count == int number of bytes sent except OSError as ex: - logger.error("Error send UDP from %s to %s.\n %s\n", self.ha, da, ex) - cnt = 0 + # ex.args[0] == ex.errno for better compat + # the value of a given errno.XXXXX may be different on each os + # EAGAIN: BSD 35, Linux 11, Windows 11 + # EWOULDBLOCK: BSD 35 Linux 11 Windows 140 + if (ex.args[0] in (errno.EAGAIN, + errno.EWOULDBLOCK, + errno.ENOBUFS, + errno.ENOMEM)): + # not enough buffer space to send, do not consume data + return 0 # try again later with same data + + else: + logger.error("Error send UDP from %s to %s.\n %s\n", self.ha, dst, ex) + cnt = 0 raise - if self.wl: # log over the wire send - self.wl.writeTx(data[:cnt], who=da) + if self.wl: # log over the wire actually sent data portion + self.wl.writeTx(data[:cnt], who=dst) return cnt def service(self): + """Service sends and receives + + Stub Override in subclass """ - Service sends and receives - """ + pass diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index ab5aa4b..521d191 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -322,7 +322,7 @@ def send(self, data, dst): """Perform non blocking send on socket. Returns: - cnt (int): number of bytes actually sent + cnt (int): number of bytes actually sent, may be less than len(data). Parameters: data (bytes): payload to send @@ -331,9 +331,21 @@ def send(self, data, dst): try: cnt = self.ls.sendto(data, dst) # count == int number of bytes sent except OSError as ex: - logger.error("Error send UXD from %s to %s.\n %s\n", self.path, dst, ex) - cnt = 0 - raise + # ex.args[0] == ex.errno for better compat + # the value of a given errno.XXXXX may be different on each os + # EAGAIN: BSD 35, Linux 11, Windows 11 + # EWOULDBLOCK: BSD 35 Linux 11 Windows 140 + if (ex.args[0] in (errno.EAGAIN, + errno.EWOULDBLOCK, + errno.ENOBUFS, + errno.ENOMEM)): + # not enough buffer space to send, do not consume data + return 0 # try again later with same data + + else: + logger.error("Error send UXD from %s to %s.\n %s\n", self.path, dst, ex) + cnt = 0 + raise if self.wl:# log over the wire send self.wl.writeTx(data[:cnt], who=dst) @@ -341,10 +353,11 @@ def send(self, data, dst): return cnt def service(self): - """ - Service sends and receives - """ + """Service sends and receives + Stub Override in subclass + """ + pass class PeerDoer(doing.Doer): From bd9673ee3d623f9a39f8dcbcff229765c1111ee7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 17 Feb 2025 19:33:23 -0700 Subject: [PATCH 13/33] basic tests for MemoGram --- src/hio/core/memoing.py | 178 ++++++++++++++++++++++++++++--------- tests/core/test_memoing.py | 107 ++++++++++++++++++++++ 2 files changed, 242 insertions(+), 43 deletions(-) create mode 100644 tests/core/test_memoing.py diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index d2c343b..0d2d28a 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -100,21 +100,54 @@ import errno from collections import deque, namedtuple +from contextlib import contextmanager - -from .. import hioing -from ...base import tyming, doing -from ... import help +from hio import hioing, help +from hio.base import tyming, doing logger = help.ogler.getLogger() # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") -class Memogram(hioing.Mixin): + +@contextmanager +def openMG(cls=None, name="test", **kwa): """ - Memogram mixin base class to adds memogram support to a transport class. - Memogram supports asynchronous memograms. Provides common methods for subclasses. + Wrapper to create and open MemoGram instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of MemoGram peer. + Enables management of transport by name. + Usage: + with openMemoGram() as mg: + mg.receive() + + with openMemoGram(cls=MemoGramSub) as mg: + mg.receive() + + """ + peer = None + + if cls is None: + cls = MemoGram + try: + peer = cls(name=name, **kwa) + peer.reopen() + + yield peer + + finally: + if peer: + peer.close() + + +class MemoGram(hioing.Mixin): + """ + MemoGram mixin base class to adds memogram support to a transport class. + MemoGram supports asynchronous memograms. Provides common methods for subclasses. A memogram is a higher level construct that sits on top of a datagram. A memogram supports the segmentation and desegmentation of memos to @@ -129,7 +162,7 @@ class Memogram(hioing.Mixin): Usage: Do not instantiate directly but use as a mixin with a transport class in order to create a new subclass that adds memogram support to the - transport class. For example MemogramUdp or MemoGramUxd + transport class. For example MemoGramUdp or MemoGramUxd Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. @@ -167,6 +200,11 @@ class Memogram(hioing.Mixin): Version (Versionage): default version consisting of namedtuple of form (major: int, minor: int) + Inherited Attributes: + name (str): unique name for MemoGram transport. Used to manage. + opened (bool): True means transport open for use + False otherwise + Attributes: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) @@ -196,6 +234,7 @@ class Memogram(hioing.Mixin): Version = Versionage(major=0, minor=0) # default version def __init__(self, + name='main', version=None, rxgs=None, rxms=None, @@ -207,7 +246,7 @@ def __init__(self, """Setup instance Inherited Parameters: - + name (str): unique name for MemoGram transport. Used to manage. Parameters: version (Versionage): version for this memoir instance consisting of @@ -240,28 +279,60 @@ def __init__(self, self.txgs = txgs if txgs is not None else deque() self.txbs = txbs if txbs is not None else (bytearray(), None) - super(Memogram, self).__init__(**kwa) + super(MemoGram, self).__init__(name=name, **kwa) + + + + if not hasattr(self, "name"): # stub so mixin works in isolation + self.name = name # mixed with subclass should provide this. if not hasattr(self, "opened"): # stub so mixin works in isolation self.opened = False # mixed with subclass should provide this. + def open(self): + """Opens transport in nonblocking mode + + This is a stub. Override in transport specific subclass + """ + self.opened = True + return True - def receive(self): + def reopen(self): + """Idempotently open transport + """ + self.close() + return self.open() + + + def close(self): + """Closes transport + + This is a stub. Override in transport subclass + """ + self.opened = False + + + def receive(self, *, echo=None): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. + Parameters: + echo (tuple): returns echo for debugging purposes where echo is + duple of form (gram: bytes, src: str) otherwise returns duple + that indicates nothing to receive of form (b'', None) + Returns: duple (tuple): of form (data: bytes, src: str) where data is the bytes of received data and src is the source address. When no data the duple is (b'', None) """ - return (b'', None) + return echo if echo else (b'', None) - def _serviceOneReceived(self): + def _serviceOneReceived(self, *, echo=None): """Service one received duple (raw, src) raw packet data. Always returns complete datagram. @@ -271,9 +342,15 @@ def _serviceOneReceived(self): return enables greedy callers to keep calling until no more data to receive from transport + + Parameters: + echo (tuple): returns echo for debugging purposes where echo is + duple of form (gram: bytes, src: str) otherwise returns duple + that indicates nothing to receive of form (b'', None) + """ try: - gram, src = self.receive() # if no data the duple is (b'', None) + gram, src = self.receive(echo=echo) # if no data the duple is (b'', None) except socket.error as ex: # OSError.errno always .args[0] for compat #if (ex.args[0] == (errno.ECONNREFUSED, #errno.ECONNRESET, @@ -301,19 +378,30 @@ def _serviceOneReceived(self): return True # received data - def serviceReceivesOnce(self): + def serviceReceivesOnce(self, *, echo=None): """Service receives once (non-greedy) and queue up + + Parameters: + echo (tuple): returns echo for debugging purposes where echo is + duple of form (gram: bytes, src: str) otherwise returns duple + that indicates nothing to receive of form (b'', None) """ if self.opened: - self._serviceOneReceived() + self._serviceOneReceived(echo=echo) - def serviceReceives(self): + def serviceReceives(self, *, echo=None): """Service all receives (greedy) and queue up + + Parameters: + echo (tuple): returns echo for debugging purposes where echo is + duple of form (gram: bytes, src: str) otherwise returns duple + that indicates nothing to receive of form (b'', None) """ while self.opened: - if not self._serviceOneReceived(): + if not self._serviceOneReceived(echo=echo): break + echo = None # only echo once def desegment(self, grams): @@ -328,7 +416,13 @@ def desegment(self, grams): Parameters: grams (deque): gram segments """ - return "" + try: + gram = grams.popleft() + memo = gram.decode() + except IndexError: + return None + + return memo def _serviceOnceRxGrams(self): @@ -361,11 +455,11 @@ def _serviceOnceRxGrams(self): if memo is not None: # allows for empty "" memo for some src self.rxms.append((memo, src)) - if not self.txgs[src]: # deque at src empty so remove deque - del self.txgs[src] # makes src unordered until next memo at src + if not self.rxgs[src]: # deque at src empty so remove deque + del self.rxgs[src] # makes src unordered until next memo at src if memo is not None: # received and desegmented memo - if src in self.txgs: # more received gram(s) at src + if src in self.rxgs: # more received gram(s) at src goods[i] = True # indicate memo recieved with more received gram(s) return any(goods) # at least one memo from a source with more received grams @@ -406,8 +500,7 @@ def serviceRxMemosOnce(self): Override in subclass to handle result and put it somewhere """ try: - pass - #memo, src = self._serviceOneRxMemo() + memo, src = self._serviceOneRxMemo() except IndexError: pass @@ -418,8 +511,7 @@ def serviceRxMemos(self): Override in subclass to handle result(s) and put them somewhere """ while self.rxms: - break - # memo, src = self._serviceOneRxMsg() + memo, src = self._serviceOneRxMemo() def serviceAllRxOnce(self): @@ -473,7 +565,7 @@ def segment(self, memo): Parameters: memo (str): to be segmented into grams """ - grams = [] + grams = [memo] return grams @@ -560,7 +652,7 @@ def _serviceOnceTxGrams(self): if dst == None: try: gram, dst = self.txgs.popleft() - gram = bytearray(gram) + gram = bytearray(gram.encode()) except IndexError: return False # nothing more to send, return False to try later @@ -652,13 +744,13 @@ def serviceAll(self): -class MemogramDoer(doing.Doer): - """Memogram Doer for reliable transports that do not require retry tymers. +class MemoGramDoer(doing.Doer): + """MemoGram Doer for reliable transports that do not require retry tymers. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (Memogram): underlying transport instance subclass of Memogram + .peer (MemoGram): underlying transport instance subclass of MemoGram """ @@ -666,9 +758,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (Peer): is Memogram Subclass instance + peer (Peer): is MemoGram Subclass instance """ - super(MemogramDoer, self).__init__(**kwa) + super(MemoGramDoer, self).__init__(**kwa) self.peer = peer @@ -688,8 +780,8 @@ def exit(self): -class TymeeMemogram(tyming.Tymee): - """TymeeMemogram mixin base class to add tymer support for unreliable transports +class TymeeMemoGram(tyming.Tymee): + """TymeeMemoGram mixin base class to add tymer support for unreliable transports that need retry tymers. Subclass of tyming.Tymee @@ -720,7 +812,7 @@ def __init__(self, *, tymeout=None, **kwa): tymeout (float): default for retry tymer if any """ - super(TymeeMemogram, self).__init__(**kwa) + super(TymeeMemoGram, self).__init__(**kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer @@ -730,7 +822,7 @@ def wind(self, tymth): Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemogram, self).wind(tymth) + super(TymeeMemoGram, self).wind(tymth) #self.tymer.wind(tymth) def serviceTymers(self): @@ -764,13 +856,13 @@ def serviceAll(self): self.serviceAllTx() -class TymeeMemogramDoer(doing.Doer): - """TymeeMemogram Doer for unreliable transports that require retry tymers. +class TymeeMemoGramDoer(doing.Doer): + """TymeeMemoGram Doer for unreliable transports that require retry tymers. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (TymeeMemogram) is underlying transport instance subclass of TymeeMemogram + .peer (TymeeMemoGram) is underlying transport instance subclass of TymeeMemoGram """ @@ -778,9 +870,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (TymeeMemogram): subclass instance + peer (TymeeMemoGram): subclass instance """ - super(TymeeMemogramDoer, self).__init__(**kwa) + super(TymeeMemoGramDoer, self).__init__(**kwa) self.peer = peer if self.tymth: self.peer.wind(self.tymth) @@ -790,7 +882,7 @@ def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemogramDoer, self).wind(tymth) + super(TymeeMemoGramDoer, self).wind(tymth) self.peer.wind(tymth) diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py new file mode 100644 index 0000000..8f036bf --- /dev/null +++ b/tests/core/test_memoing.py @@ -0,0 +1,107 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_memoing module + +""" + + +import pytest + +from hio.base import doing +from hio.core import memoing + + +def test_memogram_basic(): + """Test MemoGram class basic + """ + peer = memoing.MemoGram() + assert peer.name == "main" + assert peer.opened == False + peer.reopen() + assert peer.opened == True + + assert not peer.txms + assert not peer.txgs + gram, dst = peer.txbs + assert not dst + peer.service() + gram, dst = peer.txbs + assert not dst + + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms + peer.serviceTxMemos() + assert not peer.txms + assert peer.txgs + peer.serviceTxGrams() + assert not peer.txgs + + assert not peer.rxgs + assert not peer.rxms + echo = (b"Bye yall", "beta") + peer.serviceReceives(echo=echo) + assert peer.rxgs + peer.serviceRxGrams() + assert not peer.rxgs + assert peer.rxms + peer.serviceRxMemos() + assert not peer.rxms + + + peer.close() + assert peer.opened == False + + + + """ End Test """ + + +def test_open_mg(): + """Test contextmanager decorator openMG + """ + with (memoing.openMG(name='zeta') as zeta): + + assert zeta.opened + assert zeta.name == 'zeta' + + + assert not zeta.opened + + """ End Test """ + + +def test_memogram_doer(): + """Test MemoGramDoer class + """ + tock = 0.03125 + ticks = 4 + limit = ticks * tock + doist = doing.Doist(tock=tock, real=True, limit=limit) + assert doist.tyme == 0.0 # on next cycle + assert doist.tock == tock == 0.03125 + assert doist.real == True + assert doist.limit == limit == 0.125 + assert doist.doers == [] + + peer = memoing.MemoGram() + + mgdoer = memoing.MemoGramDoer(peer=peer) + assert mgdoer.peer == peer + assert not mgdoer.peer.opened + assert mgdoer.tock == 0.0 # ASAP + + doers = [mgdoer] + doist.do(doers=doers) + assert doist.tyme == limit + assert mgdoer.peer.opened == False + + """End Test """ + + +if __name__ == "__main__": + test_memogram_basic() + test_open_mg() + test_memogram_doer() + From 6ce4e4fcbb6e27e10eb01fc3b620444c79b01642 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 18 Feb 2025 13:32:21 -0700 Subject: [PATCH 14/33] added unit tests --- src/hio/core/memoing.py | 47 +++++++++++--- tests/core/test_memoing.py | 126 +++++++++++++++++++++++++++++++++---- 2 files changed, 152 insertions(+), 21 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 0d2d28a..3d1bc31 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -122,11 +122,11 @@ def openMG(cls=None, name="test", **kwa): name (str): unique identifer of MemoGram peer. Enables management of transport by name. Usage: - with openMemoGram() as mg: - mg.receive() + with openMemoGram() as peer: + peer.receive() - with openMemoGram(cls=MemoGramSub) as mg: - mg.receive() + with openMemoGram(cls=MemoGramSub) as peer: + peer.receive() """ peer = None @@ -565,7 +565,7 @@ def segment(self, memo): Parameters: memo (str): to be segmented into grams """ - grams = [memo] + grams = [memo.encode()] return grams @@ -605,7 +605,7 @@ def gramit(self, gram, dst): """Append (gram, dst) duple to .txgs deque Parameters: - gram (str): gram to be sent + gram (bytes): gram to be sent dst (str): address of remote destination of gram """ self.txgs.append((gram, dst)) @@ -652,7 +652,7 @@ def _serviceOnceTxGrams(self): if dst == None: try: gram, dst = self.txgs.popleft() - gram = bytearray(gram.encode()) + gram = bytearray(gram) except IndexError: return False # nothing more to send, return False to try later @@ -778,9 +778,40 @@ def exit(self): """""" self.peer.close() +@contextmanager +def openTMG(cls=None, name="test", **kwa): + """ + Wrapper to create and open TymeeMemoGram instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of MemoGram peer. + Enables management of transport by name. + Usage: + with openMemoGram() as peer: + peer.receive() + + with openMemoGram(cls=MemoGramSub) as peer: + peer.receive() + + """ + peer = None + + if cls is None: + cls = TymeeMemoGram + try: + peer = cls(name=name, **kwa) + peer.reopen() + + yield peer + + finally: + if peer: + peer.close() -class TymeeMemoGram(tyming.Tymee): +class TymeeMemoGram(tyming.Tymee, MemoGram): """TymeeMemoGram mixin base class to add tymer support for unreliable transports that need retry tymers. Subclass of tyming.Tymee diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 8f036bf..4289068 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -7,7 +7,7 @@ import pytest -from hio.base import doing +from hio.base import doing, tyming from hio.core import memoing @@ -22,39 +22,34 @@ def test_memogram_basic(): assert not peer.txms assert not peer.txgs - gram, dst = peer.txbs - assert not dst + assert peer.txbs == (b'', None) peer.service() - gram, dst = peer.txbs - assert not dst + assert peer.txbs == (b'', None) memo = "Hello There" dst = "beta" peer.memoit(memo, dst) - assert peer.txms + assert peer.txms[0] == ('Hello There', 'beta') peer.serviceTxMemos() assert not peer.txms - assert peer.txgs + assert peer.txgs[0] == (b'Hello There', 'beta') peer.serviceTxGrams() assert not peer.txgs + assert peer.txbs == (b'', None) assert not peer.rxgs assert not peer.rxms echo = (b"Bye yall", "beta") peer.serviceReceives(echo=echo) - assert peer.rxgs + assert peer.rxgs["beta"][0] == b"Bye yall" peer.serviceRxGrams() assert not peer.rxgs - assert peer.rxms + assert peer.rxms[0] == ('Bye yall', 'beta') peer.serviceRxMemos() assert not peer.rxms - peer.close() assert peer.opened == False - - - """ End Test """ @@ -99,9 +94,114 @@ def test_memogram_doer(): """End Test """ +def test_tymeememogram_basic(): + """Test TymeeMemoGram class basic + """ + peer = memoing.TymeeMemoGram() + assert peer.tymeout == 0.0 + assert peer.name == "main" + assert peer.opened == False + peer.reopen() + assert peer.opened == True + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + peer.service() + assert peer.txbs == (b'', None) + + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert peer.txgs[0] == (b'Hello There', 'beta') + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.rxms + echo = (b"Bye yall", "beta") + peer.serviceReceives(echo=echo) + assert peer.rxgs["beta"][0] == b"Bye yall" + peer.serviceRxGrams() + assert not peer.rxgs + assert peer.rxms[0] == ('Bye yall', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + tymist = tyming.Tymist(tock=1.0) + peer.wind(tymth=tymist.tymen()) + assert peer.tyme == tymist.tyme == 0.0 + tymist.tick() + assert peer.tyme == tymist.tyme == 1.0 + + + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_open_tmg(): + """Test contextmanager decorator openTMG + """ + with (memoing.openTMG(name='zeta') as zeta): + + assert zeta.opened + assert zeta.name == 'zeta' + assert zeta.tymeout == 0.0 + + + assert not zeta.opened + + """ End Test """ + + +def test_tymeememogram_doer(): + """Test TymeeMemoGramDoer class + """ + tock = 0.03125 + ticks = 4 + limit = ticks * tock + doist = doing.Doist(tock=tock, real=True, limit=limit) + assert doist.tyme == 0.0 # on next cycle + assert doist.tock == tock == 0.03125 + assert doist.real == True + assert doist.limit == limit == 0.125 + assert doist.doers == [] + + peer = memoing.TymeeMemoGram() + + tmgdoer = memoing.TymeeMemoGramDoer(peer=peer) + assert tmgdoer.peer == peer + assert not tmgdoer.peer.opened + assert tmgdoer.tock == 0.0 # ASAP + + doers = [tmgdoer] + doist.do(doers=doers) + assert doist.tyme == limit + assert tmgdoer.peer.opened == False + + tymist = tyming.Tymist(tock=1.0) + tmgdoer.wind(tymth=tymist.tymen()) + assert tmgdoer.tyme == tymist.tyme == 0.0 + assert peer.tyme == tymist.tyme == 0.0 + tymist.tick() + assert tmgdoer.tyme == tymist.tyme == 1.0 + assert peer.tyme == tymist.tyme == 1.0 + + """End Test """ + + if __name__ == "__main__": test_memogram_basic() test_open_mg() test_memogram_doer() + test_tymeememogram_basic() + test_open_tmg() + test_tymeememogram_doer() From b358f5bec3cee182c7d5acf24313c66d3a6c87a7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Wed, 19 Feb 2025 11:22:29 -0700 Subject: [PATCH 15/33] added ENOENT for no file or directory for eerno for MemoGram send (ENOENT is relevant to uxd) --- src/hio/core/memoing.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 3d1bc31..519a842 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -661,6 +661,7 @@ def _serviceOnceTxGrams(self): cnt = self.send(gram, dst) # assumes .opened == True except socket.error as ex: # OSError.errno always .args[0] for compat if (ex.args[0] in (errno.ECONNREFUSED, + errno.ENOENT, errno.ECONNRESET, errno.ENETRESET, errno.ENETUNREACH, @@ -671,7 +672,8 @@ def _serviceOnceTxGrams(self): errno.ETIME)): # far peer problem # try again later usually won't work here so we log error # and drop gram so as to allow grams to other destinations - # to get sent. + # to get sent. When uxd, ECONNREFUSED and ENOENT means dest + # uxd file path is not available to send to. logger.error("Error send from %s to %s\n %s\n", self.name, dst, ex) self.txbs = (bytearray(), None) # far peer unavailable, so drop. From 46259af93e8d6045e81dfbf6cde4a716ef7b93d2 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 11:27:40 -0700 Subject: [PATCH 16/33] started porting over packeting and packing moduels from ioflo some more refinement work on uxd --- src/hio/core/memoing.py | 127 ++++++++++++-- src/hio/core/packeting.py | 350 +++++++++++++++++++++++++++++++++++++ src/hio/core/uxd/uxding.py | 31 +++- src/hio/help/helping.py | 60 +++++++ src/hio/help/packing.py | 208 ++++++++++++++++++++++ src/hio/hioing.py | 24 +-- tests/core/uxd/test_uxd.py | 7 +- tests/help/test_helping.py | 103 +++++++++++ 8 files changed, 870 insertions(+), 40 deletions(-) create mode 100644 src/hio/core/packeting.py create mode 100644 src/hio/help/packing.py diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 519a842..de1144b 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -61,6 +61,9 @@ protocol headers could be added in some circumstances. A more conservative value of around 300-400 bytes may be preferred instead. +Usually ip headers are only 20 bytes so with UDP header (src, dst) of 8 bytes +the maximum allowed payload is 576 - 28 = 548 + Any UDP payload this size or smaller is guaranteed to be deliverable over IP (though not guaranteed to be delivered). Anything larger is allowed to be outright dropped by any router for any reason. @@ -95,12 +98,88 @@ https://unix.stackexchange.com/questions/38043/size-of-data-that-can-be-written-to-read-from-sockets https://stackoverflow.com/questions/21856517/whats-the-practical-limit-on-the-size-of-single-packet-transmitted-over-domain +Memo +Memo partitioning into parts. + +Default part is of form part = head + sep + body. +Some subclasses might have part of form part = head + sep + body + tail. +In that case encoding of body + tail must provide a way to separate body from tail. +Typically tail would be a signature on the fore part = head + sep + body + + +Separator sep is b'|' must not be a base64 character. + +The head consists of three fields in base64 +mid = memo ID +pn = part number of part in memo zero based +pc = part count of total parts in memo may be zero when body is empty + +Sep = b'|' +assert not helping.Reb64.match(Sep) + +body = b"demogram" +pn = 0 +pc = 12 + +pn = helping.intToB64b(pn, l=4) +pc = helping.intToB64b(pc, l=4) + +PartLeader = struct.Struct('!16s4s4s') +PartLeader.size == 24 +head = PartLeader.pack(mid, pn, pc) +part = Sep.join(head, body) + + +head, sep, body = part.partition(Sep) +assert helping.Reb64.match(head) +mid, pn, pc = PartLeader.unpack(head) +pn = helping.b64ToInt(pn) +pc = helping.b64ToInt(pc) + +# test PartHead +code = MtrDex.PartHead +assert code == '0P' +codeb = code.encode() + +mid = 1 +midb = mid.to_bytes(16) +assert midb == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' +pn = 1 +pnb = pn.to_bytes(3) +assert pnb == b'\x00\x00\x01' +pc = 2 +pcb = pc.to_bytes(3) +assert pcb == b'\x00\x00\x02' +raw = midb + pnb + pcb +assert raw == (b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x01\x00\x00\x02') + +assert mid == int.from_bytes(raw[:16]) +assert pn == int.from_bytes(raw[16:19]) +assert pc == int.from_bytes(raw[19:22]) + +midb64 = encodeB64(bytes([0] * 2) + midb) # prepad +pnb64 = encodeB64(pnb) +pcb64 = encodeB64(pcb) + +qb64b = codeb + midb64[2:] + pnb64 + pcb64 +assert qb64b == b'0PAAAAAAAAAAAAAAAAAAAAABAAABAAAC' +qb64 = qb64b.decode() +qb2 = decodeB64(qb64b) + +assert mid == int.from_bytes(decodeB64(b'AA' + qb64b[2:24])) +assert pn == int.from_bytes(decodeB64(qb64b[24:28])) +assert pc == int.from_bytes(decodeB64(qb64b[28:32])) + """ import socket import errno +import struct from collections import deque, namedtuple from contextlib import contextmanager +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 from hio import hioing, help from hio.base import tyming, doing @@ -204,6 +283,7 @@ class MemoGram(hioing.Mixin): name (str): unique name for MemoGram transport. Used to manage. opened (bool): True means transport open for use False otherwise + bc (int | None): count of transport buffers of MaxGramSize Attributes: version (Versionage): version for this memoir instance consisting of @@ -225,16 +305,28 @@ class MemoGram(hioing.Mixin): for (gram, dst) + .rxgs dict keyed by mid (memo id) that holds incomplete memo parts. + The mid appears in every gram part from the same memo. + each val is dict of received gram parts keyed by part number. + + srcs is dict keyed by mid that holds the src for that mid. This allows + reattaching src to memo when placing in rxms deque + + + .mids is dict of dicts keyed by src and each value dict keyed by Properties: - """ + + """ Version = Versionage(major=0, minor=0) # default version + BufCount = 64 # default bufcount bc for transport def __init__(self, name='main', + bc=None, version=None, rxgs=None, rxms=None, @@ -247,6 +339,7 @@ def __init__(self, Inherited Parameters: name (str): unique name for MemoGram transport. Used to manage. + bc (int | None): count of transport buffers of MaxGramSize Parameters: version (Versionage): version for this memoir instance consisting of @@ -254,9 +347,9 @@ def __init__(self, rxgs (dict): holding rx (receive) (data) gram deques of grams. Each item in dict has key=src and val=deque of grams received from transport. Each item of form (src: str, gram: deque) - rxms (deque): holding rx (receive) memo duples desegmented from rxgs grams + rxms (deque): holding rx (receive) memo duples fused from rxgs grams each entry in deque is duple of form (memo: str, dst: str) - txms (deque): holding tx (transmit) memo tuples to be segmented into + txms (deque): holding tx (transmit) memo tuples to be partitioned into txgs grams where each entry in deque is duple of form (memo: str, dst: str) txgs (deque): grams to transmit, each entry is duple of form: @@ -268,6 +361,7 @@ def __init__(self, for (gram, dst) + """ # initialize attributes self.version = version if version is not None else self.Version @@ -279,15 +373,14 @@ def __init__(self, self.txgs = txgs if txgs is not None else deque() self.txbs = txbs if txbs is not None else (bytearray(), None) - super(MemoGram, self).__init__(name=name, **kwa) - + bc = bc if bc is not None else self.BufCount # use bufcount to calc .bs + super(MemoGram, self).__init__(name=name, bc=bc, **kwa) - if not hasattr(self, "name"): # stub so mixin works in isolation + if not hasattr(self, "name"): # stub so mixin works in isolation. self.name = name # mixed with subclass should provide this. - - if not hasattr(self, "opened"): # stub so mixin works in isolation + if not hasattr(self, "opened"): # stub so mixin works in isolation. self.opened = False # mixed with subclass should provide this. @@ -404,17 +497,17 @@ def serviceReceives(self, *, echo=None): echo = None # only echo once - def desegment(self, grams): - """Desegment deque of grams as segments into whole memo. If grams is - missing all the segments then returns None. + def fuse(self, grams): + """Fuse gram parts from frams deque into whole memo. If grams is + missing any the parts for a whole memo then returns None. Returns: - memo (str | None): desegmented memo or None if incomplete. + memo (str | None): fused memo or None if incomplete. Override in subclass Parameters: - grams (deque): gram segments + grams (deque): gram segments from which to fuse memo parts. """ try: gram = grams.popleft() @@ -451,7 +544,7 @@ def _serviceOnceRxGrams(self): # service grams to desegment for i, src in enumerate(list(self.rxgs.keys())): # list since items may be deleted in loop # if src then grams deque at src must not be empty - memo = self.desegment(self.rxgs[src]) + memo = self.fuse(self.rxgs[src]) if memo is not None: # allows for empty "" memo for some src self.rxms.append((memo, src)) @@ -555,8 +648,8 @@ def memoit(self, memo, dst): self.txms.append((memo, dst)) - def segment(self, memo): - """Segment and package up memo into grams. + def part(self, memo): + """Partition memo into parts as grams. This is a stub method meant to be overridden in subclass Returns: @@ -579,7 +672,7 @@ def _serviceOneTxMemo(self): Appends duples of (gram, dst) from grams to .txgs deque. """ memo, dst = self.txms.popleft() # raises IndexError if empty deque - grams = self.segment(memo) + grams = self.part(memo) for gram in grams: self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) diff --git a/src/hio/core/packeting.py b/src/hio/core/packeting.py new file mode 100644 index 0000000..e6e93ad --- /dev/null +++ b/src/hio/core/packeting.py @@ -0,0 +1,350 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.packeting Module + + +""" + + +import struct +from binascii import hexlify +from collections import deque, namedtuple +import enum + +from .. import hioing + +#from ...aid.byting import packify, packifyInto, unpackify +# see hio versions in help.packing + + + +class Part(hioing.MixIn): + """ + Base class for packet part classes with .packed .size and len + """ + Size = 0 + + def __init__(self, size=None, packed=None, **kwa): + """ + Initialization method for instance. + + Parameters: + size is initial size of .packed if packed not provided + packed is initial .packed + + Attributes: + .packed is bytearray of packed binary data + + Properties: + .size is length of .packed + + """ + super(Part, self).__init__(**kwa) + + if packed is None: + size = size if size is not None else self.Size + self.packed = bytearray([0 for i in range(0, size)]) + else: + self.packed = bytearray(packed) + + def __len__(self): + """ + Returns the length of .packed + """ + return len(self.packed) + + @property + def size(self): + """ + Property size + """ + return self.__len__() + + def show(self): + """ + Returns descriptive string for display purposes + """ + name = self.__class__.__name__ + result = (" {0}: packed=0x{1}\n".format(name, + hexlify(self.packed).decode('ascii'))) + return result + + +class PackerPart(Part): + """ + Base class for packet packer part classes with .packer and .fmt + .fmt is the struct string format for the fixed size portion of part + """ + Format = '!' # default empty packer struct format string for packed + + def __init__(self, fmt=None, raw=None, **kwa): + """ + Initialization method for instance. + + Inherited Parameters: + size is initial size of .packed + + Parameters: + fmt is struct format string to pack into .packed + raw is input bytearray of data to parse(unpack) + + Inherited Attributes: + .packed is bytearray of packed binary data using .packer + + Attributes: + .fmt is struct format string + .packer is compiled struct packer + + Class Attributes: + .Format is struct packer format string for packed + + Inherited Properties: + .size is length of .packed + + """ + self.fmt = fmt if fmt is not None else self.Format + self.packer = struct.Struct(self.fmt) #precompile struct for packing unpacking + kwa['size'] = self.packer.size # override size to match .packer.size + super(PackerPart, self).__init__(**kwa) + + if raw is not None: + self.parse(raw=raw) + + def verifySize(self, raw=bytearray(b'')): + """ + Return True if len(raw) is at least long enough for packed size + """ + return (len(raw) >= (self.packer.size)) + + def parse(self, raw): + """Parse raw bytearray and assign to fields + Return offset into raw of unparsed portion + Base method to be overridden in subclass + """ + if (raw is None) or (not self.verifySize(raw)): + raise ValueError("Parse Packer: Not enough raw data for packer. " + "Need {0} bytes, got {1} bytes.".format(self.size, + len(raw))) + + result = self.packer.unpack_from(raw) # result should be empty tuple + self.packed[:] = raw[0:self.packer.size] + + return self.size #return offset to start of unparsed portion of data + + def pack(self, **kwa): + """ + Return .packed with data if any + Base method to be overridden in sub class + """ + + self.packer.pack_into(self.packed, 0) # empty + + if self.size != self.packer.size : + raise ValueError("Build Packer: size packed={0} not match " + "format={1}".format(self.size, self.packer.size)) + return self.packed + + +class PackifierPart(Part): + """ + Base class for packet packifier part classes with packify .fmt + .fmt is the packify string format for the fixed size portion of part + packify allows bit field packing + + packify/unpackify format is string of white space separated bit field lengths + The packed values are provided as sequence of bit field values + that are packed into bytearray of size bytes using fmt string. + + Each white space separated field of fmt is the length of the associated bit field + If not provided size is the least integer number of bytes that hold the fmt. + If reverse is true reverse the order of the bytes in the byte array before + returning. This is useful for converting between bigendian and littleendian. + + Assumes unsigned fields values. + Assumes network big endian so first fields element is high order bits. + Each field in format string is number of bits for the associated bit field + Fields with length of 1 are treated as has having boolean truthy field values + that is, nonzero is True and packs as a 1 + for 2+ length bit fields the field element is truncated to the number of + low order bits in the bit field + if sum of number of bits in fmt less than size bytes then the last byte in + the bytearray is right zero padded + if sum of number of bits in fmt greater than size bytes returns exception + to pad just use 0 value in source field. + example + packify("1 3 2 2", (True, 4, 0, 3)). returns bytearry([0xc3]) + """ + Format = '' # default packer struct format string for packed + + def __init__(self, fmt=None, raw=None, **kwa): + """ + Initialization method for instance. + + Inherited Parameters: + size is initial size of .packed + + Parameters: + fmt is packify format string to pack into .packed + raw is input bytearray of data to parse(unpack) + + Inherited Attributes: + .packed is bytearray of packed binary data + + Attributes: + .fmt is packify format string + .fmtSize is size given by format string + + Inherited Properties: + .size is length of .packed + + Properties + .fmtSize is size given by .fmt + + """ + self.fmt = fmt if fmt is not None else self.Format + kwa['size'] = self.fmtSize # override size to match packify size of whole bytes + super(PackifierPart, self).__init__(**kwa) + + if raw is not None: + self.parse(raw=raw) + + @property + def fmtSize(self): + """ + Property fmtSize + """ + tbfl = sum((int(x) for x in self.fmt.split())) + size = (tbfl // 8) + 1 if tbfl % 8 else tbfl // 8 + return size + + def verifySize(self, raw=bytearray(b'')): + """ + Return True if len(raw) is at least long enough for formatted size + """ + return (len(raw) >= (self.fmtSize)) + + def parse(self, raw): + """Parse raw bytearray and assign to fields + Return offset into raw of unparsed portion + Base method to be overridden in subclass + """ + if (not raw) or (not self.verifySize(raw)): + raise ValueError("Parse Packifier: Not enough raw data for packifier. " + "Need {0} bytes, got {1} bytes.".format(self.size, + len(raw))) + + result = unpackify(self.fmt, raw, boolean=True, size=self.fmtSize) # empty result + self.packed[:] = raw[0:self.size] + + return self.size #return offset to start of unparsed portion of data + + def pack(self, **kwa): + """ + Return .packed with data if any + Base method to be overridden in sub class + """ + size = packifyInto(self.packed, fmt=self.fmt, fields=()) + + if self.size != size : + raise ValueError("Build Packifier: size packed={0} not match " + "format={1}".format(self.size, size)) + return self.packed + + def show(self): + """ + Returns descriptive string for display purposes + """ + name = self.__class__.__name__ + result = (" {0}: packed=0x{1}\n".format(name, + hexlify(self.packed).decode('ascii'))) + return result + + +class PacketPart(Part): + """ + PacketPart base class for parts of packets. + Allows PacketPart to reference other parts of its Packet + """ + + def __init__(self, packet=None, **kwa): + """ + Initialization method for instance. + Base class method to be overridden in subclass + Need to add parts to packet in subclass + + Inherited Parameters: + size is initial size of .packed + + Parameters: + packet is Packet instance that holds this part + + Inherited Attributes: + .packed is bytearray of packed binary data + + Attributes: + .packet is Packet instance that holds this part + + Properties: + .size is length of .packed + + """ + self.packet = packet # do this first in case mixin needs attribute + super(PacketPart, self).__init__(**kwa) + + + def show(self): + """ + Returns descriptive string for display purposes + """ + name = self.__class__.__name__ + result = (" {0}: packed=0x{1}\n".format(name, + hexlify(self.packed).decode('ascii'))) + return result + + +class Packet(Part): + """ + Packet base class + Allows packet to reference its stack + """ + + def __init__(self, stack=None, **kwa): + """ + Initialization method for instance. + Base class method to be overridden in subclass + Need to add parts to packet in subclass + + Inherited Parameters: + size is initial size of .packed + + Parameters: + stack is I/O stack that handles this packet + + Inherited Attributes: + .packed is bytearray of packed binary data + + Attributes: + .stack is I/O stack that handles this packet + + Inherited Properties: + .size is length of .packed + + + """ + super(Packet, self).__init__(**kwa) + self.stack = stack + + + def parse(self, raw): + """ + Parse raw data into .packed + """ + self.packed = bytearray(raw) + return self.size + + def pack(self): + """ + Pack into .packed + """ + return self.packed + diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 521d191..14c3535 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -110,13 +110,17 @@ class Peer(filing.Filer): Class Attributes: Umask (int): octal default umask permissions such as 0o022 MaxUxdPathSize (int:) max characters in uxd file path - UxdBufSize (int): used to set buffer size for UXD datagram buffers - MaxUxdPayloadSize (int): limit datagram payload to this size + MaxGramSize (int): max bytes in in datagram for this transport + BufSize (int): used to set buffer size for transport datagram buffers + Attributes: umask (int): unpermission mask for uxd file, usually octal 0o022 .umask is applied after .perm is set if any - bs (int): buffer size + bc (int | None): count of transport buffers of MaxGramSize + bs (int): buffer size of transport buffers. When .bc then .bs is calculated + by multiplying, .bs = .bc * .MaxGramSize. When .bc is None then .bs + is provided value or default .BufSize wl (WireLog): instance ref for debug logging of over the wire tx and rx ls (socket.socket): local socket of this Peer @@ -135,10 +139,11 @@ class Peer(filing.Filer): Fext = "uxd" Umask = 0o022 # default MaxUxdPathSize = 108 - UxdBufSize = 65535 # 2 ** 16 - 1 - MaxUxdGramSize = UxdBufSize + MaxGramSize = 65535 # 2 ** 16 - 1 default gram size override in subclass + BufSize = 65535 # 2 ** 16 - 1 default buffersize + - def __init__(self, *, umask=None, bs = None, wl=None, + def __init__(self, *, umask=None, bc=None, bs=None, wl=None, reopen=False, clear=True, filed=False, extensioned=True, **kwa): """Initialization method for instance. @@ -158,11 +163,21 @@ def __init__(self, *, umask=None, bs = None, wl=None, Parameters: umask (int): unpermission mask for uxd file, usually octal 0o022 - bs (int): buffer size + bc (int | None): count of transport buffers of MaxGramSize + bs (int | None): buffer size of transport buffers. When .bc is provided + then .bs is calculated by multiplying, .bs = .bc * .MaxGramSize. + When .bc is not provided, then if .bs is provided use provided + value else use default .BufSize wl (WireLog): instance ref for debug logging of over the wire tx and rx """ self.umask = umask # only change umask if umask is not None below - self.bs = bs if bs is not None else self.UxdBufSize + self.bc = bc + if self.bc: + self.bs = self.MaxGramSize * self.bc + else: + self.bs = bs if bs is not None else self.BufSize + + self.wl = wl self.ls = None # local socket of this Peer, needs to be opened/bound diff --git a/src/hio/help/helping.py b/src/hio/help/helping.py index 05d3ea5..59cddca 100644 --- a/src/hio/help/helping.py +++ b/src/hio/help/helping.py @@ -11,6 +11,10 @@ import stat import json +import base64 +import re + +from collections import deque from collections.abc import Iterable, Sequence, Generator from abc import ABCMeta @@ -418,3 +422,59 @@ def load(path): return it + +# Base64 utilities + +# Mappings between Base64 Encode Index and Decode Characters +# B64ChrByIdx is dict where each key is a B64 index and each value is the B64 char +# B64IdxByChr is dict where each key is a B64 char and each value is the B64 index +# Map Base64 index to char +B64ChrByIdx = dict((index, char) for index, char in enumerate([chr(x) for x in range(65, 91)])) +B64ChrByIdx.update([(index + 26, char) for index, char in enumerate([chr(x) for x in range(97, 123)])]) +B64ChrByIdx.update([(index + 52, char) for index, char in enumerate([chr(x) for x in range(48, 58)])]) +B64ChrByIdx[62] = '-' +B64ChrByIdx[63] = '_' +# Map char to Base64 index +B64IdxByChr = {char: index for index, char in B64ChrByIdx.items()} + +B64REX = rb'^[0-9A-Za-z_-]*\Z' # [A-Za-z0-9\-\_] bytes +# Usage: if Reb64.match(bext): or if not Reb64.match(bext): bext is bytes +Reb64 = re.compile(B64REX) # compile is faster + +def intToB64(i, l=1): + """ + Returns conversion of int i to Base64 str + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + d = deque() # deque of characters base64 + + while l: + d.appendleft(B64ChrByIdx[i % 64]) + i = i // 64 + if not i: + break + for j in range(l - len(d)): # range(x) x <= 0 means do not iterate + d.appendleft("A") + return ("".join(d)) + + +def intToB64b(i, l=1): + """ + Returns conversion of int i to Base64 bytes + l is min number of b64 digits left padded with Base64 0 == "A" char + """ + return (intToB64(i=i, l=l).encode("utf-8")) + + +def b64ToInt(s): + """ + Returns conversion of Base64 str s or bytes to int + """ + if not s: + raise ValueError("Empty string, conversion undefined.") + if hasattr(s, 'decode'): + s = s.decode("utf-8") + i = 0 + for e, c in enumerate(reversed(s)): + i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) + return i diff --git a/src/hio/help/packing.py b/src/hio/help/packing.py new file mode 100644 index 0000000..9fa3a60 --- /dev/null +++ b/src/hio/help/packing.py @@ -0,0 +1,208 @@ +# -*- encoding: utf-8 -*- +""" +hio.help.packing Module + +Packs and unpacks bit fields using format string whereas struct in std lib only +packs and unpacks byte fields. + +""" + + + +def packify(fmt=u'8', fields=[0x00], size=None, reverse=False): + """ + Packs fields sequence of bit fields into bytearray of size bytes using fmt string. + Each white space separated field of fmt is the length of the associated bit field + If not provided size is the least integer number of bytes that hold the fmt. + If reverse is true reverse the order of the bytes in the byte array before + returning. This is useful for converting between bigendian and littleendian. + + Assumes unsigned fields values. + Assumes network big endian so first fields element is high order bits. + Each field in format string is number of bits for the associated bit field + Fields with length of 1 are treated as has having boolean truthy field values + that is, nonzero is True and packs as a 1 + for 2+ length bit fields the field element is truncated to the number of + low order bits in the bit field + if sum of number of bits in fmt less than size bytes then the last byte in + the bytearray is right zero padded + if sum of number of bits in fmt greater than size bytes returns exception + to pad just use 0 value in source field. + example + packify("1 3 2 2", (True, 4, 0, 3)). returns bytearry([0xc3]) + """ + tbfl = sum((int(x) for x in fmt.split())) + if size is None: + size = (tbfl // 8) + 1 if tbfl % 8 else tbfl // 8 + + if not (0 <= tbfl <= (size * 8)): + raise ValueError("Total bit field lengths in fmt not in [0, {0}]".format(size * 8)) + + n = 0 + bfp = 8 * size # starting bit field position + bu = 0 # bits used + + for i, bfmt in enumerate(fmt.split()): + bits = 0x00 + bfl = int(bfmt) + bu += bfl + + if bfl == 1: + if fields[i]: + bits = 0x01 + else: + bits = 0x00 + else: + bits = fields[i] & (2**bfl - 1) # bit-and mask out high order bits + + bits <<= (bfp - bfl) #shift left to bit position less bit field size + + n |= bits # bit-or in bits + bfp -= bfl #adjust bit field position for next element + + return bytify(n=n, size=size, reverse=reverse, strict=True) # use int.to_bytes + +def packifyInto(b, fmt=u'8', fields=[0x00], size=None, offset=0, reverse=False): + """ + Packs fields sequence of bit fields using fmt string into bytearray b + starting at offset and packing into size bytes + Each white space separated field of fmt is the length of the associated bit field + If not provided size is the least integer number of bytes that hold the fmt. + Extends the length of b to accomodate size after offset if not enough. + Returns actual size of portion packed into. + The default assumes big endian. + If reverse is True then reverses the byte order before extending. Useful for + little endian. + + Assumes unsigned fields values. + Assumes network big endian so first fields element is high order bits. + Each field in format string is number of bits for the associated bit field + Fields with length of 1 are treated as has having boolean truthy field values + that is, nonzero is True and packs as a 1 + for 2+ length bit fields the field element is truncated + to the number of low order bits in the bit field + if sum of number of bits in fmt less than size bytes then the last byte in + the bytearray is right zero padded + if sum of number of bits in fmt greater than size bytes returns exception + to pad just use 0 value in source field. + example + packify("1 3 2 2", (True, 4, 0, 3)). returns bytearry([0xc3]) + """ + tbfl = sum((int(x) for x in fmt.split())) + if size is None: + size = (tbfl // 8) + 1 if tbfl % 8 else tbfl // 8 + + if not (0 <= tbfl <= (size * 8)): + raise ValueError("Total bit field lengths in fmt not in [0, {0}]".format(size * 8)) + + if len(b) < (offset + size): + b.extend([0x00]*(offset + size - len(b))) + + n = 0 + bfp = 8 * size # starting bit field position + bu = 0 # bits used + + for i, bfmt in enumerate(fmt.split()): + bits = 0x00 + bfl = int(bfmt) + bu += bfl + + if bfl == 1: + if fields[i]: + bits = 0x01 + else: + bits = 0x00 + else: + bits = fields[i] & (2**bfl - 1) # bit-and mask out high order bits + + bits <<= (bfp - bfl) #shift left to bit position less bit field size + + n |= bits # bit-or in bits + bfp -= bfl #adjust bit field position for next element + + bp = bytify(n=n, size=size, reverse=reverse, strict=True) # use int.to_bytes + + b[offset:offset + len(bp)] = bp + + #count = 0 + #while n or (count < size): + #b[offset + size - 1 - count] = n & 0xFF + #count += 1 + #n >>= 8 + + return size + +def unpackify(fmt=u'1 1 1 1 1 1 1 1', + b=bytearray([0x00]), + boolean=False, + size=None, + reverse=False): + """ + Returns tuple of unsigned int bit field values that are unpacked from the + bytearray b according to fmt string. b maybe an integer iterator + If not provided size is the least integer number of bytes that hold the fmt. + The default assumes big endian. + If reverse is True then reverse the byte order of b before unpackifing. This + is useful for little endian. + + Each white space separated field of fmt is the length of the associated bit field. + returns unsigned fields values. + + Assumes network big endian so first fmt is high order bits. + Format string is number of bits per bit field + If boolean parameter is True then return boolean values for + bit fields of length 1 + + if sum of number of bits in fmt less than 8 * size) then remaining + bits are returned as additional field in result. + + if sum of number of bits in fmt greater 8 * len(b) returns exception + + example: + unpackify(u"1 3 2 2", bytearray([0xc3]), False) returns (1, 4, 0, 3) + unpackify(u"1 3 2 2", 0xc3, True) returns (True, 4, 0, 3) + """ + b = bytearray(b) + if reverse: + b.reverse() + + tbfl = sum((int(x) for x in fmt.split())) + if size is None: + size = (tbfl // 8) + 1 if tbfl % 8 else tbfl // 8 + + if not (0 <= tbfl <= (size * 8)): + raise ValueError("Total bit field lengths in fmt not in [0, {0}]".format(size * 8)) + + b = b[:size] + fields = [] # list of bit fields + bfp = 8 * size # bit field position + bu = 0 # bits used + n = unbytify(b) # unsigned int equivalent of b # use int.from_bytes instead + + for i, bfmt in enumerate(fmt.split()): + bfl = int(bfmt) + bu += bfl + + mask = (2**bfl - 1) << (bfp - bfl) # make mask + bits = n & mask # mask off other bits + bits >>= (bfp - bfl) # right shift to low order bits + if bfl == 1 and boolean: #convert to boolean + if bits: + bits = True + else: + bits = False + fields.append(bits) #assign to fields list + bfp -= bfl #adjust bit field position for next element + + if bfp != 0: # remaining bits + bfl = bfp + mask = (2**bfl - 1) # make mask + bits = n & mask # mask off other bits + if bfl == 1 and boolean: #convert to boolean + if bits: + bits = True + else: + bits = False + fields.append(bits) #assign to fields list + return tuple(fields) #convert to tuple + diff --git a/src/hio/hioing.py b/src/hio/hioing.py index a6f9f2b..4a99915 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -11,12 +11,23 @@ Versionage = namedtuple("Versionage", "major minor") -Version = Versionage(major=1, minor=0) # KERI Protocol Version +Version = Versionage(major=1, minor=0) # Protocol Version SEPARATOR = "\r\n\r\n" SEPARATOR_BYTES = SEPARATOR.encode("utf-8") +class Mixin(): + """ + Base class to enable consistent MRO for mixin multiple inheritance + Allows each subclass to call + super(MixinSubClass, self).__init__(*pa, **kwa) + So the __init__ propagates to common top of Tree + https://medium.com/geekculture/cooperative-multiple-inheritance-in-python-practice-60e3ac5f91cc + """ + def __init__(self, *pa, **kwa): + pass + class HioError(Exception): """ @@ -72,14 +83,3 @@ class NamerError(HioError): raise NamerError("error message") """ - -class Mixin(): - """ - Base class to enable consistent MRO for mixin multiple inheritance - Allows each subclass to call - super(MixinSubClass, self).__init__(*pa, **kwa) - So the __init__ propagates to common top of Tree - https://medium.com/geekculture/cooperative-multiple-inheritance-in-python-practice-60e3ac5f91cc - """ - def __init__(self, *pa, **kwa): - pass diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxd.py index 56619f0..188b750 100644 --- a/tests/core/uxd/test_uxd.py +++ b/tests/core/uxd/test_uxd.py @@ -23,12 +23,13 @@ def test_uxd_basic(): """ tymist = tyming.Tymist() with (wiring.openWL(samed=True, filed=True) as wl): - - alpha = uxding.Peer(name="alpha", temp=True, umask=0o077, wl=wl) + bc = 64 + alpha = uxding.Peer(name="alpha", temp=True, umask=0o077, bc=bc, wl=wl) assert not alpha.opened assert alpha.reopen() assert alpha.opened assert alpha.path.endswith("alpha.uxd") + assert alpha.actualBufSizes() == (4194240, 4194240) == (bc * alpha.MaxGramSize, bc * alpha.MaxGramSize) beta = uxding.Peer(name="beta", temp=True, umask=0o077) assert beta.reopen() @@ -157,7 +158,7 @@ def test_open_peer(): assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert alpha.actualBufSizes() == (65535, 65535) == (alpha.UxdBufSize, alpha.UxdBufSize) + assert alpha.actualBufSizes() == (65535, 65535) == (alpha.BufSize, alpha.BufSize) assert beta.opened assert beta.path.endswith("beta.uxd") diff --git a/tests/help/test_helping.py b/tests/help/test_helping.py index 8eb3773..639dbb1 100644 --- a/tests/help/test_helping.py +++ b/tests/help/test_helping.py @@ -288,7 +288,110 @@ def test_ocfn_load_dump(): +def test_b64_helpers(): + """ + Test Base64 conversion utility routines + """ + + cs = helping.intToB64(0) + assert cs == "A" + i = helping.b64ToInt(cs) + assert i == 0 + + cs = helping.intToB64(0, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = helping.b64ToInt(cs) + + cs = helping.intToB64(None, l=0) + assert cs == "" + with pytest.raises(ValueError): + i = helping.b64ToInt(cs) + + cs = helping.intToB64b(0) + assert cs == b"A" + i = helping.b64ToInt(cs) + assert i == 0 + + cs = helping.intToB64(27) + assert cs == "b" + i = helping.b64ToInt(cs) + assert i == 27 + + cs = helping.intToB64b(27) + assert cs == b"b" + i = helping.b64ToInt(cs) + assert i == 27 + + cs = helping.intToB64(27, l=2) + assert cs == "Ab" + i = helping.b64ToInt(cs) + assert i == 27 + + cs = helping.intToB64b(27, l=2) + assert cs == b"Ab" + i = helping.b64ToInt(cs) + assert i == 27 + + cs = helping.intToB64(80) + assert cs == "BQ" + i = helping.b64ToInt(cs) + assert i == 80 + + cs = helping.intToB64b(80) + assert cs == b"BQ" + i = helping.b64ToInt(cs) + assert i == 80 + + cs = helping.intToB64(4095) + assert cs == '__' + i = helping.b64ToInt(cs) + assert i == 4095 + + cs = helping.intToB64b(4095) + assert cs == b'__' + i = helping.b64ToInt(cs) + assert i == 4095 + + cs = helping.intToB64(4096) + assert cs == 'BAA' + i = helping.b64ToInt(cs) + assert i == 4096 + + cs = helping.intToB64b(4096) + assert cs == b'BAA' + i = helping.b64ToInt(cs) + assert i == 4096 + + cs = helping.intToB64(6011) + assert cs == "Bd7" + i = helping.b64ToInt(cs) + assert i == 6011 + + cs = helping.intToB64b(6011) + assert cs == b"Bd7" + i = helping.b64ToInt(cs) + assert i == 6011 + + text = b"-A-Bg-1-3-cd" + match = helping.Reb64.match(text) + assert match + assert match is not None + + text = b'' + match = helping.Reb64.match(text) + assert match + assert match is not None + + text = b'123#$' + match = helping.Reb64.match(text) + assert not match + assert match is None + + """End Test""" + if __name__ == "__main__": test_attributize() + test_b64_helpers() From 2526c6a96ffa6896fd7758436528f87b108ec816 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 12:08:36 -0700 Subject: [PATCH 17/33] redid echo echoic support --- src/hio/core/memoing.py | 93 +++++++++++++++++++++++++------------- tests/core/test_memoing.py | 6 ++- 2 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index de1144b..5c847bd 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -366,6 +366,7 @@ def __init__(self, # initialize attributes self.version = version if version is not None else self.Version + self.echos = deque() # only used in testing as echoed tx self.rxgs = rxgs if rxgs is not None else dict() self.rxms = rxms if rxms is not None else deque() @@ -408,24 +409,35 @@ def close(self): self.opened = False - def receive(self, *, echo=None): + def receive(self, *, echoic=False): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. Parameters: - echo (tuple): returns echo for debugging purposes where echo is - duple of form (gram: bytes, src: str) otherwise returns duple - that indicates nothing to receive of form (b'', None) + echoic (bool): True means use .echos in .receive debugging purposes + where echo is duple of form: (gram: bytes, src: str) + False means do not use .echos default is duple that + indicates nothing to receive of form (b'', None) Returns: duple (tuple): of form (data: bytes, src: str) where data is the bytes of received data and src is the source address. - When no data the duple is (b'', None) + When no data the duple is (b'', None) unless echoic is True + then pop off echo from .echos """ - return echo if echo else (b'', None) + if echoic: + try: + result = self.echos.popleft() + except IndexError: + result = (b'', None) + else: + result = (b'', None) + + return result - def _serviceOneReceived(self, *, echo=None): + + def _serviceOneReceived(self, *, echoic=False): """Service one received duple (raw, src) raw packet data. Always returns complete datagram. @@ -437,13 +449,14 @@ def _serviceOneReceived(self, *, echo=None): more data to receive from transport Parameters: - echo (tuple): returns echo for debugging purposes where echo is - duple of form (gram: bytes, src: str) otherwise returns duple - that indicates nothing to receive of form (b'', None) + echoic (bool): True means use .echos in .receive debugging purposes + where echo is duple of form: (gram: bytes, src: str) + False means do not use .echos default is duple that + indicates nothing to receive of form (b'', None) """ try: - gram, src = self.receive(echo=echo) # if no data the duple is (b'', None) + gram, src = self.receive(echoic=echoic) # if no data the duple is (b'', None) except socket.error as ex: # OSError.errno always .args[0] for compat #if (ex.args[0] == (errno.ECONNREFUSED, #errno.ECONNRESET, @@ -471,28 +484,30 @@ def _serviceOneReceived(self, *, echo=None): return True # received data - def serviceReceivesOnce(self, *, echo=None): + def serviceReceivesOnce(self, *, echoic=False): """Service receives once (non-greedy) and queue up Parameters: - echo (tuple): returns echo for debugging purposes where echo is - duple of form (gram: bytes, src: str) otherwise returns duple - that indicates nothing to receive of form (b'', None) + echoic (bool): True means use .echos in .receive debugging purposes + where echo is duple of form: (gram: bytes, src: str) + False means do not use .echos default is duple that + indicates nothing to receive of form (b'', None) """ if self.opened: - self._serviceOneReceived(echo=echo) + self._serviceOneReceived(echoic=echoic) - def serviceReceives(self, *, echo=None): + def serviceReceives(self, *, echoic=False): """Service all receives (greedy) and queue up Parameters: - echo (tuple): returns echo for debugging purposes where echo is - duple of form (gram: bytes, src: str) otherwise returns duple - that indicates nothing to receive of form (b'', None) + echoic (bool): True means use .echos in .receive debugging purposes + where echo is duple of form: (gram: bytes, src: str) + False means do not use .echos default is duple that + indicates nothing to receive of form (b'', None) """ while self.opened: - if not self._serviceOneReceived(echo=echo): + if not self._serviceOneReceived(echoic=echoic): break echo = None # only echo once @@ -623,7 +638,7 @@ def serviceAllRx(self): self.serviceRxMemos() - def send(self, txbs, dst): + def send(self, txbs, dst, *, echoic=False): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. @@ -633,7 +648,11 @@ def send(self, txbs, dst): Parameters: txbs (bytes | bytearray): of bytes to send dst (str): remote destination address + echoic (bool): True means echo sends into receives via. echos + False measn do not echo """ + if echoic: + self.echos.append((bytes(txbs), dst)) # make copy return len(txbs) # all sent @@ -704,10 +723,14 @@ def gramit(self, gram, dst): self.txgs.append((gram, dst)) - def _serviceOnceTxGrams(self): + def _serviceOnceTxGrams(self, *, echoic=False): """Service one pass over .txgs dict for each unique dst in .txgs and then services one pass over the .txbs of those outgoing grams + Parameters: + echoic (bool): True means echo sends into receives via. echos + False measn do not echo + Each entry in.txgs is duple of form: (gram: bytes, dst: str) where: gram (bytes): is outgoing gram segment from associated memo @@ -751,7 +774,7 @@ def _serviceOnceTxGrams(self): try: - cnt = self.send(gram, dst) # assumes .opened == True + cnt = self.send(gram, dst, echoic=echoic) # assumes .opened == True except socket.error as ex: # OSError.errno always .args[0] for compat if (ex.args[0] in (errno.ECONNREFUSED, errno.ENOENT, @@ -776,28 +799,36 @@ def _serviceOnceTxGrams(self): del gram[:cnt] # remove from buffer those bytes sent if not gram: # all sent dst = None # indicate by setting dst to None - self.txbs = (gram, dst) + self.txbs = (gram, dst) # update txbs to indicate if completely sent return (False if dst else True) # incomplete return False, else True - def serviceTxGramsOnce(self): + def serviceTxGramsOnce(self, *, echoic=False): """Service one pass (non-greedy) over all unique destinations in .txgs dict if any for blocked destination or unblocked with pending outgoing grams. + + Parameters: + echoic (bool): True means echo sends into receives via. echos + False measn do not echo """ if self.opened and self.txgs: - self._serviceOnceTxGrams() + self._serviceOnceTxGrams(echoic=echoic) - def serviceTxGrams(self): + def serviceTxGrams(self, *, echoic=False): """Service multiple passes (greedy) over all unqique destinations in .txgs dict if any for blocked destinations or unblocked with pending outgoing grams until there is no unblocked destination with a pending gram. + + Parameters: + echoic (bool): True means echo sends into receives via. echos + False measn do not echo """ - while self.opened and self.txgs: - if not self._serviceOnceTxGrams(): # no pending gram on any unblocked dst, - break # so try again later + while self.opened and self.txgs: # pending gram(s) + if not self._serviceOnceTxGrams(echoic=echoic): # send incomplete + break # try again later def serviceAllTxOnce(self): diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 4289068..792655d 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -40,7 +40,8 @@ def test_memogram_basic(): assert not peer.rxgs assert not peer.rxms echo = (b"Bye yall", "beta") - peer.serviceReceives(echo=echo) + peer.echos.append(echo) + peer.serviceReceives(echoic=True) assert peer.rxgs["beta"][0] == b"Bye yall" peer.serviceRxGrams() assert not peer.rxgs @@ -124,7 +125,8 @@ def test_tymeememogram_basic(): assert not peer.rxgs assert not peer.rxms echo = (b"Bye yall", "beta") - peer.serviceReceives(echo=echo) + peer.echos.append(echo) + peer.serviceReceives(echoic=True) assert peer.rxgs["beta"][0] == b"Bye yall" peer.serviceRxGrams() assert not peer.rxgs From 7e2a83b318c039900e687a69fc3fb460a33c69af Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 12:13:24 -0700 Subject: [PATCH 18/33] added tests for new echoic --- tests/core/test_memoing.py | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 792655d..1b73258 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -49,6 +49,31 @@ def test_memogram_basic(): peer.serviceRxMemos() assert not peer.rxms + memo = "See ya later!" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('See ya later!', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert peer.txgs[0] == (b'See ya later!', 'beta') + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert peer.echos + + assert not peer.rxgs + assert not peer.rxms + peer.serviceReceives(echoic=True) + assert peer.rxgs["beta"][0] == b"See ya later!" + assert not peer.echos + peer.serviceRxGrams() + assert not peer.rxgs + assert peer.rxms[0] == ('See ya later!', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + + peer.close() assert peer.opened == False """ End Test """ @@ -134,6 +159,29 @@ def test_tymeememogram_basic(): peer.serviceRxMemos() assert not peer.rxms + memo = "See ya later!" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('See ya later!', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert peer.txgs[0] == (b'See ya later!', 'beta') + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert peer.echos + + assert not peer.rxgs + assert not peer.rxms + peer.serviceReceives(echoic=True) + assert peer.rxgs["beta"][0] == b"See ya later!" + assert not peer.echos + peer.serviceRxGrams() + assert not peer.rxgs + assert peer.rxms[0] == ('See ya later!', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + tymist = tyming.Tymist(tock=1.0) peer.wind(tymth=tymist.tymen()) assert peer.tyme == tymist.tyme == 0.0 From 72f640c2e9f4cb6ce5c828de786d8930100d1ee4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 12:26:37 -0700 Subject: [PATCH 19/33] some clean up --- src/hio/help/packing.py | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/src/hio/help/packing.py b/src/hio/help/packing.py index 9fa3a60..082c74a 100644 --- a/src/hio/help/packing.py +++ b/src/hio/help/packing.py @@ -43,15 +43,13 @@ def packify(fmt=u'8', fields=[0x00], size=None, reverse=False): bu = 0 # bits used for i, bfmt in enumerate(fmt.split()): - bits = 0x00 + bits = 0x0 bfl = int(bfmt) bu += bfl if bfl == 1: - if fields[i]: - bits = 0x01 - else: - bits = 0x00 + bits = 0x1 if fields[i] else 0x0 + else: bits = fields[i] & (2**bfl - 1) # bit-and mask out high order bits @@ -62,6 +60,7 @@ def packify(fmt=u'8', fields=[0x00], size=None, reverse=False): return bytify(n=n, size=size, reverse=reverse, strict=True) # use int.to_bytes + def packifyInto(b, fmt=u'8', fields=[0x00], size=None, offset=0, reverse=False): """ Packs fields sequence of bit fields using fmt string into bytearray b @@ -103,15 +102,12 @@ def packifyInto(b, fmt=u'8', fields=[0x00], size=None, offset=0, reverse=False): bu = 0 # bits used for i, bfmt in enumerate(fmt.split()): - bits = 0x00 + bits = 0x0 bfl = int(bfmt) bu += bfl if bfl == 1: - if fields[i]: - bits = 0x01 - else: - bits = 0x00 + bits = 0x1 if fields[i] else 0x0 else: bits = fields[i] & (2**bfl - 1) # bit-and mask out high order bits @@ -124,14 +120,9 @@ def packifyInto(b, fmt=u'8', fields=[0x00], size=None, offset=0, reverse=False): b[offset:offset + len(bp)] = bp - #count = 0 - #while n or (count < size): - #b[offset + size - 1 - count] = n & 0xFF - #count += 1 - #n >>= 8 - return size + def unpackify(fmt=u'1 1 1 1 1 1 1 1', b=bytearray([0x00]), boolean=False, @@ -187,10 +178,8 @@ def unpackify(fmt=u'1 1 1 1 1 1 1 1', bits = n & mask # mask off other bits bits >>= (bfp - bfl) # right shift to low order bits if bfl == 1 and boolean: #convert to boolean - if bits: - bits = True - else: - bits = False + bits = True if bits else False + fields.append(bits) #assign to fields list bfp -= bfl #adjust bit field position for next element @@ -199,10 +188,8 @@ def unpackify(fmt=u'1 1 1 1 1 1 1 1', mask = (2**bfl - 1) # make mask bits = n & mask # mask off other bits if bfl == 1 and boolean: #convert to boolean - if bits: - bits = True - else: - bits = False + bits = True if bits else False + fields.append(bits) #assign to fields list return tuple(fields) #convert to tuple From 66cf9cefc5b4c9efc18872b1314926f304b6ab3a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 19:40:29 -0700 Subject: [PATCH 20/33] started provide support for MemoGram part headers --- src/hio/core/memoing.py | 208 ++++++++++++++++++++++++++++++++++--- tests/core/test_memoing.py | 10 ++ 2 files changed, 203 insertions(+), 15 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 5c847bd..0f6b7eb 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -44,7 +44,8 @@ which is a little less than 2^16 (65536) bytes. https://people.computing.clemson.edu/~westall/853/notes/udpsend.pdf -However, the existence of the UDP and IP +However, the existence of the UDP and IPmid = uuid.uuid4().bytes # 16 byte random uuid + headers should also limit. Maximum IPV4 header size is 60 bytes and UDP header is 8 bytes so maximum UDP @@ -58,7 +59,8 @@ The maximum safe UDP payload is 508 bytes. This is a packet size of 576 (the "minimum maximum reassembly buffer size"), minus the maximum 60-byte IP header and the 8-byte UDP header. As others have mentioned, additional -protocol headers could be added in some circumstances. A more conservative +protocol headers could be added in some circumstances. A more conservamid = uuid.uuid4().bytes # 16 byte random uuid +tive value of around 300-400 bytes may be preferred instead. Usually ip headers are only 20 bytes so with UDP header (src, dst) of 8 bytes @@ -96,7 +98,8 @@ the undoubled value. https://unix.stackexchange.com/questions/38043/size-of-data-that-can-be-written-to-read-from-sockets -https://stackoverflow.com/questions/21856517/whats-the-practical-limit-on-the-size-of-single-packet-transmitted-over-domain +https://stackoverflow.com/questions/21856517/whats-the-practical-limitmid = uuid.uuid4().bytes # 16 byte random uuid +-on-the-size-of-single-packet-transmitted-over-domain Memo Memo partitioning into parts. @@ -110,7 +113,8 @@ Separator sep is b'|' must not be a base64 character. The head consists of three fields in base64 -mid = memo ID +mid = memo IDmid = uuid.uuid4().bytes # 16 byte random uuid + pn = part number of part in memo zero based pc = part count of total parts in memo may be zero when body is empty @@ -139,7 +143,12 @@ # test PartHead code = MtrDex.PartHead assert code == '0P' -codeb = code.encode() +codeb = code.encode()mid = uuid.uuid4().bytes # 16 byte random uuid + + + +mid = uuid.uuid4().bytes # 16 byte random uuid + mid = 1 midb = mid.to_bytes(16) @@ -152,17 +161,18 @@ assert pcb == b'\x00\x00\x02' raw = midb + pnb + pcb assert raw == (b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x01\x00\x00\x02') + b'\x00\x00\x01\x00\x00\x02')mid = uuid.uuid4().bytes # 16 byte random uuid + assert mid == int.from_bytes(raw[:16]) assert pn == int.from_bytes(raw[16:19]) assert pc == int.from_bytes(raw[19:22]) -midb64 = encodeB64(bytes([0] * 2) + midb) # prepad -pnb64 = encodeB64(pnb) -pcb64 = encodeB64(pcb) +midb64b = encodeB64(bytes([0] * 2) + midb)[2:] # prepad ans strip +pnb64b = encodeB64(pnb) +pcb64b = encodeB64(pcb) -qb64b = codeb + midb64[2:] + pnb64 + pcb64 +qb64b = codeb + midb64b + pnb64b + pcb64b assert qb64b == b'0PAAAAAAAAAAAAAAAAAAAAABAAABAAAC' qb64 = qb64b.decode() qb2 = decodeB64(qb64b) @@ -171,6 +181,123 @@ assert pn == int.from_bytes(decodeB64(qb64b[24:28])) assert pc == int.from_bytes(decodeB64(qb64b[28:32])) + +codeb64b = b'__" + +# 16 byte random uuid +midb64b = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad then strip +pn = 1 +pnb64b = helping.intToB64b(pn, l=3) +pc = 2 +pcb64b = helping.intToB64b(pc, l=3) + +headb64b = codeb64b + midb64b + pnb64b + pcb64b + +fmt = '!2s22s4s4s' +PartLeader = struct.Struct(fmt) +PartLeader.size == 32 +head = PartLeader.pack(codeb64b, midb64b, pnb64b, pcb64b) + +assert helping.Reb64.match(head) +codeb64b, midb64b, pnb64b, pcb64b = PartLeader.unpack(head) +codeb64b, midb64b, pnb64b, pcb64b = PartLeader unpack_from(fmt, part) + + +mid = helping.b64ToInt(b'AA' + midb64b)) +pn = helping.b64ToInt(pnb64b)) +pc = helping.b64ToInt(pcb64b)) + +HeadSize = 32 # base64 chars +codeb64b = b'__" +midb64b = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip +pn = 1 +pnb64b = helping.intToB64b(pn, l=3) +pc = 2 +pcb64b = helping.intToB64b(pc, l=3) + +headb64b = codeb64b + midb64b + pnb64b + pcb64b +assert helping.Reb64.match(headb64b) + +codeb64b = headb64b[:2] +mid = helping.b64ToInt(b'AA' + headb64b[2:24])) # prepad and convert +pn = helping.b64ToInt(headb64b[24:28])) +pc = helping.b64ToInt(headb64b[28:32])) + + +PartCode Typing/Versions uses a different base64 two char code. +Codes always start with `_` and second char in code starts at `_` +and walks backwards through B64 code table for each part type-version (tv) code. +Total head length including code must be a multiple of 4 characters so +converts to/from B2 without padding. So to change length in order to change +field lengths or add or remove fields need a new part (tv) code. + +To reiterate, if ever need to change anything ned a new part tv code. There are +no versions per type. This is deemed sufficient because anticipate a very +limited set of possible fields ever needed for memogram transport types. + +tv 0 is '__' +tv 1 is '_-' +tv 3 is '_9' + +Because the part headers are valid mid-pad B64 primitives then can be losslessly +transformed to/from CESR primitives merely by translated the part tv code to +an equivalent entry in the two char cesr primitive code table. +The CESR codes albeit different are one-to-one. This enables representing headers +as CESR primitives for management but not over the wire. I.e. can transform +Memogram Parts to CESR primitives to tunnel in some CESR stream. + +Normally CESR streams are tunneled inside Memograms sent over-the-wire. When +the MemoGram is the wrapper then use the part tv code not the equivalent CESR code. +By using so the far reserved and unused '_' CESR op code table selector char +from CESR it makes it obvious that its a Memogram +tv Part Code not a CESR code when debugging. Even when the op codes are defined +its not likely to be confusing since the context of CESR op codes is different. +Morover when the MemoGram is a CESR Stream, a memogram parser will parse and +strip the MemoGram wrappers to construct the MemoGram, so no collisions. + + + +HeadCode hc = b'__' +HeadCodeSize hcs = 2 +MemoIDSize mis = 22 +PartNumSize pns = 4 +PartCntSize pcs = 4 +PartHeadSize phs = 32 + + +MaxPartBodySize = ()2** 16 -1) - HeadSize +PartBodySize pbs = something <= MaxBodySize +PartSize = PartHeadSize + PartBodySize +PartSize <= 2**16 -1 + +MaxMemoSize = (2**16-1)**2 +MaxPartCount for UDP PartSize of 508 is ceil(MaxMemoSize/508) = 8454403 which is < 2**24-1 +not partsize includes head +MaxPartCount for UXD PartSize of 2**16 -1 is ceil(MaxMemoSize/(2**16-1))= 2**16-1 +MinPartSize = MaxMemoSize // (2**24-1) = 255 = 2**8-1 +MaxPartSize = (2**16-1) +For any given PartSize there is a MaxPartCount of 2**24-1 so if fix the PartSize +this limits the effective +mms = min((2**16-1)**2), (PartSize - PartHeadSize) * (2**24-1)) +mms = min((2**16-1)**2), (PartBodySize) * (2**24-1)) + +so ps, pbs, and mms are variables specific to transport +PHS is fixed for the transport type reliable unreliable with header fields as defined +The desired ps for a given transport instance may be smaller than the allowed +maximum to accomodate buffers etc. +Given the fixed part head size PHS once can calculate the maximum memo size +that includes the header overhead given the contraint of no more than 2**24-1 +parts in a given memo. + +So for a given transport: +ps == min(ps, 2**16-1) +pbs = ps - PHS + +mms = min((2**16-1)**2), pbs * (2**24-1)) +So compare memo size to mms and if greater raise error and drop memo +otherwise partition (part) memo into parts with headers and transmit + + """ import socket import errno @@ -189,6 +316,15 @@ # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") +# namedtuple for size entries in MemoGram Pizes (part sizes) table +# cs is the part head size int number of chars in part code +# phs is the soft size int number of chars in soft (unstable) part of code +# xs is the xtra size into number of xtra (pre-pad) chars as part of soft +# fs is the full size int number of chars in code plus appended material if any +# ls is the lead size int number of bytes to pre-pad pre-converted raw binary +Sizage = namedtuple("Sizage", "hs ss xs fs ls") + + @contextmanager def openMG(cls=None, name="test", **kwa): @@ -300,12 +436,19 @@ class MemoGram(hioing.Mixin): (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: (gram: bytearray, dst: str). gram bytearray may hold untransmitted - portion when datagram is not able to be sent all at once so can + portion when datagram is not able to be sent all at once so can# namedtuple for size entries in Matter and Counter derivation code tables +# hs is the hard size int number of chars in hard (stable) part of code +# ss is the soft size int number of chars in soft (unstable) part of code +# xs is the xtra size into number of xtra (pre-pad) chars as part of soft +# fs is the full size int number of chars in code plus appended material if any +# ls is the lead size int number of bytes to pre-pad pre-converted raw binary +Sizage = namedtuple("Sizage", "hs ss xs fs ls") + keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - .rxgs dict keyed by mid (memo id) that holds incomplete memo parts. + rxgs dict keyed by mid (memo id) that holds incomplete memo parts. The mid appears in every gram part from the same memo. each val is dict of received gram parts keyed by part number. @@ -313,7 +456,18 @@ class MemoGram(hioing.Mixin): reattaching src to memo when placing in rxms deque - .mids is dict of dicts keyed by src and each value dict keyed by + mids (dict): of dicts keyed by src and each value dict keyed by + + echos (deque): holding echo receive duples for testing. Each duple of + + pc (bytes | None): part code for part header + form: (gram: bytes, dst: str). + ps (int): part size for this instance for transport. Part size + = part head size + part body size. Part size limited by + MaxPartSize and PartHeadSize + 1 + mms (int): max memo size relative to part size. Limited by MaxMemoSize + and MaxPartCount for the part size. + Properties: @@ -323,6 +477,12 @@ class MemoGram(hioing.Mixin): """ Version = Versionage(major=0, minor=0) # default version BufCount = 64 # default bufcount bc for transport + PartCode = b'__' # default part type-version code for head + MaxMemoSize = 4294836225 # (2**16-1)**2 absolute max memo size + MaxPartSize = 65535 # (2**16-1) absolute max part size + MaxPartCount = 16777215 # (2**24-1) absolute max part count + PartHeadSizes = {b'__': 32} # dict of part head sizes keyed by part codes + def __init__(self, name='main', @@ -333,6 +493,8 @@ def __init__(self, txms=None, txgs=None, txbs=None, + pc=None, + ps=None, **kwa ): """Setup instance @@ -359,14 +521,21 @@ def __init__(self, portion when datagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) + pc (bytes | None): part code for part header + ps (int): part size for this instance for transport. Part size + = part head size + part body size. Part size limited by + MaxPartSize and MaxPartCount relative to MaxMemoSize as well + as minimum part body size of 1 + pbs (int): part body size = part size - part head size for given + part code. Part body size must be at least 1. + mms (int): max memo size relative to part size and limited by + MaxMemoSize and MaxPartCount for given part size. """ # initialize attributes self.version = version if version is not None else self.Version - - self.echos = deque() # only used in testing as echoed tx self.rxgs = rxgs if rxgs is not None else dict() self.rxms = rxms if rxms is not None else deque() @@ -376,6 +545,15 @@ def __init__(self, bc = bc if bc is not None else self.BufCount # use bufcount to calc .bs + self.echos = deque() # only used in testing as echoed tx + + self.pc = pc if pc is not None else self.PartCode + phs = self.PartHeadSizes[self.pc] # part head size + ps = ps if ps is not None else self.MaxPartSize + self.ps = max(min(ps, self.MaxPartSize), phs + 1) + self.pbs = (self.ps - phs) # part body size + self.mms = min(self.MaxMemoSize, self.pbs * self.MaxPartCount) + super(MemoGram, self).__init__(name=name, bc=bc, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 1b73258..4501946 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -17,6 +17,11 @@ def test_memogram_basic(): peer = memoing.MemoGram() assert peer.name == "main" assert peer.opened == False + assert peer.pc == b'__' == peer.PartCode + assert peer.ps == peer.MaxPartSize + assert peer.pbs == peer.ps - peer.PartHeadSizes[peer.pc] + assert peer.mms == peer.MaxMemoSize + peer.reopen() assert peer.opened == True @@ -127,6 +132,11 @@ def test_tymeememogram_basic(): assert peer.tymeout == 0.0 assert peer.name == "main" assert peer.opened == False + assert peer.pc == b'__' == peer.PartCode + assert peer.ps == peer.MaxPartSize + assert peer.pbs == peer.ps - peer.PartHeadSizes[peer.pc] + assert peer.mms == peer.MaxMemoSize + peer.reopen() assert peer.opened == True From 620532bad7425b0d9ae88c1e1db612b846604d9d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 20 Feb 2025 20:13:00 -0700 Subject: [PATCH 21/33] more work on part header --- src/hio/core/memoing.py | 44 +++++++++++++++++++++++++---------------- src/hio/hioing.py | 7 +++++++ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 0f6b7eb..381a978 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -316,14 +316,8 @@ # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") -# namedtuple for size entries in MemoGram Pizes (part sizes) table -# cs is the part head size int number of chars in part code -# phs is the soft size int number of chars in soft (unstable) part of code -# xs is the xtra size into number of xtra (pre-pad) chars as part of soft -# fs is the full size int number of chars in code plus appended material if any -# ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Sizage = namedtuple("Sizage", "hs ss xs fs ls") - +Encodage = namedtuple("Encodage", 'txt bny') # part head encoding +Encodes = Encodage(txt='txt', bny='bny') @contextmanager @@ -436,14 +430,7 @@ class MemoGram(hioing.Mixin): (gram: bytes, dst: str). txbs (tuple): current transmisstion duple of form: (gram: bytearray, dst: str). gram bytearray may hold untransmitted - portion when datagram is not able to be sent all at once so can# namedtuple for size entries in Matter and Counter derivation code tables -# hs is the hard size int number of chars in hard (stable) part of code -# ss is the soft size int number of chars in soft (unstable) part of code -# xs is the xtra size into number of xtra (pre-pad) chars as part of soft -# fs is the full size int number of chars in code plus appended material if any -# ls is the lead size int number of bytes to pre-pad pre-converted raw binary -Sizage = namedtuple("Sizage", "hs ss xs fs ls") - + portion when Encodesdatagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) @@ -527,7 +514,7 @@ def __init__(self, MaxPartSize and MaxPartCount relative to MaxMemoSize as well as minimum part body size of 1 pbs (int): part body size = part size - part head size for given - part code. Part body size must be at least 1. + paEncodesrt code. Part body size must be at least 1. mms (int): max memo size relative to part size and limited by MaxMemoSize and MaxPartCount for given part size. @@ -587,6 +574,29 @@ def close(self): self.opened = False + def wiff(self, part): + """ + Returns encoding format of part bytes header as either base64 text 'txt' + or base2 binary 'bny' from first 6 bits (sextet) of first byte in part. + All part head codes start with '_' in base64 text or in base2 binary. + Given only allowed chars are from the set of base64 then can determine + if header is in base64 or base2. + + First sextet: + + 0o2 = 010 means first sextet of '_' in base64 =>'txt' + 0o7 = 111 means frist sextet of '_' in base2 => 'bny' + """ + + sextet = part[0] >> 2 + if sextet == 0o123: + return Encodes.txt # 'txt' + if sextet == 0o321: + return Encodes.bny # 'bny' + + raise hioing.MemoGramError(f"Unexpected {sextet=} at part head start.") + + def receive(self, *, echoic=False): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. diff --git a/src/hio/hioing.py b/src/hio/hioing.py index 4a99915..368842f 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -83,3 +83,10 @@ class NamerError(HioError): raise NamerError("error message") """ +class MemoGramError(HioError): + """ + Error using or configuring Remoter + + Usage: + raise MemoGramError("error message") + """ From e012e4009836cf9e3f91f73ac53a9aa191520006 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 22 Feb 2025 18:38:03 -0700 Subject: [PATCH 22/33] MemoGram complete first pass with tests --- src/hio/core/memoing.py | 612 +++++++++++++++++++++++++------------ src/hio/help/helping.py | 23 +- tests/core/test_memoing.py | 400 ++++++++++++++++++++++-- 3 files changed, 817 insertions(+), 218 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 381a978..e48da4c 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -207,28 +207,40 @@ pn = helping.b64ToInt(pnb64b)) pc = helping.b64ToInt(pcb64b)) -HeadSize = 32 # base64 chars +Standard header in all parts consists of +code + mid + pn + +The first part with pn == 0 also has an additional field called the neck +that is the total part count pc + +Standard +HeadSize = 28 # base64 chars or 21 base2 bytes +NeckSize = 4 # base 64 chars or 21 base2 bytes + codeb64b = b'__" midb64b = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip pn = 1 -pnb64b = helping.intToB64b(pn, l=3) +pnb64b = helping.intToB64b(pn, l=4) pc = 2 -pcb64b = helping.intToB64b(pc, l=3) +pcb64b = helping.intToB64b(pc, l=4) -headb64b = codeb64b + midb64b + pnb64b + pcb64b +headb64b = codeb64b + midb64b + pnb64b +neckb64b = pcb64b assert helping.Reb64.match(headb64b) +assert helping.Reb64.match(neckb64b) codeb64b = headb64b[:2] mid = helping.b64ToInt(b'AA' + headb64b[2:24])) # prepad and convert pn = helping.b64ToInt(headb64b[24:28])) -pc = helping.b64ToInt(headb64b[28:32])) +pc = helping.b64ToInt(neckb64b[28:32])) PartCode Typing/Versions uses a different base64 two char code. Codes always start with `_` and second char in code starts at `_` and walks backwards through B64 code table for each part type-version (tv) code. -Total head length including code must be a multiple of 4 characters so -converts to/from B2 without padding. So to change length in order to change +Each of total head length including code and total neck length must be a +multiple of 4 characters. so eacch converts to/from B2 without padding. +Head type-version code governs the neck as well. So to change length in order to change field lengths or add or remove fields need a new part (tv) code. To reiterate, if ever need to change anything ned a new part tv code. There are @@ -239,9 +251,11 @@ tv 1 is '_-' tv 3 is '_9' -Because the part headers are valid mid-pad B64 primitives then can be losslessly +Because the part head and neck are valid mid-pad B64 primitives then can be losslessly transformed to/from CESR primitives merely by translated the part tv code to -an equivalent entry in the two char cesr primitive code table. +an equivalent entry in the two char cesr primitive code table. When doing so +The neck is always attached and then stripped when not needed. + The CESR codes albeit different are one-to-one. This enables representing headers as CESR primitives for management but not over the wire. I.e. can transform Memogram Parts to CESR primitives to tunnel in some CESR stream. @@ -262,7 +276,8 @@ MemoIDSize mis = 22 PartNumSize pns = 4 PartCntSize pcs = 4 -PartHeadSize phs = 32 +PartHeadSize phs = 28 +PartHeadNeckSize phns = 28 + 4 = 32 MaxPartBodySize = ()2** 16 -1) - HeadSize @@ -280,6 +295,7 @@ this limits the effective mms = min((2**16-1)**2), (PartSize - PartHeadSize) * (2**24-1)) mms = min((2**16-1)**2), (PartBodySize) * (2**24-1)) +Note there is an extra 4 characters in first part for the neck so ps, pbs, and mms are variables specific to transport PHS is fixed for the transport type reliable unreliable with header fields as defined @@ -298,59 +314,100 @@ otherwise partition (part) memo into parts with headers and transmit +Note max memo size is (2**16-1)**2 = 4294836225. +But accounting for the part header overhead the largest payload max memo size is +max part size = (2**16-1) == 65535 +max part count = (2**24-1) == 16777215 +standard header size hs = 28 +neck size ns = 4 +bs = maxpartsize - hs = (65535 - hs) = 65507 # max standard part body size +max theoretical memogram payload size mms is +mms = (bs * maxpartcount) - ns = 65507 * 16777215 - 4 = 1099025023001 +but this is too big so we limit it to 4294836225 but we can have a payload +of 4294836225 when using the maxpartsize for each part. + +The maximum number of bytes in a CESR big group frame body is +(2**30-1)*4 = 4294967292. +The maximum group count is (2**30-1) == 1073741823 +Groups code count in quadlets (4) and the max count value is (2**30-1) for 5 +6 bit b64 digits. +The group code is not part of the counted quadlets so the largest group +with its group code of length 8 is 4294967292 + 8 = 4294967300 +which is (2**30-1)*4 + 8. + +This is bigger than the max effective MemoGram payload size by +4294967300 - 4294836225 = 131075 + + +So when sending via segmented MemoGram parts the maximum big group count frame +with code as payload is + +(4294836225 - 8) // 4 = 1073709054 + +This is less than the absolute max group count by +1073741823 - 1073709054 = 32769 + +So biggest memogram payload is relatively slightly smaller than the max group +frame size with group code. + +To fix this we would have to allow for a bigger part gram size than 2**16-1 +For UXD datagrams this is not a hard limit +but it is a hard limit for UDP datagrams + +It is unlikely we would ever reach those limits in a practical application. + + """ import socket import errno -import struct +import math +import uuid +#import struct from collections import deque, namedtuple from contextlib import contextmanager from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 +from dataclasses import dataclass, astuple -from hio import hioing, help -from hio.base import tyming, doing +from .. import hioing, help +from ..base import tyming, doing +from ..help import helping logger = help.ogler.getLogger() # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") -Encodage = namedtuple("Encodage", 'txt bny') # part head encoding -Encodes = Encodage(txt='txt', bny='bny') +Wiffage = namedtuple("Wiffage", 'txt bny') # part head encoding +Wiffs = Wiffage(txt='txt', bny='bny') -@contextmanager -def openMG(cls=None, name="test", **kwa): - """ - Wrapper to create and open MemoGram instances - When used in with statement block, calls .close() on exit of with block - - Parameters: - cls (Class): instance of subclass instance - name (str): unique identifer of MemoGram peer. - Enables management of transport by name. - Usage: - with openMemoGram() as peer: - peer.receive() +# namedtuple for size entries in MemoGram code tables +# cs is the code size int number of chars in code portion +# ms is the mid size int number of chars in the mid portion (memoID) +# ns is the neck size int number of chars for the part number in all parts and +# the additional neck that also appears in first part +# hs is the head size int number of chars hs = cs + ms + ns and for the first +# part the head of size hs is followed by a neck of size ns. So the total +# overhead on first part is os = hs + ns - with openMemoGram(cls=MemoGramSub) as peer: - peer.receive() +Sizage = namedtuple("Sizage", "cs ms ns hs") +@dataclass(frozen=True) +class PartCodex: """ - peer = None + PartCodex is codex of all Part Head Codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + Basic: str = '__' # Basic Part Head Code - if cls is None: - cls = MemoGram - try: - peer = cls(name=name, **kwa) - peer.reopen() + def __iter__(self): + return iter(astuple(self)) - yield peer - finally: - if peer: - peer.close() +PartDex = PartCodex() # Make instance class MemoGram(hioing.Mixin): @@ -408,6 +465,13 @@ class MemoGram(hioing.Mixin): Class Attributes: Version (Versionage): default version consisting of namedtuple of form (major: int, minor: int) + BufCount (int): default bufcount bc for transport 64 + Code (str): default part type-version head code '__' + MaxMemoSize (int): absolute max memo size 4294836225 + MaxPartSize (int): absolute max part size 65535 + MaxPartCount (int): absolute max part count 16777215 + Sizes (dict): part sizes Sizage instances keyed by part codes + Mode (str): default encoding mode for tx part header b64 'txt' Wiffs.txt Inherited Attributes: name (str): unique name for MemoGram transport. Used to manage. @@ -420,7 +484,16 @@ class MemoGram(hioing.Mixin): namedtuple of form (major: int, minor: int) rxgs (dict): holding rx (receive) (data) gram deques of grams. Each item in dict has key=src and val=deque of grames received - from transport. Each item of form (src: str, gram: deque) + rxgs (dict): keyed by mid (memoID) with value of dict where each deque + holds incomplete memo part grams for that mid. + The mid appears in every gram part from the same memo. + The value dict is keyed by the part number pn, with value + that is the part bytes. + sources (dict): keyed by mid (memoID) that holds the src for the memo. + This enables reattaching src to fused memo in rxms deque tuple. + counts (dict): keyed by mid (memoID) that holds the part count (pc) from + the first part for the memo. This enables lookup of the pc when + fusing its parts. rxms (deque): holding rx (receive) memo duples desegmented from rxgs grams each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into @@ -433,42 +506,28 @@ class MemoGram(hioing.Mixin): portion when Encodesdatagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - - - rxgs dict keyed by mid (memo id) that holds incomplete memo parts. - The mid appears in every gram part from the same memo. - each val is dict of received gram parts keyed by part number. - - srcs is dict keyed by mid that holds the src for that mid. This allows - reattaching src to memo when placing in rxms deque - - - mids (dict): of dicts keyed by src and each value dict keyed by - echos (deque): holding echo receive duples for testing. Each duple of + form: (gram: bytes, dst: str). + code (bytes | None): part head code for part header + size (int): part size for this instance for transport. Part size + = head size + body size. Part size limited by + MaxPartSize and head size + 1 + mode (str): encoding mode for part header when transmitting, + b64 'txt' or b2 'bny' - pc (bytes | None): part code for part header - form: (gram: bytes, dst: str). - ps (int): part size for this instance for transport. Part size - = part head size + part body size. Part size limited by - MaxPartSize and PartHeadSize + 1 - mms (int): max memo size relative to part size. Limited by MaxMemoSize - and MaxPartCount for the part size. Properties: - - - """ Version = Versionage(major=0, minor=0) # default version BufCount = 64 # default bufcount bc for transport - PartCode = b'__' # default part type-version code for head + Code = PartDex.Basic # default part type-version head code '__' MaxMemoSize = 4294836225 # (2**16-1)**2 absolute max memo size MaxPartSize = 65535 # (2**16-1) absolute max part size MaxPartCount = 16777215 # (2**24-1) absolute max part count - PartHeadSizes = {b'__': 32} # dict of part head sizes keyed by part codes + Sizes = {'__': Sizage(cs=2, ms=22, ns=4, hs=28)} # dict of part sizes keyed by part codes + Mode = Wiffs.txt # encoding mode for tx part header b64 'txt' or b2 'bny' def __init__(self, @@ -476,12 +535,15 @@ def __init__(self, bc=None, version=None, rxgs=None, + sources=None, + counts=None, rxms=None, txms=None, txgs=None, txbs=None, - pc=None, - ps=None, + code=None, + size=None, + mode=None, **kwa ): """Setup instance @@ -493,9 +555,17 @@ def __init__(self, Parameters: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - rxgs (dict): holding rx (receive) (data) gram deques of grams. - Each item in dict has key=src and val=deque of grams received - from transport. Each item of form (src: str, gram: deque) + rxgs (dict): keyed by mid (memoID) with value of dict where each deque + holds incomplete memo part grams for that mid. + The mid appears in every gram part from the same memo. + The value dict is keyed by the part number pn, with value + that is the part bytes. + sources (dict): keyed by mid that holds the src for the memo indexed by its + mid (memoID). This enables reattaching src to memo when + placing fused memo in rxms deque. + counts (dict): keyed by mid that holds the part count (pc) for the + memo indexed by its mid (memoID). This enables lookup of the pc + to memo when fusing its parts. rxms (deque): holding rx (receive) memo duples fused from rxgs grams each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be partitioned into @@ -508,15 +578,14 @@ def __init__(self, portion when datagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - pc (bytes | None): part code for part header - ps (int): part size for this instance for transport. Part size - = part head size + part body size. Part size limited by - MaxPartSize and MaxPartCount relative to MaxMemoSize as well - as minimum part body size of 1 - pbs (int): part body size = part size - part head size for given - paEncodesrt code. Part body size must be at least 1. - mms (int): max memo size relative to part size and limited by - MaxMemoSize and MaxPartCount for given part size. + code (bytes | None): part code for part header + size (int): part size for this instance for transport. Part size + = head size + body size. Part size limited by + MaxPartSize and MaxPartCount relative to MaxMemoSize as well + as minimum part body size of 1 + mode (str): encoding mode for part header when transmitting, + b64 'txt' or b2 'bny' + @@ -524,6 +593,8 @@ def __init__(self, # initialize attributes self.version = version if version is not None else self.Version self.rxgs = rxgs if rxgs is not None else dict() + self.sources = sources if sources is not None else dict() + self.counts = counts if counts is not None else dict() self.rxms = rxms if rxms is not None else deque() self.txms = txms if txms is not None else deque() @@ -534,12 +605,11 @@ def __init__(self, self.echos = deque() # only used in testing as echoed tx - self.pc = pc if pc is not None else self.PartCode - phs = self.PartHeadSizes[self.pc] # part head size - ps = ps if ps is not None else self.MaxPartSize - self.ps = max(min(ps, self.MaxPartSize), phs + 1) - self.pbs = (self.ps - phs) # part body size - self.mms = min(self.MaxMemoSize, self.pbs * self.MaxPartCount) + self.code = code if code is not None else self.Code + _, _, ns, hs = self.Sizes[self.code] + size = size if size is not None else self.MaxPartSize + self.size = max(min(size, self.MaxPartSize), hs + ns + 1) + self.mode = mode if mode is not None else self.Mode super(MemoGram, self).__init__(name=name, bc=bc, **kwa) @@ -575,28 +645,119 @@ def close(self): def wiff(self, part): - """ - Returns encoding format of part bytes header as either base64 text 'txt' - or base2 binary 'bny' from first 6 bits (sextet) of first byte in part. + """Determines encoding format of part bytes header as either base64 text + 'txt' or base2 binary 'bny' from first 6 bits (sextet) in part. + + Returns: + wiff (Wiffs): Wiffs.txt if B64 text, Wiffs.bny if B2 binary + Otherwise raises hioing.MemoGramError + All part head codes start with '_' in base64 text or in base2 binary. Given only allowed chars are from the set of base64 then can determine if header is in base64 or base2. - First sextet: - - 0o2 = 010 means first sextet of '_' in base64 =>'txt' - 0o7 = 111 means frist sextet of '_' in base2 => 'bny' + First sextet: (0b is binary, 0o is octal, 0x is hexadecimal) + + 0o27 = 0b010111 means first sextet of '_' in base64 => 'txt' + 0o77 = 0b111111 means first sextet of '_' in base2 => 'bny' + + Assuming that only '_' is allowed as the first char of a valid part head, + in either Base64 or Base2 encoding, a parser can unambiguously determine + if the part header encoding is binary Base2 or text Base64 because + 0o27 != 0o77. + + Furthermore, it just so happens that 0o27 is a unique first sextet + among Base64 ascii chars. So an invalid part header when in Base64 encoding + is detectable. However, there is one collision (see below) with invalid + part headers in Base2 encoding. So an erroneously constructed part might + not be detected merely by looking at the first sextet. Therefore + unreliable transports must provide some additional mechanism such as a + hash or signature on the part. + + All Base2 codes for Base64 chars are unique since the B2 codes are just + the count from 0 to 63. There is one collision, however, between Base2 and + Base 64 for invalid part headers. This is because 0o27 is the + base2 code for ascii 'V'. This means that Base2 encodings + that begin 0o27 witch is the Base2 encoding of the ascii 'V, would be + incorrectly detected as text not binary. """ sextet = part[0] >> 2 - if sextet == 0o123: - return Encodes.txt # 'txt' - if sextet == 0o321: - return Encodes.bny # 'bny' + if sextet == 0o27: + return Wiffs.txt # 'txt' + if sextet == 0o77: + return Wiffs.bny # 'bny' raise hioing.MemoGramError(f"Unexpected {sextet=} at part head start.") + def parse(self, part): + """Parse and strips header from gram part bytearray and returns + (mid, pn, pc) when first part, (mid, pn, None) when other part + Otherwise raises MemoGramError is unrecognized header + + Returns: + result (tuple): tuple of form: + (mid: str, pn: int, pc: int | None) where: + mid is memoID, pn is part number, pc is part count. + When first first part (zeroth) returns (mid, 0, pc). + When other part returns (mid, pn, None) + Otherwise raises memogram error. + + When valid recognized header, strips header bytes from front of part + leaving the part body part bytearray. + + Parameters: + part (bytearray): memo part gram from which to parse and strip its header. + + """ + cs, ms, ns, hs = self.Sizes[self.code] + + wiff = self.wiff(part) + if wiff == Wiffs.bny: + bhs = 3 * (hs) // 4 # binary head size + head = part[:bhs] # slice takes a copy + if len(head) < bhs: # not enough bytes for head + raise hioing.MemoGramError(f"Part size b2 too small < {bhs}" + f" for head.") + head = encodeB64(head) # convert to Base64 + del part[:bhs] # strip head off part + else: # wiff == Wiffs.txt: + head = part[:hs] # slice takes a copy + if len(head) < hs: # not enough bytes for head + raise hioing.MemoGramError(f"Part size b64 too small < {hs} for" + f"head.") + del part[:hs] # strip off head off + + code = head[:cs].decode() + if code != self.code: + raise hioing.MemoGramError(f"Unrecognized part {code=}.") + + mid = head[cs:cs+ms].decode() + pn = helping.b64ToInt(head[cs+ms:cs+ms+ns]) + + pc = None + if pn == 0: # first (zeroth) part so get neck + if wiff == Wiffs.bny: + bns = 3 * (ns) // 4 # binary neck size + neck = part[:bns] # slice takes a copy + if len(neck) < bns: # not enough bytes for neck + raise hioing.MemoGramError(f"Part size b2 too small < {bns}" + f" for neck.") + neck = encodeB64(neck) # convert to Base64 + del part[:bns] # strip off neck + else: # wiff == Wiffs.txt: + neck = part[:ns] # slice takes a copy + if len(neck) < ns: # not enough bytes for neck + raise hioing.MemoGramError(f"Part size b64 too small < {ns}" + f" for neck.") + del part[:ns] # strip off neck + + pc = helping.b64ToInt(neck) + + return (mid, pn, pc) + + def receive(self, *, echoic=False): """Attemps to send bytes in txbs to remote destination dst. Must be overridden in subclass. This is a stub to define mixin interface. @@ -664,12 +825,32 @@ def _serviceOneReceived(self, *, echoic=False): if not gram: # no received data return False - if src not in self.rxgs: - self.rxgs[src] = deque() + part = bytearray(gram)# make copy bytearray so can strip off header + + try: + mid, pn, pc = self.parse(part) # parse and strip off head leaving body + except hioing.MemoGramError as ex: # invalid part so drop + logger.error("Unrecognized MemoGram part from %s.\n %s.", src, ex) + return True # did receive data to can keep receiving + + if mid not in self.rxgs: + self.rxgs[mid] = dict() + + # save stripped part gram to be fused later + if pn not in self.rxgs[mid]: # make idempotent first only no replay + self.rxgs[mid][pn] = part # index body by its part number + + if pc is not None: + if mid not in self.counts: # make idempotent first only no replay + self.counts[mid] = pc # save part count for mid - self.rxgs[src].append(gram) + # assumes unique mid across all possible sources. No replay by different + # source only first source for a given mid is ever recognized + if mid not in self.sources: # make idempotent first only no replay + self.sources[mid] = src # save source for later - return True # received data + + return True # received valid def serviceReceivesOnce(self, *, echoic=False): @@ -697,12 +878,11 @@ def serviceReceives(self, *, echoic=False): while self.opened: if not self._serviceOneReceived(echoic=echoic): break - echo = None # only echo once - def fuse(self, grams): - """Fuse gram parts from frams deque into whole memo. If grams is - missing any the parts for a whole memo then returns None. + def fuse(self, parts, cnt): + """Fuse cnt gram part bodies from parts dict into whole memo . If any + parts are missing then returns None. Returns: memo (str | None): fused memo or None if incomplete. @@ -710,55 +890,38 @@ def fuse(self, grams): Override in subclass Parameters: - grams (deque): gram segments from which to fuse memo parts. + parts (dict): memo part bodies each keyed by part number from which + to fuse memo. + cnt (int): part count for mid """ - try: - gram = grams.popleft() - memo = gram.decode() - except IndexError: + if len(parts) < cnt: # must be missing one or more parts return None - return memo + memo = bytearray() + for i in range(cnt): # iterate in numeric order, items are insertion ordered + memo.extend(parts[i]) # extend memo with part body at part i + return memo.decode() # convert bytearray to str - def _serviceOnceRxGrams(self): - """Service one pass over .rxgs dict for each unique src in .rxgs - - Returns: - result (bool): True means at least one src has received a memo and - has writing grams so can keep desegmenting. - False means all sources waiting for more grams - so try again later. - The return value True or False enables back pressure on greedy callers - so they know when to block waiting for at least one source with received - memo and additional grams to desegment. - Deleting an item from a dict at a key (since python dicts are key creation - ordered) means that the next time an item is created at that key, that - item will be last in order. In order to dynamically change the ordering - of iteration over sources, when there are no received grams from a - given source we remove its dict item. This reorders the source - as last when a new gram is received and avoids iterating over sources - with no received grams. + def _serviceOnceRxGrams(self): + """Service one pass over .rxgs dict for each unique mid in .rxgs + Deleting an item from a dict at a key (since python dicts are key + insertion ordered) means that the next time an item is created it will + be last. """ - goods = [False] * len(self.rxgs) # list by src, True memo False no-memo - # service grams to desegment - for i, src in enumerate(list(self.rxgs.keys())): # list since items may be deleted in loop - # if src then grams deque at src must not be empty - memo = self.fuse(self.rxgs[src]) + for i, mid in enumerate(list(self.rxgs.keys())): # items may be deleted in loop + # if mid then parts dict at mid must not be empty + if not mid in self.counts: # missing first part so skip + continue + memo = self.fuse(self.rxgs[mid], self.counts[mid]) if memo is not None: # allows for empty "" memo for some src - self.rxms.append((memo, src)) - - if not self.rxgs[src]: # deque at src empty so remove deque - del self.rxgs[src] # makes src unordered until next memo at src - - if memo is not None: # received and desegmented memo - if src in self.rxgs: # more received gram(s) at src - goods[i] = True # indicate memo recieved with more received gram(s) - - return any(goods) # at least one memo from a source with more received grams + self.rxms.append((memo, self.sources[mid])) + del self.rxgs[mid] + del self.counts[mid] + del self.sources[mid] def serviceRxGramsOnce(self): @@ -770,13 +933,12 @@ def serviceRxGramsOnce(self): def serviceRxGrams(self): - """Service multiple passes (greedy) over all unqique sources in - .rxgs dict if any for sources with complete desegmented memos and more - incoming grams. + """Service one pass (non-greedy) over all unique sources in .rxgs + dict if any for received incoming grams. No different from + serviceRxGramsOnce because service all mids each pass. """ - while self.rxgs: - if not self._serviceOnceRxGrams(): - break + if self.rxgs: + self._serviceOnceRxGrams() def _serviceOneRxMemo(self): @@ -855,18 +1017,54 @@ def memoit(self, memo, dst): self.txms.append((memo, dst)) - def part(self, memo): - """Partition memo into parts as grams. - This is a stub method meant to be overridden in subclass + def rend(self, memo): + """Partition memo into packed part grams with headers. Returns: - grams (list[bytes]): packed segments of memo, each seqment is bytes + parts (list[bytes]): list of part grams with headers. Parameters: - memo (str): to be segmented into grams + memo (str): to be partitioned into gram parts with headers + + Note first part has head + neck overhead, hs + ns so bs is smaller by ns + non-first parts have just head overhead hs so bs is bigger by ns """ - grams = [memo.encode()] - return grams + parts = [] + memo = bytearray(memo.encode()) # convert and copy to bytearray + + cs, ms, ns, hs = self.Sizes[self.code] + # self.size is max part size + bs = (self.size - hs) # max standard part body size, + mms = min(self.MaxMemoSize, (bs * self.MaxPartCount) - ns) # max memo payload + ml = len(memo) + if ml > mms: + raise hioing.MemoGramError(f"Memo length={ml} exceeds max={mms}.") + + # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned + midb = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip + #mid = midb.decode() + # compute part count account for added neck overhead in first part + pc = math.ceil((ml + ns - bs)/bs + 1) # includes added neck ns overhead + neck = helping.intToB64b(pc, l=ns) + if self.mode == Wiffs.bny: + neck = decodeB64(neck) + pn = 0 + while memo: + pnb = helping.intToB64b(pn, l=ns) # pn size always == neck size + head = self.code.encode() + midb + pnb + if self.mode == Wiffs.bny: + head = decodeB64(head) + if pn == 0: + part = head + neck + memo[:bs-ns] # copy slice past end just copies to end + del memo[:bs-ns] # del slice past end just deletes to end + else: + part = head + memo[:bs] # copy slice past end just copies to end + del memo[:bs] # del slice past end just deletes to end + parts.append(part) + + pn += 1 + + return parts def _serviceOneTxMemo(self): @@ -879,9 +1077,9 @@ def _serviceOneTxMemo(self): Appends duples of (gram, dst) from grams to .txgs deque. """ memo, dst = self.txms.popleft() # raises IndexError if empty deque - grams = self.part(memo) - for gram in grams: - self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) + + for part in self.rend(memo): # partition memo into gram parts with head + self.txgs.append((part, dst)) # append duples (gram: bytes, dst: str) def serviceTxMemosOnce(self): @@ -1057,6 +1255,38 @@ def serviceAll(self): service = serviceAll # alias override peer service method +@contextmanager +def openMG(cls=None, name="test", **kwa): + """ + Wrapper to create and open MemoGram instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of MemoGram peer. + Enables management of transport by name. + Usage: + with openMemoGram() as peer: + peer.receive() + + with openMemoGram(cls=MemoGramSub) as peer: + peer.receive() + + """ + peer = None + + if cls is None: + cls = MemoGram + try: + peer = cls(name=name, **kwa) + peer.reopen() + + yield peer + + finally: + if peer: + peer.close() + class MemoGramDoer(doing.Doer): """MemoGram Doer for reliable transports that do not require retry tymers. @@ -1092,37 +1322,6 @@ def exit(self): """""" self.peer.close() -@contextmanager -def openTMG(cls=None, name="test", **kwa): - """ - Wrapper to create and open TymeeMemoGram instances - When used in with statement block, calls .close() on exit of with block - - Parameters: - cls (Class): instance of subclass instance - name (str): unique identifer of MemoGram peer. - Enables management of transport by name. - Usage: - with openMemoGram() as peer: - peer.receive() - - with openMemoGram(cls=MemoGramSub) as peer: - peer.receive() - - """ - peer = None - - if cls is None: - cls = TymeeMemoGram - try: - peer = cls(name=name, **kwa) - peer.reopen() - - yield peer - - finally: - if peer: - peer.close() class TymeeMemoGram(tyming.Tymee, MemoGram): @@ -1201,6 +1400,39 @@ def serviceAll(self): self.serviceAllTx() +@contextmanager +def openTMG(cls=None, name="test", **kwa): + """ + Wrapper to create and open TymeeMemoGram instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of MemoGram peer. + Enables management of transport by name. + Usage: + with openMemoGram() as peer: + peer.receive() + + with openMemoGram(cls=MemoGramSub) as peer: + peer.receive() + + """ + peer = None + + if cls is None: + cls = TymeeMemoGram + try: + peer = cls(name=name, **kwa) + peer.reopen() + + yield peer + + finally: + if peer: + peer.close() + + class TymeeMemoGramDoer(doing.Doer): """TymeeMemoGram Doer for unreliable transports that require retry tymers. diff --git a/src/hio/help/helping.py b/src/hio/help/helping.py index 59cddca..62e01bc 100644 --- a/src/hio/help/helping.py +++ b/src/hio/help/helping.py @@ -442,9 +442,15 @@ def load(path): Reb64 = re.compile(B64REX) # compile is faster def intToB64(i, l=1): - """ - Returns conversion of int i to Base64 str + """Converts int i to at least l Base64 chars as str. + Returns: + b64 (str): Base64 converstion of i of length minimum l. If more than + l chars are needed to represent i in Base64 then returned + str is lengthed appropriately. When less then l chars is + needed then returned str is prepadded with 'A' character. + l is min number of b64 digits left padded with Base64 0 == "A" char + The length of return bytes extended to accomodate full Base64 encoding of i """ d = deque() # deque of characters base64 @@ -459,9 +465,16 @@ def intToB64(i, l=1): def intToB64b(i, l=1): - """ - Returns conversion of int i to Base64 bytes - l is min number of b64 digits left padded with Base64 0 == "A" char + """Converts int i to at least l Base64 chars as bytes. + + Returns: + b64 (bytes): Base64 converstion of i of length minimum l. If more than + l bytes are needed to represent i in Base64 then returned + bytes is lengthed appropriately. When less then l bytes + is needed then returned bytes is prepadded with b'A' bytes. + + l is min number of b64 digits left padded with Base64 0 == b"A" bytes + The length of return bytes extended to accomodate full Base64 encoding of i """ return (intToB64(i=i, l=l).encode("utf-8")) diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 4501946..bc7aeb2 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -3,7 +3,8 @@ tests.core.test_memoing module """ - +from base64 import urlsafe_b64encode as encodeB64 +from base64 import urlsafe_b64decode as decodeB64 import pytest @@ -17,10 +18,10 @@ def test_memogram_basic(): peer = memoing.MemoGram() assert peer.name == "main" assert peer.opened == False - assert peer.pc == b'__' == peer.PartCode - assert peer.ps == peer.MaxPartSize - assert peer.pbs == peer.ps - peer.PartHeadSizes[peer.pc] - assert peer.mms == peer.MaxMemoSize + assert peer.code == '__' == peer.Code + assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + assert peer.size == peer.MaxPartSize + peer.reopen() assert peer.opened == True @@ -37,30 +38,45 @@ def test_memogram_basic(): assert peer.txms[0] == ('Hello There', 'beta') peer.serviceTxMemos() assert not peer.txms - assert peer.txgs[0] == (b'Hello There', 'beta') + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.txt + assert m.endswith(memo.encode()) + assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) assert not peer.rxgs + assert not peer.counts + assert not peer.sources assert not peer.rxms - echo = (b"Bye yall", "beta") + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + gram = ('__' + mid + 'AAAA' + 'AAAB' + "Hello There").encode() + echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) - assert peer.rxgs["beta"][0] == b"Bye yall" + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' peer.serviceRxGrams() assert not peer.rxgs - assert peer.rxms[0] == ('Bye yall', 'beta') + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') peer.serviceRxMemos() assert not peer.rxms + # send and receive via echo memo = "See ya later!" dst = "beta" peer.memoit(memo, dst) assert peer.txms[0] == ('See ya later!', 'beta') peer.serviceTxMemos() assert not peer.txms - assert peer.txgs[0] == (b'See ya later!', 'beta') + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.txt + assert m.endswith(memo.encode()) + assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) assert not peer.txgs assert peer.txbs == (b'', None) @@ -68,22 +84,299 @@ def test_memogram_basic(): assert not peer.rxgs assert not peer.rxms + assert not peer.counts + assert not peer.sources peer.serviceReceives(echoic=True) - assert peer.rxgs["beta"][0] == b"See ya later!" + mid = list(peer.rxgs.keys())[0] + assert peer.rxgs[mid][0] == b'See ya later!' + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' assert not peer.echos peer.serviceRxGrams() assert not peer.rxgs + assert not peer.counts + assert not peer.sources + peer.rxms[0] assert peer.rxms[0] == ('See ya later!', 'beta') peer.serviceRxMemos() assert not peer.rxms + # test binary q2 encoding of transmission part header + peer.mode = memoing.Wiffs.bny + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.bny + assert m.endswith(memo.encode()) + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) + gram = headneck + b"Hello There" + assert peer.wiff(gram) == memoing.Wiffs.bny + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms peer.close() assert peer.opened == False """ End Test """ +def test_memogram_small_part_size(): + """Test MemoGram class with small part size + """ + peer = memoing.MemoGram(size=6) + assert peer.name == "main" + assert peer.opened == False + assert peer.code == '__' == peer.Code + assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + assert peer.size == 33 # can't be smaller than head + neck + 1 + + peer = memoing.MemoGram(size=38) + assert peer.size == 38 + peer.reopen() + assert peer.opened == True + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + peer.service() + assert peer.txbs == (b'', None) + + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 2 + for m, d in peer.txgs: + assert peer.wiff(m) == memoing.Wiffs.txt + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + b'__DFymLrtlZG6bp0HhlUsR6uAAAAAAACHello ' + b'__DFymLrtlZG6bp0HhlUsR6uAAABThere' + mid = 'DFymLrtlZG6bp0HhlUsR6u' + gram = ('__' + mid + 'AAAA' + 'AAAC' + "Hello ").encode() + echo = (gram, "beta") + peer.echos.append(echo) + gram = ('__' + mid + 'AAAB' + "There").encode() + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert len(peer.rxgs[mid]) == 2 + assert peer.counts[mid] == 2 + assert peer.sources[mid] == 'beta' + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'There') + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # send and receive via echo + memo = "See ya later!" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('See ya later!', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 2 + for m, d in peer.txgs: + assert peer.wiff(m) == memoing.Wiffs.txt + assert d == dst == 'beta' + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert peer.echos + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + peer.serviceReceives(echoic=True) + mid = list(peer.rxgs.keys())[0] + assert len(peer.rxgs[mid]) == 2 + assert peer.counts[mid] == 2 + assert peer.sources[mid] == 'beta' + assert peer.rxgs[mid][0] == bytearray(b'See ya') + assert peer.rxgs[mid][1] == bytearray(b' later!') + assert not peer.echos + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + peer.rxms[0] + assert peer.rxms[0] == ('See ya later!', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # test binary q2 encoding of transmission part header + peer.mode = memoing.Wiffs.bny + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 2 + for m, d in peer.txgs: + assert peer.wiff(m) == memoing.Wiffs.bny + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + + headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) + gram = headneck + b"Hello There" + assert peer.wiff(gram) == memoing.Wiffs.bny + echo = (gram, "beta") + peer.echos.append(echo) + + b'__DFymLrtlZG6bp0HhlUsR6uAAAAAAACHello ' + b'__DFymLrtlZG6bp0HhlUsR6uAAABThere' + mid = 'DFymLrtlZG6bp0HhlUsR6u' + headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAC').encode()) + gram = headneck + b"Hello " + assert peer.wiff(gram) == memoing.Wiffs.bny + echo = (gram, "beta") + peer.echos.append(echo) + head = decodeB64(('__' + mid + 'AAAB').encode()) + gram = head + b"There" + assert peer.wiff(gram) == memoing.Wiffs.bny + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert len(peer.rxgs[mid]) == 2 + assert peer.counts[mid] == 2 + assert peer.sources[mid] == 'beta' + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'There') + peer.serviceReceives(echoic=True) + assert len(peer.rxgs[mid]) == 2 + assert peer.counts[mid] == 2 + assert peer.sources[mid] == 'beta' + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'There') + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + peer.close() + assert peer.opened == False + """ End Test """ + + +def test_memogram_multiples(): + """Test MemoGram class with small part size and multiple queued memos + """ + peer = memoing.MemoGram(size=38) + assert peer.size == 38 + assert peer.name == "main" + assert peer.opened == False + assert peer.code == '__' == peer.Code + assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 4 + for m, d in peer.txgs: + assert peer.wiff(m) == memoing.Wiffs.txt + assert d in ("alpha", "beta") + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceReceives(echoic=True) + assert not peer.echos + assert len(peer.rxgs) == 2 + assert len(peer.counts) == 2 + assert len(peer.sources) == 2 + + mid = list(peer.rxgs.keys())[0] + assert peer.sources[mid] == 'alpha' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'there.') + + mid = list(peer.rxgs.keys())[1] + assert peer.sources[mid] == 'beta' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'How ya') + assert peer.rxgs[mid][1] == bytearray(b' doing?') + + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert len(peer.rxms) == 2 + assert peer.rxms[0] == ('Hello there.', 'alpha') + assert peer.rxms[1] == ('How ya doing?', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + + peer.close() + assert peer.opened == False + """ End Test """ + + + + def test_open_mg(): """Test contextmanager decorator openMG """ @@ -132,10 +425,9 @@ def test_tymeememogram_basic(): assert peer.tymeout == 0.0 assert peer.name == "main" assert peer.opened == False - assert peer.pc == b'__' == peer.PartCode - assert peer.ps == peer.MaxPartSize - assert peer.pbs == peer.ps - peer.PartHeadSizes[peer.pc] - assert peer.mms == peer.MaxMemoSize + assert peer.code == '__' == peer.Code + assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + assert peer.size == peer.MaxPartSize peer.reopen() assert peer.opened == True @@ -152,30 +444,45 @@ def test_tymeememogram_basic(): assert peer.txms[0] == ('Hello There', 'beta') peer.serviceTxMemos() assert not peer.txms - assert peer.txgs[0] == (b'Hello There', 'beta') + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.txt + assert m.endswith(memo.encode()) + assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs assert peer.txbs == (b'', None) assert not peer.rxgs + assert not peer.counts + assert not peer.sources assert not peer.rxms - echo = (b"Bye yall", "beta") + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + gram = ('__' + mid + 'AAAA' + 'AAAB' + "Hello There").encode() + echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) - assert peer.rxgs["beta"][0] == b"Bye yall" + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' peer.serviceRxGrams() assert not peer.rxgs - assert peer.rxms[0] == ('Bye yall', 'beta') + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') peer.serviceRxMemos() assert not peer.rxms + # send and receive via echo memo = "See ya later!" dst = "beta" peer.memoit(memo, dst) assert peer.txms[0] == ('See ya later!', 'beta') peer.serviceTxMemos() assert not peer.txms - assert peer.txgs[0] == (b'See ya later!', 'beta') + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.txt + assert m.endswith(memo.encode()) + assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) assert not peer.txgs assert peer.txbs == (b'', None) @@ -183,23 +490,68 @@ def test_tymeememogram_basic(): assert not peer.rxgs assert not peer.rxms + assert not peer.counts + assert not peer.sources peer.serviceReceives(echoic=True) - assert peer.rxgs["beta"][0] == b"See ya later!" + mid = list(peer.rxgs.keys())[0] + assert peer.rxgs[mid][0] == b'See ya later!' + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' assert not peer.echos peer.serviceRxGrams() assert not peer.rxgs + assert not peer.counts + assert not peer.sources + peer.rxms[0] assert peer.rxms[0] == ('See ya later!', 'beta') peer.serviceRxMemos() assert not peer.rxms + # test binary q2 encoding of transmission part header + peer.mode = memoing.Wiffs.bny + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + m, d = peer.txgs[0] + assert peer.wiff(m) == memoing.Wiffs.bny + assert m.endswith(memo.encode()) + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) + gram = headneck + b"Hello There" + assert peer.wiff(gram) == memoing.Wiffs.bny + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # Test wind tymist = tyming.Tymist(tock=1.0) peer.wind(tymth=tymist.tymen()) assert peer.tyme == tymist.tyme == 0.0 tymist.tick() assert peer.tyme == tymist.tyme == 1.0 - - peer.close() assert peer.opened == False """ End Test """ @@ -259,6 +611,8 @@ def test_tymeememogram_doer(): if __name__ == "__main__": test_memogram_basic() + test_memogram_small_part_size() + test_memogram_multiples() test_open_mg() test_memogram_doer() test_tymeememogram_basic() From 3f01b3cbf6ab2715a5c536c0ae2dc569a33443d4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sun, 23 Feb 2025 15:02:51 -0700 Subject: [PATCH 23/33] refactor MemoGram to Memoer to better match convention --- src/hio/core/memoing.py | 129 +++++++++++++++++++++---------------- src/hio/hioing.py | 2 +- tests/core/test_memoing.py | 34 +++++----- 3 files changed, 92 insertions(+), 73 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index e48da4c..a183e2f 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -2,7 +2,10 @@ """ hio.core.memoing Module -Mixin Base Classes that add support for MemoGrams to datagram based transports. +Mixin Base Classes that add support for memograms over datagram based transports. +In this context, a memogram is a larger memogram that is partitioned into +smaller parts as transport specific datagrams. This allows larger messages to +be transported over datagram transports than the underlying transport can support. Provide mixin classes. Takes of advantage of multiple inheritance to enable mixtures of behaviors @@ -260,14 +263,20 @@ as CESR primitives for management but not over the wire. I.e. can transform Memogram Parts to CESR primitives to tunnel in some CESR stream. -Normally CESR streams are tunneled inside Memograms sent over-the-wire. When -the MemoGram is the wrapper then use the part tv code not the equivalent CESR code. -By using so the far reserved and unused '_' CESR op code table selector char -from CESR it makes it obvious that its a Memogram -tv Part Code not a CESR code when debugging. Even when the op codes are defined -its not likely to be confusing since the context of CESR op codes is different. -Morover when the MemoGram is a CESR Stream, a memogram parser will parse and -strip the MemoGram wrappers to construct the MemoGram, so no collisions. +Normally CESR streams are tunneled inside Memoer memograms sent over-the-wire. +In this case, the memogram is the wrapper and each part uses the part tv code +defined herein not the equivalent CESR code. +Using so far reserved and unused '_' code which is reserved as the selector +for theCESR op code table for memogram parts makes it more apparent that a part +belongs to a memogram and not a CESR stream. +Eventually when CESR op codes eventually become defined, it's not likely to be +confusing since the context of CESR op codes is different that transport +wrappers. + +Morover when a MemoGram payload is a tunneled CESR Stream, the memogram parser +will parse and strip the MemoGram part wrappers to construct the tunneled, +stream so the wrapper is transarent to the CESR stream and the CESR stream +payload is opaque to the memogram wrapper, i.e. no ambiguity. @@ -356,6 +365,16 @@ It is unlikely we would ever reach those limits in a practical application. +ToDo: +change maxmemosize to accomodate big group + code +Change mode to boolean b2mode True False no more Wiffs +Add code '_-' for signed parts vs verification id size ss signatures size +expanes Sizes for vs and ss +Signed first part code + mid + vid + pn + pc + body + sig +Signed other parts code + mid + vid + pn + body + sig +Add sign and verify methods +Change name of MemoGram to Memoer to match convention + """ import socket @@ -383,7 +402,7 @@ Wiffs = Wiffage(txt='txt', bny='bny') -# namedtuple for size entries in MemoGram code tables +# namedtuple for size entries in Memoer code tables # cs is the code size int number of chars in code portion # ms is the mid size int number of chars in the mid portion (memoID) # ns is the neck size int number of chars for the part number in all parts and @@ -410,10 +429,10 @@ def __iter__(self): PartDex = PartCodex() # Make instance -class MemoGram(hioing.Mixin): +class Memoer(hioing.Mixin): """ - MemoGram mixin base class to adds memogram support to a transport class. - MemoGram supports asynchronous memograms. Provides common methods for subclasses. + Memoer mixin base class to adds memogram support to a transport class. + Memoer supports asynchronous memograms. Provides common methods for subclasses. A memogram is a higher level construct that sits on top of a datagram. A memogram supports the segmentation and desegmentation of memos to @@ -428,7 +447,7 @@ class MemoGram(hioing.Mixin): Usage: Do not instantiate directly but use as a mixin with a transport class in order to create a new subclass that adds memogram support to the - transport class. For example MemoGramUdp or MemoGramUxd + transport class. For example MemoerUdp or MemoerUxd Each direction of dataflow uses a tiered set of buffers that respect the constraints of non-blocking asynchronous IO with datagram transports. @@ -474,7 +493,7 @@ class MemoGram(hioing.Mixin): Mode (str): default encoding mode for tx part header b64 'txt' Wiffs.txt Inherited Attributes: - name (str): unique name for MemoGram transport. Used to manage. + name (str): unique name for Memoer transport. Used to manage. opened (bool): True means transport open for use False otherwise bc (int | None): count of transport buffers of MaxGramSize @@ -549,7 +568,7 @@ def __init__(self, """Setup instance Inherited Parameters: - name (str): unique name for MemoGram transport. Used to manage. + name (str): unique name for Memoer transport. Used to manage. bc (int | None): count of transport buffers of MaxGramSize Parameters: @@ -611,7 +630,7 @@ def __init__(self, self.size = max(min(size, self.MaxPartSize), hs + ns + 1) self.mode = mode if mode is not None else self.Mode - super(MemoGram, self).__init__(name=name, bc=bc, **kwa) + super(Memoer, self).__init__(name=name, bc=bc, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. self.name = name # mixed with subclass should provide this. @@ -650,7 +669,7 @@ def wiff(self, part): Returns: wiff (Wiffs): Wiffs.txt if B64 text, Wiffs.bny if B2 binary - Otherwise raises hioing.MemoGramError + Otherwise raises hioing.MemoerError All part head codes start with '_' in base64 text or in base2 binary. Given only allowed chars are from the set of base64 then can determine @@ -688,13 +707,13 @@ def wiff(self, part): if sextet == 0o77: return Wiffs.bny # 'bny' - raise hioing.MemoGramError(f"Unexpected {sextet=} at part head start.") + raise hioing.Memoer(f"Unexpected {sextet=} at part head start.") def parse(self, part): """Parse and strips header from gram part bytearray and returns (mid, pn, pc) when first part, (mid, pn, None) when other part - Otherwise raises MemoGramError is unrecognized header + Otherwise raises MemoerError is unrecognized header Returns: result (tuple): tuple of form: @@ -718,20 +737,20 @@ def parse(self, part): bhs = 3 * (hs) // 4 # binary head size head = part[:bhs] # slice takes a copy if len(head) < bhs: # not enough bytes for head - raise hioing.MemoGramError(f"Part size b2 too small < {bhs}" + raise hioing.Memoer(f"Part size b2 too small < {bhs}" f" for head.") head = encodeB64(head) # convert to Base64 del part[:bhs] # strip head off part else: # wiff == Wiffs.txt: head = part[:hs] # slice takes a copy if len(head) < hs: # not enough bytes for head - raise hioing.MemoGramError(f"Part size b64 too small < {hs} for" + raise hioing.Memoer(f"Part size b64 too small < {hs} for" f"head.") del part[:hs] # strip off head off code = head[:cs].decode() if code != self.code: - raise hioing.MemoGramError(f"Unrecognized part {code=}.") + raise hioing.Memoer(f"Unrecognized part {code=}.") mid = head[cs:cs+ms].decode() pn = helping.b64ToInt(head[cs+ms:cs+ms+ns]) @@ -742,14 +761,14 @@ def parse(self, part): bns = 3 * (ns) // 4 # binary neck size neck = part[:bns] # slice takes a copy if len(neck) < bns: # not enough bytes for neck - raise hioing.MemoGramError(f"Part size b2 too small < {bns}" + raise hioing.Memoer(f"Part size b2 too small < {bns}" f" for neck.") neck = encodeB64(neck) # convert to Base64 del part[:bns] # strip off neck else: # wiff == Wiffs.txt: neck = part[:ns] # slice takes a copy if len(neck) < ns: # not enough bytes for neck - raise hioing.MemoGramError(f"Part size b64 too small < {ns}" + raise hioing.Memoer(f"Part size b64 too small < {ns}" f" for neck.") del part[:ns] # strip off neck @@ -829,8 +848,8 @@ def _serviceOneReceived(self, *, echoic=False): try: mid, pn, pc = self.parse(part) # parse and strip off head leaving body - except hioing.MemoGramError as ex: # invalid part so drop - logger.error("Unrecognized MemoGram part from %s.\n %s.", src, ex) + except hioing.Memoer as ex: # invalid part so drop + logger.error("Unrecognized Memoer part from %s.\n %s.", src, ex) return True # did receive data to can keep receiving if mid not in self.rxgs: @@ -1038,7 +1057,7 @@ def rend(self, memo): mms = min(self.MaxMemoSize, (bs * self.MaxPartCount) - ns) # max memo payload ml = len(memo) if ml > mms: - raise hioing.MemoGramError(f"Memo length={ml} exceeds max={mms}.") + raise hioing.Memoer(f"Memo length={ml} exceeds max={mms}.") # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned midb = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip @@ -1256,27 +1275,27 @@ def serviceAll(self): @contextmanager -def openMG(cls=None, name="test", **kwa): +def openMemoer(cls=None, name="test", **kwa): """ - Wrapper to create and open MemoGram instances + Wrapper to create and open Memoer instances When used in with statement block, calls .close() on exit of with block Parameters: cls (Class): instance of subclass instance - name (str): unique identifer of MemoGram peer. + name (str): unique identifer of Memoer peer. Enables management of transport by name. Usage: - with openMemoGram() as peer: + with openMemoer() as peer: peer.receive() - with openMemoGram(cls=MemoGramSub) as peer: + with openMemoer(cls=MemoerSub) as peer: peer.receive() """ peer = None if cls is None: - cls = MemoGram + cls = Memoer try: peer = cls(name=name, **kwa) peer.reopen() @@ -1288,13 +1307,13 @@ def openMG(cls=None, name="test", **kwa): peer.close() -class MemoGramDoer(doing.Doer): - """MemoGram Doer for reliable transports that do not require retry tymers. +class MemoerDoer(doing.Doer): + """Memoer Doer for reliable transports that do not require retry tymers. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (MemoGram): underlying transport instance subclass of MemoGram + .peer (Memoer): underlying transport instance subclass of Memoer """ @@ -1302,9 +1321,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (Peer): is MemoGram Subclass instance + peer (Peer): is Memoer Subclass instance """ - super(MemoGramDoer, self).__init__(**kwa) + super(MemoerDoer, self).__init__(**kwa) self.peer = peer @@ -1324,8 +1343,8 @@ def exit(self): -class TymeeMemoGram(tyming.Tymee, MemoGram): - """TymeeMemoGram mixin base class to add tymer support for unreliable transports +class TymeeMemoer(tyming.Tymee, Memoer): + """TymeeMemoer mixin base class to add tymer support for unreliable transports that need retry tymers. Subclass of tyming.Tymee @@ -1356,7 +1375,7 @@ def __init__(self, *, tymeout=None, **kwa): tymeout (float): default for retry tymer if any """ - super(TymeeMemoGram, self).__init__(**kwa) + super(TymeeMemoer, self).__init__(**kwa) self.tymeout = tymeout if tymeout is not None else self.Tymeout #self.tymer = tyming.Tymer(tymth=self.tymth, duration=self.tymeout) # retry tymer @@ -1366,7 +1385,7 @@ def wind(self, tymth): Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoGram, self).wind(tymth) + super(TymeeMemoer, self).wind(tymth) #self.tymer.wind(tymth) def serviceTymers(self): @@ -1401,27 +1420,27 @@ def serviceAll(self): @contextmanager -def openTMG(cls=None, name="test", **kwa): +def openTM(cls=None, name="test", **kwa): """ - Wrapper to create and open TymeeMemoGram instances + Wrapper to create and open TymeeMemoer instances When used in with statement block, calls .close() on exit of with block Parameters: cls (Class): instance of subclass instance - name (str): unique identifer of MemoGram peer. + name (str): unique identifer of Memoer peer. Enables management of transport by name. Usage: - with openMemoGram() as peer: + with openTM() as peer: peer.receive() - with openMemoGram(cls=MemoGramSub) as peer: + with openTM(cls=MemoerSub) as peer: peer.receive() """ peer = None if cls is None: - cls = TymeeMemoGram + cls = TymeeMemoer try: peer = cls(name=name, **kwa) peer.reopen() @@ -1433,13 +1452,13 @@ def openTMG(cls=None, name="test", **kwa): peer.close() -class TymeeMemoGramDoer(doing.Doer): - """TymeeMemoGram Doer for unreliable transports that require retry tymers. +class TymeeMemoerDoer(doing.Doer): + """TymeeMemoer Doer for unreliable transports that require retry tymers. See Doer for inherited attributes, properties, and methods. Attributes: - .peer (TymeeMemoGram) is underlying transport instance subclass of TymeeMemoGram + .peer (TymeeMemoer) is underlying transport instance subclass of TymeeMemoer """ @@ -1447,9 +1466,9 @@ def __init__(self, peer, **kwa): """Initialize instance. Parameters: - peer (TymeeMemoGram): subclass instance + peer (TymeeMemoer): subclass instance """ - super(TymeeMemoGramDoer, self).__init__(**kwa) + super(TymeeMemoerDoer, self).__init__(**kwa) self.peer = peer if self.tymth: self.peer.wind(self.tymth) @@ -1459,7 +1478,7 @@ def wind(self, tymth): """Inject new tymist.tymth as new ._tymth. Changes tymist.tyme base. Updates winds .tymer .tymth """ - super(TymeeMemoGramDoer, self).wind(tymth) + super(TymeeMemoerDoer, self).wind(tymth) self.peer.wind(tymth) diff --git a/src/hio/hioing.py b/src/hio/hioing.py index 368842f..aa4610b 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -83,7 +83,7 @@ class NamerError(HioError): raise NamerError("error message") """ -class MemoGramError(HioError): +class Memoer(HioError): """ Error using or configuring Remoter diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index bc7aeb2..db8468b 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -12,10 +12,10 @@ from hio.core import memoing -def test_memogram_basic(): - """Test MemoGram class basic +def test_memoer_basic(): + """Test Memoer class basic """ - peer = memoing.MemoGram() + peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False assert peer.code == '__' == peer.Code @@ -147,14 +147,14 @@ def test_memogram_basic(): def test_memogram_small_part_size(): """Test MemoGram class with small part size """ - peer = memoing.MemoGram(size=6) + peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False assert peer.code == '__' == peer.Code assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size assert peer.size == 33 # can't be smaller than head + neck + 1 - peer = memoing.MemoGram(size=38) + peer = memoing.Memoer(size=38) assert peer.size == 38 peer.reopen() assert peer.opened == True @@ -308,10 +308,10 @@ def test_memogram_small_part_size(): """ End Test """ -def test_memogram_multiples(): - """Test MemoGram class with small part size and multiple queued memos +def test_memoer_multiple(): + """Test Memoer class with small part size and multiple queued memos """ - peer = memoing.MemoGram(size=38) + peer = memoing.Memoer(size=38) assert peer.size == 38 assert peer.name == "main" assert peer.opened == False @@ -380,7 +380,7 @@ def test_memogram_multiples(): def test_open_mg(): """Test contextmanager decorator openMG """ - with (memoing.openMG(name='zeta') as zeta): + with (memoing.openMemoer(name='zeta') as zeta): assert zeta.opened assert zeta.name == 'zeta' @@ -404,9 +404,9 @@ def test_memogram_doer(): assert doist.limit == limit == 0.125 assert doist.doers == [] - peer = memoing.MemoGram() + peer = memoing.Memoer() - mgdoer = memoing.MemoGramDoer(peer=peer) + mgdoer = memoing.MemoerDoer(peer=peer) assert mgdoer.peer == peer assert not mgdoer.peer.opened assert mgdoer.tock == 0.0 # ASAP @@ -421,7 +421,7 @@ def test_memogram_doer(): def test_tymeememogram_basic(): """Test TymeeMemoGram class basic """ - peer = memoing.TymeeMemoGram() + peer = memoing.TymeeMemoer() assert peer.tymeout == 0.0 assert peer.name == "main" assert peer.opened == False @@ -560,7 +560,7 @@ def test_tymeememogram_basic(): def test_open_tmg(): """Test contextmanager decorator openTMG """ - with (memoing.openTMG(name='zeta') as zeta): + with (memoing.openTM(name='zeta') as zeta): assert zeta.opened assert zeta.name == 'zeta' @@ -585,9 +585,9 @@ def test_tymeememogram_doer(): assert doist.limit == limit == 0.125 assert doist.doers == [] - peer = memoing.TymeeMemoGram() + peer = memoing.TymeeMemoer() - tmgdoer = memoing.TymeeMemoGramDoer(peer=peer) + tmgdoer = memoing.TymeeMemoerDoer(peer=peer) assert tmgdoer.peer == peer assert not tmgdoer.peer.opened assert tmgdoer.tock == 0.0 # ASAP @@ -610,9 +610,9 @@ def test_tymeememogram_doer(): if __name__ == "__main__": - test_memogram_basic() + test_memoer_basic() test_memogram_small_part_size() - test_memogram_multiples() + test_memoer_multiple() test_open_mg() test_memogram_doer() test_tymeememogram_basic() From 9b2f22b9dad0ceea04cf154e6622588f5cb66d9d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 24 Feb 2025 12:36:25 -0700 Subject: [PATCH 24/33] refactor terminology to be more precise gram not part and gram part is a subset. --- src/hio/core/memoing.py | 602 +++++++++++++++++++------------------ src/hio/hioing.py | 2 +- tests/core/test_memoing.py | 84 +++--- 3 files changed, 352 insertions(+), 336 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index a183e2f..9bca4c4 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -105,12 +105,12 @@ -on-the-size-of-single-packet-transmitted-over-domain Memo -Memo partitioning into parts. +Memo partitioning into grams. -Default part is of form part = head + sep + body. -Some subclasses might have part of form part = head + sep + body + tail. +Default gram is of form gram = head + sep + body. +Some subclasses might have gram of form gram = head + sep + body + tail. In that case encoding of body + tail must provide a way to separate body from tail. -Typically tail would be a signature on the fore part = head + sep + body +Typically tail would be a signature on the fore gram = head + sep + body Separator sep is b'|' must not be a base64 character. @@ -118,33 +118,33 @@ The head consists of three fields in base64 mid = memo IDmid = uuid.uuid4().bytes # 16 byte random uuid -pn = part number of part in memo zero based -pc = part count of total parts in memo may be zero when body is empty +gn = gram number of gram in memo zero based +gc = gram count of total grams in memo may be zero when body is empty Sep = b'|' assert not helping.Reb64.match(Sep) body = b"demogram" -pn = 0 -pc = 12 +gn = 0 +gc = 12 -pn = helping.intToB64b(pn, l=4) -pc = helping.intToB64b(pc, l=4) +gn = helping.intToB64b(gn, l=4) +gc = helping.intToB64b(gc, l=4) -PartLeader = struct.Struct('!16s4s4s') -PartLeader.size == 24 -head = PartLeader.pack(mid, pn, pc) -part = Sep.join(head, body) +GramLeader = struct.Struct('!16s4s4s') +GramLeader.size == 24 +head = GramLeader.pack(mid, gn, gc) +gram = Sep.join(head, body) -head, sep, body = part.partition(Sep) +head, sep, body = gram.partition(Sep) assert helping.Reb64.match(head) -mid, pn, pc = PartLeader.unpack(head) -pn = helping.b64ToInt(pn) -pc = helping.b64ToInt(pc) +mid, gn, gc = GramLeader.unpack(head) +gn = helping.b64ToInt(gn) +gc = helping.b64ToInt(gc) -# test PartHead -code = MtrDex.PartHead +# test GramHead +code = MtrDex.GramHead assert code == '0P' codeb = code.encode()mid = uuid.uuid4().bytes # 16 byte random uuid @@ -156,65 +156,65 @@ mid = 1 midb = mid.to_bytes(16) assert midb == b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' -pn = 1 -pnb = pn.to_bytes(3) -assert pnb == b'\x00\x00\x01' -pc = 2 -pcb = pc.to_bytes(3) -assert pcb == b'\x00\x00\x02' -raw = midb + pnb + pcb +gn = 1 +gnb = gn.to_bytes(3) +assert gnb == b'\x00\x00\x01' +gc = 2 +gcb = gc.to_bytes(3) +assert gcb == b'\x00\x00\x02' +raw = midb + gnb + gcb assert raw == (b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' b'\x00\x00\x01\x00\x00\x02')mid = uuid.uuid4().bytes # 16 byte random uuid assert mid == int.from_bytes(raw[:16]) -assert pn == int.from_bytes(raw[16:19]) -assert pc == int.from_bytes(raw[19:22]) +assert gn == int.from_bytes(raw[16:19]) +assert gc == int.from_bytes(raw[19:22]) midb64b = encodeB64(bytes([0] * 2) + midb)[2:] # prepad ans strip -pnb64b = encodeB64(pnb) -pcb64b = encodeB64(pcb) +gnb64b = encodeB64(gnb) +gcb64b = encodeB64(gcb) -qb64b = codeb + midb64b + pnb64b + pcb64b +qb64b = codeb + midb64b + gnb64b + gcb64b assert qb64b == b'0PAAAAAAAAAAAAAAAAAAAAABAAABAAAC' qb64 = qb64b.decode() qb2 = decodeB64(qb64b) assert mid == int.from_bytes(decodeB64(b'AA' + qb64b[2:24])) -assert pn == int.from_bytes(decodeB64(qb64b[24:28])) -assert pc == int.from_bytes(decodeB64(qb64b[28:32])) +assert gn == int.from_bytes(decodeB64(qb64b[24:28])) +assert gc == int.from_bytes(decodeB64(qb64b[28:32])) codeb64b = b'__" # 16 byte random uuid midb64b = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad then strip -pn = 1 -pnb64b = helping.intToB64b(pn, l=3) -pc = 2 -pcb64b = helping.intToB64b(pc, l=3) +gn = 1 +gnb64b = helping.intToB64b(gn, l=3) +gc = 2 +gcb64b = helping.intToB64b(gc, l=3) -headb64b = codeb64b + midb64b + pnb64b + pcb64b +headb64b = codeb64b + midb64b + gnb64b + gcb64b fmt = '!2s22s4s4s' -PartLeader = struct.Struct(fmt) -PartLeader.size == 32 -head = PartLeader.pack(codeb64b, midb64b, pnb64b, pcb64b) +GramLeader = struct.Struct(fmt) +GramLeader.size == 32 +head = GramLeader.pack(codeb64b, midb64b, gnb64b, gcb64b) assert helping.Reb64.match(head) -codeb64b, midb64b, pnb64b, pcb64b = PartLeader.unpack(head) -codeb64b, midb64b, pnb64b, pcb64b = PartLeader unpack_from(fmt, part) +codeb64b, midb64b, gnb64b, gcb64b = GramLeader.unpack(head) +codeb64b, midb64b, gnb64b, gcb64b = GramLeader unpack_from(fmt, gram) mid = helping.b64ToInt(b'AA' + midb64b)) -pn = helping.b64ToInt(pnb64b)) -pc = helping.b64ToInt(pcb64b)) +gn = helping.b64ToInt(gnb64b)) +gc = helping.b64ToInt(gcb64b)) -Standard header in all parts consists of -code + mid + pn +Standard header in all grams consists of +code + mid + gn -The first part with pn == 0 also has an additional field called the neck -that is the total part count pc +The first gram with gn == 0 also has an additional field called the neck +that is the total gram count gc Standard HeadSize = 28 # base64 chars or 21 base2 bytes @@ -222,31 +222,31 @@ codeb64b = b'__" midb64b = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip -pn = 1 -pnb64b = helping.intToB64b(pn, l=4) -pc = 2 -pcb64b = helping.intToB64b(pc, l=4) +gn = 1 +gnb64b = helping.intToB64b(gn, l=4) +gc = 2 +gcb64b = helping.intToB64b(gc, l=4) -headb64b = codeb64b + midb64b + pnb64b -neckb64b = pcb64b +headb64b = codeb64b + midb64b + gnb64b +neckb64b = gcb64b assert helping.Reb64.match(headb64b) assert helping.Reb64.match(neckb64b) codeb64b = headb64b[:2] mid = helping.b64ToInt(b'AA' + headb64b[2:24])) # prepad and convert -pn = helping.b64ToInt(headb64b[24:28])) -pc = helping.b64ToInt(neckb64b[28:32])) +gn = helping.b64ToInt(headb64b[24:28])) +gc = helping.b64ToInt(neckb64b[28:32])) -PartCode Typing/Versions uses a different base64 two char code. +GramCode Typing/Versions uses a different base64 two char code. Codes always start with `_` and second char in code starts at `_` -and walks backwards through B64 code table for each part type-version (tv) code. +and walks backwards through B64 code table for each gram type-version (tv) code. Each of total head length including code and total neck length must be a multiple of 4 characters. so eacch converts to/from B2 without padding. Head type-version code governs the neck as well. So to change length in order to change -field lengths or add or remove fields need a new part (tv) code. +field lengths or add or remove fields need a new gram (tv) code. -To reiterate, if ever need to change anything ned a new part tv code. There are +To reiterate, if ever need to change anything ned a new gram tv code. There are no versions per type. This is deemed sufficient because anticipate a very limited set of possible fields ever needed for memogram transport types. @@ -254,27 +254,27 @@ tv 1 is '_-' tv 3 is '_9' -Because the part head and neck are valid mid-pad B64 primitives then can be losslessly -transformed to/from CESR primitives merely by translated the part tv code to +Because the gram head and neck are valid mid-pad B64 primitives then can be losslessly +transformed to/from CESR primitives merely by translated the gram tv code to an equivalent entry in the two char cesr primitive code table. When doing so The neck is always attached and then stripped when not needed. The CESR codes albeit different are one-to-one. This enables representing headers as CESR primitives for management but not over the wire. I.e. can transform -Memogram Parts to CESR primitives to tunnel in some CESR stream. +Memogram Grams to CESR primitives to tunnel in some CESR stream. Normally CESR streams are tunneled inside Memoer memograms sent over-the-wire. -In this case, the memogram is the wrapper and each part uses the part tv code +In this case, the memogram is the wrapper and each gram uses the gram tv code defined herein not the equivalent CESR code. Using so far reserved and unused '_' code which is reserved as the selector -for theCESR op code table for memogram parts makes it more apparent that a part +for theCESR op code table for memogram grams makes it more apparent that a gram belongs to a memogram and not a CESR stream. Eventually when CESR op codes eventually become defined, it's not likely to be confusing since the context of CESR op codes is different that transport wrappers. Morover when a MemoGram payload is a tunneled CESR Stream, the memogram parser -will parse and strip the MemoGram part wrappers to construct the tunneled, +will parse and strip the MemoGram gram wrappers to construct the tunneled, stream so the wrapper is transarent to the CESR stream and the CESR stream payload is opaque to the memogram wrapper, i.e. no ambiguity. @@ -283,36 +283,36 @@ HeadCode hc = b'__' HeadCodeSize hcs = 2 MemoIDSize mis = 22 -PartNumSize pns = 4 -PartCntSize pcs = 4 -PartHeadSize phs = 28 -PartHeadNeckSize phns = 28 + 4 = 32 +GramNumSize gns = 4 +GramCntSize gcs = 4 +GramHeadSize phs = 28 +GramHeadNeckSize phns = 28 + 4 = 32 -MaxPartBodySize = ()2** 16 -1) - HeadSize -PartBodySize pbs = something <= MaxBodySize -PartSize = PartHeadSize + PartBodySize -PartSize <= 2**16 -1 +MaxGramBodySize = ()2** 16 -1) - HeadSize +GramBodySize pbs = something <= MaxBodySize +GramSize = GramHeadSize + GramBodySize +GramSize <= 2**16 -1 MaxMemoSize = (2**16-1)**2 -MaxPartCount for UDP PartSize of 508 is ceil(MaxMemoSize/508) = 8454403 which is < 2**24-1 -not partsize includes head -MaxPartCount for UXD PartSize of 2**16 -1 is ceil(MaxMemoSize/(2**16-1))= 2**16-1 -MinPartSize = MaxMemoSize // (2**24-1) = 255 = 2**8-1 -MaxPartSize = (2**16-1) -For any given PartSize there is a MaxPartCount of 2**24-1 so if fix the PartSize +MaxGramCount for UDP GramSize of 508 is ceil(MaxMemoSize/508) = 8454403 which is < 2**24-1 +not gramsize includes head +MaxGramCount for UXD GramSize of 2**16 -1 is ceil(MaxMemoSize/(2**16-1))= 2**16-1 +MinGramSize = MaxMemoSize // (2**24-1) = 255 = 2**8-1 +MaxGramSize = (2**16-1) +For any given GramSize there is a MaxGramCount of 2**24-1 so if fix the GramSize this limits the effective -mms = min((2**16-1)**2), (PartSize - PartHeadSize) * (2**24-1)) -mms = min((2**16-1)**2), (PartBodySize) * (2**24-1)) -Note there is an extra 4 characters in first part for the neck +mms = min((2**16-1)**2), (GramSize - GramHeadSize) * (2**24-1)) +mms = min((2**16-1)**2), (GramBodySize) * (2**24-1)) +Note there is an extra 4 characters in first gram for the neck so ps, pbs, and mms are variables specific to transport PHS is fixed for the transport type reliable unreliable with header fields as defined The desired ps for a given transport instance may be smaller than the allowed maximum to accomodate buffers etc. -Given the fixed part head size PHS once can calculate the maximum memo size +Given the fixed gram head size PHS once can calculate the maximum memo size that includes the header overhead given the contraint of no more than 2**24-1 -parts in a given memo. +grams in a given memo. So for a given transport: ps == min(ps, 2**16-1) @@ -320,60 +320,66 @@ mms = min((2**16-1)**2), pbs * (2**24-1)) So compare memo size to mms and if greater raise error and drop memo -otherwise partition (part) memo into parts with headers and transmit +otherwise partition (gram) memo into grams with headers and transmit Note max memo size is (2**16-1)**2 = 4294836225. -But accounting for the part header overhead the largest payload max memo size is -max part size = (2**16-1) == 65535 -max part count = (2**24-1) == 16777215 +But accounting for the gram header overhead the largest payload max memo size is +max gram size = (2**16-1) == 65535 +max gram count = (2**24-1) == 16777215 standard header size hs = 28 neck size ns = 4 -bs = maxpartsize - hs = (65535 - hs) = 65507 # max standard part body size +bs = maxgramsize - hs = (65535 - hs) = 65507 # max standard gram body size max theoretical memogram payload size mms is -mms = (bs * maxpartcount) - ns = 65507 * 16777215 - 4 = 1099025023001 +mms = (bs * maxgramcount) - ns = 65507 * 16777215 - 4 = 1099025023001 but this is too big so we limit it to 4294836225 but we can have a payload -of 4294836225 when using the maxpartsize for each part. - -The maximum number of bytes in a CESR big group frame body is +of 4294836225 when using the maxgramsize for each gram. + +CESR groups code count in quadlets (4 chars). +The big group count codes have five digits of 6 bits each or 30 bits to count the +size of the following framed chars. +The max count value is therefore (2**30-1). Cesr group codes count the number +of quadlets (4 chars) in the framed group. The maximum number of bytes in a +CESR big group frame body is therefore: (2**30-1)*4 = 4294967292. -The maximum group count is (2**30-1) == 1073741823 -Groups code count in quadlets (4) and the max count value is (2**30-1) for 5 -6 bit b64 digits. -The group code is not part of the counted quadlets so the largest group -with its group code of length 8 is 4294967292 + 8 = 4294967300 -which is (2**30-1)*4 + 8. - -This is bigger than the max effective MemoGram payload size by -4294967300 - 4294836225 = 131075 - - -So when sending via segmented MemoGram parts the maximum big group count frame -with code as payload is +The group code is not gram of the counted quadlets in the frame body. +The big group codes are 8 chars long. Thus with its group code the total big +frame size is: +(2**30-1)*4 + 8 = 4294967300 + +The largest value for a 32 bit number is +(2 ** 32 - 1) = 4294967295 which is 5 less than the largest group frame size with code. +4294967300 - 4294967295 = 5 +Largest total frame size with group code = 2**32+4 which exceeds the largest +value of a 32 bit number. + +Restricting the largest memogram payload to be 2**32-1 should be sufficient. +It is anticipated that other limits such as memory or processing power will +constrain memogram sizes to be much smaller than 2**32-1 much less 2**32+4. +So MaximumMemoSize is 2**32-1 = 4294967295 +MaxGramSize with overhead is 2**16-1 = 65535 which is a limit on some datagram +transports for the maximum sized datagram. This is true of UDP although the +UDP MTU size is much smaller than this. It may not longer be true on some +OSes that the maximum datagram size for UXD transports is 65535. On some +OSes it may be bigger. But this seems like a good practical limit nonetheless +as OS buffer sizes are usually in the order of magnitude of 65535. -(4294836225 - 8) // 4 = 1073709054 -This is less than the absolute max group count by -1073741823 - 1073709054 = 32769 +The maximum group count is (2**30-1) == 1073741823 -So biggest memogram payload is relatively slightly smaller than the max group -frame size with group code. -To fix this we would have to allow for a bigger part gram size than 2**16-1 -For UXD datagrams this is not a hard limit -but it is a hard limit for UDP datagrams -It is unlikely we would ever reach those limits in a practical application. ToDo: -change maxmemosize to accomodate big group + code -Change mode to boolean b2mode True False no more Wiffs -Add code '_-' for signed parts vs verification id size ss signatures size +Make gram body size calculation a function of the current .mode for header +encoding as b2 headers are smaller so the body payload is bigger. + +Add code '_-' for signed grams vs verification id size ss signatures size expanes Sizes for vs and ss -Signed first part code + mid + vid + pn + pc + body + sig -Signed other parts code + mid + vid + pn + body + sig +Signed first gram code + mid + vid + gn + gc + body + sig +Signed other grams code + mid + vid + gn + body + sig Add sign and verify methods -Change name of MemoGram to Memoer to match convention + """ @@ -398,35 +404,33 @@ # namedtuple of ints (major: int, minor: int) Versionage = namedtuple("Versionage", "major minor") -Wiffage = namedtuple("Wiffage", 'txt bny') # part head encoding -Wiffs = Wiffage(txt='txt', bny='bny') - # namedtuple for size entries in Memoer code tables # cs is the code size int number of chars in code portion # ms is the mid size int number of chars in the mid portion (memoID) -# ns is the neck size int number of chars for the part number in all parts and -# the additional neck that also appears in first part +# ns is the neck size int number of chars for the gram number in all grams and +# the additional neck that also appears in first gram # hs is the head size int number of chars hs = cs + ms + ns and for the first -# part the head of size hs is followed by a neck of size ns. So the total -# overhead on first part is os = hs + ns +# gram the head of size hs is followed by a neck of size ns. So the total +# overhead on first gram is os = hs + ns Sizage = namedtuple("Sizage", "cs ms ns hs") @dataclass(frozen=True) -class PartCodex: +class GramCodes: """ - PartCodex is codex of all Part Head Codes. + GramCodes is codex of all Gram Codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ - Basic: str = '__' # Basic Part Head Code + Basic: str = '__' # Basic gram code for basic overhead parts + Signed: str = '_-' # Basic gram code for signed overhead parts def __iter__(self): return iter(astuple(self)) -PartDex = PartCodex() # Make instance +GramDex = GramCodes() # Make instance class Memoer(hioing.Mixin): @@ -484,13 +488,10 @@ class Memoer(hioing.Mixin): Class Attributes: Version (Versionage): default version consisting of namedtuple of form (major: int, minor: int) - BufCount (int): default bufcount bc for transport 64 - Code (str): default part type-version head code '__' - MaxMemoSize (int): absolute max memo size 4294836225 - MaxPartSize (int): absolute max part size 65535 - MaxPartCount (int): absolute max part count 16777215 - Sizes (dict): part sizes Sizage instances keyed by part codes - Mode (str): default encoding mode for tx part header b64 'txt' Wiffs.txt + MaxMemoSize (int): absolute max memo size + MaxGramSize (int): absolute max gram size on tx with overhead + MaxGramCount (int): absolute max gram count + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes Inherited Attributes: name (str): unique name for Memoer transport. Used to manage. @@ -504,15 +505,15 @@ class Memoer(hioing.Mixin): rxgs (dict): holding rx (receive) (data) gram deques of grams. Each item in dict has key=src and val=deque of grames received rxgs (dict): keyed by mid (memoID) with value of dict where each deque - holds incomplete memo part grams for that mid. - The mid appears in every gram part from the same memo. - The value dict is keyed by the part number pn, with value - that is the part bytes. + holds incomplete memo grams for that mid. + The mid appears in every gram from the same memo. + The value dict is keyed by the gram number with value + that is the gram bytes. sources (dict): keyed by mid (memoID) that holds the src for the memo. This enables reattaching src to fused memo in rxms deque tuple. - counts (dict): keyed by mid (memoID) that holds the part count (pc) from - the first part for the memo. This enables lookup of the pc when - fusing its parts. + counts (dict): keyed by mid (memoID) that holds the gram count from + the first gram for the memo. This enables lookup of the gram count when + fusing its grams. rxms (deque): holding rx (receive) memo duples desegmented from rxgs grams each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be segmented into @@ -527,12 +528,15 @@ class Memoer(hioing.Mixin): for (gram, dst) echos (deque): holding echo receive duples for testing. Each duple of form: (gram: bytes, dst: str). - code (bytes | None): part head code for part header - size (int): part size for this instance for transport. Part size - = head size + body size. Part size limited by - MaxPartSize and head size + 1 - mode (str): encoding mode for part header when transmitting, - b64 'txt' or b2 'bny' + code (bytes | None): gram code for gram header when rending for tx + mode (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = over head size + neck size + body size. + other gram size = over head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. @@ -540,17 +544,15 @@ class Memoer(hioing.Mixin): """ Version = Versionage(major=0, minor=0) # default version - BufCount = 64 # default bufcount bc for transport - Code = PartDex.Basic # default part type-version head code '__' - MaxMemoSize = 4294836225 # (2**16-1)**2 absolute max memo size - MaxPartSize = 65535 # (2**16-1) absolute max part size - MaxPartCount = 16777215 # (2**24-1) absolute max part count - Sizes = {'__': Sizage(cs=2, ms=22, ns=4, hs=28)} # dict of part sizes keyed by part codes - Mode = Wiffs.txt # encoding mode for tx part header b64 'txt' or b2 'bny' + MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size + MaxGramSize = 65535 # (2**16-1) absolute max gram size + MaxGramCount = 16777215 # (2**24-1) absolute max gram count + # dict of gram header part sizes keyed by gram codes + Sizes = {'__': Sizage(cs=2, ms=22, ns=4, hs=28)} def __init__(self, - name='main', + name=None, bc=None, version=None, rxgs=None, @@ -560,55 +562,59 @@ def __init__(self, txms=None, txgs=None, txbs=None, - code=None, + code=GramDex.Basic, + mode=False, size=None, - mode=None, **kwa ): """Setup instance Inherited Parameters: name (str): unique name for Memoer transport. Used to manage. + opened (bool): True means opened for transport + False otherwise bc (int | None): count of transport buffers of MaxGramSize Parameters: version (Versionage): version for this memoir instance consisting of namedtuple of form (major: int, minor: int) - rxgs (dict): keyed by mid (memoID) with value of dict where each deque - holds incomplete memo part grams for that mid. - The mid appears in every gram part from the same memo. - The value dict is keyed by the part number pn, with value - that is the part bytes. - sources (dict): keyed by mid that holds the src for the memo indexed by its - mid (memoID). This enables reattaching src to memo when + rxgs (dict): keyed by mid (memoID) with value of dict where each + value dict holds grams from memo keyed by gram number. + Grams have been stripped of their headers. + The mid appears in every gram from the same memo. + The value dict is keyed by the gram number gn, with value bytes. + sources (dict): keyed by mid that holds the src for the memo indexed + by its mid (memoID). This enables reattaching src to memo when placing fused memo in rxms deque. - counts (dict): keyed by mid that holds the part count (pc) for the - memo indexed by its mid (memoID). This enables lookup of the pc - to memo when fusing its parts. + counts (dict): keyed by mid that holds the gram count for the + memo indexed by its mid (memoID). This enables lookup of the + gram count for a given memo to know when it has received all its + constituent grams in order to fuse back into the memo. rxms (deque): holding rx (receive) memo duples fused from rxgs grams each entry in deque is duple of form (memo: str, dst: str) txms (deque): holding tx (transmit) memo tuples to be partitioned into txgs grams where each entry in deque is duple of form (memo: str, dst: str) txgs (deque): grams to transmit, each entry is duple of form: - (gram: bytes, dst: str). + (gram: bytes, dst: str). Grams include gram headers. txbs (tuple): current transmisstion duple of form: (gram: bytearray, dst: str). gram bytearray may hold untransmitted portion when datagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - code (bytes | None): part code for part header - size (int): part size for this instance for transport. Part size - = head size + body size. Part size limited by - MaxPartSize and MaxPartCount relative to MaxMemoSize as well - as minimum part body size of 1 - mode (str): encoding mode for part header when transmitting, - b64 'txt' or b2 'bny' - - + code (bytes | None): gram code for gram header + mode (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + size (int): gram size when rending for tx. + first gram size = head size + neck size + body size. + other gram size = head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount relative to + MaxMemoSize. """ + # initialize attributes self.version = version if version is not None else self.Version self.rxgs = rxgs if rxgs is not None else dict() @@ -620,24 +626,28 @@ def __init__(self, self.txgs = txgs if txgs is not None else deque() self.txbs = txbs if txbs is not None else (bytearray(), None) - bc = bc if bc is not None else self.BufCount # use bufcount to calc .bs + self.echos = deque() # only used in testing as echoed tx self.code = code if code is not None else self.Code _, _, ns, hs = self.Sizes[self.code] - size = size if size is not None else self.MaxPartSize - self.size = max(min(size, self.MaxPartSize), hs + ns + 1) - self.mode = mode if mode is not None else self.Mode + self.mode = mode + size = size if size is not None else self.MaxGramSize + self.size = max(min(size, self.MaxGramSize), hs + ns + 1) - super(Memoer, self).__init__(name=name, bc=bc, **kwa) + + super(Memoer, self).__init__(name, bc, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. - self.name = name # mixed with subclass should provide this. + self.name = name if name is not None else "main" # mixed with subclass should provide this. if not hasattr(self, "opened"): # stub so mixin works in isolation. self.opened = False # mixed with subclass should provide this. + if not hasattr(self, "bc"): # stub so mixin works in isolation. + self.bc = bc if bc is not None else 64 # mixed with subclass should provide this. + def open(self): """Opens transport in nonblocking mode @@ -663,118 +673,119 @@ def close(self): self.opened = False - def wiff(self, part): - """Determines encoding format of part bytes header as either base64 text - 'txt' or base2 binary 'bny' from first 6 bits (sextet) in part. + def wiff(self, gram): + """Determines encoding mode of gram bytes header when parsing grams. + The mode maybe either base2 or base64. + Returns: - wiff (Wiffs): Wiffs.txt if B64 text, Wiffs.bny if B2 binary - Otherwise raises hioing.MemoerError + mode (bool): True means base2 + False means base64 + Otherwise raises hioing.MemoerError - All part head codes start with '_' in base64 text or in base2 binary. + All gram head codes start with '_' in base64 text or in base2 binary. Given only allowed chars are from the set of base64 then can determine if header is in base64 or base2. First sextet: (0b is binary, 0o is octal, 0x is hexadecimal) - 0o27 = 0b010111 means first sextet of '_' in base64 => 'txt' - 0o77 = 0b111111 means first sextet of '_' in base2 => 'bny' + 0o27 = 0b010111 means first sextet of '_' in base64 + 0o77 = 0b111111 means first sextet of '_' in base2 - Assuming that only '_' is allowed as the first char of a valid part head, + Assuming that only '_' is allowed as the first char of a valid gram head, in either Base64 or Base2 encoding, a parser can unambiguously determine - if the part header encoding is binary Base2 or text Base64 because + if the gram header encoding is binary Base2 or text Base64 because 0o27 != 0o77. Furthermore, it just so happens that 0o27 is a unique first sextet - among Base64 ascii chars. So an invalid part header when in Base64 encoding + among Base64 ascii chars. So an invalid gram header when in Base64 encoding is detectable. However, there is one collision (see below) with invalid - part headers in Base2 encoding. So an erroneously constructed part might + gram headers in Base2 encoding. So an erroneously constructed gram might not be detected merely by looking at the first sextet. Therefore unreliable transports must provide some additional mechanism such as a - hash or signature on the part. + hash or signature on the gram. All Base2 codes for Base64 chars are unique since the B2 codes are just the count from 0 to 63. There is one collision, however, between Base2 and - Base 64 for invalid part headers. This is because 0o27 is the + Base 64 for invalid gram headers. This is because 0o27 is the base2 code for ascii 'V'. This means that Base2 encodings that begin 0o27 witch is the Base2 encoding of the ascii 'V, would be incorrectly detected as text not binary. """ - sextet = part[0] >> 2 + sextet = gram[0] >> 2 if sextet == 0o27: - return Wiffs.txt # 'txt' + return False # base64 text mode if sextet == 0o77: - return Wiffs.bny # 'bny' + return True # base2 binary mode - raise hioing.Memoer(f"Unexpected {sextet=} at part head start.") + raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") - def parse(self, part): - """Parse and strips header from gram part bytearray and returns - (mid, pn, pc) when first part, (mid, pn, None) when other part - Otherwise raises MemoerError is unrecognized header + def parse(self, gram): + """Parse and strips header from gram bytearray and returns + (mid, gn, gc). Raises MemoerError is unrecognized header Returns: result (tuple): tuple of form: - (mid: str, pn: int, pc: int | None) where: - mid is memoID, pn is part number, pc is part count. - When first first part (zeroth) returns (mid, 0, pc). - When other part returns (mid, pn, None) - Otherwise raises memogram error. + (mid: str, gn: int, gc: int | None) where: + mid is memoID, gn is gram number, gc is gram count. + When first gram (zeroth) returns (mid, 0, gc). + When other gram returns (mid, gn, None) + Otherwise raises MemoerError error. - When valid recognized header, strips header bytes from front of part - leaving the part body part bytearray. + When valid recognized header, strips header bytes from front of gram + leaving the gram body part bytearray. Parameters: - part (bytearray): memo part gram from which to parse and strip its header. + gram (bytearray): memo gram from which to parse and strip its header. """ cs, ms, ns, hs = self.Sizes[self.code] - wiff = self.wiff(part) - if wiff == Wiffs.bny: + wiff = self.wiff(gram) + if wiff: # base2 binary mode bhs = 3 * (hs) // 4 # binary head size - head = part[:bhs] # slice takes a copy + head = gram[:bhs] # slice takes a copy if len(head) < bhs: # not enough bytes for head - raise hioing.Memoer(f"Part size b2 too small < {bhs}" - f" for head.") + raise hioing.MemoerError(f"Not enough rx bytes for base2 gram" + f"header < {bhs}.") head = encodeB64(head) # convert to Base64 - del part[:bhs] # strip head off part - else: # wiff == Wiffs.txt: - head = part[:hs] # slice takes a copy + del gram[:bhs] # strip head off gram + else: # base64 text mode + head = gram[:hs] # slice takes a copy if len(head) < hs: # not enough bytes for head - raise hioing.Memoer(f"Part size b64 too small < {hs} for" - f"head.") - del part[:hs] # strip off head off + raise hioing.MemoerError(f"Not enough rx chars for base64 gram " + f"header < {hs}.") + del gram[:hs] # strip off head off code = head[:cs].decode() if code != self.code: - raise hioing.Memoer(f"Unrecognized part {code=}.") + raise hioing.MemoerError(f"Unrecognized gram {code=}.") mid = head[cs:cs+ms].decode() - pn = helping.b64ToInt(head[cs+ms:cs+ms+ns]) + gn = helping.b64ToInt(head[cs+ms:cs+ms+ns]) - pc = None - if pn == 0: # first (zeroth) part so get neck - if wiff == Wiffs.bny: + gc = None + if gn == 0: # first (zeroth) gram so get neck + if wiff: # base2 binary mode bns = 3 * (ns) // 4 # binary neck size - neck = part[:bns] # slice takes a copy + neck = gram[:bns] # slice takes a copy if len(neck) < bns: # not enough bytes for neck - raise hioing.Memoer(f"Part size b2 too small < {bns}" - f" for neck.") + raise hioing.MemoerError(f"Not enough rx bytes for base2" + f" gram neck < {bns}.") neck = encodeB64(neck) # convert to Base64 - del part[:bns] # strip off neck - else: # wiff == Wiffs.txt: - neck = part[:ns] # slice takes a copy + del gram[:bns] # strip off neck + else: # # base64 text mode + neck = gram[:ns] # slice takes a copy if len(neck) < ns: # not enough bytes for neck - raise hioing.Memoer(f"Part size b64 too small < {ns}" - f" for neck.") - del part[:ns] # strip off neck + raise hioing.MemoerError(f"Not enough rx chars for base64" + f" gram neck < {ns}.") + del gram[:ns] # strip off neck - pc = helping.b64ToInt(neck) + gc = helping.b64ToInt(neck) - return (mid, pn, pc) + return (mid, gn, gc) def receive(self, *, echoic=False): @@ -844,31 +855,30 @@ def _serviceOneReceived(self, *, echoic=False): if not gram: # no received data return False - part = bytearray(gram)# make copy bytearray so can strip off header + gram = bytearray(gram)# make copy bytearray so can strip off header try: - mid, pn, pc = self.parse(part) # parse and strip off head leaving body - except hioing.Memoer as ex: # invalid part so drop - logger.error("Unrecognized Memoer part from %s.\n %s.", src, ex) + mid, gn, gc = self.parse(gram) # parse and strip off head leaving body + except hioing.MemoerError as ex: # invalid gram so drop + logger.error("Unrecognized Memoer gram from %s.\n %s.", src, ex) return True # did receive data to can keep receiving if mid not in self.rxgs: self.rxgs[mid] = dict() - # save stripped part gram to be fused later - if pn not in self.rxgs[mid]: # make idempotent first only no replay - self.rxgs[mid][pn] = part # index body by its part number + # save stripped gram to be fused later + if gn not in self.rxgs[mid]: # make idempotent first only no replay + self.rxgs[mid][gn] = gram # index body by its gram number - if pc is not None: + if gc is not None: if mid not in self.counts: # make idempotent first only no replay - self.counts[mid] = pc # save part count for mid + self.counts[mid] = gc # save gram count for mid # assumes unique mid across all possible sources. No replay by different # source only first source for a given mid is ever recognized if mid not in self.sources: # make idempotent first only no replay self.sources[mid] = src # save source for later - return True # received valid @@ -899,9 +909,9 @@ def serviceReceives(self, *, echoic=False): break - def fuse(self, parts, cnt): - """Fuse cnt gram part bodies from parts dict into whole memo . If any - parts are missing then returns None. + def fuse(self, grams, cnt): + """Fuse cnt gram body parts from grams dict into whole memo . If any + grams are missing then returns None. Returns: memo (str | None): fused memo or None if incomplete. @@ -909,16 +919,16 @@ def fuse(self, parts, cnt): Override in subclass Parameters: - parts (dict): memo part bodies each keyed by part number from which - to fuse memo. - cnt (int): part count for mid + grams (dict): memo gram body parts each keyed by gram number from which + to fuse memo. Headers have been stripped. + cnt (int): gram count for mid """ - if len(parts) < cnt: # must be missing one or more parts + if len(grams) < cnt: # must be missing one or more grams return None memo = bytearray() for i in range(cnt): # iterate in numeric order, items are insertion ordered - memo.extend(parts[i]) # extend memo with part body at part i + memo.extend(grams[i]) # extend memo with gram body part at gram i return memo.decode() # convert bytearray to str @@ -932,8 +942,8 @@ def _serviceOnceRxGrams(self): be last. """ for i, mid in enumerate(list(self.rxgs.keys())): # items may be deleted in loop - # if mid then parts dict at mid must not be empty - if not mid in self.counts: # missing first part so skip + # if mid then grams dict at mid must not be empty + if not mid in self.counts: # missing first gram so skip continue memo = self.fuse(self.rxgs[mid], self.counts[mid]) if memo is not None: # allows for empty "" memo for some src @@ -1037,53 +1047,53 @@ def memoit(self, memo, dst): def rend(self, memo): - """Partition memo into packed part grams with headers. + """Partition memo into packed grams with headers. Returns: - parts (list[bytes]): list of part grams with headers. + grams (list[bytes]): list of grams with headers. Parameters: - memo (str): to be partitioned into gram parts with headers + memo (str): to be partitioned into grams with headers - Note first part has head + neck overhead, hs + ns so bs is smaller by ns - non-first parts have just head overhead hs so bs is bigger by ns + Note first gram has head + neck overhead, hs + ns so bs is smaller by ns + non-first grams have just head overhead hs so bs is bigger by ns """ - parts = [] + grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray cs, ms, ns, hs = self.Sizes[self.code] - # self.size is max part size - bs = (self.size - hs) # max standard part body size, - mms = min(self.MaxMemoSize, (bs * self.MaxPartCount) - ns) # max memo payload + # self.size is max gram size + bs = (self.size - hs) # max standard gram body size, + mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload ml = len(memo) if ml > mms: - raise hioing.Memoer(f"Memo length={ml} exceeds max={mms}.") + raise hioing.MemoerError(f"Memo length={ml} exceeds max={mms}.") # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned midb = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip #mid = midb.decode() - # compute part count account for added neck overhead in first part - pc = math.ceil((ml + ns - bs)/bs + 1) # includes added neck ns overhead - neck = helping.intToB64b(pc, l=ns) - if self.mode == Wiffs.bny: + # compute gram count account for added neck overhead in first gram + gc = math.ceil((ml + ns - bs)/bs + 1) # includes added neck ns overhead + neck = helping.intToB64b(gc, l=ns) + if self.mode: # b2 mode binary neck = decodeB64(neck) - pn = 0 + gn = 0 while memo: - pnb = helping.intToB64b(pn, l=ns) # pn size always == neck size - head = self.code.encode() + midb + pnb - if self.mode == Wiffs.bny: + gnb = helping.intToB64b(gn, l=ns) # gn size always == neck size + head = self.code.encode() + midb + gnb + if self.mode: # b2 binary head = decodeB64(head) - if pn == 0: - part = head + neck + memo[:bs-ns] # copy slice past end just copies to end + if gn == 0: + gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end del memo[:bs-ns] # del slice past end just deletes to end else: - part = head + memo[:bs] # copy slice past end just copies to end + gram = head + memo[:bs] # copy slice past end just copies to end del memo[:bs] # del slice past end just deletes to end - parts.append(part) + grams.append(gram) - pn += 1 + gn += 1 - return parts + return grams def _serviceOneTxMemo(self): @@ -1091,14 +1101,14 @@ def _serviceOneTxMemo(self): (memo: str, dst: str) where: memo is outgoing memo dst is destination address - Calls .segment method to process the segmentation and packing as - appropriate to convert memo into gram(s). + Calls .rent method to process the partitioning and packing as + appropriate to convert memo into grams with headers. Appends duples of (gram, dst) from grams to .txgs deque. """ memo, dst = self.txms.popleft() # raises IndexError if empty deque - for part in self.rend(memo): # partition memo into gram parts with head - self.txgs.append((part, dst)) # append duples (gram: bytes, dst: str) + for gram in self.rend(memo): # partition memo into gram parts with head + self.txgs.append((gram, dst)) # append duples (gram: bytes, dst: str) def serviceTxMemosOnce(self): diff --git a/src/hio/hioing.py b/src/hio/hioing.py index aa4610b..6eee0b4 100644 --- a/src/hio/hioing.py +++ b/src/hio/hioing.py @@ -83,7 +83,7 @@ class NamerError(HioError): raise NamerError("error message") """ -class Memoer(HioError): +class MemoerError(HioError): """ Error using or configuring Remoter diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index db8468b..8e966e4 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -18,10 +18,11 @@ def test_memoer_basic(): peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False - assert peer.code == '__' == peer.Code + assert peer.bc == 64 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.mode assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size - assert peer.size == peer.MaxPartSize - + assert peer.size == peer.MaxGramSize peer.reopen() assert peer.opened == True @@ -39,7 +40,7 @@ def test_memoer_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams() @@ -74,7 +75,7 @@ def test_memoer_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) @@ -101,8 +102,8 @@ def test_memoer_basic(): peer.serviceRxMemos() assert not peer.rxms - # test binary q2 encoding of transmission part header - peer.mode = memoing.Wiffs.bny + # test binary q2 encoding of transmission gram header + peer.mode = True # set to binary base2 memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -110,7 +111,7 @@ def test_memoer_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.bny + assert peer.wiff(m) # base2 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams() @@ -124,7 +125,7 @@ def test_memoer_basic(): mid = 'ALBI68S1ZIxqwFOSWFF1L2' headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) gram = headneck + b"Hello There" - assert peer.wiff(gram) == memoing.Wiffs.bny + assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -144,13 +145,15 @@ def test_memoer_basic(): """ End Test """ -def test_memogram_small_part_size(): - """Test MemoGram class with small part size +def test_memogram_small_gram_size(): + """Test MemoGram class with small gram size """ peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False - assert peer.code == '__' == peer.Code + assert peer.bc == 64 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.mode assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size assert peer.size == 33 # can't be smaller than head + neck + 1 @@ -173,7 +176,7 @@ def test_memogram_small_part_size(): assert not peer.txms assert len(peer.txgs) == 2 for m, d in peer.txgs: - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs @@ -215,7 +218,7 @@ def test_memogram_small_part_size(): assert not peer.txms assert len(peer.txgs) == 2 for m, d in peer.txgs: - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) assert not peer.txgs @@ -243,8 +246,8 @@ def test_memogram_small_part_size(): peer.serviceRxMemos() assert not peer.rxms - # test binary q2 encoding of transmission part header - peer.mode = memoing.Wiffs.bny + # test binary q2 encoding of transmission gram header + peer.mode = True # set to binary base2 memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -253,7 +256,7 @@ def test_memogram_small_part_size(): assert not peer.txms assert len(peer.txgs) == 2 for m, d in peer.txgs: - assert peer.wiff(m) == memoing.Wiffs.bny + assert peer.wiff(m) # base2 assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs @@ -266,7 +269,7 @@ def test_memogram_small_part_size(): headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) gram = headneck + b"Hello There" - assert peer.wiff(gram) == memoing.Wiffs.bny + assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) @@ -275,12 +278,12 @@ def test_memogram_small_part_size(): mid = 'DFymLrtlZG6bp0HhlUsR6u' headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAC').encode()) gram = headneck + b"Hello " - assert peer.wiff(gram) == memoing.Wiffs.bny + assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) head = decodeB64(('__' + mid + 'AAAB').encode()) gram = head + b"There" - assert peer.wiff(gram) == memoing.Wiffs.bny + assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -309,14 +312,16 @@ def test_memogram_small_part_size(): def test_memoer_multiple(): - """Test Memoer class with small part size and multiple queued memos + """Test Memoer class with small gram size and multiple queued memos """ peer = memoing.Memoer(size=38) assert peer.size == 38 assert peer.name == "main" assert peer.opened == False - assert peer.code == '__' == peer.Code - assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + assert peer.bc == 64 + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.mode + peer.reopen() assert peer.opened == True @@ -328,7 +333,7 @@ def test_memoer_multiple(): assert not peer.txms assert len(peer.txgs) == 4 for m, d in peer.txgs: - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert d in ("alpha", "beta") peer.serviceTxGrams(echoic=True) assert not peer.txgs @@ -377,8 +382,8 @@ def test_memoer_multiple(): -def test_open_mg(): - """Test contextmanager decorator openMG +def test_open_memoer(): + """Test contextmanager decorator openMemoer """ with (memoing.openMemoer(name='zeta') as zeta): @@ -425,9 +430,10 @@ def test_tymeememogram_basic(): assert peer.tymeout == 0.0 assert peer.name == "main" assert peer.opened == False - assert peer.code == '__' == peer.Code + assert peer.code == memoing.GramDex.Basic == '__' + assert not peer.mode assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size - assert peer.size == peer.MaxPartSize + assert peer.size == peer.MaxGramSize peer.reopen() assert peer.opened == True @@ -445,7 +451,7 @@ def test_tymeememogram_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams() @@ -480,7 +486,7 @@ def test_tymeememogram_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.txt + assert not peer.wiff(m) # base64 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) @@ -507,8 +513,8 @@ def test_tymeememogram_basic(): peer.serviceRxMemos() assert not peer.rxms - # test binary q2 encoding of transmission part header - peer.mode = memoing.Wiffs.bny + # test binary q2 encoding of transmission gram header + peer.mode = True # set to binary base2 memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -516,7 +522,7 @@ def test_tymeememogram_basic(): peer.serviceTxMemos() assert not peer.txms m, d = peer.txgs[0] - assert peer.wiff(m) == memoing.Wiffs.bny + assert peer.wiff(m) # base2 assert m.endswith(memo.encode()) assert d == dst == 'beta' peer.serviceTxGrams() @@ -530,7 +536,7 @@ def test_tymeememogram_basic(): mid = 'ALBI68S1ZIxqwFOSWFF1L2' headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) gram = headneck + b"Hello There" - assert peer.wiff(gram) == memoing.Wiffs.bny + assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) peer.serviceReceives(echoic=True) @@ -557,8 +563,8 @@ def test_tymeememogram_basic(): """ End Test """ -def test_open_tmg(): - """Test contextmanager decorator openTMG +def test_open_tm(): + """Test contextmanager decorator openTM for openTymeeMemoer """ with (memoing.openTM(name='zeta') as zeta): @@ -611,11 +617,11 @@ def test_tymeememogram_doer(): if __name__ == "__main__": test_memoer_basic() - test_memogram_small_part_size() + test_memogram_small_gram_size() test_memoer_multiple() - test_open_mg() + test_open_memoer() test_memogram_doer() test_tymeememogram_basic() - test_open_tmg() + test_open_tm() test_tymeememogram_doer() From b099e64dfe5d5d56d60da258e0c16cc67100063d Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 24 Feb 2025 13:21:38 -0700 Subject: [PATCH 25/33] added properties for code, mode, and size --- src/hio/core/memoing.py | 106 +++++++++++++++++++++++++++++-------- tests/core/test_memoing.py | 9 ++-- 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 9bca4c4..674498e 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -405,16 +405,18 @@ Versionage = namedtuple("Versionage", "major minor") -# namedtuple for size entries in Memoer code tables -# cs is the code size int number of chars in code portion -# ms is the mid size int number of chars in the mid portion (memoID) -# ns is the neck size int number of chars for the gram number in all grams and -# the additional neck that also appears in first gram +# namedtuple for gram header part size entries in Memoer code tables +# cs is the code part size int number of chars in code part +# ms is the mid part size int number of chars in the mid part (memoID) +# ns is the neck part size int number of chars for the gram number in all grams +# and the additional neck that also appears in first gram # hs is the head size int number of chars hs = cs + ms + ns and for the first # gram the head of size hs is followed by a neck of size ns. So the total # overhead on first gram is os = hs + ns +# vs is the vid part size int number of chars in the vid part (verification ID) +# ss is the signature part size int number of chars in the signature part. -Sizage = namedtuple("Sizage", "cs ms ns hs") +Sizage = namedtuple("Sizage", "cs ms vs ns hs ss") @dataclass(frozen=True) class GramCodes: @@ -528,6 +530,15 @@ class Memoer(hioing.Mixin): for (gram, dst) echos (deque): holding echo receive duples for testing. Each duple of form: (gram: bytes, dst: str). + + + Hidden: + _code (bytes | None): see size property + _mode (bool): see mode property + _size (int): see size property + + + Properties: code (bytes | None): gram code for gram header when rending for tx mode (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 @@ -538,17 +549,16 @@ class Memoer(hioing.Mixin): Gram size also limited by MaxGramSize and MaxGramCount relative to MaxMemoSize. - - - Properties: - """ Version = Versionage(major=0, minor=0) # default version MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size MaxGramSize = 65535 # (2**16-1) absolute max gram size MaxGramCount = 16777215 # (2**24-1) absolute max gram count # dict of gram header part sizes keyed by gram codes - Sizes = {'__': Sizage(cs=2, ms=22, ns=4, hs=28)} + Sizes = { + '__': Sizage(cs=2, ms=22, vs=0, ns=4, hs=28, ss=0), + '_-': Sizage(cs=2, ms=22, vs=44, ns=4, hs=72, ss=88), + } def __init__(self, @@ -602,7 +612,7 @@ def __init__(self, portion when datagram is not able to be sent all at once so can keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) - code (bytes | None): gram code for gram header + code (bytes): gram code for gram header mode (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 size (int): gram size when rending for tx. @@ -626,16 +636,11 @@ def __init__(self, self.txgs = txgs if txgs is not None else deque() self.txbs = txbs if txbs is not None else (bytearray(), None) - - self.echos = deque() # only used in testing as echoed tx - self.code = code if code is not None else self.Code - _, _, ns, hs = self.Sizes[self.code] - self.mode = mode - size = size if size is not None else self.MaxGramSize - self.size = max(min(size, self.MaxGramSize), hs + ns + 1) - + self._code = code + self._mode = mode + self.size = size # property sets size given .code and constraints super(Memoer, self).__init__(name, bc, **kwa) @@ -649,6 +654,63 @@ def __init__(self, self.bc = bc if bc is not None else 64 # mixed with subclass should provide this. + @property + def code(self): + """Property getter for ._code + + Returns: + code (str): two char base64 gram code + """ + return self._code + + @property + def mode(self): + """Property getter for ._mode + + Returns: + mode (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + """ + return self._mode + + @mode.setter + def mode(self, mode): + """Property setter for ._mode + + Paramaters: + mode (bool): True means when rending for tx encode header in base2 + False means when rending for tx encode header in base64 + """ + self._mode = mode + + @property + def size(self): + """Property getter for ._size + + Returns: + size (int): gram size when rending for tx. + First gram size = head size + neck size + body size. + Other gram size = head size + body size. + Min gram body size is one. + Gram size also limited by MaxGramSize and MaxGramCount + relative to MaxMemoSize. + """ + return self._size + + @size.setter + def size(self, size): + """Property setter for ._size + + Parameters: + size (int | None): gram size for rending memo + + """ + _, _, _, ns, hs, ss = self.Sizes[self.code] # "cs ms vs ns hs ss" + size = size if size is not None else self.MaxGramSize + # mininum size must be big enough for first gram header and 1 body byte + self._size = max(min(size, self.MaxGramSize), hs + ns + ss + 1) + + def open(self): """Opens transport in nonblocking mode @@ -741,7 +803,7 @@ def parse(self, gram): gram (bytearray): memo gram from which to parse and strip its header. """ - cs, ms, ns, hs = self.Sizes[self.code] + cs, ms, vs, ns, hs, ss = self.Sizes[self.code] #"cs ms vs ns hs ss" wiff = self.wiff(gram) if wiff: # base2 binary mode @@ -1061,7 +1123,7 @@ def rend(self, memo): grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray - cs, ms, ns, hs = self.Sizes[self.code] + cs, ms, vs, ns, hs, ss = self.Sizes[self.code] # "cs ms vs ns hs ss" # self.size is max gram size bs = (self.size - hs) # max standard gram body size, mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 8e966e4..a8744cc 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -21,7 +21,8 @@ def test_memoer_basic(): assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode - assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + # (code, mid, vid, neck, head, sig) part sizes + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) assert peer.size == peer.MaxGramSize peer.reopen() @@ -154,7 +155,8 @@ def test_memogram_small_gram_size(): assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode - assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + # (code, mid, vid, neck, head, sig) part sizes + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) assert peer.size == 33 # can't be smaller than head + neck + 1 peer = memoing.Memoer(size=38) @@ -432,7 +434,8 @@ def test_tymeememogram_basic(): assert peer.opened == False assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode - assert peer.Sizes[peer.code] == (2, 22, 4, 28) # (code, mid, neck, head) size + # (code, mid, vid, neck, head, sig) part sizes + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) assert peer.size == peer.MaxGramSize peer.reopen() From fe20f7dac0caaa13c8316babca167587ed9994eb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 24 Feb 2025 15:30:28 -0700 Subject: [PATCH 26/33] started adding support for signed memo grams --- src/hio/core/memoing.py | 92 +++++++++++++++++++++++++------------- tests/core/test_memoing.py | 37 +++++++-------- 2 files changed, 77 insertions(+), 52 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 674498e..22a2a07 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -408,15 +408,18 @@ # namedtuple for gram header part size entries in Memoer code tables # cs is the code part size int number of chars in code part # ms is the mid part size int number of chars in the mid part (memoID) -# ns is the neck part size int number of chars for the gram number in all grams -# and the additional neck that also appears in first gram -# hs is the head size int number of chars hs = cs + ms + ns and for the first -# gram the head of size hs is followed by a neck of size ns. So the total -# overhead on first gram is os = hs + ns # vs is the vid part size int number of chars in the vid part (verification ID) +# ns is the neck part size int number of chars for the gram number in all grams +# and the additional neck that also appears in first gram for gram count # ss is the signature part size int number of chars in the signature part. +# the signature part is discontinuously attached after the body part but +# its size is included in over head computation for the body size +# hs is the head size int number of chars for other grams no-neck. +# hs = cs + ms + vs + ns + ss. The size for the first gram with neck is +# hs + ns -Sizage = namedtuple("Sizage", "cs ms vs ns hs ss") + +Sizage = namedtuple("Sizage", "cs ms vs ss ns hs") @dataclass(frozen=True) class GramCodes: @@ -554,10 +557,10 @@ class Memoer(hioing.Mixin): MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size MaxGramSize = 65535 # (2**16-1) absolute max gram size MaxGramCount = 16777215 # (2**24-1) absolute max gram count - # dict of gram header part sizes keyed by gram codes + # dict of gram header part sizes keyed by gram codes: cs ms vs ss ns hs Sizes = { - '__': Sizage(cs=2, ms=22, vs=0, ns=4, hs=28, ss=0), - '_-': Sizage(cs=2, ms=22, vs=44, ns=4, hs=72, ss=88), + '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), + '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), } @@ -705,10 +708,10 @@ def size(self, size): size (int | None): gram size for rending memo """ - _, _, _, ns, hs, ss = self.Sizes[self.code] # "cs ms vs ns hs ss" + _, _, _, _, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs size = size if size is not None else self.MaxGramSize # mininum size must be big enough for first gram header and 1 body byte - self._size = max(min(size, self.MaxGramSize), hs + ns + ss + 1) + self._size = max(min(size, self.MaxGramSize), hs + ns + 1) def open(self): @@ -803,7 +806,7 @@ def parse(self, gram): gram (bytearray): memo gram from which to parse and strip its header. """ - cs, ms, vs, ns, hs, ss = self.Sizes[self.code] #"cs ms vs ns hs ss" + cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs wiff = self.wiff(gram) if wiff: # base2 binary mode @@ -1122,34 +1125,63 @@ def rend(self, memo): """ grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray - - cs, ms, vs, ns, hs, ss = self.Sizes[self.code] # "cs ms vs ns hs ss" # self.size is max gram size - bs = (self.size - hs) # max standard gram body size, - mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload + cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + ps = (3 - ((ms) % 3)) % 3 # net pad size for mid + if ps != (cs % 4): # code + mid must lie on 24 bit boundary + raise hioing.MemoerError(f"Invalid combination of code size={cs}" + f" and mid size={ms}.") + + # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned + mid = encodeB64(bytes([0] * ps) + uuid.uuid4().bytes)[ps:] # prepad, convert, and strip + fore = self.code.encode() + mid # forehead of header + vid = b'A' * vs + sig = b'A' * ss ml = len(memo) + + if self.mode: # rend header parts in base2 instead of base64 + hs = 3 * hs // 4 # mode b2 means head part sizes smaller by 3/4 + ns = 3 * ns // 4 # mode b2 means head part sizes smaller by 3/4 + vs = 3 * vs // 4 # mode b2 means head part sizes smaller by 3/4 + ss = 3 * ss // 4 # mode b2 means head part sizes smaller by 3/4 + fore = decodeB64(fore) + vid = decodeB64(vid) + sig = decodeB64(sig) + + bs = (self.size - hs) # max standard gram body size without neck + # compute gram count based on overhead note added neck overhead in first gram + # first gram is special its header is longer by ns than the other grams + # which means its payload body is shorter by ns than the other gram bodies + # so take ml and subtract first payload size = ml - (bs-ns) to get the + # portion of memo carried by other grams. Divide this by bs rounded up + # (ceil) to get cnt of other grams and add 1 for the first gram to get + # total gram cnt. + # gc = ceil((ml-(bs-ns))/bs + 1) = ceil((ml-bs+ns)/bs + 1) + gc = math.ceil((ml-bs+ns)/bs+1) # includes added neck ns overhead + mms = min(self.MaxMemoSize, (bs * self.MaxGramCount) - ns) # max memo payload + if ml > mms: raise hioing.MemoerError(f"Memo length={ml} exceeds max={mms}.") - # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned - midb = encodeB64(bytes([0] * 2) + uuid.uuid4().bytes)[2:] # prepad, convert, and strip - #mid = midb.decode() - # compute gram count account for added neck overhead in first gram - gc = math.ceil((ml + ns - bs)/bs + 1) # includes added neck ns overhead - neck = helping.intToB64b(gc, l=ns) - if self.mode: # b2 mode binary - neck = decodeB64(neck) + if self.mode: + neck = gc.to_bytes(ns) + else: + neck = helping.intToB64b(gc, l=ns) + gn = 0 while memo: - gnb = helping.intToB64b(gn, l=ns) # gn size always == neck size - head = self.code.encode() + midb + gnb - if self.mode: # b2 binary - head = decodeB64(head) + if self.mode: + num = gn.to_bytes(ns) # num size must always be neck size + else: + num = helping.intToB64b(gn, l=ns) # num size must always be neck size + + head = fore + vid + num + if gn == 0: - gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end + gram = head + neck + memo[:bs-ns] + sig # copy slice past end just copies to end del memo[:bs-ns] # del slice past end just deletes to end else: - gram = head + memo[:bs] # copy slice past end just copies to end + gram = head + memo[:bs] + sig # copy slice past end just copies to end del memo[:bs] # del slice past end just deletes to end grams.append(gram) diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index a8744cc..724833b 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -22,7 +22,7 @@ def test_memoer_basic(): assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) + assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize peer.reopen() @@ -156,7 +156,7 @@ def test_memogram_small_gram_size(): assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) + assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == 33 # can't be smaller than head + neck + 1 peer = memoing.Memoer(size=38) @@ -210,6 +210,7 @@ def test_memogram_small_gram_size(): assert peer.rxms[0] == ('Hello There', 'beta') peer.serviceRxMemos() assert not peer.rxms + assert not peer.echos # send and receive via echo memo = "See ya later!" @@ -232,6 +233,7 @@ def test_memogram_small_gram_size(): assert not peer.counts assert not peer.sources peer.serviceReceives(echoic=True) + assert not peer.echos mid = list(peer.rxgs.keys())[0] assert len(peer.rxgs[mid]) == 2 assert peer.counts[mid] == 2 @@ -250,10 +252,10 @@ def test_memogram_small_gram_size(): # test binary q2 encoding of transmission gram header peer.mode = True # set to binary base2 - memo = "Hello There" + memo = 'See ya later alligator!' dst = "beta" peer.memoit(memo, dst) - assert peer.txms[0] == ('Hello There', 'beta') + assert peer.txms[0] == ('See ya later alligator!', 'beta') peer.serviceTxMemos() assert not peer.txms assert len(peer.txgs) == 2 @@ -268,43 +270,34 @@ def test_memogram_small_gram_size(): assert not peer.counts assert not peer.sources assert not peer.rxms - - headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAB').encode()) - gram = headneck + b"Hello There" - assert peer.wiff(gram) # base2 - echo = (gram, "beta") - peer.echos.append(echo) + assert not peer.echos b'__DFymLrtlZG6bp0HhlUsR6uAAAAAAACHello ' b'__DFymLrtlZG6bp0HhlUsR6uAAABThere' mid = 'DFymLrtlZG6bp0HhlUsR6u' headneck = decodeB64(('__' + mid + 'AAAA' + 'AAAC').encode()) - gram = headneck + b"Hello " + gram = headneck + b"See ya later a" assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) head = decodeB64(('__' + mid + 'AAAB').encode()) - gram = head + b"There" + gram = head + b"lligator!" assert peer.wiff(gram) # base2 echo = (gram, "beta") peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert not peer.echos assert len(peer.rxgs[mid]) == 2 assert peer.counts[mid] == 2 assert peer.sources[mid] == 'beta' - assert peer.rxgs[mid][0] == bytearray(b'Hello ') - assert peer.rxgs[mid][1] == bytearray(b'There') - peer.serviceReceives(echoic=True) - assert len(peer.rxgs[mid]) == 2 - assert peer.counts[mid] == 2 - assert peer.sources[mid] == 'beta' - assert peer.rxgs[mid][0] == bytearray(b'Hello ') - assert peer.rxgs[mid][1] == bytearray(b'There') + assert peer.rxgs[mid][0] == bytearray(b'See ya later a') + assert peer.rxgs[mid][1] == bytearray(b'lligator!') peer.serviceRxGrams() assert not peer.rxgs assert not peer.counts assert not peer.sources - assert peer.rxms[0] == ('Hello There', 'beta') + assert peer.rxms[0] == ('See ya later alligator!', 'beta') peer.serviceRxMemos() assert not peer.rxms @@ -435,7 +428,7 @@ def test_tymeememogram_basic(): assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode # (code, mid, vid, neck, head, sig) part sizes - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 28, 0) + assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize peer.reopen() From 88adf000709f4c1633c707c551c9a62388779a7b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Mon, 24 Feb 2025 20:21:44 -0700 Subject: [PATCH 27/33] Added support for stubs for signed grams. Still need more unit tests. --- src/hio/core/memoing.py | 206 ++++++++++++++++++++++++++----------- src/hio/help/helping.py | 126 +++++++++++++++++++++-- tests/core/test_memoing.py | 46 +++++++++ tests/help/test_helping.py | 93 +++++++++++++++++ 4 files changed, 403 insertions(+), 68 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 22a2a07..82ea48d 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -393,7 +393,7 @@ from contextlib import contextmanager from base64 import urlsafe_b64encode as encodeB64 from base64 import urlsafe_b64decode as decodeB64 -from dataclasses import dataclass, astuple +from dataclasses import dataclass, astuple, asdict from .. import hioing, help from ..base import tyming, doing @@ -422,9 +422,9 @@ Sizage = namedtuple("Sizage", "cs ms vs ss ns hs") @dataclass(frozen=True) -class GramCodes: +class GramCodex: """ - GramCodes is codex of all Gram Codes. + GramCodex is codex of all Gram Codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ @@ -435,7 +435,7 @@ def __iter__(self): return iter(astuple(self)) -GramDex = GramCodes() # Make instance +GramDex = GramCodex() # Make instance class Memoer(hioing.Mixin): @@ -493,10 +493,14 @@ class Memoer(hioing.Mixin): Class Attributes: Version (Versionage): default version consisting of namedtuple of form (major: int, minor: int) + Codex (GramDex): dataclass ref to gram codex + Codes (dict): maps codex names to codex values + Names (dict): maps codex values to codex names + Sizes (dict): gram head part sizes Sizage instances keyed by gram codes MaxMemoSize (int): absolute max memo size MaxGramSize (int): absolute max gram size on tx with overhead MaxGramCount (int): absolute max gram count - Sizes (dict): gram head part sizes Sizage instances keyed by gram codes + Inherited Attributes: name (str): unique name for Memoer transport. Used to manage. @@ -554,14 +558,18 @@ class Memoer(hioing.Mixin): """ Version = Versionage(major=0, minor=0) # default version - MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size - MaxGramSize = 65535 # (2**16-1) absolute max gram size - MaxGramCount = 16777215 # (2**24-1) absolute max gram count + Codex = GramDex + Codes = asdict(Codex) # map code name to code + Names = {val : key for key, val in Codes.items()} # invert map code to code name # dict of gram header part sizes keyed by gram codes: cs ms vs ss ns hs Sizes = { '__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160), } + #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) + MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size + MaxGramSize = 65535 # (2**16-1) absolute max gram size + MaxGramCount = 16777215 # (2**24-1) absolute max gram count def __init__(self, @@ -787,9 +795,31 @@ def wiff(self, gram): raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") - def parse(self, gram): - """Parse and strips header from gram bytearray and returns - (mid, gn, gc). Raises MemoerError is unrecognized header + def verify(self, sig, ser, vid): + """Verify signature sig on signed part of gram, ser, using key from vid. + Stub override in subclass to perform real signature verification + + Returns: + result (bool): True if signature verifies + False otherwise + + Parameters: + sig (bytes | str): qualified base64 qb64b + ser (bytes): signed portion of gram in delivered format + vid (bytes | str): qualified base64 qb64b of verifier ID + """ + if hasattr(sig, 'encode'): + sig = sig.encode() + + if hasattr(vid, 'encode'): + vid = vid.encode() + + return True + + + def pick(self, gram): + """Strips header from gram bytearray leaving only gram body in gram and + returns (mid, gn, gc). Raises MemoerError if unrecognized header Returns: result (tuple): tuple of form: @@ -805,50 +835,81 @@ def parse(self, gram): Parameters: gram (bytearray): memo gram from which to parse and strip its header. + """ - cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + mode = self.wiff(gram) # rx gram encoding mode True=B2 or False=B64 + if mode: # base2 binary mode + if len(gram) < 2: # assumes len(code) must be 2 + raise hioing.MemoerError(f"Gram length={len(gram)} to short to " + f"hold code.") + code = helping.codeB2ToB64(gram, 2) # assumes len(code) must be 2 + cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs + ps = (3 - ((ms) % 3)) % 3 # net pad size for mid + cms = 3 * (cs + ms) // 4 # cs + ms are aligned on 24 bit boundary + hs = 3 * hs // 4 # mode b2 means head part sizes smaller by 3/4 + ns = 3 * ns // 4 # mode b2 means head part sizes smaller by 3/4 + vs = 3 * vs // 4 # mode b2 means head part sizes smaller by 3/4 + ss = 3 * ss // 4 # mode b2 means head part sizes smaller by 3/4 + + if len(gram) < (hs + 1): # not big enough for non-first gram + raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" + f" < {hs + 1}.") + + mid = encodeB64(bytes([0] * ps) + gram[cs:cms])[ps:].decode() # prepad, convert, strip + vid = encodeB64(gram[cms:cms+vs]) # must be on 24 bit boundary + gn = int.from_bytes(gram[cms+vs:cms+vs+ns]) + if gn == 0: # first (zeroth) gram so get neck + if len(gram) < hs + ns + 1: + raise hioing.MemoerError(f"Not enough rx bytes for b2 " + f"gram < {hs + ns + 1}.") + neck = gram[cms+vs+ns:cms+vs+2*ns] # slice takes a copy + gc = int.from_bytes(neck) # convert to int + sig = encodeB64(gram[-ss if ss else len(gram):]) # last ss bytes are signature + del gram[-ss if ss else len(gram):] # strip sig + signed = bytes(gram[:]) # copy signed portion of gram + del gram[:hs-ss+ns] # strip of fore head leaving body in gram + else: # non-first gram no neck + gc = None + sig = encodeB64(gram[-ss if ss else len(gram):]) + del gram[-ss if ss else len(gram):] # strip sig + signed = bytes(gram[:]) # copy signed portion of gram + del gram[:hs-ss] # strip of fore head leaving body in gram - wiff = self.wiff(gram) - if wiff: # base2 binary mode - bhs = 3 * (hs) // 4 # binary head size - head = gram[:bhs] # slice takes a copy - if len(head) < bhs: # not enough bytes for head - raise hioing.MemoerError(f"Not enough rx bytes for base2 gram" - f"header < {bhs}.") - head = encodeB64(head) # convert to Base64 - del gram[:bhs] # strip head off gram else: # base64 text mode - head = gram[:hs] # slice takes a copy - if len(head) < hs: # not enough bytes for head - raise hioing.MemoerError(f"Not enough rx chars for base64 gram " - f"header < {hs}.") - del gram[:hs] # strip off head off - - code = head[:cs].decode() - if code != self.code: - raise hioing.MemoerError(f"Unrecognized gram {code=}.") - - mid = head[cs:cs+ms].decode() - gn = helping.b64ToInt(head[cs+ms:cs+ms+ns]) - - gc = None - if gn == 0: # first (zeroth) gram so get neck - if wiff: # base2 binary mode - bns = 3 * (ns) // 4 # binary neck size - neck = gram[:bns] # slice takes a copy - if len(neck) < bns: # not enough bytes for neck - raise hioing.MemoerError(f"Not enough rx bytes for base2" - f" gram neck < {bns}.") - neck = encodeB64(neck) # convert to Base64 - del gram[:bns] # strip off neck - else: # # base64 text mode - neck = gram[:ns] # slice takes a copy - if len(neck) < ns: # not enough bytes for neck - raise hioing.MemoerError(f"Not enough rx chars for base64" - f" gram neck < {ns}.") - del gram[:ns] # strip off neck - - gc = helping.b64ToInt(neck) + if len(gram) < 2: # assumes len(code) must be 2 + raise hioing.MemoerError(f"Gram length={len(gram)} to short to " + f"hold code.") + code = gram[:2].decode() # assumes len(code) must be 2 + cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs + + if len(gram) < (hs + 1): # not big enough for non-first gram + raise hioing.MemoerError(f"Not enough rx bytes for b64 gram" + f" < {hs + 1}.") + + mid = bytes(gram[cs:cs+ms]).decode() + vid = bytes(gram[cs+ms:cs+ms+vs]) # must be on 24 bit boundary + gn = helping.b64ToInt(gram[cs+ms+vs:cs+ms+vs+ns]) + if gn == 0: # first (zeroth) gram so get neck + if len(gram) < hs + ns + 1: + raise hioing.MemoerError(f"Not enough rx bytes for b64 " + f"gram < {hs + ns + 1}.") + neck = gram[cs+ms+vs+ns:cs+ms+vs+2*ns] # slice takes a copy + gc = helping.b64ToInt(neck) # convert to int + sig = bytes(gram[-ss if ss else len(gram):]) # last ss bytes are signature + del gram[-ss if ss else len(gram):] # strip sig + signed = bytes(gram[:]) # copy signed portion of gram + del gram[:hs-ss+ns] # strip of fore head leaving body in gram + else: # non-first gram no neck + gc = None + sig = bytes(gram[-ss if ss else len(gram):]) + del gram[-ss if ss else len(gram):] # strip sig + signed = bytes(gram[:]) # copy signed portion of gram + del gram[:hs-ss] # strip of fore head leaving body in gram + + if sig: # signature not empty + if not self.verify(sig, signed, vid): + raise hioing.MemoerError(f"Invalid signature on gram from " + f"verifier {vid=}.") return (mid, gn, gc) @@ -923,7 +984,7 @@ def _serviceOneReceived(self, *, echoic=False): gram = bytearray(gram)# make copy bytearray so can strip off header try: - mid, gn, gc = self.parse(gram) # parse and strip off head leaving body + mid, gn, gc = self.pick(gram) # parse and strip off head leaving body except hioing.MemoerError as ex: # invalid gram so drop logger.error("Unrecognized Memoer gram from %s.\n %s.", src, ex) return True # did receive data to can keep receiving @@ -1111,7 +1172,25 @@ def memoit(self, memo, dst): self.txms.append((memo, dst)) - def rend(self, memo): + def sign(self, ser, vid): + """Sign serialization ser using private key for verifier ID vid + Stub override in subclass to fetch private key for vid and sign + + Returns: + sig(bytes): qb64b qualified base64 representation of signature + + Parameters: + ser (bytes): signed portion of gram is delivered format + vid (str | bytes): qualified base64 qb64 of verifier ID + + """ + if hasattr(vid, 'encode'): + vid = vid.encode() + cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs + return b'A' * ss + + + def rend(self, memo, *, vid=None): """Partition memo into packed grams with headers. Returns: @@ -1119,6 +1198,7 @@ def rend(self, memo): Parameters: memo (str): to be partitioned into grams with headers + vid (str) Note first gram has head + neck overhead, hs + ns so bs is smaller by ns non-first grams have just head overhead hs so bs is bigger by ns @@ -1128,15 +1208,11 @@ def rend(self, memo): # self.size is max gram size cs, ms, vs, ss, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs ps = (3 - ((ms) % 3)) % 3 # net pad size for mid - if ps != (cs % 4): # code + mid must lie on 24 bit boundary - raise hioing.MemoerError(f"Invalid combination of code size={cs}" - f" and mid size={ms}.") + vid = vid if vid is not None else b'A' * vs # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned mid = encodeB64(bytes([0] * ps) + uuid.uuid4().bytes)[ps:] # prepad, convert, and strip fore = self.code.encode() + mid # forehead of header - vid = b'A' * vs - sig = b'A' * ss ml = len(memo) if self.mode: # rend header parts in base2 instead of base64 @@ -1146,7 +1222,6 @@ def rend(self, memo): ss = 3 * ss // 4 # mode b2 means head part sizes smaller by 3/4 fore = decodeB64(fore) vid = decodeB64(vid) - sig = decodeB64(sig) bs = (self.size - hs) # max standard gram body size without neck # compute gram count based on overhead note added neck overhead in first gram @@ -1178,11 +1253,18 @@ def rend(self, memo): head = fore + vid + num if gn == 0: - gram = head + neck + memo[:bs-ns] + sig # copy slice past end just copies to end + gram = head + neck + memo[:bs-ns] # copy slice past end just copies to end del memo[:bs-ns] # del slice past end just deletes to end else: - gram = head + memo[:bs] + sig # copy slice past end just copies to end + gram = head + memo[:bs] # copy slice past end just copies to end del memo[:bs] # del slice past end just deletes to end + + if ss: # sign + sig = self.sign(gram, vid) + if mode: + sig = decodeB64(sig) + gram = gram + sig + grams.append(gram) gn += 1 diff --git a/src/hio/help/helping.py b/src/hio/help/helping.py index 62e01bc..84a122f 100644 --- a/src/hio/help/helping.py +++ b/src/hio/help/helping.py @@ -22,6 +22,32 @@ import cbor2 as cbor +# Utilities +def isign(i): + """ + Integer sign function + Returns: + (int): 1 if i > 0, -1 if i < 0, 0 otherwise + + """ + return (1 if i > 0 else -1 if i < 0 else 0) + + +def sceil(r): + """ + Symmetric ceiling function + Returns: + sceil (int): value that is symmetric ceiling of r away from zero + + Because int() provides a symmetric floor towards zero, just inc int(r) by: + 1 when r - int(r) > 0 (r positive) + -1 when r - int(r) < 0 (r negative) + 0 when r - int(r) == 0 (r integral already) + abs(r) > abs(int(r) or 0 when abs(r) + """ + return (int(r) + isign(r - int(r))) + + def copyfunc(f, name=None): """ Copy a function in detail. @@ -470,18 +496,26 @@ def intToB64b(i, l=1): Returns: b64 (bytes): Base64 converstion of i of length minimum l. If more than l bytes are needed to represent i in Base64 then returned - bytes is lengthed appropriately. When less then l bytes + bytes is extended appropriately. When less then l bytes is needed then returned bytes is prepadded with b'A' bytes. - l is min number of b64 digits left padded with Base64 0 == b"A" bytes - The length of return bytes extended to accomodate full Base64 encoding of i + Parameters: + i (int): to be converted + l (int): min number of b64 digits. When empty these are left padded with + Base64 0 == b'A' digits. + The length of return bytes is extended to accomodate full + Base64 encoding of i regardless of l. """ - return (intToB64(i=i, l=l).encode("utf-8")) + return (intToB64(i=i, l=l).encode()) def b64ToInt(s): - """ - Returns conversion of Base64 str s or bytes to int + """Converts Base64 to int + Returns: + i (int): conversion s (str | bytes) to int + + Parameters: + s (str | bytes): to be converted """ if not s: raise ValueError("Empty string, conversion undefined.") @@ -491,3 +525,83 @@ def b64ToInt(s): for e, c in enumerate(reversed(s)): i |= B64IdxByChr[c] << (e * 6) # same as i += B64IdxByChr[c] * (64 ** e) return i + + + +def codeB64ToB2(s): + """Convert Base64 chars in s to B2 bytes + + Returns: + bs (bytes): conversion (decode) of s Base64 chars to Base2 bytes. + Where the number of total bytes returned is equal to the minimun number of + chars (octet) sufficient to hold the total converted concatenated chars from s, + with one sextet per each Base64 char of s. Assumes no pad chars in s. + + + Sextets are left aligned with pad bits in last (rightmost) byte to support + mid padding of code portion with respect to rest of primitive. + This is useful for decoding as bytes, code characters from the front of + a Base64 encoded string of characters. + + Parameters: + s (str | bytes): Base64 str or bytes to convert + + """ + i = b64ToInt(s) + i <<= 2 * (len(s) % 4) # add 2 bits right zero padding for each sextet + n = sceil(len(s) * 3 / 4) # compute min number of ocetets to hold all sextets + return (i.to_bytes(n, 'big')) + + +def codeB2ToB64(b, l): + """Convert l sextets from base2 b to base64 str + + Returns: + code (str): conversion (encode) of l Base2 sextets from front of b + to Base64 chars. + + One char for each of l sextets from front (left) of b. + This is useful for encoding as code characters, sextets from the front of + a Base2 bytes (byte string). Must provide l because of ambiguity between l=3 + and l=4. Both require 3 bytes in b. Trailing pad bits are removed so + returned sextets as characters are right aligned . + + Parameters: + b (bytes | str): target from which to nab sextets + l (int): number of sextets to convert from front of b + """ + if hasattr(b, 'encode'): + b = b.encode("utf-8") # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') # convert only first n bytes to int + # check if prepad bits are zero + tbs = 2 * (l % 4) # trailing bit size in bits + i >>= tbs # right shift out trailing bits to make right aligned + return (intToB64(i, l)) # return as B64 + + +def nabSextets(b, l): + """Nab l sextets from front of b + Returns: + sextets (bytes): first l sextets from front (left) of b as bytes + (byte string). Length of bytes returned is minimum sufficient to hold + all l sextets. Last byte returned is right bit padded with zeros which + is compatible with mid padded codes on front of primitives + + Parameters: + b (bytes | str): target from which to nab sextets + l (int): number of sextets to nab from front of b + + """ + if hasattr(b, 'encode'): + b = b.encode() # convert to bytes + n = sceil(l * 3 / 4) # number of bytes needed for l sextets + if n > len(b): + raise ValueError("Not enough bytes in {} to nab {} sextets.".format(b, l)) + i = int.from_bytes(b[:n], 'big') + p = 2 * (l % 4) + i >>= p # strip of last bits + i <<= p # pad with empty bits + return (i.to_bytes(n, 'big')) diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 724833b..8a0b063 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -10,6 +10,51 @@ from hio.base import doing, tyming from hio.core import memoing +from hio.core.memoing import Memoer, Versionage, Sizage + +def test_memoer_class(): + """Test class attributes of Memoer class""" + + assert Memoer.Version == Versionage(major=0, minor=0) + assert Memoer.Codex == memoing.GramDex + + assert Memoer.Codes == {'Basic': '__', 'Signed': '_-'} + assert Memoer.Names == {'__': 'Basic', '_-': 'Signed'} + + + # Codes table with sizes of code (hard) and full primitive material + assert Memoer.Sizes == {'__': Sizage(cs=2, ms=22, vs=0, ss=0, ns=4, hs=28), + '_-': Sizage(cs=2, ms=22, vs=44, ss=88, ns=4, hs=160)} + + + # verify all Codes + for code, val in Memoer.Sizes.items(): + cs = val.cs + ms = val.ms + vs = val.vs + ss = val.ss + ns = val.ns + hs = val.hs + + assert len(code) == cs == 2 + assert code[0] == '_' + code[1] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_' + assert hs > 0 + assert hs == cs + ms + vs + ss + ns + ps = (3 - ((ms) % 3)) % 3 # net pad size for mid + assert ps == (cs % 4) # combined code + mid size must lie on 24 bit boundary + assert not vs % 4 # vid size must be on 24 bit boundary + assert not ss % 4 # sig size must be on 24 bit boundary + assert not ns % 4 # neck (num or cnt) size must be on 24 bit boundary + assert not hs % 4 # head size must be on 24 bit boundary + + #assert Memoer.Bodes == {b'\xff\xf0': '__', b'\xff\xe0': '_-'} + + assert Memoer.MaxMemoSize == (2**32-1) # absolute max memo payload size + assert Memoer.MaxGramSize == (2**16-1) # absolute max gram size + assert Memoer.MaxGramCount == (2**24-1) # absolute max gram count + + """Done Test""" def test_memoer_basic(): @@ -612,6 +657,7 @@ def test_tymeememogram_doer(): if __name__ == "__main__": + test_memoer_class() test_memoer_basic() test_memogram_small_gram_size() test_memoer_multiple() diff --git a/tests/help/test_helping.py b/tests/help/test_helping.py index 639dbb1..9aa4100 100644 --- a/tests/help/test_helping.py +++ b/tests/help/test_helping.py @@ -3,9 +3,50 @@ tests.help.test_helping module """ +import fractions + import pytest from hio.help import helping, Hict +from hio.help.helping import isign, sceil + +def test_utilities(): + """ + Test utility functions + """ + assert isign(1) == 1 + assert isign(-1) == -1 + assert isign(0) == 0 + assert isign(2) == 1 + assert isign(-1) == -1 + + assert isign(1.0) == 1 + assert isign(-1.0) == -1 + assert isign(0.0) == 0 + + assert isign(1.5) == 1 + assert isign(-1.5) == -1 + assert isign(0.5) == 1 + assert isign(-0.5) == -1 + + + assert sceil(0.5) == 1 + assert sceil(-0.5) == -1 + assert sceil(1) == 1 + assert sceil(-1) == -1 + assert sceil(0) == 0 + assert sceil(0.0) == 0 + + assert sceil(1.1) == 2 + assert sceil(-1.1) == -2 + assert sceil(2.8) == 3 + assert sceil(-2.8) == -3 + + assert sceil(fractions.Fraction(3, 2)) == 2 + assert sceil(fractions.Fraction(-3, 2)) == -2 + assert sceil(fractions.Fraction(0)) == 0 + + def test_copy_func(): """ @@ -373,6 +414,57 @@ def test_b64_helpers(): i = helping.b64ToInt(cs) assert i == 6011 + s = "-BAC" + b = helping.codeB64ToB2(s[:]) + assert len(b) == 3 + assert b == b'\xf8\x10\x02' + t = helping.codeB2ToB64(b, 4) + assert t == s[:] + i = int.from_bytes(b, 'big') + assert i == 0o76010002 + i >>= 2 * (len(s) % 4) + assert i == 0o76010002 + p = helping.nabSextets(b, 4) + assert p == b'\xf8\x10\x02' + + b = helping.codeB64ToB2(s[:3]) + assert len(b) == 3 + assert b == b'\xf8\x10\x00' + t = helping.codeB2ToB64(b, 3) + assert t == s[:3] + i = int.from_bytes(b, 'big') + assert i == 0o76010000 + i >>= 2 * (len(s[:3]) % 4) + assert i == 0o760100 + p = helping.nabSextets(b, 3) + assert p == b'\xf8\x10\x00' + + b = helping.codeB64ToB2(s[:2]) + assert len(b) == 2 + assert b == b'\xf8\x10' + t = helping.codeB2ToB64(b, 2) + assert t == s[:2] + i = int.from_bytes(b, 'big') + assert i == 0o174020 + i >>= 2 * (len(s[:2]) % 4) + assert i == 0o7601 + p = helping.nabSextets(b, 2) + assert p == b'\xf8\x10' + + b = helping.codeB64ToB2(s[:1]) + assert len(b) == 1 + assert b == b'\xf8' + t = helping.codeB2ToB64(b, 1) + assert t == s[:1] + i = int.from_bytes(b, 'big') + assert i == 0o370 + i >>= 2 * (len(s[:1]) % 4) + assert i == 0o76 + p = helping.nabSextets(b, 1) + assert p == b'\xf8' + + + text = b"-A-Bg-1-3-cd" match = helping.Reb64.match(text) assert match @@ -393,5 +485,6 @@ def test_b64_helpers(): if __name__ == "__main__": + test_utilities() test_attributize() test_b64_helpers() From 17de212e80485a4a575eaed6f16ae49b7e51c458 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 10:25:52 -0700 Subject: [PATCH 28/33] unit tests for signed grams and some clean up and resultant fixes --- src/hio/core/memoing.py | 30 +++- tests/core/test_memoing.py | 274 ++++++++++++++++++++++++++++++++++++- 2 files changed, 296 insertions(+), 8 deletions(-) diff --git a/src/hio/core/memoing.py b/src/hio/core/memoing.py index 82ea48d..9b30550 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memoing.py @@ -545,6 +545,7 @@ class Memoer(hioing.Mixin): _size (int): see size property + Properties: code (bytes | None): gram code for gram header when rending for tx mode (bool): True means when rending for tx encode header in base2 @@ -556,6 +557,7 @@ class Memoer(hioing.Mixin): Gram size also limited by MaxGramSize and MaxGramCount relative to MaxMemoSize. + """ Version = Versionage(major=0, minor=0) # default version Codex = GramDex @@ -632,8 +634,6 @@ def __init__(self, Min gram body size is one. Gram size also limited by MaxGramSize and MaxGramCount relative to MaxMemoSize. - - """ # initialize attributes @@ -649,8 +649,8 @@ def __init__(self, self.echos = deque() # only used in testing as echoed tx - self._code = code - self._mode = mode + self.code = code + self.mode = mode self.size = size # property sets size given .code and constraints super(Memoer, self).__init__(name, bc, **kwa) @@ -674,6 +674,20 @@ def code(self): """ return self._code + @code.setter + def code(self, code): + """Property setter for ._code + + Paramaters: + code (str): two char base64 gram code + """ + if code not in self.Codex: + raise hioing.MemoerError(f"Invalid {code=}.") + + self._code = code + if hasattr(self, "_size"): + self.size = self._size # refresh size given new code + @property def mode(self): """Property getter for ._mode @@ -693,6 +707,8 @@ def mode(self, mode): False means when rending for tx encode header in base64 """ self._mode = mode + if hasattr(self, "_size"): + self.size = self._size # refresh size given new mode @property def size(self): @@ -718,6 +734,10 @@ def size(self, size): """ _, _, _, _, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs size = size if size is not None else self.MaxGramSize + if self.mode: # minimum header smaller when in base2 mode + hs = 3 * hs // 4 + ns = 3 * ns // 4 + # mininum size must be big enough for first gram header and 1 body byte self._size = max(min(size, self.MaxGramSize), hs + ns + 1) @@ -1261,7 +1281,7 @@ def rend(self, memo, *, vid=None): if ss: # sign sig = self.sign(gram, vid) - if mode: + if self.mode: sig = decodeB64(sig) gram = gram + sig diff --git a/tests/core/test_memoing.py b/tests/core/test_memoing.py index 8a0b063..e992521 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/test_memoing.py @@ -8,9 +8,10 @@ import pytest +from hio.help import helping from hio.base import doing, tyming from hio.core import memoing -from hio.core.memoing import Memoer, Versionage, Sizage +from hio.core.memoing import Memoer, Versionage, Sizage, GramDex def test_memoer_class(): """Test class attributes of Memoer class""" @@ -66,7 +67,7 @@ def test_memoer_basic(): assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode - # (code, mid, vid, neck, head, sig) part sizes + # (code, mid, vid, sig, neck, head) part sizes assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize @@ -150,6 +151,7 @@ def test_memoer_basic(): # test binary q2 encoding of transmission gram header peer.mode = True # set to binary base2 + assert peer.mode memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -200,7 +202,7 @@ def test_memogram_small_gram_size(): assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.mode - # (code, mid, vid, neck, head, sig) part sizes + # (code, mid, vid, sig, neck, head) part sizes assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == 33 # can't be smaller than head + neck + 1 @@ -297,6 +299,8 @@ def test_memogram_small_gram_size(): # test binary q2 encoding of transmission gram header peer.mode = True # set to binary base2 + assert peer.mode + assert peer.size == 38 memo = 'See ya later alligator!' dst = "beta" peer.memoit(memo, dst) @@ -419,7 +423,269 @@ def test_memoer_multiple(): assert peer.opened == False """ End Test """ +def test_memoer_basic_signed(): + """Test Memoer class basic signed code + """ + peer = memoing.Memoer(code=GramDex.Signed) + assert peer.name == "main" + assert peer.opened == False + assert peer.bc == 64 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.mode + # (code, mid, vid, sig, neck, head) part sizes + assert peer.Sizes[peer.code] == (2, 22, 44, 88, 4, 160) # cs ms vs ss ns hs + assert peer.size == peer.MaxGramSize + + peer.reopen() + assert peer.opened == True + + assert not peer.txms + assert not peer.txgs + assert peer.txbs == (b'', None) + peer.service() + assert peer.txbs == (b'', None) + + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + g, d = peer.txgs[0] + assert not peer.wiff(g) # base64 + assert g.find(memo.encode()) != -1 + assert len(g) == 160 + 4 + len(memo) + assert g[:2].decode() == GramDex.Signed + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + gram = ('_-' + mid + 'AAAA' + ('A' * 44) + 'AAAB' + "Hello There" + ('A' * 88)).encode() + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # send and receive via echo + memo = "See ya later!" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('See ya later!', 'beta') + peer.serviceTxMemos() + assert not peer.txms + g, d = peer.txgs[0] + assert not peer.wiff(g) # base64 + assert g.find(memo.encode()) != -1 + assert len(g) == 160 + 4 + len(memo) + assert g[:2].decode() == GramDex.Signed + assert d == dst == 'beta' + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert peer.echos + + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + peer.serviceReceives(echoic=True) + mid = list(peer.rxgs.keys())[0] + assert peer.rxgs[mid][0] == b'See ya later!' + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' + assert not peer.echos + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + peer.rxms[0] + assert peer.rxms[0] == ('See ya later!', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # test binary q2 encoding of transmission gram header + peer.mode = True # set to binary base2 + memo = "Hello There" + dst = "beta" + peer.memoit(memo, dst) + assert peer.txms[0] == ('Hello There', 'beta') + peer.serviceTxMemos() + assert not peer.txms + g, d = peer.txgs[0] + assert peer.wiff(g) # base64 + assert g.find(memo.encode()) != -1 + assert len(g) == 3 * (160 + 4) // 4 + len(memo) + assert helping.codeB2ToB64(g, 2) == GramDex.Signed + assert d == dst == 'beta' + peer.serviceTxGrams() + assert not peer.txgs + assert peer.txbs == (b'', None) + + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert not peer.rxms + mid = 'ALBI68S1ZIxqwFOSWFF1L2' + head = decodeB64(('_-' + mid + 'AAAA' + ('A' * 44) + 'AAAB').encode()) + tail = decodeB64(('A' * 88).encode()) + gram = head + memo.encode() + tail + assert peer.wiff(gram) # base2 + assert len(gram) == 3 * (160 + 4) // 4 + len(memo) + echo = (gram, "beta") + peer.echos.append(echo) + peer.serviceReceives(echoic=True) + assert peer.rxgs[mid][0] == bytearray(b'Hello There') + assert peer.counts[mid] == 1 + assert peer.sources[mid] == 'beta' + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert peer.rxms[0] == ('Hello There', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + peer.close() + assert peer.opened == False + """ End Test """ + +def test_memoer_multiple_signed(): + """Test Memoer class with small gram size and multiple queued memos signed + """ + peer = memoing.Memoer(code=GramDex.Signed, size=170) + assert peer.size == 170 + assert peer.name == "main" + assert peer.opened == False + assert peer.bc == 64 + assert peer.code == memoing.GramDex.Signed == '_-' + assert not peer.mode + + peer.reopen() + assert peer.opened == True + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 4 + for g, d in peer.txgs: + assert not peer.wiff(g) # base64 + assert d in ("alpha", "beta") + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceReceives(echoic=True) + assert not peer.echos + assert len(peer.rxgs) == 2 + assert len(peer.counts) == 2 + assert len(peer.sources) == 2 + + mid = list(peer.rxgs.keys())[0] + assert peer.sources[mid] == 'alpha' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'there.') + + mid = list(peer.rxgs.keys())[1] + assert peer.sources[mid] == 'beta' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'How ya') + assert peer.rxgs[mid][1] == bytearray(b' doing?') + + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert len(peer.rxms) == 2 + assert peer.rxms[0] == ('Hello there.', 'alpha') + assert peer.rxms[1] == ('How ya doing?', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + # test in base2 mode + peer.mode = True + assert peer.mode + peer.size = 129 + assert peer.size == 129 + + # send and receive multiple via echo + peer.memoit("Hello there.", "alpha") + peer.memoit("How ya doing?", "beta") + assert len(peer.txms) == 2 + peer.serviceTxMemos() + assert not peer.txms + assert len(peer.txgs) == 4 + for g, d in peer.txgs: + assert peer.wiff(g) # base64 + assert d in ("alpha", "beta") + peer.serviceTxGrams(echoic=True) + assert not peer.txgs + assert peer.txbs == (b'', None) + assert len(peer.echos) == 4 + assert not peer.rxgs + assert not peer.rxms + assert not peer.counts + assert not peer.sources + + peer.serviceReceives(echoic=True) + assert not peer.echos + assert len(peer.rxgs) == 2 + assert len(peer.counts) == 2 + assert len(peer.sources) == 2 + + mid = list(peer.rxgs.keys())[0] + assert peer.sources[mid] == 'alpha' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'Hello ') + assert peer.rxgs[mid][1] == bytearray(b'there.') + + mid = list(peer.rxgs.keys())[1] + assert peer.sources[mid] == 'beta' + assert peer.counts[mid] == 2 + assert len(peer.rxgs[mid]) == 2 + assert peer.rxgs[mid][0] == bytearray(b'How ya') + assert peer.rxgs[mid][1] == bytearray(b' doing?') + + peer.serviceRxGrams() + assert not peer.rxgs + assert not peer.counts + assert not peer.sources + assert len(peer.rxms) == 2 + assert peer.rxms[0] == ('Hello there.', 'alpha') + assert peer.rxms[1] == ('How ya doing?', 'beta') + peer.serviceRxMemos() + assert not peer.rxms + + + peer.close() + assert peer.opened == False + """ End Test """ def test_open_memoer(): @@ -661,6 +927,8 @@ def test_tymeememogram_doer(): test_memoer_basic() test_memogram_small_gram_size() test_memoer_multiple() + test_memoer_basic_signed() + test_memoer_multiple_signed() test_open_memoer() test_memogram_doer() test_tymeememogram_basic() From e2d493b8f82b19626374192405b32c467695a744 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 15:00:00 -0700 Subject: [PATCH 29/33] refactored memoing as package and then created subclass of UXD Peer and Memoer with tests --- src/hio/base/filing.py | 7 +- src/hio/core/memo/__init__.py | 10 ++ src/hio/core/{ => memo}/memoing.py | 125 ++++++++++++---------- src/hio/core/udp/udping.py | 8 +- src/hio/core/uxd/__init__.py | 2 +- src/hio/core/uxd/peermemoing.py | 53 ++++++++++ src/hio/core/uxd/uxding.py | 110 +++++++++---------- tests/core/memo/__init__.py | 4 + tests/core/{ => memo}/test_memoing.py | 32 +++--- tests/core/uxd/test_peer_memoer.py | 145 ++++++++++++++++++++++++++ 10 files changed, 363 insertions(+), 133 deletions(-) create mode 100644 src/hio/core/memo/__init__.py rename src/hio/core/{ => memo}/memoing.py (95%) create mode 100644 src/hio/core/uxd/peermemoing.py create mode 100644 tests/core/memo/__init__.py rename tests/core/{ => memo}/test_memoing.py (98%) create mode 100644 tests/core/uxd/test_peer_memoer.py diff --git a/src/hio/base/filing.py b/src/hio/base/filing.py index c9841ac..8562c5f 100644 --- a/src/hio/base/filing.py +++ b/src/hio/base/filing.py @@ -381,6 +381,7 @@ def remake(self, *, name="", base="", temp=None, headDirPath=None, perm=None, os.makedirs(head) if filed: file = ocfn(path, mode=mode, perm=perm) + else: os.makedirs(path) else: @@ -403,13 +404,15 @@ def remake(self, *, name="", base="", temp=None, headDirPath=None, perm=None, os.makedirs(head) if filed: file = ocfn(path, mode=mode, perm=perm) + else: os.makedirs(path) else: if filed: file = ocfn(path, mode=mode, perm=perm) - os.chmod(path, perm) # set dir/file permissions + if not extensioned: + os.chmod(path, perm) # set dir/file permissions return path, file @@ -501,7 +504,7 @@ def close(self, clear=False): if clear: self._clearPath() - return self.opened + return not self.opened # True means closed False means still opened def _clearPath(self): diff --git a/src/hio/core/memo/__init__.py b/src/hio/core/memo/__init__.py new file mode 100644 index 0000000..205bc75 --- /dev/null +++ b/src/hio/core/memo/__init__.py @@ -0,0 +1,10 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.memo Package +""" + + +from .memoing import (Versionage, Sizage, GramDex, + openMemoer, Memoer, MemoerDoer, + openTM, TymeeMemoer, TymeeMemoerDoer) + diff --git a/src/hio/core/memoing.py b/src/hio/core/memo/memoing.py similarity index 95% rename from src/hio/core/memoing.py rename to src/hio/core/memo/memoing.py index 9b30550..ee8669a 100644 --- a/src/hio/core/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -371,7 +371,7 @@ ToDo: -Make gram body size calculation a function of the current .mode for header +Make gram body size calculation a function of the current .curt for header encoding as b2 headers are smaller so the body payload is bigger. Add code '_-' for signed grams vs verification id size ss signatures size @@ -395,9 +395,9 @@ from base64 import urlsafe_b64decode as decodeB64 from dataclasses import dataclass, astuple, asdict -from .. import hioing, help -from ..base import tyming, doing -from ..help import helping +from ... import hioing, help +from ...base import tyming, doing +from ...help import helping logger = help.ogler.getLogger() @@ -489,6 +489,8 @@ class Memoer(hioing.Mixin): Memo segmentation/desegmentation information is embedded in the grams. + Inherited Class Attributes: + MaxGramSize (int): absolute max gram size on tx with overhead Class Attributes: Version (Versionage): default version consisting of namedtuple of form @@ -498,7 +500,6 @@ class Memoer(hioing.Mixin): Names (dict): maps codex values to codex names Sizes (dict): gram head part sizes Sizage instances keyed by gram codes MaxMemoSize (int): absolute max memo size - MaxGramSize (int): absolute max gram size on tx with overhead MaxGramCount (int): absolute max gram count @@ -541,14 +542,14 @@ class Memoer(hioing.Mixin): Hidden: _code (bytes | None): see size property - _mode (bool): see mode property + _curt (bool): see curt property _size (int): see size property Properties: code (bytes | None): gram code for gram header when rending for tx - mode (bool): True means when rending for tx encode header in base2 + curt (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 size (int): gram size when rending for tx. first gram size = over head size + neck size + body size. @@ -570,11 +571,11 @@ class Memoer(hioing.Mixin): } #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) MaxMemoSize = 4294967295 # (2**32-1) absolute max memo payload size - MaxGramSize = 65535 # (2**16-1) absolute max gram size MaxGramCount = 16777215 # (2**24-1) absolute max gram count + MaxGramSize = 65535 # (2**16-1) Overridden in subclass - def __init__(self, + def __init__(self, *, name=None, bc=None, version=None, @@ -586,7 +587,7 @@ def __init__(self, txgs=None, txbs=None, code=GramDex.Basic, - mode=False, + curt=False, size=None, **kwa ): @@ -626,7 +627,7 @@ def __init__(self, keep trying. Nothing to send indicated by (bytearray(), None) for (gram, dst) code (bytes): gram code for gram header - mode (bool): True means when rending for tx encode header in base2 + curt (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 size (int): gram size when rending for tx. first gram size = head size + neck size + body size. @@ -650,10 +651,10 @@ def __init__(self, self.echos = deque() # only used in testing as echoed tx self.code = code - self.mode = mode + self.curt = curt self.size = size # property sets size given .code and constraints - super(Memoer, self).__init__(name, bc, **kwa) + super(Memoer, self).__init__(name=name, bc=bc, **kwa) if not hasattr(self, "name"): # stub so mixin works in isolation. self.name = name if name is not None else "main" # mixed with subclass should provide this. @@ -689,26 +690,26 @@ def code(self, code): self.size = self._size # refresh size given new code @property - def mode(self): - """Property getter for ._mode + def curt(self): + """Property getter for ._curt Returns: - mode (bool): True means when rending for tx encode header in base2 + curt (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 """ - return self._mode + return self._curt - @mode.setter - def mode(self, mode): - """Property setter for ._mode + @curt.setter + def curt(self, curt): + """Property setter for ._curt Paramaters: - mode (bool): True means when rending for tx encode header in base2 + curt (bool): True means when rending for tx encode header in base2 False means when rending for tx encode header in base64 """ - self._mode = mode + self._curt = curt if hasattr(self, "_size"): - self.size = self._size # refresh size given new mode + self.size = self._size # refresh size given new curt @property def size(self): @@ -734,7 +735,7 @@ def size(self, size): """ _, _, _, _, ns, hs = self.Sizes[self.code] # cs ms vs ss ns hs size = size if size is not None else self.MaxGramSize - if self.mode: # minimum header smaller when in base2 mode + if self.curt: # minimum header smaller when in base2 curt hs = 3 * hs // 4 ns = 3 * ns // 4 @@ -753,6 +754,8 @@ def open(self): def reopen(self): """Idempotently open transport + + This is a stub. Override in transport specific subclass """ self.close() return self.open() @@ -767,13 +770,13 @@ def close(self): def wiff(self, gram): - """Determines encoding mode of gram bytes header when parsing grams. - The mode maybe either base2 or base64. + """Determines encoding of gram bytes header when parsing grams. + The encoding maybe either base2 or base64. Returns: - mode (bool): True means base2 - False means base64 + curt (bool): True means base2 encoding + False means base64 encoding Otherwise raises hioing.MemoerError All gram head codes start with '_' in base64 text or in base2 binary. @@ -808,16 +811,17 @@ def wiff(self, gram): sextet = gram[0] >> 2 if sextet == 0o27: - return False # base64 text mode + return False # base64 text encoding if sextet == 0o77: - return True # base2 binary mode + return True # base2 binary encoding raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") def verify(self, sig, ser, vid): """Verify signature sig on signed part of gram, ser, using key from vid. - Stub override in subclass to perform real signature verification + Must be overriden in subclass to perform real signature verification. + This is a stub. Returns: result (bool): True if signature verifies @@ -857,8 +861,8 @@ def pick(self, gram): """ - mode = self.wiff(gram) # rx gram encoding mode True=B2 or False=B64 - if mode: # base2 binary mode + curt = self.wiff(gram) # rx gram encoding True=B2 or False=B64 + if curt: # base2 binary encoding if len(gram) < 2: # assumes len(code) must be 2 raise hioing.MemoerError(f"Gram length={len(gram)} to short to " f"hold code.") @@ -866,10 +870,10 @@ def pick(self, gram): cs, ms, vs, ss, ns, hs = self.Sizes[code] # cs ms vs ss ns hs ps = (3 - ((ms) % 3)) % 3 # net pad size for mid cms = 3 * (cs + ms) // 4 # cs + ms are aligned on 24 bit boundary - hs = 3 * hs // 4 # mode b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # mode b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # mode b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # mode b2 means head part sizes smaller by 3/4 + hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 + ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 + vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 + ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 if len(gram) < (hs + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" @@ -895,7 +899,7 @@ def pick(self, gram): signed = bytes(gram[:]) # copy signed portion of gram del gram[:hs-ss] # strip of fore head leaving body in gram - else: # base64 text mode + else: # base64 text encoding if len(gram) < 2: # assumes len(code) must be 2 raise hioing.MemoerError(f"Gram length={len(gram)} to short to " f"hold code.") @@ -935,8 +939,9 @@ def pick(self, gram): def receive(self, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. Must be - overridden in subclass. This is a stub to define mixin interface. + """Attemps to send bytes in txbs to remote destination dst. + Must be overridden in subclass. + This is a stub to define mixin interface. Parameters: echoic (bool): True means use .echos in .receive debugging purposes @@ -1164,8 +1169,9 @@ def serviceAllRx(self): def send(self, txbs, dst, *, echoic=False): - """Attemps to send bytes in txbs to remote destination dst. Must be - overridden in subclass. This is a stub to define mixin interface. + """Attemps to send bytes in txbs to remote destination dst. + Must be overridden in subclass. + This is a stub to define mixin interface. Returns: count (int): bytes actually sent @@ -1194,7 +1200,8 @@ def memoit(self, memo, dst): def sign(self, ser, vid): """Sign serialization ser using private key for verifier ID vid - Stub override in subclass to fetch private key for vid and sign + Must be overriden in subclass to fetch private key for vid and sign. + This is a stub. Returns: sig(bytes): qb64b qualified base64 representation of signature @@ -1235,11 +1242,11 @@ def rend(self, memo, *, vid=None): fore = self.code.encode() + mid # forehead of header ml = len(memo) - if self.mode: # rend header parts in base2 instead of base64 - hs = 3 * hs // 4 # mode b2 means head part sizes smaller by 3/4 - ns = 3 * ns // 4 # mode b2 means head part sizes smaller by 3/4 - vs = 3 * vs // 4 # mode b2 means head part sizes smaller by 3/4 - ss = 3 * ss // 4 # mode b2 means head part sizes smaller by 3/4 + if self.curt: # rend header parts in base2 instead of base64 + hs = 3 * hs // 4 # encoding b2 means head part sizes smaller by 3/4 + ns = 3 * ns // 4 # encoding b2 means head part sizes smaller by 3/4 + vs = 3 * vs // 4 # encoding b2 means head part sizes smaller by 3/4 + ss = 3 * ss // 4 # encoding b2 means head part sizes smaller by 3/4 fore = decodeB64(fore) vid = decodeB64(vid) @@ -1258,14 +1265,14 @@ def rend(self, memo, *, vid=None): if ml > mms: raise hioing.MemoerError(f"Memo length={ml} exceeds max={mms}.") - if self.mode: + if self.curt: neck = gc.to_bytes(ns) else: neck = helping.intToB64b(gc, l=ns) gn = 0 while memo: - if self.mode: + if self.curt: num = gn.to_bytes(ns) # num size must always be neck size else: num = helping.intToB64b(gn, l=ns) # num size must always be neck size @@ -1281,7 +1288,7 @@ def rend(self, memo, *, vid=None): if ss: # sign sig = self.sign(gram, vid) - if self.mode: + if self.curt: sig = decodeB64(sig) gram = gram + sig @@ -1383,7 +1390,7 @@ def _serviceOnceTxGrams(self, *, echoic=False): except IndexError: return False # nothing more to send, return False to try later - + cnt = 0 try: cnt = self.send(gram, dst, echoic=echoic) # assumes .opened == True except socket.error as ex: # OSError.errno always .args[0] for compat @@ -1404,13 +1411,15 @@ def _serviceOnceTxGrams(self, *, echoic=False): logger.error("Error send from %s to %s\n %s\n", self.name, dst, ex) self.txbs = (bytearray(), None) # far peer unavailable, so drop. + dst = None # dropped is same as all sent else: raise # unexpected error - del gram[:cnt] # remove from buffer those bytes sent - if not gram: # all sent - dst = None # indicate by setting dst to None - self.txbs = (gram, dst) # update txbs to indicate if completely sent + if cnt: + del gram[:cnt] # remove from buffer those bytes sent + if not gram: # all sent + dst = None # indicate by setting dst to None + self.txbs = (gram, dst) # update txbs to indicate if completely sent return (False if dst else True) # incomplete return False, else True @@ -1597,7 +1606,8 @@ def wind(self, tymth): def serviceTymers(self): """Service all retry tymers - Stub override in subclass + Must be overriden in subclass. + This is a stub. """ pass @@ -1701,3 +1711,4 @@ def recur(self, tyme): def exit(self): """""" self.peer.close() + diff --git a/src/hio/core/udp/udping.py b/src/hio/core/udp/udping.py index 2a29c6a..15ad717 100644 --- a/src/hio/core/udp/udping.py +++ b/src/hio/core/udp/udping.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -hio.core.udping Module +hio.core.udp.udping Module """ import sys import platform @@ -213,7 +213,9 @@ def close(self): self.ls = None self.opened = False - def receive(self): + return not self.opened # True means closed successfully + + def receive(self, **kwa): """Perform non blocking read on socket. Returns: @@ -240,7 +242,7 @@ def receive(self): return (data, sa) - def send(self, data, dst): + def send(self, data, dst, **kwa): """Perform non blocking send on socket. Returns: diff --git a/src/hio/core/uxd/__init__.py b/src/hio/core/uxd/__init__.py index 11602d6..5f29125 100644 --- a/src/hio/core/uxd/__init__.py +++ b/src/hio/core/uxd/__init__.py @@ -4,5 +4,5 @@ """ -from .uxding import openPeer, Peer, PeerDoer +from .uxding import Peer, openPeer, PeerDoer diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py new file mode 100644 index 0000000..6735909 --- /dev/null +++ b/src/hio/core/uxd/peermemoing.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +""" +hio.core.uxd.peermemoing Module +""" + +from ... import help +from ... import hioing +from ...base import doing +from ..uxd import Peer +from ..memo import Memoer, GramDex + +logger = help.ogler.getLogger() + + +class PeerMemoer(Peer, Memoer): + """Class for sending memograms over UXD transport + Mixin base classes Peer and Memoer to attain memogram over uxd transport. + + + Inherited Class Attributes: + MaxGramSize (int): absolute max gram size on tx with overhead + See memoing.Memoer Class + See Peer Class + + Inherited Attributes: + See memoing.Memoer Class + See Peer Class + + Class Attributes: + + + Attributes: + + + """ + + + def __init__(self, *, bc=64, **kwa): + """Initialization method for instance. + + Inherited Parameters: + bc (int | None): count of transport buffers of MaxGramSize + + See memoing.Memoer for other inherited paramters + See Peer for other inherited paramters + + + Parameters: + + """ + super(PeerMemoer, self).__init__(bc=bc, **kwa) + + diff --git a/src/hio/core/uxd/uxding.py b/src/hio/core/uxd/uxding.py index 14c3535..323f5e5 100644 --- a/src/hio/core/uxd/uxding.py +++ b/src/hio/core/uxd/uxding.py @@ -1,6 +1,6 @@ # -*- encoding: utf-8 -*- """ -hio.core.uxding Module +hio.core.uxd.uxding Module """ import sys import platform @@ -19,55 +19,6 @@ logger = help.ogler.getLogger() -@contextmanager -def openPeer(cls=None, name="test", temp=True, reopen=True, clear=True, - filed=False, extensioned=True, **kwa): - """ - Wrapper to create and open UXD Peer instances - When used in with statement block, calls .close() on exit of with block - - Parameters: - cls (Class): instance of subclass instance - name (str): unique identifer of peer. Unique path part so can have many - Peers each at different paths that each use different dirs or files - temp (bool): True means open in temporary directory, clear on close - Otherwise open in persistent directory, do not clear on close - reopen (bool): True (re)open with this init - False not (re)open with this init but later (default) - clear (bool): True means remove directory upon close when reopening - False means do not remove directory upon close when reopening - filed (bool): True means .path is file path not directory path - False means .path is directiory path not file path - extensioned (bool): When not filed: - True means ensure .path ends with fext - False means do not ensure .path ends with fext - - See filing.Filer and uxding.Peer for other keyword parameter passthroughs - - Usage: - with openPeer() as peer0: - peer0.receive() - - with openPeer(cls=PeerBig) as peer0: - peer0.receive() - - """ - peer = None - if cls is None: - cls = Peer - try: - peer = cls(name=name, temp=temp, reopen=reopen, clear=clear, - filed=filed, extensioned=extensioned, **kwa) - - yield peer - - finally: - if peer: - peer.close(clear=peer.temp or clear) - - - - class Peer(filing.Filer): """Class to manage non-blocking io on UXD (unix domain) socket. @@ -297,15 +248,15 @@ def close(self, clear=True): self.opened = False try: - super(Peer, self).close(clear=clear) # removes uxd file at end of path only + result = super(Peer, self).close(clear=clear) # removes uxd file at end of path only except OSError: if os.path.exists(self.path): raise - return self.opened + return result # True means closed successfully - def receive(self): + def receive(self, **kwa): """Perform non blocking receive on socket. Returns: @@ -333,7 +284,7 @@ def receive(self): return (data, src) - def send(self, data, dst): + def send(self, data, dst, **kwa): """Perform non blocking send on socket. Returns: @@ -375,6 +326,56 @@ def service(self): pass + +@contextmanager +def openPeer(cls=None, name="test", temp=True, reopen=True, clear=True, + filed=False, extensioned=True, **kwa): + """ + Wrapper to create and open UXD Peer instances + When used in with statement block, calls .close() on exit of with block + + Parameters: + cls (Class): instance of subclass instance + name (str): unique identifer of peer. Unique path part so can have many + Peers each at different paths that each use different dirs or files + temp (bool): True means open in temporary directory, clear on close + Otherwise open in persistent directory, do not clear on close + reopen (bool): True (re)open with this init + False not (re)open with this init but later (default) + clear (bool): True means remove directory upon close when reopening + False means do not remove directory upon close when reopening + filed (bool): True means .path is file path not directory path + False means .path is directiory path not file path + extensioned (bool): When not filed: + True means ensure .path ends with fext + False means do not ensure .path ends with fext + + See filing.Filer and uxding.Peer for other keyword parameter passthroughs + + Usage: + with openPeer() as peer0: + peer0.receive() + + with openPeer(cls=PeerBig) as peer0: + peer0.receive() + + """ + peer = None + if cls is None: + cls = Peer + try: + peer = cls(name=name, temp=temp, reopen=reopen, clear=clear, + filed=filed, extensioned=extensioned, **kwa) + + yield peer + + finally: + if peer: + peer.close(clear=peer.temp or clear) + + + + class PeerDoer(doing.Doer): """ Basic UXD Peer Doer @@ -414,3 +415,4 @@ def recur(self, tyme): def exit(self): """""" self.peer.close(clear=True) + diff --git a/tests/core/memo/__init__.py b/tests/core/memo/__init__.py new file mode 100644 index 0000000..ea8ec44 --- /dev/null +++ b/tests/core/memo/__init__.py @@ -0,0 +1,4 @@ +""" +pytest module +""" + diff --git a/tests/core/test_memoing.py b/tests/core/memo/test_memoing.py similarity index 98% rename from tests/core/test_memoing.py rename to tests/core/memo/test_memoing.py index e992521..adfea47 100644 --- a/tests/core/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -10,8 +10,8 @@ from hio.help import helping from hio.base import doing, tyming -from hio.core import memoing -from hio.core.memoing import Memoer, Versionage, Sizage, GramDex +from hio.core.memo import memoing +from hio.core.memo import Versionage, Sizage, GramDex, Memoer def test_memoer_class(): """Test class attributes of Memoer class""" @@ -66,7 +66,7 @@ def test_memoer_basic(): assert peer.opened == False assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' - assert not peer.mode + assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize @@ -150,8 +150,8 @@ def test_memoer_basic(): assert not peer.rxms # test binary q2 encoding of transmission gram header - peer.mode = True # set to binary base2 - assert peer.mode + peer.curt = True # set to binary base2 + assert peer.curt memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -201,7 +201,7 @@ def test_memogram_small_gram_size(): assert peer.opened == False assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' - assert not peer.mode + assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == 33 # can't be smaller than head + neck + 1 @@ -298,8 +298,8 @@ def test_memogram_small_gram_size(): assert not peer.rxms # test binary q2 encoding of transmission gram header - peer.mode = True # set to binary base2 - assert peer.mode + peer.curt = True # set to binary base2 + assert peer.curt assert peer.size == 38 memo = 'See ya later alligator!' dst = "beta" @@ -364,7 +364,7 @@ def test_memoer_multiple(): assert peer.opened == False assert peer.bc == 64 assert peer.code == memoing.GramDex.Basic == '__' - assert not peer.mode + assert not peer.curt peer.reopen() assert peer.opened == True @@ -431,7 +431,7 @@ def test_memoer_basic_signed(): assert peer.opened == False assert peer.bc == 64 assert peer.code == memoing.GramDex.Signed == '_-' - assert not peer.mode + assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes assert peer.Sizes[peer.code] == (2, 22, 44, 88, 4, 160) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize @@ -519,7 +519,7 @@ def test_memoer_basic_signed(): assert not peer.rxms # test binary q2 encoding of transmission gram header - peer.mode = True # set to binary base2 + peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" peer.memoit(memo, dst) @@ -573,7 +573,7 @@ def test_memoer_multiple_signed(): assert peer.opened == False assert peer.bc == 64 assert peer.code == memoing.GramDex.Signed == '_-' - assert not peer.mode + assert not peer.curt peer.reopen() assert peer.opened == True @@ -628,8 +628,8 @@ def test_memoer_multiple_signed(): assert not peer.rxms # test in base2 mode - peer.mode = True - assert peer.mode + peer.curt = True + assert peer.curt peer.size = 129 assert peer.size == 129 @@ -737,7 +737,7 @@ def test_tymeememogram_basic(): assert peer.name == "main" assert peer.opened == False assert peer.code == memoing.GramDex.Basic == '__' - assert not peer.mode + assert not peer.curt # (code, mid, vid, neck, head, sig) part sizes assert peer.Sizes[peer.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert peer.size == peer.MaxGramSize @@ -821,7 +821,7 @@ def test_tymeememogram_basic(): assert not peer.rxms # test binary q2 encoding of transmission gram header - peer.mode = True # set to binary base2 + peer.curt = True # set to binary base2 memo = "Hello There" dst = "beta" peer.memoit(memo, dst) diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoer.py new file mode 100644 index 0000000..078a1a9 --- /dev/null +++ b/tests/core/uxd/test_peer_memoer.py @@ -0,0 +1,145 @@ +# -*- encoding: utf-8 -*- +""" +tests.core.test_peer_memoer module + +""" +import os + +import pytest + +from hio.help import helping +from hio.base import doing, tyming +from hio.core.memo import GramDex +from hio.core.uxd import uxding, peermemoing + + + + +def test_memoer_peer_basic(): + """Test MemoerPeer class""" + + alpha = peermemoing.PeerMemoer(name="alpha", temp=True, size=38) + assert alpha.name == "alpha" + assert alpha.code == GramDex.Basic + assert not alpha.curt + # (code, mid, vid, sig, neck, head) part sizes + assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs + assert alpha.size == 38 + + assert alpha.bc == 64 + assert not alpha.opened + assert alpha.reopen() + assert alpha.opened + assert alpha.path.endswith("alpha.uxd") + assert alpha.actualBufSizes() == (4194240, 4194240) == (alpha.bc * alpha.MaxGramSize, + alpha.bc * alpha.MaxGramSize) + + beta = peermemoing.PeerMemoer(name="beta", temp=True, size=38) + assert beta.reopen() + assert beta.path.endswith("beta.uxd") + + # alpha sends + alpha.memoit("Hello there.", beta.path) + alpha.memoit("How ya doing?", beta.path) + assert len(alpha.txms) == 2 + alpha.serviceTxMemos() + assert not alpha.txms + assert len(alpha.txgs) == 4 + for m, d in alpha.txgs: + assert not alpha.wiff(m) # base64 + assert d == beta.path + alpha.serviceTxGrams() + assert not alpha.txgs + assert alpha.txbs == (b'', None) + assert not alpha.rxgs + assert not alpha.rxms + assert not alpha.counts + assert not alpha.sources + + # beta receives + beta.serviceReceives() + assert not beta.echos + assert len(beta.rxgs) == 2 + assert len(beta.counts) == 2 + assert len(beta.sources) == 2 + + mid = list(beta.rxgs.keys())[0] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'Hello ') + assert beta.rxgs[mid][1] == bytearray(b'there.') + + mid = list(beta.rxgs.keys())[1] + assert beta.sources[mid] == alpha.path + assert beta.counts[mid] == 2 + assert len(beta.rxgs[mid]) == 2 + assert beta.rxgs[mid][0] == bytearray(b'How ya') + assert beta.rxgs[mid][1] == bytearray(b' doing?') + + beta.serviceRxGrams() + assert not beta.rxgs + assert not beta.counts + assert not beta.sources + assert len(beta.rxms) == 2 + assert beta.rxms[0] == ('Hello there.', alpha.path) + assert beta.rxms[1] == ('How ya doing?', alpha.path) + beta.serviceRxMemos() + assert not beta.rxms + + # beta sends + beta.memoit("Well is not this a fine day?", alpha.path) + beta.memoit("Do you want to do lunch?", alpha.path) + assert len(beta.txms) == 2 + beta.serviceTxMemos() + assert not beta.txms + beta.serviceTxGrams() + assert not beta.txgs + assert beta.txbs == (b'', None) + assert not beta.rxgs + assert not beta.rxms + assert not beta.counts + assert not beta.sources + + # alpha receives + alpha.serviceReceives() + assert not alpha.echos + assert len(alpha.rxgs) == 2 + assert len(alpha.counts) == 2 + assert len(alpha.sources) == 2 + + mid = list(alpha.rxgs.keys())[0] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 4 + assert len(alpha.rxgs[mid]) == 4 + + mid = list(alpha.rxgs.keys())[1] + assert alpha.sources[mid] == beta.path + assert alpha.counts[mid] == 3 + assert len(alpha.rxgs[mid]) == 3 + + alpha.serviceRxGrams() + assert not alpha.rxgs + assert not alpha.counts + assert not alpha.sources + assert len(alpha.rxms) == 2 + assert alpha.rxms[0] == ('Well is not this a fine day?', beta.path) + assert alpha.rxms[1] == ('Do you want to do lunch?', beta.path) + alpha.serviceRxMemos() + assert not alpha.rxms + + + assert beta.close() + assert not beta.opened + assert not os.path.exists(beta.path) + + assert alpha.close() + assert not alpha.opened + assert not os.path.exists(alpha.path) + + + """Done Test""" + +if __name__ == "__main__": + test_memoer_peer_basic() + From 5703c45d62e699eae2395c2730837a4b4e1d85fb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 18:44:19 -0700 Subject: [PATCH 30/33] commented out unit test that does not work on github cause buffer size limitation on test os --- tests/core/uxd/test_peer_memoer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoer.py index 078a1a9..541f201 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoer.py @@ -31,8 +31,8 @@ def test_memoer_peer_basic(): assert alpha.reopen() assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert alpha.actualBufSizes() == (4194240, 4194240) == (alpha.bc * alpha.MaxGramSize, - alpha.bc * alpha.MaxGramSize) + #assert alpha.actualBufSizes() == (4194240, 4194240) == (alpha.bc * alpha.MaxGramSize, + # alpha.bc * alpha.MaxGramSize) beta = peermemoing.PeerMemoer(name="beta", temp=True, size=38) assert beta.reopen() From 9406e86adeca84506146c8aac05e1faafdb23bba Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 19:01:57 -0700 Subject: [PATCH 31/33] update setup.py so using latest stuff being tested Fix some buffers for uxd so not to big for github --- setup.py | 20 ++++++++++---------- src/hio/__init__.py | 2 +- src/hio/core/memo/memoing.py | 2 +- src/hio/core/uxd/peermemoing.py | 2 +- tests/core/memo/test_memoing.py | 10 +++++----- tests/core/uxd/test_peer_memoer.py | 2 +- 6 files changed, 19 insertions(+), 19 deletions(-) diff --git a/setup.py b/setup.py index 44cae83..22e1740 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( name='hio', - version='0.6.14', # also change in src/hio/__init__.py + version='0.6.15', # also change in src/hio/__init__.py license='Apache Software License 2.0', description='Hierarchical Concurrency with Async IO', long_description=("HIO Hierarchical Concurrency and Asynchronous IO Library. " @@ -66,7 +66,7 @@ 'Operating System :: Unix', 'Operating System :: POSIX', 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: Implementation :: CPython', # uncomment if you test on these interpreters: #'Programming Language :: Python :: Implementation :: PyPy', @@ -86,13 +86,13 @@ "structured concurrency", # eg: 'keyword1', 'keyword2', 'keyword3', ], - python_requires='>=3.12.2', + python_requires='>=3.13.1', install_requires=[ - 'lmdb>=1.4.1', - 'msgpack>=1.0.8', - 'cbor2>=5.6.2', - 'multidict>=6.0.5', - 'falcon>=3.1.3', + 'lmdb>=1.6.2', + 'msgpack>=1.1.0', + 'cbor2>=5.6.5', + 'multidict>=6.1.0', + 'falcon>=4.0.2', 'ordered-set>=4.1.0', ], @@ -102,8 +102,8 @@ # ':python_version=="2.6"': ['argparse'], }, tests_require=[ - 'coverage>=7.4.4', - 'pytest>=8.1.1', + 'coverage>=7.6.10', + 'pytest>=8.3.4', ], setup_requires=[ ], diff --git a/src/hio/__init__.py b/src/hio/__init__.py index 010fe25..a50ae81 100644 --- a/src/hio/__init__.py +++ b/src/hio/__init__.py @@ -3,6 +3,6 @@ hio package """ -__version__ = '0.6.14' # also change in setup.py +__version__ = '0.6.15' # also change in setup.py from .hioing import Mixin, HioError, ValidationError, VersionError diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index ee8669a..5efcf3d 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -663,7 +663,7 @@ def __init__(self, *, self.opened = False # mixed with subclass should provide this. if not hasattr(self, "bc"): # stub so mixin works in isolation. - self.bc = bc if bc is not None else 64 # mixed with subclass should provide this. + self.bc = bc if bc is not None else 4 # mixed with subclass should provide this. @property diff --git a/src/hio/core/uxd/peermemoing.py b/src/hio/core/uxd/peermemoing.py index 6735909..2f4effd 100644 --- a/src/hio/core/uxd/peermemoing.py +++ b/src/hio/core/uxd/peermemoing.py @@ -35,7 +35,7 @@ class PeerMemoer(Peer, Memoer): """ - def __init__(self, *, bc=64, **kwa): + def __init__(self, *, bc=4, **kwa): """Initialization method for instance. Inherited Parameters: diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index adfea47..30d3b7c 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -64,7 +64,7 @@ def test_memoer_basic(): peer = memoing.Memoer() assert peer.name == "main" assert peer.opened == False - assert peer.bc == 64 + assert peer.bc == 4 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -199,7 +199,7 @@ def test_memogram_small_gram_size(): peer = memoing.Memoer(size=6) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 64 + assert peer.bc == 4 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -362,7 +362,7 @@ def test_memoer_multiple(): assert peer.size == 38 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 64 + assert peer.bc == 4 assert peer.code == memoing.GramDex.Basic == '__' assert not peer.curt @@ -429,7 +429,7 @@ def test_memoer_basic_signed(): peer = memoing.Memoer(code=GramDex.Signed) assert peer.name == "main" assert peer.opened == False - assert peer.bc == 64 + assert peer.bc == 4 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt # (code, mid, vid, sig, neck, head) part sizes @@ -571,7 +571,7 @@ def test_memoer_multiple_signed(): assert peer.size == 170 assert peer.name == "main" assert peer.opened == False - assert peer.bc == 64 + assert peer.bc == 4 assert peer.code == memoing.GramDex.Signed == '_-' assert not peer.curt diff --git a/tests/core/uxd/test_peer_memoer.py b/tests/core/uxd/test_peer_memoer.py index 541f201..0ed8696 100644 --- a/tests/core/uxd/test_peer_memoer.py +++ b/tests/core/uxd/test_peer_memoer.py @@ -26,7 +26,7 @@ def test_memoer_peer_basic(): assert alpha.Sizes[alpha.code] == (2, 22, 0, 0, 4, 28) # cs ms vs ss ns hs assert alpha.size == 38 - assert alpha.bc == 64 + assert alpha.bc == 4 assert not alpha.opened assert alpha.reopen() assert alpha.opened From 06fa9bfda902e1f4249d2d1bd87185f4c90ff69a Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 19:11:17 -0700 Subject: [PATCH 32/33] update to v0.6.16 --- setup.py | 4 ++-- src/hio/__init__.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 22e1740..1446cab 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ setup( name='hio', - version='0.6.15', # also change in src/hio/__init__.py + version='0.6.16', # also change in src/hio/__init__.py license='Apache Software License 2.0', description='Hierarchical Concurrency with Async IO', long_description=("HIO Hierarchical Concurrency and Asynchronous IO Library. " @@ -66,7 +66,7 @@ 'Operating System :: Unix', 'Operating System :: POSIX', 'Operating System :: Microsoft :: Windows', - 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'Programming Language :: Python :: Implementation :: CPython', # uncomment if you test on these interpreters: #'Programming Language :: Python :: Implementation :: PyPy', diff --git a/src/hio/__init__.py b/src/hio/__init__.py index a50ae81..139064f 100644 --- a/src/hio/__init__.py +++ b/src/hio/__init__.py @@ -3,6 +3,6 @@ hio package """ -__version__ = '0.6.15' # also change in setup.py +__version__ = '0.6.16' # also change in setup.py from .hioing import Mixin, HioError, ValidationError, VersionError From 25e23d97fa7a1d2761c6c74d98cf44d982207deb Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Tue, 25 Feb 2025 19:15:34 -0700 Subject: [PATCH 33/33] fixed unit tests buffer size issue with linux on github test matrix --- tests/core/uxd/test_uxd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/core/uxd/test_uxd.py b/tests/core/uxd/test_uxd.py index 188b750..60a7437 100644 --- a/tests/core/uxd/test_uxd.py +++ b/tests/core/uxd/test_uxd.py @@ -23,13 +23,13 @@ def test_uxd_basic(): """ tymist = tyming.Tymist() with (wiring.openWL(samed=True, filed=True) as wl): - bc = 64 + bc = 4 alpha = uxding.Peer(name="alpha", temp=True, umask=0o077, bc=bc, wl=wl) assert not alpha.opened assert alpha.reopen() assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert alpha.actualBufSizes() == (4194240, 4194240) == (bc * alpha.MaxGramSize, bc * alpha.MaxGramSize) + #assert alpha.actualBufSizes() == (4194240, 4194240) == (bc * alpha.MaxGramSize, bc * alpha.MaxGramSize) beta = uxding.Peer(name="beta", temp=True, umask=0o077) assert beta.reopen() @@ -158,7 +158,7 @@ def test_open_peer(): assert alpha.opened assert alpha.path.endswith("alpha.uxd") - assert alpha.actualBufSizes() == (65535, 65535) == (alpha.BufSize, alpha.BufSize) + #assert alpha.actualBufSizes() == (65535, 65535) == (alpha.BufSize, alpha.BufSize) assert beta.opened assert beta.path.endswith("beta.uxd")