From c60e7fe6f00dea6dbd23fcbed7a1dbde163664f7 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:04:42 +0200 Subject: [PATCH 01/15] API overhaul - send_message new signature, auto sender information - send_acl_message deleted - new package message MangoMessage which contains all meta data for distribution - new express API - container factory methods overhaul --- docs/source/codecs.rst | 2 +- docs/source/getting_started.rst | 54 +- docs/source/message exchange.rst | 10 +- docs/source/migration.rst | 18 + docs/source/scheduling.rst | 69 +- docs/source/tutorial.rst | 108 ++- examples/distributed_clock/README.md | 16 - examples/distributed_clock/clock_agent.py | 108 --- examples/distributed_clock/clock_manager.py | 146 ---- examples/distributed_clock/docker-compose.yml | 10 - examples/distributed_clock/mqtt.conf | 6 - examples/external_clock.py | 50 -- examples/getting_started/v_1.py | 12 - examples/getting_started/v_2.py | 24 - examples/getting_started/v_3.py | 14 - examples/getting_started/v_4.py | 49 -- examples/ping_pong_termination.py | 68 -- examples/rrule_event.py | 60 -- examples/simple_agent.py | 116 ---- examples/tcp_container_example.py | 43 -- .../v1_basic_setup_and_message_passing.py | 50 -- ...ainer_messaging_and_basic_functionality.py | 199 ------ examples/tutorial/v3_codecs_and_typing.py | 213 ------ examples/tutorial/v4_scheduling_and_roles.py | 326 --------- mango/__init__.py | 8 +- mango/agent/core.py | 229 +++---- mango/agent/role.py | 252 ++++--- mango/container/core.py | 645 ++---------------- mango/container/external_coupling.py | 53 +- mango/container/factory.py | 270 ++------ mango/container/mp.py | 513 ++++++++++++++ mango/container/mqtt.py | 214 +++++- mango/container/protocol.py | 8 +- mango/container/tcp.py | 61 +- mango/express/api.py | 225 ++++++ mango/messages/codecs.py | 9 +- mango/messages/mango_message.proto | 6 + mango/messages/mango_message_pb2.py | 36 + mango/messages/message.py | 106 ++- mango/modules/__init__.py | 0 mango/modules/base_module.py | 72 -- mango/modules/mqtt_module.py | 141 ---- mango/modules/rabbit_module.py | 119 ---- mango/modules/zero_module.py | 88 --- mango/util/clock.py | 5 +- mango/util/distributed_clock.py | 64 +- mango/util/multiprocessing.py | 18 +- mango/util/scheduling.py | 22 +- mango/util/termination_detection.py | 8 +- readme.md | 54 +- setup.py | 2 +- tests/integration_tests/__init__.py | 39 ++ .../test_distributed_clock.py | 92 +-- .../test_message_roundtrip.py | 82 +-- .../test_message_roundtrip_mp.py | 53 +- .../test_single_container_termination.py | 445 +++++------- tests/unit_tests/container/test_mp.py | 127 ++-- tests/unit_tests/container/test_tcp.py | 34 +- tests/unit_tests/core/test_agent.py | 78 +-- tests/unit_tests/core/test_container.py | 69 +- .../test_external_scheduling_container.py | 93 ++- tests/unit_tests/express/test_api.py | 129 ++++ tests/unit_tests/messages/test_codecs.py | 12 +- tests/unit_tests/role/role_test.py | 2 +- tests/unit_tests/role_agent_test.py | 134 ++-- tests/unit_tests/test_agents.py | 98 ++- 66 files changed, 2361 insertions(+), 4125 deletions(-) delete mode 100644 examples/distributed_clock/README.md delete mode 100644 examples/distributed_clock/clock_agent.py delete mode 100644 examples/distributed_clock/clock_manager.py delete mode 100644 examples/distributed_clock/docker-compose.yml delete mode 100644 examples/distributed_clock/mqtt.conf delete mode 100644 examples/external_clock.py delete mode 100644 examples/getting_started/v_1.py delete mode 100644 examples/getting_started/v_2.py delete mode 100644 examples/getting_started/v_3.py delete mode 100644 examples/getting_started/v_4.py delete mode 100644 examples/ping_pong_termination.py delete mode 100644 examples/rrule_event.py delete mode 100644 examples/simple_agent.py delete mode 100644 examples/tcp_container_example.py delete mode 100644 examples/tutorial/v1_basic_setup_and_message_passing.py delete mode 100644 examples/tutorial/v2_inter_container_messaging_and_basic_functionality.py delete mode 100644 examples/tutorial/v3_codecs_and_typing.py delete mode 100644 examples/tutorial/v4_scheduling_and_roles.py create mode 100644 mango/container/mp.py create mode 100644 mango/express/api.py create mode 100644 mango/messages/mango_message.proto create mode 100644 mango/messages/mango_message_pb2.py delete mode 100644 mango/modules/__init__.py delete mode 100644 mango/modules/base_module.py delete mode 100644 mango/modules/mqtt_module.py delete mode 100644 mango/modules/rabbit_module.py delete mode 100644 mango/modules/zero_module.py create mode 100644 tests/unit_tests/express/test_api.py diff --git a/docs/source/codecs.rst b/docs/source/codecs.rst index b145071..270f716 100644 --- a/docs/source/codecs.rst +++ b/docs/source/codecs.rst @@ -116,7 +116,7 @@ All that is left to do now is to pass our codec to the container. This is done d # agents can now directly pass content of type MyClass to each other my_object = MyClass("abc", 123) - await sending_container.send_acl_message( + await sending_container.send_message( content=my_object, receiver_addr=("localhost", 5555), receiver_id="agent0" ) diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index d552045..563824e 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -98,21 +98,16 @@ Creating a proactive Agent Let's implement another agent that is able to send a hello world message to another agent: -.. code-block:: python3 +.. code-block:: python from mango import Agent - class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_task(coroutine=self.context.send_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!") - ) + class HelloWorldAgent(Agent): + async def greet(self, other_addr): + await self.send_message("Hello world!", other_addr) - def handle_message(self, content, meta): - print(f"Received a message with the following content: {content}") + def handle_message(self, content, meta): + print(f"Received a message with the following content: {content}") We are using the scheduling API, which is explained in further detail in the section :doc:`scheduling`. @@ -122,11 +117,10 @@ Connecting two agents We can now connect an instance of a HelloWorldAgent with an instance of a RepeatingAgent and let them run. -.. code-block:: python3 +.. code-block:: python import asyncio - from mango import Agent - from mango import create_container + from mango import Agent, create_tcp_container, activate class RepeatingAgent(Agent): @@ -139,35 +133,29 @@ a RepeatingAgent and let them run. # This method defines what the agent will do with incoming messages. print(f"Received a message with the following content: {content}") - class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!" - ) + async def greet(self, other_addr): + await self.send_message("Hello world!", other_addr) def handle_message(self, content, meta): print(f"Received a message with the following content: {content}") async def run_container_and_two_agents(first_addr, second_addr): - first_container = await create_container(addr=first_addr) - second_container = await create_container(addr=second_addr) - first_agent = RepeatingAgent(first_container) - second_agent = HelloWorldAgent(second_container, first_container.addr, first_agent.aid) - await asyncio.sleep(1) - await first_agent.shutdown() - await second_agent.shutdown() - await first_container.shutdown() - await second_container.shutdown() + first_container = create_tcp_container(addr=first_addr) + second_container = create_tcp_container(addr=second_addr) + + first_agent = first_container.include(RepeatingAgent()) + second_agent = second_container.include(HelloWorldAgent()) + + async with activate(first_container, second_container) as cl: + await second_agent.greet(first_agent.addr) - if __name__ == '__main__': + def test_second_example(): asyncio.run(run_container_and_two_agents( - first_addr=('localhost', 5555), second_addr=('localhost', 5556))) + first_addr=('localhost', 5555), second_addr=('localhost', 5556)) + ) You should now see the following output: diff --git a/docs/source/message exchange.rst b/docs/source/message exchange.rst index bdb8321..b11cd11 100644 --- a/docs/source/message exchange.rst +++ b/docs/source/message exchange.rst @@ -63,15 +63,7 @@ has to be provided. This will appear as the `content` argument at the receivers handle_message() method. -If you want to send an ACL-message use the method ``container.send_acl_message``, which will wrap the content in a ACLMessage using ``create_acl`` internally. - -.. code-block:: python3 - - async def send_acl_message(self, content, - receiver_addr: Union[str, Tuple[str, int]], *, - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs) -> bool: +If you want to send an ACL-message use the method ``create_acl`` to create the ACL content and send it with the regular ``send_message``-method internally. The argument ``acl_metadata`` enables to set all meta information of an acl message. It expects a dictionary with the field name as string as a key and the field value as key. diff --git a/docs/source/migration.rst b/docs/source/migration.rst index 1b17a08..a886cda 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -3,7 +3,25 @@ Migration Some mango versions will break the API; in this case, we may provide instructions on the migration on this page. +******************** +mango 1.2.X to 2.0.0 +******************** +* The methods send_acl_message and schedule_instant_acl_message have been removed, use send_message/schedule_instant_message with create_acl instead if you explicitly need an ACLMessage, otherwise you can just replace it with send_message/schedule_instant_message. + * Background: In the past, mango relied on ACL messages for message routing, in 2.0.0 mango introduces an internal message container for all types of content, which provides necessary routing information. That way all mango messages can always be routed to the correct target. +* The container factory method create_container has been removed and instead there is one factory method per container type now: create_tcp_container, create_mqtt_container, create_external_container +* The container factory methods are not async anymore! +* The startup for the TCP-server/MQTT-client has been moved to `container.start()`. + * Use the async context manager provided by `activate(containers)` as described in the documentation + * Background: This has the advantage that calling shutdown is not necessary anymore, in that way it is not possible to forget and ressource leaks are avoided at any time +* Consider to use the async context manager provided by run_with_X, these managers will provide suitable containers internally, that way the user does not need to handle container at all +* Agents are created without container now + * This implies that Agents need to be registered explicitly using `register` + * Background: This is necessary to provide the run_with_X methods as in this case the Agents need to be created before the Containers. Furthermore it generally provides more flexibility for the user and for us to add new features in the future. +* The main way to route messages is now to use AgentAddress to specific the destination. Just replace receiver_addr and receiver_id with the convenience methods agent.addr or sender_addr(agent) (from mango). +* The signature methods send_message and schedule_instant_message have been changed, you can omit specifying sender_id and sender_addr most of the time now (if you use convenience methods of Agent or Role). + Further the method now requires an AgentAddress +* ******************** mango 0.4.0 to 1.0.0 diff --git a/docs/source/scheduling.rst b/docs/source/scheduling.rst index bacde2e..e310562 100644 --- a/docs/source/scheduling.rst +++ b/docs/source/scheduling.rst @@ -38,16 +38,9 @@ Furthermore there are convenience methods to get rid of the class imports when u class ScheduleAgent(Agent): def __init__(self, container, other_addr, other_id): - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!") - ) - # equivalent to - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!")) + self.schedule_instant_message( + "Hello world!" + other_addr ) def handle_message(self, content, meta: Dict[str, Any]): @@ -94,13 +87,13 @@ In mango the following process tasks are available: class ScheduleAgent(Agent): def __init__(self, container, other_addr, other_id): - self.schedule_instant_process_task(coroutine_creator=lambda: self.send_acl_message( + self.schedule_instant_process_task(coroutine_creator=lambda: self.send_message( receiver_addr=other_addr, receiver_id=other_id, content="Hello world!") ) # equivalent to - self.schedule_process_task(InstantScheduledProcessTask(coroutine_creator=lambda: self.send_acl_message( + self.schedule_process_task(InstantScheduledProcessTask(coroutine_creator=lambda: self.send_message( receiver_addr=other_addr, receiver_id=other_id, content="Hello world!")) @@ -137,7 +130,7 @@ control how fast or slow time passes within your agent system: timestamp=self.current_timestamp + 5) async def send_hello_world(self, receiver_addr, receiver_id): - await self.send_acl_message(receiver_addr=receiver_addr, + await self.send_message(receiver_addr=receiver_addr, receiver_id=receiver_id, content='Hello World') @@ -175,7 +168,7 @@ If you change the clock to an ``ExternalClock`` by uncommenting the ExternalCloc the program won't terminate as the time of the clock is not proceeded by an external process. If you comment in the ExternalClock and change your main() as follows, the program will terminate after one second: -.. code-block:: python3 +.. code-block:: python async def main(): # clock = AsyncioClock() @@ -190,3 +183,51 @@ If you comment in the ExternalClock and change your main() as follows, the progr clock.set_time(clock.time + 5) await receiver.wait_for_reply await c.shutdown() + + +******************************* +Using a distributed clock +******************************* +To distribute simulations, mango provides a distributed clock, which is implemented with by two Agents: +1. DistributedClockAgent: this agent needs to be present in every participating container +2. DistributedClockManager: this agent shall exist exactly once + +The clock is distributed by an DistributedClockManager Agent on the managing container, which listens to the current time. + +1. In the other container DistributedClockAgent's are running, which listen to messages from the ClockManager. +2. The ClockAgent sets the received time on the clock of its container with `set_time` and responds with its `get_next_activity()` after making sure that all tasks which are due at the current timestamp are finished. +3. The ClockManager only acts after all connected Containers have finished and have sent their next timestamp as response. +4. The response is then added as a Future on the manager, which makes sure, that the managers `get_next_activity()` shows the next action needed to run on all containers. + +Caution: it is needed, that all agents are connected before starting the manager + +In the following a simple example is shown. + +.. testcode:: + + import asyncio + from mango import DistributedClockAgent, DistributedClockManager, create_tcp_container, activate, ExternalClock + + async def main(): + container_man = create_tcp_container(("localhost", 1555), clock=ExternalClock()) + container_ag = create_tcp_container(("localhost", 1556), clock=ExternalClock()) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager( + receiver_clock_addresses=[clock_agent.addr] + )) + + async with activate(container_man, container_ag) as cl: + # increasing the time + container_man.clock.set_time(100) + # first distribute the time - then wait for the agent to finish + next_event = await clock_manager.distribute_time() + # here no second distribute to wait for retrieval is needed + # the clock_manager distributed the time to the other container + assert container_ag.clock.time == 100 + print("Time has been distributed!") + + asyncio.run(main()) +.. testoutput:: + + Time has been distributed! \ No newline at end of file diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 36e1efc..f0b01d9 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -140,7 +140,7 @@ This example covers: - message passing between different containers - basic task scheduling - setting custom agent ids - - use of ACL metadata + - use of metadata .. raw:: html @@ -167,11 +167,8 @@ As an additional feature, we will make it possible to manually set the agent of Next, we set up its ``handle_message`` function. The controller needs to distinguish between two message types: The replies to feed_in requests and later the acknowledgements that a new maximum feed_in was set by a pv agent. -We use the ``performative`` field of the ACL message to do this. We set the ``performative`` field to ``inform`` -for feed_in replies and to ``accept_proposal`` for feed_in change acknowledgements. All messages between containers -are expected to be ACL Messages (or implement the split_content_and_meta function). Since we do not want to create -the full ACL object ourselves every time, within this example we always use the convenience method -``container.send_acl_message``, which also supports setting the acl metadata. +We use the assign the key ``performative``of the metadata of the message to do this. We set the ``performative`` field to ``inform`` +for feed_in replies and to ``accept_proposal`` for feed_in change acknowledgements. .. code-block:: python @@ -226,10 +223,10 @@ We do the same for our PV agents. We will also enable user defined agent ids her When a PV agent receives a request from the controller, it immediately answers. Note two important changes to the first -example here: First, within our message handling methods we can not ``await send_acl_message`` directly -because ``handle_message`` is not a coroutine. Instead, we pass ``send_acl_message`` as a task to the scheduler to be +example here: First, within our message handling methods we can not ``await send_message`` directly +because ``handle_message`` is not a coroutine. Instead, we pass ``send_message`` as a task to the scheduler to be executed at once via the ``schedule_instant_task`` method. -Second, we set ``acl_meta`` to contain the typing information of our message. +Second, we set ``meta`` to contain the typing information of our message. .. code-block:: python @@ -240,29 +237,24 @@ Second, we set ``acl_meta`` to contain the typing information of our message. reported_feed_in = PV_FEED_IN[self.aid] # PV_FEED_IN must be defined at the top content = reported_feed_in - acl_meta = {"sender_addr": self.addr, "sender_id": self.aid, + meta = {"sender_addr": self.addr, "sender_id": self.aid, "performative": Performatives.inform} - # Note, could be shortened using self.schedule_instant_acl_message - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=sender_addr, - receiver_id=sender_id, - acl_metadata=acl_meta, - ) + self.schedule_instant_message( + content=content, + receiver_addr=sender_addr, + receiver_id=sender_id, + **meta, ) def handle_set_feed_in_max(self, max_feed_in, sender_addr, sender_id): self.max_feed_in = float(max_feed_in) print(f"{self.aid}: Limiting my feed_in to {max_feed_in}") - self.schedule_instant_task( - self.send_acl_message( - content=None, - receiver_addr=sender_addr, - receiver_id=sender_id, - acl_metadata={'performative': Performatives.accept_proposal}, - ) + self.schedule_instant_message( + content=None, + receiver_addr=sender_addr, + receiver_id=sender_id, + performative=Performatives.accept_proposal, ) Now both of our agents can handle their respective messages. The last thing to do is make the controller actually @@ -284,24 +276,16 @@ perform its active actions. We do this by implementing a ``run`` function with t self.reports_done = asyncio.Future() self.acks_done = asyncio.Future() - # Note: For messages passed between different containers (i.e. over the network socket) it is expected - # that the message is an ACLMessage object. We can let the container wrap our content in such an - # object with using the send_acl_message method. - # We distinguish the types of messages we send by adding a type field to our content. - # ask pv agent feed-ins for addr, aid in self.known_agents: content = None - acl_meta = {"sender_addr": self.addr, "sender_id": self.aid, + meta = {"sender_addr": self.addr, "sender_id": self.aid, "performative": Performatives.request} - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) + self.schedule_instant_message( + content=content, + receiver_addr=addr, + receiver_id=aid, + **meta, ) # wait for both pv agents to answer @@ -313,17 +297,14 @@ perform its active actions. We do this by implementing a ``run`` function with t for addr, aid in self.known_agents: content = min_feed_in - acl_meta = {"sender_addr": self.addr, "sender_id": self.aid, + meta = {"sender_addr": self.addr, "sender_id": self.aid, "performative": Performatives.propose} - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) + self.schedule_instant_message( + content=content, + receiver_addr=addr, + receiver_id=aid, + **meta, ) # wait for both pv agents to acknowledge the change @@ -541,7 +522,7 @@ it should forward these messages to this role. The ``subscribe_message`` method expects, besides the role and a handle method, a message condition function. The idea of the condition function is to allow to define a condition filtering incoming messages. Another idea is that sending messages from the role is now done via its context with the method: -``self.context.send_acl_message``. +``self.context.send_message``. We first create the ``Ping`` role, which has to periodically send out its messages. We can use mango's scheduling API to handle @@ -574,11 +555,11 @@ also run tasks at specific times. For a full overview we refer to the documentat msg = Ping(ping_id) meta = {"sender_addr": self.context.addr, "sender_id": self.context.aid} - await self.context.send_acl_message( + await self.context.send_message( msg, receiver_addr=addr, receiver_id=aid, - acl_metadata=meta, + **meta, ) self.expected_pongs.append(ping_id) self.ping_counter += 1 @@ -645,12 +626,10 @@ The ``Pong`` role is associated with the PV Agents and purely reactive. print(f"Ping {self.context.aid}: Received a ping with ID: {ping_id}") # message sending from roles is done via the RoleContext - self.context.schedule_instant_task( - self.context.send_acl_message( - answer, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) + self.context.schedule_message( + answer, + receiver_addr=sender_addr, + receiver_id=sender_id, ) @@ -677,22 +656,19 @@ unchanged and is simply moved to the PVRole. def handle_ask_feed_in(self, content, meta): """...""" - self.context.schedule_instant_task( - self.context.send_acl_message( + self.context.schedule_instant_message( content=msg, receiver_addr=sender_addr, receiver_id=sender_id, ) - ) + def handle_set_feed_in_max(self, content, meta): """...""" - self.context.schedule_instant_task( - self.context.send_acl_message( - content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) + self.context.schedule_instant_message( + content=msg, + receiver_addr=sender_addr, + receiver_id=sender_id, ) diff --git a/examples/distributed_clock/README.md b/examples/distributed_clock/README.md deleted file mode 100644 index e2509c9..0000000 --- a/examples/distributed_clock/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Distributed Simulation - -To run a distributed simulation, a clock manager is used. -The clock is distributed by an DistributedClockManager Agent on the managing container, which listens to the current time. - -1. In the other container a DistributedClockAgent is running, which listens to messages from the other container. -2. The ClockAgent sets the received time on the clock of its container with `set_time` and responds with its `get_next_activity()` after making sure that all tasks which are due at the current timestamp are finished. -3. The ClockManager only acts after all connected Containers have finished and have sent their next timestamp as response. -4. The response is then added as a Future on the manager, which makes sure, that the managers `get_next_activity()` shows the next action needed to run on all containers. - -A simple example is added here, which sends messages every 900 seconds from the manager to the agent, while the agent itself sends a message every 300 seconds to its manager. -If the `clock_agent`s routine takes longer, the manager will wait for it to finish before setting the next time. - -Caution: it is needed, that all agents are connected before starting the manager - -This example is tested with MQTT as well as by using TCP connection diff --git a/examples/distributed_clock/clock_agent.py b/examples/distributed_clock/clock_agent.py deleted file mode 100644 index f1c4328..0000000 --- a/examples/distributed_clock/clock_agent.py +++ /dev/null @@ -1,108 +0,0 @@ -import asyncio -import logging -from typing import TypedDict - -import numpy as np - -from mango import Role, RoleAgent, create_container -from mango.messages.message import Performatives -from mango.util.clock import ExternalClock -from mango.util.distributed_clock import DistributedClockAgent - -logger = logging.getLogger(__name__) - - -class SimpleBid(TypedDict): - price: float - volume: float - - -class BiddingRole(Role): - def __init__(self, receiver_addr, receiver_id, volume=100, price=0.05): - super().__init__() - self.receiver_addr = receiver_addr - self.receiver_id = receiver_id - self.volume = volume - self.price = price - - def setup(self): - self.volume = self.volume - self.price = self.price - self.context.subscribe_message( - self, self.handle_message, lambda content, meta: True - ) - self.context.schedule_periodic_task(coroutine_func=self.say_hello, delay=1200) - - async def say_hello(self): - await self.context.send_acl_message( - content="Hello Market", - receiver_addr=self.receiver_addr, - receiver_id=self.receiver_id, - acl_metadata={ - "sender_id": self.context.aid, - "sender_addr": self.context.addr, - }, - ) - - def handle_message(self, content, meta): - self.context.schedule_instant_task(coroutine=self.set_bids()) - logger.debug("current_time %s", self.context.current_timestamp) - - async def set_bids(self): - # await asyncio.sleep(1) - price = self.price + 0.01 * self.price * np.random.random() - logger.debug("did set bids at %s", self.context._scheduler.clock.time) - - acl_metadata = { - "performative": Performatives.inform, - "sender_id": self.context.aid, - "sender_addr": self.context.addr, - "conversation_id": "conversation01", - } - await self.context.send_acl_message( - content={"price": price, "volume": self.volume}, - receiver_addr=self.receiver_addr, - receiver_id=self.receiver_id, - acl_metadata=acl_metadata, - ) - - -async def main(): - clock = ExternalClock(start_time=0) - # connection_type = 'mqtt' - connection_type = "tcp" - - if connection_type == "mqtt": - addr = "c2" - other_container_addr = "c1" - else: - addr = ("localhost", 5556) - other_container_addr = ("localhost", 5555) - container_kwargs = { - "connection_type": connection_type, - "addr": addr, - "clock": clock, - "mqtt_kwargs": { - "client_id": "container_2", - "broker_addr": ("localhost", 1883, 60), - "transport": "tcp", - }, - } - - c = await create_container(**container_kwargs) - - clock_agent = DistributedClockAgent(c) - - for i in range(2): - agent = RoleAgent(c, suggested_aid=f"a{i}") - agent.add_role( - BiddingRole(other_container_addr, "market", price=0.05 * (i % 9)) - ) - - await clock_agent.stopped - await c.shutdown() - - -if __name__ == "__main__": - logging.basicConfig(level="INFO") - asyncio.run(main()) diff --git a/examples/distributed_clock/clock_manager.py b/examples/distributed_clock/clock_manager.py deleted file mode 100644 index dfabbad..0000000 --- a/examples/distributed_clock/clock_manager.py +++ /dev/null @@ -1,146 +0,0 @@ -import asyncio -import logging -from datetime import datetime -from typing import TypedDict - -import pandas as pd -from tqdm import tqdm - -from mango import Role, RoleAgent, create_container -from mango.messages.message import Performatives -from mango.util.clock import ExternalClock -from mango.util.distributed_clock import DistributedClockManager -from mango.util.termination_detection import tasks_complete_or_sleeping - -logger = logging.getLogger(__name__) - - -class SimpleBid(TypedDict): - price: float - volume: float - - -class OneSidedMarketRole(Role): - def __init__(self, demand=1000, receiver_ids=[]): - super().__init__() - self.demand = demand - self.bids = [] - self.receiver_ids = receiver_ids - - def setup(self): - self.results = [] - self.demands = [] - - self.context.subscribe_message( - self, self.handle_message, lambda content, meta: isinstance(content, dict) - ) - - self.context.subscribe_message( - self, self.handle_other, lambda content, meta: not isinstance(content, dict) - ) - self.context.schedule_periodic_task(coroutine_func=self.clear_market, delay=900) - self.starttime = self.context.current_timestamp - - async def clear_market(self): - time = datetime.fromtimestamp(self.context.current_timestamp) - if self.context.current_timestamp > self.starttime: - i = time.hour + time.minute / 60 - df = pd.DataFrame.from_dict(self.bids) - self.bids = [] - price = 0 - if df.empty: - logger.info("Did not receive any bids!") - else: - # simple merit order calculation - df = df.sort_values("price") - df["cumsum"] = df["volume"].cumsum() - filtered = df[df["cumsum"] >= self.demand] - if filtered.empty: - # demand could not be matched - price = 100 - else: - price = filtered["price"].values[0] - self.results.append(price) - self.demands.append(self.demand) - else: - logger.info("First opening does not have anything to clear") - price = 0 - acl_metadata = { - "performative": Performatives.inform, - "sender_id": self.context.aid, - "sender_addr": self.context.addr, - "conversation_id": "conversation01", - } - resp = [] - for receiver_addr, receiver_id in self.receiver_ids: - r = self.context.send_acl_message( - content={"message": f"Current time is {time}", "price": price}, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - ) - resp.append(r) - for r in resp: - await r - - def handle_message(self, content, meta): - # content is SimpleBid - content["sender_id"] = meta["sender_id"] - self.bids.append(content) - - def handle_other(self, content, meta): - # content is other - print(f'got {content} from {meta.get("sender_id")}') - - async def on_stop(self): - logger.info(self.results) - - -async def main(start): - clock = ExternalClock(start_time=start.timestamp()) - # connection_type = 'mqtt' - connection_type = "tcp" - - if connection_type == "mqtt": - addr = "c1" - other_container_addr = "c2" - else: - addr = ("localhost", 5555) - other_container_addr = ("localhost", 5556) - container_kwargs = { - "connection_type": connection_type, - "addr": addr, - "clock": clock, - "mqtt_kwargs": { - "client_id": "container_1", - "broker_addr": ("localhost", 1883, 60), - "transport": "tcp", - }, - } - - c = await create_container(**container_kwargs) - clock_agent = DistributedClockManager( - c, receiver_clock_addresses=[(other_container_addr, "clock_agent")] - ) - # distribute time here, so that containers already have correct start time - next_event = await clock_agent.distribute_time() - market = RoleAgent(c, suggested_aid="market") - receiver_ids = [(other_container_addr, "a0"), (other_container_addr, "a1")] - market.add_role(OneSidedMarketRole(demand=1000, receiver_ids=receiver_ids)) - - if isinstance(clock, ExternalClock): - for i in tqdm(range(30)): - next_event = await clock_agent.distribute_time() - logger.info("current step: %s", clock.time) - await tasks_complete_or_sleeping(c) - # comment next line to see that the first message is not received in correct timings - # also comment sleep(0) in termination_detection to see other timing problems - next_event = await clock_agent.distribute_time() - clock.set_time(next_event) - await c.shutdown() - - -if __name__ == "__main__": - logging.basicConfig(level="INFO") - start = datetime(2023, 1, 1) - asyncio.run(main(start)) diff --git a/examples/distributed_clock/docker-compose.yml b/examples/distributed_clock/docker-compose.yml deleted file mode 100644 index 99b35ea..0000000 --- a/examples/distributed_clock/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ -version: '3' -services: - mqtt-broker: - container_name: mango-broker - image: eclipse-mosquitto:2 - restart: always - volumes: - - ./mqtt.conf:/mosquitto/config/mosquitto.conf - ports: - - "1883:1883/tcp" diff --git a/examples/distributed_clock/mqtt.conf b/examples/distributed_clock/mqtt.conf deleted file mode 100644 index bf704e1..0000000 --- a/examples/distributed_clock/mqtt.conf +++ /dev/null @@ -1,6 +0,0 @@ -# https://github.com/eclipse/mosquitto/blob/master/mosquitto.conf -listener 1883 -allow_anonymous true -# https://blog.jaimyn.dev/mqtt-use-acls-multiple-user-accounts/ - -max_keepalive 3600 diff --git a/examples/external_clock.py b/examples/external_clock.py deleted file mode 100644 index a31485f..0000000 --- a/examples/external_clock.py +++ /dev/null @@ -1,50 +0,0 @@ -import asyncio - -from mango import Agent, create_container -from mango.util.clock import ExternalClock - - -class Caller(Agent): - def __init__(self, container, receiver_addr, receiver_id): - super().__init__(container) - self.schedule_timestamp_task( - coroutine=self.send_hello_world(receiver_addr, receiver_id), - timestamp=self.current_timestamp + 5, - ) - - async def send_hello_world(self, receiver_addr, receiver_id): - await self.send_acl_message( - receiver_addr=receiver_addr, receiver_id=receiver_id, content="Hello World" - ) - - def handle_message(self, content, meta): - pass - - -class Receiver(Agent): - def __init__(self, container): - super().__init__(container) - self.wait_for_reply = asyncio.Future() - - def handle_message(self, content, meta): - print(f"Received a message with the following content {content}.") - self.wait_for_reply.set_result(True) - - -async def main(): - # clock = AsyncioClock() - clock = ExternalClock(start_time=1000) - addr = ("127.0.0.1", 5555) - - c = await create_container(addr=addr, clock=clock) - receiver = Receiver(c) - caller = Caller(c, addr, receiver.aid) - if isinstance(clock, ExternalClock): - await asyncio.sleep(1) - clock.set_time(clock.time + 5) - await receiver.wait_for_reply - await c.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/getting_started/v_1.py b/examples/getting_started/v_1.py deleted file mode 100644 index dd88853..0000000 --- a/examples/getting_started/v_1.py +++ /dev/null @@ -1,12 +0,0 @@ -from mango.agent.core import Agent - - -class RepeatingAgent(Agent): - def __init__(self, container): - # We must pass a reference of the container to "mango.Agent": - super().__init__(container) - print(f"Hello world! My id is {self.aid}.") - - def handle_message(self, content, meta): - # This method defines what the agent will do with incoming messages. - print(f"Received a message with the following content: {content}") diff --git a/examples/getting_started/v_2.py b/examples/getting_started/v_2.py deleted file mode 100644 index 3a4db6a..0000000 --- a/examples/getting_started/v_2.py +++ /dev/null @@ -1,24 +0,0 @@ -import asyncio - -from mango import Agent, create_container - - -class RepeatingAgent(Agent): - def __init__(self, container): - # We must pass a ref. to the container to "mango.Agent": - super().__init__(container) - print(f"Hello world! My id is {self.aid}.") - - def handle_message(self, content, meta): - # This method defines what the agent will do with incoming messages. - print(f"Received a message with the following content: {content}") - - -async def run_container_and_agent(addr, duration): - first_container = await create_container(addr=addr) - RepeatingAgent(first_container) - await asyncio.sleep(duration) - await first_container.shutdown() - - -asyncio.run(run_container_and_agent(addr=("localhost", 5555), duration=3)) diff --git a/examples/getting_started/v_3.py b/examples/getting_started/v_3.py deleted file mode 100644 index d798646..0000000 --- a/examples/getting_started/v_3.py +++ /dev/null @@ -1,14 +0,0 @@ -from mango import Agent - - -class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!", - ) - - def handle_message(self, content, meta): - print(f"Received a message with the following content: {content}") diff --git a/examples/getting_started/v_4.py b/examples/getting_started/v_4.py deleted file mode 100644 index 6d4e391..0000000 --- a/examples/getting_started/v_4.py +++ /dev/null @@ -1,49 +0,0 @@ -import asyncio - -from mango import Agent, create_container - - -class RepeatingAgent(Agent): - def __init__(self, container): - # We must pass a ref. to the container to "mango.Agent": - super().__init__(container) - print(f"Hello world! My id is {self.aid}.") - - def handle_message(self, content, meta): - # This method defines what the agent will do with incoming messages. - print(f"Received a message with the following content: {content}") - - -class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!", - ) - - def handle_message(self, content, meta): - print(f"Received a message with the following content: {content}") - - -async def run_container_and_two_agents(first_addr, second_addr): - first_container = await create_container(addr=first_addr) - second_container = await create_container(addr=second_addr) - first_agent = RepeatingAgent(first_container) - second_agent = HelloWorldAgent( - second_container, first_container.addr, first_agent.aid - ) - await asyncio.sleep(1) - await first_agent.shutdown() - await second_agent.shutdown() - await first_container.shutdown() - await second_container.shutdown() - - -if __name__ == "__main__": - asyncio.run( - run_container_and_two_agents( - first_addr=("localhost", 5555), second_addr=("localhost", 5556) - ) - ) diff --git a/examples/ping_pong_termination.py b/examples/ping_pong_termination.py deleted file mode 100644 index 9bf644f..0000000 --- a/examples/ping_pong_termination.py +++ /dev/null @@ -1,68 +0,0 @@ -import asyncio - -from mango import Agent, create_container -from mango.util.clock import ExternalClock -from mango.util.termination_detection import tasks_complete_or_sleeping - - -class Caller(Agent): - def __init__(self, container, receiver_addr, receiver_id): - super().__init__(container) - self.schedule_timestamp_task( - coroutine=self.send_hello_world(receiver_addr, receiver_id), - timestamp=self.current_timestamp + 5, - ) - self.i = 0 - - async def send_hello_world(self, receiver_addr, receiver_id): - await self.send_acl_message( - receiver_addr=receiver_addr, receiver_id=receiver_id, content="Hello World" - ) - - def handle_message(self, content, meta): - print(f"{self.aid} Received a message with the following content {content}.") - self.i += 1 - if self.i < 100: - self.schedule_instant_acl_message( - receiver_addr=self.addr, receiver_id="agent0", content=self.i - ) - - -class Receiver(Agent): - def __init__(self, container): - super().__init__(container) - - def handle_message(self, content, meta): - print(f"{self.aid} Received a message with the following content {content}.") - self.schedule_instant_acl_message( - receiver_addr=self.addr, receiver_id="agent1", content=content - ) - - -async def main(): - # clock = AsyncioClock() - clock = ExternalClock(start_time=1000) - addr = ("127.0.0.1", 5555) - - c = await create_container(addr=addr, clock=clock) - receiver = Receiver(c) - caller = Caller(c, addr, receiver.aid) - if isinstance(clock, ExternalClock): - await asyncio.sleep(1) - clock.set_time(clock.time + 5) - - # wait until each agent is done with all tasks - # this does not end correctly - await receiver._scheduler.tasks_complete_or_sleeping() - await caller._scheduler.tasks_complete_or_sleeping() - - # checking tasks completed for each agent does not help here, as they are sleeping alternating - # the following container-wide function catches this behavior in a single container - # to solve this situation for multiple containers a distributed termination detection - # is needed - await tasks_complete_or_sleeping(c) - await c.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/rrule_event.py b/examples/rrule_event.py deleted file mode 100644 index 6c4a6fc..0000000 --- a/examples/rrule_event.py +++ /dev/null @@ -1,60 +0,0 @@ -import asyncio -from datetime import datetime - -from dateutil import rrule - -from mango import Agent, create_container -from mango.util.clock import ExternalClock - - -class Caller(Agent): - def __init__(self, container, receiver_addr, receiver_id, recurrency): - super().__init__(container) - self.receiver_addr = receiver_addr - self.receiver_id = receiver_id - self.schedule_recurrent_task( - coroutine_func=self.send_hello_world, recurrency=recurrency - ) - - async def send_hello_world(self): - time = datetime.fromtimestamp(self._scheduler.clock.time) - await self.send_acl_message( - receiver_addr=self.receiver_addr, - receiver_id=self.receiver_id, - content=f"Current time is {time}", - ) - - def handle_message(self, content, meta): - pass - - -class Receiver(Agent): - def __init__(self, container): - super().__init__(container) - self.wait_for_reply = asyncio.Future() - - def handle_message(self, content, meta): - print(f"Received a message with the following content: {content}.") - - -async def main(start): - clock = ExternalClock(start_time=start.timestamp()) - addr = ("127.0.0.1", 5555) - # market acts every 15 minutes - recurrency = rrule.rrule(rrule.MINUTELY, interval=15, dtstart=start) - - c = await create_container(addr=addr, clock=clock) - receiver = Receiver(c) - caller = Caller(c, addr, receiver.aid, recurrency) - if isinstance(clock, ExternalClock): - for i in range(100): - await asyncio.sleep(0.01) - clock.set_time(clock.time + 60) - await c.shutdown() - - -if __name__ == "__main__": - from dateutil.parser import parse - - start = parse("202301010000") - asyncio.run(main(start)) diff --git a/examples/simple_agent.py b/examples/simple_agent.py deleted file mode 100644 index 733c655..0000000 --- a/examples/simple_agent.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Simple Agent to show basic functionality""" - -import asyncio -import logging - -from mango import Agent -from mango.messages import other_proto_msgs_pb2 as other_proto_msg -from mango.messages.acl_message_pb2 import ACLMessage as ACLMessage_proto -from mango.messages.message import ACLMessage as ACLMessage_json -from mango.messages.message import Performatives - -logger = logging.getLogger(__name__) - - -class SimpleAgent(Agent): - def __init__(self, container, other_aid=None, other_addr=None, codec="json"): - super().__init__(container) - self.other_aid = other_aid - self.other_addr = other_addr - self.conversations = [] - self.codec = codec - if isinstance(container.addr, (list, tuple)): - self.addr_str = f"{container.addr[0]}:{container.addr[1]}" - else: - self.addr_str = container.addr - - async def send_greeting(self): - # if we have the address of another agent we send a greeting - if self.other_aid is not None: - if self.codec == "json": - message_content = "Hi there" - elif self.codec == "protobuf": - message_content = other_proto_msg.GenericMsg() - message_content.text = "Hi there" - acl_meta = { - "reply_with": "greeting", - "conversation_id": f"{self.aid}_1", - "performative": Performatives.inform.value, - "sender_id": self.aid, - } - await self.send_acl_message( - message_content, - self.other_addr, - receiver_id=self.other_aid, - acl_metadata=acl_meta, - ) - - def handle_message(self, content, meta): - """ - decide which actions shall be performed as reaction to message - :param content: the content of the mssage - :param meta: meta information - """ - logger.info("Received message: %s with meta %s", content, meta) - - # so far we only expect and react to greetings - t = asyncio.create_task(self.react_to_greeting(content, meta)) - t.add_done_callback(self.raise_exceptions) - - async def react_to_greeting(self, msg_in, meta_in): - conversation_id = meta_in["conversation_id"] - reply_with = meta_in["reply_with"] - sender_id = meta_in["sender_id"] - sender_addr = meta_in["sender_addr"] - if not reply_with: - # No reply necessary - remove conversation id - if conversation_id in self.conversations: - self.conversations.remove(conversation_id) - else: - # prepare a reply - if conversation_id not in self.conversations: - self.conversations.append(conversation_id) - if reply_with == "greeting": - # greet back - message_out_content = "Hi there too" - reply_key = "greeting2" - elif reply_with == "greeting2": - # back greeting received, send good bye - message_out_content = "Good Bye" - # end conversation - reply_key = None - self.conversations.remove(conversation_id) - else: - assert False, f"got strange reply_with: {reply_with}" - if self.codec == "json": - message = ACLMessage_json( - sender_id=self.aid, - sender_addr=self.addr_str, - receiver_id=sender_id, - receiver_addr=sender_addr, - content=message_out_content, - in_reply_to=reply_with, - reply_with=reply_key, - conversation_id=conversation_id, - performative=Performatives.inform, - ) - elif self.codec == "protobuf": - message = ACLMessage_proto() - message.sender_id = self.aid - message.sender_addr = self.addr_str - message.receiver_id = sender_id - message.in_reply_to = reply_with - if reply_key: - message.reply_with = reply_key - message.conversation_id = conversation_id - message.performative = Performatives.inform.value - sub_msg = other_proto_msg.GenericMsg() - sub_msg.text = message_out_content - message.content_class = type(sub_msg).__name__ - message.content = sub_msg.SerializeToString() - logger.debug("Going to send %s", message) - await self.send_message(message, sender_addr) - - # shutdown if no more open conversations - if len(self.conversations) == 0: - await self.shutdown() diff --git a/examples/tcp_container_example.py b/examples/tcp_container_example.py deleted file mode 100644 index 3022697..0000000 --- a/examples/tcp_container_example.py +++ /dev/null @@ -1,43 +0,0 @@ -import asyncio - -from examples.simple_agent import SimpleAgent -from mango import create_container - - -async def one_container_two_agents(): - # ip and port of container - addr1 = ("127.0.0.1", 5555) - from mango.messages.codecs import JSON - - codec = JSON() - container1 = await create_container(connection_type="tcp", codec=codec, addr=addr1) - agent_a = SimpleAgent(container1, codec=codec) - - agent_id1 = agent_a.aid - - # initialize an agent in container2 and tell him the id of the first agent - agent_b = SimpleAgent(container1, agent_id1, addr1, codec=codec) - - # let agent_b start a conversation by sending a greeting - await agent_b.send_greeting() - try: - # wait for all agents to be shutdown in the container - await asyncio.wait_for(asyncio.gather(container1._no_agents_running), timeout=3) - except KeyboardInterrupt: - print("KeyboardInterrupt") - finally: - print(f"[{addr1}]: Shutting down container") - await container1.shutdown() - - -def two_container_two_agents(): - addr1 = (("127.0.0.1", 5555),) - addr2 = (("127.0.0.1", 5555),) - - -if __name__ == "__main__": - import logging - - logging.basicConfig(level="INFO") - - asyncio.run(one_container_two_agents()) diff --git a/examples/tutorial/v1_basic_setup_and_message_passing.py b/examples/tutorial/v1_basic_setup_and_message_passing.py deleted file mode 100644 index 2432be1..0000000 --- a/examples/tutorial/v1_basic_setup_and_message_passing.py +++ /dev/null @@ -1,50 +0,0 @@ -import asyncio - -from mango import Agent, create_container - -""" -For your first mango tutorial you will learn the fundamentals of creating mango agents and containers as well -as making them communicate with each other. - -This example covers: - - container - - agent creation - - basic message passing - - clean shutdown of containers -""" - -PV_CONTAINER_ADDRESS = ("localhost", 5555) - - -class PVAgent(Agent): - def __init__(self, container): - super().__init__(container) - print(f"Hello I am a PV agent! My id is {self.aid}.") - - def handle_message(self, content, meta): - print(f"Received message with content: {content} and meta {meta}.") - - -async def main(): - # defaults to tcp connection - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS) - - # agents always live inside a container - pv_agent_0 = PVAgent(pv_container) - pv_agent_1 = PVAgent(pv_container) - - # we can now send a simple message to an agent and observe that it is received: - # Note that as of now agent IDs are set automatically as agent0, agent1, ... in order of instantiation. - await pv_container.send_message( - "Hello, this is a simple message.", - receiver_addr=PV_CONTAINER_ADDRESS, - receiver_id="agent0", - ) - - # don't forget to properly shut down containers at the end of your program - # otherwise you will get an asyncio.exceptions.CancelledError - await pv_container.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/tutorial/v2_inter_container_messaging_and_basic_functionality.py b/examples/tutorial/v2_inter_container_messaging_and_basic_functionality.py deleted file mode 100644 index 906e016..0000000 --- a/examples/tutorial/v2_inter_container_messaging_and_basic_functionality.py +++ /dev/null @@ -1,199 +0,0 @@ -import asyncio - -from mango import Agent, create_container -from mango.messages.message import Performatives - -""" -In the previous example, you have learned how to create mango agents and containers and -how to send basic messages between them. -In this example, you expand upon this. We introduce a controller agent that asks the current feed_in of our PV agents -and subsequently limits the output of both to their minimum. - -This example covers: - - message passing between different containers - - basic task scheduling - - setting custom agent ids - - use of ACL metadata -""" - -PV_CONTAINER_ADDRESS = ("localhost", 5555) -CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) -PV_FEED_IN = { - "PV Agent 0": 2.0, - "PV Agent 1": 1.0, -} - - -class PVAgent(Agent): - def __init__(self, container, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.max_feed_in = -1 - - def handle_message(self, content, meta): - performative = meta["performative"] - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - - if performative == Performatives.request: - # ask_feed_in message - self.handle_ask_feed_in(sender_addr, sender_id) - elif performative == Performatives.propose: - # set_max_feed_in message - self.handle_set_feed_in_max(content, sender_addr, sender_id) - else: - print( - f"{self.aid}: Received an unexpected message with content {content} and meta {meta}" - ) - - def handle_ask_feed_in(self, sender_addr, sender_id): - reported_feed_in = PV_FEED_IN[self.aid] # PV_FEED_IN must be defined at the top - content = reported_feed_in - - acl_meta = { - "sender_addr": self.addr, - "sender_id": self.aid, - "performative": Performatives.inform, - } - - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=sender_addr, - receiver_id=sender_id, - acl_metadata=acl_meta, - ) - ) - - def handle_set_feed_in_max(self, max_feed_in, sender_addr, sender_id): - self.max_feed_in = float(max_feed_in) - print(f"{self.aid}: Limiting my feed_in to {max_feed_in}") - self.schedule_instant_task( - self.send_acl_message( - content=None, - receiver_addr=sender_addr, - receiver_id=sender_id, - acl_metadata={"performative": Performatives.accept_proposal}, - ) - ) - - -class ControllerAgent(Agent): - def __init__(self, container, known_agents, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.known_agents = known_agents - self.reported_feed_ins = [] - self.reported_acks = 0 - self.reports_done = None - self.acks_done = None - - def handle_message(self, content, meta): - performative = meta["performative"] - if performative == Performatives.inform: - # feed_in_reply message - self.handle_feed_in_reply(content) - elif performative == Performatives.accept_proposal: - # set_max_ack message - self.handle_set_max_ack() - else: - print( - f"{self.aid}: Received an unexpected message with content {content} and meta {meta}" - ) - - def handle_feed_in_reply(self, feed_in_value): - self.reported_feed_ins.append(float(feed_in_value)) - if len(self.reported_feed_ins) == len(self.known_agents): - if self.reports_done is not None: - self.reports_done.set_result(True) - - def handle_set_max_ack(self): - self.reported_acks += 1 - if self.reported_acks == len(self.known_agents): - if self.acks_done is not None: - self.acks_done.set_result(True) - - async def run(self): - # we define an asyncio future to await replies from all known pv agents: - self.reports_done = asyncio.Future() - self.acks_done = asyncio.Future() - - # Note: For messages passed between different containers (i.e. over the network socket) it is expected - # that the message is an ACLMessage object. We can let the container wrap our content in such an - # object with using the send_acl_message method. - # We distinguish the types of messages we send by adding a type field to our content. - - # ask pv agent feed-ins - for addr, aid in self.known_agents: - content = None - acl_meta = { - "sender_addr": self.addr, - "sender_id": self.aid, - "performative": Performatives.request, - } - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - ) - - # wait for both pv agents to answer - await self.reports_done - - # limit both pv agents to the smaller ones feed-in - print(f"{self.aid}: received feed_ins: {self.reported_feed_ins}") - min_feed_in = min(self.reported_feed_ins) - - for addr, aid in self.known_agents: - content = min_feed_in - acl_meta = { - "sender_addr": self.addr, - "sender_id": self.aid, - "performative": Performatives.propose, - } - - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=content, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - ) - - # wait for both pv agents to acknowledge the change - await self.acks_done - - -async def main(): - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS) - controller_container = await create_container(addr=CONTROLLER_CONTAINER_ADDRESS) - - # agents always live inside a container - pv_agent_0 = PVAgent(pv_container, suggested_aid="PV Agent 0") - pv_agent_1 = PVAgent(pv_container, suggested_aid="PV Agent 1") - - # We pass info of the pv agents addresses to the controller here directly. - # In reality, we would use some kind of discovery mechanism for this. - known_agents = [ - (PV_CONTAINER_ADDRESS, pv_agent_0.aid), - (PV_CONTAINER_ADDRESS, pv_agent_1.aid), - ] - - controller_agent = ControllerAgent( - controller_container, known_agents, suggested_aid="Controller" - ) - - # the only active component in this setup - await controller_agent.run() - - # always properly shut down your containers - await pv_container.shutdown() - await controller_container.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/tutorial/v3_codecs_and_typing.py b/examples/tutorial/v3_codecs_and_typing.py deleted file mode 100644 index 1eedb04..0000000 --- a/examples/tutorial/v3_codecs_and_typing.py +++ /dev/null @@ -1,213 +0,0 @@ -import asyncio -from dataclasses import dataclass - -from mango import Agent, create_container -from mango.messages import codecs - -""" -In example 2 we created some basic agent functionality and established inter-container communication. -To distinguish message types we used a corresponding field in our content dictionary. This approach is -tedious and prone to error. A better way is to use dedicated message objects and using their types to distinguish -messages. Arbitrary objects can be encoded for messaging between agents by mangos codecs. - -This example covers: - - message classes - - codec basics - - the json_serializable decorator -""" - -PV_CONTAINER_ADDRESS = ("localhost", 5555) -CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) -PV_FEED_IN = { - "PV Agent 0": 2.0, - "PV Agent 1": 1.0, -} - - -@codecs.json_serializable -@dataclass -class AskFeedInMsg: - pass - - -@codecs.json_serializable -@dataclass -class FeedInReplyMsg: - feed_in: int - - -@codecs.json_serializable -@dataclass -class SetMaxFeedInMsg: - max_feed_in: int - - -@codecs.json_serializable -@dataclass -class MaxFeedInAck: - pass - - -class PVAgent(Agent): - def __init__(self, container, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.max_feed_in = -1 - - def handle_message(self, content, meta): - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - - if isinstance(content, AskFeedInMsg): - self.handle_ask_feed_in(sender_addr, sender_id) - elif isinstance(content, SetMaxFeedInMsg): - self.handle_set_feed_in_max(content.max_feed_in, sender_addr, sender_id) - else: - print(f"{self.aid}: Received a message of unknown type {type(content)}") - - def handle_ask_feed_in(self, sender_addr, sender_id): - reported_feed_in = PV_FEED_IN[self.aid] # PV_FEED_IN must be defined at the top - msg = FeedInReplyMsg(reported_feed_in) - - self.schedule_instant_task( - self.send_acl_message( - content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) - ) - - def handle_set_feed_in_max(self, max_feed_in, sender_addr, sender_id): - self.max_feed_in = float(max_feed_in) - print(f"{self.aid}: Limiting my feed_in to {max_feed_in}") - msg = MaxFeedInAck() - - self.schedule_instant_task( - self.send_acl_message( - content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) - ) - - -class ControllerAgent(Agent): - def __init__(self, container, known_agents, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.known_agents = known_agents - self.reported_feed_ins = [] - self.reported_acks = 0 - self.reports_done = None - self.acks_done = None - - def handle_message(self, content, meta): - if isinstance(content, FeedInReplyMsg): - self.handle_feed_in_reply(content.feed_in) - elif isinstance(content, MaxFeedInAck): - self.handle_set_max_ack() - else: - print(f"{self.aid}: Received a message of unknown type {type(content)}") - - def handle_feed_in_reply(self, feed_in_value): - self.reported_feed_ins.append(float(feed_in_value)) - if len(self.reported_feed_ins) == len(self.known_agents): - if self.reports_done is not None: - self.reports_done.set_result(True) - - def handle_set_max_ack(self): - self.reported_acks += 1 - if self.reported_acks == len(self.known_agents): - if self.acks_done is not None: - self.acks_done.set_result(True) - - async def run(self): - # we define an asyncio future to await replies from all known pv agents: - self.reports_done = asyncio.Future() - self.acks_done = asyncio.Future() - - # Note: For messages passed between different containers (i.e. over the network socket) it is expected - # that the message is an ACLMessage object. We can let the container wrap our content in such an - # object using the send_acl_message method. - # We distinguish the types of messages we send by adding a type field to our content. - - # ask pv agent feed-ins - for addr, aid in self.known_agents: - msg = AskFeedInMsg() - acl_meta = {"sender_addr": self.addr, "sender_id": self.aid} - - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=msg, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - ) - - # wait for both pv agents to answer - await self.reports_done - - # limit both pv agents to the smaller ones feed-in - print(f"{self.aid} received feed_ins: {self.reported_feed_ins}") - min_feed_in = min(self.reported_feed_ins) - - for addr, aid in self.known_agents: - msg = SetMaxFeedInMsg(min_feed_in) - acl_meta = {"sender_addr": self.addr, "sender_id": self.aid} - - # alternatively we could call send_acl_message here directly and await it - self.schedule_instant_task( - self.send_acl_message( - content=msg, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - ) - - # wait for both pv agents to acknowledge the change - await self.acks_done - - -async def main(): - # If no codec is given, every container automatically creates a new JSON codec. - # Now, we set up the codecs explicitely and pass them the neccessary extra serializers. - - # Both types of agents need to be able to handle the same message types (either for serialization - # or deserializaion). In general, a serializer is passed to the codec by three values: - # (type, serialize_method, deserialize_method) - # - # the @json_serializable decorater creates these automatically for simple classes and - # provides them as a tuple via the __serializer__ method on the class. - my_codec = codecs.JSON() - my_codec.add_serializer(*AskFeedInMsg.__serializer__()) - my_codec.add_serializer(*SetMaxFeedInMsg.__serializer__()) - my_codec.add_serializer(*FeedInReplyMsg.__serializer__()) - my_codec.add_serializer(*MaxFeedInAck.__serializer__()) - - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS, codec=my_codec) - - controller_container = await create_container( - addr=CONTROLLER_CONTAINER_ADDRESS, codec=my_codec - ) - - pv_agent_0 = PVAgent(pv_container, suggested_aid="PV Agent 0") - pv_agent_1 = PVAgent(pv_container, suggested_aid="PV Agent 1") - - known_agents = [ - (PV_CONTAINER_ADDRESS, pv_agent_0.aid), - (PV_CONTAINER_ADDRESS, pv_agent_1.aid), - ] - - controller_agent = ControllerAgent( - controller_container, known_agents, suggested_aid="Controller" - ) - await controller_agent.run() - - # always properly shut down your containers - await pv_container.shutdown() - await controller_container.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/examples/tutorial/v4_scheduling_and_roles.py b/examples/tutorial/v4_scheduling_and_roles.py deleted file mode 100644 index f9f6585..0000000 --- a/examples/tutorial/v4_scheduling_and_roles.py +++ /dev/null @@ -1,326 +0,0 @@ -import asyncio -from dataclasses import dataclass - -# note that our imports changed because we now use the specialized RoleAgent superclass -from mango import Role, RoleAgent, create_container -from mango.messages import codecs - -""" -In example 3, you restructured your code to use codecs for easier handling of typed message objects. -Now we want to expand the functionality of our controller. In addition to setting the maximum feed_in -of the pv agents, the controller should now also periodically check if the pv agents are still reachable. - -To achieve this, the controller should seend a regular "ping" message to each pv agent that is in turn answered -by a corresponding "pong". To properly serparate different responsibilities within agents, mango has a role -system where each role covers the functionalities of a responsibility. - -A role is a python object that can be assigned to a RoleAgent. The two main functions each role implements are: - __init__ - where you do the initial object setup - setup - which is called when the role is assigned to an agent - -This distinction is relevant because only within `setup` the -RoleContext (i.e. access to the parent agent and container) exist. -Thus, things like message handlers that require container knowledge are introduced there. - -This example covers: - - scheduling and periodic tasks - - role API basics -""" - -PV_CONTAINER_ADDRESS = ("localhost", 5555) -CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) -PV_FEED_IN = { - "PV Agent 0": 2.0, - "PV Agent 1": 1.0, -} - - -# To separate the agents functionalities we introduce four roles: -# - a ping role for sending out periodic "are you alive" messages -# - a pong role for replying to ping messages -# - a pvrole for setting and reporting feed_in -# - a controller role for sending out feed_in requests and setting maximum feed_ins -class PingRole(Role): - def __init__(self, ping_recipients, time_between_pings): - super().__init__() - self.ping_recipients = ping_recipients - self.time_between_pings = time_between_pings - self.ping_counter = 0 - self.expected_pongs = [] - - def setup(self): - self.context.subscribe_message( - self, self.handle_pong, lambda content, meta: isinstance(content, Pong) - ) - - self.context.schedule_periodic_task(self.send_pings, self.time_between_pings) - - async def send_pings(self): - for addr, aid in self.ping_recipients: - ping_id = self.ping_counter - msg = Ping(ping_id) - meta = {"sender_addr": self.context.addr, "sender_id": self.context.aid} - - await self.context.send_acl_message( - msg, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=meta, - ) - self.expected_pongs.append(ping_id) - self.ping_counter += 1 - - def handle_pong(self, content, meta): - if content.pong_id in self.expected_pongs: - print( - f"Pong {self.context.aid}: Received an expected pong with ID: {content.pong_id}" - ) - self.expected_pongs.remove(content.pong_id) - else: - print( - f"Pong {self.context.aid}: Received an unexpected pong with ID: {content.pong_id}" - ) - - -class PongRole(Role): - def setup(self): - self.context.subscribe_message( - self, self.handle_ping, lambda content, meta: isinstance(content, Ping) - ) - - def handle_ping(self, content, meta): - ping_id = content.ping_id - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - answer = Pong(ping_id) - - print(f"Ping {self.context.aid}: Received a ping with ID: {ping_id}") - - # message sending from roles is done via the RoleContext - self.context.schedule_instant_task( - self.context.send_acl_message( - answer, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) - ) - - -class PVRole(Role): - def __init__(self): - super().__init__() - self.max_feed_in = -1 - - def setup(self): - self.context.subscribe_message( - self, - self.handle_ask_feed_in, - lambda content, meta: isinstance(content, AskFeedInMsg), - ) - self.context.subscribe_message( - self, - self.handle_set_feed_in_max, - lambda content, meta: isinstance(content, SetMaxFeedInMsg), - ) - - def handle_ask_feed_in(self, content, meta): - reported_feed_in = PV_FEED_IN[ - self.context.aid - ] # PV_FEED_IN must be defined at the top - msg = FeedInReplyMsg(reported_feed_in) - - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - - self.context.schedule_instant_task( - self.context.send_acl_message( - content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) - ) - - def handle_set_feed_in_max(self, content, meta): - max_feed_in = float(content.max_feed_in) - self.max_feed_in = max_feed_in - print(f"{self.context.aid}: Limiting my feed_in to {max_feed_in}") - - msg = MaxFeedInAck() - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - - self.context.schedule_instant_task( - self.context.send_acl_message( - content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, - ) - ) - - -class ControllerRole(Role): - def __init__(self, known_agents): - super().__init__() - self.known_agents = known_agents - self.reported_feed_ins = [] - self.reported_acks = 0 - self.reports_done = None - self.acks_done = None - - def setup(self): - self.context.subscribe_message( - self, - self.handle_feed_in_reply, - lambda content, meta: isinstance(content, FeedInReplyMsg), - ) - - self.context.subscribe_message( - self, - self.handle_set_max_ack, - lambda content, meta: isinstance(content, MaxFeedInAck), - ) - - self.context.schedule_instant_task(self.run()) - - def handle_feed_in_reply(self, content, meta): - feed_in_value = float(content.feed_in) - - self.reported_feed_ins.append(feed_in_value) - if len(self.reported_feed_ins) == len(self.known_agents): - if self.reports_done is not None: - self.reports_done.set_result(True) - - def handle_set_max_ack(self, content, meta): - self.reported_acks += 1 - if self.reported_acks == len(self.known_agents): - if self.acks_done is not None: - self.acks_done.set_result(True) - - async def run(self): - # we define an asyncio future to await replies from all known pv agents: - self.reports_done = asyncio.Future() - self.acks_done = asyncio.Future() - - # ask pv agent feed-ins - for addr, aid in self.known_agents: - msg = AskFeedInMsg() - acl_meta = {"sender_addr": self.context.addr, "sender_id": self.context.aid} - - await self.context.send_acl_message( - content=msg, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - - # wait for both pv agents to answer - await self.reports_done - - # limit both pv agents to the smaller ones feed-in - print(f"Controller received feed_ins: {self.reported_feed_ins}") - min_feed_in = min(self.reported_feed_ins) - - for addr, aid in self.known_agents: - msg = SetMaxFeedInMsg(min_feed_in) - acl_meta = {"sender_addr": self.context.addr, "sender_id": self.context.aid} - - await self.context.send_acl_message( - content=msg, - receiver_addr=addr, - receiver_id=aid, - acl_metadata=acl_meta, - ) - - # wait for both pv agents to acknowledge the change - await self.acks_done - - -@codecs.json_serializable -@dataclass -class AskFeedInMsg: - pass - - -@codecs.json_serializable -@dataclass -class FeedInReplyMsg: - feed_in: int - - -@codecs.json_serializable -@dataclass -class SetMaxFeedInMsg: - max_feed_in: int - - -@codecs.json_serializable -@dataclass -class MaxFeedInAck: - pass - - -@codecs.json_serializable -@dataclass -class Ping: - ping_id: int - - -@codecs.json_serializable -@dataclass -class Pong: - pong_id: int - - -class PVAgent(RoleAgent): - def __init__(self, container, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.add_role(PongRole()) - self.add_role(PVRole()) - - -class ControllerAgent(RoleAgent): - def __init__(self, container, known_agents, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) - self.add_role(PingRole(known_agents, 2)) - self.add_role(ControllerRole(known_agents)) - - -async def main(): - my_codec = codecs.JSON() - my_codec.add_serializer(*AskFeedInMsg.__serializer__()) - my_codec.add_serializer(*SetMaxFeedInMsg.__serializer__()) - my_codec.add_serializer(*FeedInReplyMsg.__serializer__()) - my_codec.add_serializer(*MaxFeedInAck.__serializer__()) - - # dont forget to add our new serializers - my_codec.add_serializer(*Ping.__serializer__()) - my_codec.add_serializer(*Pong.__serializer__()) - - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS, codec=my_codec) - - controller_container = await create_container( - addr=CONTROLLER_CONTAINER_ADDRESS, codec=my_codec - ) - - pv_agent_0 = PVAgent(pv_container, suggested_aid="PV Agent 0") - pv_agent_1 = PVAgent(pv_container, suggested_aid="PV Agent 1") - - known_agents = [ - (PV_CONTAINER_ADDRESS, pv_agent_0.aid), - (PV_CONTAINER_ADDRESS, pv_agent_1.aid), - ] - - controller_agent = ControllerAgent( - controller_container, known_agents, suggested_aid="Controller" - ) - - # no more run call since everything now happens automatically within the roles - await asyncio.sleep(5) - - # always properly shut down your containers - await pv_container.shutdown() - await controller_container.shutdown() - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/mango/__init__.py b/mango/__init__.py index d9dc7de..1e893d6 100644 --- a/mango/__init__.py +++ b/mango/__init__.py @@ -1,3 +1,7 @@ -from .agent.core import Agent +from .messages.message import create_acl +from .agent.core import Agent, AgentAddress from .agent.role import Role, RoleAgent, RoleContext -from .container.factory import create as create_container +from .container.factory import create_tcp as create_tcp_container, create_mqtt as create_mqtt_container, create_external_coupling as create_ec_container +from .express.api import activate, run_with_mqtt, run_with_tcp, agent_composed_of, PrintingAgent, sender_addr, addr +from .util.distributed_clock import DistributedClockAgent, DistributedClockManager +from .util.clock import ExternalClock \ No newline at end of file diff --git a/mango/agent/core.py b/mango/agent/core.py index 205033f..e610ba2 100644 --- a/mango/agent/core.py +++ b/mango/agent/core.py @@ -4,21 +4,25 @@ Every agent must live in a container. Containers are responsible for making connections to other agents. """ - +from dataclasses import dataclass import asyncio import logging from abc import ABC -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any -from ..container.core import Container from ..util.scheduling import ScheduledProcessTask, ScheduledTask, Scheduler +from ..util.clock import Clock logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class AgentAddress: + addr: Any + aid: str class AgentContext: def __init__(self, container) -> None: - self._container: Container = container + self._container = container @property def current_timestamp(self) -> float: @@ -27,6 +31,10 @@ def current_timestamp(self) -> float: """ return self._container.clock.time + @property + def clock(self) -> Clock: + return self._container.clock + @property def addr(self): return self._container.addr @@ -41,91 +49,72 @@ def deregister_agent(self, aid): async def send_message( self, content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, + receiver_addr: AgentAddress, + sender_id: None | str = None, **kwargs, ): """ See container.send_message(...) """ return await self._container.send_message( - content, receiver_addr=receiver_addr, receiver_id=receiver_id, **kwargs - ) - - async def send_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs, - ): - """ - See container.send_acl_message(...) - """ - return await self._container.send_acl_message( - content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - **kwargs, + content, receiver_addr=receiver_addr, sender_id=sender_id, **kwargs ) class AgentDelegates: - def __init__(self, context, scheduler) -> None: - self._context: AgentContext = context - self._scheduler: Scheduler = scheduler + def __init__(self) -> None: + self.context: AgentContext = None + self.scheduler: Scheduler = None + self._aid = None + + def on_start(self): + """Called when container started in which the agent is contained + """ + pass + + def on_ready(self): + """Called when all container has been started using activate(...). + """ + pass @property def current_timestamp(self) -> float: """ Method that returns the current unix timestamp given the clock within the container """ - return self._context.current_timestamp + return self.context.current_timestamp + + @property + def aid(self): + return self._aid @property def addr(self): - return self._context.addr + """Return the address of the agent as AgentAddress + + Returns: + _type_: AgentAddress + """ + return AgentAddress(self.context.addr, self.aid) async def send_message( self, content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, + receiver_addr: AgentAddress, **kwargs, ): """ See container.send_message(...) """ - return await self._context.send_message( - content, receiver_addr=receiver_addr, receiver_id=receiver_id, **kwargs + return await self.context.send_message( + content, receiver_addr=receiver_addr, sender_id=self.aid, **kwargs ) - async def send_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs, - ): - """ - See container.send_acl_message(...) - """ - return await self._context.send_acl_message( - content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - **kwargs, - ) def schedule_instant_message( self, content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, + receiver_addr: AgentAddress, **kwargs, ): """ @@ -134,44 +123,13 @@ def schedule_instant_message( :param content: The content of the message :param receiver_addr: The address passed to the container - :param receiver_id: The agent id of the receiver :param kwargs: Additional parameters to provide protocol specific settings :returns: asyncio.Task for the scheduled coroutine """ return self.schedule_instant_task( self.send_message( - content, receiver_addr=receiver_addr, receiver_id=receiver_id, **kwargs - ) - ) - - def schedule_instant_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs, - ): - """ - Schedules sending an acl message without any delay. This is equivalent to using the schedulers 'schedule_instant_task' with the coroutine created by - 'container.send_acl_message'. - - :param content: The content of the message - :param receiver_addr: The address passed to the container - :param receiver_id: The agent id of the receiver - :param acl_metadata: Metadata for the acl message - :param kwargs: Additional parameters to provide protocol specific settings - :returns: asyncio.Task for the scheduled coroutine - """ - - return self.schedule_instant_task( - self.send_acl_message( - content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - **kwargs, + content, receiver_addr=receiver_addr, **kwargs ) ) @@ -194,7 +152,7 @@ def schedule_conditional_process_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_conditional_process_task( + return self.scheduler.schedule_conditional_process_task( coroutine_creator=coroutine_creator, condition_func=condition_func, lookup_delay=lookup_delay, @@ -216,7 +174,7 @@ def schedule_conditional_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_conditional_task( + return self.scheduler.schedule_conditional_task( coroutine=coroutine, condition_func=condition_func, lookup_delay=lookup_delay, @@ -236,7 +194,7 @@ def schedule_timestamp_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_timestamp_task( + return self.scheduler.schedule_timestamp_task( coroutine=coroutine, timestamp=timestamp, on_stop=on_stop, src=src ) @@ -252,7 +210,7 @@ def schedule_timestamp_process_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_timestamp_process_task( + return self.scheduler.schedule_timestamp_process_task( coroutine_creator=coroutine_creator, timestamp=timestamp, on_stop=on_stop, @@ -273,7 +231,7 @@ def schedule_periodic_process_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_periodic_process_task( + return self.scheduler.schedule_periodic_process_task( coroutine_creator=coroutine_creator, delay=delay, on_stop=on_stop, src=src ) @@ -289,7 +247,7 @@ def schedule_periodic_task(self, coroutine_func, delay, on_stop=None, src=None): :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_periodic_task( + return self.scheduler.schedule_periodic_task( coroutine_func=coroutine_func, delay=delay, on_stop=on_stop, src=src ) @@ -307,7 +265,7 @@ def schedule_recurrent_process_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_recurrent_process_task( + return self.scheduler.schedule_recurrent_process_task( coroutine_creator=coroutine_creator, recurrency=recurrency, on_stop=on_stop, @@ -328,7 +286,7 @@ def schedule_recurrent_task( :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_recurrent_task( + return self.scheduler.schedule_recurrent_task( coroutine_func=coroutine_func, recurrency=recurrency, on_stop=on_stop, @@ -345,7 +303,7 @@ def schedule_instant_process_task(self, coroutine_creator, on_stop=None, src=Non :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_instant_process_task( + return self.scheduler.schedule_instant_process_task( coroutine_creator=coroutine_creator, on_stop=on_stop, src=src ) @@ -359,7 +317,7 @@ def schedule_instant_task(self, coroutine, on_stop=None, src=None): :param src: creator of the task :type src: Object """ - return self._scheduler.schedule_instant_task( + return self.scheduler.schedule_instant_task( coroutine=coroutine, on_stop=on_stop, src=src ) @@ -370,7 +328,7 @@ def schedule_process_task(self, task: ScheduledProcessTask, src=None): :param task: task to be scheduled :param src: object, which represents the source of the task (for example the object in which the task got created) """ - return self._scheduler.schedule_process_task(task, src=src) + return self.scheduler.schedule_process_task(task, src=src) def schedule_task(self, task: ScheduledTask, src=None): """Schedule a task with asyncio. When the task is finished, if finite, its automatically @@ -379,14 +337,14 @@ def schedule_task(self, task: ScheduledTask, src=None): :param task: task to be scheduled :param src: object, which represents the source of the task (for example the object in which the task got created) """ - return self._scheduler.schedule_task(task, src=src) + return self.scheduler.schedule_task(task, src=src) async def tasks_complete(self, timeout=1): """Wait for all scheduled tasks to complete using a timeout. :param timeout: waiting timeout. Defaults to 1. """ - await self._scheduler.tasks_complete(timeout=timeout) + await self.scheduler.tasks_complete(timeout=timeout) class Agent(ABC, AgentDelegates): @@ -394,34 +352,51 @@ class Agent(ABC, AgentDelegates): def __init__( self, - container: Container, - suggested_aid: str = None, - suspendable_tasks=True, - observable_tasks=True, ): - """Initialize an agent and register it with its container - :param container: The container that the agent lives in. Must be a Container - :param suggested_aid: (Optional) suggested aid, if the aid is already taken, a generated aid is used. - Using the generated aid-style ("agentX") is not allowed. - """ - scheduler = Scheduler( - clock=container.clock, - suspendable=suspendable_tasks, - observable=observable_tasks, - ) - context = AgentContext(container) - self.aid = context.register_agent(self, suggested_aid) + """ + Initialize an agent + """ + + super().__init__() + self.inbox = asyncio.Queue() - super().__init__(context, scheduler) + @property + def observable_tasks(self): + return self.scheduler.observable + + @observable_tasks.setter + def observable_tasks(self, value: bool): + self.scheduler.observable = value + + @property + def suspendable_tasks(self): + return self.scheduler.suspendable + @suspendable_tasks.setter + def suspendable_tasks(self, value: bool): + self.self.scheduler.suspendable = value + + def on_register(self): + """ + Hook-in to define behavior of the agent directly after it got registered by a container + """ + pass + + def _do_register(self, container, aid): self._check_inbox_task = asyncio.create_task(self._check_inbox()) - self._check_inbox_task.add_done_callback(self.raise_exceptions) + self._check_inbox_task.add_done_callback(self._raise_exceptions) self._stopped = asyncio.Future() + self._aid = aid + self.context = AgentContext(container) + self.scheduler = Scheduler( + suspendable=True, + observable=True, + clock=container.clock + ) + self.on_register() - logger.info("Agent %s: start running in container %s", self.aid, self.addr) - - def raise_exceptions(self, fut: asyncio.Future): + def _raise_exceptions(self, fut: asyncio.Future): """ Inline function used as a callback to raise exceptions :param fut: The Future object of the task @@ -455,7 +430,7 @@ async def _check_inbox(self): except Exception: logger.exception("The check inbox task of %s failed!", self.aid) - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): """ Has to be implemented by the user. @@ -472,21 +447,21 @@ async def shutdown(self): if not self._stopped.done(): self._stopped.set_result(True) - self._context.deregister_agent(self.aid) + self.context.deregister_agent(self.aid) try: # Shutdown reactive inbox task - self._check_inbox_task.remove_done_callback(self.raise_exceptions) + self._check_inbox_task.remove_done_callback(self._raise_exceptions) self._check_inbox_task.cancel() await self._check_inbox_task except asyncio.CancelledError: pass try: - await self._scheduler.stop() + await self.scheduler.stop() except asyncio.CancelledError: pass try: - await self._scheduler.shutdown() + await self.scheduler.shutdown() except asyncio.CancelledError: pass finally: - logger.info("Agent %s: Shutdown successful", self.aid) + logger.info("Agent %s: Shutdown successful", self.aid) \ No newline at end of file diff --git a/mango/agent/role.py b/mango/agent/role.py index 4a9ea05..e05ab44 100644 --- a/mango/agent/role.py +++ b/mango/agent/role.py @@ -34,9 +34,9 @@ import asyncio from abc import ABC -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable -from mango.agent.core import Agent, AgentContext, AgentDelegates +from mango.agent.core import Agent, AgentContext, AgentDelegates, AgentAddress from mango.util.scheduling import Scheduler @@ -61,63 +61,10 @@ def update(self, data: dict): self.__setattr__(k, v) -class RoleContext: +class Role: pass -class Role(ABC): - """General role class, defining the API every role can use. A role implements one responsibility - of an agent. - - Every role - must be added to a :class:`RoleAgent` and is defined by some lifecycle methods: - - * :func:`Role.setup` is called when the Role is added to the agent, so its the perfect place for - initialization and scheduling of tasks - * :func:`Role.on_stop` is called when the container the agent lives in, is shut down - - To interact with the environment you have to use the context, accessible via :func:Role.context. - """ - - def __init__(self) -> None: - """Initialize the roles internals. - !!Care!! the role context is unknown at this point! - """ - self._context = None - - def bind(self, context: RoleContext) -> None: - """Method used internal to set the context, do not override! - - :param context: the role context - """ - self._context = context - - @property - def context(self) -> RoleContext: - """Return the context of the role. This context can be send as bridge to the agent. - - :return: the context of the role - """ - return self._context - - def setup(self) -> None: - """Lifecycle hook in, which will be called on adding the role to agent. The role context - is known from hereon. - """ - - def on_change_model(self, model) -> None: - """Will be invoked when a subscribed model changes via :func:`RoleContext.update`. - - :param model: the model - """ - - def on_deactivation(self, src) -> None: - """Hook in, which will be called when another role deactivates this instance (temporarily)""" - - async def on_stop(self) -> None: - """Lifecycle hook in, which will be called when the container is shut down or if the role got removed.""" - - class RoleHandler: """Contains all roles and their models. Implements the communication between roles.""" @@ -191,7 +138,7 @@ def remove_role(self, role: Role) -> None: del self._role_to_active[role] @property - def roles(self) -> List[Role]: + def roles(self) -> list[Role]: """Returns all roles Returns: @@ -227,7 +174,7 @@ async def on_stop(self): for role in self._roles: await role.on_stop() - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): """Handle an incoming message, delegating it to all applicable subscribers .. code-block:: python @@ -239,55 +186,22 @@ def handle_message(self, content, meta: Dict[str, Any]): :param content: content :param meta: meta """ + for role in self.roles: + role.handle_message(content, meta) for role, message_condition, method, _ in self._message_subs: if self._is_role_active(role) and message_condition(content, meta): method(content, meta) - def _notify_send_message_subs(self, content, receiver_addr, receiver_id, **kwargs): + def _notify_send_message_subs(self, content, receiver_addr: AgentAddress, **kwargs): for role in self._send_msg_subs: for sub in self._send_msg_subs[role]: if self._is_role_active(role): sub( content=content, receiver_addr=receiver_addr, - receiver_id=receiver_id, **kwargs, ) - async def send_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - **kwargs, - ): - self._notify_send_message_subs(content, receiver_addr, receiver_id, **kwargs) - return await self._agent_context.send_message( - content=content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - **kwargs, - ) - - async def send_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs, - ): - self._notify_send_message_subs(content, receiver_addr, receiver_id, **kwargs) - return await self._agent_context.send_acl_message( - content=content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - **kwargs, - ) - def subscribe_message(self, role, method, message_condition, priority=0): if len(self._message_subs) == 0: self._message_subs.append((role, message_condition, method, priority)) @@ -319,23 +233,28 @@ def subscribe_event(self, role: Role, event_type: type, method: Callable): self._role_event_type_to_handler[event_type] = [] self._role_event_type_to_handler[event_type] += [(role, method)] - + + def on_start(self): + for role in self.roles: + role.on_start() + + def on_ready(self): + for role in self.roles: + role.on_ready() class RoleContext(AgentDelegates): """Implementation of the RoleContext.""" def __init__( self, - agent_context: AgentContext, - scheduler: Scheduler, role_handler: RoleHandler, aid: str, inbox, ): - self._agent_context = agent_context + super().__init__() + self._agent_context = None self._role_handler = role_handler self._aid = aid - self._scheduler = scheduler self._inbox = inbox @property @@ -394,7 +313,7 @@ def remove_role(self, role: Role): self._role_handler.remove_role(role) asyncio.create_task(role.on_stop()) - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): """Handle an incoming message, delegating it to all applicable subscribers .. code-block:: python @@ -411,34 +330,17 @@ def handle_message(self, content, meta: Dict[str, Any]): async def send_message( self, content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, + receiver_addr: AgentAddress, **kwargs, ): - return await self._role_handler.send_message( + self._role_handler._notify_send_message_subs(content, receiver_addr, **kwargs) + return await self._agent_context.send_message( content=content, receiver_addr=receiver_addr, - receiver_id=receiver_id, + sender_id=self.aid, **kwargs, ) - async def send_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - **kwargs, - ): - return await self._role_handler.send_acl_message( - content=content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - **kwargs, - ) def emit_event(self, event: Any, event_source: Any = None): """Emit an custom event to other roles. @@ -474,6 +376,12 @@ def deactivate(self, role) -> None: def activate(self, role) -> None: self._role_handler.activate(role) + + def on_start(self): + self._role_handler.on_start() + + def on_ready(self): + self._role_handler.on_ready() class RoleAgent(Agent): @@ -482,11 +390,7 @@ class RoleAgent(Agent): """ def __init__( - self, - container, - suggested_aid: str = None, - suspendable_tasks=True, - observable_tasks=True, + self ): """Create a role-agent @@ -494,17 +398,24 @@ def __init__( :param suggested_aid: (Optional) suggested aid, if the aid is already taken, a generated aid is used. Using the generated aid-style ("agentX") is not allowed. """ - super().__init__( - container, - suggested_aid=suggested_aid, - suspendable_tasks=suspendable_tasks, - observable_tasks=observable_tasks, - ) - - self._role_handler = RoleHandler(self._context, self._scheduler) + super().__init__() + self._role_handler = RoleHandler(None, None) self._role_context = RoleContext( - self._context, self._scheduler, self._role_handler, self.aid, self.inbox + self._role_handler, self.aid, self.inbox ) + + def on_start(self): + self._role_context.on_start() + + def on_ready(self): + self._role_context.on_ready() + + def on_register(self): + self._role_context._agent_context = self.context + self._role_handler._agent_context = self.context + self._role_context.scheduler = self.scheduler + self._role_handler._scheduler = self.scheduler + self._role_context._aid = self.aid def add_role(self, role: Role): """Add a role to the agent. This will lead to the call of :func:`Role.setup`. @@ -522,16 +433,83 @@ def remove_role(self, role: Role): self._role_context.remove_role(role) @property - def roles(self) -> List[Role]: + def roles(self) -> list[Role]: """Returns list of roles :return: list of roles """ return self._role_handler.roles - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): self._role_context.handle_message(content, meta) async def shutdown(self): await self._role_handler.on_stop() await super().shutdown() + + + +class Role(ABC): + """General role class, defining the API every role can use. A role implements one responsibility + of an agent. + + Every role + must be added to a :class:`RoleAgent` and is defined by some lifecycle methods: + + * :func:`Role.setup` is called when the Role is added to the agent, so its the perfect place for + initialization and scheduling of tasks + * :func:`Role.on_stop` is called when the container the agent lives in, is shut down + + To interact with the environment you have to use the context, accessible via :func:Role.context. + """ + + def __init__(self) -> None: + """Initialize the roles internals. + !!Care!! the role context is unknown at this point! + """ + self._context = None + + def bind(self, context: RoleContext) -> None: + """Method used internal to set the context, do not override! + + :param context: the role context + """ + self._context = context + + @property + def context(self) -> RoleContext: + """Return the context of the role. This context can be send as bridge to the agent. + + :return: the context of the role + """ + return self._context + + def setup(self) -> None: + """Lifecycle hook in, which will be called on adding the role to agent. The role context + is known from hereon. + """ + + def on_change_model(self, model) -> None: + """Will be invoked when a subscribed model changes via :func:`RoleContext.update`. + + :param model: the model + """ + + def on_deactivation(self, src) -> None: + """Hook in, which will be called when another role deactivates this instance (temporarily)""" + + async def on_stop(self) -> None: + """Lifecycle hook in, which will be called when the container is shut down or if the role got removed.""" + + def on_start(self) -> None: + """Called when container started in which the agent is contained + """ + pass + + def on_ready(self): + """Called after the start of all container using activate + """ + pass + + def handle_message(self, content: Any, meta: dict): + pass diff --git a/mango/container/core.py b/mango/container/core.py index d8c3689..1318d0a 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -1,521 +1,19 @@ import asyncio import copy import logging -import os -import warnings from abc import ABC, abstractmethod -from dataclasses import dataclass -from multiprocessing import Event, Process -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, TypeVar -import dill # noqa F401 # do not remove! Necessary for the auto loaded pickle reg extensions - -from ..messages.codecs import ACLMessage, Codec +from ..messages.codecs import Codec from ..util.clock import Clock -from ..util.multiprocessing import AioDuplex, PipeToWriteQueue, aioduplex +from .mp import MirrorContainerProcessManager, MainContainerProcessManager, cancel_and_wait_for_task +from ..agent.core import Agent, AgentAddress logger = logging.getLogger(__name__) AGENT_PATTERN_NAME_PRE = "agent" -WAIT_STEP = 0.01 - - -class IPCEventType(enumerate): - """Available IPC event types for event process container communication""" - - AIDS = 1 - DISPATCH = 2 - - -@dataclass -class IPCEvent: - """IPCEvent data container.""" - - type: IPCEventType - data: object - pid: int - - -@dataclass -class ContainerMirrorData: - """Container for the data necessary for setting up a mirror container in another process""" - - message_pipe: AioDuplex - event_pipe: AioDuplex - terminate_event: Event - main_queue: asyncio.Queue - - -@dataclass -class ContainerData: - """Container for the data neccessary for the creation of all container implementations""" - - addr: object - codec: Codec - clock: Clock - kwargs: dict - - -async def cancel_and_wait_for_task(task): - """Utility to cancel and wait for a task. - - :param task: task to be canceled - :type task: asyncio.Task - """ - try: - task.cancel() - await task - except asyncio.CancelledError: - pass - except EOFError: - pass - - -def create_agent_process_environment( - container_data: ContainerData, - agent_creator, - mirror_container_creator, - message_pipe: AioDuplex, - main_queue: asyncio.Queue, - event_pipe: AioDuplex, - terminate_event: Event, - process_initialized_event: Event, -): - """Create the agent process environment for using agent subprocesses - in a mango container. This routine will create a new event loop and run - the so-called agent-loop, which will - - 1. initialize the mirror-container and the agents - 2. will wait and return to the event loop until there is a terminate signal - * while this step, the container and its agents are responsive - 3. shutdown the mirror container - - :param container_data: data for the mirror container creation - :type container_data: ContainerData - :param agent_creator: function, which will be called with the mirror container to create and initialize all agents - :type agent_creator: Function(Container) - :param mirror_container_creator: function, which will create a mirror container - :type mirror_container_creator: Function(ContainerData, AsyncIoEventLoop, AioDuplex, AioDuplex, ) - :param message_pipe: Pipe for messages - :type message_pipe: AioDuplex - :param event_pipe: Pipe for events - :type event_pipe: AioDuplex - :param terminate_event: Event which signals termination of the main container - :type terminate_event: Event - :param process_initialized_event: Event signaling to the main container, that the environment is done with initializing - :type process_initialized_event: Event - """ - asyncio.set_event_loop(asyncio.new_event_loop()) - - async def start_agent_loop(): - container = mirror_container_creator( - container_data, - asyncio.get_event_loop(), - message_pipe, - main_queue, - event_pipe, - terminate_event, - ) - if asyncio.iscoroutinefunction(agent_creator): - await agent_creator(container) - else: - agent_creator(container) - process_initialized_event.set() - - while not terminate_event.is_set(): - await asyncio.sleep(WAIT_STEP) - await container.shutdown() - - asyncio.run(start_agent_loop()) - - -@dataclass -class AgentProcessHandle: - """Represents the handle for the agent process. It is awaitable for - the initializing of agent process. Further it contains the pid of the process. - """ - - init: asyncio.Task - pid: int - - def __await__(self): - return self.init.__await__() - - -class BaseContainerProcessManager: - """Base class for the two different container process manager types: mirror process manager, and main process manager. - These process manager change the flow of the container to add the subprocess feature natively in the container - itself. For these purposes this base class defines some hook in's which are commonly used by all implementations. - - However, there are some methods exclusive to one of the two types of process managers. These methods will either, - return None, or raise a NotImplementedError. - """ - - @property - def aids(self): - """ - List of aids living in subprocesses. - """ - return [] - - def handle_message_in_sp(self, message, receiver_id, priority, meta): - """Called when a message should be handled by the process manager. This - happens when the receiver id is unknown to the main container itself. - - :param message: the message - :type message: Any - :param receiver_id: aid of the receiver - :type receiver_id: str - :param priority: prio - :type priority: int - :param meta: meta - :type meta: dict - :raises NotImplementedError: generally not implemented in mirror container manager - """ - - raise NotImplementedError( - f"{self}-{receiver_id}-{self._container._agents.keys()}" - ) - - def create_agent_process(self, agent_creator, container, mirror_container_creator): - """Creates a process with an agent, which is created by agent_creator. Further, - the mirror_container_creator will create a mirror container, which replaces the - original container for all agents which live in its process. - - :param agent_creator: function with one argument 'container', which creates all agents, which - shall live in the process - :type agent_creator: Function(Container) - :param container: the main container - :type container: Container - :param mirror_container_creator: function, which creates a mirror container, given container - data and IPC connection data - :type mirror_container_creator: Function(ContainerData, AsyncioLoop, AioDuplex, AioDuplex, Event) - :raises NotImplementedError: generally raised if the manager is a mirror manager - """ - raise NotImplementedError() - - def pre_hook_send_internal_message( - self, message, receiver_id, priority, default_meta - ): - """Hook in before an internal message is sent. Capable of preventing the default - send_internal_message call. - Therefore this method is able to reroute messages without side effects. - - :param message: the message - :type message: Any - :param receiver_id: aid - :type receiver_id: str - :param priority: prio - :type priority: 0 - :param default_meta: meta - :type default_meta: dict - :return: Tuple, first the status (True, False = successful, unsuccessful and prevent - the original send_internal_message, None = Continue original call), second the Queue-like - inbox, in which the message should be redirected in - :rtype: Tuple[Boolean, Queue-like] - """ - return None, None - - def pre_hook_reserve_aid(self, suggested_aid=None): - """Hook in before an aid is reserved. Capable of preventing the default - reserve_aid call. - - :param suggested_aid: the aid, defaults to None - :type suggested_aid: str, optional - :return: aid, can be None if the original reserve_aid should be executed - :rtype: str - """ - return None - - def dispatch_to_agent_process(self, pid: int, coro_func, *args): - """Dispatches a coroutine function to another process. The `coroutine_func` - and its arguments are serialized and sent to the agent process, in which it - is executed with the Container as first argument (followed by the defined arguments). - - :param pid: the pid - :type pid: int - :param coro_func: coro function, which shall be executed - :type coro_func: async def - :raises NotImplementedError: raises a NotImplementedError if mirror manager - """ - raise NotImplementedError() - - async def shutdown(self): - """Clean up all process related stuff. - - :raises NotImplementedError: Should never be raised - """ - raise NotImplementedError() - - -class MirrorContainerProcessManager(BaseContainerProcessManager): - """Internal Manager class, responsible for the implementation of operations necessary for the agent processes - in the mirror container. - """ - - def __init__( - self, - container, - mirror_data: ContainerMirrorData, - ) -> None: - self._container = container - self._mirror_data = mirror_data - self._out_queue = asyncio.Queue() - - self._fetch_from_ipc_task = asyncio.create_task( - self._move_incoming_messages_to_inbox( - self._mirror_data.message_pipe, - self._mirror_data.terminate_event, - ) - ) - self._send_to_message_pipe_task = asyncio.create_task( - self._send_to_message_pipe( - self._mirror_data.message_pipe, - self._mirror_data.terminate_event, - ) - ) - self._execute_dispatch_events_task = asyncio.create_task( - self._execute_dispatch_event(self._mirror_data.event_pipe) - ) - - async def _execute_dispatch_event(self, event_pipe: AioDuplex): - try: - async with event_pipe.open_readonly() as rx: - while True: - event: IPCEvent = await rx.read_object() - if event.type == IPCEventType.DISPATCH: - coro_func, args = event.data - try: - await coro_func(self._container, *args) - except Exception: - logger.exception("A dispatched coroutine has failed!") - except EOFError: - # other side disconnected -> task not necessry anymore - pass - except Exception: - logger.exception("The Dispatch Event Loop has failed!") - - async def _move_incoming_messages_to_inbox( - self, message_pipe: AioDuplex, terminate_event: Event - ): - try: - async with message_pipe.open_readonly() as rx: - while not terminate_event.is_set(): - priority, message, meta = await rx.read_object() - - receiver = self._container._agents.get(meta["receiver_id"], None) - if receiver is None: - logger.error( - "A message was routed to the wrong process, as the %s doesn't contain a known receiver-id", - meta, - ) - target_inbox = receiver.inbox - target_inbox.put_nowait((priority, message, meta)) - - except Exception: - logger.exception("The Move Message Task Loop has failed!") - - async def _send_to_message_pipe( - self, message_pipe: AioDuplex, terminate_event: Event - ): - try: - async with message_pipe.open_writeonly() as tx: - while not terminate_event.is_set(): - data = await self._out_queue.get() - - tx.write_object(data) - await tx.drain() - - except Exception: - logger.exception("The Send Message Task Loop has failed!") - - def pre_hook_send_internal_message( - self, message, receiver_id, priority, default_meta - ): - self._out_queue.put_nowait((message, receiver_id, priority, default_meta)) - return True, None - - def pre_hook_reserve_aid(self, suggested_aid=None): - ipc_event = IPCEvent(IPCEventType.AIDS, suggested_aid, os.getpid()) - self._mirror_data.event_pipe.write_connection.send(ipc_event) - return self._mirror_data.event_pipe.read_connection.recv() - - async def shutdown(self): - await cancel_and_wait_for_task(self._fetch_from_ipc_task) - await cancel_and_wait_for_task(self._execute_dispatch_events_task) - await cancel_and_wait_for_task(self._send_to_message_pipe_task) - - -class MainContainerProcessManager(BaseContainerProcessManager): - """Internal Manager class, responsible for the implementation of operations necessary for the agent processes - in the main container. - """ - - def __init__( - self, - container, - ) -> None: - self._active = False - self._container = container - self._mp_enabled = False - - def _init_mp(self): - # For agent multiprocessing support - self._agent_processes = [] - self._terminate_sub_processes = Event() - self._pid_to_message_pipe = {} - self._pid_to_pipe = {} - self._pid_to_aids = {} - self._handle_process_events_tasks: List[asyncio.Task] = [] - self._handle_sp_messages_tasks: List[asyncio.Task] = [] - self._main_queue = None - self._mp_enabled = True - - @property - def aids(self): - all_aids = [] - if self._active: - for _, aids in self._pid_to_aids.items(): - all_aids += aids - return all_aids - - async def _handle_process_events(self, pipe): - try: - async with pipe.open() as (rx, tx): - while True: - event: IPCEvent = await rx.read_object() - if event.type == IPCEventType.AIDS: - aid = self._container._reserve_aid(event.data) - if event.pid not in self._pid_to_aids: - self._pid_to_aids[event.pid] = set() - self._pid_to_aids[event.pid].add(aid) - tx.write_object(aid) - await tx.drain() - except EOFError: - # other side disconnected -> task not necessry anymore - pass - except Exception: - logger.exception("The Process Event Loop has failed!") - - async def _handle_process_message(self, pipe: AioDuplex): - try: - async with pipe.open_readonly() as rx: - while True: - ( - message, - receiver_id, - prio, - meta, - ) = await rx.read_object() - - self._container._send_internal_message( - message=message, - receiver_id=receiver_id, - priority=prio, - default_meta=meta, - ) - - except EOFError: - # other side disconnected -> task not necessry anymore - pass - except Exception: - logger.exception("The Process Message Loop has failed!") - - def pre_hook_send_internal_message( - self, message, receiver_id, priority, default_meta - ): - target_inbox = None - if self._active: - target_inbox = self._find_sp_queue(receiver_id) - default_meta["receiver_id"] = receiver_id - return None, target_inbox - - def _find_sp_queue(self, aid): - if not self._mp_enabled: - return None - for pid, aids in self._pid_to_aids.items(): - if aid in aids: - return PipeToWriteQueue(self._pid_to_message_pipe[pid]) - raise ValueError(f"The aid '{aid}' does not exist in any subprocess.") - - def create_agent_process(self, agent_creator, container, mirror_container_creator): - if not self._active: - self._init_mp() - self._active = True - - from_pipe_message, to_pipe_message = aioduplex() - from_pipe, to_pipe = aioduplex() - process_initialized = Event() - with to_pipe.detach() as to_pipe, to_pipe_message.detach() as to_pipe_message: - agent_process = Process( - target=create_agent_process_environment, - args=( - ContainerData( - addr=container.addr, - codec=container.codec, - clock=container.clock, - kwargs=container._kwargs, - ), - agent_creator, - mirror_container_creator, - to_pipe_message, - self._main_queue, - to_pipe, - self._terminate_sub_processes, - process_initialized, - ), - ) - self._agent_processes.append(agent_process) - agent_process.daemon = True - agent_process.start() - - self._pid_to_message_pipe[agent_process.pid] = from_pipe_message - self._pid_to_pipe[agent_process.pid] = from_pipe - self._handle_process_events_tasks.append( - asyncio.create_task(self._handle_process_events(from_pipe)) - ) - self._handle_sp_messages_tasks.append( - asyncio.create_task(self._handle_process_message(from_pipe_message)) - ) - - async def wait_for_process_initialized(): - while not process_initialized.is_set(): - await asyncio.sleep(WAIT_STEP) - - return AgentProcessHandle( - asyncio.create_task(wait_for_process_initialized()), agent_process.pid - ) - - def dispatch_to_agent_process(self, pid: int, coro_func, *args): - assert pid in self._pid_to_pipe - - ipc_event = IPCEvent(IPCEventType.DISPATCH, (coro_func, args), pid) - self._pid_to_pipe[pid].write_connection.send(ipc_event) - - def handle_message_in_sp(self, message, receiver_id, priority, meta): - sp_queue_of_agent = self._find_sp_queue(receiver_id) - if sp_queue_of_agent is None: - logger.warning("Received a message for an unknown receiver;%s", receiver_id) - else: - sp_queue_of_agent.put_nowait((priority, message, meta)) - - async def shutdown(self): - if self._active: - # send a signal to all sub processes to terminate their message feed in's - self._terminate_sub_processes.set() - - for task in self._handle_process_events_tasks: - await cancel_and_wait_for_task(task) - - for task in self._handle_sp_messages_tasks: - await cancel_and_wait_for_task(task) - - # wait for and tidy up processes - for process in self._agent_processes: - process.join() - process.terminate() - process.close() +A = TypeVar("A") class Container(ABC): """Superclass for a mango container""" @@ -541,7 +39,7 @@ def __init__( self.loop: asyncio.AbstractEventLoop = loop # dict of agents. aid: agent instance - self._agents: Dict = {} + self._agents: dict = {} self._aid_counter: int = 0 # counter for aids self.running: bool = True # True until self.shutdown() is called @@ -607,7 +105,7 @@ def _reserve_aid(self, suggested_aid=None): self._aid_counter += 1 return aid - def register_agent(self, agent, suggested_aid: str = None): + def register_agent(self, agent: Agent, suggested_aid: str = None): """ Register *agent* and return the agent id @@ -621,9 +119,24 @@ def register_agent(self, agent, suggested_aid: str = None): self._no_agents_running = asyncio.Future() aid = self._reserve_aid(suggested_aid) self._agents[aid] = agent + agent._do_register(self, aid) logger.debug("Successfully registered agent;%s", aid) return aid + def include(self, agent: A, suggested_aid: str = None) -> A: + """Include the agent in the container. Return the agent for + convenience. + + Args: + agent (Agent): the agent to be included + suggested_aid (str, optional): suggested aid for registration + + Returns: + _type_: the agent included + """ + self.register_agent(agent, suggested_aid=suggested_aid) + return agent + def deregister_agent(self, aid): """ Deregister an agent @@ -638,109 +151,22 @@ def deregister_agent(self, aid): @abstractmethod async def send_message( self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - **kwargs, + content: Any, + receiver_addr: AgentAddress, + sender_id: None | str = None, + **kwargs ) -> bool: """ The Container sends a message to an agent according the container protocol. :param content: The content of the message - :param receiver_addr: In case of TCP this is a tuple of host, port - In case of MQTT this is the topic to publish to. - :param receiver_id: The agent id of the receiver - :param kwargs: Additional parameters to provide protocol specific settings + :param receiver_addr: The address the message is sent to, should be constructed using + agent_address(protocol_addr, aid) or address(agent) on sending messages, + and sender_address(meta) on replying to messages. + :param kwargs: Can contain additional meta information """ raise NotImplementedError - async def send_acl_message( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - is_anonymous_acl=False, - **kwargs, - ) -> bool: - """ - The Container sends a message, wrapped in an ACL message, to an agent according the container protocol. - - :param content: The content of the message - :param receiver_addr: In case of TCP this is a tuple of host, port - In case of MQTT this is the topic to publish to. - :param receiver_id: The agent id of the receiver - :param acl_metadata: metadata for the acl_header. - :param is_anonymous_acl: If set to True, the sender information won't be written in the ACL header - :param kwargs: Additional parameters to provide protocol specific settings - """ - return await self.send_message( - self._create_acl( - content, - receiver_addr=receiver_addr, - receiver_id=receiver_id, - acl_metadata=acl_metadata, - is_anonymous_acl=is_anonymous_acl, - ), - receiver_addr=receiver_addr, - receiver_id=receiver_id, - **kwargs, - ) - - def _create_acl( - self, - content, - receiver_addr: Union[str, Tuple[str, int]], - receiver_id: Optional[str] = None, - acl_metadata: Optional[Dict[str, Any]] = None, - is_anonymous_acl=False, - ): - """ - :param content: - :param receiver_addr: - :param receiver_id: - :param acl_metadata: - :return: - """ - acl_metadata = {} if acl_metadata is None else acl_metadata.copy() - # analyse and complete acl_metadata - if "receiver_addr" not in acl_metadata.keys(): - acl_metadata["receiver_addr"] = receiver_addr - elif acl_metadata["receiver_addr"] != receiver_addr: - warnings.warn( - f"The argument receiver_addr ({receiver_addr}) is not equal to " - f"acl_metadata['receiver_addr'] ({acl_metadata['receiver_addr']}). \ - For consistency, the value in acl_metadata['receiver_addr'] " - f"was overwritten with receiver_addr.", - UserWarning, - ) - acl_metadata["receiver_addr"] = receiver_addr - if receiver_id: - if "receiver_id" not in acl_metadata.keys(): - acl_metadata["receiver_id"] = receiver_id - elif acl_metadata["receiver_id"] != receiver_id: - warnings.warn( - f"The argument receiver_id ({receiver_id}) is not equal to " - f"acl_metadata['receiver_id'] ({acl_metadata['receiver_id']}). \ - For consistency, the value in acl_metadata['receiver_id'] " - f"was overwritten with receiver_id.", - UserWarning, - ) - acl_metadata["receiver_id"] = receiver_id - # add sender_addr if not defined and not anonymous - if not is_anonymous_acl: - if "sender_addr" not in acl_metadata.keys() and self.addr is not None: - acl_metadata["sender_addr"] = self.addr - - message = ACLMessage() - message.content = content - - for key, value in acl_metadata.items(): - setattr(message, key, value) - return message - def _send_internal_message( self, message, @@ -814,7 +240,7 @@ def raise_exceptions(result): self.inbox.task_done() # signals that the queue object is # processed - async def _handle_message(self, *, priority: int, content, meta: Dict[str, Any]): + async def _handle_message(self, *, priority: int, content, meta: dict[str, Any]): """ This is called as a separate task for every message that is read :param priority: priority of the message @@ -871,6 +297,15 @@ def dispatch_to_agent_process(self, pid: int, coro_func, *args): """ self._container_process_manager.dispatch_to_agent_process(pid, coro_func, *args) + async def start(self): + """Start the container. It totally depends on the implementation for what is actually happening.""" + for agent in self._agents.values(): + agent.on_start() + + def on_ready(self): + for agent in self._agents.values(): + agent.on_ready() + async def shutdown(self): """Shutdown all agents in the container and the container itself""" self.running = False diff --git a/mango/container/external_coupling.py b/mango/container/external_coupling.py index 570e9f3..eeee64f 100644 --- a/mango/container/external_coupling.py +++ b/mango/container/external_coupling.py @@ -2,11 +2,13 @@ import logging import time from dataclasses import dataclass -from typing import List, Optional, Tuple, Union -from mango.container.core import Container, ContainerMirrorData +from mango.container.core import Container +from mango.container.mp import ContainerMirrorData +from mango.agent.core import AgentAddress from ..messages.codecs import Codec +from ..messages.message import MangoMessage from ..util.clock import ExternalClock from ..util.termination_detection import tasks_complete_or_sleeping @@ -23,8 +25,8 @@ class ExternalAgentMessage: @dataclass class ExternalSchedulingContainerOutput: duration: float - messages: List[ExternalAgentMessage] - next_activity: Optional[float] + messages: list[ExternalAgentMessage] + next_activity: None | float def ext_mirror_container_creator( @@ -85,30 +87,37 @@ def __init__( async def send_message( self, content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, + receiver_addr: AgentAddress, + sender_id: None | str = None, **kwargs, ) -> bool: """ The Container sends a message to an agent according the container protocol. :param content: The content of the message - :param receiver_addr: Address if the receiving container - :param receiver_id: The agent id of the receiver + :param receiver_addr: Address of the receiving container :param kwargs: Additional parameters to provide protocol specific settings """ message = content - if receiver_addr == self.addr: - if not receiver_id: - receiver_id = message.receiver_id - default_meta = {"network_protocol": "external_connection"} + meta = {} + for key, value in kwargs.items(): + meta[key] = value + meta["sender_id"] = sender_id + meta["sender_addr"] = self.addr + meta["receiver_id"] = receiver_addr.aid + meta["receiver_addr"] = receiver_addr.addr + + if receiver_addr.addr == self.addr: + receiver_id = receiver_addr.aid + meta.update({"network_protocol": "external_connection"}) success = self._send_internal_message( - message=message, receiver_id=receiver_id, default_meta=default_meta + message=message, receiver_id=receiver_id, default_meta=meta ) self._new_internal_message = True else: + if not hasattr(content, "split_content_and_meta"): + message = MangoMessage(content, meta) success = await self._send_external_message(receiver_addr, message) return success @@ -125,14 +134,14 @@ async def _send_external_message(self, addr, message) -> bool: self.message_buffer.append( ExternalAgentMessage( time=time.time() - self.current_start_time_of_step + self.clock.time, - receiver=addr, + receiver=addr.addr, message=encoded_msg, ) ) return True async def step( - self, simulation_time: float, incoming_messages: List[bytes] + self, simulation_time: float, incoming_messages: list[bytes] ) -> ExternalSchedulingContainerOutput: if self.message_buffer: logger.warning( @@ -147,10 +156,10 @@ async def step( for encoded_msg in incoming_messages: message = self.codec.decode(encoded_msg) - content, acl_meta = message.split_content_and_meta() - acl_meta["network_protocol"] = "external_connection" + content, meta = message.split_content_and_meta() + meta["network_protocol"] = "external_connection" - await self.inbox.put((0, content, acl_meta)) + await self.inbox.put((0, content, meta)) # now wait for the msg_queue to be empty await self.inbox.join() @@ -185,9 +194,3 @@ def as_agent_process( agent_creator=agent_creator, mirror_container_creator=mirror_container_creator, ) - - async def shutdown(self): - """ - calls shutdown() from super class Container - """ - await super().shutdown() diff --git a/mango/container/factory.py b/mango/container/factory.py index 8b446bf..38a0a6a 100644 --- a/mango/container/factory.py +++ b/mango/container/factory.py @@ -1,8 +1,6 @@ import asyncio import logging -from typing import Any, Dict, Optional, Tuple, Union - -import paho.mqtt.client as paho +from typing import Any from mango.container.core import Container from mango.container.external_coupling import ExternalSchedulingContainer @@ -19,223 +17,75 @@ MQTT_CONNECTION = "mqtt" EXTERNAL_CONNECTION = "external_connection" +def create_mqtt(broker_addr: tuple | dict | str, + client_id: str, + codec: Codec = None, + clock: Clock = None, + inbox_topic: str | None = None, + copy_internal_messages: bool = False, + **kwargs, +): + if codec is None: + codec = JSON() + if clock is None: + clock = AsyncioClock() + + return MQTTContainer( + client_id=client_id, + broker_addr=broker_addr, + loop=asyncio.get_running_loop(), + clock=clock, + codec=codec, + inbox_topic=inbox_topic, + copy_internal_messages=copy_internal_messages, + **kwargs, + ) + +def create_external_coupling( + codec: Codec = None, + clock: Clock = None, + addr: None | str | tuple[str, int] = None, + **kwargs: dict[str, Any], +): + if codec is None: + codec = JSON() + if clock is None: + clock = ExternalClock() + + return ExternalSchedulingContainer( + addr=addr, loop=asyncio.get_running_loop(), codec=codec, clock=clock, **kwargs + ) + -async def create( - *, - connection_type: str = "tcp", +def create_tcp( + addr: str | tuple[str, int], codec: Codec = None, clock: Clock = None, - addr: Optional[Union[str, Tuple[str, int]]] = None, copy_internal_messages: bool = False, - mqtt_kwargs: Dict[str, Any] = None, - **kwargs: Dict[str, Any], + **kwargs: dict[str, Any], ) -> Container: """ - This method is called to instantiate a container instance, either - a TCPContainer or a MQTTContainer, depending on the parameter - connection_type. + This method is called to instantiate a tcp container - :param connection_type: Defines the connection type. So far only 'tcp' - or 'mqtt' are allowed :param codec: Defines the codec to use. Defaults to JSON :param clock: The clock that the scheduler of the agent should be based on. Defaults to the AsyncioClock - :param addr: the address to use. If connection_type == 'tcp': it has - to be a tuple of (host, port). If connection_type == 'mqtt' this can - optionally define an inbox_topic that is used similarly than - a tcp address. - :param mqtt_kwargs: Dictionary of keyword arguments for connection to a mqtt broker. At least - the keys 'broker_addr' and 'client_id' have to be provided. - Ignored if connection_type != 'mqtt' - - :return: The instance of a MQTTContainer or a TCPContainer + :param addr: the address to use. it has to be a tuple of (host, port). + + :return: The instance of a TCPContainer """ - connection_type = connection_type.lower() - if connection_type not in [TCP_CONNECTION, MQTT_CONNECTION, EXTERNAL_CONNECTION]: - raise ValueError(f"Unknown connection type {connection_type}") - - loop = asyncio.get_running_loop() - - # not initialized by the default parameter because then - # containers could unexpectedly share the same codec object if codec is None: codec = JSON() - if clock is None: - if connection_type == EXTERNAL_CONNECTION: - clock = ExternalClock() - else: - clock = AsyncioClock() - - if connection_type == TCP_CONNECTION: - # initialize TCPContainer - container = TCPContainer( - addr=addr, - codec=codec, - loop=loop, - clock=clock, - copy_internal_messages=copy_internal_messages, - **kwargs, - ) - await container.setup() - return container - - if connection_type == EXTERNAL_CONNECTION: - return ExternalSchedulingContainer( - addr=addr, loop=loop, codec=codec, clock=clock - ) - - if connection_type == MQTT_CONNECTION: - # get and check relevant kwargs from mqtt_kwargs - # client_id - client_id = mqtt_kwargs.pop("client_id", None) - if not client_id: - raise ValueError("client_id is requested within mqtt_kwargs") - - # broker_addr - broker_addr = mqtt_kwargs.pop("broker_addr", None) - if not broker_addr: - raise ValueError("broker_addr is requested within mqtt_kwargs") - - # get parameters for Client.init() - init_kwargs = {} - possible_init_kwargs = ( - "clean_session", - "userdata", - "protocol", - "transport", - ) - for possible_kwarg in possible_init_kwargs: - if possible_kwarg in mqtt_kwargs.keys(): - init_kwargs[possible_kwarg] = mqtt_kwargs.pop(possible_kwarg) - - # check if addr is a valid topic without wildcards - if addr is not None and ( - not isinstance(addr, str) or "#" in addr or "+" in addr - ): - raise ValueError( - "addr is not set correctly. It is used as " - "inbox topic and must be a string without " - "any wildcards ('#' or '+')" - ) - - # create paho.Client object for mqtt communication - mqtt_messenger: paho.Client = paho.Client( - paho.CallbackAPIVersion.VERSION2, client_id=client_id, **init_kwargs - ) - - # set TLS options if provided - # expected as a dict: - # {ca_certs, certfile, keyfile, cert_eqs, tls_version, ciphers} - tls_kwargs = mqtt_kwargs.pop("tls_kwargs", None) - if tls_kwargs: - mqtt_messenger.tls_set(**tls_kwargs) - - # Future that is triggered, on successful connection - connected = asyncio.Future() - - # callbacks to check for successful connection - def on_con(client, userdata, flags, reason_code, properties): - logger.info("Connection Callback with the following flags: %s", flags) - loop.call_soon_threadsafe(connected.set_result, reason_code) - - mqtt_messenger.on_connect = on_con - - # check broker_addr input and connect - if isinstance(broker_addr, tuple): - if not 0 < len(broker_addr) < 4: - raise ValueError("Invalid broker address argument count") - if len(broker_addr) > 0 and not isinstance(broker_addr[0], str): - raise ValueError("Invalid broker address - host must be str") - if len(broker_addr) > 1 and not isinstance(broker_addr[1], int): - raise ValueError("Invalid broker address - port must be int") - if len(broker_addr) > 2 and not isinstance(broker_addr[2], int): - raise ValueError("Invalid broker address - keepalive must be int") - mqtt_messenger.connect(*broker_addr, **mqtt_kwargs) - - elif isinstance(broker_addr, dict): - if "hostname" not in broker_addr.keys(): - raise ValueError("Invalid broker address - host not given") - mqtt_messenger.connect(**broker_addr, **mqtt_kwargs) - - else: - if not isinstance(broker_addr, str): - raise ValueError("Invalid broker address") - mqtt_messenger.connect(broker_addr, **mqtt_kwargs) - - logger.info("[%s]: Going to connect to broker at %s..", client_id, broker_addr) - - counter = 0 - # process MQTT messages for maximum of 10 seconds to - # receive connection callback - while not connected.done() and counter < 100: - mqtt_messenger.loop() - # wait for the thread to trigger the future - await asyncio.sleep(0.1) - counter += 1 - - if not connected.done(): - # timeout - raise ConnectionError( - f"Connection to {broker_addr} could not be " - f"established after {counter * 0.1} seconds" - ) - if connected.result() != 0: - raise ConnectionError( - f"Connection to {broker_addr} could not be " - f"set up. Callback returner error code " - f"{connected.result()}" - ) - - logger.info("sucessfully connected to mqtt broker") - if addr is not None: - # connection has been set up, subscribe to inbox topic now - logger.info( - "[%s]: Going to subscribe to %s as inbox topic..", client_id, addr - ) - - # create Future that is triggered on successful subscription - subscribed = asyncio.Future() - - # set up subscription callback - def on_sub(client, userdata, mid, reason_code_list, properties): - loop.call_soon_threadsafe(subscribed.set_result, True) - - mqtt_messenger.on_subscribe = on_sub - - # subscribe topic - result, _ = mqtt_messenger.subscribe(addr, 2) - if result != paho.MQTT_ERR_SUCCESS: - # subscription to inbox topic was not successful - mqtt_messenger.disconnect() - raise ConnectionError( - f"Subscription request to {addr} at {broker_addr} " - f"returned error code: {result}" - ) - - counter = 0 - while not subscribed.done() and counter < 100: - # wait for subscription - mqtt_messenger.loop(timeout=0.1) - await asyncio.sleep(0.1) - counter += 1 - if not subscribed.done(): - raise ConnectionError( - f"Subscription request to {addr} at {broker_addr} " - f"did not succeed after {counter * 0.1} seconds." - ) - logger.info("successfully subscribed to topic") - - # connection and subscription is successful, remove callbacks - mqtt_messenger.on_subscribe = None - mqtt_messenger.on_connect = None - - return MQTTContainer( - client_id=client_id, - addr=addr, - loop=loop, - clock=clock, - mqtt_client=mqtt_messenger, - codec=codec, - copy_internal_messages=copy_internal_messages, - **kwargs, - ) + clock = AsyncioClock() + if isinstance(addr, str): + addr = tuple(addr.split(":")) + + # initialize TCPContainer + return TCPContainer( + addr=addr, + codec=codec, + loop=asyncio.get_running_loop(), + clock=clock, + copy_internal_messages=copy_internal_messages, + **kwargs, + ) diff --git a/mango/container/mp.py b/mango/container/mp.py new file mode 100644 index 0000000..691e928 --- /dev/null +++ b/mango/container/mp.py @@ -0,0 +1,513 @@ + +import asyncio +import os +import logging +from dataclasses import dataclass +from multiprocessing import Event, Process +from multiprocessing.synchronize import Event as MultiprocessingEvent + +import dill # noqa F401 # do not remove! Necessary for the auto loaded pickle reg extensions + +from ..messages.codecs import Codec +from ..util.clock import Clock +from ..util.multiprocessing import AioDuplex, PipeToWriteQueue, aioduplex + +logger = logging.getLogger(__name__) + +WAIT_STEP = 0.01 + +class IPCEventType(enumerate): + """Available IPC event types for event process container communication""" + + AIDS = 1 + DISPATCH = 2 + + +@dataclass +class IPCEvent: + """IPCEvent data container.""" + + type: IPCEventType + data: object + pid: int + + +@dataclass +class ContainerMirrorData: + """Container for the data necessary for setting up a mirror container in another process""" + + message_pipe: AioDuplex + event_pipe: AioDuplex + terminate_event: MultiprocessingEvent + main_queue: asyncio.Queue + + +@dataclass +class ContainerData: + """Container for the data neccessary for the creation of all container implementations""" + + addr: object + codec: Codec + clock: Clock + kwargs: dict + + +async def cancel_and_wait_for_task(task): + """Utility to cancel and wait for a task. + + :param task: task to be canceled + :type task: asyncio.Task + """ + try: + task.cancel() + await task + except asyncio.CancelledError: + pass + except EOFError: + pass + + +def create_agent_process_environment( + container_data: ContainerData, + agent_creator, + mirror_container_creator, + message_pipe: AioDuplex, + main_queue: asyncio.Queue, + event_pipe: AioDuplex, + terminate_event: MultiprocessingEvent, + process_initialized_event: MultiprocessingEvent, +): + """Create the agent process environment for using agent subprocesses + in a mango container. This routine will create a new event loop and run + the so-called agent-loop, which will + + 1. initialize the mirror-container and the agents + 2. will wait and return to the event loop until there is a terminate signal + * while this step, the container and its agents are responsive + 3. shutdown the mirror container + + :param container_data: data for the mirror container creation + :type container_data: ContainerData + :param agent_creator: function, which will be called with the mirror container to create and initialize all agents + :type agent_creator: Function(Container) + :param mirror_container_creator: function, which will create a mirror container + :type mirror_container_creator: Function(ContainerData, AsyncIoEventLoop, AioDuplex, AioDuplex, ) + :param message_pipe: Pipe for messages + :type message_pipe: AioDuplex + :param event_pipe: Pipe for events + :type event_pipe: AioDuplex + :param terminate_event: Event which signals termination of the main container + :type terminate_event: Event + :param process_initialized_event: Event signaling to the main container, that the environment is done with initializing + :type process_initialized_event: Event + """ + asyncio.set_event_loop(asyncio.new_event_loop()) + + async def start_agent_loop(): + container = mirror_container_creator( + container_data, + asyncio.get_event_loop(), + message_pipe, + main_queue, + event_pipe, + terminate_event, + ) + if asyncio.iscoroutinefunction(agent_creator): + await agent_creator(container) + else: + agent_creator(container) + process_initialized_event.set() + + while not terminate_event.is_set(): + await asyncio.sleep(WAIT_STEP) + await container.shutdown() + + asyncio.run(start_agent_loop()) + + +@dataclass +class AgentProcessHandle: + """Represents the handle for the agent process. It is awaitable for + the initializing of agent process. Further it contains the pid of the process. + """ + + init: asyncio.Task + pid: int + + def __await__(self): + return self.init.__await__() + + +class BaseContainerProcessManager: + """Base class for the two different container process manager types: mirror process manager, and main process manager. + These process manager change the flow of the container to add the subprocess feature natively in the container + itself. For these purposes this base class defines some hook in's which are commonly used by all implementations. + + However, there are some methods exclusive to one of the two types of process managers. These methods will either, + return None, or raise a NotImplementedError. + """ + + @property + def aids(self): + """ + List of aids living in subprocesses. + """ + return [] + + def handle_message_in_sp(self, message, receiver_id, priority, meta): + """Called when a message should be handled by the process manager. This + happens when the receiver id is unknown to the main container itself. + + :param message: the message + :type message: Any + :param receiver_id: aid of the receiver + :type receiver_id: str + :param priority: prio + :type priority: int + :param meta: meta + :type meta: dict + :raises NotImplementedError: generally not implemented in mirror container manager + """ + + raise NotImplementedError( + f"{self}-{receiver_id}-{self._container._agents.keys()}" + ) + + def create_agent_process(self, agent_creator, container, mirror_container_creator): + """Creates a process with an agent, which is created by agent_creator. Further, + the mirror_container_creator will create a mirror container, which replaces the + original container for all agents which live in its process. + + :param agent_creator: function with one argument 'container', which creates all agents, which + shall live in the process + :type agent_creator: Function(Container) + :param container: the main container + :type container: Container + :param mirror_container_creator: function, which creates a mirror container, given container + data and IPC connection data + :type mirror_container_creator: Function(ContainerData, AsyncioLoop, AioDuplex, AioDuplex, Event) + :raises NotImplementedError: generally raised if the manager is a mirror manager + """ + raise NotImplementedError() + + def pre_hook_send_internal_message( + self, message, receiver_id, priority, default_meta + ): + """Hook in before an internal message is sent. Capable of preventing the default + send_internal_message call. + Therefore this method is able to reroute messages without side effects. + + :param message: the message + :type message: Any + :param receiver_id: aid + :type receiver_id: str + :param priority: prio + :type priority: 0 + :param default_meta: meta + :type default_meta: dict + :return: Tuple, first the status (True, False = successful, unsuccessful and prevent + the original send_internal_message, None = Continue original call), second the Queue-like + inbox, in which the message should be redirected in + :rtype: Tuple[Boolean, Queue-like] + """ + return None, None + + def pre_hook_reserve_aid(self, suggested_aid=None): + """Hook in before an aid is reserved. Capable of preventing the default + reserve_aid call. + + :param suggested_aid: the aid, defaults to None + :type suggested_aid: str, optional + :return: aid, can be None if the original reserve_aid should be executed + :rtype: str + """ + return None + + def dispatch_to_agent_process(self, pid: int, coro_func, *args): + """Dispatches a coroutine function to another process. The `coroutine_func` + and its arguments are serialized and sent to the agent process, in which it + is executed with the Container as first argument (followed by the defined arguments). + + :param pid: the pid + :type pid: int + :param coro_func: coro function, which shall be executed + :type coro_func: async def + :raises NotImplementedError: raises a NotImplementedError if mirror manager + """ + raise NotImplementedError() + + async def shutdown(self): + """Clean up all process related stuff. + + :raises NotImplementedError: Should never be raised + """ + raise NotImplementedError() + + +class MirrorContainerProcessManager(BaseContainerProcessManager): + """Internal Manager class, responsible for the implementation of operations necessary for the agent processes + in the mirror container. + """ + + def __init__( + self, + container, + mirror_data: ContainerMirrorData, + ) -> None: + self._container = container + self._mirror_data = mirror_data + self._out_queue = asyncio.Queue() + + self._fetch_from_ipc_task = asyncio.create_task( + self._move_incoming_messages_to_inbox( + self._mirror_data.message_pipe, + self._mirror_data.terminate_event, + ) + ) + self._send_to_message_pipe_task = asyncio.create_task( + self._send_to_message_pipe( + self._mirror_data.message_pipe, + self._mirror_data.terminate_event, + ) + ) + self._execute_dispatch_events_task = asyncio.create_task( + self._execute_dispatch_event(self._mirror_data.event_pipe) + ) + + async def _execute_dispatch_event(self, event_pipe: AioDuplex): + try: + async with event_pipe.open_readonly() as rx: + while True: + event: IPCEvent = await rx.read_object() + if event.type == IPCEventType.DISPATCH: + coro_func, args = event.data + try: + await coro_func(self._container, *args) + except Exception: + logger.exception("A dispatched coroutine has failed!") + except EOFError: + # other side disconnected -> task not necessry anymore + pass + except Exception: + logger.exception("The Dispatch Event Loop has failed!") + + async def _move_incoming_messages_to_inbox( + self, message_pipe: AioDuplex, terminate_event: MultiprocessingEvent + ): + try: + async with message_pipe.open_readonly() as rx: + while not terminate_event.is_set(): + priority, message, meta = await rx.read_object() + + receiver = self._container._agents.get(meta["receiver_id"], None) + if receiver is None: + logger.error( + "A message was routed to the wrong process, as the %s doesn't contain a known receiver-id", + meta, + ) + target_inbox = receiver.inbox + target_inbox.put_nowait((priority, message, meta)) + + except Exception: + logger.exception("The Move Message Task Loop has failed!") + + async def _send_to_message_pipe( + self, message_pipe: AioDuplex, terminate_event: MultiprocessingEvent + ): + try: + async with message_pipe.open_writeonly() as tx: + while not terminate_event.is_set(): + data = await self._out_queue.get() + + tx.write_object(data) + await tx.drain() + + except Exception: + logger.exception("The Send Message Task Loop has failed!") + + def pre_hook_send_internal_message( + self, message, receiver_id, priority, default_meta + ): + self._out_queue.put_nowait((message, receiver_id, priority, default_meta)) + return True, None + + def pre_hook_reserve_aid(self, suggested_aid=None): + ipc_event = IPCEvent(IPCEventType.AIDS, suggested_aid, os.getpid()) + self._mirror_data.event_pipe.write_connection.send(ipc_event) + return self._mirror_data.event_pipe.read_connection.recv() + + async def shutdown(self): + await cancel_and_wait_for_task(self._fetch_from_ipc_task) + await cancel_and_wait_for_task(self._execute_dispatch_events_task) + await cancel_and_wait_for_task(self._send_to_message_pipe_task) + + +class MainContainerProcessManager(BaseContainerProcessManager): + """Internal Manager class, responsible for the implementation of operations necessary for the agent processes + in the main container. + """ + + def __init__( + self, + container, + ) -> None: + self._active = False + self._container = container + self._mp_enabled = False + + def _init_mp(self): + # For agent multiprocessing support + self._agent_processes = [] + self._terminate_sub_processes = Event() + self._pid_to_message_pipe = {} + self._pid_to_pipe = {} + self._pid_to_aids = {} + self._handle_process_events_tasks: list[asyncio.Task] = [] + self._handle_sp_messages_tasks: list[asyncio.Task] = [] + self._main_queue = None + self._mp_enabled = True + + @property + def aids(self): + all_aids = [] + if self._active: + for _, aids in self._pid_to_aids.items(): + all_aids += aids + return all_aids + + async def _handle_process_events(self, pipe): + try: + async with pipe.open() as (rx, tx): + while True: + event: IPCEvent = await rx.read_object() + if event.type == IPCEventType.AIDS: + aid = self._container._reserve_aid(event.data) + if event.pid not in self._pid_to_aids: + self._pid_to_aids[event.pid] = set() + self._pid_to_aids[event.pid].add(aid) + tx.write_object(aid) + await tx.drain() + except EOFError: + # other side disconnected -> task not necessry anymore + pass + except Exception: + logger.exception("The Process Event Loop has failed!") + + async def _handle_process_message(self, pipe: AioDuplex): + try: + async with pipe.open_readonly() as rx: + while True: + ( + message, + receiver_id, + prio, + meta, + ) = await rx.read_object() + + self._container._send_internal_message( + message=message, + receiver_id=receiver_id, + priority=prio, + default_meta=meta, + ) + + except EOFError: + # other side disconnected -> task not necessry anymore + pass + except Exception: + logger.exception("The Process Message Loop has failed!") + + def pre_hook_send_internal_message( + self, message, receiver_id, priority, default_meta + ): + target_inbox = None + if self._active: + target_inbox = self._find_sp_queue(receiver_id) + default_meta["receiver_id"] = receiver_id + return None, target_inbox + + def _find_sp_queue(self, aid): + if not self._mp_enabled: + return None + for pid, aids in self._pid_to_aids.items(): + if aid in aids: + return PipeToWriteQueue(self._pid_to_message_pipe[pid]) + raise ValueError(f"The aid '{aid}' does not exist in any subprocess.") + + def create_agent_process(self, agent_creator, container, mirror_container_creator): + if not self._active: + self._init_mp() + self._active = True + + from_pipe_message, to_pipe_message = aioduplex() + from_pipe, to_pipe = aioduplex() + process_initialized = Event() + with to_pipe.detach() as to_pipe, to_pipe_message.detach() as to_pipe_message: + agent_process = Process( + target=create_agent_process_environment, + args=( + ContainerData( + addr=container.addr, + codec=container.codec, + clock=container.clock, + kwargs=container._kwargs, + ), + agent_creator, + mirror_container_creator, + to_pipe_message, + self._main_queue, + to_pipe, + self._terminate_sub_processes, + process_initialized, + ), + ) + self._agent_processes.append(agent_process) + agent_process.daemon = True + agent_process.start() + + self._pid_to_message_pipe[agent_process.pid] = from_pipe_message + self._pid_to_pipe[agent_process.pid] = from_pipe + self._handle_process_events_tasks.append( + asyncio.create_task(self._handle_process_events(from_pipe)) + ) + self._handle_sp_messages_tasks.append( + asyncio.create_task(self._handle_process_message(from_pipe_message)) + ) + + async def wait_for_process_initialized(): + while not process_initialized.is_set(): + await asyncio.sleep(WAIT_STEP) + + return AgentProcessHandle( + asyncio.create_task(wait_for_process_initialized()), agent_process.pid + ) + + def dispatch_to_agent_process(self, pid: int, coro_func, *args): + assert pid in self._pid_to_pipe + + ipc_event = IPCEvent(IPCEventType.DISPATCH, (coro_func, args), pid) + self._pid_to_pipe[pid].write_connection.send(ipc_event) + + def handle_message_in_sp(self, message, receiver_id, priority, meta): + sp_queue_of_agent = self._find_sp_queue(receiver_id) + if sp_queue_of_agent is None: + logger.warning("Received a message for an unknown receiver;%s", receiver_id) + else: + sp_queue_of_agent.put_nowait((priority, message, meta)) + + async def shutdown(self): + if self._active: + # send a signal to all sub processes to terminate their message feed in's + self._terminate_sub_processes.set() + + for task in self._handle_process_events_tasks: + await cancel_and_wait_for_task(task) + + for task in self._handle_sp_messages_tasks: + await cancel_and_wait_for_task(task) + + # wait for and tidy up processes + for process in self._agent_processes: + process.join() + process.terminate() + process.close() diff --git a/mango/container/mqtt.py b/mango/container/mqtt.py index 0d08097..9cb477c 100644 --- a/mango/container/mqtt.py +++ b/mango/container/mqtt.py @@ -1,13 +1,15 @@ import asyncio import logging from functools import partial -from typing import Any, Dict, Optional, Set, Tuple, Union +from typing import Any import paho.mqtt.client as paho -from mango.container.core import Container, ContainerMirrorData +from mango.container.mp import ContainerMirrorData +from mango.container.core import Container, AgentAddress from ..messages.codecs import ACLMessage, Codec +from ..messages.message import MangoMessage from ..util.clock import Clock logger = logging.getLogger(__name__) @@ -15,7 +17,7 @@ def mqtt_mirror_container_creator( client_id, - mqtt_client, + inbox_topic, container_data, loop, message_pipe, @@ -25,11 +27,11 @@ def mqtt_mirror_container_creator( ): return MQTTContainer( client_id=client_id, - addr=container_data.addr, + inbox_topic=inbox_topic, + broker_addr=container_data.addr, codec=container_data.codec, clock=container_data.clock, loop=loop, - mqtt_client=mqtt_client, mirror_data=ContainerMirrorData( message_pipe=message_pipe, event_pipe=event_pipe, @@ -52,11 +54,11 @@ def __init__( self, *, client_id: str, - addr: Optional[str], + broker_addr: tuple | dict | str, loop: asyncio.AbstractEventLoop, clock: Clock, - mqtt_client: paho.Client, codec: Codec, + inbox_topic: None | str = None, **kwargs, ): """ @@ -72,23 +74,163 @@ def __init__( allowed """ super().__init__( - codec=codec, addr=addr, loop=loop, clock=clock, name=client_id, **kwargs + codec=codec, addr=broker_addr, loop=loop, clock=clock, name=client_id, **kwargs ) self.client_id: str = client_id - # the configured and connected paho client - self.mqtt_client: paho.Client = mqtt_client - self.inbox_topic: Optional[str] = addr + # the client will be created on start. + self.mqtt_client: paho.Client = None + self.inbox_topic: None | str = inbox_topic # dict mapping additionally subscribed topics to a set of aids - self.additional_subscriptions: Dict[str, Set[str]] = {} + self.additional_subscriptions: dict[str, set[str]] = {} # Future for pending sub requests - self.pending_sub_request: Optional[asyncio.Future] = None + self.pending_sub_request: None | asyncio.Future = None + + async def start(self): + if not self.client_id: + raise ValueError("client_id is required!") + if not self.addr: + raise ValueError("broker_addr is required!") + + # get parameters for Client.init() + init_kwargs = {} + possible_init_kwargs = ( + "clean_session", + "userdata", + "protocol", + "transport", + ) + for possible_kwarg in possible_init_kwargs: + if possible_kwarg in self._kwargs.keys(): + init_kwargs[possible_kwarg] = self._kwargs.pop(possible_kwarg) + + # check if addr is a valid topic without wildcards + if self.inbox_topic is not None and ( + not isinstance(self.inbox_topic, str) or "#" in self.inbox_topic or "+" in self.inbox_topic + ): + raise ValueError( + "inbox topic is not set correctly. It must be a string without any wildcards ('#' or '+')!" + ) + + # create paho.Client object for mqtt communication + mqtt_messenger: paho.Client = paho.Client( + paho.CallbackAPIVersion.VERSION2, client_id=self.client_id, **init_kwargs + ) + + # set TLS options if provided + # expected as a dict: + # {ca_certs, certfile, keyfile, cert_eqs, tls_version, ciphers} + tls_kwargs = self._kwargs.pop("tls_kwargs", None) + if tls_kwargs: + mqtt_messenger.tls_set(**tls_kwargs) + + # Future that is triggered, on successful connection + connected = asyncio.Future() + + # callbacks to check for successful connection + def on_con(client, userdata, flags, reason_code, properties): + logger.info("Connection Callback with the following flags: %s", flags) + self.loop.call_soon_threadsafe(connected.set_result, reason_code) + + mqtt_messenger.on_connect = on_con + + # check broker_addr input and connect + if isinstance(self.addr, tuple): + if not 0 < len(self.addr) < 4: + raise ValueError("Invalid broker address argument count") + if len(self.addr) > 0 and not isinstance(self.addr[0], str): + raise ValueError("Invalid broker address - host must be str") + if len(self.addr) > 1 and not isinstance(self.addr[1], int): + raise ValueError("Invalid broker address - port must be int") + if len(self.addr) > 2 and not isinstance(self.addr[2], int): + raise ValueError("Invalid broker address - keepalive must be int") + mqtt_messenger.connect(*self.addr, **self._kwargs) + + elif isinstance(self.addr, dict): + if "hostname" not in self.addr.keys(): + raise ValueError("Invalid broker address - host not given") + mqtt_messenger.connect(**self.addr, **self._kwargs) + + else: + if not isinstance(self.addr, str): + raise ValueError("Invalid broker address") + mqtt_messenger.connect(self.addr, **self._kwargs) + + logger.info("[%s]: Going to connect to broker at %s..", self.client_id, self.addr) + + counter = 0 + # process MQTT messages for maximum of 10 seconds to + # receive connection callback + while not connected.done() and counter < 100: + mqtt_messenger.loop() + # wait for the thread to trigger the future + await asyncio.sleep(0.1) + counter += 1 + + if not connected.done(): + # timeout + raise ConnectionError( + f"Connection to {self.addr} could not be " + f"established after {counter * 0.1} seconds" + ) + if connected.result() != 0: + raise ConnectionError( + f"Connection to {self.addr} could not be " + f"set up. Callback returner error code " + f"{connected.result()}" + ) + + logger.info("sucessfully connected to mqtt broker") + if self.inbox_topic is not None: + # connection has been set up, subscribe to inbox topic now + logger.info( + "[%s]: Going to subscribe to %s as inbox topic..", self.client_id, self.inbox_topic + ) + # create Future that is triggered on successful subscription + subscribed = asyncio.Future() + + # set up subscription callback + def on_sub(client, userdata, mid, reason_code_list, properties): + self.loop.call_soon_threadsafe(subscribed.set_result, True) + + mqtt_messenger.on_subscribe = on_sub + + # subscribe topic + result, _ = mqtt_messenger.subscribe(self.inbox_topic, 2) + if result != paho.MQTT_ERR_SUCCESS: + # subscription to inbox topic was not successful + mqtt_messenger.disconnect() + raise ConnectionError( + f"Subscription request to {self.inbox_topic} at {self.addr} " + f"returned error code: {result}" + ) + + counter = 0 + while not subscribed.done() and counter < 100: + # wait for subscription + mqtt_messenger.loop(timeout=0.1) + await asyncio.sleep(0.1) + counter += 1 + if not subscribed.done(): + raise ConnectionError( + f"Subscription request to {self.inbox_topic} at {self.addr} " + f"did not succeed after {counter * 0.1} seconds." + ) + logger.info("successfully subscribed to topic") + + # connection and subscription is successful, remove callbacks + mqtt_messenger.on_subscribe = None + mqtt_messenger.on_connect = None + + self.mqtt_client = mqtt_messenger # set the callbacks self._set_mqtt_callbacks() - # start the mqtt client self.mqtt_client.loop_start() + + await super().start() + def _set_mqtt_callbacks(self): """ @@ -149,14 +291,14 @@ def decode_mqtt_message(self, *, topic, payload): content = None decoded = self.codec.decode(payload) - if isinstance(decoded, ACLMessage): + if hasattr(decoded, "split_content_and_meta"): content, meta = decoded.split_content_and_meta() else: content = decoded return content, meta - async def _handle_message(self, *, priority: int, content, meta: Dict[str, Any]): + async def _handle_message(self, *, priority: int, content, meta: dict[str, Any]): """ This is called as a separate task for every message that is read :param priority: priority of the message @@ -192,18 +334,17 @@ async def _handle_message(self, *, priority: int, content, meta: Dict[str, Any]) async def send_message( self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - **kwargs, + content: Any, + receiver_addr: AgentAddress, + sender_id: None | str = None, + **kwargs ): """ The container sends the message of one of its own agents to a specific topic. :param content: The content of the message :param receiver_addr: The topic to publish to. - :param receiver_id: The agent id of the receiver + :param sender_id: The sender aid :param kwargs: Additional parameters to provide protocol specific settings Possible fields: qos: The quality of service to use for publishing @@ -211,29 +352,36 @@ async def send_message( Ignored if connection_type != 'mqtt' """ - # the message is already complete - message = content - # internal message first (if retain Flag is set, it has to be sent to # the broker + meta = {} + for key, value in kwargs.items(): + meta[key] = value + meta["sender_id"] = sender_id + meta["sender_addr"] = self.inbox_topic + meta["receiver_id"] = receiver_addr.aid + actual_mqtt_kwargs = {} if kwargs is None else kwargs if ( - self.addr - and receiver_addr == self.addr + self.inbox_topic + and receiver_addr == self.inbox_topic and not actual_mqtt_kwargs.get("retain", False) ): - meta = { - "topic": self.addr, + meta.update({ + "topic": self.inbox_topic, "qos": actual_mqtt_kwargs.get("qos", 0), "retain": False, "network_protocol": "mqtt", - } + }) return self._send_internal_message( - message, receiver_id, default_meta=meta, inbox=self.inbox + content, receiver_addr.aid, default_meta=meta, inbox=self.inbox ) else: - self._send_external_message(topic=receiver_addr, message=message) + message = content + if not hasattr(content, "split_content_and_meta"): + message = MangoMessage(content, meta) + self._send_external_message(topic=receiver_addr.addr, message=message) return True def _send_external_message(self, *, topic: str, message): @@ -300,7 +448,7 @@ def as_agent_process( mirror_container_creator = partial( mqtt_mirror_container_creator, self.client_id, - self.mqtt_client, + self.inbox_topic, ) return super().as_agent_process( agent_creator=agent_creator, diff --git a/mango/container/protocol.py b/mango/container/protocol.py index 1520eb6..56b4f58 100644 --- a/mango/container/protocol.py +++ b/mango/container/protocol.py @@ -91,14 +91,14 @@ def data_received(self, data): message = self.codec.decode(data) if hasattr(message, "split_content_and_meta"): - content, acl_meta = message.split_content_and_meta() - acl_meta["network_protocol"] = "tcp" + content, meta = message.split_content_and_meta() + meta["network_protocol"] = "tcp" else: - content, acl_meta = message, message + content, meta = message, None # TODO priority is now always 0, # but should be encoded in the message - self.container.inbox.put_nowait((0, content, acl_meta)) + self.container.inbox.put_nowait((0, content, meta)) else: # No complete message in the buffer, nothing more to do. diff --git a/mango/container/tcp.py b/mango/container/tcp.py index 5c36b99..525bfca 100644 --- a/mango/container/tcp.py +++ b/mango/container/tcp.py @@ -6,10 +6,12 @@ import asyncio import logging import time -from typing import Optional, Tuple, Union +from typing import Any -from mango.container.core import Container, ContainerMirrorData +from mango.container.core import Container, AgentAddress +from mango.container.mp import ContainerMirrorData +from ..messages.message import MangoMessage from ..messages.codecs import Codec from ..util.clock import Clock from .protocol import ContainerProtocol @@ -160,7 +162,7 @@ class TCPContainer(Container): def __init__( self, *, - addr: Tuple[str, int], + addr: tuple[str, int], codec: Codec, loop: asyncio.AbstractEventLoop, clock: Clock, @@ -168,7 +170,7 @@ def __init__( ): """ Initializes a TCP container. Do not directly call this method but use - the factory method of **Container** instead + the factory method create_container instead. :param addr: The container address :param codec: The codec to use :param loop: Current event loop @@ -182,7 +184,7 @@ def __init__( **kwargs, ) - self.server = None # will be set within setup + self.server = None # will be set within start self.running = True self._tcp_connection_pool = TCPConnectionPool( loop, @@ -190,7 +192,7 @@ def __init__( max_connections_per_target=kwargs.get(TCP_MAX_CONNECTIONS_PER_TARGET, 10), ) - async def setup(self): + async def start(self): # create a TCP server bound to host and port that uses the # specified protocol self.server = await self.loop.create_server( @@ -198,14 +200,14 @@ async def setup(self): self.addr[0], self.addr[1], ) + await super().start() async def send_message( self, - content, - receiver_addr: Union[str, Tuple[str, int]], - *, - receiver_id: Optional[str] = None, - **kwargs, + content: Any, + receiver_addr: AgentAddress, + sender_id: None | str = None, + **kwargs ) -> bool: """ The Container sends a message to an agent using TCP. @@ -215,31 +217,38 @@ async def send_message( :param receiver_id: The agent id of the receiver :param kwargs: Additional parameters to provide protocol specific settings """ - if isinstance(receiver_addr, str) and ":" in receiver_addr: - receiver_addr = receiver_addr.split(":") - elif isinstance(receiver_addr, (tuple, list)) and len(receiver_addr) == 2: - receiver_addr = tuple(receiver_addr) + protocol_addr = receiver_addr.addr + if isinstance(protocol_addr, str) and ":" in protocol_addr: + protocol_addr = protocol_addr.split(":") + elif isinstance(protocol_addr, (tuple, list)) and len(protocol_addr) == 2: + protocol_addr = tuple(protocol_addr) else: - logger.warning("Address for sending message is not valid;%s", receiver_addr) + logger.warning("Address for sending message is not valid;%s", protocol_addr) return False - message = content - - if receiver_addr == self.addr: - if not receiver_id: - receiver_id = message.receiver_id - + meta = {} + for key, value in kwargs.items(): + meta[key] = value + meta["sender_id"] = sender_id + meta["sender_addr"] = self.addr + meta["receiver_id"] = receiver_addr.aid + + if protocol_addr == self.addr: # internal message - meta = {"network_protocol": "tcp"} + meta["network_protocol"] = "tcp" success = self._send_internal_message( - message, receiver_id, default_meta=meta + content, receiver_addr.aid, default_meta=meta ) else: - success = await self._send_external_message(receiver_addr, message) + message = content + # if the user does not provide a splittable content, we create the default one + if not hasattr(content, "split_content_and_meta"): + message = MangoMessage(content, meta) + success = await self._send_external_message(receiver_addr.addr, message, meta) return success - async def _send_external_message(self, addr, message) -> bool: + async def _send_external_message(self, addr, message, meta) -> bool: """ Sends *message* to another container at *addr* :param addr: Tuple of (host, port) diff --git a/mango/express/api.py b/mango/express/api.py new file mode 100644 index 0000000..72ddddd --- /dev/null +++ b/mango/express/api.py @@ -0,0 +1,225 @@ +import logging +from abc import ABC, abstractmethod +from typing import Any + +from ..agent.role import Role, RoleAgent +from ..agent.core import Agent +from ..container.core import Container, AgentAddress +from ..container.factory import create_tcp, create_mqtt +from ..messages.codecs import Codec + +logger = logging.getLogger(__name__) + +class ContainerActivationManager: + + def __init__(self, containers: list[Container]) -> None: + self._containers = containers + + async def __aenter__(self): + for container in self._containers: + await container.start() + for container in self._containers: + container.on_ready() + if len(self._containers) == 1: + return self._containers[0] + return self._containers + + async def __aexit__(self, exc_type, exc, tb): + for container in self._containers: + await container.shutdown() + +class RunWithContainer(ABC): + def __init__(self, num: int, *agents: tuple[Agent, dict]) -> None: + self._num = num + self._agents = agents + self.__activation_cm = None + + @abstractmethod + def create_container_list(self, num) -> list[Container]: + pass + + async def after_start(self, container_list, agents): + pass + + async def __aenter__(self): + actual_number_container = self._num + if self._num > len(self._agents): + actual_number_container = len(self._agents) + container_list = self.create_container_list(actual_number_container) + for (i, agent_tuple) in enumerate(self._agents): + container_id = i % actual_number_container + container = container_list[container_id] + actual_agent = agent_tuple[0] + agent_params = agent_tuple[1] + container.register_agent(actual_agent, suggested_aid=agent_params.get("aid", None)) + self.__activation_cm = activate(container_list) + await self.__activation_cm.__aenter__() + await self.after_start(container_list, self._agents) + return container_list + + async def __aexit__(self, exc_type, exc, tb): + await self.__activation_cm.__aexit__(exc_type, exc, tb) + +class RunWithTCPManager(RunWithContainer): + + def __init__(self, + num: int, + *agents: Agent | tuple[Agent, dict], + addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None) -> None: + agents = [agent if isinstance(agent, tuple) else (agent, dict()) for agent in agents[0]] + super().__init__(num, *agents) + + self._addr = addr + self._codec = codec + + def create_container_list(self, num): + return [create_tcp((self._addr[0], self._addr[1]+i), codec=self._codec) for i in range(num)] + +class RunWithMQTTManager(RunWithContainer): + + def __init__(self, + num: int, + *agents: Agent | tuple[Agent, dict], + broker_addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None) -> None: + agents = [agent if isinstance(agent, tuple) else (agent, dict()) for agent in agents[0]] + super().__init__(num, *agents) + + self._broker_addr = broker_addr + self._codec = codec + + def create_container_list(self, num): + return [create_mqtt((self._broker_addr[0], self._broker_addr[1]+i), + client_id=f"client{i}", + codec=self._codec) + for i in range(num)] + + async def after_start(self, container_list, agents): + for (i, agent_tuple) in enumerate(agents): + container_id = i % len(container_list) + container = container_list[container_id] + actual_agent = agent_tuple[0] + agent_params = agent_tuple[1] + topics = agent_params.get("topics", []) + for topic in topics: + await container.subscribe_for_agent(aid=actual_agent.aid, topic=topic) + + +def activate(*containers: Container) -> ContainerActivationManager: + """Create and return an async activation context manager. This can be used with the + `async with` syntax to run code while the container(s) are active. The containers + are started first, after your code under `async with` will run, and at the end + the container will shut down (even when an error occurs). + + Example: + ```python + # Single container + async with activate(container) as container: + # do your stuff + + # Multiple container + async with activate(container_list) as container_list: + # do your stuff + ``` + + Args: + containers (Container | list): a single container or a list of containers + + Returns: + ContainerActivationManager: The context manager to be used as described + """ + if isinstance(containers[0], list): + containers = containers[0] + return ContainerActivationManager(list(containers)) + +def run_with_tcp(num: int, + *agents: Agent | tuple[Agent, dict], + addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None) -> RunWithTCPManager: + """Create and return an async context manager, which can be used to run the given + agents in `num` automatically created tcp container. The agents are distributed + evenly. + + Example: + ```python + async with run_with_tcp(2, Agent(), Agent(), (Agent(), dict(aid="my_agent_id"))) as c: + # do your stuff + ``` + + Args: + num (int): number of tcp container + agents (args): list of agents which shall run + + Returns: + RunWithTCPManager: the async context manager to run the agents with + """ + return RunWithTCPManager(num, agents, addr=addr, codec=codec) + +def run_with_mqtt(num: int, + *agents: tuple[Agent, dict], + broker_addr: tuple[str, int] = ("127.0.0.1", 1883), + codec: None | Codec = None) -> RunWithMQTTManager: + """Create and return an async context manager, which can be used to run the given + agents in `num` automatically created mqtt container. The agents are distributed according + to the topic + + Args: + num (int): _description_ + agents (args): list of agents which shall run, it is possible to provide a tuple + (Agent, dict), the dict supports "aid" for the suggested_aid and "topics" as list of topics the agent + wants to subscribe to. + broker_addr (tuple[str, int], optional): Address of the broker the container shall connect to. Defaults to ("127.0.0.1", 5555). + codec (None | Codec, optional): The codec of the container + + Returns: + RunWithMQTTManager: _description_ + """ + return RunWithMQTTManager(num, agents, broker_addr=broker_addr, codec=codec) + +class ComposedAgent(RoleAgent): + pass + +def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAgent: + """Create an agent composed of the given `roles`. If a container is provided, + the created agent is automatically registered with the container `register_in`. + + Args: + *roles Role: The roles which are added to the agent + register_in (None | Container): container in which the created agent is registered, + if provided + """ + agent = ComposedAgent() + for role in roles: + agent.add_role(role) + if register_in is not None: + register_in.register_agent(agent) + return agent + +class PrintingAgent(Agent): + def handle_message(self, content, meta: dict[str, Any]): + logging.info(f"Received: {content} with {meta}") + +def sender_addr(meta: dict) -> AgentAddress: + """Extract the sender_addr from the meta dict. + + Args: + meta (dict): the meta you received + + Returns: + AgentAddress: Extracted agent address to be used for replying to messages + """ + # convert decoded addr list to tuple for hashability + return AgentAddress(tuple(meta["sender_addr"]) if isinstance(meta["sender_addr"], list) else meta["sender_addr"], meta["sender_id"]) + +def addr(protocol_part: Any, aid: str) -> AgentAddress: + """Create an Address from the topic. + + Args: + protocol_part (Any): protocol part of the address, e.g. topic for mqtt, or host/port for tcp, ... + aid (str): the agent id + + Returns: + AgentAddress: the address + """ + return AgentAddress(protocol_part, aid) \ No newline at end of file diff --git a/mango/messages/codecs.py b/mango/messages/codecs.py index 8f9f567..1fa9a35 100644 --- a/mango/messages/codecs.py +++ b/mango/messages/codecs.py @@ -16,10 +16,11 @@ import msgspec -from mango.messages.message import ACLMessage, Performatives, enum_serializer +from mango.messages.message import ACLMessage, Performatives, enum_serializer, MangoMessage from ..messages.acl_message_pb2 import ACLMessage as ACLProto from ..messages.other_proto_msgs_pb2 import GenericMsg as GenericProtoMsg +from ..messages.mango_message_pb2 import MangoMessage as MMProto def json_serializable(cls=None, repr=True): @@ -166,6 +167,7 @@ class JSON(Codec): def __init__(self): super().__init__() self.add_serializer(*ACLMessage.__json_serializer__()) + self.add_serializer(*MangoMessage.__json_serializer__()) self.add_serializer(*enum_serializer(Performatives)) def encode(self, data): @@ -179,11 +181,12 @@ class FastJSON(Codec): def __init__(self): super().__init__() self.add_serializer(*ACLMessage.__json_serializer__()) + self.add_serializer(*MangoMessage.__json_serializer__()) self.add_serializer(*enum_serializer(Performatives)) self.encoder = msgspec.json.Encoder(enc_hook=self.serialize_obj) self.decoder = msgspec.json.Decoder( - dec_hook=lambda _, b: self.deserialize_obj(b), type=ACLMessage + dec_hook=lambda _, b: self.deserialize_obj(b), type=MangoMessage ) def encode(self, data): @@ -202,6 +205,7 @@ def __init__(self): # the codec merely handles the mapping of object types to these methods # it does not require any knowledge of the actual proto classes self.add_serializer(ACLMessage, self._acl_to_proto, self._proto_to_acl) + self.add_serializer(*MangoMessage.__protoserializer__()) def encode(self, data): # All known proto messages are wrapped in this generic proto msg. @@ -304,3 +308,4 @@ def _proto_to_acl(self, data): acl.content = self.deserialize_obj(obj_repr) return acl + diff --git a/mango/messages/mango_message.proto b/mango/messages/mango_message.proto new file mode 100644 index 0000000..bbbfd5f --- /dev/null +++ b/mango/messages/mango_message.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message MangoMessage { + bytes content = 1; + bytes meta = 2; +} diff --git a/mango/messages/mango_message_pb2.py b/mango/messages/mango_message_pb2.py new file mode 100644 index 0000000..38bb275 --- /dev/null +++ b/mango/messages/mango_message_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: mango_message.proto +# Protobuf Python Version: 5.27.2 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 5, + 27, + 2, + '', + 'mango_message.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13mango_message.proto\"-\n\x0cMangoMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04meta\x18\x02 \x01(\x0c\x62\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mango_message_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR._loaded_options = None + _globals['_MANGOMESSAGE']._serialized_start=23 + _globals['_MANGOMESSAGE']._serialized_end=68 +# @@protoc_insertion_point(module_scope) diff --git a/mango/messages/message.py b/mango/messages/message.py index 228891e..6a3f7c0 100644 --- a/mango/messages/message.py +++ b/mango/messages/message.py @@ -6,15 +6,68 @@ It also includes the enum classes for the message Performative and Type """ - +from dataclasses import dataclass +from abc import ABC, abstractmethod import pickle from enum import Enum -from typing import Any, Dict +from typing import Any +import warnings from ..messages.acl_message_pb2 import ACLMessage as ACLProto +from ..messages.mango_message_pb2 import MangoMessage as MangoMsg + +class Message(ABC): + @abstractmethod + def split_content_and_meta(self): + pass + + def __asdict__(self): + return vars(self) + +@dataclass +class MangoMessage(Message): + content: Any = None + meta: dict[str,Any] = None + + def split_content_and_meta(self): + return self.content, self.meta + @classmethod + def __fromdict__(cls, attrs): + msg = MangoMessage() + for key, value in attrs.items(): + setattr(msg, key, value) + return msg -class ACLMessage: + @classmethod + def __json_serializer__(cls): + return cls, cls.__asdict__, cls.__fromdict__ + + def __toproto__(self): + msg = MangoMsg() + msg.content = pickle.dumps(self.content) + msg.meta = pickle.dumps(self.meta) + return msg + + @classmethod + def __fromproto__(cls, data): + msg = MangoMsg() + msg.ParseFromString(data) + + mango_message = cls() + + mango_message.content = pickle.loads(bytes(msg.content)) if msg.content else None + mango_message.meta = pickle.loads(bytes(msg.meta)) if msg.meta else None + + return mango_message + + @classmethod + def __protoserializer__(cls): + return cls, cls.__toproto__, cls.__fromproto__ + + + +class ACLMessage(Message): """ The ACL Message is the standard header used for the communication between mango agents. This class is based on the FIPA ACL standard: http://www.fipa.org/specs/fipa00061/SC00061G.html @@ -137,7 +190,7 @@ def __fromproto__(cls, data): return acl - def extract_meta(self) -> Dict[str, Any]: + def extract_meta(self) -> dict[str, Any]: meta_dict = self.message_dict meta_dict.pop("content") return meta_dict @@ -184,3 +237,48 @@ class Performatives(Enum): inform_if = 20 proxy = 21 propagate = 22 + +def create_acl( + content, + receiver_addr: str | tuple[str, int], + sender_addr: str | tuple[str, int], + receiver_id: None | str = None, + acl_metadata: None | dict[str, Any] = None, + is_anonymous_acl=False, +): + acl_metadata = {} if acl_metadata is None else acl_metadata.copy() + # analyse and complete acl_metadata + if "receiver_addr" not in acl_metadata.keys(): + acl_metadata["receiver_addr"] = receiver_addr + elif acl_metadata["receiver_addr"] != receiver_addr: + warnings.warn( + f"The argument receiver_addr ({receiver_addr}) is not equal to " + f"acl_metadata['receiver_addr'] ({acl_metadata['receiver_addr']}). \ + For consistency, the value in acl_metadata['receiver_addr'] " + f"was overwritten with receiver_addr.", + UserWarning, + ) + acl_metadata["receiver_addr"] = receiver_addr + if receiver_id: + if "receiver_id" not in acl_metadata.keys(): + acl_metadata["receiver_id"] = receiver_id + elif acl_metadata["receiver_id"] != receiver_id: + warnings.warn( + f"The argument receiver_id ({receiver_id}) is not equal to " + f"acl_metadata['receiver_id'] ({acl_metadata['receiver_id']}). \ + For consistency, the value in acl_metadata['receiver_id'] " + f"was overwritten with receiver_id.", + UserWarning, + ) + acl_metadata["receiver_id"] = receiver_id + # add sender_addr if not defined and not anonymous + if not is_anonymous_acl: + if "sender_addr" not in acl_metadata.keys() and sender_addr is not None: + acl_metadata["sender_addr"] = sender_addr + + message = ACLMessage() + message.content = content + + for key, value in acl_metadata.items(): + setattr(message, key, value) + return message diff --git a/mango/modules/__init__.py b/mango/modules/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mango/modules/base_module.py b/mango/modules/base_module.py deleted file mode 100644 index 54715a4..0000000 --- a/mango/modules/base_module.py +++ /dev/null @@ -1,72 +0,0 @@ -"""This module contains the base class for basic modules that can be used -inside agents to encapsulate complex functionality""" - -import traceback - -from .mqtt_module import MQTTModule -from .rabbit_module import RabbitModule -from .zero_module import ZeroModule - - -class BaseModule: - """An agent can have multiple specialized modules which inherit - from BaseModule. The all need to specify which messaging framework should - be used for the internal message exchange between the modules. - TODO write more - """ - - frameworks = {"mqtt": MQTTModule, "rabbit": RabbitModule, "zero": ZeroModule} - - def __init__( - self, *, name: str, framework="mqtt", subscr_topics, pub_topics, broker - ): - """ - Initialization of the module - - :param name: name of the module (str) - :param subscr_topics: List of string and integer tuples for subscribed - topics:[ (topic, qos)] e.g.[("my/topic", 0), ("another/topic", 2)] - :param pub_topics: List of string and integer tuples for publishing - topics:[ (topic, qos)] - :param broker: MQTT broker - :param log_level - - """ - super().__init__() - self.name = name - self.subscr_topics = subscr_topics - self.pub_topics = pub_topics - self.broker = broker - - self.messenger = BaseModule.frameworks[framework]( - name=self.name, - subscr_topics=self.subscr_topics, - pub_topics=self.pub_topics, - broker=self.broker, - ) - - self.add_message_callback = self.messenger.add_message_callback - self.start_mq_thread = self.messenger.start_mq_thread - self.end_mq_thread = self.messenger.end_mq_thread - self.publish_mq_message = self.messenger.publish_mq_message - self.bind_callback = self.messenger.bind_callback - - def raise_exceptions(self, result): - """ - Function used as a callback to raise exceptions - :param result: result of the task - """ - exception = result.exception() - if exception is not None: - tb = traceback.format_exc() - print(tb) - print(f"exception in {self.name}") - print(f"exception: {exception}") - raise exception - - # def handle_exception(loop, context): - # # context["message"] will always be there; but context["exception"] may not - # msg = context.get("exception", context["message"]) - # logging.error("Caught exception: %s", msg) - # logging.info("Shutting down...") - # asyncio.create_task(shutdown(loop)) diff --git a/mango/modules/mqtt_module.py b/mango/modules/mqtt_module.py deleted file mode 100644 index 0465890..0000000 --- a/mango/modules/mqtt_module.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -TODO -""" - -from functools import partial - -import paho.mqtt.client as paho - - -class MQTTModule: - """ - Module wrapper for a paho mqtt client used in the mango base module. - """ - - def __init__(self, *, name: str, subscr_topics, pub_topics, broker): - """ - - :param name: name of the module (str) - :param subscr_topics: List of string and integer tuples for subscribed - topics:[ (topic, qos)] e.g.[("my/topic", 0), ("another/topic", 2)] - :param pub_topics: List of string and integer tuples for publishing - topics:[ (topic, qos)] - :param broker: MQTT broker - """ - self.client_id = name - self.client = paho.Client(client_id=name) - self.client.on_connect = self.conn - self.client.on_message = self.on_mqtt_msg - self.client.on_disconnect = self.on_disconnect - self.client.connect(broker[0], broker[1], broker[3]) - self.subscr_topics = subscr_topics - self.client.subscribe(subscr_topics) # subscribe - self.pub_topics = pub_topics - - def add_message_callback(self, topic, fkt): - """ - Add a callback method in the mqtt client - :param topic: the topic to bind the method to - :param fkt: the method to bind - :return: None - """ - # bind function to general callback signature - partial_function = partial(self.bind_callback, fkt) - self.client.message_callback_add(topic, partial_function) - - def start_mq_thread(self): - """ - Start the message listening thread - :return: None - """ - self.client.loop_start() - - def end_mq_thread(self): - """ - Stop the message listening thread and disconnect the client - :return: - """ - self.client.loop_stop() - self.client.disconnect() - - def publish_mq_message(self, topic, payload, retain=False): - """ - Publish a message to the mqtt broker. - :param topic: the topic to publish on - :param payload: the message to publish - :param retain: flag indicating if the message should be retained on - shutdown - :return: None - """ - self.client.publish(topic, payload, retain=retain) - - @staticmethod - def bind_callback(func, client, userdata, message): - # pylint: disable=unused-argument - """ - Function to call our generic callback function that has only two - parameters from the framework specific callback with four parameters by - binding it as a partial function. - - :param func: The wanted callback function - :param client: the messaging client (unused) - :param userdata: the userdata object (unused) - :param message: the message payload - - :return: None - """ - func(topic=message.topic, payload=message.payload) - - def conn(self, client, userdata, flags, reason_code, properties): - # pylint: disable=unused-argument - """ - Callback method on broker connection on paho mqtt framework - - :param client: the connecting client - :param userdata: userdata object - :param flags: returned flags - :param reason_code: reason code - :param properties: properties - - :return: None - """ - - def on_disconnect( - self, client, userdata, disconnect_flags, reason_code, properties - ): - # pylint: disable=unused-argument - """ - Callback method on broker disconnect on paho mqtt framework - - :param client: the connecting client - :param userdata: userdata object - :param reason_code: reason code - :param properties: properties - - :return: None - """ - - def log(self, client, userdata, level, buf): - # pylint: disable=unused-argument - """ - Client log method - - :param client: the mqtt client - :param userdata: userdata object - :param level: log level - :param buf: data buffer - - :return: None - """ - - def on_mqtt_msg(self, client, userdata, message): - """ - Each module has to implement this to handle all messages it receives - in subscribed topics - - :param client: - :param userdata: - :param message: - - :return: - """ diff --git a/mango/modules/rabbit_module.py b/mango/modules/rabbit_module.py deleted file mode 100644 index cbef1cd..0000000 --- a/mango/modules/rabbit_module.py +++ /dev/null @@ -1,119 +0,0 @@ -import threading -import time -from abc import ABC -from functools import partial - -import pika - - -class RabbitModule(ABC): - def __init__(self, *, name: str, subscr_topics, pub_topics, broker): - """ - - :param name: name of the module (str) - :param subscr_topics: List of string and integer tuples for subscribed topics:[ (topic, qos)] - e.g.[("my/topic", 0), ("another/topic", 2)] - :param pub_topics: List of string and integer tuples for publishing topics:[ (topic, qos)] - :param broker: MQTT broker - :param log_level - - """ - super().__init__() - self.subscr_topics = subscr_topics - - self.pub_topics = pub_topics - - self.thread_active = False - self.thread_running = False - self.mq_thread = threading.Thread(target=self.run_mq) - - self.known_registers = [] - self.sub_channel = None - self.sub_connection = None - - self.setup_done = False - - # set up publishing connection - # we separate this because each thread needs its own connection and we want to be able to call - # publish from outside - self.pub_connection = pika.BlockingConnection( - pika.ConnectionParameters(host=broker[0], port=broker[1]) - ) - - self.pub_channel = self.pub_connection.channel() - - for topic in self.pub_topics: - # self.pub_channel.queue_declare(queue=topic[0]) - self.pub_channel.exchange_declare(exchange=topic[0], exchange_type="fanout") - - # run the sub thread - self.mq_thread.start() - - # make sure all our connections etc. are set up before we attempt to access them from outside - while not self.thread_active: - continue - - ### The following methods might be changed, in case we use another MQ paradigme such as rabbitMQ, zeroMQ...) - def add_message_callback(self, topic, fkt): - self.known_registers.append((topic, fkt)) - - def start_mq_thread(self): - self.thread_running = True - - # wait for all the exchanges and queues to be set up - while not self.setup_done: - continue - - def run_mq(self): - # set up stuff - self.sub_connection = pika.BlockingConnection( - pika.ConnectionParameters(host="localhost") - ) - self.sub_channel = self.sub_connection.channel() - - for topic in self.subscr_topics: - # self.sub_channel.queue_declare(queue=topic[0]) - self.sub_channel.exchange_declare(exchange=topic[0], exchange_type="fanout") - - self.thread_active = True - - # run loop - while self.thread_active: - if self.thread_running: - # set up saved callback - for cb in self.known_registers: - queues = [] - result = self.sub_channel.queue_declare(queue="", exclusive=True) - queues.append(result.method.queue) - self.sub_channel.queue_bind(exchange=cb[0], queue=queues[-1]) - - f = partial(self.bind_callback, cb[0], cb[1]) - self.sub_channel.basic_consume( - queue=queues[-1], on_message_callback=f, auto_ack=True - ) - - self.setup_done = True - - # this loops until it is stopped from outside - # essentially what happens in start_consuming with extra flag check - while self.sub_channel._consumer_infos and self.thread_running: - self.sub_channel.connection.process_data_events(time_limit=1) - - self.sub_channel.cancel() - - else: - time.sleep(1) - - # teardown - - def end_mq_thread(self): - # self.pub_connection.close() - self.thread_active = False - self.thread_running = False - - def publish_mq_message(self, topic, payload): - self.pub_channel.basic_publish(exchange=topic, routing_key="", body=payload) - - # the actual binding to whatever signature the frameworks callbacks have happens here - def bind_callback(self, topic, func, ch, method, properties, message): - func(payload=message, topic=topic) diff --git a/mango/modules/zero_module.py b/mango/modules/zero_module.py deleted file mode 100644 index 2aba48c..0000000 --- a/mango/modules/zero_module.py +++ /dev/null @@ -1,88 +0,0 @@ -import pickle -import threading -import time -from abc import ABC - -import zmq - - -class ZeroModule(ABC): - def __init__(self, *, name: str, subscr_topics, pub_topics, broker): - """ - - :param name: name of the module (str) - :param subscr_topics: List of string and integer tuples for subscribed topics:[ (topic, qos)] - e.g.[("my/topic", 0), ("another/topic", 2)] - :param pub_topics: List of string and integer tuples for publishing topics:[ (topic, qos)] - :param broker: MQTT broker - :param log_level - - """ - super().__init__() - self.name = name - - self.subscr_topics = subscr_topics - self.pub_topics = pub_topics - - self.thread_active = False - self.mq_thread = threading.Thread(target=self.run_mq) - - # set up pub - self.pub_context = zmq.Context() - self.pub_socket = self.pub_context.socket(zmq.PUB) - self.pub_socket.connect(f"tcp://{broker[0]}:{broker[1]}") - - # set up sub - self.sub_context = zmq.Context() - self.sub_socket = self.sub_context.socket(zmq.SUB) - self.sub_socket.connect(f"tcp://{broker[0]}:{broker[2]}") - - for topic in self.subscr_topics: - self.sub_socket.setsockopt_string(zmq.SUBSCRIBE, topic[0]) - - self.known_callbacks = [] - - ### The following methods might be changed, in case we use another MQ paradigme such as rabbitMQ, zeroMQ...) - def add_message_callback(self, topic, fkt): - # bind function to general callback signature - self.known_callbacks.append((topic, fkt)) - - def start_mq_thread(self): - self.thread_active = True - self.mq_thread.start() - - def end_mq_thread(self): - self.thread_active = False - # self.sub_context.destroy() - self.mq_thread.join() - - def publish_mq_message(self, topic, payload): - pl = pickle.dumps(payload) - self.pub_socket.send_multipart([topic.encode("utf-8"), pl]) - - # the actual binding to whatever signature the frameworks callbacks have happens here - def bind_callback(self, func, client, userdata, message): - pass - - def run_mq(self): - got_message = False - - while self.thread_active: - # TODO there probably is a clean way to terminate this - try: - [topic, msg] = self.sub_socket.recv_multipart(flags=zmq.NOBLOCK) - got_message = True - except Exception: - # didnt get a message - got_message = False - - if not got_message: - time.sleep(0.5) - continue - - topic = topic.decode("utf-8") - msg = pickle.loads(msg) - - for callback in self.known_callbacks: - if callback[0] == topic: - callback[1](topic=topic, payload=msg) diff --git a/mango/util/clock.py b/mango/util/clock.py index 0231357..fb24fde 100644 --- a/mango/util/clock.py +++ b/mango/util/clock.py @@ -2,7 +2,6 @@ import bisect import time from abc import ABC, abstractmethod -from typing import List, Tuple class Clock(ABC): @@ -51,8 +50,8 @@ class ExternalClock(Clock): def __init__(self, start_time: float = 0): self._time: float = start_time - self._futures: List[ - Tuple[float, asyncio.Future] + self._futures: list[ + tuple[float, asyncio.Future] ] = [] # list of all futures to be triggered @property diff --git a/mango/util/distributed_clock.py b/mango/util/distributed_clock.py index 3990ad9..fe1f409 100644 --- a/mango/util/distributed_clock.py +++ b/mango/util/distributed_clock.py @@ -1,7 +1,7 @@ import asyncio import logging -from mango import Agent +from mango import Agent, sender_addr from .termination_detection import tasks_complete_or_sleeping @@ -10,33 +10,32 @@ class ClockAgent(Agent): async def wait_all_done(self): - await tasks_complete_or_sleeping(self._context._container) + await tasks_complete_or_sleeping(self.context._container) class DistributedClockManager(ClockAgent): - def __init__(self, container, receiver_clock_addresses: list): - super().__init__(container, "clock") + def __init__(self, receiver_clock_addresses: list): + super().__init__() + self.receiver_clock_addresses = receiver_clock_addresses self.schedules = [] self.futures = {} + + def on_ready(self): self.schedule_instant_task(self.wait_all_online()) def handle_message(self, content: float, meta): - if isinstance(meta["sender_addr"], list): - sender_addr = tuple(meta["sender_addr"]) - else: - sender_addr = meta["sender_addr"] - - logger.debug("clockmanager: %s from %s", content, sender_addr) + sender = sender_addr(meta) + logger.debug("clockmanager: %s from %s", content, sender) if content: assert isinstance(content, (int, float)), f"{content} was {type(content)}" self.schedules.append(content) - if not self.futures[sender_addr].done(): - self.futures[sender_addr].set_result(True) + if not self.futures[sender].done(): + self.futures[sender].set_result(True) else: # with as_agent_process - messages can come from ourselves - logger.debug("got another message from agent %s - %s", sender_addr, content) + logger.debug("got another message from agent %s - %s", sender, content) async def broadcast(self, message, add_futures=True): """ @@ -47,15 +46,13 @@ async def broadcast(self, message, add_futures=True): message (object): the given message add_futures (bool, optional): Adds futures which can be awaited until a response to a message is given. Defaults to True. """ - for receiver_addr, receiver_aid in self.receiver_clock_addresses: + for receiver_addr in self.receiver_clock_addresses: logger.debug("clockmanager send: %s - %s", message, receiver_addr) # in MQTT we can not be sure if the message was delivered # checking the return code here would only help for TCP - await self.send_acl_message( + await self.send_message( message, - receiver_addr, - receiver_aid, - acl_metadata={"sender_id": self.aid}, + receiver_addr ) if add_futures: self.futures[receiver_addr] = asyncio.Future() @@ -73,7 +70,7 @@ async def send_current_time(self, time=None): Args: time (number, optional): The current time which is set. Defaults to None. """ - time = time or self._scheduler.clock.time + time = time or self.scheduler.clock.time await self.broadcast(time, add_futures=False) async def wait_for_futures(self): @@ -118,7 +115,7 @@ async def get_next_event(self): # wait for our container too await self.wait_all_done() - next_activity = self._scheduler.clock.get_next_activity() + next_activity = self.scheduler.clock.get_next_activity() if next_activity is not None: # logger.error(f"{next_activity}") @@ -128,11 +125,11 @@ async def get_next_event(self): next_event = min(self.schedules) else: logger.warning("%s: no new events, time stands still", self.aid) - next_event = self._scheduler.clock.time + next_event = self.scheduler.clock.time - if next_event < self._scheduler.clock.time: + if next_event < self.scheduler.clock.time: logger.warning("%s: got old event, time stands still", self.aid) - next_event = self._scheduler.clock.time + next_event = self.scheduler.clock.time logger.debug("next event at %s", next_event) return next_event @@ -156,14 +153,13 @@ async def distribute_time(self, time=None): class DistributedClockAgent(ClockAgent): - def __init__(self, container, suggested_aid="clock_agent"): - super().__init__(container, suggested_aid=suggested_aid) + def __init__(self): + super().__init__() self.stopped = asyncio.Future() def handle_message(self, content: float, meta): - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] - logger.info("clockagent: %s from %s", content, sender_addr) + sender = sender_addr(meta) + logger.info("clockagent: %s from %s", content, sender) if content == "stop": if not self.stopped.done(): self.stopped.set_result(True) @@ -178,16 +174,14 @@ def respond(fut: asyncio.Future = None): if self.stopped.done(): return - next_time = self._scheduler.clock.get_next_activity() - - self.schedule_instant_acl_message( + next_time = self.scheduler.clock.get_next_activity() + + self.schedule_instant_message( next_time, - sender_addr, - sender_id, - acl_metadata={"sender_id": self.aid}, + sender_addr(meta) ) t.add_done_callback(respond) else: assert isinstance(content, (int, float)), f"{content} was {type(content)}" - self._scheduler.clock.set_time(content) + self.scheduler.clock.set_time(content) diff --git a/mango/util/multiprocessing.py b/mango/util/multiprocessing.py index c557a6d..0d04fab 100644 --- a/mango/util/multiprocessing.py +++ b/mango/util/multiprocessing.py @@ -29,12 +29,12 @@ from contextlib import asynccontextmanager, contextmanager from multiprocessing.connection import Connection from multiprocessing.reduction import ForkingPickler -from typing import Any, AsyncContextManager, ContextManager, Tuple +from typing import Any, AsyncContextManager, ContextManager import dill -def aiopipe() -> Tuple["AioPipeReader", "AioPipeWriter"]: +def aiopipe() -> tuple["AioPipeReader", "AioPipeWriter"]: """Create a pair of pipe endpoints, both readable and writable (duplex). :return: Reader-, Writer-Pair @@ -43,7 +43,7 @@ def aiopipe() -> Tuple["AioPipeReader", "AioPipeWriter"]: return AioPipeReader(rx), AioPipeWriter(tx) -def aioduplex() -> Tuple["AioDuplex", "AioDuplex"]: +def aioduplex() -> tuple["AioDuplex", "AioDuplex"]: """Create a pair of pipe endpoints, both readable and writable (duplex). :return: AioDuplex-Pair @@ -75,11 +75,11 @@ async def open(self): await asyncio.sleep(0) - async def _open(self) -> Tuple[asyncio.BaseTransport, Any]: + async def _open(self) -> tuple[asyncio.BaseTransport, Any]: raise NotImplementedError() @contextmanager - def detach(self) -> ContextManager["AioPipeStream"]: + def detach(self): try: os.set_inheritable(self._fd, True) yield self @@ -262,26 +262,26 @@ def __init__(self, rx: AioPipeReader, tx: AioPipeWriter): self._tx = tx @contextmanager - def detach(self) -> ContextManager["AioDuplex"]: + def detach(self): with self._rx.detach(), self._tx.detach(): yield self @asynccontextmanager async def open( self, - ) -> AsyncContextManager[Tuple["asyncio.StreamReader", "asyncio.StreamWriter"]]: + ): async with self._rx.open() as rx, self._tx.open() as tx: yield rx, tx @asynccontextmanager - async def open_readonly(self) -> AsyncContextManager[Tuple["asyncio.StreamReader"]]: + async def open_readonly(self): async with self._rx.open() as rx: yield rx @asynccontextmanager async def open_writeonly( self, - ) -> AsyncContextManager[Tuple["asyncio.StreamWriter"]]: + ): async with self._tx.open() as tx: yield tx diff --git a/mango/util/scheduling.py b/mango/util/scheduling.py index 14d16b7..b207d20 100644 --- a/mango/util/scheduling.py +++ b/mango/util/scheduling.py @@ -463,8 +463,8 @@ def __init__( self._manager = None self._num_process_parallel = num_process_parallel self._process_pool_exec = None - self._suspendable = suspendable - self._observable = observable + self.suspendable = suspendable + self.observable = observable @staticmethod def _run_task_in_p_context( @@ -500,7 +500,7 @@ def schedule_task(self, task: ScheduledTask, src=None) -> asyncio.Task: :type: Object """ l_task = None - if self._suspendable: + if self.suspendable: coro = Suspendable(task.run()) l_task = asyncio.ensure_future(coro) else: @@ -531,7 +531,7 @@ def schedule_timestamp_task( timestamp=timestamp, clock=self.clock, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -551,7 +551,7 @@ def schedule_instant_task(self, coroutine, on_stop=None, src=None): coroutine=coroutine, clock=self.clock, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -574,7 +574,7 @@ def schedule_periodic_task(self, coroutine_func, delay, on_stop=None, src=None): delay=delay, clock=self.clock, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -599,7 +599,7 @@ def schedule_recurrent_task( recurrency=recurrency, clock=self.clock, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -632,7 +632,7 @@ def schedule_conditional_task( clock=self.clock, lookup_delay=lookup_delay, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -657,7 +657,7 @@ def schedule_awaiting_task( awaited_coroutine=awaited_coroutine, clock=self.clock, on_stop=on_stop, - observable=self._observable, + observable=self.observable, ), src=src, ) @@ -827,7 +827,7 @@ def suspend(self, given_src): :param given_src: the src object :type given_src: object """ - if not self._suspendable: + if not self.suspendable: raise Exception("The scheduler is configured as non-suspendable!") for _, _, coro, src in self._scheduled_tasks: @@ -843,7 +843,7 @@ def resume(self, given_src): :param given_src: the src object :type given_src: object """ - if not self._suspendable: + if not self.suspendable: raise Exception("The scheduler is configured as non-suspendable!") for _, _, coro, src in self._scheduled_tasks: diff --git a/mango/util/termination_detection.py b/mango/util/termination_detection.py index a4f6512..cf79b5b 100644 --- a/mango/util/termination_detection.py +++ b/mango/util/termination_detection.py @@ -26,8 +26,8 @@ async def tasks_complete_or_sleeping(container: Container, except_sources=["no_w # python does not have do while pattern for agent in container._agents.values(): await agent.inbox.join() - task_list.extend(agent._scheduler._scheduled_tasks) - task_list.extend(agent._scheduler._scheduled_process_tasks) + task_list.extend(agent.scheduler._scheduled_tasks) + task_list.extend(agent.scheduler._scheduled_process_tasks) task_list = list(filter(lambda x: x[3] not in except_sources, task_list)) while len(task_list) > len(sleeping_tasks): @@ -50,6 +50,6 @@ async def tasks_complete_or_sleeping(container: Container, except_sources=["no_w task_list = [] for agent in container._agents.values(): await agent.inbox.join() - task_list.extend(agent._scheduler._scheduled_tasks) - task_list.extend(agent._scheduler._scheduled_process_tasks) + task_list.extend(agent.scheduler._scheduled_tasks) + task_list.extend(agent.scheduler._scheduled_process_tasks) task_list = list(filter(lambda x: x[3] not in except_sources, task_list)) diff --git a/readme.md b/readme.md index 378db92..cc13051 100644 --- a/readme.md +++ b/readme.md @@ -117,29 +117,27 @@ the agent does yet not receive any messages. Let's implement another agent that is able to send a hello world message to another agent: -```python3 +```python - from mango import Agent - from mango.util.scheduling import InstantScheduledTask +from mango import Agent - class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!") - ) + class HelloWorldAgent(Agent): + def __init__(self, container, other_addr, other_id): + super().__init__(container) + self.schedule_instant_message( + receiver_addr=other_addr, + receiver_id=other_id, + content="Hello world!" + ) - def handle_message(self, content, meta): - print(f"Received a message with the following content: {content}") + def handle_message(self, content, meta): + print(f"Received a message with the following content: {content}") ``` #### Connecting two agents We can now connect an instance of a HelloWorldAgent with an instance of a RepeatingAgent and run them. ```python import asyncio -from mango import Agent, create_container -from mango.util.scheduling import InstantScheduledTask +from mango import Agent, create_tcp_container, activate class RepeatingAgent(Agent): @@ -153,28 +151,22 @@ class RepeatingAgent(Agent): print(f"Received a message with the following content: {content}") class HelloWorldAgent(Agent): - def __init__(self, container, other_addr, other_id): - super().__init__(container) - self.schedule_instant_acl_message( - receiver_addr=other_addr, - receiver_id=other_id, - content="Hello world!") - ) + async def greet(self, other_addr): + await self.send_message("Hello world!", other_addr) def handle_message(self, content, meta): print(f"Received a message with the following content: {content}") async def run_container_and_two_agents(first_addr, second_addr): - first_container = await create_container(addr=first_addr) - second_container = await create_container(addr=second_addr) - first_agent = RepeatingAgent(first_container) - second_agent = HelloWorldAgent(second_container, first_container.addr, first_agent.aid) - await asyncio.sleep(1) - await first_agent.shutdown() - await second_agent.shutdown() - await first_container.shutdown() - await second_container.shutdown() + first_container = create_tcp_container(addr=first_addr) + second_container = create_tcp_container(addr=second_addr) + + first_agent = first_container.include(RepeatingAgent()) + second_agent = second_container.include(HelloWorldAgent()) + + async with activate(first_container, second_container) as cl: + await second_agent.greet(first_agent.addr) def test_second_example(): diff --git a/setup.py b/setup.py index a9319c9..af1c7d1 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ URL = "https://github.com/OFFIS-DAI/mango" EMAIL = "mango@offis.de" AUTHOR = "mango Team" -REQUIRES_PYTHON = ">=3.8.0" +REQUIRES_PYTHON = ">=3.9.0" VERSION = "1.2.0" # What packages are required for this module to be executed? diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py index e69de29..6624f6b 100644 --- a/tests/integration_tests/__init__.py +++ b/tests/integration_tests/__init__.py @@ -0,0 +1,39 @@ +from mango import create_tcp_container, create_mqtt_container +from mango.util.clock import ExternalClock + +def create_test_container(type, init_addr, repl_addr, codec): + broker = ("localhost", 1883, 60) + + clock_man = ExternalClock(5) + if type == "tcp": + container_man = create_tcp_container( + addr=init_addr, + codec=codec, + clock=clock_man, + ) + elif type == "mqtt": + container_man = create_mqtt_container( + broker_addr=broker, + client_id="container_1", + clock=clock_man, + codec=codec, + inbox_topic=init_addr, + transport="tcp" + ) + clock_ag = ExternalClock() + if type == "tcp": + container_ag = create_tcp_container( + addr=repl_addr, + codec=codec, + clock=clock_ag, + ) + elif type == "mqtt": + container_ag = create_mqtt_container( + broker_addr=broker, + client_id="container_2", + clock=clock_ag, + codec=codec, + inbox_topic=repl_addr, + transport="tcp" + ) + return container_man, container_ag \ No newline at end of file diff --git a/tests/integration_tests/test_distributed_clock.py b/tests/integration_tests/test_distributed_clock.py index 26c353a..fd1b06a 100644 --- a/tests/integration_tests/test_distributed_clock.py +++ b/tests/integration_tests/test_distributed_clock.py @@ -2,80 +2,44 @@ import pytest -from mango import create_container +from mango import activate, addr from mango.messages.codecs import JSON -from mango.util.clock import ExternalClock from mango.util.distributed_clock import DistributedClockAgent, DistributedClockManager +from . import create_test_container + JSON_CODEC = JSON() async def setup_and_run_test_case(connection_type, codec): - comm_topic = "test_topic" init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" + + container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager(receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + )) + + async with activate(container_man, container_ag) as cl: + # increasing the time + container_man.clock.set_time(100) + # first distribute the time - then wait for the agent to finish + next_event = await clock_manager.distribute_time() + # here no second distribute to wait for retrieval is needed + # the clock_manager distributed the time to the other container + assert container_ag.clock.time == 100 + container_man.clock.set_time(1000) + next_event = await clock_manager.distribute_time() + # here no second distribute to wait for retrieval is needed + + assert container_ag.clock.time == 1000 + container_man.clock.set_time(2000) + # distribute the new time + await clock_manager.distribute_time() + # did work the second time too + assert container_ag.clock.time == 2000 - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - clock_man = ExternalClock() - container_man = await create_container( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - clock=clock_man, - ) - clock_ag = ExternalClock() - container_ag = await create_container( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - clock=clock_ag, - ) - - clock_agent = DistributedClockAgent(container_ag) - clock_manager = DistributedClockManager( - container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] - ) - - # increasing the time - clock_man.set_time(100) - # first distribute the time - then wait for the agent to finish - next_event = await clock_manager.distribute_time() - # here no second distribute to wait for retrieval is needed - # the clock_manager distributed the time to the other container - assert clock_ag.time == 100 - clock_man.set_time(1000) - next_event = await clock_manager.distribute_time() - # here no second distribute to wait for retrieval is needed - - assert clock_ag.time == 1000 - clock_man.set_time(2000) - # distribute the new time - await clock_manager.distribute_time() - # did work the second time too - assert clock_ag.time == 2000 - - await clock_manager.shutdown() - await clock_agent.shutdown() - - # finally shut down - await asyncio.gather( - container_man.shutdown(), - container_ag.shutdown(), - ) @pytest.mark.asyncio diff --git a/tests/integration_tests/test_message_roundtrip.py b/tests/integration_tests/test_message_roundtrip.py index dc43697..b31a6fc 100644 --- a/tests/integration_tests/test_message_roundtrip.py +++ b/tests/integration_tests/test_message_roundtrip.py @@ -2,9 +2,10 @@ import pytest -import mango.container.factory as container_factory +from mango import create_mqtt_container, create_tcp_container, activate, addr from mango.agent.core import Agent from mango.messages.codecs import JSON, PROTOBUF, FastJSON +from . import create_test_container from ..unit_tests.messages.msg_pb2 import MyMsg @@ -40,31 +41,7 @@ async def setup_and_run_test_case(connection_type, codec): init_addr = ("localhost", 1555) if connection_type == "tcp" else None repl_addr = ("localhost", 1556) if connection_type == "tcp" else None - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - container_1 = await container_factory.create( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - ) - container_2 = await container_factory.create( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - ) + container_1, container_2 = create_test_container(connection_type, init_addr, repl_addr, codec) if connection_type == "mqtt": init_target = repl_target = comm_topic @@ -72,18 +49,13 @@ async def setup_and_run_test_case(connection_type, codec): init_target = repl_addr repl_target = init_addr - init_agent = InitiatorAgent(container_1, init_target) - repl_agent = ReplierAgent(container_2, repl_target) - - repl_agent.other_aid = init_agent.aid - init_agent.other_aid = repl_agent.aid - - await asyncio.gather(repl_agent.start(), init_agent.start()) - await asyncio.gather( - container_1.shutdown(), - container_2.shutdown(), - ) + init_agent = container_1.include(InitiatorAgent(container_1)) + repl_agent = container_2.include(ReplierAgent(container_2)) + repl_agent.target = addr(repl_target, init_agent.aid) + init_agent.target = addr(init_target, repl_agent.aid) + async with activate(container_1, container_2) as cl: + await asyncio.gather(repl_agent.start(), init_agent.start()) # InitiatorAgent: # - send "Hello" @@ -91,13 +63,11 @@ async def setup_and_run_test_case(connection_type, codec): # - answers to reply # - shuts down class InitiatorAgent(Agent): - def __init__(self, container, target): - super().__init__(container) - self.target = target - self.other_aid = None - self.container = container - + def __init__(self, container): + super().__init__() + self.target = None self.got_reply = asyncio.Future() + self.container = container def handle_message(self, content, meta): if content == M2: @@ -105,29 +75,25 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): - await self.container.subscribe_for_agent(aid=self.aid, topic=self.target) + await self.container.subscribe_for_agent(aid=self.aid, topic=self.target.addr) await asyncio.sleep(0.1) # send initial message - await self.send_acl_message( + await self.send_message( M1, - self.target, - receiver_id=self.other_aid, + self.target ) # await reply await self.got_reply # answer to reply - await self.send_acl_message( + await self.send_message( M3, - self.target, - receiver_id=self.other_aid, + self.target ) - # shut down - # ReplierAgent: # - awaits "Hello" @@ -135,9 +101,9 @@ async def start(self): # - awaits reply # - shuts down class ReplierAgent(Agent): - def __init__(self, container, target): - super().__init__(container) - self.target = target + def __init__(self, container): + super().__init__() + self.target = None self.other_aid = None self.got_first = asyncio.Future() @@ -153,19 +119,17 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): - await self.container.subscribe_for_agent(aid=self.aid, topic=self.target) + await self.container.subscribe_for_agent(aid=self.aid, topic=self.target.addr) # await "Hello" await self.got_first # send reply - await self.send_acl_message(M2, self.target, receiver_id=self.other_aid) + await self.send_message(M2, self.target, receiver_id=self.other_aid) # await reply await self.got_second - # shut down - @pytest.mark.asyncio async def test_tcp_json(): diff --git a/tests/integration_tests/test_message_roundtrip_mp.py b/tests/integration_tests/test_message_roundtrip_mp.py index 800fba8..c1e820a 100644 --- a/tests/integration_tests/test_message_roundtrip_mp.py +++ b/tests/integration_tests/test_message_roundtrip_mp.py @@ -2,8 +2,8 @@ import pytest -import mango.container.factory as container_factory from mango.agent.core import Agent +from mango import AgentAddress, sender_addr, create_tcp_container, activate class PingPongAgent(Agent): @@ -15,17 +15,10 @@ def handle_message(self, content, meta): # get addr and id from sender if self.test_counter == 1: - receiver_host, receiver_port = meta["sender_addr"] - receiver_id = meta["sender_id"] # send back pong, providing your own details - self.current_task = self.schedule_instant_acl_message( + self.current_task = self.schedule_instant_message( content="pong", - receiver_addr=(receiver_host, receiver_port), - receiver_id=receiver_id, - acl_metadata={ - "sender_addr": self.addr, - "sender_id": self.aid, - }, + receiver_addr=sender_addr(meta), ) @@ -36,40 +29,32 @@ async def test_mp_simple_ping_pong_multi_container_tcp(): aid1 = "c1_p1_agent" aid2 = "c2_p1_agent" - container_1 = await container_factory.create( + container_1 = create_tcp_container( addr=init_addr, ) - container_2 = await container_factory.create( + container_2 = create_tcp_container( addr=repl_addr, ) await container_1.as_agent_process( - agent_creator=lambda c: PingPongAgent(c, suggested_aid=aid1) + agent_creator=lambda c: c.include(PingPongAgent(), suggested_aid=aid1) ) await container_2.as_agent_process( - agent_creator=lambda c: PingPongAgent(c, suggested_aid=aid2) + agent_creator=lambda c: c.include(PingPongAgent(), suggested_aid=aid2) ) - agent = PingPongAgent(container_1) + agent = container_1.include(PingPongAgent()) - await agent.send_acl_message( - "Message To Process Agent1", - receiver_addr=container_1.addr, - receiver_id=aid1, - acl_metadata={"sender_id": agent.aid}, - ) + async with activate(container_1, container_2) as cl: + await agent.send_message( + "Message To Process Agent1", + receiver_addr=AgentAddress(container_1.addr,aid1) + ) - await agent.send_acl_message( - "Message To Process Agent2", - receiver_addr=container_2.addr, - receiver_id=aid2, - acl_metadata={"sender_id": agent.aid}, - ) + await agent.send_message( + "Message To Process Agent2", + receiver_addr=AgentAddress(container_2.addr,aid2) + ) - while agent.test_counter != 2: - await asyncio.sleep(0.1) + while agent.test_counter != 2: + await asyncio.sleep(0.1) assert agent.test_counter == 2 - - await asyncio.gather( - container_1.shutdown(), - container_2.shutdown(), - ) diff --git a/tests/integration_tests/test_single_container_termination.py b/tests/integration_tests/test_single_container_termination.py index 8458dc1..bc5c723 100644 --- a/tests/integration_tests/test_single_container_termination.py +++ b/tests/integration_tests/test_single_container_termination.py @@ -2,43 +2,43 @@ import pytest -from mango import Agent, create_container +from mango import Agent, create_ec_container, sender_addr, activate, addr from mango.util.clock import ExternalClock from mango.util.distributed_clock import DistributedClockAgent, DistributedClockManager from mango.util.termination_detection import tasks_complete_or_sleeping - +from . import create_test_container class Caller(Agent): def __init__( self, - container, receiver_addr, - receiver_id, send_response_messages=False, max_count=100, schedule_timestamp=False, ): - super().__init__(container) - self.schedule_timestamp_task( - coroutine=self.send_hello_world(receiver_addr, receiver_id), - timestamp=self.current_timestamp + 5, - ) + super().__init__() self.i = 0 self.send_response_messages = send_response_messages self.max_count = max_count self.schedule_timestamp = schedule_timestamp self.done = asyncio.Future() + self.target = receiver_addr - async def send_hello_world(self, receiver_addr, receiver_id): - await self.send_acl_message( - receiver_addr=receiver_addr, receiver_id=receiver_id, content="Hello World" + def on_ready(self): + self.schedule_timestamp_task( + coroutine=self.send_hello_world(self.target), + timestamp=self.current_timestamp + 5, + ) + + async def send_hello_world(self, receiver_addr): + await self.send_message( + receiver_addr=receiver_addr, content="Hello World" ) async def send_ordered(self, meta): - await self.send_acl_message( - receiver_addr=meta["sender_addr"], - receiver_id=meta["sender_id"], + await self.send_message( content=self.i, + receiver_addr=sender_addr(meta), ) def handle_message(self, content, meta): @@ -55,17 +55,10 @@ def handle_message(self, content, meta): class Receiver(Agent): - def __init__(self, container, receiver_addr=None, receiver_id=None): - super().__init__(container) - self.receiver_addr = receiver_addr - self.receiver_id = receiver_id - def handle_message(self, content, meta): - self.schedule_instant_acl_message( - receiver_addr=self.receiver_addr or self.addr, - receiver_id=self.receiver_id, + self.schedule_instant_message( + receiver_addr=sender_addr(meta), content=content, - acl_metadata={"sender_id": self.aid}, ) @@ -73,89 +66,55 @@ def handle_message(self, content, meta): async def test_termination_single_container(): clock = ExternalClock(start_time=1000) - c = await create_container(connection_type="external_connection", clock=clock) - receiver = Receiver(c, receiver_id="agent1") - caller = Caller(c, c.addr, receiver.aid, send_response_messages=True) - if isinstance(clock, ExternalClock): + c = create_ec_container(clock=clock) + receiver = c.include(Receiver()) + caller = c.include(Caller(receiver.addr, send_response_messages=True)) + + async with activate(c) as c: await asyncio.sleep(0.1) clock.set_time(clock.time + 5) + # wait until each agent is done with all tasks at some point + await receiver.scheduler.tasks_complete_or_sleeping() + await caller.scheduler.tasks_complete_or_sleeping() - # wait until each agent is done with all tasks at some point - await receiver._scheduler.tasks_complete_or_sleeping() - await caller._scheduler.tasks_complete_or_sleeping() - # this does not end far too early - assert caller.i < 30 - - # checking tasks completed for each agent does not help here, as they are sleeping alternating - # the following container-wide function catches this behavior in a single container - # to solve this situation for multiple containers a distributed termination detection - # is needed - await tasks_complete_or_sleeping(c) - assert caller.i == caller.max_count - await c.shutdown() + # this does not end far too early + assert caller.i < 30 + # checking tasks completed for each agent does not help here, as they are sleeping alternating + # the following container-wide function catches this behavior in a single container + # to solve this situation for multiple containers a distributed termination detection + # is needed + await tasks_complete_or_sleeping(c) + + assert caller.i == caller.max_count async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - clock_man = ExternalClock(5) - container_man = await create_container( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - clock=clock_man, - ) - clock_ag = ExternalClock() - container_ag = await create_container( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - clock=clock_ag, - ) - - clock_agent = DistributedClockAgent(container_ag) - clock_manager = DistributedClockManager( - container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] - ) - receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller( - container_man, - repl_addr, - receiver.aid, + container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + )) + receiver = container_ag.include(Receiver()) + caller = container_man.include(Caller( + addr(repl_addr, receiver.aid), send_response_messages=True, max_count=max_count, - ) + )) - clock_man.set_time(clock_man.time + 5) - # we do not have distributed termination detection yet in core - assert caller.i < caller.max_count + async with activate(container_man, container_ag) as c: + container_man.clock.set_time(container_man.clock.time + 5) - await caller.done + # we do not have distributed termination detection yet in core + assert caller.i < caller.max_count + await caller.done + assert caller.i == caller.max_count - # finally shut down - await asyncio.gather( - container_man.shutdown(), - container_ag.shutdown(), - ) - async def distribute_ping_pong_test_timestamp( connection_type, codec=None, max_count=10 @@ -163,76 +122,41 @@ async def distribute_ping_pong_test_timestamp( init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - clock_man = ExternalClock(5) - container_man = await create_container( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - clock=clock_man, - ) - clock_ag = ExternalClock() - container_ag = await create_container( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - clock=clock_ag, - ) - - clock_agent = DistributedClockAgent(container_ag) - clock_manager = DistributedClockManager( - container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] - ) - receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller( - container_man, - repl_addr, - receiver.aid, + container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + )) + receiver = container_ag.include(Receiver()) + caller = container_man.include(Caller( + addr(repl_addr, receiver.aid), send_response_messages=True, max_count=max_count, schedule_timestamp=True, - ) + )) # we do not have distributed termination detection yet in core - assert caller.i < caller.max_count + async with activate(container_man, container_ag) as cl: + assert caller.i < caller.max_count - import time + import time - tt = 0 - if isinstance(clock_man, ExternalClock): - for i in range(caller.max_count): - await tasks_complete_or_sleeping(container_man) - t = time.time() - await clock_manager.send_current_time() - next_event = await clock_manager.get_next_event() - tt += time.time() - t + tt = 0 + if isinstance(container_man.clock, ExternalClock): + for i in range(caller.max_count): + await tasks_complete_or_sleeping(container_man) + t = time.time() + await clock_manager.send_current_time() + next_event = await clock_manager.get_next_event() + tt += time.time() - t - clock_man.set_time(next_event) + container_man.clock.set_time(next_event) - await caller.done + await caller.done assert caller.i == caller.max_count - # finally shut down - await asyncio.gather( - container_man.shutdown(), - container_ag.shutdown(), - ) - @pytest.mark.asyncio async def test_distribute_ping_pong_tcp(): @@ -258,156 +182,91 @@ async def distribute_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - clock_man = ExternalClock(5) - container_man = await create_container( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - clock=clock_man, - ) - clock_ag = ExternalClock() - container_ag = await create_container( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - clock=clock_ag, - ) - - clock_agent = DistributedClockAgent(container_ag) - clock_manager = DistributedClockManager( - container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] - ) - receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller(container_man, repl_addr, receiver.aid) - - assert receiver._scheduler.clock.time == 0 - # first synchronize the clock to the receiver - next_event = await clock_manager.distribute_time(clock_man.time) - await tasks_complete_or_sleeping(container_man) - # this is to early, as we did not wait a whole roundtrip - assert receiver._scheduler.clock.time == 0 - # increase the time, triggering an action in the caller - clock_man.set_time(10) - # distribute the new time to the clock_manager - next_event = await clock_manager.distribute_time() - # wait until everything is done - await tasks_complete_or_sleeping(container_man) - # also wait for the result in the agent container - next_event = await clock_manager.distribute_time() - assert receiver._scheduler.clock.time == 10 - # now the response should be received - await tasks_complete_or_sleeping(container_man) - assert caller.i == 1, "received one message" - clock_man.set_time(15) - next_event = await clock_manager.distribute_time() - await tasks_complete_or_sleeping(container_man) - next_event = await clock_manager.distribute_time() - # the clock_manager distributed the time to the other container - assert clock_ag.time == 15 - clock_man.set_time(1000) - next_event = await clock_manager.distribute_time() - next_event = await clock_manager.distribute_time() - # did work the second time too - assert clock_ag.time == 1000 - - # finally shut down - await asyncio.gather( - container_man.shutdown(), - container_ag.shutdown(), - ) + container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + )) + receiver = container_ag.include(Receiver()) + caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) + + async with activate(container_man, container_ag) as cl: + assert receiver.scheduler.clock.time == 0 + # first synchronize the clock to the receiver + next_event = await clock_manager.distribute_time(container_man.clock.time) + await tasks_complete_or_sleeping(container_man) + # this is to early, as we did not wait a whole roundtrip + assert receiver.scheduler.clock.time == 0 + # increase the time, triggering an action in the caller + container_man.clock.set_time(10) + # distribute the new time to the clock_manager + next_event = await clock_manager.distribute_time() + # wait until everything is done + await tasks_complete_or_sleeping(container_man) + # also wait for the result in the agent container + next_event = await clock_manager.distribute_time() + assert receiver.scheduler.clock.time == 10 + # now the response should be received + await tasks_complete_or_sleeping(container_man) + assert caller.i == 1, "received one message" + container_man.clock.set_time(15) + next_event = await clock_manager.distribute_time() + await tasks_complete_or_sleeping(container_man) + next_event = await clock_manager.distribute_time() + # the clock_manager distributed the time to the other container + assert container_ag.clock.time == 15 + container_man.clock.set_time(1000) + next_event = await clock_manager.distribute_time() + next_event = await clock_manager.distribute_time() + # did work the second time too + assert container_ag.clock.time == 1000 async def send_current_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - broker = ("localhost", 1883, 60) - mqtt_kwargs_1 = { - "client_id": "container_1", - "broker_addr": broker, - "transport": "tcp", - } - - mqtt_kwargs_2 = { - "client_id": "container_2", - "broker_addr": broker, - "transport": "tcp", - } - - clock_man = ExternalClock(5) - container_man = await create_container( - connection_type=connection_type, - codec=codec, - addr=init_addr, - mqtt_kwargs=mqtt_kwargs_1, - clock=clock_man, - ) - clock_ag = ExternalClock() - container_ag = await create_container( - connection_type=connection_type, - codec=codec, - addr=repl_addr, - mqtt_kwargs=mqtt_kwargs_2, - clock=clock_ag, - ) - - clock_agent = DistributedClockAgent(container_ag) - clock_manager = DistributedClockManager( - container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] - ) - receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller(container_man, repl_addr, receiver.aid) - await tasks_complete_or_sleeping(container_man) - - assert receiver._scheduler.clock.time == 0 - # first synchronize the clock to the receiver - await clock_manager.send_current_time() - # just waiting until it is done is not enough - await tasks_complete_or_sleeping(container_man) - # as we return to soon and did not yet have set the time - assert receiver._scheduler.clock.time == 0 - # increase the time, triggering an action in the caller - clock_man.set_time(10) - # distribute the new time to the clock_manager - await clock_manager.send_current_time() - # and wait until everything is done - await tasks_complete_or_sleeping(container_man) - # also wait for the result in the agent container - next_event = await clock_manager.get_next_event() - assert receiver._scheduler.clock.time == 10 - # now the response should be received - assert caller.i == 1, "received one message" - clock_man.set_time(15) - await clock_manager.send_current_time() - next_event = await clock_manager.get_next_event() - # the clock_manager distributed the time to the other container - assert clock_ag.time == 15 - clock_man.set_time(1000) - await clock_manager.send_current_time() - next_event = await clock_manager.get_next_event() - # did work the second time too - assert clock_ag.time == 1000 - - # finally shut down - await asyncio.gather( - container_man.shutdown(), - container_ag.shutdown(), - ) + container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + clock_agent = container_ag.include(DistributedClockAgent()) + clock_manager = container_man.include(DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + )) + receiver = container_ag.include(Receiver()) + caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) + + async with activate(container_man, container_ag) as cl: + await tasks_complete_or_sleeping(container_man) + + assert receiver.scheduler.clock.time == 0 + # first synchronize the clock to the receiver + await clock_manager.send_current_time() + # just waiting until it is done is not enough + await tasks_complete_or_sleeping(container_man) + # as we return to soon and did not yet have set the time + assert receiver.scheduler.clock.time == 0 + # increase the time, triggering an action in the caller + container_man.clock.set_time(10) + # distribute the new time to the clock_manager + await clock_manager.send_current_time() + # and wait until everything is done + await tasks_complete_or_sleeping(container_man) + # also wait for the result in the agent container + next_event = await clock_manager.get_next_event() + assert receiver.scheduler.clock.time == 10 + # now the response should be received + assert caller.i == 1, "received one message" + container_man.clock.set_time(15) + await clock_manager.send_current_time() + next_event = await clock_manager.get_next_event() + # the clock_manager distributed the time to the other container + assert container_ag.clock.time == 15 + container_man.clock.set_time(1000) + await clock_manager.send_current_time() + next_event = await clock_manager.get_next_event() + # did work the second time too + assert container_ag.clock.time == 1000 @pytest.mark.asyncio diff --git a/tests/unit_tests/container/test_mp.py b/tests/unit_tests/container/test_mp.py index 4cd511d..33fcd18 100644 --- a/tests/unit_tests/container/test_mp.py +++ b/tests/unit_tests/container/test_mp.py @@ -2,7 +2,7 @@ import pytest -from mango import Agent, create_container +from mango import Agent, create_tcp_container, sender_addr, AgentAddress, activate, addr class MyAgent(Agent): @@ -14,17 +14,10 @@ def handle_message(self, content, meta): # get addr and id from sender if self.test_counter == 1: - receiver_host, receiver_port = meta["sender_addr"] - receiver_id = meta["sender_id"] # send back pong, providing your own details - self.current_task = self.schedule_instant_acl_message( + self.current_task = self.schedule_instant_message( content="pong", - receiver_addr=(receiver_host, receiver_port), - receiver_id=receiver_id, - acl_metadata={ - "sender_addr": self.addr, - "sender_id": self.aid, - }, + receiver_addr=sender_addr(meta) ) @@ -37,21 +30,17 @@ def handle_message(self, content, meta): class P2PTestAgent(Agent): - def __init__(self, container, response_aid, suggested_aid: str = None): - super().__init__(container, suggested_aid) - self._response_aid = response_aid + receiver_id: str + + def __init__(self, receiver_id): + super().__init__() + self.receiver_id = receiver_id def handle_message(self, content, meta): - receiver_host, receiver_port = meta["sender_addr"] # send back pong, providing your own details - self.current_task = self.schedule_instant_acl_message( + self.current_task = self.schedule_instant_message( content="pong", - receiver_addr=(receiver_host, receiver_port), - receiver_id=self._response_aid, - acl_metadata={ - "sender_addr": self.addr, - "sender_id": self.aid, - }, + receiver_addr=addr(meta["sender_addr"], self.receiver_id) ) @@ -74,108 +63,90 @@ def handle_message(self, content, meta): ) async def test_agent_processes_ping_pong(num_sp_agents, num_sp): # GIVEN - c = await create_container(addr=("127.0.0.2", 15589), copy_internal_messages=False) + c = create_tcp_container(addr=("127.0.0.2", 15589), copy_internal_messages=False) for i in range(num_sp): await c.as_agent_process( agent_creator=lambda container: [ - MyAgent(container, suggested_aid=f"process_agent{i},{j}") + container.include(MyAgent(), suggested_aid=f"process_agent{i},{j}") for j in range(num_sp_agents) ] ) - agent = MyAgent(c) + agent = c.include(MyAgent()) # WHEN - for i in range(num_sp): - for j in range(num_sp_agents): - await agent.send_acl_message( - "Message To Process Agent", - receiver_addr=c.addr, - receiver_id=f"process_agent{i},{j}", - acl_metadata={"sender_id": agent.aid}, - ) - while agent.test_counter != num_sp_agents * num_sp: - await asyncio.sleep(0.1) + async with activate(c) as c: + for i in range(num_sp): + for j in range(num_sp_agents): + await agent.send_message( + "Message To Process Agent", + receiver_addr=addr(c.addr, f"process_agent{i},{j}") + ) + while agent.test_counter != num_sp_agents * num_sp: + await asyncio.sleep(0.1) assert agent.test_counter == num_sp_agents * num_sp - await c.shutdown() - @pytest.mark.asyncio async def test_agent_processes_ping_pong_p_to_p(): # GIVEN - addr = ("127.0.0.2", 5826) + addr = ("127.0.0.2", 5829) aid_main_agent = "main_agent" - c = await create_container(addr=addr, copy_internal_messages=False) + c = create_tcp_container(addr=addr, copy_internal_messages=False) await c.as_agent_process( - agent_creator=lambda container: P2PTestAgent( - container, aid_main_agent, suggested_aid="process_agent1" - ) + agent_creator=lambda container: container.include(P2PTestAgent(aid_main_agent), + suggested_aid="process_agent1") ) - main_agent = P2PMainAgent(c, suggested_aid=aid_main_agent) + main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) # WHEN def agent_init(c): - agent = MyAgent(c, suggested_aid="process_agent2") - agent.schedule_instant_acl_message( + agent = c.include(MyAgent(), suggested_aid="process_agent2") + agent.schedule_instant_message( "Message To Process Agent", - receiver_addr=addr, - receiver_id="process_agent1", - acl_metadata={"sender_id": agent.aid}, + receiver_addr=AgentAddress(addr, "process_agent1") ) return agent - await c.as_agent_process(agent_creator=agent_init) + async with activate(c) as c: + await c.as_agent_process(agent_creator=agent_init) - while main_agent.test_counter != 1: - await asyncio.sleep(0.1) + while main_agent.test_counter != 1: + await asyncio.sleep(0.1) assert main_agent.test_counter == 1 - await c.shutdown() - @pytest.mark.asyncio async def test_async_agent_processes_ping_pong_p_to_p(): # GIVEN - addr = ("127.0.0.2", 5826) + addr = ("127.0.0.2", 5811) aid_main_agent = "main_agent" - c = await create_container(addr=addr, copy_internal_messages=False) - main_agent = P2PMainAgent(c, suggested_aid=aid_main_agent) + c = create_tcp_container(addr=addr, copy_internal_messages=False) + main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) async def agent_creator(container): - p2pta = P2PTestAgent(container, aid_main_agent, suggested_aid="process_agent1") + p2pta = container.include(P2PTestAgent(aid_main_agent), suggested_aid="process_agent1") await p2pta.send_message( content="pong", - receiver_addr=addr, - receiver_id=aid_main_agent, - acl_metadata={ - "sender_addr": p2pta.addr, - "sender_id": p2pta.aid, - }, + receiver_addr=main_agent.addr ) - await c.as_agent_process(agent_creator=agent_creator) + async with activate(c) as c: + await c.as_agent_process(agent_creator=agent_creator) - # WHEN - def agent_init(c): - agent = MyAgent(c, suggested_aid="process_agent2") - agent.schedule_instant_acl_message( - "Message To Process Agent", - receiver_addr=addr, - receiver_id="process_agent1", - acl_metadata={"sender_id": agent.aid}, - ) - return agent + # WHEN + def agent_init(c): + agent = c.include(MyAgent(), suggested_aid="process_agent2") + agent.schedule_instant_message("Message To Process Agent", AgentAddress(addr, "process_agent1")) + return agent - await c.as_agent_process(agent_creator=agent_init) + await c.as_agent_process(agent_creator=agent_init) - while main_agent.test_counter != 2: - await asyncio.sleep(0.1) + while main_agent.test_counter != 2: + await asyncio.sleep(0.1) assert main_agent.test_counter == 2 - await c.shutdown() - if __name__ == "__main__": asyncio.run(test_agent_processes_ping_pong(5, 5)) diff --git a/tests/unit_tests/container/test_tcp.py b/tests/unit_tests/container/test_tcp.py index b04ccda..7ff2ec4 100644 --- a/tests/unit_tests/container/test_tcp.py +++ b/tests/unit_tests/container/test_tcp.py @@ -2,21 +2,24 @@ import pytest -from mango import create_container +from mango import create_tcp_container from mango.container.protocol import ContainerProtocol from mango.container.tcp import TCPConnectionPool @pytest.mark.asyncio async def test_connection_open_close(): - c = await create_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + await c.start() await c.shutdown() @pytest.mark.asyncio async def test_connection_pool_obtain_release(): - c = await create_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) - c2 = await create_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + c2 = create_tcp_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) + await c.start() + await c2.start() addr = "127.0.0.2", 5556 connection_pool = TCPConnectionPool(asyncio.get_event_loop()) @@ -40,9 +43,11 @@ async def test_connection_pool_obtain_release(): @pytest.mark.asyncio async def test_connection_pool_double_obtain_release(): - c = await create_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) - c2 = await create_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) - + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + c2 = create_tcp_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) + await c.start() + await c2.start() + addr = "127.0.0.2", 5556 connection_pool = TCPConnectionPool(asyncio.get_event_loop()) raw_prot = ContainerProtocol( @@ -80,9 +85,12 @@ async def test_connection_pool_double_obtain_release(): async def test_ttl(): addr = "127.0.0.2", 5556 addr2 = "127.0.0.2", 5557 - c = await create_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) - c2 = await create_container(addr=addr, copy_internal_messages=False) - c3 = await create_container(addr=addr2, copy_internal_messages=False) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + c2 = create_tcp_container(addr=addr, copy_internal_messages=False) + c3 = create_tcp_container(addr=addr2, copy_internal_messages=False) + await c.start() + await c2.start() + await c3.start() connection_pool = TCPConnectionPool(asyncio.get_event_loop(), ttl_in_sec=0.1) raw_prot = ContainerProtocol( @@ -120,8 +128,10 @@ async def test_ttl(): @pytest.mark.asyncio async def test_max_connections(): - c = await create_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) - c2 = await create_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) + c2 = create_tcp_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) + await c.start() + await c2.start() addr = "127.0.0.2", 5556 connection_pool = TCPConnectionPool( diff --git a/tests/unit_tests/core/test_agent.py b/tests/unit_tests/core/test_agent.py index df8c436..d0b47f5 100644 --- a/tests/unit_tests/core/test_agent.py +++ b/tests/unit_tests/core/test_agent.py @@ -1,33 +1,33 @@ import asyncio -from typing import Any, Dict +from typing import Any import pytest -from mango import create_container +from mango import create_tcp_container, create_acl, activate from mango.agent.core import Agent class MyAgent(Agent): test_counter: int = 0 - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): self.test_counter += 1 @pytest.mark.asyncio async def test_periodic_facade(): # GIVEN - c = await create_container(addr=("127.0.0.2", 5555)) - agent = MyAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) l = [] async def increase_counter(): l.append(1) # WHEN - t = agent.schedule_periodic_task(increase_counter, 2) + t = agent.schedule_periodic_task(increase_counter, 0.2) try: - await asyncio.wait_for(t, timeout=3) + await asyncio.wait_for(t, timeout=0.3) except asyncio.exceptions.TimeoutError: pass @@ -39,68 +39,62 @@ async def increase_counter(): @pytest.mark.asyncio async def test_send_message(): # GIVEN - c = await create_container(addr=("127.0.0.2", 5555)) - agent = MyAgent(c) - agent2 = MyAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) + agent2 = c.include(MyAgent()) - await agent.send_message("", receiver_addr=agent.addr, receiver_id=agent2.aid) - msg = await agent2.inbox.get() - _, content, meta = msg - agent2.handle_message(content=content, meta=meta) + async with activate(c) as c: + await agent.send_message("", receiver_addr=agent2.addr) + msg = await agent2.inbox.get() + _, content, meta = msg + agent2.handle_message(content=content, meta=meta) # THEN assert agent2.test_counter == 1 - await c.shutdown() @pytest.mark.asyncio async def test_send_acl_message(): # GIVEN - c = await create_container(addr=("127.0.0.2", 5555)) - agent = MyAgent(c) - agent2 = MyAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) + agent2 = c.include(MyAgent()) - await agent.send_acl_message("", receiver_addr=agent.addr, receiver_id=agent2.aid) - msg = await agent2.inbox.get() - _, content, meta = msg - agent2.handle_message(content=content, meta=meta) + async with activate(c) as c: + await agent.send_message(create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), receiver_addr=agent2.addr) + msg = await agent2.inbox.get() + _, content, meta = msg + agent2.handle_message(content=content, meta=meta) # THEN assert agent2.test_counter == 1 - await c.shutdown() @pytest.mark.asyncio async def test_schedule_message(): # GIVEN - c = await create_container(addr=("127.0.0.2", 5555)) - agent = MyAgent(c) - agent2 = MyAgent(c) - - agent.schedule_instant_message("", receiver_addr=agent.addr, receiver_id=agent2.aid) - msg = await agent2.inbox.get() - _, content, meta = msg - agent2.handle_message(content=content, meta=meta) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) + agent2 = c.include(MyAgent()) + async with activate(c) as c: + await agent.schedule_instant_message("", receiver_addr=agent2.addr) + # THEN assert agent2.test_counter == 1 - await c.shutdown() @pytest.mark.asyncio async def test_schedule_acl_message(): # GIVEN - c = await create_container(addr=("127.0.0.2", 5555)) - agent = MyAgent(c) - agent2 = MyAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) + agent2 = c.include(MyAgent()) - agent.schedule_instant_acl_message( - "", receiver_addr=agent.addr, receiver_id=agent2.aid - ) - msg = await agent2.inbox.get() - _, content, meta = msg - agent2.handle_message(content=content, meta=meta) + async with activate(c) as c: + await agent.schedule_instant_message( + create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), receiver_addr=agent2.addr + ) # THEN assert agent2.test_counter == 1 - await c.shutdown() diff --git a/tests/unit_tests/core/test_container.py b/tests/unit_tests/core/test_container.py index c36c2b8..4601138 100644 --- a/tests/unit_tests/core/test_container.py +++ b/tests/unit_tests/core/test_container.py @@ -1,18 +1,20 @@ import pytest -import mango.container.factory as container_factory from mango.agent.core import Agent - +from mango import create_tcp_container, create_acl class LooksLikeAgent: async def shutdown(self): pass + def _do_register(self, container, aid): + pass + @pytest.mark.asyncio async def test_register_aid_pattern_match(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() suggested_aid = "agent12" @@ -27,7 +29,7 @@ async def test_register_aid_pattern_match(): @pytest.mark.asyncio async def test_register_aid_success(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() suggested_aid = "cagent12" @@ -42,7 +44,7 @@ async def test_register_aid_success(): @pytest.mark.asyncio async def test_register_no_suggested(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() # WHEN @@ -56,7 +58,7 @@ async def test_register_no_suggested(): @pytest.mark.asyncio async def test_register_pattern_half_match(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() suggested_aid = "agentABC" @@ -71,7 +73,7 @@ async def test_register_pattern_half_match(): @pytest.mark.asyncio async def test_register_existing(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() suggested_aid = "agentABC" @@ -88,7 +90,7 @@ async def test_register_existing(): @pytest.mark.asyncio async def test_is_aid_available(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) aid_to_check = "agentABC" # WHEN @@ -102,7 +104,7 @@ async def test_is_aid_available(): @pytest.mark.asyncio async def test_is_aid_available_but_match(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) aid_to_check = "agent5" # WHEN @@ -116,7 +118,7 @@ async def test_is_aid_available_but_match(): @pytest.mark.asyncio async def test_is_aid_not_available(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) c.register_agent(LooksLikeAgent(), "abc") aid_to_check = "abc" @@ -131,7 +133,7 @@ async def test_is_aid_not_available(): @pytest.mark.asyncio async def test_is_aid_not_available_and_match(): # GIVEN - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) c.register_agent(LooksLikeAgent()) aid_to_check = "agent0" @@ -145,10 +147,11 @@ async def test_is_aid_not_available_and_match(): @pytest.mark.asyncio async def test_create_acl_no_modify(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) common_acl_q = {} - actual_acl_message = c._create_acl( - "", receiver_addr="", receiver_id="", acl_metadata=common_acl_q + actual_acl_message = create_acl( + "", receiver_addr="", receiver_id="", acl_metadata=common_acl_q, + sender_addr=c.addr ) assert "reeiver_addr" not in common_acl_q @@ -160,9 +163,10 @@ async def test_create_acl_no_modify(): @pytest.mark.asyncio async def test_create_acl_anon(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) - actual_acl_message = c._create_acl( - "", receiver_addr="", receiver_id="", is_anonymous_acl=True + c = create_tcp_container(addr=("127.0.0.2", 5555)) + actual_acl_message = create_acl( + "", receiver_addr="", receiver_id="", is_anonymous_acl=True, + sender_addr=c.addr ) assert actual_acl_message.sender_addr is None @@ -171,9 +175,9 @@ async def test_create_acl_anon(): @pytest.mark.asyncio async def test_create_acl_not_anon(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) - actual_acl_message = c._create_acl( - "", receiver_addr="", receiver_id="", is_anonymous_acl=False + c = create_tcp_container(addr=("127.0.0.2", 5555)) + actual_acl_message = create_acl( + "", receiver_addr="", receiver_id="", is_anonymous_acl=False, sender_addr=c.addr ) assert actual_acl_message.sender_addr is not None @@ -191,14 +195,14 @@ class Data: @pytest.mark.asyncio async def test_send_message_no_copy(): - c = await container_factory.create( + c = create_tcp_container( addr=("127.0.0.2", 5555), copy_internal_messages=False ) - agent1 = ExampleAgent(c) + agent1 = c.include(ExampleAgent()) message_to_send = Data() - await c.send_acl_message( - message_to_send, receiver_addr=c.addr, receiver_id=agent1.aid + await c.send_message( + message_to_send, receiver_addr=agent1.addr ) await c.shutdown() @@ -207,14 +211,14 @@ async def test_send_message_no_copy(): @pytest.mark.asyncio async def test_send_message_copy(): - c = await container_factory.create( + c = create_tcp_container( addr=("127.0.0.2", 5555), copy_internal_messages=True ) - agent1 = ExampleAgent(c) + agent1 = c.include(ExampleAgent()) message_to_send = Data() - await c.send_acl_message( - message_to_send, receiver_addr=c.addr, receiver_id=agent1.aid + await c.send_message( + message_to_send, receiver_addr=agent1.addr ) await c.shutdown() @@ -223,13 +227,14 @@ async def test_send_message_copy(): @pytest.mark.asyncio async def test_create_acl_diff_receiver(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) + c = create_tcp_container(addr=("127.0.0.2", 5555)) with pytest.warns(UserWarning) as record: - actual_acl_message = c._create_acl( + actual_acl_message = create_acl( "", receiver_addr="A", receiver_id="A", acl_metadata={"receiver_id": "B", "receiver_addr": "B"}, + sender_addr=c.addr, is_anonymous_acl=False, ) @@ -241,8 +246,8 @@ async def test_create_acl_diff_receiver(): @pytest.mark.asyncio async def test_containers_dont_share_default_codec(): - c1 = await container_factory.create(addr=("127.0.0.2", 5555)) - c2 = await container_factory.create(addr=("127.0.0.2", 5556)) + c1 = create_tcp_container(addr=("127.0.0.2", 5555)) + c2 = create_tcp_container(addr=("127.0.0.2", 5556)) assert c1.codec is not c2.codec diff --git a/tests/unit_tests/core/test_external_scheduling_container.py b/tests/unit_tests/core/test_external_scheduling_container.py index 021a4fc..3d5e8de 100644 --- a/tests/unit_tests/core/test_external_scheduling_container.py +++ b/tests/unit_tests/core/test_external_scheduling_container.py @@ -3,18 +3,18 @@ import pytest -import mango.container.factory as container_factory from mango.agent.core import Agent from mango.container.external_coupling import ExternalAgentMessage from mango.container.factory import EXTERNAL_CONNECTION from mango.messages.message import ACLMessage from mango.util.clock import ExternalClock +from mango import AgentAddress, create_ec_container, AgentAddress, sender_addr, create_acl @pytest.mark.asyncio async def test_init(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1234", connection_type=EXTERNAL_CONNECTION + external_scheduling_container = create_ec_container( + addr="external_eid_1234" ) assert external_scheduling_container.addr == "external_eid_1234" assert isinstance(external_scheduling_container.clock, ExternalClock) @@ -23,11 +23,9 @@ async def test_init(): @pytest.mark.asyncio async def test_send_msg(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1234", connection_type=EXTERNAL_CONNECTION - ) - await external_scheduling_container.send_acl_message( - content="test", receiver_addr="eid321", receiver_id="Agent0" + external_scheduling_container = create_ec_container(addr="external_eid_1234") + await external_scheduling_container.send_message( + content="test", receiver_addr=AgentAddress("eid321", aid="Agent0"), sender_id="" ) assert len(external_scheduling_container.message_buffer) == 1 external_agent_msg: ExternalAgentMessage = ( @@ -36,18 +34,16 @@ async def test_send_msg(): assert external_agent_msg.receiver == "eid321" decoded_msg = external_scheduling_container.codec.decode(external_agent_msg.message) assert decoded_msg.content == "test" - assert decoded_msg.receiver_addr == "eid321" - assert decoded_msg.receiver_id == "Agent0" + assert decoded_msg.meta["receiver_addr"] == "eid321" + assert decoded_msg.meta["receiver_id"] == "Agent0" await external_scheduling_container.shutdown() @pytest.mark.asyncio async def test_step(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1234", connection_type=EXTERNAL_CONNECTION - ) - await external_scheduling_container.send_acl_message( - content="test", receiver_addr="eid321", receiver_id="Agent0" + external_scheduling_container = create_ec_container(addr="external_eid_1234") + await external_scheduling_container.send_message( + content="test", receiver_addr=AgentAddress("eid321", aid="Agent0") ) step_output = await external_scheduling_container.step( simulation_time=12, incoming_messages=[] @@ -61,23 +57,24 @@ async def test_step(): assert external_msg.receiver == "eid321" decoded_msg = external_scheduling_container.codec.decode(external_msg.message) assert decoded_msg.content == "test" - assert decoded_msg.receiver_addr == "eid321" - assert decoded_msg.receiver_id == "Agent0" + assert decoded_msg.meta["receiver_addr"] == "eid321" + assert decoded_msg.meta["receiver_id"] == "Agent0" await external_scheduling_container.shutdown() class ReplyAgent(Agent): - def __init__(self, container): - super().__init__(container) + def __init__(self): + super().__init__() self.current_ping = 0 self.tasks = [] + + def on_register(self): self.tasks.append(self.schedule_periodic_task(self.send_ping, delay=10)) async def send_ping(self): - await self.send_acl_message( + await self.send_message( content=f"ping{self.current_ping}", - receiver_addr="ping_receiver_addr", - receiver_id="ping_receiver_id", + receiver_addr=AgentAddress("ping_receiver_addr","ping_receiver_id") ) self.current_ping += 1 @@ -85,16 +82,14 @@ def handle_message(self, content, meta: Dict[str, Any]): self.schedule_instant_task(self.sleep_and_answer(content, meta)) async def sleep_and_answer(self, content, meta): - await self.send_acl_message( + await self.send_message( content=f"I received {content}", - receiver_addr=meta["sender_addr"], - receiver_id=["sender_id"], + receiver_addr=sender_addr(meta) ) await asyncio.sleep(0.1) - await self.send_acl_message( + await self.send_message( content=f"Thanks for sending {content}", - receiver_addr=meta["sender_addr"], - receiver_id=["sender_id"], + receiver_addr=sender_addr(meta) ) async def stop_tasks(self): @@ -107,10 +102,12 @@ async def stop_tasks(self): class WaitForMessageAgent(Agent): - def __init__(self, container): - super().__init__(container) + def __init__(self): + super().__init__() self.received_msg = False + + def on_register(self): self.schedule_conditional_task( condition_func=lambda: self.received_msg, coroutine=self.print_cond_task_finished(), @@ -126,10 +123,10 @@ def handle_message(self, content, meta: Dict[str, Any]): @pytest.mark.asyncio async def test_step_with_cond_task(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1", connection_type=EXTERNAL_CONNECTION + external_scheduling_container = create_ec_container( + addr="external_eid_1" ) - agent_1 = WaitForMessageAgent(external_scheduling_container) + agent_1 = external_scheduling_container.include(WaitForMessageAgent()) print("Agent init") current_time = 0 @@ -152,10 +149,11 @@ async def test_step_with_cond_task(): ) # create and send message in next step - message = external_scheduling_container._create_acl( + message = create_acl( content="", receiver_addr=external_scheduling_container.addr, receiver_id=agent_1.aid, + sender_addr=external_scheduling_container.addr ) encoded_msg = external_scheduling_container.codec.encode(message) print("created message") @@ -184,8 +182,8 @@ async def test_step_with_cond_task(): class SelfSendAgent(Agent): - def __init__(self, container, final_number=3): - super().__init__(container) + def __init__(self, final_number=3): + super().__init__() self.no_received_msg = 0 self.final_no = final_number @@ -198,25 +196,24 @@ def handle_message(self, content, meta: Dict[str, Any]): i += 1 # send message to yourself if necessary if self.no_received_msg < self.final_no: - self.schedule_instant_acl_message( - receiver_addr=self.addr, receiver_id=self.aid, content=content + self.schedule_instant_message( + receiver_addr=self.addr, content=content ) else: - self.schedule_instant_acl_message( - receiver_addr="AnyOtherAddr", receiver_id="AnyOtherId", content=content - ) + self.schedule_instant_message(content, AgentAddress("AnyOtherAddr", "AnyOtherId")) @pytest.mark.asyncio async def test_send_internal_messages(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1", connection_type=EXTERNAL_CONNECTION + external_scheduling_container = create_ec_container( + addr="external_eid_1" ) - agent_1 = SelfSendAgent(container=external_scheduling_container, final_number=3) - message = external_scheduling_container._create_acl( + agent_1 = external_scheduling_container.include(SelfSendAgent(final_number=3)) + message = create_acl( content="", receiver_addr=external_scheduling_container.addr, receiver_id=agent_1.aid, + sender_addr=external_scheduling_container.addr ) encoded_msg = external_scheduling_container.codec.encode(message) return_values = await external_scheduling_container.step( @@ -229,10 +226,10 @@ async def test_send_internal_messages(): @pytest.mark.asyncio async def test_step_with_replying_agent(): - external_scheduling_container = await container_factory.create( - addr="external_eid_1", connection_type=EXTERNAL_CONNECTION + external_scheduling_container = create_ec_container( + addr="external_eid_1" ) - reply_agent = ReplyAgent(container=external_scheduling_container) + reply_agent = external_scheduling_container.include(ReplyAgent()) new_acl_msg = ACLMessage() new_acl_msg.content = "hello you" new_acl_msg.receiver_addr = "external_eid_1" diff --git a/tests/unit_tests/express/test_api.py b/tests/unit_tests/express/test_api.py new file mode 100644 index 0000000..dc9453c --- /dev/null +++ b/tests/unit_tests/express/test_api.py @@ -0,0 +1,129 @@ +from typing import Any +import pytest +import asyncio + +from mango import agent_composed_of, activate, create_tcp_container, Role, sender_addr, Agent, run_with_tcp, run_with_mqtt, addr + +class PingPongRole(Role): + counter: int = 0 + + def handle_message(self, content: Any, meta: dict): + if self.counter >= 5: + return + + self.counter += 1 + + if content == "Ping": + self.context.schedule_instant_message("Pong", sender_addr(meta)) + elif content == "Pong": + self.context.schedule_instant_message("Ping", sender_addr(meta)) + + + +@pytest.mark.asyncio +async def test_activate_pingpong(): + container = create_tcp_container("127.0.0.1:5555") + ping_pong_agent = agent_composed_of(PingPongRole(), register_in=container) + ping_pong_agent_two = agent_composed_of(PingPongRole(), register_in=container) + + async with activate(container) as c: + await c.send_message("Ping", ping_pong_agent.addr, sender_id=ping_pong_agent_two.aid) + while ping_pong_agent.roles[0].counter < 5: + await asyncio.sleep(0.01) + + assert ping_pong_agent.roles[0].counter == 5 + + +class MyAgent(Agent): + test_counter: int = 0 + + def handle_message(self, content, meta: dict[str, Any]): + self.test_counter += 1 + + +@pytest.mark.asyncio +async def test_activate_api_style_agent(): + # GIVEN + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(MyAgent()) + agent2 = c.include(MyAgent()) + + # WHEN + async with activate(c) as c: + await agent.schedule_instant_message( + "", receiver_addr=agent2.addr + ) + + # THEN + assert agent2.test_counter == 1 + + +@pytest.mark.asyncio +async def test_run_api_style_agent(): + # GIVEN + run_agent = MyAgent() + run_agent2 = MyAgent() + + # WHEN + async with run_with_tcp(1, run_agent, run_agent2) as c: + await run_agent.schedule_instant_message( + "", receiver_addr=run_agent2.addr + ) + + # THEN + assert run_agent2.test_counter == 1 + + +@pytest.mark.asyncio +async def test_run_api_style_agent(): + # GIVEN + run_agent = MyAgent() + run_agent2 = MyAgent() + + # WHEN + async with run_with_tcp(1, run_agent, run_agent2) as c: + await run_agent.schedule_instant_message( + "", receiver_addr=run_agent2.addr + ) + + # THEN + assert run_agent2.test_counter == 1 + + +@pytest.mark.asyncio +async def test_run_api_style_agent_with_aid(): + # GIVEN + run_agent = MyAgent() + run_agent2 = MyAgent() + + # WHEN + async with run_with_tcp(1, run_agent, (run_agent2, dict(aid="my_custom_aid"))) as c: + await run_agent.schedule_instant_message( + "", receiver_addr=run_agent2.addr + ) + + # THEN + assert run_agent2.test_counter == 1 + assert run_agent2.aid == "my_custom_aid" + assert run_agent.aid == "agent0" + +@pytest.mark.asyncio +async def test_run_api_style_agent_with_aid_mqtt(): + # GIVEN + run_agent = MyAgent() + run_agent2 = MyAgent() + + # WHEN + async with run_with_mqtt(1, + (run_agent, dict(topics=["my_top"])), + (run_agent2, dict(topics=["your_top"], aid="my_custom_aid"))) as c: + await run_agent.schedule_instant_message( + "", addr("your_top", run_agent2.aid) + ) + while run_agent2.test_counter == 0: + await asyncio.sleep(0.01) + + # THEN + assert run_agent2.test_counter == 1 + assert run_agent2.aid == "my_custom_aid" + assert run_agent.aid == "agent0" diff --git a/tests/unit_tests/messages/test_codecs.py b/tests/unit_tests/messages/test_codecs.py index 31d2a96..51ef449 100644 --- a/tests/unit_tests/messages/test_codecs.py +++ b/tests/unit_tests/messages/test_codecs.py @@ -10,7 +10,7 @@ SerializationError, json_serializable, ) -from mango.messages.message import ACLMessage, Performatives +from mango.messages.message import ACLMessage, Performatives, MangoMessage from .msg_pb2 import MyMsg @@ -162,6 +162,16 @@ def test_codec_known(codec): assert vars(msg_new) == vars(msg) +@pytest.mark.parametrize("codec", testcodecs) +def test_codec_mango_message(codec): + msg = MangoMessage("abc", dict(sender_id=2)) + + my_codec = codec() + msg_new = my_codec.decode(my_codec.encode(msg)) + + assert vars(msg_new) == vars(msg) + + def test_json_data_class(): my_codec = JSON() my_codec.add_serializer(*SomeDataClass.__serializer__()) diff --git a/tests/unit_tests/role/role_test.py b/tests/unit_tests/role/role_test.py index 2a9a0f8..5c41977 100644 --- a/tests/unit_tests/role/role_test.py +++ b/tests/unit_tests/role/role_test.py @@ -125,7 +125,7 @@ def setup(self) -> None: def test_emit_event(): # GIVEN role_handler = RoleHandler(None, None) - context = RoleContext(None, None, role_handler, None, None) + context = RoleContext(role_handler, None, None) ex_role = SubRole() ex_role2 = RoleHandlingEvents() context.add_role(ex_role) diff --git a/tests/unit_tests/role_agent_test.py b/tests/unit_tests/role_agent_test.py index dbfe2de..b896f8b 100644 --- a/tests/unit_tests/role_agent_test.py +++ b/tests/unit_tests/role_agent_test.py @@ -1,21 +1,19 @@ import asyncio import datetime -from abc import abstractmethod from typing import Any, Dict import pytest -import mango.container.factory as container_factory from mango.agent.role import Role, RoleAgent, RoleContext from mango.util.scheduling import TimestampScheduledTask +from mango import sender_addr, create_tcp_container, activate class SimpleReactiveRole(Role): def setup(self): - self.context.subscribe_message(self, self.handle_message, self.is_applicable) + self.context.subscribe_message(self, self.react_handle_message, self.is_applicable) - @abstractmethod - def handle_message(self, content, meta: Dict[str, Any]) -> None: + def react_handle_message(self, content, meta: Dict[str, Any]) -> None: pass def is_applicable(self, content, meta: Dict[str, Any]) -> bool: @@ -27,21 +25,13 @@ def __init__(self): super().__init__() self.sending_tasks = [] - def handle_message(self, content, meta: Dict[str, Any]): + def react_handle_message(self, content, meta: Dict[str, Any]): assert "sender_addr" in meta.keys() and "sender_id" in meta.keys() - # get addr and id from sender - receiver_host, receiver_port = meta["sender_addr"] - receiver_id = meta["sender_id"] # send back pong, providing your own details - t = self.context.schedule_instant_acl_message( + t = self.context.schedule_instant_message( content="pong", - receiver_addr=(receiver_host, receiver_port), - receiver_id=receiver_id, - acl_metadata={ - "sender_addr": self.context.addr, - "sender_id": self.context.aid, - }, + receiver_addr=sender_addr(meta) ) self.sending_tasks.append(t) @@ -51,51 +41,43 @@ def is_applicable(self, content, meta): class PingRole(SimpleReactiveRole): - def __init__(self, addr, expect_no_answer=False): + def __init__(self, target, expect_no_answer=False): self.open_ping_requests = {} - self._addr = addr + self.target = target self._expect_no_answer = expect_no_answer - def handle_message(self, content, meta: Dict[str, Any]): + def react_handle_message(self, content, meta: Dict[str, Any]): assert "sender_addr" in meta.keys() and "sender_id" in meta.keys() - # get host, port and id from sender - sender_host, sender_port = meta["sender_addr"] - sender_id = meta["sender_id"] - assert ((sender_host, sender_port), sender_id) in self.open_ping_requests.keys() + sender = sender_addr(meta) + assert sender in self.open_ping_requests.keys() - self.open_ping_requests[((sender_host, sender_port), sender_id)].set_result( + self.open_ping_requests[sender].set_result( True ) def is_applicable(self, content, meta): return content == "pong" - def setup(self): - super().setup() + def on_ready(self): for task in list( map( lambda a: TimestampScheduledTask( - self.send_ping_to_other(a[0], a[1], self.context), + self.send_ping_to_other(a, self.context), datetime.datetime.now().timestamp(), ), - self._addr, + self.target, ) ): self.context.schedule_task(task) async def send_ping_to_other( - self, other_addr, other_id, agent_context: RoleContext + self, other_addr, agent_context: RoleContext ): # create - self.open_ping_requests[(other_addr, other_id)] = asyncio.Future() - success = await agent_context.send_acl_message( + self.open_ping_requests[other_addr] = asyncio.Future() + success = await agent_context.send_message( content="ping", receiver_addr=other_addr, - receiver_id=other_id, - acl_metadata={ - "sender_addr": agent_context.addr, - "sender_id": agent_context.aid, - }, ) assert success @@ -103,13 +85,13 @@ async def on_stop(self): await self.wait_for_pong_replies() async def wait_for_pong_replies(self, timeout=1): - for addr_tuple, fut in self.open_ping_requests.items(): + for addr, fut in self.open_ping_requests.items(): try: await asyncio.wait_for(fut, timeout=timeout) assert not self._expect_no_answer except asyncio.TimeoutError: print( - f"Timeout occurred while waiting for the ping response of {addr_tuple}, " + f"Timeout occurred while waiting for the ping response of {addr}, " "going to check if all messages could be send" ) assert ( @@ -144,7 +126,7 @@ async def test_send_ping_pong(num_agents, num_containers): # create containers containers = [] for i in range(num_containers): - c = await container_factory.create(addr=("127.0.0.2", 5555 + i)) + c = create_tcp_container(addr=("127.0.0.2", 5555 + i)) containers.append(c) # create agents @@ -152,29 +134,25 @@ async def test_send_ping_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = RoleAgent(c) + a = c.include(RoleAgent()) a.add_role(PongRole()) agents.append(a) - addrs.append((c.addr, a.aid)) + addrs.append(a.addr) # all agents send ping request to all agents (including themselves) for a in agents: a.add_role(PingRole(addrs)) - for a in agents: - if a._check_inbox_task.done(): - if a._check_inbox_task.exception() is not None: - raise a._check_inbox_task.exception() - else: - assert False, "check_inbox terminated unexpectedly." + async with activate(containers) as cl: + for a in agents: + if a._check_inbox_task.done(): + if a._check_inbox_task.exception() is not None: + raise a._check_inbox_task.exception() + else: + assert False, "check_inbox terminated unexpectedly." - for a in agents: - await a.tasks_complete() - - # gracefully shutdown - for a in agents: - await a.shutdown() - await asyncio.gather(*[c.shutdown() for c in containers]) + for a in agents: + await a.tasks_complete() assert len(asyncio.all_tasks()) == 1 @@ -185,7 +163,7 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): # create containers containers = [] for i in range(num_containers): - c = await container_factory.create(addr=("127.0.0.2", 5555 + i)) + c = create_tcp_container(addr=("127.0.0.2", 5555 + i)) containers.append(c) # create agents @@ -193,7 +171,7 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = RoleAgent(c) + a = c.include(RoleAgent()) a.add_role(PongRole()) agents.append(a) addrs.append((c.addr, a.aid)) @@ -204,50 +182,44 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): a.add_role(ping_role) a.add_role(DeactivateAllRoles([ping_role])) - for a in agents: - if a._check_inbox_task.done(): - if a._check_inbox_task.exception() is not None: - raise a._check_inbox_task.exception() - else: - assert False, "check_inbox terminated unexpectedly." - - for a in agents: - await a.tasks_complete() + async with activate(containers) as cl: + for a in agents: + if a._check_inbox_task.done(): + if a._check_inbox_task.exception() is not None: + raise a._check_inbox_task.exception() + else: + assert False, "check_inbox terminated unexpectedly." - # gracefully shutdown - for a in agents: - await a.shutdown() - await asyncio.gather(*[c.shutdown() for c in containers]) + for a in agents: + await a.tasks_complete() assert len(asyncio.all_tasks()) == 1 @pytest.mark.asyncio async def test_role_add_remove(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) - agent = RoleAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(RoleAgent()) role = SampleRole() agent.add_role(role) - assert agent._role_handler.roles[0] == role + assert agent.roles[0] == role agent.remove_role(role) - assert len(agent._role_handler.roles) == 0 - await c.shutdown() + assert len(agent.roles) == 0 @pytest.mark.asyncio async def test_role_add_remove_context(): - c = await container_factory.create(addr=("127.0.0.2", 5555)) - agent = RoleAgent(c) + c = create_tcp_container(addr=("127.0.0.2", 5555)) + agent = c.include(RoleAgent()) role = SampleRole() - agent._role_context.add_role(role) + agent.add_role(role) assert role.setup_called - assert agent._role_handler.roles[0] == role + assert agent.roles[0] == role - agent._role_context.remove_role(role) + agent.remove_role(role) - assert len(agent._role_handler.roles) == 0 - await c.shutdown() + assert len(agent.roles) == 0 diff --git a/tests/unit_tests/test_agents.py b/tests/unit_tests/test_agents.py index 6069425..1972153 100644 --- a/tests/unit_tests/test_agents.py +++ b/tests/unit_tests/test_agents.py @@ -1,61 +1,47 @@ import asyncio -from typing import Any, Dict +from typing import Any import pytest -import mango.container.factory as container_factory -from mango.agent.core import Agent - +from mango import sender_addr, create_tcp_container, activate, Agent class PingPongAgent(Agent): """ Simple PingPongAgent that can send ping and receive pong messages """ - def __init__(self, container): - super().__init__(container) + def __init__(self): + super().__init__() self.open_ping_requests = {} self.sending_tasks = [] - def handle_message(self, content, meta: Dict[str, Any]): + def handle_message(self, content, meta: dict[str, Any]): # answer on ping if content == "ping": assert "sender_addr" in meta.keys() and "sender_id" in meta.keys() - # get addr and id from sender - receiver_host, receiver_port = meta["sender_addr"] - receiver_id = meta["sender_id"] # send back pong, providing your own details - t = self.schedule_instant_acl_message( + t = self.schedule_instant_message( content="pong", - receiver_addr=(receiver_host, receiver_port), - receiver_id=receiver_id, - acl_metadata={"sender_addr": self.addr, "sender_id": self.aid}, + receiver_addr=sender_addr(meta) ) self.sending_tasks.append(t) elif content == "pong": assert "sender_addr" in meta.keys() and "sender_id" in meta.keys() # get host, port and id from sender - sender_host, sender_port = meta["sender_addr"] - sender_id = meta["sender_id"] - assert ( - (sender_host, sender_port), - sender_id, - ) in self.open_ping_requests.keys() + assert sender_addr(meta) in self.open_ping_requests.keys() - self.open_ping_requests[((sender_host, sender_port), sender_id)].set_result( + self.open_ping_requests[sender_addr(meta)].set_result( True ) - async def send_ping_to_other(self, other_addr, other_id): + async def send_ping_to_other(self, other_addr): # create - self.open_ping_requests[(other_addr, other_id)] = asyncio.Future() - success = await self.send_acl_message( + self.open_ping_requests[other_addr] = asyncio.Future() + success = await self.send_message( content="ping", - receiver_addr=other_addr, - receiver_id=other_id, - acl_metadata={"sender_addr": self.addr, "sender_id": self.aid}, + receiver_addr=other_addr ) assert success @@ -86,14 +72,14 @@ async def wait_for_pong_replies(self, timeout=1): @pytest.mark.asyncio async def test_init_and_shutdown(): - c = await container_factory.create(addr=("127.0.0.1", 5555)) - a = PingPongAgent(c) - assert a.aid is not None - assert not a._check_inbox_task.done() - assert not c._check_inbox_task.done() - await a.shutdown() - await c.shutdown() - assert a._stopped + c = create_tcp_container(addr=("127.0.0.1", 5555)) + a = c.include(PingPongAgent()) + + async with activate(c) as c: + assert a.aid is not None + assert not a._check_inbox_task.done() + assert not c._check_inbox_task.done() + assert a._stopped assert not c.running assert len(asyncio.all_tasks()) == 1 @@ -106,7 +92,7 @@ async def test_send_ping_pong(num_agents, num_containers): # create containers containers = [] for i in range(num_containers): - c = await container_factory.create(addr=("127.0.0.2", 5555 + i)) + c = create_tcp_container(addr=("127.0.0.2", 5555 + i)) containers.append(c) # create agents @@ -114,28 +100,24 @@ async def test_send_ping_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = PingPongAgent(c) + a = c.include(PingPongAgent()) agents.append(a) - addrs.append((c.addr, a.aid)) - - # all agents send ping request to all agents (including themselves) - for a in agents: - for receiver_addr, receiver_id in addrs: - await a.send_ping_to_other(other_addr=receiver_addr, other_id=receiver_id) - - for a in agents: - if a._check_inbox_task.done(): - if a._check_inbox_task.exception() is not None: - raise a._check_inbox_task.exception() - else: - assert False, "check_inbox terminated unexpectedly." - for a in agents: - # await a.wait_for_sending_messages() - await a.wait_for_pong_replies() - - # gracefully shutdown - for a in agents: - await a.shutdown() - await asyncio.gather(*[c.shutdown() for c in containers]) + addrs.append(a.addr) + + async with activate(containers) as cl: + # all agents send ping request to all agents (including themselves) + for a in agents: + for receiver_addr in addrs: + await a.send_ping_to_other(receiver_addr) + + for a in agents: + if a._check_inbox_task.done(): + if a._check_inbox_task.exception() is not None: + raise a._check_inbox_task.exception() + else: + assert False, "check_inbox terminated unexpectedly." + for a in agents: + # await a.wait_for_sending_messages() + await a.wait_for_pong_replies() assert len(asyncio.all_tasks()) == 1 From 28f7b15bd0d6ba9feff908fd3bfdc9ee6e74c550 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:07:27 +0200 Subject: [PATCH 02/15] 2.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index af1c7d1..e135455 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ EMAIL = "mango@offis.de" AUTHOR = "mango Team" REQUIRES_PYTHON = ">=3.9.0" -VERSION = "1.2.0" +VERSION = "2.0.0" # What packages are required for this module to be executed? REQUIRED = [ From c4bd9b66442ab73c4a124ae9797376b43f24515f Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:24:25 +0200 Subject: [PATCH 03/15] Fix ruff linting errors. --- .github/workflows/test-mango.yml | 2 +- mango/agent/core.py | 7 ++----- mango/agent/role.py | 5 +---- mango/container/core.py | 8 ++++++-- mango/container/external_coupling.py | 2 +- mango/container/mp.py | 2 +- mango/container/mqtt.py | 4 ++-- mango/container/tcp.py | 4 ++-- mango/express/api.py | 8 ++++---- mango/messages/codecs.py | 8 ++++++-- mango/messages/mango_message_pb2.py | 2 +- mango/messages/message.py | 7 ++++--- mango/util/multiprocessing.py | 2 +- 13 files changed, 32 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index 2743745..260e5e3 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/mango/agent/core.py b/mango/agent/core.py index e610ba2..cb39cc1 100644 --- a/mango/agent/core.py +++ b/mango/agent/core.py @@ -4,14 +4,14 @@ Every agent must live in a container. Containers are responsible for making connections to other agents. """ -from dataclasses import dataclass import asyncio import logging from abc import ABC +from dataclasses import dataclass from typing import Any -from ..util.scheduling import ScheduledProcessTask, ScheduledTask, Scheduler from ..util.clock import Clock +from ..util.scheduling import ScheduledProcessTask, ScheduledTask, Scheduler logger = logging.getLogger(__name__) @@ -70,12 +70,10 @@ def __init__(self) -> None: def on_start(self): """Called when container started in which the agent is contained """ - pass def on_ready(self): """Called when all container has been started using activate(...). """ - pass @property def current_timestamp(self) -> float: @@ -381,7 +379,6 @@ def on_register(self): """ Hook-in to define behavior of the agent directly after it got registered by a container """ - pass def _do_register(self, container, aid): self._check_inbox_task = asyncio.create_task(self._check_inbox()) diff --git a/mango/agent/role.py b/mango/agent/role.py index e05ab44..5e78047 100644 --- a/mango/agent/role.py +++ b/mango/agent/role.py @@ -36,8 +36,7 @@ from abc import ABC from typing import Any, Callable -from mango.agent.core import Agent, AgentContext, AgentDelegates, AgentAddress -from mango.util.scheduling import Scheduler +from mango.agent.core import Agent, AgentAddress, AgentDelegates class DataContainer: @@ -504,12 +503,10 @@ async def on_stop(self) -> None: def on_start(self) -> None: """Called when container started in which the agent is contained """ - pass def on_ready(self): """Called after the start of all container using activate """ - pass def handle_message(self, content: Any, meta: dict): pass diff --git a/mango/container/core.py b/mango/container/core.py index 1318d0a..f24df4d 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -4,10 +4,14 @@ from abc import ABC, abstractmethod from typing import Any, TypeVar +from ..agent.core import Agent, AgentAddress from ..messages.codecs import Codec from ..util.clock import Clock -from .mp import MirrorContainerProcessManager, MainContainerProcessManager, cancel_and_wait_for_task -from ..agent.core import Agent, AgentAddress +from .mp import ( + MainContainerProcessManager, + MirrorContainerProcessManager, + cancel_and_wait_for_task, +) logger = logging.getLogger(__name__) diff --git a/mango/container/external_coupling.py b/mango/container/external_coupling.py index eeee64f..317e044 100644 --- a/mango/container/external_coupling.py +++ b/mango/container/external_coupling.py @@ -3,9 +3,9 @@ import time from dataclasses import dataclass +from mango.agent.core import AgentAddress from mango.container.core import Container from mango.container.mp import ContainerMirrorData -from mango.agent.core import AgentAddress from ..messages.codecs import Codec from ..messages.message import MangoMessage diff --git a/mango/container/mp.py b/mango/container/mp.py index 691e928..30d8bd0 100644 --- a/mango/container/mp.py +++ b/mango/container/mp.py @@ -1,7 +1,7 @@ import asyncio -import os import logging +import os from dataclasses import dataclass from multiprocessing import Event, Process from multiprocessing.synchronize import Event as MultiprocessingEvent diff --git a/mango/container/mqtt.py b/mango/container/mqtt.py index 9cb477c..ab990f3 100644 --- a/mango/container/mqtt.py +++ b/mango/container/mqtt.py @@ -5,10 +5,10 @@ import paho.mqtt.client as paho +from mango.container.core import AgentAddress, Container from mango.container.mp import ContainerMirrorData -from mango.container.core import Container, AgentAddress -from ..messages.codecs import ACLMessage, Codec +from ..messages.codecs import Codec from ..messages.message import MangoMessage from ..util.clock import Clock diff --git a/mango/container/tcp.py b/mango/container/tcp.py index 525bfca..9d84cc9 100644 --- a/mango/container/tcp.py +++ b/mango/container/tcp.py @@ -8,11 +8,11 @@ import time from typing import Any -from mango.container.core import Container, AgentAddress +from mango.container.core import AgentAddress, Container from mango.container.mp import ContainerMirrorData -from ..messages.message import MangoMessage from ..messages.codecs import Codec +from ..messages.message import MangoMessage from ..util.clock import Clock from .protocol import ContainerProtocol diff --git a/mango/express/api.py b/mango/express/api.py index 72ddddd..732f7f0 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -2,10 +2,10 @@ from abc import ABC, abstractmethod from typing import Any -from ..agent.role import Role, RoleAgent from ..agent.core import Agent -from ..container.core import Container, AgentAddress -from ..container.factory import create_tcp, create_mqtt +from ..agent.role import Role, RoleAgent +from ..container.core import AgentAddress, Container +from ..container.factory import create_mqtt, create_tcp from ..messages.codecs import Codec logger = logging.getLogger(__name__) @@ -198,7 +198,7 @@ def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAg class PrintingAgent(Agent): def handle_message(self, content, meta: dict[str, Any]): - logging.info(f"Received: {content} with {meta}") + logging.info("Received: %s with %s", content, meta) def sender_addr(meta: dict) -> AgentAddress: """Extract the sender_addr from the meta dict. diff --git a/mango/messages/codecs.py b/mango/messages/codecs.py index 1fa9a35..afebdf6 100644 --- a/mango/messages/codecs.py +++ b/mango/messages/codecs.py @@ -16,11 +16,15 @@ import msgspec -from mango.messages.message import ACLMessage, Performatives, enum_serializer, MangoMessage +from mango.messages.message import ( + ACLMessage, + MangoMessage, + Performatives, + enum_serializer, +) from ..messages.acl_message_pb2 import ACLMessage as ACLProto from ..messages.other_proto_msgs_pb2 import GenericMsg as GenericProtoMsg -from ..messages.mango_message_pb2 import MangoMessage as MMProto def json_serializable(cls=None, repr=True): diff --git a/mango/messages/mango_message_pb2.py b/mango/messages/mango_message_pb2.py index 38bb275..f3e0095 100644 --- a/mango/messages/mango_message_pb2.py +++ b/mango/messages/mango_message_pb2.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: mango_message.proto @@ -9,6 +8,7 @@ from google.protobuf import runtime_version as _runtime_version from google.protobuf import symbol_database as _symbol_database from google.protobuf.internal import builder as _builder + _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, 5, diff --git a/mango/messages/message.py b/mango/messages/message.py index 6a3f7c0..4bbf72e 100644 --- a/mango/messages/message.py +++ b/mango/messages/message.py @@ -6,16 +6,17 @@ It also includes the enum classes for the message Performative and Type """ -from dataclasses import dataclass -from abc import ABC, abstractmethod import pickle +import warnings +from abc import ABC, abstractmethod +from dataclasses import dataclass from enum import Enum from typing import Any -import warnings from ..messages.acl_message_pb2 import ACLMessage as ACLProto from ..messages.mango_message_pb2 import MangoMessage as MangoMsg + class Message(ABC): @abstractmethod def split_content_and_meta(self): diff --git a/mango/util/multiprocessing.py b/mango/util/multiprocessing.py index 0d04fab..4ccab97 100644 --- a/mango/util/multiprocessing.py +++ b/mango/util/multiprocessing.py @@ -29,7 +29,7 @@ from contextlib import asynccontextmanager, contextmanager from multiprocessing.connection import Connection from multiprocessing.reduction import ForkingPickler -from typing import Any, AsyncContextManager, ContextManager +from typing import Any import dill From fa215b136ab49cf5261606c3a666fde970afca29 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:35:03 +0200 Subject: [PATCH 04/15] Ruff formatting. --- mango/__init__.py | 18 ++- mango/agent/core.py | 30 +++-- mango/agent/role.py | 25 ++-- mango/container/core.py | 7 +- mango/container/external_coupling.py | 2 +- mango/container/factory.py | 25 ++-- mango/container/mp.py | 2 +- mango/container/mqtt.py | 42 ++++--- mango/container/tcp.py | 10 +- mango/express/api.py | 108 ++++++++++++------ mango/messages/codecs.py | 1 - mango/messages/mango_message_pb2.py | 22 ++-- mango/messages/message.py | 14 ++- mango/util/distributed_clock.py | 16 +-- tests/integration_tests/__init__.py | 11 +- .../test_distributed_clock.py | 18 +-- .../test_message_roundtrip.py | 27 ++--- .../test_message_roundtrip_mp.py | 6 +- .../test_single_container_termination.py | 88 ++++++++------ tests/unit_tests/container/test_mp.py | 29 ++--- tests/unit_tests/container/test_tcp.py | 2 +- tests/unit_tests/core/test_agent.py | 12 +- tests/unit_tests/core/test_container.py | 29 ++--- .../test_external_scheduling_container.py | 37 +++--- tests/unit_tests/express/test_api.py | 64 +++++------ tests/unit_tests/messages/test_codecs.py | 2 +- tests/unit_tests/role_agent_test.py | 17 ++- tests/unit_tests/test_agents.py | 15 +-- 28 files changed, 362 insertions(+), 317 deletions(-) diff --git a/mango/__init__.py b/mango/__init__.py index 1e893d6..c9d99e6 100644 --- a/mango/__init__.py +++ b/mango/__init__.py @@ -1,7 +1,19 @@ from .messages.message import create_acl from .agent.core import Agent, AgentAddress from .agent.role import Role, RoleAgent, RoleContext -from .container.factory import create_tcp as create_tcp_container, create_mqtt as create_mqtt_container, create_external_coupling as create_ec_container -from .express.api import activate, run_with_mqtt, run_with_tcp, agent_composed_of, PrintingAgent, sender_addr, addr +from .container.factory import ( + create_tcp as create_tcp_container, + create_mqtt as create_mqtt_container, + create_external_coupling as create_ec_container, +) +from .express.api import ( + activate, + run_with_mqtt, + run_with_tcp, + agent_composed_of, + PrintingAgent, + sender_addr, + addr, +) from .util.distributed_clock import DistributedClockAgent, DistributedClockManager -from .util.clock import ExternalClock \ No newline at end of file +from .util.clock import ExternalClock diff --git a/mango/agent/core.py b/mango/agent/core.py index cb39cc1..36159df 100644 --- a/mango/agent/core.py +++ b/mango/agent/core.py @@ -4,6 +4,7 @@ Every agent must live in a container. Containers are responsible for making connections to other agents. """ + import asyncio import logging from abc import ABC @@ -15,11 +16,13 @@ logger = logging.getLogger(__name__) + @dataclass(frozen=True) class AgentAddress: addr: Any aid: str + class AgentContext: def __init__(self, container) -> None: self._container = container @@ -64,16 +67,14 @@ async def send_message( class AgentDelegates: def __init__(self) -> None: self.context: AgentContext = None - self.scheduler: Scheduler = None - self._aid = None + self.scheduler: Scheduler = None + self._aid = None def on_start(self): - """Called when container started in which the agent is contained - """ + """Called when container started in which the agent is contained""" def on_ready(self): - """Called when all container has been started using activate(...). - """ + """Called when all container has been started using activate(...).""" @property def current_timestamp(self) -> float: @@ -108,7 +109,6 @@ async def send_message( content, receiver_addr=receiver_addr, sender_id=self.aid, **kwargs ) - def schedule_instant_message( self, content, @@ -126,9 +126,7 @@ def schedule_instant_message( """ return self.schedule_instant_task( - self.send_message( - content, receiver_addr=receiver_addr, **kwargs - ) + self.send_message(content, receiver_addr=receiver_addr, **kwargs) ) def schedule_conditional_process_task( @@ -359,15 +357,15 @@ def __init__( self.inbox = asyncio.Queue() - @property + @property def observable_tasks(self): return self.scheduler.observable @observable_tasks.setter def observable_tasks(self, value: bool): self.scheduler.observable = value - - @property + + @property def suspendable_tasks(self): return self.scheduler.suspendable @@ -387,9 +385,7 @@ def _do_register(self, container, aid): self._aid = aid self.context = AgentContext(container) self.scheduler = Scheduler( - suspendable=True, - observable=True, - clock=container.clock + suspendable=True, observable=True, clock=container.clock ) self.on_register() @@ -461,4 +457,4 @@ async def shutdown(self): except asyncio.CancelledError: pass finally: - logger.info("Agent %s: Shutdown successful", self.aid) \ No newline at end of file + logger.info("Agent %s: Shutdown successful", self.aid) diff --git a/mango/agent/role.py b/mango/agent/role.py index 5e78047..a8ff2a9 100644 --- a/mango/agent/role.py +++ b/mango/agent/role.py @@ -232,15 +232,16 @@ def subscribe_event(self, role: Role, event_type: type, method: Callable): self._role_event_type_to_handler[event_type] = [] self._role_event_type_to_handler[event_type] += [(role, method)] - + def on_start(self): for role in self.roles: role.on_start() - + def on_ready(self): for role in self.roles: role.on_ready() + class RoleContext(AgentDelegates): """Implementation of the RoleContext.""" @@ -340,7 +341,6 @@ async def send_message( **kwargs, ) - def emit_event(self, event: Any, event_source: Any = None): """Emit an custom event to other roles. @@ -375,7 +375,7 @@ def deactivate(self, role) -> None: def activate(self, role) -> None: self._role_handler.activate(role) - + def on_start(self): self._role_handler.on_start() @@ -388,9 +388,7 @@ class RoleAgent(Agent): a RoleAgent as base for your agents. A role can be added with :func:`RoleAgent.add_role`. """ - def __init__( - self - ): + def __init__(self): """Create a role-agent :param container: container the agent lives in @@ -399,10 +397,8 @@ def __init__( """ super().__init__() self._role_handler = RoleHandler(None, None) - self._role_context = RoleContext( - self._role_handler, self.aid, self.inbox - ) - + self._role_context = RoleContext(self._role_handler, self.aid, self.inbox) + def on_start(self): self._role_context.on_start() @@ -447,7 +443,6 @@ async def shutdown(self): await super().shutdown() - class Role(ABC): """General role class, defining the API every role can use. A role implements one responsibility of an agent. @@ -501,12 +496,10 @@ async def on_stop(self) -> None: """Lifecycle hook in, which will be called when the container is shut down or if the role got removed.""" def on_start(self) -> None: - """Called when container started in which the agent is contained - """ + """Called when container started in which the agent is contained""" def on_ready(self): - """Called after the start of all container using activate - """ + """Called after the start of all container using activate""" def handle_message(self, content: Any, meta: dict): pass diff --git a/mango/container/core.py b/mango/container/core.py index f24df4d..2de6992 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -19,6 +19,7 @@ A = TypeVar("A") + class Container(ABC): """Superclass for a mango container""" @@ -128,7 +129,7 @@ def register_agent(self, agent: Agent, suggested_aid: str = None): return aid def include(self, agent: A, suggested_aid: str = None) -> A: - """Include the agent in the container. Return the agent for + """Include the agent in the container. Return the agent for convenience. Args: @@ -158,14 +159,14 @@ async def send_message( content: Any, receiver_addr: AgentAddress, sender_id: None | str = None, - **kwargs + **kwargs, ) -> bool: """ The Container sends a message to an agent according the container protocol. :param content: The content of the message :param receiver_addr: The address the message is sent to, should be constructed using - agent_address(protocol_addr, aid) or address(agent) on sending messages, + agent_address(protocol_addr, aid) or address(agent) on sending messages, and sender_address(meta) on replying to messages. :param kwargs: Can contain additional meta information """ diff --git a/mango/container/external_coupling.py b/mango/container/external_coupling.py index 317e044..0981c10 100644 --- a/mango/container/external_coupling.py +++ b/mango/container/external_coupling.py @@ -102,7 +102,7 @@ async def send_message( meta = {} for key, value in kwargs.items(): - meta[key] = value + meta[key] = value meta["sender_id"] = sender_id meta["sender_addr"] = self.addr meta["receiver_id"] = receiver_addr.aid diff --git a/mango/container/factory.py b/mango/container/factory.py index 38a0a6a..a1d352a 100644 --- a/mango/container/factory.py +++ b/mango/container/factory.py @@ -17,7 +17,9 @@ MQTT_CONNECTION = "mqtt" EXTERNAL_CONNECTION = "external_connection" -def create_mqtt(broker_addr: tuple | dict | str, + +def create_mqtt( + broker_addr: tuple | dict | str, client_id: str, codec: Codec = None, clock: Clock = None, @@ -29,7 +31,7 @@ def create_mqtt(broker_addr: tuple | dict | str, codec = JSON() if clock is None: clock = AsyncioClock() - + return MQTTContainer( client_id=client_id, broker_addr=broker_addr, @@ -40,18 +42,19 @@ def create_mqtt(broker_addr: tuple | dict | str, copy_internal_messages=copy_internal_messages, **kwargs, ) - + + def create_external_coupling( - codec: Codec = None, - clock: Clock = None, - addr: None | str | tuple[str, int] = None, - **kwargs: dict[str, Any], + codec: Codec = None, + clock: Clock = None, + addr: None | str | tuple[str, int] = None, + **kwargs: dict[str, Any], ): if codec is None: codec = JSON() if clock is None: clock = ExternalClock() - + return ExternalSchedulingContainer( addr=addr, loop=asyncio.get_running_loop(), codec=codec, clock=clock, **kwargs ) @@ -69,8 +72,8 @@ def create_tcp( :param codec: Defines the codec to use. Defaults to JSON :param clock: The clock that the scheduler of the agent should be based on. Defaults to the AsyncioClock - :param addr: the address to use. it has to be a tuple of (host, port). - + :param addr: the address to use. it has to be a tuple of (host, port). + :return: The instance of a TCPContainer """ if codec is None: @@ -79,7 +82,7 @@ def create_tcp( clock = AsyncioClock() if isinstance(addr, str): addr = tuple(addr.split(":")) - + # initialize TCPContainer return TCPContainer( addr=addr, diff --git a/mango/container/mp.py b/mango/container/mp.py index 30d8bd0..51861bf 100644 --- a/mango/container/mp.py +++ b/mango/container/mp.py @@ -1,4 +1,3 @@ - import asyncio import logging import os @@ -16,6 +15,7 @@ WAIT_STEP = 0.01 + class IPCEventType(enumerate): """Available IPC event types for event process container communication""" diff --git a/mango/container/mqtt.py b/mango/container/mqtt.py index ab990f3..1b87da5 100644 --- a/mango/container/mqtt.py +++ b/mango/container/mqtt.py @@ -74,7 +74,12 @@ def __init__( allowed """ super().__init__( - codec=codec, addr=broker_addr, loop=loop, clock=clock, name=client_id, **kwargs + codec=codec, + addr=broker_addr, + loop=loop, + clock=clock, + name=client_id, + **kwargs, ) self.client_id: str = client_id @@ -106,7 +111,9 @@ async def start(self): # check if addr is a valid topic without wildcards if self.inbox_topic is not None and ( - not isinstance(self.inbox_topic, str) or "#" in self.inbox_topic or "+" in self.inbox_topic + not isinstance(self.inbox_topic, str) + or "#" in self.inbox_topic + or "+" in self.inbox_topic ): raise ValueError( "inbox topic is not set correctly. It must be a string without any wildcards ('#' or '+')!" @@ -156,7 +163,9 @@ def on_con(client, userdata, flags, reason_code, properties): raise ValueError("Invalid broker address") mqtt_messenger.connect(self.addr, **self._kwargs) - logger.info("[%s]: Going to connect to broker at %s..", self.client_id, self.addr) + logger.info( + "[%s]: Going to connect to broker at %s..", self.client_id, self.addr + ) counter = 0 # process MQTT messages for maximum of 10 seconds to @@ -184,7 +193,9 @@ def on_con(client, userdata, flags, reason_code, properties): if self.inbox_topic is not None: # connection has been set up, subscribe to inbox topic now logger.info( - "[%s]: Going to subscribe to %s as inbox topic..", self.client_id, self.inbox_topic + "[%s]: Going to subscribe to %s as inbox topic..", + self.client_id, + self.inbox_topic, ) # create Future that is triggered on successful subscription @@ -222,15 +233,14 @@ def on_sub(client, userdata, mid, reason_code_list, properties): # connection and subscription is successful, remove callbacks mqtt_messenger.on_subscribe = None mqtt_messenger.on_connect = None - + self.mqtt_client = mqtt_messenger # set the callbacks self._set_mqtt_callbacks() # start the mqtt client self.mqtt_client.loop_start() - - await super().start() + await super().start() def _set_mqtt_callbacks(self): """ @@ -337,7 +347,7 @@ async def send_message( content: Any, receiver_addr: AgentAddress, sender_id: None | str = None, - **kwargs + **kwargs, ): """ The container sends the message of one of its own agents to a specific topic. @@ -356,7 +366,7 @@ async def send_message( # the broker meta = {} for key, value in kwargs.items(): - meta[key] = value + meta[key] = value meta["sender_id"] = sender_id meta["sender_addr"] = self.inbox_topic meta["receiver_id"] = receiver_addr.aid @@ -367,12 +377,14 @@ async def send_message( and receiver_addr == self.inbox_topic and not actual_mqtt_kwargs.get("retain", False) ): - meta.update({ - "topic": self.inbox_topic, - "qos": actual_mqtt_kwargs.get("qos", 0), - "retain": False, - "network_protocol": "mqtt", - }) + meta.update( + { + "topic": self.inbox_topic, + "qos": actual_mqtt_kwargs.get("qos", 0), + "retain": False, + "network_protocol": "mqtt", + } + ) return self._send_internal_message( content, receiver_addr.aid, default_meta=meta, inbox=self.inbox ) diff --git a/mango/container/tcp.py b/mango/container/tcp.py index 9d84cc9..9b188f9 100644 --- a/mango/container/tcp.py +++ b/mango/container/tcp.py @@ -207,7 +207,7 @@ async def send_message( content: Any, receiver_addr: AgentAddress, sender_id: None | str = None, - **kwargs + **kwargs, ) -> bool: """ The Container sends a message to an agent using TCP. @@ -228,11 +228,11 @@ async def send_message( meta = {} for key, value in kwargs.items(): - meta[key] = value + meta[key] = value meta["sender_id"] = sender_id meta["sender_addr"] = self.addr meta["receiver_id"] = receiver_addr.aid - + if protocol_addr == self.addr: # internal message meta["network_protocol"] = "tcp" @@ -244,7 +244,9 @@ async def send_message( # if the user does not provide a splittable content, we create the default one if not hasattr(content, "split_content_and_meta"): message = MangoMessage(content, meta) - success = await self._send_external_message(receiver_addr.addr, message, meta) + success = await self._send_external_message( + receiver_addr.addr, message, meta + ) return success diff --git a/mango/express/api.py b/mango/express/api.py index 732f7f0..fd9941a 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -10,8 +10,8 @@ logger = logging.getLogger(__name__) -class ContainerActivationManager: +class ContainerActivationManager: def __init__(self, containers: list[Container]) -> None: self._containers = containers @@ -28,6 +28,7 @@ async def __aexit__(self, exc_type, exc, tb): for container in self._containers: await container.shutdown() + class RunWithContainer(ABC): def __init__(self, num: int, *agents: tuple[Agent, dict]) -> None: self._num = num @@ -46,12 +47,14 @@ async def __aenter__(self): if self._num > len(self._agents): actual_number_container = len(self._agents) container_list = self.create_container_list(actual_number_container) - for (i, agent_tuple) in enumerate(self._agents): + for i, agent_tuple in enumerate(self._agents): container_id = i % actual_number_container container = container_list[container_id] actual_agent = agent_tuple[0] agent_params = agent_tuple[1] - container.register_agent(actual_agent, suggested_aid=agent_params.get("aid", None)) + container.register_agent( + actual_agent, suggested_aid=agent_params.get("aid", None) + ) self.__activation_cm = activate(container_list) await self.__activation_cm.__aenter__() await self.after_start(container_list, self._agents) @@ -60,43 +63,60 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc, tb): await self.__activation_cm.__aexit__(exc_type, exc, tb) -class RunWithTCPManager(RunWithContainer): - def __init__(self, - num: int, - *agents: Agent | tuple[Agent, dict], - addr: tuple[str, int] = ("127.0.0.1", 5555), - codec: None | Codec = None) -> None: - agents = [agent if isinstance(agent, tuple) else (agent, dict()) for agent in agents[0]] +class RunWithTCPManager(RunWithContainer): + def __init__( + self, + num: int, + *agents: Agent | tuple[Agent, dict], + addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None, + ) -> None: + agents = [ + agent if isinstance(agent, tuple) else (agent, dict()) + for agent in agents[0] + ] super().__init__(num, *agents) self._addr = addr self._codec = codec def create_container_list(self, num): - return [create_tcp((self._addr[0], self._addr[1]+i), codec=self._codec) for i in range(num)] + return [ + create_tcp((self._addr[0], self._addr[1] + i), codec=self._codec) + for i in range(num) + ] -class RunWithMQTTManager(RunWithContainer): - def __init__(self, - num: int, - *agents: Agent | tuple[Agent, dict], - broker_addr: tuple[str, int] = ("127.0.0.1", 5555), - codec: None | Codec = None) -> None: - agents = [agent if isinstance(agent, tuple) else (agent, dict()) for agent in agents[0]] +class RunWithMQTTManager(RunWithContainer): + def __init__( + self, + num: int, + *agents: Agent | tuple[Agent, dict], + broker_addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None, + ) -> None: + agents = [ + agent if isinstance(agent, tuple) else (agent, dict()) + for agent in agents[0] + ] super().__init__(num, *agents) self._broker_addr = broker_addr self._codec = codec def create_container_list(self, num): - return [create_mqtt((self._broker_addr[0], self._broker_addr[1]+i), - client_id=f"client{i}", - codec=self._codec) - for i in range(num)] + return [ + create_mqtt( + (self._broker_addr[0], self._broker_addr[1] + i), + client_id=f"client{i}", + codec=self._codec, + ) + for i in range(num) + ] async def after_start(self, container_list, agents): - for (i, agent_tuple) in enumerate(agents): + for i, agent_tuple in enumerate(agents): container_id = i % len(container_list) container = container_list[container_id] actual_agent = agent_tuple[0] @@ -110,14 +130,14 @@ def activate(*containers: Container) -> ContainerActivationManager: """Create and return an async activation context manager. This can be used with the `async with` syntax to run code while the container(s) are active. The containers are started first, after your code under `async with` will run, and at the end - the container will shut down (even when an error occurs). + the container will shut down (even when an error occurs). Example: ```python # Single container async with activate(container) as container: # do your stuff - + # Multiple container async with activate(container_list) as container_list: # do your stuff @@ -133,10 +153,13 @@ def activate(*containers: Container) -> ContainerActivationManager: containers = containers[0] return ContainerActivationManager(list(containers)) -def run_with_tcp(num: int, - *agents: Agent | tuple[Agent, dict], - addr: tuple[str, int] = ("127.0.0.1", 5555), - codec: None | Codec = None) -> RunWithTCPManager: + +def run_with_tcp( + num: int, + *agents: Agent | tuple[Agent, dict], + addr: tuple[str, int] = ("127.0.0.1", 5555), + codec: None | Codec = None, +) -> RunWithTCPManager: """Create and return an async context manager, which can be used to run the given agents in `num` automatically created tcp container. The agents are distributed evenly. @@ -156,10 +179,13 @@ def run_with_tcp(num: int, """ return RunWithTCPManager(num, agents, addr=addr, codec=codec) -def run_with_mqtt(num: int, - *agents: tuple[Agent, dict], - broker_addr: tuple[str, int] = ("127.0.0.1", 1883), - codec: None | Codec = None) -> RunWithMQTTManager: + +def run_with_mqtt( + num: int, + *agents: tuple[Agent, dict], + broker_addr: tuple[str, int] = ("127.0.0.1", 1883), + codec: None | Codec = None, +) -> RunWithMQTTManager: """Create and return an async context manager, which can be used to run the given agents in `num` automatically created mqtt container. The agents are distributed according to the topic @@ -177,16 +203,18 @@ def run_with_mqtt(num: int, """ return RunWithMQTTManager(num, agents, broker_addr=broker_addr, codec=codec) + class ComposedAgent(RoleAgent): pass + def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAgent: """Create an agent composed of the given `roles`. If a container is provided, the created agent is automatically registered with the container `register_in`. Args: *roles Role: The roles which are added to the agent - register_in (None | Container): container in which the created agent is registered, + register_in (None | Container): container in which the created agent is registered, if provided """ agent = ComposedAgent() @@ -196,10 +224,12 @@ def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAg register_in.register_agent(agent) return agent + class PrintingAgent(Agent): def handle_message(self, content, meta: dict[str, Any]): logging.info("Received: %s with %s", content, meta) + def sender_addr(meta: dict) -> AgentAddress: """Extract the sender_addr from the meta dict. @@ -210,7 +240,13 @@ def sender_addr(meta: dict) -> AgentAddress: AgentAddress: Extracted agent address to be used for replying to messages """ # convert decoded addr list to tuple for hashability - return AgentAddress(tuple(meta["sender_addr"]) if isinstance(meta["sender_addr"], list) else meta["sender_addr"], meta["sender_id"]) + return AgentAddress( + tuple(meta["sender_addr"]) + if isinstance(meta["sender_addr"], list) + else meta["sender_addr"], + meta["sender_id"], + ) + def addr(protocol_part: Any, aid: str) -> AgentAddress: """Create an Address from the topic. @@ -222,4 +258,4 @@ def addr(protocol_part: Any, aid: str) -> AgentAddress: Returns: AgentAddress: the address """ - return AgentAddress(protocol_part, aid) \ No newline at end of file + return AgentAddress(protocol_part, aid) diff --git a/mango/messages/codecs.py b/mango/messages/codecs.py index afebdf6..cadc00c 100644 --- a/mango/messages/codecs.py +++ b/mango/messages/codecs.py @@ -312,4 +312,3 @@ def _proto_to_acl(self, data): acl.content = self.deserialize_obj(obj_repr) return acl - diff --git a/mango/messages/mango_message_pb2.py b/mango/messages/mango_message_pb2.py index f3e0095..ae618b3 100644 --- a/mango/messages/mango_message_pb2.py +++ b/mango/messages/mango_message_pb2.py @@ -3,6 +3,7 @@ # source: mango_message.proto # Protobuf Python Version: 5.27.2 """Generated protocol buffer code.""" + from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import runtime_version as _runtime_version @@ -10,27 +11,22 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( - _runtime_version.Domain.PUBLIC, - 5, - 27, - 2, - '', - 'mango_message.proto' + _runtime_version.Domain.PUBLIC, 5, 27, 2, "", "mango_message.proto" ) # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13mango_message.proto\"-\n\x0cMangoMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04meta\x18\x02 \x01(\x0c\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( + b'\n\x13mango_message.proto"-\n\x0cMangoMessage\x12\x0f\n\x07\x63ontent\x18\x01 \x01(\x0c\x12\x0c\n\x04meta\x18\x02 \x01(\x0c\x62\x06proto3' +) _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'mango_message_pb2', _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "mango_message_pb2", _globals) if not _descriptor._USE_C_DESCRIPTORS: - DESCRIPTOR._loaded_options = None - _globals['_MANGOMESSAGE']._serialized_start=23 - _globals['_MANGOMESSAGE']._serialized_end=68 + DESCRIPTOR._loaded_options = None + _globals["_MANGOMESSAGE"]._serialized_start = 23 + _globals["_MANGOMESSAGE"]._serialized_end = 68 # @@protoc_insertion_point(module_scope) diff --git a/mango/messages/message.py b/mango/messages/message.py index 4bbf72e..6dbfec6 100644 --- a/mango/messages/message.py +++ b/mango/messages/message.py @@ -6,6 +6,7 @@ It also includes the enum classes for the message Performative and Type """ + import pickle import warnings from abc import ABC, abstractmethod @@ -21,14 +22,15 @@ class Message(ABC): @abstractmethod def split_content_and_meta(self): pass - + def __asdict__(self): return vars(self) + @dataclass class MangoMessage(Message): content: Any = None - meta: dict[str,Any] = None + meta: dict[str, Any] = None def split_content_and_meta(self): return self.content, self.meta @@ -57,16 +59,17 @@ def __fromproto__(cls, data): mango_message = cls() - mango_message.content = pickle.loads(bytes(msg.content)) if msg.content else None + mango_message.content = ( + pickle.loads(bytes(msg.content)) if msg.content else None + ) mango_message.meta = pickle.loads(bytes(msg.meta)) if msg.meta else None return mango_message - + @classmethod def __protoserializer__(cls): return cls, cls.__toproto__, cls.__fromproto__ - class ACLMessage(Message): """ @@ -239,6 +242,7 @@ class Performatives(Enum): proxy = 21 propagate = 22 + def create_acl( content, receiver_addr: str | tuple[str, int], diff --git a/mango/util/distributed_clock.py b/mango/util/distributed_clock.py index fe1f409..b5dd345 100644 --- a/mango/util/distributed_clock.py +++ b/mango/util/distributed_clock.py @@ -16,11 +16,11 @@ async def wait_all_done(self): class DistributedClockManager(ClockAgent): def __init__(self, receiver_clock_addresses: list): super().__init__() - + self.receiver_clock_addresses = receiver_clock_addresses self.schedules = [] self.futures = {} - + def on_ready(self): self.schedule_instant_task(self.wait_all_online()) @@ -50,10 +50,7 @@ async def broadcast(self, message, add_futures=True): logger.debug("clockmanager send: %s - %s", message, receiver_addr) # in MQTT we can not be sure if the message was delivered # checking the return code here would only help for TCP - await self.send_message( - message, - receiver_addr - ) + await self.send_message(message, receiver_addr) if add_futures: self.futures[receiver_addr] = asyncio.Future() @@ -175,11 +172,8 @@ def respond(fut: asyncio.Future = None): return next_time = self.scheduler.clock.get_next_activity() - - self.schedule_instant_message( - next_time, - sender_addr(meta) - ) + + self.schedule_instant_message(next_time, sender_addr(meta)) t.add_done_callback(respond) else: diff --git a/tests/integration_tests/__init__.py b/tests/integration_tests/__init__.py index 6624f6b..10559b8 100644 --- a/tests/integration_tests/__init__.py +++ b/tests/integration_tests/__init__.py @@ -1,6 +1,7 @@ from mango import create_tcp_container, create_mqtt_container from mango.util.clock import ExternalClock + def create_test_container(type, init_addr, repl_addr, codec): broker = ("localhost", 1883, 60) @@ -15,10 +16,10 @@ def create_test_container(type, init_addr, repl_addr, codec): container_man = create_mqtt_container( broker_addr=broker, client_id="container_1", - clock=clock_man, + clock=clock_man, codec=codec, inbox_topic=init_addr, - transport="tcp" + transport="tcp", ) clock_ag = ExternalClock() if type == "tcp": @@ -31,9 +32,9 @@ def create_test_container(type, init_addr, repl_addr, codec): container_ag = create_mqtt_container( broker_addr=broker, client_id="container_2", - clock=clock_ag, + clock=clock_ag, codec=codec, inbox_topic=repl_addr, - transport="tcp" + transport="tcp", ) - return container_man, container_ag \ No newline at end of file + return container_man, container_ag diff --git a/tests/integration_tests/test_distributed_clock.py b/tests/integration_tests/test_distributed_clock.py index fd1b06a..5fb6de2 100644 --- a/tests/integration_tests/test_distributed_clock.py +++ b/tests/integration_tests/test_distributed_clock.py @@ -1,5 +1,3 @@ -import asyncio - import pytest from mango import activate, addr @@ -14,13 +12,18 @@ async def setup_and_run_test_case(connection_type, codec): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - - container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + + container_man, container_ag = create_test_container( + connection_type, init_addr, repl_addr, codec + ) clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager(receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] - )) - + clock_manager = container_man.include( + DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + ) + ) + async with activate(container_man, container_ag) as cl: # increasing the time container_man.clock.set_time(100) @@ -41,7 +44,6 @@ async def setup_and_run_test_case(connection_type, codec): assert container_ag.clock.time == 2000 - @pytest.mark.asyncio async def test_tcp_json(): await setup_and_run_test_case("tcp", JSON_CODEC) diff --git a/tests/integration_tests/test_message_roundtrip.py b/tests/integration_tests/test_message_roundtrip.py index b31a6fc..82477da 100644 --- a/tests/integration_tests/test_message_roundtrip.py +++ b/tests/integration_tests/test_message_roundtrip.py @@ -2,12 +2,12 @@ import pytest -from mango import create_mqtt_container, create_tcp_container, activate, addr +from mango import activate, addr from mango.agent.core import Agent from mango.messages.codecs import JSON, PROTOBUF, FastJSON -from . import create_test_container from ..unit_tests.messages.msg_pb2 import MyMsg +from . import create_test_container M1 = "Hello" M2 = "Hello2" @@ -41,7 +41,9 @@ async def setup_and_run_test_case(connection_type, codec): init_addr = ("localhost", 1555) if connection_type == "tcp" else None repl_addr = ("localhost", 1556) if connection_type == "tcp" else None - container_1, container_2 = create_test_container(connection_type, init_addr, repl_addr, codec) + container_1, container_2 = create_test_container( + connection_type, init_addr, repl_addr, codec + ) if connection_type == "mqtt": init_target = repl_target = comm_topic @@ -57,6 +59,7 @@ async def setup_and_run_test_case(connection_type, codec): async with activate(container_1, container_2) as cl: await asyncio.gather(repl_agent.start(), init_agent.start()) + # InitiatorAgent: # - send "Hello" # - awaits reply @@ -75,24 +78,20 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): - await self.container.subscribe_for_agent(aid=self.aid, topic=self.target.addr) + await self.container.subscribe_for_agent( + aid=self.aid, topic=self.target.addr + ) await asyncio.sleep(0.1) # send initial message - await self.send_message( - M1, - self.target - ) + await self.send_message(M1, self.target) # await reply await self.got_reply # answer to reply - await self.send_message( - M3, - self.target - ) + await self.send_message(M3, self.target) # ReplierAgent: @@ -119,7 +118,9 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): - await self.container.subscribe_for_agent(aid=self.aid, topic=self.target.addr) + await self.container.subscribe_for_agent( + aid=self.aid, topic=self.target.addr + ) # await "Hello" await self.got_first diff --git a/tests/integration_tests/test_message_roundtrip_mp.py b/tests/integration_tests/test_message_roundtrip_mp.py index c1e820a..5b7596b 100644 --- a/tests/integration_tests/test_message_roundtrip_mp.py +++ b/tests/integration_tests/test_message_roundtrip_mp.py @@ -2,8 +2,8 @@ import pytest +from mango import AgentAddress, activate, create_tcp_container, sender_addr from mango.agent.core import Agent -from mango import AgentAddress, sender_addr, create_tcp_container, activate class PingPongAgent(Agent): @@ -46,12 +46,12 @@ async def test_mp_simple_ping_pong_multi_container_tcp(): async with activate(container_1, container_2) as cl: await agent.send_message( "Message To Process Agent1", - receiver_addr=AgentAddress(container_1.addr,aid1) + receiver_addr=AgentAddress(container_1.addr, aid1), ) await agent.send_message( "Message To Process Agent2", - receiver_addr=AgentAddress(container_2.addr,aid2) + receiver_addr=AgentAddress(container_2.addr, aid2), ) while agent.test_counter != 2: diff --git a/tests/integration_tests/test_single_container_termination.py b/tests/integration_tests/test_single_container_termination.py index bc5c723..a25a230 100644 --- a/tests/integration_tests/test_single_container_termination.py +++ b/tests/integration_tests/test_single_container_termination.py @@ -2,12 +2,14 @@ import pytest -from mango import Agent, create_ec_container, sender_addr, activate, addr +from mango import Agent, activate, addr, create_ec_container, sender_addr from mango.util.clock import ExternalClock from mango.util.distributed_clock import DistributedClockAgent, DistributedClockManager from mango.util.termination_detection import tasks_complete_or_sleeping + from . import create_test_container + class Caller(Agent): def __init__( self, @@ -31,9 +33,7 @@ def on_ready(self): ) async def send_hello_world(self, receiver_addr): - await self.send_message( - receiver_addr=receiver_addr, content="Hello World" - ) + await self.send_message(receiver_addr=receiver_addr, content="Hello World") async def send_ordered(self, meta): await self.send_message( @@ -70,7 +70,7 @@ async def test_termination_single_container(): receiver = c.include(Receiver()) caller = c.include(Caller(receiver.addr, send_response_messages=True)) - async with activate(c) as c: + async with activate(c) as c: await asyncio.sleep(0.1) clock.set_time(clock.time + 5) # wait until each agent is done with all tasks at some point @@ -88,23 +88,29 @@ async def test_termination_single_container(): assert caller.i == caller.max_count + async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + container_man, container_ag = create_test_container( + connection_type, init_addr, repl_addr, codec + ) clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager( - receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] - )) + clock_manager = container_man.include( + DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + ) + ) receiver = container_ag.include(Receiver()) - caller = container_man.include(Caller( - addr(repl_addr, receiver.aid), - send_response_messages=True, - max_count=max_count, - )) - + caller = container_man.include( + Caller( + addr(repl_addr, receiver.aid), + send_response_messages=True, + max_count=max_count, + ) + ) async with activate(container_man, container_ag) as c: container_man.clock.set_time(container_man.clock.time + 5) @@ -112,7 +118,7 @@ async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): # we do not have distributed termination detection yet in core assert caller.i < caller.max_count await caller.done - + assert caller.i == caller.max_count @@ -122,19 +128,25 @@ async def distribute_ping_pong_test_timestamp( init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + container_man, container_ag = create_test_container( + connection_type, init_addr, repl_addr, codec + ) clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager( - receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] - )) + clock_manager = container_man.include( + DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + ) + ) receiver = container_ag.include(Receiver()) - caller = container_man.include(Caller( - addr(repl_addr, receiver.aid), - send_response_messages=True, - max_count=max_count, - schedule_timestamp=True, - )) + caller = container_man.include( + Caller( + addr(repl_addr, receiver.aid), + send_response_messages=True, + max_count=max_count, + schedule_timestamp=True, + ) + ) # we do not have distributed termination detection yet in core async with activate(container_man, container_ag) as cl: @@ -182,12 +194,16 @@ async def distribute_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + container_man, container_ag = create_test_container( + connection_type, init_addr, repl_addr, codec + ) clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager( - receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] - )) + clock_manager = container_man.include( + DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + ) + ) receiver = container_ag.include(Receiver()) caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) @@ -227,12 +243,16 @@ async def send_current_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" - container_man, container_ag = create_test_container(connection_type, init_addr, repl_addr, codec) + container_man, container_ag = create_test_container( + connection_type, init_addr, repl_addr, codec + ) clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager( - receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] - )) + clock_manager = container_man.include( + DistributedClockManager( + receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] + ) + ) receiver = container_ag.include(Receiver()) caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) diff --git a/tests/unit_tests/container/test_mp.py b/tests/unit_tests/container/test_mp.py index 33fcd18..742be68 100644 --- a/tests/unit_tests/container/test_mp.py +++ b/tests/unit_tests/container/test_mp.py @@ -2,7 +2,7 @@ import pytest -from mango import Agent, create_tcp_container, sender_addr, AgentAddress, activate, addr +from mango import Agent, AgentAddress, activate, addr, create_tcp_container, sender_addr class MyAgent(Agent): @@ -16,8 +16,7 @@ def handle_message(self, content, meta): if self.test_counter == 1: # send back pong, providing your own details self.current_task = self.schedule_instant_message( - content="pong", - receiver_addr=sender_addr(meta) + content="pong", receiver_addr=sender_addr(meta) ) @@ -39,8 +38,7 @@ def __init__(self, receiver_id): def handle_message(self, content, meta): # send back pong, providing your own details self.current_task = self.schedule_instant_message( - content="pong", - receiver_addr=addr(meta["sender_addr"], self.receiver_id) + content="pong", receiver_addr=addr(meta["sender_addr"], self.receiver_id) ) @@ -79,13 +77,14 @@ async def test_agent_processes_ping_pong(num_sp_agents, num_sp): for j in range(num_sp_agents): await agent.send_message( "Message To Process Agent", - receiver_addr=addr(c.addr, f"process_agent{i},{j}") + receiver_addr=addr(c.addr, f"process_agent{i},{j}"), ) while agent.test_counter != num_sp_agents * num_sp: await asyncio.sleep(0.1) assert agent.test_counter == num_sp_agents * num_sp + @pytest.mark.asyncio async def test_agent_processes_ping_pong_p_to_p(): # GIVEN @@ -93,8 +92,9 @@ async def test_agent_processes_ping_pong_p_to_p(): aid_main_agent = "main_agent" c = create_tcp_container(addr=addr, copy_internal_messages=False) await c.as_agent_process( - agent_creator=lambda container: container.include(P2PTestAgent(aid_main_agent), - suggested_aid="process_agent1") + agent_creator=lambda container: container.include( + P2PTestAgent(aid_main_agent), suggested_aid="process_agent1" + ) ) main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) @@ -103,7 +103,7 @@ def agent_init(c): agent = c.include(MyAgent(), suggested_aid="process_agent2") agent.schedule_instant_message( "Message To Process Agent", - receiver_addr=AgentAddress(addr, "process_agent1") + receiver_addr=AgentAddress(addr, "process_agent1"), ) return agent @@ -125,11 +125,10 @@ async def test_async_agent_processes_ping_pong_p_to_p(): main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) async def agent_creator(container): - p2pta = container.include(P2PTestAgent(aid_main_agent), suggested_aid="process_agent1") - await p2pta.send_message( - content="pong", - receiver_addr=main_agent.addr + p2pta = container.include( + P2PTestAgent(aid_main_agent), suggested_aid="process_agent1" ) + await p2pta.send_message(content="pong", receiver_addr=main_agent.addr) async with activate(c) as c: await c.as_agent_process(agent_creator=agent_creator) @@ -137,7 +136,9 @@ async def agent_creator(container): # WHEN def agent_init(c): agent = c.include(MyAgent(), suggested_aid="process_agent2") - agent.schedule_instant_message("Message To Process Agent", AgentAddress(addr, "process_agent1")) + agent.schedule_instant_message( + "Message To Process Agent", AgentAddress(addr, "process_agent1") + ) return agent await c.as_agent_process(agent_creator=agent_init) diff --git a/tests/unit_tests/container/test_tcp.py b/tests/unit_tests/container/test_tcp.py index 7ff2ec4..7e21a17 100644 --- a/tests/unit_tests/container/test_tcp.py +++ b/tests/unit_tests/container/test_tcp.py @@ -47,7 +47,7 @@ async def test_connection_pool_double_obtain_release(): c2 = create_tcp_container(addr=("127.0.0.2", 5556), copy_internal_messages=False) await c.start() await c2.start() - + addr = "127.0.0.2", 5556 connection_pool = TCPConnectionPool(asyncio.get_event_loop()) raw_prot = ContainerProtocol( diff --git a/tests/unit_tests/core/test_agent.py b/tests/unit_tests/core/test_agent.py index d0b47f5..4c82a63 100644 --- a/tests/unit_tests/core/test_agent.py +++ b/tests/unit_tests/core/test_agent.py @@ -3,7 +3,7 @@ import pytest -from mango import create_tcp_container, create_acl, activate +from mango import activate, create_acl, create_tcp_container from mango.agent.core import Agent @@ -61,7 +61,10 @@ async def test_send_acl_message(): agent2 = c.include(MyAgent()) async with activate(c) as c: - await agent.send_message(create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), receiver_addr=agent2.addr) + await agent.send_message( + create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), + receiver_addr=agent2.addr, + ) msg = await agent2.inbox.get() _, content, meta = msg agent2.handle_message(content=content, meta=meta) @@ -79,7 +82,7 @@ async def test_schedule_message(): async with activate(c) as c: await agent.schedule_instant_message("", receiver_addr=agent2.addr) - + # THEN assert agent2.test_counter == 1 @@ -93,7 +96,8 @@ async def test_schedule_acl_message(): async with activate(c) as c: await agent.schedule_instant_message( - create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), receiver_addr=agent2.addr + create_acl("", receiver_addr=agent2.addr, sender_addr=c.addr), + receiver_addr=agent2.addr, ) # THEN diff --git a/tests/unit_tests/core/test_container.py b/tests/unit_tests/core/test_container.py index 4601138..96c0c03 100644 --- a/tests/unit_tests/core/test_container.py +++ b/tests/unit_tests/core/test_container.py @@ -1,7 +1,8 @@ import pytest +from mango import create_acl, create_tcp_container from mango.agent.core import Agent -from mango import create_tcp_container, create_acl + class LooksLikeAgent: async def shutdown(self): @@ -150,8 +151,11 @@ async def test_create_acl_no_modify(): c = create_tcp_container(addr=("127.0.0.2", 5555)) common_acl_q = {} actual_acl_message = create_acl( - "", receiver_addr="", receiver_id="", acl_metadata=common_acl_q, - sender_addr=c.addr + "", + receiver_addr="", + receiver_id="", + acl_metadata=common_acl_q, + sender_addr=c.addr, ) assert "reeiver_addr" not in common_acl_q @@ -165,8 +169,7 @@ async def test_create_acl_no_modify(): async def test_create_acl_anon(): c = create_tcp_container(addr=("127.0.0.2", 5555)) actual_acl_message = create_acl( - "", receiver_addr="", receiver_id="", is_anonymous_acl=True, - sender_addr=c.addr + "", receiver_addr="", receiver_id="", is_anonymous_acl=True, sender_addr=c.addr ) assert actual_acl_message.sender_addr is None @@ -195,15 +198,11 @@ class Data: @pytest.mark.asyncio async def test_send_message_no_copy(): - c = create_tcp_container( - addr=("127.0.0.2", 5555), copy_internal_messages=False - ) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) agent1 = c.include(ExampleAgent()) message_to_send = Data() - await c.send_message( - message_to_send, receiver_addr=agent1.addr - ) + await c.send_message(message_to_send, receiver_addr=agent1.addr) await c.shutdown() assert agent1.content is message_to_send @@ -211,15 +210,11 @@ async def test_send_message_no_copy(): @pytest.mark.asyncio async def test_send_message_copy(): - c = create_tcp_container( - addr=("127.0.0.2", 5555), copy_internal_messages=True - ) + c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=True) agent1 = c.include(ExampleAgent()) message_to_send = Data() - await c.send_message( - message_to_send, receiver_addr=agent1.addr - ) + await c.send_message(message_to_send, receiver_addr=agent1.addr) await c.shutdown() assert agent1.content is not message_to_send diff --git a/tests/unit_tests/core/test_external_scheduling_container.py b/tests/unit_tests/core/test_external_scheduling_container.py index 3d5e8de..b816aeb 100644 --- a/tests/unit_tests/core/test_external_scheduling_container.py +++ b/tests/unit_tests/core/test_external_scheduling_container.py @@ -3,19 +3,16 @@ import pytest +from mango import AgentAddress, create_acl, create_ec_container, sender_addr from mango.agent.core import Agent from mango.container.external_coupling import ExternalAgentMessage -from mango.container.factory import EXTERNAL_CONNECTION from mango.messages.message import ACLMessage from mango.util.clock import ExternalClock -from mango import AgentAddress, create_ec_container, AgentAddress, sender_addr, create_acl @pytest.mark.asyncio async def test_init(): - external_scheduling_container = create_ec_container( - addr="external_eid_1234" - ) + external_scheduling_container = create_ec_container(addr="external_eid_1234") assert external_scheduling_container.addr == "external_eid_1234" assert isinstance(external_scheduling_container.clock, ExternalClock) await external_scheduling_container.shutdown() @@ -74,7 +71,7 @@ def on_register(self): async def send_ping(self): await self.send_message( content=f"ping{self.current_ping}", - receiver_addr=AgentAddress("ping_receiver_addr","ping_receiver_id") + receiver_addr=AgentAddress("ping_receiver_addr", "ping_receiver_id"), ) self.current_ping += 1 @@ -83,13 +80,11 @@ def handle_message(self, content, meta: Dict[str, Any]): async def sleep_and_answer(self, content, meta): await self.send_message( - content=f"I received {content}", - receiver_addr=sender_addr(meta) + content=f"I received {content}", receiver_addr=sender_addr(meta) ) await asyncio.sleep(0.1) await self.send_message( - content=f"Thanks for sending {content}", - receiver_addr=sender_addr(meta) + content=f"Thanks for sending {content}", receiver_addr=sender_addr(meta) ) async def stop_tasks(self): @@ -123,9 +118,7 @@ def handle_message(self, content, meta: Dict[str, Any]): @pytest.mark.asyncio async def test_step_with_cond_task(): - external_scheduling_container = create_ec_container( - addr="external_eid_1" - ) + external_scheduling_container = create_ec_container(addr="external_eid_1") agent_1 = external_scheduling_container.include(WaitForMessageAgent()) print("Agent init") @@ -153,7 +146,7 @@ async def test_step_with_cond_task(): content="", receiver_addr=external_scheduling_container.addr, receiver_id=agent_1.aid, - sender_addr=external_scheduling_container.addr + sender_addr=external_scheduling_container.addr, ) encoded_msg = external_scheduling_container.codec.encode(message) print("created message") @@ -196,24 +189,22 @@ def handle_message(self, content, meta: Dict[str, Any]): i += 1 # send message to yourself if necessary if self.no_received_msg < self.final_no: + self.schedule_instant_message(receiver_addr=self.addr, content=content) + else: self.schedule_instant_message( - receiver_addr=self.addr, content=content + content, AgentAddress("AnyOtherAddr", "AnyOtherId") ) - else: - self.schedule_instant_message(content, AgentAddress("AnyOtherAddr", "AnyOtherId")) @pytest.mark.asyncio async def test_send_internal_messages(): - external_scheduling_container = create_ec_container( - addr="external_eid_1" - ) + external_scheduling_container = create_ec_container(addr="external_eid_1") agent_1 = external_scheduling_container.include(SelfSendAgent(final_number=3)) message = create_acl( content="", receiver_addr=external_scheduling_container.addr, receiver_id=agent_1.aid, - sender_addr=external_scheduling_container.addr + sender_addr=external_scheduling_container.addr, ) encoded_msg = external_scheduling_container.codec.encode(message) return_values = await external_scheduling_container.step( @@ -226,9 +217,7 @@ async def test_send_internal_messages(): @pytest.mark.asyncio async def test_step_with_replying_agent(): - external_scheduling_container = create_ec_container( - addr="external_eid_1" - ) + external_scheduling_container = create_ec_container(addr="external_eid_1") reply_agent = external_scheduling_container.include(ReplyAgent()) new_acl_msg = ACLMessage() new_acl_msg.content = "hello you" diff --git a/tests/unit_tests/express/test_api.py b/tests/unit_tests/express/test_api.py index dc9453c..11195e6 100644 --- a/tests/unit_tests/express/test_api.py +++ b/tests/unit_tests/express/test_api.py @@ -1,8 +1,20 @@ +import asyncio from typing import Any + import pytest -import asyncio -from mango import agent_composed_of, activate, create_tcp_container, Role, sender_addr, Agent, run_with_tcp, run_with_mqtt, addr +from mango import ( + Agent, + Role, + activate, + addr, + agent_composed_of, + create_tcp_container, + run_with_mqtt, + run_with_tcp, + sender_addr, +) + class PingPongRole(Role): counter: int = 0 @@ -10,14 +22,13 @@ class PingPongRole(Role): def handle_message(self, content: Any, meta: dict): if self.counter >= 5: return - + self.counter += 1 if content == "Ping": self.context.schedule_instant_message("Pong", sender_addr(meta)) elif content == "Pong": self.context.schedule_instant_message("Ping", sender_addr(meta)) - @pytest.mark.asyncio @@ -27,7 +38,9 @@ async def test_activate_pingpong(): ping_pong_agent_two = agent_composed_of(PingPongRole(), register_in=container) async with activate(container) as c: - await c.send_message("Ping", ping_pong_agent.addr, sender_id=ping_pong_agent_two.aid) + await c.send_message( + "Ping", ping_pong_agent.addr, sender_id=ping_pong_agent_two.aid + ) while ping_pong_agent.roles[0].counter < 5: await asyncio.sleep(0.01) @@ -50,9 +63,7 @@ async def test_activate_api_style_agent(): # WHEN async with activate(c) as c: - await agent.schedule_instant_message( - "", receiver_addr=agent2.addr - ) + await agent.schedule_instant_message("", receiver_addr=agent2.addr) # THEN assert agent2.test_counter == 1 @@ -66,25 +77,7 @@ async def test_run_api_style_agent(): # WHEN async with run_with_tcp(1, run_agent, run_agent2) as c: - await run_agent.schedule_instant_message( - "", receiver_addr=run_agent2.addr - ) - - # THEN - assert run_agent2.test_counter == 1 - - -@pytest.mark.asyncio -async def test_run_api_style_agent(): - # GIVEN - run_agent = MyAgent() - run_agent2 = MyAgent() - - # WHEN - async with run_with_tcp(1, run_agent, run_agent2) as c: - await run_agent.schedule_instant_message( - "", receiver_addr=run_agent2.addr - ) + await run_agent.schedule_instant_message("", receiver_addr=run_agent2.addr) # THEN assert run_agent2.test_counter == 1 @@ -98,15 +91,14 @@ async def test_run_api_style_agent_with_aid(): # WHEN async with run_with_tcp(1, run_agent, (run_agent2, dict(aid="my_custom_aid"))) as c: - await run_agent.schedule_instant_message( - "", receiver_addr=run_agent2.addr - ) + await run_agent.schedule_instant_message("", receiver_addr=run_agent2.addr) # THEN assert run_agent2.test_counter == 1 assert run_agent2.aid == "my_custom_aid" assert run_agent.aid == "agent0" + @pytest.mark.asyncio async def test_run_api_style_agent_with_aid_mqtt(): # GIVEN @@ -114,12 +106,12 @@ async def test_run_api_style_agent_with_aid_mqtt(): run_agent2 = MyAgent() # WHEN - async with run_with_mqtt(1, - (run_agent, dict(topics=["my_top"])), - (run_agent2, dict(topics=["your_top"], aid="my_custom_aid"))) as c: - await run_agent.schedule_instant_message( - "", addr("your_top", run_agent2.aid) - ) + async with run_with_mqtt( + 1, + (run_agent, dict(topics=["my_top"])), + (run_agent2, dict(topics=["your_top"], aid="my_custom_aid")), + ) as c: + await run_agent.schedule_instant_message("", addr("your_top", run_agent2.aid)) while run_agent2.test_counter == 0: await asyncio.sleep(0.01) diff --git a/tests/unit_tests/messages/test_codecs.py b/tests/unit_tests/messages/test_codecs.py index 51ef449..6c91887 100644 --- a/tests/unit_tests/messages/test_codecs.py +++ b/tests/unit_tests/messages/test_codecs.py @@ -10,7 +10,7 @@ SerializationError, json_serializable, ) -from mango.messages.message import ACLMessage, Performatives, MangoMessage +from mango.messages.message import ACLMessage, MangoMessage, Performatives from .msg_pb2 import MyMsg diff --git a/tests/unit_tests/role_agent_test.py b/tests/unit_tests/role_agent_test.py index b896f8b..cd70c5d 100644 --- a/tests/unit_tests/role_agent_test.py +++ b/tests/unit_tests/role_agent_test.py @@ -4,14 +4,16 @@ import pytest +from mango import activate, create_tcp_container, sender_addr from mango.agent.role import Role, RoleAgent, RoleContext from mango.util.scheduling import TimestampScheduledTask -from mango import sender_addr, create_tcp_container, activate class SimpleReactiveRole(Role): def setup(self): - self.context.subscribe_message(self, self.react_handle_message, self.is_applicable) + self.context.subscribe_message( + self, self.react_handle_message, self.is_applicable + ) def react_handle_message(self, content, meta: Dict[str, Any]) -> None: pass @@ -30,8 +32,7 @@ def react_handle_message(self, content, meta: Dict[str, Any]): # send back pong, providing your own details t = self.context.schedule_instant_message( - content="pong", - receiver_addr=sender_addr(meta) + content="pong", receiver_addr=sender_addr(meta) ) self.sending_tasks.append(t) @@ -51,9 +52,7 @@ def react_handle_message(self, content, meta: Dict[str, Any]): sender = sender_addr(meta) assert sender in self.open_ping_requests.keys() - self.open_ping_requests[sender].set_result( - True - ) + self.open_ping_requests[sender].set_result(True) def is_applicable(self, content, meta): return content == "pong" @@ -70,9 +69,7 @@ def on_ready(self): ): self.context.schedule_task(task) - async def send_ping_to_other( - self, other_addr, agent_context: RoleContext - ): + async def send_ping_to_other(self, other_addr, agent_context: RoleContext): # create self.open_ping_requests[other_addr] = asyncio.Future() success = await agent_context.send_message( diff --git a/tests/unit_tests/test_agents.py b/tests/unit_tests/test_agents.py index 1972153..5e70912 100644 --- a/tests/unit_tests/test_agents.py +++ b/tests/unit_tests/test_agents.py @@ -3,7 +3,8 @@ import pytest -from mango import sender_addr, create_tcp_container, activate, Agent +from mango import Agent, activate, create_tcp_container, sender_addr + class PingPongAgent(Agent): """ @@ -22,8 +23,7 @@ def handle_message(self, content, meta: dict[str, Any]): # send back pong, providing your own details t = self.schedule_instant_message( - content="pong", - receiver_addr=sender_addr(meta) + content="pong", receiver_addr=sender_addr(meta) ) self.sending_tasks.append(t) @@ -32,17 +32,12 @@ def handle_message(self, content, meta: dict[str, Any]): # get host, port and id from sender assert sender_addr(meta) in self.open_ping_requests.keys() - self.open_ping_requests[sender_addr(meta)].set_result( - True - ) + self.open_ping_requests[sender_addr(meta)].set_result(True) async def send_ping_to_other(self, other_addr): # create self.open_ping_requests[other_addr] = asyncio.Future() - success = await self.send_message( - content="ping", - receiver_addr=other_addr - ) + success = await self.send_message(content="ping", receiver_addr=other_addr) assert success async def wait_for_sending_messages(self, timeout=1): From b6043aca9419fb32fbe467ac015fe85d2fda6ce4 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:38:42 +0200 Subject: [PATCH 05/15] Up to 3.10 --- .github/workflows/test-mango.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index 260e5e3..eab6c15 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -18,7 +18,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/setup.py b/setup.py index e135455..8101744 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ URL = "https://github.com/OFFIS-DAI/mango" EMAIL = "mango@offis.de" AUTHOR = "mango Team" -REQUIRES_PYTHON = ">=3.9.0" +REQUIRES_PYTHON = ">=3.10.0" VERSION = "2.0.0" # What packages are required for this module to be executed? From 5b951f9960ade50a6bea12b1e85c4c81ca4286fd Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:43:17 +0200 Subject: [PATCH 06/15] Fixing test. --- tests/unit_tests/role_agent_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/role_agent_test.py b/tests/unit_tests/role_agent_test.py index cd70c5d..147845b 100644 --- a/tests/unit_tests/role_agent_test.py +++ b/tests/unit_tests/role_agent_test.py @@ -171,7 +171,7 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): a = c.include(RoleAgent()) a.add_role(PongRole()) agents.append(a) - addrs.append((c.addr, a.aid)) + addrs.append(a.addr) # add Ping Role and deactivate it immediately for a in agents: From e21d30c80d7cf2bf230b07063882055de1f09c9c Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Sun, 13 Oct 2024 16:59:51 +0200 Subject: [PATCH 07/15] Formatting. --- mango/express/api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mango/express/api.py b/mango/express/api.py index fd9941a..f60632d 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -1,3 +1,4 @@ +import asyncio import logging from abc import ABC, abstractmethod from typing import Any @@ -16,8 +17,7 @@ def __init__(self, containers: list[Container]) -> None: self._containers = containers async def __aenter__(self): - for container in self._containers: - await container.start() + await asyncio.gather(*[c.start() for c in self._containers]) for container in self._containers: container.on_ready() if len(self._containers) == 1: @@ -25,8 +25,7 @@ async def __aenter__(self): return self._containers async def __aexit__(self, exc_type, exc, tb): - for container in self._containers: - await container.shutdown() + await asyncio.gather(*[c.shutdown() for c in self._containers]) class RunWithContainer(ABC): From 945f23c7458823226c95098c343170ff6658ae75 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 17:51:36 +0200 Subject: [PATCH 08/15] Review and getting started guide reworked, doctests are integrated in the CI now. --- .github/workflows/test-mango.yml | 4 + docs/source/ACL messages.rst | 0 docs/source/_static/Logo_mango_ohne_sub.svg | 1 + .../_static/Logo_mango_ohne_sub_white.svg | 1 + docs/source/api_ref/index.rst | 21 ++- docs/source/api_ref/mango.agent.rst | 6 +- docs/source/api_ref/mango.container.rst | 20 +-- docs/source/api_ref/mango.express.rst | 10 ++ docs/source/api_ref/mango.messages.rst | 28 +--- docs/source/api_ref/mango.modules.rst | 34 ----- docs/source/api_ref/mango.util.rst | 18 +-- docs/source/conf.py | 2 +- docs/source/customizing-container.rst | 9 -- docs/source/getting_started.rst | 141 +++++++++++------- docs/source/impressum.rst | 46 ------ docs/source/index.rst | 9 +- docs/source/installation.rst | 2 +- docs/source/legals.rst | 47 ++++++ docs/source/scheduling.rst | 11 +- images/mango_basics.jpg | Bin 61566 -> 0 bytes mango/agent/core.py | 14 +- mango/agent/role.py | 4 +- mango/container/core.py | 42 ++++-- mango/container/external_coupling.py | 9 +- mango/container/factory.py | 7 +- mango/container/mqtt.py | 24 ++- mango/container/protocol.py | 4 +- mango/container/tcp.py | 25 ++-- mango/express/api.py | 104 +++++++------ readme.md | 18 ++- .../test_distributed_clock.py | 4 +- .../test_message_roundtrip.py | 4 +- .../test_message_roundtrip_mp.py | 6 +- .../test_single_container_termination.py | 36 ++--- tests/unit_tests/container/test_mp.py | 16 +- tests/unit_tests/core/test_agent.py | 18 +-- tests/unit_tests/core/test_container.py | 20 +-- .../test_external_scheduling_container.py | 6 +- tests/unit_tests/express/test_api.py | 4 +- tests/unit_tests/role_agent_test.py | 8 +- tests/unit_tests/test_agents.py | 4 +- 41 files changed, 403 insertions(+), 384 deletions(-) delete mode 100644 docs/source/ACL messages.rst create mode 100644 docs/source/_static/Logo_mango_ohne_sub.svg create mode 100644 docs/source/_static/Logo_mango_ohne_sub_white.svg create mode 100644 docs/source/api_ref/mango.express.rst delete mode 100644 docs/source/api_ref/mango.modules.rst delete mode 100644 docs/source/customizing-container.rst delete mode 100644 docs/source/impressum.rst delete mode 100644 images/mango_basics.jpg diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index eab6c15..aee10ef 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -44,6 +44,10 @@ jobs: source venv/bin/activate ruff check . ruff format --check . + - name: Doctests + run: | + source venv/bin/activate + docs/make doctest - name: Test+Coverage run: | source venv/bin/activate diff --git a/docs/source/ACL messages.rst b/docs/source/ACL messages.rst deleted file mode 100644 index e69de29..0000000 diff --git a/docs/source/_static/Logo_mango_ohne_sub.svg b/docs/source/_static/Logo_mango_ohne_sub.svg new file mode 100644 index 0000000..6a7f4aa --- /dev/null +++ b/docs/source/_static/Logo_mango_ohne_sub.svg @@ -0,0 +1 @@ + diff --git a/docs/source/_static/Logo_mango_ohne_sub_white.svg b/docs/source/_static/Logo_mango_ohne_sub_white.svg new file mode 100644 index 0000000..915b9c4 --- /dev/null +++ b/docs/source/_static/Logo_mango_ohne_sub_white.svg @@ -0,0 +1 @@ + diff --git a/docs/source/api_ref/index.rst b/docs/source/api_ref/index.rst index 43c8f97..2d77faa 100644 --- a/docs/source/api_ref/index.rst +++ b/docs/source/api_ref/index.rst @@ -2,17 +2,30 @@ API reference ============= -The API reference provides detailed descriptions of mango's classes and +The API reference provides detailed descriptions of the mango's classes and functions. -Subpackages ------------ +.. automodule:: mango + :members: + :undoc-members: + :imported-members: + :inherited-members: + + +.. note:: + Note that, most classes and functions described in the API reference + should be imported using `from mango import ...`, as the stable and public API + generally will be available by using `mango` and the internal module structure + might change, even in minor releases. + +By subpackages +--------------- .. toctree:: :maxdepth: 2 + mango.express mango.agent mango.container mango.messages - mango.modules mango.util diff --git a/docs/source/api_ref/mango.agent.rst b/docs/source/api_ref/mango.agent.rst index e5cff1e..3d7e4a1 100644 --- a/docs/source/api_ref/mango.agent.rst +++ b/docs/source/api_ref/mango.agent.rst @@ -1,7 +1,7 @@ -mango.agent package +Agents API =================== -mango.agent.core module +Agent core ----------------------- .. automodule:: mango.agent.core @@ -9,7 +9,7 @@ mango.agent.core module :undoc-members: :show-inheritance: -mango.agent.role module +Roles ----------------------- .. automodule:: mango.agent.role diff --git a/docs/source/api_ref/mango.container.rst b/docs/source/api_ref/mango.container.rst index c2d40f2..8c882c3 100644 --- a/docs/source/api_ref/mango.container.rst +++ b/docs/source/api_ref/mango.container.rst @@ -1,7 +1,7 @@ -mango.container package +Container API ======================= -mango.container.core module +Container core --------------------------- .. automodule:: mango.container.core @@ -9,7 +9,7 @@ mango.container.core module :undoc-members: :show-inheritance: -mango.container.external\_coupling module +Container external coupling ----------------------------------------- .. automodule:: mango.container.external_coupling @@ -17,7 +17,7 @@ mango.container.external\_coupling module :undoc-members: :show-inheritance: -mango.container.factory module +Container creation ------------------------------ .. automodule:: mango.container.factory @@ -25,7 +25,7 @@ mango.container.factory module :undoc-members: :show-inheritance: -mango.container.mqtt module +MQTT Container --------------------------- .. automodule:: mango.container.mqtt @@ -33,15 +33,7 @@ mango.container.mqtt module :undoc-members: :show-inheritance: -mango.container.protocol module -------------------------------- - -.. automodule:: mango.container.protocol - :members: - :undoc-members: - :show-inheritance: - -mango.container.tcp module +TCP Container -------------------------- .. automodule:: mango.container.tcp diff --git a/docs/source/api_ref/mango.express.rst b/docs/source/api_ref/mango.express.rst new file mode 100644 index 0000000..b8f6d5e --- /dev/null +++ b/docs/source/api_ref/mango.express.rst @@ -0,0 +1,10 @@ +Express API +=================== + +API +------------------------------ + +.. automodule:: mango.express.api + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/api_ref/mango.messages.rst b/docs/source/api_ref/mango.messages.rst index ae14fc8..6c6d0f5 100644 --- a/docs/source/api_ref/mango.messages.rst +++ b/docs/source/api_ref/mango.messages.rst @@ -1,34 +1,10 @@ -mango.messages package +Codecs ====================== -mango.messages.acl\_message\_pb2 module ---------------------------------------- - -.. automodule:: mango.messages.acl_message_pb2 - :members: - :undoc-members: - :show-inheritance: - -mango.messages.codecs module +Codec implementations ---------------------------- .. automodule:: mango.messages.codecs :members: :undoc-members: :show-inheritance: - -mango.messages.message module ------------------------------ - -.. automodule:: mango.messages.message - :members: - :undoc-members: - :show-inheritance: - -mango.messages.other\_proto\_msgs\_pb2 module ---------------------------------------------- - -.. automodule:: mango.messages.other_proto_msgs_pb2 - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api_ref/mango.modules.rst b/docs/source/api_ref/mango.modules.rst deleted file mode 100644 index 7f887f3..0000000 --- a/docs/source/api_ref/mango.modules.rst +++ /dev/null @@ -1,34 +0,0 @@ -mango.modules package -===================== - -mango.modules.base\_module module ---------------------------------- - -.. automodule:: mango.modules.base_module - :members: - :undoc-members: - :show-inheritance: - -mango.modules.mqtt\_module module ---------------------------------- - -.. automodule:: mango.modules.mqtt_module - :members: - :undoc-members: - :show-inheritance: - -mango.modules.rabbit\_module module ------------------------------------ - -.. automodule:: mango.modules.rabbit_module - :members: - :undoc-members: - :show-inheritance: - -mango.modules.zero\_module module ---------------------------------- - -.. automodule:: mango.modules.zero_module - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/api_ref/mango.util.rst b/docs/source/api_ref/mango.util.rst index c81ccc9..9352083 100644 --- a/docs/source/api_ref/mango.util.rst +++ b/docs/source/api_ref/mango.util.rst @@ -1,7 +1,7 @@ -mango.util package +Utilities ================== -mango.util.clock module +Clock ----------------------- .. automodule:: mango.util.clock @@ -9,7 +9,7 @@ mango.util.clock module :undoc-members: :show-inheritance: -mango.util.distributed\_clock module +Distributed clock ------------------------------------ .. automodule:: mango.util.distributed_clock @@ -17,15 +17,7 @@ mango.util.distributed\_clock module :undoc-members: :show-inheritance: -mango.util.multiprocessing module ---------------------------------- - -.. automodule:: mango.util.multiprocessing - :members: - :undoc-members: - :show-inheritance: - -mango.util.scheduling module +Scheduling ---------------------------- .. automodule:: mango.util.scheduling @@ -33,7 +25,7 @@ mango.util.scheduling module :undoc-members: :show-inheritance: -mango.util.termination\_detection module +Termination detection ---------------------------------------- .. automodule:: mango.util.termination_detection diff --git a/docs/source/conf.py b/docs/source/conf.py index b2b9e26..c8e222a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,7 @@ author = "mango team" # The full version, including alpha/beta/rc tags -version = release = "1.1.4" +version = release = "2.0.0" # -- General configuration --------------------------------------------------- diff --git a/docs/source/customizing-container.rst b/docs/source/customizing-container.rst deleted file mode 100644 index 515dad4..0000000 --- a/docs/source/customizing-container.rst +++ /dev/null @@ -1,9 +0,0 @@ -============================= -Customizing a mango container -============================= - -A mango container can be customized regarding its way it connects to other containers. - -*************** -connection type -*************** diff --git a/docs/source/getting_started.rst b/docs/source/getting_started.rst index 563824e..f4f7ddd 100644 --- a/docs/source/getting_started.rst +++ b/docs/source/getting_started.rst @@ -1,4 +1,4 @@ -=============== + Getting started =============== In this section you will get to know the necessary steps to create a simple multi-agent system @@ -11,85 +11,109 @@ Creating an agent In our first example, we create a very simple agent that simply prints the content of all messages it receives: -.. code-block:: python3 +.. testcode:: from mango import Agent - class RepeatingAgent(Agent): - def __init__(self, container): - # We must pass a reference of the container to "mango.Agent": - super().__init__(container) - print(f"Hello world! My id is {self.aid}.") + + def __init__(self): + super().__init__() + print(f"Creating a RepeatingAgent. At this point self.addr={self.addr}") def handle_message(self, content, meta): # This method defines what the agent will do with incoming messages. - print(f"Received a message with the following content: {content}") + print(f"Received a message with the following content: {content}!") + + def on_register(self): + print(f"The agent has been registered to a container: {self.addr}!") + + def on_ready(self): + print("All containers have been activated!") -Agents must be a subclass of :class:`Agent`. This base class needs -a reference to the container that the agents live in, so you must forward -a *container* argument to it if you override ``__init__()``. + RepeatingAgent() + +.. testoutput:: + + Creating a RepeatingAgent. At this point self.addr=None + +Agents must be a subclass of :class:`mango.Agent`. Agent's are notified when they are registered :meth:`mango.Agent.on_register` +and when the container(s) has been activated :meth:`mango.Agent.on_ready`. Consequenty, most agent features (like scheduling, +sending internal messages, the agent address) are available after registration, and only after :meth:`mango.Agent.on_ready` has +been called, all features are available (sending external messages). ******************** Creating a container ******************** -Agents live in a container, so we need to know how to create a mango container. +Agents live in containers, so we need to know how to create a mango container. The container is responsible for message exchange between agents. More information about container and agents can be -found in :doc:`Agents and container` +found in :doc:`Agents and container ` + +.. testcode:: + + from mango import create_tcp_container -.. code-block:: python3 + # Containers have to be created using a factory method + # Other container types are available through create_mqtt_container and create_ec_container + container = create_tcp_container(addr=('localhost', 5555)) + print(container.addr) - from mango import create_container - # Containers need to be started via a factory function. - # This method is a coroutine so it needs to be called from a coroutine using the - # await statement - async def get_container(): - return await create_container(addr=('localhost', 5555)) +.. testoutput:: -This is how a container is created. Since the method :py:meth:`create_container()` is a -coroutine__ we need to await its result. + ('localhost', 5555) -__ https://docs.python.org/3.10/library/asyncio-task.html + +This is how a tcp container is created. While container creation, it is possible to set the codec, the address information (depending on the type) +and the clock (see :ref:`ClockDocs`). ******************************************* Running your first agent within a container ******************************************* -To put it all together we will wrap the creation of a container and the agent into a coroutine -and execute it using :py:meth:`asyncio.run()`. -The following script will create a RepeatingAgent -and let it run within a container for three seconds and -then shutdown the container: -.. code-block:: python3 +The container and the contained agents need `asyncio` (see `asyncio docs `_) to work, therefore we need write a coroutine +function and execute it using `asyncio.run`. - import asyncio - from mango import Agent - from mango import create_container +The following script will create a RepeatingAgent, register it, and let it run within a container for 50ms and then shutdown the container: +.. testcode:: + + import asyncio + from mango import create_tcp_container, Agent, activate class RepeatingAgent(Agent): - def __init__(self, container): - # We must pass a ref. to the container to "mango.Agent": - super().__init__(container) - print(f"Hello world! My id is {self.aid}.") + def __init__(self): + super().__init__() + print(f"Creating a RepeatingAgent. At this point self.addr={self.addr}") def handle_message(self, content, meta): - # This method defines what the agent will do with incoming messages. - print(f"Received a message with the following content: {content}") + print(f"Received a message with the following content: {content}!") + + def on_register(self): + print(f"The agent has been registered to a container: {self.addr}!") + + def on_ready(self): + print("All containers have been activated!") async def run_container_and_agent(addr, duration): - first_container = await create_container(addr=addr) - first_agent = RepeatingAgent(first_container) - await asyncio.sleep(duration) - await first_container.shutdown() + first_container = create_tcp_container(addr=addr) + first_agent = first_container.register(RepeatingAgent()) + + async with activate(first_container) as container: + await asyncio.sleep(duration) + + asyncio.run(run_container_and_agent(addr=('localhost', 5555), duration=0.05)) - asyncio.run(run_container_and_agent(addr=('localhost', 5555), duration=3)) +.. testoutput:: + Creating a RepeatingAgent. At this point self.addr=None + The agent has been registered to a container: AgentAddress(protocol_addr=('localhost', 5555), aid='agent0')! + All containers have been activated! -The only output you should see is "Hello world! My id is agent0.", because -the agent does not receive any other messages. +In this example no messages are sent, nor does the Agent do anything, but the call order of the hook-in functions is clearly visible. +The function :py:meth:`mango.activate` will start the container and shut it down after the +code in its scope has been execute (here, after the sleep). ************************** Creating a proactive Agent @@ -98,8 +122,9 @@ Creating a proactive Agent Let's implement another agent that is able to send a hello world message to another agent: -.. code-block:: python +.. testcode:: + import asyncio from mango import Agent class HelloWorldAgent(Agent): @@ -109,7 +134,23 @@ to another agent: def handle_message(self, content, meta): print(f"Received a message with the following content: {content}") -We are using the scheduling API, which is explained in further detail in the section :doc:`scheduling`. + async def run_container_and_agent(addr, duration): + first_container = create_tcp_container(addr=addr) + first_hello_agent = first_container.register(HelloWorldAgent()) + second_hello_agent = first_container.register(HelloWorldAgent()) + + async with activate(first_container) as container: + await first_hello_agent.greet(second_hello_agent.addr) + + asyncio.run(run_container_and_agent(addr=('localhost', 5555), duration=0.05)) + +.. testoutput:: + + Received a message with the following content: Hello world! + + +If you do not want to await sending the message, and just let asyncio/mango schedule it, you can use :meth:`mango.Agent.schedule_instant_message` instead of +:meth:`mango.Agent.send_message`. ********************* Connecting two agents @@ -144,9 +185,9 @@ a RepeatingAgent and let them run. async def run_container_and_two_agents(first_addr, second_addr): first_container = create_tcp_container(addr=first_addr) second_container = create_tcp_container(addr=second_addr) - - first_agent = first_container.include(RepeatingAgent()) - second_agent = second_container.include(HelloWorldAgent()) + + first_agent = first_container.register(RepeatingAgent()) + second_agent = second_container.register(HelloWorldAgent()) async with activate(first_container, second_container) as cl: await second_agent.greet(first_agent.addr) diff --git a/docs/source/impressum.rst b/docs/source/impressum.rst deleted file mode 100644 index 31455c9..0000000 --- a/docs/source/impressum.rst +++ /dev/null @@ -1,46 +0,0 @@ -========= -Impressum -========= - - -**Anschrift** - -| OFFIS e. V. -| Escherweg 2 -| 26121 Oldenburg -| Telefon +49 441 9722-0 -| Fax +49 441 9722-102 -| E-Mail: `institut [ A T ] offis.de `_ -| Internet: `www.offis.de `_ - - -**Vertretungsberechtigter Vorstand** - -| Prof. Dr. Sebastian Lehnhoff (Vorsitzender) -| Prof. Dr. techn. Susanne Boll-Westermann -| Prof. Dr.-Ing. Andreas Hein -| Prof. Dr.-Ing. Astrid Nieße - - -**Registergericht** - -| Amtsgericht Oldenburg -| Registernummer VR 1956 - - -**Umsatzsteuer-Identifikationsnummer (USt-IdNr.)** - -DE 811582102 - - -**Verantwortlich im Sinne der Presse** - -| Dr. Ing. Jürgen Meister (Bereichsleiter) -| OFFIS e.V. -| Escherweg 2 -| 26121 Oldenburg - - -**Datenschutz** - -Mehr zum Thema Datenschutz finden Sie :doc:`hier `. diff --git a/docs/source/index.rst b/docs/source/index.rst index 44d61d4..185b386 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -14,7 +14,7 @@ The main features of mango are listed below. Features ================================= - Container mechanism to speedup local message exchange -- Message definition based on the FIPA ACL standard +- Support for using FIPA message based on the FIPA ACL standard - Structuring complex agents with loose coupling and agent roles - Built-in codecs: `JSON `_ and `protobuf `_ - Supports communication between agents directly via TCP or via an external MQTT broker in the middle @@ -25,19 +25,18 @@ Features installation getting_started - migration tutorial agents-container message exchange + role-api scheduling codecs - role-api - development api_ref/index + migration + development privacy legals datenschutz - impressum diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 442b358..4418ddc 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -1,6 +1,6 @@ Installation ============ -*mango* requires Python >= 3.8 and runs on Linux, OSX and Windows. +*mango* requires Python >= 3.10 and runs on Linux, OSX and Windows. For installation of mango you could use virtualenv__ which can create isolated Python environments for different projects. diff --git a/docs/source/legals.rst b/docs/source/legals.rst index 8c9f36b..bc73aa1 100644 --- a/docs/source/legals.rst +++ b/docs/source/legals.rst @@ -45,3 +45,50 @@ DE 811582102 **Disclaimer** Despite careful control OFFIS assumes no liability for the content of external links. The operators of such a website are solely responsible for its content. At the time of linking the concerned sites were checked for possible violations of law. Illegal contents were not identifiable at that time. A permanent control of the linked pages is not reasonable without specific indications of a violation. Upon notification of violations, OFFIS will remove such links immediately. + +========= +Impressum +========= + + +**Anschrift** + +| OFFIS e. V. +| Escherweg 2 +| 26121 Oldenburg +| Telefon +49 441 9722-0 +| Fax +49 441 9722-102 +| E-Mail: `institut [ A T ] offis.de `_ +| Internet: `www.offis.de `_ + + +**Vertretungsberechtigter Vorstand** + +| Prof. Dr. Sebastian Lehnhoff (Vorsitzender) +| Prof. Dr. techn. Susanne Boll-Westermann +| Prof. Dr.-Ing. Andreas Hein +| Prof. Dr.-Ing. Astrid Nieße + + +**Registergericht** + +| Amtsgericht Oldenburg +| Registernummer VR 1956 + + +**Umsatzsteuer-Identifikationsnummer (USt-IdNr.)** + +DE 811582102 + + +**Verantwortlich im Sinne der Presse** + +| Dr. Ing. Jürgen Meister (Bereichsleiter) +| OFFIS e.V. +| Escherweg 2 +| 26121 Oldenburg + + +**Datenschutz** + +Mehr zum Thema Datenschutz finden Sie :doc:`hier `. diff --git a/docs/source/scheduling.rst b/docs/source/scheduling.rst index e310562..b6008ff 100644 --- a/docs/source/scheduling.rst +++ b/docs/source/scheduling.rst @@ -102,6 +102,7 @@ In mango the following process tasks are available: def handle_message(self, content, meta: Dict[str, Any]): pass +.. _ClockDocs: ******************************* Using an external clock ******************************* @@ -188,7 +189,7 @@ If you comment in the ExternalClock and change your main() as follows, the progr ******************************* Using a distributed clock ******************************* -To distribute simulations, mango provides a distributed clock, which is implemented with by two Agents: +To distribute simulations, mango provides a distributed clock, which is implemented with by two Agents: 1. DistributedClockAgent: this agent needs to be present in every participating container 2. DistributedClockManager: this agent shall exist exactly once @@ -212,8 +213,8 @@ In the following a simple example is shown. container_man = create_tcp_container(("localhost", 1555), clock=ExternalClock()) container_ag = create_tcp_container(("localhost", 1556), clock=ExternalClock()) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include(DistributedClockManager( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register(DistributedClockManager( receiver_clock_addresses=[clock_agent.addr] )) @@ -226,8 +227,8 @@ In the following a simple example is shown. # the clock_manager distributed the time to the other container assert container_ag.clock.time == 100 print("Time has been distributed!") - + asyncio.run(main()) .. testoutput:: - Time has been distributed! \ No newline at end of file + Time has been distributed! diff --git a/images/mango_basics.jpg b/images/mango_basics.jpg deleted file mode 100644 index eb77fe7762cd787aebe08bf23eaf35515fb61203..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 61566 zcmeFZ1yEewmM`A8gdjnJYZDxTI|K+GAP`&=Ah^3XfdIiB0txQY5Zoa+1PdPAoyN7H zn}6rKGxP3u|L@k!d-uMonW<@N7k&DiKE2o4OMh$a^YH6o6@afKrzi(NLP7$(Mf?FC z7638;OmuV%bTmv13=AwROl%yY$2gB3;gCHc#3Q02r=g}Ir=+9>ax&4 zKj-G-wR9ACvOXQPT1JAHN>H0|+pYQ&8DZkmv!( z1V|_ZNDn;#8UO$Z4Kdr_6aJSM5;6)Z8af6h7WN~=1$FoUWF!<6WKqlo4bdnm$y&w$B@vlPvH>>iAl*Rsh`u* zbMx{G3X6(MN~>#X>*^aCo0_}2dwTo&2L^vkOioSD%>J62Ut8bU+}hsR-P=DsJHNQR zy1u!E{-z5Ffbtiz{*AJ~p^E@P7cwd;3M$5Lx{#1P5d(z)6^)J?{fU%1#ye+1dY%uM zMAGp&Rh?K2yc#FOrY_^yBtX73#?#-V{XyA(jIf~pE6VVEZJOBp;39)!61OQ3E zHIz9w5c{7!*zojEJ)(2oa~=h&Vn9EI?%cNo2vwly+!PfXlEf-EDC^!SGL1d}Ql+3X z@dYe)??p~R4=+>W>7~y^a*GEogpj{EwmPmY;DqxF<-fasZlTt_wHm;`E(mSUE0wrA znA_Q*6-~hS=|&Tx0nQ6E(i>toyJNnLtc(=EdH_(T+xM(oPWO1<*XYjD5MSz;UD;9y zrH)?<-=DDs=GJMxZ6*dooGY|>3y@a<$zYM>t zxzT-x%U@I^^!G{7Og*Qaro{n;zS+tetV%z@W~HX)GSgk(ZkU2v;eRKl4gMLSe>8Nn68BOz2*ssTXzVLvA*afx0CO# z512SH?=mZ+ljDBjF0fpF@X6Wn1|&ewQ4gn8!u$uyFO~j!U@baxvjl<%3-s)HDfztRB!PC@Dnyq%_l)M{Ej`$WU4~T^`Q|U^(9q1iwQNKZS`Zpj zK7ddiz`r+GPL+X#z;7+`^!Pvaoun?&(`4x~Cs$+1D%u>_EA@0VqJ1t1@zjs^C25M@ zoAB!|j=Xm#9%&5~&V4)3T}6CsOmd1N!7x6t=TuWUC}wtJaF3^M=w9NoUoTE6?%+Pe zG1APO`%NS$f;k)cztKP+q?aXBWI%g~_Bk^U`}VqLelZ>1B7SZzr#hhd0Kmr0+PHky zX14yUc;55lw!+{Oqi+qU9vF-rFXT6j>5RO*_J*BIcS-}b+IYH*-n^-=`sjB)fKT)4 zQZjJ6R13zXLH@2e{@se$H$Lvc7HaLIM{b)N3=R6bzE;joXG+OA}MKvBca7+Dq{g%c%!xq7`CKa(aQaHD{E|)CR*!_DH(F=-Gp~GIu#0rm&Mph5hv4C4RluaoH~8durgBN)Rxf9yjYbQ>D_-3~f#QG*jpgMPp!mQaA%vZ72iEwq2}iY{9GMg?`6;c}iM| zFANIh26Tf?-7&xV!8+5NU`I#uruv2yJpRb$!RLwFOk((DA8uhb37$IV4}d@!=bDDp zdqNZ4DK0+I$pY)tCwkoupQy6UD2BJB*bjvr6u&b8+)AI%J!(;Ox7WC`TKD$*c0iK> ztTtmR!v*Xs9*In}gRT9EOy*N~Mx38u_FP>fKXXR`NN9bab&kVmx2B)wK+CTuoEzZ? zPaO@B{jO(f_x64PlQyP<>)k|hy0KZaqrWJL>B-NOlk9e_I$y!oh}E6|bK~sF*Djbs zW2zV3dDb0s!EcN-KvlPW=#Q;XTTbIcmNlb1*+ty=@_KwNT~a>S_a@Fs;ibXB29n%D zw=s%LAF^!nU>@1(}%53E=Mia~jlvcP*_JL&IbE?Yv zajPI3ZD5L9+y;efx34Ajs;jz6o^}0IN^13f4J_sGd8B>edo)!Fm*aU1RT;uDCB;_R z?Y&8T|5R%Ge1V1g(M$tN5AK!8I*sP3b_5@e_39wklJIQ^>YObAil})L~4*7x#O>(gNVr*})yTDH})(n0pUZ_(@$#`darHj@uj>i*d0`&$5MJF zrgCM?M{yqjT&ONy3wtA~ehT4WU6X}hpVn&w4L;!%G(01v`9k@G^y=VNozt@6vgkHO zEK*xWjP&F%*4CT7-o##uKoL3b>Dk~?w}5Dr<#N#j;G+j#Lq*bj@MEn&YMuDI>`n7$ zu(>hEmZ0Gg}#RLFc{=8DKVshQ^Y|(bcWys-b zt9)^K;wo`KXLDDj-o#1Sy{`J9jj+;rl^8L1~<;|5QVvm4%tSf^J^WYS%ItN zClgUK_)CdmH-w^rd7ER4?}hI{*oB9d#IEVNEhyQ0DU&u&9mdNEfOtfL_Rprybv$T> zC}Ko#Gpc>b_yi_aPyH*ed<+)ZQ$|g!zu0MDt~jYIC}N--TUHuha{5Rf6bxx*OwaX> z8+fE#&9@ci(^bb%G{!T9&{;BAi5dl12=7yz@Rtb$7sA@-b#(*;a$mNKj*X53ba%uY z2T*821o;}C6fXDE$a*|3+!e63B|fZ=2lqCQF;q43(zy@J)hWGoo&~OM#{8Vyo0qLf zE?QhswCFpjk(^ig&Kj}hOq0|&hI&5-=`R@#o@?J>lXVjy@p)T6>+nkMj7>cBS4*`6 z`YYsLot>lut5DOj1Cp7JIUnO3N4u21J@Y5B6x=V=tKu6I2PE2xw+Vymet7t(KrXn% zxS2oT&ZJvwGhh2SxS;#jqz7%!K`gly#(8mFiOT3704qugFk1r?G&?sbvIl_d9X?CU zmuni{)1}#y;@UC;$HO6*(l&9^9cPaM*^e^b1VMobTy#-_LCSIemCIj#WsV0;@v`pC z4v9rJo<1By`k&slq$^m|D^gf9XCG-OzQvW-@J3{|y5(GMD$@JnBFw<51o{y3_^8rh z&9EV}pIg#!BG>FIhgo5$pSsvNP(6C|kXX(+!Hf8pEs{HxvRqy8tLo($`~khqB-U(c z1t!oE3RCQtn8Gr(LhDo#4?(9#()uv_0t11pq4x}8w|P$DK9sKWAzTM17-}_euxnZ- z8Ak;$NNWM8sy-{+iGiM1kentj4Q6+TdN^i}j>vT=9UT310;;7!ox|sC3pC-`RB=qUv#Zmrb2Erth~by?HKN z%BG`V>&rF}IXBl!Shx!xVpAmgfkp9wz5lCOu|!wx_V^)PW80Kpu3ME2Q$tkMR{pIz z@9dWS+|CX2wsvC@g$KM%)n|ReR|+dbzu5IDd-(_13Re|N~5p{Mx0KCj-ZXiBX-dx>mgXLz^h02N!rsy<$vO~1Dwg(%e&oS}HD}g!gojN+8#H!;R zqyHGoO^w}9iS@NfH&BIUy-4zo-+S6urw7bk*~&iI*OV|J#igor;kc94)vDcMCHno` znY&JYn>f$D!eoA)G2ii#LEcl{?S`1w_=q=xU( z32alx_51Dgu}q@~NE_>8OargbrsBjy1ZMTxYU}TK#MICs>%yxs@89gBl=QFZY>Z!o zyOms`O&zy>W>iR1iEv>2z(CWsca{#uO}6`~=qrBESrm@rEGm;`@s>wA+s-*YK^G~0 zP)u`KB-wgb(@1)Yz9pPWL$uZVM^*9T;P^lcN~mcX2c+jJ<*Hh-nZkcr146^OkvR5d z3~Z&hmqH$zw`s6qc*};Gs0d6cCAHJ&L3@JlMV*_Q*)?6B-#mG=l|lEGQ7@Z-C-`R+ zFEVfcYaeITUF}32cuV}e&2{~2{#Ff!7bA_`e2T0N@Dm+Wl;fz0!Wv)Vg~gtqeG||6 zUp7RWxnFRH%!3FtnTyOhEvzzJ)$iP%207IjL`gs4*Ng&H2ijAq*?_4aJpv6RXu+%F6aD3TkW5*W#m#(nW?V5P9$yn5@Zyrlowm8aI{s^G~2 zY8r|GfIhA(WXkpIsBgVBE9vXo4Wb%bXw*sfBjqQ(IJs6E5ttiV&MVm2T6t?lCXb$d zo26+@cu``*FhmS2w%cjKK%pun;~No~k+JnGRY;9I8(^qkA+m?f_?lNXA@Ot#2KW7%L~K7V>~R;ri# zp3AQ$T9tBp1(CA}vz9MzL*5B2^;7V$|a_$I+6eFAdRD zpVI^Yu8=dK1&7Us`nIDgUscT&z6qIS!hAEuN8aR?4w*V!bM=C42$U%!g@&E{zoJL$ zy#g(<%8|M*lH0?dlfKirU-{03?J9o-1os5t!kSI?j#i=ZmaY2}4%%pu)O7(38NL#7$ijWj1nVkK9r0 z7k2QzMnO5L2yKNXw0@DVR3Zs_mCvvIu7mQ~Cm2~hc2yi#T;xt`zS7&wzS!V1 zXO5nk*y@!;_9e#R8;*7O$mB1CrP{-^IA^5Dvf81<`HI&wW&N}5%~XS|ti3zhR#oaVWrgg&1NZ$s81O&U=f&wV>`!1R3h6JfOPE+mJ%kVpz4k~C&2u|xU3ot_ z3%KRDn`_+3S~XsD@s~BJs|)(E!~x0G&5;U`B}L9iU=)~>>zC8zlvwcPZtY;8c8$X9 z&*hS(DLJy$h+wvF>q{^|BT`o|_r#ZXx94d6K`#6{ujB}`voh0taw2H zZ~q0tL~65s0MN@Jk}8up&@PuM$(gMLWmQn;s|F7>=7c$K4{h2ot^K9WK85cPLl`lq zNCV8?bgEVvy1b*jcE{GntCt9wA$+NlJcN0~V7`p1av=oOmK`meG@!8Jus?*%1&cX3 zhz{&3dRT0T^TygulQVD491540cYr$vcEv=uVPP8Z(Ir@Aoy6GI1E72^v191Cg>-L} z_t+ps!t<))1FV>IA8y`m_1Qv=&2ezs@+<7nVTTkFvASEpyp&A-wIH%(p=|GE>YLW` zJewmiMD*z{|3W+rxkZOyKmF@6O=FT=sz7~>6azIwGMypZwlM&?k(H%!@$*QLYoom} zOS!vfVGR_HqM5(To?klUMmEYf3L zO^w$z&G32z;|ktdF>d4eRx*S#=`z~uo zPb-VvZ)8_vCpAGtHJC_U)`_|pvD!$<0#RlycNS@ab4(uS<_n(-FBjW8AUHH&Yrb({ zIK@CDFzr`CAV*9i)!wE4om%~r{r#8X;0l+bJi5y_Q#8)1*Na~aOh<^biaLwufBZBY zP1VkEB^v3;gQZa6-7WgFNr#s69U-vQ{OA9rlsf&WDo>P>H7YS?I*LYp<_mY48$?o! zn>6#?q2vdUNdaGla&jazq<4{kxw`i5<(a_r5ZcQJfOTxqWO}gL#AeXy@`{o8^a1D6 ziic1qoXEI)aWj1~H*)XoR?BbbJ6`Iqf>MlBuLnRUJNQ{mL6=fu^EMHcxFO!oEp#@0 zcG7*v`1y?o=iKFEV&Z3FgZASlOa6EbYDEF1TJ6Izi@0I$aES=Mqba5Q1^sG66Qr&y~A^k=m03~qK>VT}wq<;4~5R6OQEx8!GYpIME1 zvP5>WhKVKg3K1) zZ6&sa%Z{*S&him<^)GC}_f7z!Ecd-zQv_nDbK_iLU68p%0wuVA4LkHL**P0=m#Z~* z8p9L1{Tf0H{HbEkCM%{!6nPy#RYLQj_&^395v%+b+&>qEO2A-FM+*Y|g(0!9+?OG- zn{3}EB*<)^-kG3;cUAByLpt$Q|&IY;{hNwqUDHppm<&DFXcKZFe+K#PbjN1Lt{#fD?XMX`K>M%ie>^@6@37N z2>XeH8-MIVgYj%FB`4a$bF*el-a^xlyojiAPN~MAZO@$Ha72%VG<_{Nfb1q_bzK1r z={P9={s7n&uC&c9SO;o!$&`sTPp+{Sp`UAM?IeY=!}A{iN_*@9lwKZV=*r#>pD9B# zW$r^U_KfU_am!cUDEuskuUV*EQ8+dH>Rdz3QF@WuVhw|GWSkg7>!n#&zH8y2!Q-%_Ftd zI5!UeJH8Y~oGb@DLIwYm)M{C(6)CxE6PojFc!v7{c&;ySOGk?J!BE|_56$lE46hXd zo#Dfv?fHd1e=|Q_N*Nh2;9oK>7^8N(tJ$uEL z_K%+AFQ29!xO$9<^vAMdNVdqpAZ=q!qoP`f&Z9$XAR>UH`!~<}uO9aQ<$s3UfvA|A zTbzj-1UlET58} zDVoXSzdxd=^Y4Z)|4>)UzqD|%MK+x-{n+d(GDpo#zH7w&^sqAwRdEiaFCH@sGTZuD(}r{wC_b5&fdv5EA&=Ux#A4rI^pSQD-c zbMluwu?K&KFfa|ONxNQ;HEsOV;V8QK@_N$LJgkd6QY^Yz(L8=`(@|T(2w_D3#7woh z5ql^hq+|#*dmpnLu`anLG$XnDy8Qvrp7j8T9)AE}^;IFPW5_k^;sJ2)P&D_d(oqfi z3bI?l04Iz#gNHmizT0~M+${agqFueg`!8~x&dbxJ^uik07tj2mcY-N~-Z6?BQ^ds6 znm;%>hbIW5rq>ZBLGdq11u<36kfS~OA(rmXCX}BzKT(r(u<&{wOYchxEOcGJd8vrB z5&xN{hf(ow`Br8`mlZan?;bnXCm*1VLx5Do_qkDHj5{9FHFJ{|L4Mg zRrvp5&;JO;e^6%>>0_+YS=5aMN#DPVlW#03n?;vB{L&%5PPLY z{ZU8qZ*?^GPaer_TDWo#Yo_oLMz(O>OS~fz_UY1K?#VjWBO8I)swjD93V|0F@3z)b zf5DS2R{DKL#nSw^TseAda5DAW557{hV3Bnm&&QgvVa(9^gWj~|HvY~1L0!(}pr1MZ z4*-9}R457#3nl-HSj_(}mj8d~`0Ft1zmCWLXRq5cF6yO6cLu)4c!qNYJwe>S1cIp# z-sh@J?6?z4Ui3!PdOR;H7`kZA_-?Y`9iS_3 zw&t?3Qv#XQy9WSwCZdW2vpO?_A0MdYRtu1A_QO_YVxLJK!E2$+NPjvT_6*^rySp5Kq|AN>mP$BRDxD)4}O3jnYN~F04>5F}S0JK0Sck^G5 z{qg&MnR(Ip3!FcuC;!Jq{zQO(;4D(tkDLIORa}Wd7+W(HTWA(sDF@NwrnYA^u%|z@ zw`(R3@j+x&dK7;EmYUO&n*_cO07Du?D$nmwYLj(_*@l=n-UA>z-PT#zEd9nOrQ|D63_%=$arFrQo4=$S9$vm?lEnC};oJm8I(D;+oP<$(33hn{ncQw>tS zyEdb_9xPRvOPFYW_SYgr%=pifsDDHn|2uq_QZi;-=k*DEJZk9ypj9zLbE#}-84KE^ zTRVo{!Ap$3O75rBrFIgMf}WUg4XlgNJpdTg!XE$!*N6t*gqG4ZzrX#2^4(qa(y{GG2beGYgAFG0=o8l-bClgYLgyHa_N_k9#T+LV}fX^;mW zR*J8L2WQR19xC&~zby_u0E{^E-`$e*T8+zBhiQR~WERx&8Va-GGho5_S#3oIk}$U@ z!SlR36gh+ikqv{RFx_ZJ6Ty;1X|L)e(_q`4SGdXd0{BcYSEI)JVt8M@$)(2fOVF?8 zM4EVDKysce*Lwsp_HS;}GOj3gn}W5Hpd3942M>Ul5J@n#M;QimLBT<1W?2RN0Pt&| zxw%A4q^5qP@?coJ%r_ryvSs?wr!B{sa`maFARn1Ssp~P})s9 zjdxgMKbqk*^XmbS0(3;gv)|PaD;pRuyWQILGi4rRGV3ZAkwcwp7XPjz>~Mm+tKCy> zXRd|DEO;-yLyL^X?p5waBckr}@?X&@O1=lcnHCGE+X1> zB0i+!%f>ms8?}BHD3G-H)oxLZ5miwP#C|drgVOY(`yejRK`;X5e3rhb@oO7S^#B;C z!Mi&~AT2s3hGE?uw&JFH3&&Ws1X&s>4nx5Ryu*S~VHP3kOqY3gPlgeYvmj#Y+N^R3 zX@G2%KA-ww&a}zw>#NCK$X?N=CFt!Q)3No;7(zZ79^dc6zmIq95+IlsI0z@sla+jr zJ0K2L+Xi_f#IykP_#+m=A7@ibSQ`Td)UGzC`U%5!fHgBLX67MA`& zP|mm0?ly<`FCn%CH8xqYxD;)iB@!#`+AUc1}@vN5jBvWs7bEqq= z4CQ&n%Zmb2AJs7Z%A`1XRyk?RmyE3tvD=NNYfhd6OR;a7EV6Zkn#$7Gr7sqpppXHh&1TXRr!GnwoUpe&uT7_CvPRl775HdF&Auteiy1y!{BK-sJ zF$gzSZZW-l;Ru+iRL2qzT2R5;-*JY2at%X>n`Xr*!q({zt%gTQhI6HEI#c5#0-K_q z2f!sFuBsU{ns#dc+fRrCXmd^w-~Egb(0ByK&ZDP+X{@IUv?@r8eSx8mWFxS?E~?u) zh}Gu=AY=Chh^80UyBL2+0n|fWHuzT_CO)iJ7CGB!2I3a=Pz(Qt_J{- zFWji9O>ZGfr>e{^o#O!jMj+&1oZm1xs;BOF#sGF6Fte-L%}C!~szCX30t@aZN(>0wql0o*TYc|@J!>xkvmpMdgrUD~;!krN*8?ri+Pe&qt0DD>^`rA2j`Rw{ z*71Z+P6S9obZP;iO*_J@>2?jb2wRaAKCSwdN>s3HVGHg{K53&(a(+v=ydcX>cDAgg z-Cu!KWN0!s|1x25sT0TX+E@`e6iPVL)Dn6}5)Ia0njf7dB-?z#jx$If=aH~#PfAJ( zG^0`S`!+_B=jb)DBI|I)-IJjIqB@y2LCPrVg_sM5gRiE)sZonxr9b?@ZLy}`w8HsXq7^F#Nk6B0^-H$>}*A1t!+D)IH5A&)_}B%ce*;)~Iru0hMw$zM`yAfe;TCj~+qw8aZ!su*?bS310)E)^oD4(O}5GVy_jru`T>I}uFU1sSPD zOQ0L%XmTALP|(jSha(l0DaQsGm@bIdepLb=P>@ z3#!jJFnMB^=EP=U3-B|<(iAGpF({f@ZOXd63)Ln=^^M5c8Yy(e_b%_(J=0siHa?V` zJJ|cB9Zk#=;d9B#YpWV;7S9-M9^IXXQ^qntpP@JL0O+Z96*;wJ3%K-^>w%pf`oaU6 z#&dbvW1W)hOMfBisJYSz5x^w}-Li(k+awD-848>b=-r^EX_^G zzd74KnOyL+${V=U(~iqdgncDcabU4?1tE5AZ0d!<5dQmTS!IRH+RA<8a|8~H=vX0k zzjO}}u#~y6Lj3UX)c>!F`?poyfA=_zy#$quz5EHE#&CZ&3K&pF;p|LRE~of!`jMzb zGO9fDWFK1qC8-XBwH*T6fa1;_sMdz>B$*iI==?=LSP-3Vu^`nfgXxT>OJ8!q!Q2(- zZ{GYI{yjx`_h_=kz)~K?LpCPa>B1>;&8BerxZ+G=!>t@U2$T5D&PkE&>4^J#>MZ0z zcp7bl9)ast|8%%G`ip*vI*kLhrpZdXI?@WxW3j?BU{qfkfglDFK4B2i^{#EEYeGrk z5Gd=``X)njuXD1-r>)&&rwS}k@Xk>(UK~>Uh9HY$OUuq@HQl`zmXWW>F|vr?hv-$> zjAM4p5x4!)=vZYnKMh_ghd6U*O3{BGPGG^X=0qR=@F7T!`=V$F=hQ~vI-27```C<9 zkum=fiv|{u_*V}~Kdr}3o%{^rLc@>im7@lgA!sckk?CA;+{++pNcl$JI{Un@yx@Z; z0AH9XbJ}=Ae-eB#$d6ao&De}`E>EA)g9E`3;(+L>?~NDChD|K9Y}GZK$K0;5zKP+% zj&8kJ_0<@A?<#l?9G8Fe0Lbq{6(+^)V|VhQ_R4j)(VO$}M>wZI=J5h|PpE$xs8itZ z7+m+rS-kvoh-@{<>6g$buIRZoYi6p*SW)+bmKn!*P_ zxcT;)S8Z~KUt6AOz|9p?<<5aVpPbJ4(Ua)9-;H#d|BSu6NIf^#aA|64L^9d2pFGzM zFR_g}J-+`aCXQ&K15LBlguygss&DAc@Ya>LZYkwIN^V-8O2Psj0EcHf?uJ}{K9)84 z#Oy}{+D3RYb35V;Z+L3)!kl|F#{4ttY)X1K$Jjk`t*uD^b$Ve5%YHSG_71ZfB>dR& zjRC?23KxiTaSp+e@??oA&l&4#YX+Pt6^vrPk@4H-R}3gNm@CSf>vtBf7!*uFqv{J^ zQ@w475u?w1!8jf6sPSC9bb7SR+L9}gZD0azafpbEPJMp?Xq-xMtz>A+559Z=me=XEYqjx1Y&wJ%SS<%Q^6s$xn8QygdFlGs`mczSU@pd5a$u zr-M9Pq&02$-Jf2QyJ`C#`1tVIF!AtJVVUXXSwG4-LJm@q^OGs`4tU$>5#@iSYxpTP zZ`Chsu1cm%#gXwkTD=%PbQiX?CwDQBxUy7N`&GR0SEI@Kv}`|fZ?*%!8`0N?39411WS$B zr*cQgajH^UqOmldl|C9}m`8|QgCI}CKsJ3!i^8BFW&aRXTum!?mPsFquOpT?IA-kQ zmLIJ5ttpK@MrSzfY;N6MksCHci+;J9DO?0Wki>E!3uO~f0jY!+KIYA0j8M{e`i_js z6aRVM;;BAu5E2IM1^rh`GC$7EA5UU?${BJ8DdhIaNb!Hjqf$>RD|_e`)Wk7V;JzKv^t|peN+l9a5t9ywc&vFCH(Gv}Mc5qc59{)QX2}-XqOC^4jQP*+e$dO}S)1 znFky0*QdDL^0Y?N8Vy%TpAl1ozft8K#9T^@GkRO^TkVOS_3Lc8K!}y>jk{1AEE`b` zk>8z7b)^5r7^-oF-A?DDg$? zr!jv+(tgY_ibK?X*&)2d%`l$=}3JiX4Bgmzf+~%~+^9$%a2;yg*QLx3wh;uP%@m=tw zS^At)yN+JvFyAY6iLl4lI^@KEz{z)#f@sz@ypJ#tCaq1W+&W@PdG#I-cJ@MIbs9Qq zvUC&iLPW(TAEu5-splR5$*(RF-plAV-XlCld>%ruL*M-{ml=6Agd{z z=Zw?kGiR3O#p+eFsHO$E-~vVqX?xK;`Wb(@zIjoDXaU>8aA_>aW~^Nx4zf&joWDH7 z;-nTAA4K}f>+pR`3?xPNZ1crHr!>TVSZE`Ncja_0x8}<8PVEE%Dw)`qU2C@-!!5jZlg*Yq%^3 zV?9ego;En`*miAxmga5gZ4Ep*)J60XFvhGp-JQqa6)2NrxHWUuIGDm9!wrw09ML?t zgBw5+i1j$4lFdXA5|FEZ_` z!g`*Cm2S+SE}_Qn>*6m)}sd|YKb(zD~z?E%2|hyVzFzi?uEDLk~qd1eN_ zb=rY&A%$(zJU41O?w6UV7$cr8tiL{)=U;sDqKc>Q2NiY-?R^h2?y>PSSeCkz32_rJ zXe9FBiOmQb=)3ZT9Px$c|_7<8*)qeF|cU=0y^I@h& zskhc<*y194ZW%5pPmehZGIA)whRK{nPj6=Gq8uNG)N0^}ye#b&$3!m5EhU-~I80r9 zwg1Crm`y0)yhl}D!TyD$ju|5L(GKRx<*h-}A_HauN<}nk5U5ItP7CNcvM?q(et?Xm zhE-;@!s`i66Rv|z6GVa|gMbv7QU2tq4F3d9%@nE0kc$re&Rhn5CkO!6!jcC7ZR83R z(b)&ZDsz9M`2dKMJ%%}fV5pM@5eAeif5M**;{N2tj7*}_e!|HSJxn$H|8OJI=(k&b zaSev)0>!{9?ox!9*o>DSbX=&YE}!SS#MTV^^SFe;p+M001CeySE6~YfmIE)u(`r=6 zBeqeS5swsjb(oATAhH^zS*ul>Up}Bxh$mi5m6Qt!zdhvAb-)X5e#-M^xV&jQ}CW-iYFq0~hI%~6P5tk;wx$~mLbh08%?oTN$vrb^W$3Se(HirE1RfhtU92Br19R zO6!8~vC%jhR}wcuACg+0ZOQi)Im)W^RXTctzIs2UB2jc}u|KfFfbvg5_84 z9h{V~g4O5u-P?~E#BX+ersvk1YiE|>6ml&QRwU1K=VWp;dT2feN-Z7C*A?ge0v=^u zR^C?mo2uv)INo3deCleCf%;CTLAdf8tk4JEw%~u>92`BX;O$!usS54~d({r@@`+Og z3>*8hePE9(kI;?%?4$_D`b@3jR;vMRbwofqM;bkYj6})p0k$1!p>79}V~`wr@#dA6 zkz^LA!CR|4s26;(k74&7+6lVi#X-8831;-fqIfSmfjf)1ycghkmOIkrYj`DSDyv)> zQFr5De=iz+_qZ5i{}6N&@zX+6&f{%q9s(03mRHjD+HP}ay>{FN$kPG3@CGdR zRS0JkvB_nJwqSmd7ylY9{*D;`k>MgJ2t-bQLktm}(c$5NW>+qi z4%ZNnTwQrP=I8F4gxdglf;GHbaz8ah_Vv38x}@{y&&TRI>)13HE4P0s_VHMmHHNGu z%gF6(C0(Ou$>P0MozIQ_Iw8yS%R#qxfR{21ZopJ|*WIzRT&XE2Fz8OwKR|mV|0Y!K zQ0lUD(C4b+n+=*~;-@1FET$^>fiHDi8@lzG)fJy@mhSPd4n1gjT%VoauGrG7Rmpks zuU|rrem>eEn(fth)^!pVWAyG{j&?kF`wvvD>x$fOBfco8q<+=J)S@W#O39^2-cu{~Hts(SkTN4jB ziwOkAXYyX_{KQ35d6wsL?Ydxn<*asPho!G~|BW;%#E{K=gUU&mCiv}S<)JCbmHtQR zIiYAy`{AhI>QLM!lzJ(N?XOV#DbF{Hy34XChHr7T6yJHgNH+?$%}<-$qlGeR&8w>)<|e+Z-c zGlxlgtwq0YJqH@B)brlzE`nO3PG`Y_5X1Xg$P%cNYM0UXU<)L9gBn|EG+4YeH?P~z zV1UYgHddv&_7#Uk_@4V%Gd7ZAiQ~Sqx*zy0{IjtUrj``&EhXjG;GbqOZEV`h-2+pV zj$%tIHgjk8eM@;%zS}j27g!gYAIdA^EfiYQaN-;kiMVMYIG7C1JACKM6a9sBEGn*m=fEcAiB;sMx#9vQTJp0^xJ0&StH=aQX1$5Jb5uDxf`L-jEFpS zw&L4c@YJwZGKF|p^T z!HECv+&gYOE6Jve$HZfxF4wg&C?cw-F@Ig6D0IUz~gn zX{A{lDP>I(=}U8k7pd1Kil@A1x}E+m1CTXiJx;-yiT(1}( z+@z;UMp1<^Q^vS4-`rVDtf9)#i1HU}=Ly;6RZEnw$}@!a2lPV6rp-fn&32@=k6L8a zzE<>gcKdEj)Up;_rY@+IoC|mpmalkQ7tvPN^43qB8LEe_T=0L%D`Gv^sY)};{iSC4xEInY*uazR z{j5=n4j6Ts)!ZKF#A?k?k=l~EpjHzBs(Gt4F}4-hW6ru}z(jiRVc?zuJp*Kh-%ve8 zPcqnJ?do7|PL=fBeSoSO>zZ$8(jl)obG`43nS2BNeFENd5)_J5oaauWJK6Wjid-%N zcabhp`HKYs3l9JT4?2<4)I;QnpFgfMO+&nqz3nn&+>hg&GrwBow9atJC6;+>pUU`~ zhIt-0Frx{gYrIK3e6gpUGtSWPMb)d&zCPfcXzDZRjRZAP;lVvse=4BA&TvxF6cGcU zkr~Q-!jY%RC|9vP|TQ|OyIWR~7x>kw^M zA7I#fDMHwfW6QCVM!yWJ%KFi_0b8_;X|wLI+1V;`IDlgwOR=q(upz2K-+}%bM=Ekh zqZ56?^W}vrwB8CeARWfD_x4k09f7Jqz6l6Sn& zoXwGIP2GHH8EGfHeB|5t806#r&#f~Al)(KdXW#Gw8F(rO6 z2s~kv981->wb#o-*t?A!konaE!1nBtn1ylwrRj1TGUz5v)n4a|?gEUiMVRJn7<9Oj z>KaXMt}c(NOr$_S^iI>GXqzOaCw5qlEu0<~&RGw>3=g`;(V_ zG(D+9XF{=wq$^+K(*Ux6}_yxdY!2>Xan(QjXWU z^i?z95Ie_#u*c#nL(*MK{A_n%AY-%L^5es#Fu0h5W$gpNQ~)RIddM8GK3BN_#e}xl;sdvElbu_lr)s{TPi%O?2ZVJsO&KLs%l*ez(X7;EHBIxXcoI|AWQnw% z{AEB&(}QxYZ0)`eO!8Ihp<%mwekf#AH{J@J5^1 znP^W`@nFsdzL6V40@m^Zi?%&o_J}zkf9ipqZJr{8+ys&7o2SlHf|Wsd@KzfrI*hZx z=3E~uFZEEH;br`tqkf-M@dG~Jhiw?)W~_~fssQHaWeM>%`gRay%UJ#_foDZsNjm=@ zaqk%v)!JprMh_EuPhob&{7YN zq=Q{JuCOD^-VVgJ_RCPvC>E0xEqg4ly<{vbR$g z+EO4w9L=INy+}qWFiTTIs@&zRJBcEDXM4TjyNiSq3}>*z%SW#xiqlWFk4w?j@T`#h z#lxfNyC=x5c%esZ>u4QcDLP-0p_MJD4iG`J?jyGiFQrZ+u5qRyi(HUEi#fe^b4%XQ zy-LQnbs1tf3hr>kO0l_G-FyFiU7-)1z2P{FVi&a05V_t*I-ki{+O3DXX6{Nn46z$_ z)z>vI`i0TLmjfAm3DA`=W01^D!t`0)6G~N}{m!Z@&7@1G&f4@8FLJ}#Za35ZFT%sx zqn-g1hrGL`wOxLL{yC-(l?YWnKj%}{aV~qz98Y_fE|;N1ZK_XUk&Vc(&c;NvfCZvr z6?KSFB~-DeaXwaic!k!2PlpU^2@BoaT?5!Cl&;SOotn~{lG=O%Iy~)PUZ-kCMJ%N{ zLWscUR>~`ECW`#y&P*iI+O`8d;6#*lWnQetb{2FS4sJ~`3_(KrO5jq4?Rgn_5x>iR zgo;P;K?M&A+@Ar!1$)H^dF_j+#e0WG9neKn>H@;IxQ>4_9!v-B2(b1wDOu4{SkkwD zO@De{z3<(|?g?c83M_t>I}i*#>P+&mzW|>Qng_xTW!ul=7d%nQv&wpAC1LAjX|~>y zM)avrKQ1OlpCdH(gs!!ltHSmzz*)qr#zadBvNO`ecBB z5qj%4Dox&urlkvkhjR9`Qir$@W2T+U!`N-ih+c!&rr+~}_N$an1puytXAzD#Nwb%1 z1SSK88>Du@r3niNnv#@5iEIDhW}!(kXbtc3Y^YpL3|yn@JAu$Q#YBj7pB=D7x7pT! z4?cH+J~RF-!BwlMQ=a91wUE$v2PsLp=PeK5xqwSt8pz$7(_a{Vm&v~)R!-C}jLwb} zFHnIR5h3zJ%(XJA21r@d_xJzG8n+#SWV1 zsL*ol?$;faZ;Mr_)o5o_j?mXB1!uEb<^XF?l@HT#=}O`o>`F_?_UCI~{uwkwiDUuN zTL2J=oxvumSBA5HL~czY2m9PPz8jUeFoD}MY z73_x5jVEG}01FVqQFc#_pfH9g*;~PVjxV!KVjBfMNX@XqiLluv-^*=q_-Uu>h{og0 zJ|yw}9W`+=@I@}xP!Rpr*@%oaG$cyg`J&qBc{XDPF3YXZ*U+u8h#DkDF+O8qsG93@ z6~|d%yZeJR@yEUUegFP1eh3);&KUX+up|GRs6Y#>9&U}#o3O~zR(53?HqfSrfCq?k z=}BDGj90IesbE6l_;_r$+h_aNT5^5g`e>aicjsws-7&MQ_@2C!*v$$Ty`idiefn`P zq{;66%cGP)$;ABme8+DWw{r5&tyI>{Zlp<%IW8fWSTyQ);zFgak)m2HHfJ(mQ7pU;8^z)foD#mX2M&s{Jrn?tsT0)~}zu;)kF5<7029dq17mDmr?gs(a8G~M$nzvKm zN|k0Y(}Xfpw?woQVPG_q=E6hNn1&6Ofn>gZy*L=AjXqj9zWD@@1z*TZk`x60bFDGs zmURW}k-Yi3Vt3+}Y(v1o;SQIlB^h1Rky3cu_kCFOG{4j{EEVkgPI)=jdPfs!$D4%; z?0=YjbXvmA*&t@KNIm ziOsB#(*Zi*EB1%Mga3BV^!4M<>)7!xx8XH#xt$-XlZq3*PaC6$?XOn*MVR=WoS(7MJ>;W5*G`374)GfrTf?jU%b|oO0e=)P4}kLHNprUjoAYEArN4 z1mV~D2w&Jl%}}?*qeHLgy`gK-3=x9kknTaGq)CbHL8VVQBWji7A_ghlV6CagK-QVe zzPVDMsJ6Po``$lPYf=$I(JNJ1dR%^K!{2-Mo}G)dz1!WqN8xtN9ti zpWg)3@-;+ZAi38#+COX_D1@9{$Cgy|hP_(-!j4Q4!xy z^s{9M-j^a{-l&d*JP?IOO;=hJIKSJUUbwy@hK+{Eiaa?hfh2`ddZvD7J@0)QyDisi zFZe{7w)&lTXC{i-TDL)AUpNJZd1oeyncw}V$feO`NRyx}ef^hvjcU0v+AsH~eH`ZV zEw5901xg#k-i{vgzQzFrtG8g9v7Y*fv=;%`1u}S*TR9*ckB97r%2mpsl zV=mV#>PHY2+z3931%m~E@BM1I?0e65u!OvJ0(s%&Z0+o%UMJp`6Q!>)AWEn6r76(wwJ-UF zF)ura(mcF|6lzKp+x9OI{eS!VAFX2lMg6Zrgh6*ojKN)qq3^%?k;@_46F;Bl@@M|U zv5+jaLKKU0u-nYOS4Kc$9B~KSEk4d^#0M?vk ziYaBD0EXZ=xdEPWfL5AP>UPlCAF7)F4f`Gizrc?lx%qPhdt0p$U84cHbT5Hrdhw4y z`%mj1;L`rrY2f12_hLS>1dkzh1|H0dhAo}9(SA+&+3bn6b)7yIW_6T<7;8w-u$LfY z9#9~fJC#hcZs&VskVe}&^!00NTTHkLhoMb`LBM?Opf}s&45qr8$hmEHH<8w3Vu0?O zu`#iwc`AZ4Azl7Jm&3p`E~x+6#rgT3E;R-x&TTU#f;A3HU2SLmCrA*tPIY1jdptWO>Pw`mOY{2pQz1IxJ4zKOGTU@DA<4zu+31m<& z)Z^5nQ=lV`PRTDmsC(|Z#41aAL4W?YI}EQ`dMMI)o*u?Ac}6Ex7KFdCF!&2Y^t2W? zBz=8zOQ+A!baZ5mY{~4}6Et`Z&nxwoSgO1yw|?OHMXp8xH|RdBD#ka#iEvgJ>{otctZGu1mPMCCJ0@@_SSG8oYxa8<17&qkA_ zg5>=U)^KlU^M6sQ4~l3u9lNASDniz+f3_rj_`I#(4 zPm9T8i_Pr^u5bFXmMG>7kAXe8%YS{CTwR-E}E z7@&^@jIcG$CKZpg8vAd+cHR9)S2#r1!aSOZ{)_&S?NBLY0dOvhoMEr9mz7Lua>S$U zpE8%@CY;}_+~vqu-4+T*IKm9Ie&o-PvJ=PsRQGG8B?yI@uT)7>)eoavgP6Ht_Pg~3YL3sVBbwxeW5%f8(2f6zUoP+dbaA(T|1iwQ9jOf}2V%DvO7&9O78P8X5nn(<}?dnhJs5hT$r zDalHg_(~uarNO z(`QA=yu%Z~iBALq-T!ao|Nju}GhjV0Ki!i&V}eFAek@h3WcsqM0dn_4%{O1H=QLI* zD2{Y-m$mfY46vfko@$EtZ942T57S7Jcaj4plj_PJadV~X*Ve9*+0_ZbT`s?^m;W7o zy0I3Cg~K~D0p;bh{G|!nBV2@F3e)q(G(in0Sfqe(5ctR&@jo$^aPNbWaT32U z3^Io6qK2#g(zPj8O#jT|mH;Sn{!l#-W2pa`Rj#r97m`I@6>+6N)1TWi6VLuGsLVw2 z09X6o>K~V^7ZlNo>)OQqAkvhyT&qAUQ#ryKJw5fZtwG66r28V@`6s&1H!HZkPkf@^H5NqTyG7$|<%r9Kr~XiToF~{qyTd z(^sP9R)wIO;W}S2!W{!$gRvDQ{DEb|(nNT+pY6qlXH43Rk7!&sP@I9+0eNyN=wc7{ zqtnP*4ZcRY1vv3tziE2%DxT^^Y|$C=Cr*RYQfy%TOX&;tG|15yc>FtLNfY2<4Js1e zoR3Yo%Znwn#;Z*CI>ubZczn!?3dcA&h5EIs=jtZQb!mq)hv`s$kDD3;?#F%x|G)41 zpE~qrJGCV6hIiu^7_0$_xmdB_rEM0#6m6|P>;OqyJ2m41dVKO?S$&s#Ve(>YypFu> zh#|;mVz+Wt+U*Ia0%~N_C(KbZm!88`e%UZ&4=w$kszT2v%sz1UMiiYkR3+(%CZ5=+ zAGoS9Gszjtn-Vv)Wfux+FB4~$|;}z0X6>oHO#CA;!c`q~8AFqqHfz8VJ_o9%HwB1^s%3Po` z`poxwP0I)GcU-pWv#i(;jVQO)Fu!;bxd4=K(uQ}vA&uF%M>>Ob4OQO*25KWEsl-^t zo3F2ZtUQ_31XR}`zPU9gynT&(;F9X_yTNwGL^J(>RwRr92lt4C-uz_{6jflC9nh7^ zX?*x1wCZNYq1wmFYohVBMrp_$El?W0V-i&x*1+i7T)Gna3#?FGIl_ZMgyj1fE3{dQ zuD-Uujv^9ctA_MOaMjnnnAssYy7u{p*r$mP@P4e4f9(>#kmtB5RvA*rX-j$aF0fgE zf7?oAIK1>TAT(&|$j$nCyN&ghogI$AfR$tvXumci6RJv12dNK%;Pc+L%ihdkiJQJ- z=%Bop=miIVYN~(J-pbT>>qY@d(Is4$uSiZt5+Ue|fK_e(4|d;{pv( zC;zD*{2h_B1!E^P`B-wNo_tAjQ*y+;&H>MX=lS_hcT^3w+3wFi))@K%UlR(6@uT!u zn>;_I%Bx@z&~Xd#%`0sXk}Xj>d4@oI?im-dcD_Swf}fR_6;ctcA!S>+-vUT>sI&gm z7tSLE`lL@436H!&e2BFyCmHkSv|gTW3zps4S#9iO6sNQDG`uAS)zoo+gZ24w> z(2elRto$LQDafq5r8z{3W|$axHR481=)u>G?c?a*cvOE;&pW?`$^FFC_ zRHJhJg%Qy-T+YM4(3XruKdY5keLb%qYdiDw7Y5?O_O87O$6##UWU|PrQ~XEc{7(xAEGE_y%_H(HzFUuStS&Oxm(N}E zUW3BL`oW4v>~V#bMB9;iww0eiLVXl(w8)+XIy62XqgNvDwuEg@$H(QeViqTNl_jNw z9)H`2mR4-dHQh}1`I)_#K{Bnw*80;ffLRXgc_U*?fy%<&Fz{+ukkgouzNCt7k(g@l zm7g(Gg~u!_Mv=huMZyc&%7gP>PRfQm1k$Z6q&gS-sJ2b}#@SmO=N$LDV){?xUGy@L zE&7TgHm-kRgz(HsFSY5|eor)b6`3=5#fmK?gUc3bLL2TCQoiiP7oa%+av zaxXyx8`k{6gAVJX8(|60JdJg7zmTq>Zp8NY+FENLq3Lm%RlY&`!?9gb*+l}XM8}t?zIY!Tg)s=8{FI^R^0EMXRxXV+i zqs%F8{e3XI6oK5mbc~%hmv*>Qh>>te1?WoMOD9b7IYQxLx{sr46~|tQm>PiJQrPx4e_mycRZa%eZr93bPvl195BKK62s*l{&;## zkX3DqGj2WbNwDqZQAKppls{uPKcgZTQ$NA~48;euMObakIb1BQ$p^^75{xhia^4*a zsb;IFz&u2TU2hJIVZ)!8_&bDTVW^oj5n^Tip!EM~3v%>jA7XH#s>xmBrBF%?r0mMR zinfzsMSvSLzya?0z}ll!1IC_dACMg2{j3%L0R~()lu(_ci`%q}tT6 z1)v~_%K%M_T*d+xu55-U&gN4;3T6h%mXBFr-XHoOcVk|3_CTH~){V2;*gdAG38nZ* zBl{qDpb}|2u%cN&ralV8P&*W-Bv04KwrFZnsTs}nD5+B#d>utEc2rx-*<~hxlcvt* z+{?n06F)xn%u@6+)E4)U^8L43ALF~9qvt8C>zsz!Is~2^3E-Y)C$$^M?~f7_=K{x) zkP9g=@w6won4X;xq~v;&hz#zc-L#rh(}}L7QFd?zLh(PUzM+LH_qdq+SwZpAR8s>d z1&q0-LZ!t`AZqNJWQ(X{tk9l?I_dG-O; zUOV~cfX%$x(vFFl%lFBR7WOGdhRTr^)mqUZ*41y!v~K*8CFA+{HONj6BnP$kMgJC_ z|C8Ntrl;dP=C1gKTw19Z%tBv-ZP`|<)5R&33(Vhd-ge^DnRBm+KY2Tc6Hc(nhS~Z? zi2KHq@E3-5N3MZJHi-v#`=Dz&j>>d{!2;ftb;2gQoRe-IB=tQ*HA?YrXJ9bs)kX;B zk@E`wu&o+k|1`fZ+5mGPy31l)nk9iP-;$4Cc-ws0CxUWglm`=Bn&ztkO=@6}tCGatFO2PCl+fGCF;?SxgwK%- zmx%a2VKsc(Mb|qTzW(Jp9@=I4 zO;{mH3R4c6Q%^4RO6VVU-Z0>#CgArY{G?8;N~|5*+Z-u3GGX@QGSx2FZ%YPM%3Lj6 zNc6aMzsnMA4&ZlImBL75XFnGKG3~|wR?SP3Z26`eA#?+|uT@E9Otsc|$e;(>xbmFP z^znF{VI(r!`LHE$pi#(|x7+>YPwN;bxPIvDLd^`i$GobP_E=fZJ$_sEt-jkS&^9cc z`6QQW7Mw6OgH*DwbIKnn$}Fo0!Fy!$g7GzIVn@^J&VI2?wnj~L2sQ&o&yqMeOcOmb z4B2Ed10J_KXx_1Yyql`hjv2cHM-+*~jlC?8-Q*^qu^BFhP*s3I?Vt+Kh`#j*Y3U{7 z_Ru#Z^Dn=&`17H7fWNr)|9K3H^T_?%*UUlZ^FSIXF3*9)Q##82%PIGs&E7p#C){gb zK>NoVUOvcJk^ajcPfJuk*(?Ojcq*Z8vo;+*19#+AjoR3)zL+u4?OQZ0Qh_}XapxIb zK7DhS_x8awx|3t_-mM)_9k;)#=;b;zD@9X-+|`Z0SfX{F@rX{|tayGa{@QnOn9X0n zgVWRO8j{XZ9`=}VoQG5mQ_T}oF)NRFl*2CfXHj0X7XU4p`~I;lNSR5p_K`@_#e8H= zI7ieS*_6$JuHOYFpsxXA90t0oTW2H^KSy%`hi&tFwBwd#5PEFi3ZD!WNE3v}OVU09 zbtg;-Y=-h~ZjyX^bNk25G-Bxt(-;d#QU`XV&z+l2i5if=SME!8()sTd-Q|;r8-ss2 zbzi1E6$6Zzij_t8<|?v>W$?g>_FK~2dsmpiojHeHHcePAbr#qEXp*{%vg|IL<_?~h z`Yh2bJJ=z64ovatr#!l~2h}Cv4%9*)7M8~>G zKU-{2bnm;y$vmE$qI?edBGQ=ZiwzihXokM?VNm~mL|>NLw#@Po{Kd;R2^8o0c;exl zuC(wQasgtls9Ey0mhO&Ivw8@7#;VKma&E5yi>&=B0Lzlt06by-x2~hfPIB)gsm*TI zqis#;2W?6iZtqWTWjTe?_S0;Q#K}6gRTWsB3%;hZ-D~L}oHHm|5FZ_v8F%TI08J%x zL_OT%y#dBCbmH=w$R6&x#!&2BiOv)>aX3(SDVaX~2-G*@2t$Qsg!ytjh6)=-i_1vt zlJ=Fp1M^ikQlUK?%E$0tNE5R0q%hU;q|kX~fMIO!*l^>5k-d zi(qoS?I++Voz!>qo@6wbD_LYPi#`_2h~=FfR0+I*`f@2316d7B|KNYzf#% zy{t*T(z)gd8;7*6bsS*w5>&FJOCq5*wCnN}!hHMy=^ah*l&~;;EXpL`z#{pRj$$cC z+PzsFgLJ-(eejI|YG>#?CT~yG?ia=e&`V#d{ZN>ao2?J-MOK{&S}|tT68w!hyS8t4 zAS~S9Dq7KnIiG!RH_TGSg<`zh?&e7eQo+p9>0I79{`v&% zt^+@AI-v6ZJ|lm0f2~)4az=M^fY#o7wvIbyHoxc58OfOCw0?Yy4Rx6xjp8bS)^*J6 z1m;Q!h5pP!cg~jM70rt;d4~16H-%n4p6k`6qPqe+y+a%)3&%o-=_n=hpa>Xbt*Tcw zUmo(LB>=@ImsUEE@(8f=k#U(bR2(>m2k$-D5&7!zH9{X6$~XvP;6c; zDofC~4lw9m^6?Rt$d-1iplgne!xdX0>#U+i^QM#6TZ?MRLN`j*t-rP^`mYy|c6(S} zJvl`w_NrH%?81pm4=RqElt+De?Y_Gx5qx0>S2!mPbP-d# zOs|{P+(CH~>+KlAtmEMI*QPC6v`5wY{acGzU*2{oIgOkYM$QRx612yj^;<-eeGsD_ z(cEmOw;gPdk|fcB>3Vm^JaIX_Ru6pFL#8OcrIvO|53>sVJ%H#x+54jWSqQGGmv|i3 zT~=|+YiO$b<_ijfQXl?E{0D6g?$ts&fM^^8C2`oLoop7m<|8?+v*6x>>tFLP64piz z1s#;)<)Wtry|2;Wz>w*%as$f*jULyE!7{8(8Jb3yWHPpmXtKzh(o#*0ZM;@g$MGup$SEV!+{>s2HF4jR1pM`oO@ThoNF|eKJD=^OsG&1`%+J4E zB${SoYw6+uo~D#=w;g*p=5~)tmU{7MOS&w+YU>)(>i1RBPBg-%itSmndbzG-9m|gx zTr4|>(ABffCHc%SE+4Z;yQ-vp_JQ$c4v|Gp7r0BC9(nnVb{cyqNHf+$lc;PD64ERa zC(M?OJRc@R!MztsM)|#Vb~LY#E8B)Y-;eL?|H!@93&#}}5*#m02`3D5X*SMlSCXq|%!_GBxnZdpP@=4t7MSuKQ0 z+tp4E*a83d>1Wu|Xq}%>hieAIzKG0%XJvO8)0dDpenY`6(3|Y4lM9&3OM4%ToLc>w zcNg^gIh6aU`xrMIbD)nfn->15;3X+{sG1w-XpiG2=t`L`^YP&DXmPGp{B=WQZ5075 zI@grc44_UI3qLRUgR6a8!?h)i0VK-wzE`=LR@4O{w?%7 z;JPY`DQkJEl$$sD!hb$EUUaCH0=OXsyg`-GmZAwwfWJWxAQ*k+`P4_;!|f?hE|DS4 zio{bMpd;MT4$W&-DGeX?Sby}HNSbx;xf;7fSldb}0jI#VhH0~&yP5q)ZQ^XtRy|pG z|4cJexsa4-(lT6Kr&c#}cE{?D-OA_f`MH268Lq>)SQg#=YKXL^$}`5C*uB?Z;cnS=q_)233skDgL=y7$UXq_%WY-SvUsjho zX3sgH!I>nJ6NDuNgGk_E+&XUcxJg2BH+LgnW?bHPT!y&ukL&aJMj(^s0#UmR;%8%? zB`4AjlGH-659j0S2p#}s&wrIdZ?V7X)`8w}YXLh%w9(5t_l&KCPDcQ- zvo4`YKtbR*su$)lvQ?#0Gv_IQq7l z$}{thkqK8#T`nFM?Yd;^0G2dSF|Kmcelg=Z`)Dt&bHyM)x&NN#gL>|cvezs3!nyD+ zNVGx>rc!kvrdTzagr z@8-LaZUv@K<2xCof-5fo=fo0C}0@BRemILj7n%o5F zyKn&hyG94*Kbw>|gdrRZ@>!<1&^!&pHC1ZP+x<~kj5oDA7 zM$2_;NivUQ-iOt3l@5UNkt}mNpf*9#IG7e_Pd!XU^mX5ldyqr+v2?zBJ0n2FV1lEa zRYVuzd@797V7*kFI$n#VKoj@m(LpHHHU=3|5yD)M=dSwWz&q={jG+wSgNB|5klPp; zYx<~$+P1=5g@vKa`OR1ehj*NgpHMj&vu^ABo1@U(<&3^ky@@Z>wu0LWZ5jnPy(VQ_ znrNL@?gAH2f8(K1&6is)*1YxEFM0w=y&eV_u;OYXp z62iCIFu0WDCY1xR2Guwv@V$p3J0}DRHbcHEYU0B{*CIc@UQ)!J!_;*SefJD&5k&iM zTn5V}c_|#R6btE=y}kaf1nzMEyC0Lb_rn)KNA$%UQ zItMEt{5~5=RO!GaUBu;U8!1lBzI!4UmDbH~NU zv0nJ+guyCT26yYpez=B7gZBcEekBh}7+w}$jftkD|CE;#j)sk&vuYd6T24gGP0I}_ zP?%{eU9C(voV*%~(HYt+iF3Uwh;ID!L3;hZ_k}^#VaI!Wg4$@s-n_RWnCjrZdvs#~I@yk8_V!;h>Gw!D}BBHckO4enbOq#Poo?TBM zf)m)7z_iV+(#aK?Ntxep^ohE45AM33oJ}+DwbZq!6OgA^l7?V}a2DQ*_*Sr^P9aIl zz|!6Id5Vg^ltJ&1dpLTFE}D~eCtb>O^iBG;H&2fApiDpiIOnypF(iUdm=_8#OD)FZ zSB;`M2hb;u5EW~y?2pF6*Q0Uu)R`nCWdcw>0fAn(f=+LMnGvZDMFAd5Qwkn)o(yf^ ztJ^w|2?#27B5fit0xiVl6{uBT7kv&96w9n+Hj*Y)$ME%sjrDUYq^LLO%24sR-ma37 zF~IS{(koR-sH4hy9;6-56X1iWAD#)7VD+n_-WGg7K=>v=ec^4oJQJItHEYb(?y7ft zX~d2)DjHIWiZFU~m2NO_ULNY}^0FYWKW83MbU;!4b}BSl;Mz}ZjgQK_a$#bYOMTo< z!y;$u;T=1+Dg5X%>RkZ7u&UCGhBcgOYK(su8(Ou|U(*$)+&A{5{2-|ed#EGJmU_Xg zMt57{mWL2{=%4XvO8%Uus{D^=R1a| zI_+>stLgMMKUyzmZmW!zu(WqTU{Tn`kOQQEn#zRR{7qpdsrT|e?MvzUNT zI1roVce;7+cC+WIYsQe_@h_TOt9vr*rD(h(8NIx>qEkJ9yv{(~XNiX={CF$rRH!c@ zY)xlt|LV=_4={<74rS>D>-HDkdK zSqZ3uL+%aK4jWv=Df=mO&xXPGiGsdckQKv+AtH|>s%Nq z5N#e4pu44~xW_SNTe|Tu-jR#eP<7u6KmM-!UiK8JaB4Y~Jk@IYdWy9jTh>ff>ME34PP>*>a#hVow$(+9E*C3HGw)(QdHgaVP+_d- z(cVp?jZx`ozl?Ectw5N(blRs9mJB61n*nAIUo}y?4g3X8eb`*X(U!_!^sREsaW<#( z__8YQ4Yy=t{i(B`+!_}7>FcEX0P<-o`^;D^>=z0+-b+Q@(qU#a&^!^I^lpHwi@tN? zcz=GaRxwgreOt4-@v6T%}FZ@dv`W}wD1P*%qUey zskb%o?Xz$<-oWD0MWeurB9J%Zh*fs{IbFDUu+I=+w(+DEsIFxqt;{U!*|6>jb3xbd zo2gI@6G7SYhr=}Ga^BZ+^R?;(ou~U*e^AuR1EH3Fpu~$km%VXmbrcG=h(O8Xi+&jr zK)0d{jhIns8FgXoQ1WerNm+l*E0~DYm=mGO2U;Jq>;Fh&vOWqN5Fx8-?^ex{`!iif zI_TejciDJ{Zct|(bT+O+KkW1Naz|{{89w>8`FQ%$RQJS>o7e;L-$;`v zy@xZnyeAb_wd%w}XCzxv2);WmqLJ8~*lxM`nGDuglcR{UuFIfofnwDR@+~9J;)>kH zRFgA!q~o_6O%9udX;JP`_Skh6{`euh{kDDkt|NT~6A5)DH~YhpQ@%Ef6{>DuL3Ad7 z|2FvlC5sran02pelUr~{)BwIW!0I?fgLV&|zTR~pfZCR4Ty249e_=f39hwjYNiv?> zAGsluM!t+ICpb6Vw(q~~o)KOz`LevY*Ct#t4VE~Bxs>=p>-xsamv{F#2K(dqOzE}+$7S+j?{>AOtj&NtcQ$#V>-^@~H4|nR zAi}qDgF!75_##3j$h@gGC<)sphI{0gcdz){yYq{uzAVwkj_UiaY(I*l++ z!crGkY!WXrZ!R%t&db`Y)WCbouU8GJYokJ8u36?;+EZ=p~3)F~G4lQ-S z{8!6&ja3wc*MQ5}9PzdPD0;`}rb{k4J$P4uIl^np;2_5 zWM?gF>2TCF#BYZ5+0oYG`ur5H-$Wxfv%k!j=e4>+`4l%mq0 zYJA6s%P-Ubn7SNAP|`jm^o%Zu4*!p;txQsnu(`G_D$&#xVQIyqoo?hV(T^6kp&5Yj zz1{SjgqQWx{(65IQ~jRS>VgPboEl3T`+DvV7dmegT=KeR@=8IT+;iCgYvx!W*Hgtw z?#S!P<9A)oMQZHZx4E3k6aA*%eJ1a>G3J5Gh6i)Z!2etX$#qA0QB{#dgr82@?w-+C zxtpC(G0~zH`Dt%EiYOI3{lYke>7oPf0#<7q04=42h3r+XXZ`8eM4?D34oy6HM0w#J z%iepBHBbANDJK)ilZ*FKNPy&z4+DM>O`DuemjsZ@!to*JUhu6}5BmKAO8=Gj#L|dH z{W%<2uqX6x5KO8tP13Y}$h}XUQu2vnN_|>U(M{THG@WuqG^d)Fc2YP7R)b@*lV}sX zv8=1*Ckf{yjIaTB9dQGzZ31esF9ncq9H;|~28Qq+Rg@xGanCfCn zlAhaVIl1SWfUv2XiUNUGjjC#j{)NGJN#mklh?~nstSQ^`XNHKT#OfOpu*1){*C4r4 zg=pS5{O^nCIE=VOq37KOKXFeAE`9D^Hpopt`9NaK4TjN^dNQWkQiR|7xp#5Vo`gNQ zdHrh3aUm`xclsD1QzWYY{Ow6J-nj6=hZlYdmNnc~xy9KmjEud&Kq&i zK6x)xnl*~Z#PyZ9irNbGq4aQ#5xb^?i(v7Q2jnou)FWN7zv=Yk!s*I;G-2~A<-}Ja z1*iMDm19v@fhtYhi-%qWKGyLkpq}HZs)Rb%*`o|S&rz6%b4<~e6ITxmVM_F%*@P5# zgrp9hjA&Zg-3Mg;HTNmbD_tjsEn{YXkoR*Yyr8ksu8fFWe0MLMzr#U&eBC;O&`N>z zLEaP%W|p7HzWbFJ?$tO?754x*gXdV54^;0)U%k~YE1K=)TxZ9@{#vmiu9@iSdAg-cmdhAy>R1~6){bBAtTUHahgo4p>5>H*kO?i&*Xp+_ATO=e z7J!t|SKkd+`(zT6Vbu-eBM;o0#;&QaPQvWcrhkuteh$;%PT;QQghQJl$K*ghP@)0! z$%kvKw!LwIJQo&meP$2fOFfRGEWA_1&DYc1oImj7^{X~BO%-@%>>0KxdH0A7+Y^=A z`l_mMIu#?a0E|dq*G|QHNV8J0g@N8(d8U25fjA0Lkq_LmCU&+qMxRp4@@IW?3&3Bx zi&kTzk8icV9k*Sp9k!$Sh!gv<2YMMyy>isSj3QBq;IhDfXszR1rw(5VQTbq$)FMNe zsTz>n3EABR$}Uz{hSO9gS3sF)+!js9(u6u$5ZQrdWjW7YK!T7D1gdSr=1a_|a*28w z&OdNvhZfkj=5Io%&hF{>A*;ez=t8?-X=IlPvvG`^aT*T!{|&k33Z0o zhS-JF`8K5iG-`PXV*Qz|>-ldJG4!=PQ5CiI951qTIP-q#;yz}Aq{6UF|`t8-r%R29ZOQoK$Uu+%2y$TQ;id0 zDKA5J=Q^BHFShh=6UR}|m7uhrX|0|W*qWsu%A(S+qojo662abxNTMHz8p-Dj4DfiAFDe@LHbyoG8zRHi_dLm8mFg@PwL)l zy3RVKn&%2G+<4lG4EAivlhl!0^ z34508tBIkt%LbkKRC6q)Ss{haqa+EDE&1Z;uLZKUK$5kA}zI_K>+Q1+~HrnCcGZ_A7f?lMDXC| zl8Ku-F_*rZQFqH_jcBRJ2kv}-*d zxjMg*n9B~qMJK1@p`6vD`mU=!U79zA;F2rR?E>bn(dj5d`%CbMsKaoWfR|jIvpygs z16Nlq?J-&=JHLHYD}K@kcsV-(>28MHRE|1cH8Em@G+$Cmh&I|Oc6E(y61WBbp3g!i zN;ABmHiAEsRu##aJcWAd0m_i`4T5zRY29&6+@}>{e`a^jPB2s>?VZ&$Z;wHmp;V-# zFAn{erm{BL!sL+6;3~-9zoDa+zLn)mfJVe2Ae(C><0_+!&OKdfa5#s8?HuKl-dw z3YZ*qu>l-gC*?W93!Yyw^jO9={B`Z8uYM6SolXZcXT`;|vQ;Cj!}GUkcvs`s(&}Qi zt*4r*hQWlI3*EjYtM0;08W*`or~35^8D=Lu&lJ9#6#(3baLzwh{_8VpQu?u0?LbJY zW&twdPq*-Bsm}mPu1LYA0cgz1_sObZTYZONIxR>iTZf2dZjXt2rIH!^9dxw`W>zp% zvm|4Laiv*IQO=<5OP&L*0ukIbmAqb=p1ReTNR0KsydnmAdQ+|Uc)5v-1PEC-1tv$) z3#`eveHZum?yhy*Ykkmtf`VurK{u`mebbQtqw|8rLMzfqj&wlBr-nZ^k_ zo953wW;5Lrl5}A{h~3L+>+2B4n=EM}%&U;C{35egsTtflsD-F{2VUbE)sX?izIDQ& zGl9?S!s7YgMZB|0fobdrqa|Cx4nskq4iMj z`f&ptUkM_5B`09Q-QU=PznOzfc=tG^B-w~DwTm%?vW=mWBi?mD&do+~%4-**mk)oh z!r6o57NGhx`o-!A9Cx^A=#{L^^O;%6x7irR@gYpXzYp@4u|}GI(Lx~#Mg}{=kI#TC zz3k;5srsK>3;^QnpNGuxj4#J`^8H!vv?>bl?Eh=vk*4O#duQxsU(;S|B!sLU*RvGx712XP!NrnGq{t=$t%z%5!j1k9A`bFB z97h(X8i{-x^p_0i_h-LJQ)CAFfEJ%Kfm4!GHI9gpnjFYf@Xk{J%GeJ0<`mq5$j^6P zVQ~Ej&@MlDDF7rsmkiW3Xh|ALJ8h4`D*AwX zSja?^03O9XPyIWT{`yJq64eM~^Qk$oTLfKdAc(yfUGI{C4doYLf(#@vmpr z83o21O|qH-9Ci_4AqE8p&?v&gZ1lrDKD6IBGWbD4-R}%Yuy;q%^ zbb9DG-%M}&wd;>S7=8EqnSMR-f^Arr+gMX&`*)G)WTd*VeBss40+NOJZpSE5TDIz_ zwDc)#r|B5=s$`kWYDB24i!F6DFWvptY<8tL{t&r*p59L*yRMUoRA$~%tJ{@<8)q$J#9VMuv z)aCB|SL-bK$3TU?2R1@ncocjDVk@^jFN%we@7Ps%RX@`o?tZAQ=NEH1yq=oL z0?(M}d%R{Vq-wwWCAaAvL&a_oQfwPQS{#{ZRV8G2I zzYMtG*w;NO>oAq)5f&Ax95cij4drnw3z_G{OL=n?u&XF_%GsR3$95EWn7CxrRMK)0 z^G+_SYuPQF6hU$`f{D*q}EXBl9W66p79|DLd9z%P-pW zl{L6>{PuIu6}>xOZ&D=ek`;Qcn&^txC65bssy0^7H=a7}X&`hx>?(v{?zEydk#Vo- z2pmQP+STAo?7C>7FgrR|O*W#^u@bIkPz@n4z_xIjV|qXH=Jj=0>t6Ef;~pvEKx3Q_ zNqajwL$^w4(lf<)Q=qUd9g(&K*{a@XOPMYK{q~?OO}cEMQ`w7SB842QA!S{;6P z@)rh|yIseT@2OLEz?mw_LEi=Tg>9_!6&FkLQ3bX;jb!I_3Cooq%G&-lgj>1-a_}nb zDj#m8WBj}hV9WBjs!iNcICnIR;Q`LK!{72J6#V&XYV4zE6I{-cyOU4N6xV ztTS)XSO5xDJ_AxClrE$IIu3JoJ(cxkB4%|?(Z#EWS(tnl%{9?8-}V|?%#^}tXCo<- zWue8Haq|M2f`1W*{_ervW|NRB9iuruu+2^+2n?d#^bF~sdgNumv#0wO5m6!kaup;7 z>8Q10VH$$+d9Xo1_0uHV;*RZGGtgNlhF9aDFWwTKnw4$p;Kba3mdBzWl6Mt+_>iY{ zu}xber||fYxX-zDnrC&$9{0lWGS|~GWUxt<%;>#)m>)3~)kc+dQ-OBy+kbU~|MiMZ zq?7%$_qYlny{5+@ONthNIjgKmteb(ET^H-n?z!@MP9MRl8k?DPe%+-7 zHJzPAGaZyC9!ET%84U7rT)(!csx>QX zxUMp=7`KsY7yo*^dL9Hgl>N~U6JJ@k&)TE`BC`bsE~hqj*^?smJYi+I`+PHa+jSWa zJEO}@G<#%^gdPeg4T<&QuA^PJF>0~1C)f5!KZh?SlxCgN20}PU-qWI^s?#Gok zls{dVCH`t^K$3tVwn}0bh**GsqUWHDy;MGGjMq7ylVn=s88X7+@^p3h*W5nG5`ZSN z5c^|UL`_*={b7YlMTithbz*7-{{jF2cJ%`pv=v?uwjieu4As%lDX9R;5)MP?2<_OWXRo0ImB zLalBYlDwyME@c6c=JL9B^?ou4U>AOJ>bI5p;c%Gm`SH~X8z)aoS~lx+qMNa+GFhnJ z9P#pa<$&|&&KPu)UlAY}K=B`@%5jX6>?=Uk2ww~}x~OgXPFF`UB6E6x2nhXU)ip2J zs(QW*u)Qu64R)KHDaD>Anj5+{q)|;f{g`&g?db3+kYHJ;zSB1~dkQ%5a(%EBhPqt{k~s8JAEokpBE50eSA0N0RthbF4pQe(ux5L^kv#qc0u< zdy4D{M{bW}XR8MwYf}RUy$3}^S$Cb<@Rx4}jGV`2ASimO0EK|^pgkQwF#0~Q3Tk%) zR6kzps3EKfpi2FL>*P=ecQHS*IKJ&W=(=`>Q1>`*y{h`x#QM(0yLX7Y!yiJ3D;-R1 zuXPK9sp=*fq!kSolFq*>?QKx*+hV93=3t4fYNJbyEs-OdRi61I>PE3ZrSwsl$d#@? z(3edxqY3q^j^3|^+zFZiWI6Mk^<`x znXK*fE)HvihLzzm8n-Fqog%pNY8&St^gMu93AhZr$P}pRA@8At9K5Ex=*~%1+4oW%oZmA@n=k>{;+|h34*>Dk8gw>Di;M2?SACw=?P%x z-QG3~OD`*NzjA4WrCYwqZT^_F(4{OqWHC-TICt$)-H>?(H>!CeJ%EtYW=eye;3h(Q zDD#|+TfD(!SoiH7w11_Xiym_#VFgv`g~HhoRwNV(a*xc&ykn3;SlO)ERF)+oq+FW| zJJkip~HAk(dfstE~ad5G`bU8(?pbXPn+8e1911lJUgXbnI@3xjc`janG zUVm8s+GwxGbdh56<>Vz~|B{)8BYyZ&bie=Y@MjoH0?_^5FP8HB^m)vEhH>fbBN3x`~L)kP*Vjl~uo6P!`g6dL5#)?#W# zG2aQ{NPa-m?TTcfUG?b9>r9&@c~ok)X!h8UGmVb_T)5Js4T9b*Gl}t&{+*J_+7H$U zm#&YGgsbx@a0xd7g&Q)~N6{ge(~KuvjA~sT!6|pO5y6tUc)@7033j4`eWbr1;L}@*FcF4kkIi_2N5xPc2U6t)>()c=hOOY0YoOHz zu|`hwQA)0v{y>!hx|x$x6))}<8?xw&ifR+->41m>g}>Fg2*JBN=Jy<@JvD0N_Vo=VF^5Qx%3%7pnJ;e00HJErafZu$Q86 zssUvOJRCj6!bEUQI@Mo@ugCN&FHJ@NS>Y!n{+y{vhim*nv=0g_kM*o*?pY&QZhl&S zxq6uVx-?|M9T$6C8nh5Gu$#~G@`cnCQ2Y3)m1Wfm9~UDXli5&c%6HJs!HKGwhv?Zk zXuzc#{&J(ASA^6~Nq6YiWXx$4u3g9oi&52XOZm7hfww*|xonPj1VN%ZO;H3)y}{C{Q2$4Q7+Xw#Wimr9e~9+dNU z5O;rz%b;mXC!_`_Q`R&6!({pW-6pF!fIWL>|KoR%-js%>UT0nPGuqId@WKlGC4!u6 z`SB+txnC5tbbJcqWe|_4iK%R?tE|{zj>!r5v&X|q<8HO!3hgPW3rUjbXg=3wkvAlC zB~87<@o9rfb@&^wPaFDu>eo%FSB-I$^rxKAC?1zX2atfHoUaX_i?07}m9HmmFgqiV|AgkPLNwQd*dDC~>kZ^! zgo<%m!+y95^v7=bOm4RslXYcVta4b5LG-XbYLn#3|2 zgPL>+VQUk$GP&HJ6XkBeud4X;?!h^0wnmFhKE}-cX?eJ|1S2Ra^TQP@mRHXN% z&&%Go{SohkXDA!ZNuEWo>Fft~YVs5i98dXyEH5_ofX=35cq-Kc$`j-g`d6F!3lRFz zgp&|2-}Uv+2l?5u*EG5brv@zny-KVo!en7~f)=!5Ojmc4j+KcZ z>R;PoMo@DCVCzs}5#2ScO4V-GA4p7SUK)P!PKf)v^Om4}Os?>V4ds{U$3C>zpIuj5 z@F%`><-)CqHU6*a5pq5nG=(zH&fRvcp_m=anekjdd8s>~$j=o0YNkOW!_AU45v?)1&xIPbW z)s(^?8}VWJz~1JaX=B|c2IjYnWFBS-Cdu`AB&(NISrw(;nc-NHy524-_?jG6C|Mdc z61BN905CBc#iYAW@QB%tjY9MGSUNVp-DmF`C0xIRNF6Z8XG9!cj=fjfv75@mlvfJK zNZc@?@t^g4QD5cZafUDUK4L3cv1LclzD=XNipoA06t$p4qc|&ic3=_Y9+ijS-F(fG zmfM*-QQ*TWqxpsJLDS=_2SU`l46zH^T0oxamt>a-Tb@bW7oxDgiApjx)(x1xP4OV_)=J*}yZfvM zKZ2Vt=yU=?$FaI8YFAvVxJwRb0Xu0{>hPt{>N`pL^V}|BUewVLJNyYC>oz<<05*| zD6PUbN5%5^ft{fRHg1OOxMbz!g9Ufqwm{EB>h9Vh=R!FwC6st~X>0{|)2IIJbza7# zy9XXKILTAVmgt_Zv5Yz#_ zU8?Cv>CG7Ro{X3my_bwjFYFxmh_aMm3Lr-!+lb|@3BdCJpn+X0KGU*%<~04jF6E5+ zu`C&b^6f8W+fxIeAidopUp(29?c-B`^om3@hyLz|ox1sjY3$sOEXckJr)C4z0ir)H zeIEI#X`z&F&64d}?!=^c*z~v*tN&=LUc4?SA&UQHSaRlDSTm6upv!2FI>H<&0DuHN z28)}3&7)>jR^Mo^qk%xtt@4K<6-F+Zb?`lotusq}C|%hE&VJDqzJ=>oDoSNu4@al* zmQPjcyPcw~ZJIEv>WCc=6s&g&ju*mH{I2%gaI|-9mujjb5nhxVHqwjlssxa% zU;l9}NuLUTTeHXTIPtI~^O-EGAOh5?mYM+NZDoFr^I?4ep0N@6i$nE8LiQtQP1bM~ z{WV8R2#?XYPppw;$wa@vh@x|`na=F`dW#Ar>8oGtS~@xT{B*%yjwnS(gh(28=o$t? z79O-dE#oII#tZc>Ky}i>vndEKpA_ggv&(p_h4dwN_<#$+j$DX`Bt0!oz(Yr$Ut&e4TfPapJVkLl~+ zE)WY(*ibWLy2Y> ziv2x|Zs>${LCcOH3`zIEZw@L>PbmS)?gK(OKg4vQRsuk|U#r{A;(sdlt2pvtzHSc? zK=>edJo4`L3xWAu+EN26yGZ*r`Q^C5OrNaPz93*`6f4Z-{|vYP`SIVP8fiIA-{U!ldn*3!)~Dm)`6_`Kcksz&2VUgSQr9!Z0qqK! z>QYNE{kP;@096dO9k<(14@oARWVy?Bo)YPUo6Hp5A99AEo#{sr(-M5%Ba&-~ucqo5 zP5TrTs=Ce0L@DWKzbW>KT=ZCB{qfn0Gq0VA@Bcs*L7#4Z2qG1gfr9??NxwcBN_D)3 zUm@K%bg9QOhg3wKR=A~*d7}uVUs>-G$NwD+Bf~=xCpUiKg*QuOg^sRBn$|w3@Tr~q2#O9KilB7Gry$0Hr1w6l+!77mm{{5 zwYsxi1GxU(@E=x#61kSd8zH~@tl#^iqi5QattTNfS40kb=e4|*lI1Fr7T;-)pRbm$ zW0xQyO;iFQM{0PfJnJG4N(CS^dvlLIS$IHZu#Rz5gv-SdwFVT*PU94)fqb zoY*VIt=tKRMu9#D1NuGDrJ4ugc-0Xf^BzwW?z;t8t z8uO(M!n>T7 zS)Zihd|)cphKT%z1O(jWFR8%)tiS)2x@Uj=f&TS5{@=`v`2bXd{3;d}`7r7=aoC3o zGyQ!cbd>SQghyF2cm1!+?oN>;`Jr&{d?(J=jei}nHRhe-l#!~oXrTMFc%!rca$%i2 zqs$Sed~C7lu5=GJ%!+pee=1^wZ5#+i9BEH$>nu1^st%i+8th8W?d<{45}<$+d&S2y;k+j?4bD)xbLIPk8Dn`b&7cw;W5E|nxdt+6*^}XJKeka&`|;!zK-md%3_y@ z?TiW39w%}27g*e>Ok5-vpU}9vU3FURn93o*;z2V2fS zVOIKp9PFMzfbqo)z@QTKjRm7)Lb%WVOd&!!{oI`ipmdfLmX!TsAw$ZEvXY`+NdT5y z2JnWY2pB3IfHeiUu>0|N-1WV{KsDVU9*4udg!#8-w-`}NFW4$c?-$e&j;(=r5>^m< z{T8SDrVZP{Je`}`>@KmXVR#+xE~cC<$wmRy_&%v6%D5^In*ok`y%co~NehzarR}Lo zPDYskHS)Lj*WDG-dCkduk!l=lLAwt?=*F(eT$s|%hO_w1a=1`v>R~o6^%ciQtt0@Q z{ml;neAgDB`aQ;-@GkBv;`<60sEu-f0+~P+&0Ze9W&3J_Cj$@`dn9fcu<3EN#D^wQJ;d^PcLDF|_p44& zSHLSj<+DF(8-JDnz}`=e*_-;;GFF$bp5X(0=@Xp$d*`8IiZz=x`1))iN23#eU8F5^ z3xeFCUr+D1?ciBwEi4?lDt@1PDAd~RtCb8cEt}zp5fQ(MUUL9S;l(^TTBty4;oV63 zn1?z~k%hv*=$ZMeCi)+w0sO6Y_>uu%^LIMCR!KsCUQ>Bk=iGB$tduK)(Qn5W8!XHM zxTh#wlJr)-_wV253ISY0` zemY&ZeTRSIU>}>GnO2@J&HH+mQJpta!qn=%>w`I-5e{{J-zXSSw;gvT!(XMEzow>jxDob6i=H7g$54)UrIJzbGnl+QW1dEkA$ET|z^ zCop{r(Oc7}9b^EukeppZM12Rv7{2ZM6B*ia9O-YD522OJ{tSB5_K}U|IP4iM|6%8H zN}B`Ei0ph3yFf|gP}&BlfKLBqZcgfXLV*fU@vU75HVFP5bTD$FT0HX6F&QrbV9qD& zIQFjq|JSDh%E(+_A_RWw`Tg?b|`^G&X8?Ls(uoAS&Y!2 z12uLc6f``%^Pu{YL7bY3{HF&uBxXdKKF=4Kac`=EjrXv}u3Jfldv#d?2dixY<9PI! z`;~P`Ae1BYwkFO>UAiplgrH!g41nBj(_AEaEK7@PwjC9!Jw3qWhWDes_(0*oi=oe7 zg>%G`3pb*%{hwc+9v?n!?Gz>+7( zH+YQ^uo(%Gd4QTVbOMY4I!(-Ug*_RVVS!bUEjSQU`KCIc3*h`<=rI*w&kB8hZptaY zI;OXpXL$|mAM91ighulyB*t#J;Rt2tbjaJ%L~eUsth`w5bG5T`<=H;oPNqd09NmX{ zF?geAh*Q{vz6SkyLkI`Ey^U1K$deiEcRp~7IZmuFkw_3~Fvx}iOp6$6*rg-bDf3uR z@zQjU!=y~Y-U?~<9&bNTfGSmqVMcL8R=_Ho@WR^=^-2SkcQ#Yrfc}HXKQWT`=PuyS z_kTqAfMiy~)Sr86%@{wy2am_UgQ_Z+(YzE-VyQj2imPBxwah7_OdsUmd?jeMC_mS`H*;AVOC!fktYE@ zz`e&qLX2(Y8d{gx0hQF~UB_1)%~A&&!VSVvy~OAYTp}#87NY%SHR&d0M6T~KGH-t? zq&Yw!=odqVCcOq&nr_&4kQOY{8Rtag4-_FbKmgKKi2nM?-XpU?#fQde0lUv073~}A z?cj=1&xP}}o=>C+VtpePhJ2n24(3P-KGz7YK_p30z0OR5GM>8&ia_S;?c|f`Xr!-J zp}Tyh&A2VBdh88GKe%1sa07xwPY>}bDiehY5C6LIC-7! z?dz-P`|h8efP5{ta9H`lo@;seK(Vxp3wwU15xgqSK)$iBU5#smo;`I>5Ac`Xo1)%& zZt(S>_$mSqDb^XDK6uMV(?hWGtj$cTmvCyU@{DG) z`h2%>Vg>=&^>tiAqh z;9whacv${C7cpo-nCbTcBc>j35O>RRLP-J@mf?e8Fl&CpW-154$#rQ9Ed%}VyU zLsNitOV^u1k(=$d!|S|sK%bNl#9^b&xGOTCWsBn%9>|kvmPmiXJ$ZVn;+{Dd|5ban z0&$u7aFt_r_Jy-24#w8E16Zcl{ZD18d!1NLCSulX21!Lv;^v}EMqDEwt#hHbdXB#N z(A&&CUIp)MAgEM-oFCV3KgfF#XkFStFvyy*S9g2+h}1oo>{b(2mMurrLq>}yS1i)$ zwY5s2z`%0+$jKJLp|Mvw#~q!IQSR6f58kj{4W4;@l9pL1k|0KL{J-#uOJLzT6xly)K?T2!+18Ee7jORa$lV~qA>+5=iOBqvIA zxW-}k3Sy2f%irkm$y4O`@|q;vqMNxa-#=Rxz0KLJV%y)Wlv8sU>X3$SxDu$o%bb`B zjn8#>E}?R6aRN(JQ>2?R;;Q{rJ0fdUd_fH1E{b!{4izzb3Ec@U3__Sz8#XlR)nvzi zBrOrb^~m?v+WM;|We*N(w`aPUnSd8{@^M$;@J!$S5A}#xeBglr{bT8>fkKt(@v9W< zqjq|0@;2%q(EYo{MXiy_(SEa$3CTQ(Kqef;KdrjoPbckf9si)Dd;?IXSfCcve+GX$ z8A9~$GO*{;-1ZSIQdQXD$nQNYWOL#<4eNMt4Gxh*dD~T}C>syt74q7c@~$a3WNlc^ zEA77fV2>Ir%6Im7be{ma}h z!K*4(&rs8hOIReNP5pNun6ZC=&X~<~R&za_ZcNzw;P9g8k&C z9Rx_gD8nYYP)i}x0du|qt}-TV7~e~cSyQ3=Q}ST4O@_HWnMOEd%M;s?zFwi)y15od zx^c2SD|G^_b}!4)YSi$zjkz!xNf1mLWPtbK(Oye|Q-cBKK4PCzhSnEm?zSL!`Lcg)%g$TQN`SOoW-dIu*&LrpJV+#)JcI|7e zC`>ZclQ!|?;9ki4qBNvL`M`jAVf~AL%Jjeq2v|F)`e610lkK_L|cUV^GdoN2?+(J>egrmOXbJMa8& z2u%p(T40rp4BYM{*pI-=TM44fZ|?~L6d#wcS2yWu8g@3oqnJ2eY!J&6PL4Fxjq?zv z5Jv#k*HEqBX)n)ric{X)pe}dXl{LA!*W<3T9AsE!84Mr`axfEiGt}1PSWKI`zB7|^ zZC$4^`T{zzE8MDsNspo%LJ3fA6h@D!KMFQgkO~q~z2@&)PD)Bq<#I?fRXAd{bKw^r zvdeMHm=MqWVk-ZfG}FSY3VzKoxuqOx!QOVQL*GtJ%u*m}@c6ciGhXa4iUW2BpxTB> z_qM^U9kYOPVV7k$-$Mf^D{rK-&>C(LtcS}}mgYIe(K9=dn?HUT&ez0cMOpRc?1&XZ zxy+P_FkfSBglt1~RYM3s>KU@)39V` z{x_Q|SFD2%)-2XA*JU?He76={V)i)Cs`K2ikLEi)UIxpV%7oIWeYUeRjfP{Wsd}rP z&3HN??bm}5nkr1{ZY%xRTRuHAJk$hPHx<6<32B5tt`B}Qf2X@k0csgfqWDV95*Wiaq#8n z=J4ErM0Lp0MO_;QM#3=S@GJM7Hnn(-cp}OkWi1T-NH;(4@pJ169RDq_@87b@KfdEh zOUi?=nRt`Fb?3c9K_Dl~*`;$3z5DpIgLFR!QL?{%j0V^#2zESWI#k9>dnJ>fBQ_D| zn+@xYg5!>Lc5>4P3e!)+BoJFEla32huDp_o1+z9%*#apr{$TiFF`aRa0BCTB>)*c0X|6iEj#!WGPDkqo%xIu+Q5~dKhp?$udBe8{Cx$7zr2a(} zn=sibIK07_);|&o(zU#B?i@=l{cb?YxZ5_CEvo#K9yyL!_KgJCLBq>i^FsXj z-Lq3->^A~GZTRn{dpv&jFi&`U#&qy?=ALnHQo2t=(Ob>+=JBra95t!^aAeln*Jha*XcpT$FU&xq24<{QP&&x)OYvy{u$#aUi&I0i)o}z{fDH!wBIL zs#~UPA6wZNE?JFBga>%C6Y6{AYn)BG(9uM{o?BW`9XfX<-SV3>$9Ny50jquK4M(C> zUO6)PZ3P*f;rF2DFW9RCT&|HG&q*egSPSKc39TV&zbiQMhf zLXO`8Hd+Asg%S%ukiBqUMjeMMDB#jak2k@%eb!TP@ZLD&bVWsCL_AUnX6%V~0iZ88 zV&Og_>|rSWgo3c|4kh^4HbN^z0Sql2*pYb?*h)bq5B(s^6)?=V^i48YFTiQvP8HE2 z{|oKEp#J}}`8Hq+d*=mEZ83YlXu-CKTtf9IOS(k&P%dS;(m^lN>}7UpFt~CJSS|J6 z{#Y_8Un&6r5_B793B#Qms{M_wfk|N-M~KBN3LAJIc)9>6L1}k@bQOx`*%=hgfIdNe zXo6ANA%5wgTEq=*0MxVhXU)O?>?^nb=v8)ZjWlUC$gf=m{^e(9R>A{9f9Wg9Upw}F zEmKf||F(_v@!sk`P9SFe zzvwvxo=AGxPF=@rsx(@~n7q(;^+;~6jbklXWS$z!U`Q(f0s^`BegyjOhPOtD0rdz& zGS!?tp%et|ojc3os|T*#>XG?>-?u)JPdJiFR@Mg03V-{0&G$vW)ko_m Xs=8#YrmSgciq@0WVh@1aqOt!4V&ebW diff --git a/mango/agent/core.py b/mango/agent/core.py index 36159df..fe55db8 100644 --- a/mango/agent/core.py +++ b/mango/agent/core.py @@ -19,7 +19,7 @@ @dataclass(frozen=True) class AgentAddress: - addr: Any + protocol_addr: Any aid: str @@ -42,12 +42,12 @@ def clock(self) -> Clock: def addr(self): return self._container.addr - def register_agent(self, agent, suggested_aid): - return self._container.register_agent(agent, suggested_aid=suggested_aid) + def register(self, agent, suggested_aid): + return self._container.register(agent, suggested_aid=suggested_aid) - def deregister_agent(self, aid): + def deregister(self, aid): if self._container.running: - self._container.deregister_agent(aid) + self._container.deregister(aid) async def send_message( self, @@ -94,6 +94,8 @@ def addr(self): Returns: _type_: AgentAddress """ + if self.context is None: + return None return AgentAddress(self.context.addr, self.aid) async def send_message( @@ -440,7 +442,7 @@ async def shutdown(self): if not self._stopped.done(): self._stopped.set_result(True) - self.context.deregister_agent(self.aid) + self.context.deregister(self.aid) try: # Shutdown reactive inbox task self._check_inbox_task.remove_done_callback(self._raise_exceptions) diff --git a/mango/agent/role.py b/mango/agent/role.py index a8ff2a9..6c2698a 100644 --- a/mango/agent/role.py +++ b/mango/agent/role.py @@ -298,7 +298,7 @@ def add_role(self, role: Role): :param role: the Role """ - role.bind(self) + role._bind(self) self._role_handler.add_role(role) # Setup role @@ -463,7 +463,7 @@ def __init__(self) -> None: """ self._context = None - def bind(self, context: RoleContext) -> None: + def _bind(self, context: RoleContext) -> None: """Method used internal to set the context, do not override! :param context: the role context diff --git a/mango/container/core.py b/mango/container/core.py index 2de6992..07a571c 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -29,7 +29,6 @@ def __init__( addr, name: str, codec, - loop, clock: Clock, copy_internal_messages=False, mirror_data=None, @@ -41,29 +40,28 @@ def __init__( self._copy_internal_messages = copy_internal_messages self.codec: Codec = codec - self.loop: asyncio.AbstractEventLoop = loop # dict of agents. aid: agent instance - self._agents: dict = {} + self._agents: dict[str, Agent] = {} self._aid_counter: int = 0 # counter for aids - self.running: bool = True # True until self.shutdown() is called - self._no_agents_running: asyncio.Future = asyncio.Future() - self._no_agents_running.set_result( - True - ) # signals that currently no agent lives in this container + self.running: bool = False # True until self.shutdown() is called + self._no_agents_running: asyncio.Future = None # inbox for all incoming messages - self.inbox: asyncio.Queue = asyncio.Queue() + self.inbox: asyncio.Queue = None # task that processes the inbox. - self._check_inbox_task: asyncio.Task = asyncio.create_task(self._check_inbox()) + self._check_inbox_task: asyncio.Task = None # multiprocessing self._kwargs = kwargs - if mirror_data is not None: + self._mirror_data = mirror_data + + # multiprocessing + if self._mirror_data is not None: self._container_process_manager = MirrorContainerProcessManager( - self, mirror_data + self, self._mirror_data ) else: self._container_process_manager = MainContainerProcessManager(self) @@ -110,7 +108,7 @@ def _reserve_aid(self, suggested_aid=None): self._aid_counter += 1 return aid - def register_agent(self, agent: Agent, suggested_aid: str = None): + def register(self, agent: Agent, suggested_aid: str = None): """ Register *agent* and return the agent id @@ -126,7 +124,7 @@ def register_agent(self, agent: Agent, suggested_aid: str = None): self._agents[aid] = agent agent._do_register(self, aid) logger.debug("Successfully registered agent;%s", aid) - return aid + return agent def include(self, agent: A, suggested_aid: str = None) -> A: """Include the agent in the container. Return the agent for @@ -139,10 +137,10 @@ def include(self, agent: A, suggested_aid: str = None) -> A: Returns: _type_: the agent included """ - self.register_agent(agent, suggested_aid=suggested_aid) + self.register(agent, suggested_aid=suggested_aid) return agent - def deregister_agent(self, aid): + def deregister(self, aid): """ Deregister an agent @@ -303,6 +301,18 @@ def dispatch_to_agent_process(self, pid: int, coro_func, *args): self._container_process_manager.dispatch_to_agent_process(pid, coro_func, *args) async def start(self): + self.running: bool = True # True until self.shutdown() is called + self._no_agents_running: asyncio.Future = asyncio.Future() + self._no_agents_running.set_result( + True + ) # signals that currently no agent lives in this container + + # inbox for all incoming messages + self.inbox: asyncio.Queue = asyncio.Queue() + + # task that processes the inbox. + self._check_inbox_task: asyncio.Task = asyncio.create_task(self._check_inbox()) + """Start the container. It totally depends on the implementation for what is actually happening.""" for agent in self._agents.values(): agent.on_start() diff --git a/mango/container/external_coupling.py b/mango/container/external_coupling.py index 0981c10..f5184ce 100644 --- a/mango/container/external_coupling.py +++ b/mango/container/external_coupling.py @@ -1,4 +1,3 @@ -import asyncio import logging import time from dataclasses import dataclass @@ -55,7 +54,6 @@ def __init__( *, addr: str, codec: Codec, - loop: asyncio.AbstractEventLoop, clock: ExternalClock = None, **kwargs, ): @@ -74,7 +72,6 @@ def __init__( addr=addr, name=addr, codec=codec, - loop=loop, clock=clock, **kwargs, ) @@ -106,9 +103,9 @@ async def send_message( meta["sender_id"] = sender_id meta["sender_addr"] = self.addr meta["receiver_id"] = receiver_addr.aid - meta["receiver_addr"] = receiver_addr.addr + meta["receiver_addr"] = receiver_addr.protocol_addr - if receiver_addr.addr == self.addr: + if receiver_addr.protocol_addr == self.addr: receiver_id = receiver_addr.aid meta.update({"network_protocol": "external_connection"}) success = self._send_internal_message( @@ -134,7 +131,7 @@ async def _send_external_message(self, addr, message) -> bool: self.message_buffer.append( ExternalAgentMessage( time=time.time() - self.current_start_time_of_step + self.clock.time, - receiver=addr.addr, + receiver=addr.protocol_addr, message=encoded_msg, ) ) diff --git a/mango/container/factory.py b/mango/container/factory.py index a1d352a..8b4437e 100644 --- a/mango/container/factory.py +++ b/mango/container/factory.py @@ -1,4 +1,3 @@ -import asyncio import logging from typing import Any @@ -35,7 +34,6 @@ def create_mqtt( return MQTTContainer( client_id=client_id, broker_addr=broker_addr, - loop=asyncio.get_running_loop(), clock=clock, codec=codec, inbox_topic=inbox_topic, @@ -55,9 +53,7 @@ def create_external_coupling( if clock is None: clock = ExternalClock() - return ExternalSchedulingContainer( - addr=addr, loop=asyncio.get_running_loop(), codec=codec, clock=clock, **kwargs - ) + return ExternalSchedulingContainer(addr=addr, codec=codec, clock=clock, **kwargs) def create_tcp( @@ -87,7 +83,6 @@ def create_tcp( return TCPContainer( addr=addr, codec=codec, - loop=asyncio.get_running_loop(), clock=clock, copy_internal_messages=copy_internal_messages, **kwargs, diff --git a/mango/container/mqtt.py b/mango/container/mqtt.py index 1b87da5..fdc372e 100644 --- a/mango/container/mqtt.py +++ b/mango/container/mqtt.py @@ -137,7 +137,9 @@ async def start(self): # callbacks to check for successful connection def on_con(client, userdata, flags, reason_code, properties): logger.info("Connection Callback with the following flags: %s", flags) - self.loop.call_soon_threadsafe(connected.set_result, reason_code) + asyncio.get_running_loop().call_soon_threadsafe( + connected.set_result, reason_code + ) mqtt_messenger.on_connect = on_con @@ -203,7 +205,9 @@ def on_con(client, userdata, flags, reason_code, properties): # set up subscription callback def on_sub(client, userdata, mid, reason_code_list, properties): - self.loop.call_soon_threadsafe(subscribed.set_result, True) + asyncio.get_running_loop().call_soon_threadsafe( + subscribed.set_result, True + ) mqtt_messenger.on_subscribe = on_sub @@ -264,7 +268,9 @@ def on_discon(client, userdata, disconnect_flags, reason_code, properties): self.mqtt_client.on_disconnect = on_discon def on_sub(client, userdata, mid, reason_code_list, properties): - self.loop.call_soon_threadsafe(self.pending_sub_request.set_result, 0) + asyncio.get_running_loop().call_soon_threadsafe( + self.pending_sub_request.set_result, 0 + ) self.mqtt_client.on_subscribe = on_sub @@ -283,7 +289,9 @@ def on_message(client, userdata, message): # update meta dict meta.update(message_meta) # put information to inbox - self.loop.call_soon_threadsafe(self.inbox.put_nowait, (0, content, meta)) + asyncio.get_running_loop().call_soon_threadsafe( + self.inbox.put_nowait, (0, content, meta) + ) self.mqtt_client.on_message = on_message self.mqtt_client.enable_logger(logger) @@ -393,7 +401,9 @@ async def send_message( message = content if not hasattr(content, "split_content_and_meta"): message = MangoMessage(content, meta) - self._send_external_message(topic=receiver_addr.addr, message=message) + self._send_external_message( + topic=receiver_addr.protocol_addr, message=message + ) return True def _send_external_message(self, *, topic: str, message): @@ -433,13 +443,13 @@ async def subscribe_for_agent(self, *, aid: str, topic: str, qos: int = 0) -> bo await self.pending_sub_request return True - def deregister_agent(self, aid): + def deregister(self, aid): """ :param aid: :return: """ - super().deregister_agent(aid) + super().deregister(aid) empty_subscriptions = [] for subscription, aid_set in self.additional_subscriptions.items(): if aid in aid_set: diff --git a/mango/container/protocol.py b/mango/container/protocol.py index 56b4f58..f35c59b 100644 --- a/mango/container/protocol.py +++ b/mango/container/protocol.py @@ -16,18 +16,16 @@ class ContainerProtocol(asyncio.Protocol): """Protocol for implementing the TCP Container connection. Internally reads the asyncio transport object into a buffer and moves the read messages async to the container inbox.""" - def __init__(self, *, container, loop, codec): + def __init__(self, *, container, codec): """ :param container: - :param loop: """ super().__init__() self.codec = codec self.transport = None # type: _SelectorTransport self.container = container - self._loop = loop self._buffer = bytearray() def connection_made(self, transport): diff --git a/mango/container/tcp.py b/mango/container/tcp.py index 9b188f9..e17ba35 100644 --- a/mango/container/tcp.py +++ b/mango/container/tcp.py @@ -164,7 +164,6 @@ def __init__( *, addr: tuple[str, int], codec: Codec, - loop: asyncio.AbstractEventLoop, clock: Clock, **kwargs, ): @@ -178,25 +177,27 @@ def __init__( super().__init__( addr=addr, codec=codec, - loop=loop, name=f"{addr[0]}:{addr[1]}", clock=clock, **kwargs, ) self.server = None # will be set within start - self.running = True + self.running = False + + async def start(self): self._tcp_connection_pool = TCPConnectionPool( - loop, - ttl_in_sec=kwargs.get(TCP_CONNECTION_TTL, 30), - max_connections_per_target=kwargs.get(TCP_MAX_CONNECTIONS_PER_TARGET, 10), + asyncio.get_running_loop(), + ttl_in_sec=self._kwargs.get(TCP_CONNECTION_TTL, 30), + max_connections_per_target=self._kwargs.get( + TCP_MAX_CONNECTIONS_PER_TARGET, 10 + ), ) - async def start(self): # create a TCP server bound to host and port that uses the # specified protocol - self.server = await self.loop.create_server( - lambda: ContainerProtocol(container=self, loop=self.loop, codec=self.codec), + self.server = await asyncio.get_running_loop().create_server( + lambda: ContainerProtocol(container=self, codec=self.codec), self.addr[0], self.addr[1], ) @@ -217,7 +218,7 @@ async def send_message( :param receiver_id: The agent id of the receiver :param kwargs: Additional parameters to provide protocol specific settings """ - protocol_addr = receiver_addr.addr + protocol_addr = receiver_addr.protocol_addr if isinstance(protocol_addr, str) and ":" in protocol_addr: protocol_addr = protocol_addr.split(":") elif isinstance(protocol_addr, (tuple, list)) and len(protocol_addr) == 2: @@ -245,7 +246,7 @@ async def send_message( if not hasattr(content, "split_content_and_meta"): message = MangoMessage(content, meta) success = await self._send_external_message( - receiver_addr.addr, message, meta + receiver_addr.protocol_addr, message, meta ) return success @@ -268,7 +269,7 @@ async def _send_external_message(self, addr, message, meta) -> bool: protocol = await self._tcp_connection_pool.obtain_connection( addr[0], addr[1], - ContainerProtocol(container=self, loop=self.loop, codec=self.codec), + ContainerProtocol(container=self, codec=self.codec), ) logger.debug("Connection established to addr; %s", addr) diff --git a/mango/express/api.py b/mango/express/api.py index f60632d..aeb57c7 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -51,7 +51,7 @@ async def __aenter__(self): container = container_list[container_id] actual_agent = agent_tuple[0] agent_params = agent_tuple[1] - container.register_agent( + container.register( actual_agent, suggested_aid=agent_params.get("aid", None) ) self.__activation_cm = activate(container_list) @@ -126,27 +126,26 @@ async def after_start(self, container_list, agents): def activate(*containers: Container) -> ContainerActivationManager: - """Create and return an async activation context manager. This can be used with the - `async with` syntax to run code while the container(s) are active. The containers - are started first, after your code under `async with` will run, and at the end - the container will shut down (even when an error occurs). + """ + Create and return an async activation context manager. + This can be used with the `async with` syntax to run code while the container(s) are active. + The containers are started first, after your code under `async with` will run, and + at the end the container will shut down (even when an error occurs). Example: - ```python - # Single container - async with activate(container) as container: - # do your stuff - # Multiple container - async with activate(container_list) as container_list: - # do your stuff - ``` + .. code-block:: python - Args: - containers (Container | list): a single container or a list of containers + # Single container + async with activate(container) as container: + # do your stuff - Returns: - ContainerActivationManager: The context manager to be used as described + # Multiple container + async with activate(container_list) as container_list: + # do your stuff + + :return: The context manager to be used as described + :rtype: ContainerActivationManager """ if isinstance(containers[0], list): containers = containers[0] @@ -159,22 +158,24 @@ def run_with_tcp( addr: tuple[str, int] = ("127.0.0.1", 5555), codec: None | Codec = None, ) -> RunWithTCPManager: - """Create and return an async context manager, which can be used to run the given + """ + Create and return an async context manager, which can be used to run the given agents in `num` automatically created tcp container. The agents are distributed evenly. - Example: - ```python - async with run_with_tcp(2, Agent(), Agent(), (Agent(), dict(aid="my_agent_id"))) as c: - # do your stuff - ``` + .. code-block:: python - Args: - num (int): number of tcp container - agents (args): list of agents which shall run + async with run_with_tcp(2, Agent(), Agent(), (Agent(), dict(aid="my_agent_id"))) as c: + # do your stuff - Returns: - RunWithTCPManager: the async context manager to run the agents with + :param num: number of tcp container + :type num: int + :param addr: the starting addr of the containers, defaults to ("127.0.0.1", 5555) + :type addr: tuple[str, int], optional + :param codec: the codec for the containers, defaults to None + :type codec: None | Codec, optional + :return: the async context manager to run the agents with + :rtype: RunWithTCPManager """ return RunWithTCPManager(num, agents, addr=addr, codec=codec) @@ -189,16 +190,18 @@ def run_with_mqtt( agents in `num` automatically created mqtt container. The agents are distributed according to the topic - Args: - num (int): _description_ - agents (args): list of agents which shall run, it is possible to provide a tuple - (Agent, dict), the dict supports "aid" for the suggested_aid and "topics" as list of topics the agent - wants to subscribe to. - broker_addr (tuple[str, int], optional): Address of the broker the container shall connect to. Defaults to ("127.0.0.1", 5555). - codec (None | Codec, optional): The codec of the container - - Returns: - RunWithMQTTManager: _description_ + The function takes a list of agents which shall run, it is possible to provide a tuple + (Agent, dict), the dict supports "aid" for the suggested_aid and "topics" as list of topics the agent + wants to subscribe to. + + :param num: _description_ + :type num: int + :param broker_addr: Address of the broker the container shall connect to, defaults to ("127.0.0.1", 1883) + :type broker_addr: tuple[str, int], optional + :param codec: _description_, defaults to None + :type codec: None | Codec, optional, The codec of the container + :return: the async context manager + :rtype: RunWithMQTTManager """ return RunWithMQTTManager(num, agents, broker_addr=broker_addr, codec=codec) @@ -208,19 +211,22 @@ class ComposedAgent(RoleAgent): def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAgent: - """Create an agent composed of the given `roles`. If a container is provided, + """ + Create an agent composed of the given `roles`. If a container is provided, the created agent is automatically registered with the container `register_in`. - Args: - *roles Role: The roles which are added to the agent - register_in (None | Container): container in which the created agent is registered, + + :param register_in: container in which the created agent is registered, if provided + :type register_in: None | Container + :return: the composed agent + :rtype: ComposedAgent """ agent = ComposedAgent() for role in roles: agent.add_role(role) if register_in is not None: - register_in.register_agent(agent) + register_in.register(agent) return agent @@ -230,7 +236,8 @@ def handle_message(self, content, meta: dict[str, Any]): def sender_addr(meta: dict) -> AgentAddress: - """Extract the sender_addr from the meta dict. + """ + Extract the sender_addr from the meta dict. Args: meta (dict): the meta you received @@ -247,14 +254,15 @@ def sender_addr(meta: dict) -> AgentAddress: ) -def addr(protocol_part: Any, aid: str) -> AgentAddress: - """Create an Address from the topic. +def addr(protocol_addr: Any, aid: str) -> AgentAddress: + """ + Create an Address from the topic. Args: - protocol_part (Any): protocol part of the address, e.g. topic for mqtt, or host/port for tcp, ... + protocol_addr (Any): the container part of the addr, e.g. topic for mqtt, or host/port for tcp, ... aid (str): the agent id Returns: AgentAddress: the address """ - return AgentAddress(protocol_part, aid) + return AgentAddress(protocol_addr, aid) diff --git a/readme.md b/readme.md index cc13051..29eadf6 100644 --- a/readme.md +++ b/readme.md @@ -1,8 +1,18 @@ -# mango +

+ +![logo](docs/source/_static/Logo_mango_ohne_sub.svg#gh-light-mode-only) +![logo](docs/source/_static/Logo_mango_ohne_sub_white.svg#gh-dark-mode-only) + +

[PyPi](https://pypi.org/project/mango-agents/) | [Read the Docs](https://mango-agents.readthedocs.io) | [Github](https://github.com/OFFIS-DAI/mango) | [mail](mailto:mango@offis.de) +![lifecycle](https://img.shields.io/badge/lifecycle-maturing-blue.svg) +[![MIT License](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/OFFIS-DAI/mango/blob/development/LICENSE) +[![Test mango-python](https://github.com/OFFIS-DAI/mango/actions/workflows/test-mango.yml/badge.svg)](https://github.com/OFFIS-DAI/mango/actions/workflows/test-mango.yml) + + **Note:** _This project is still in an early development stage. We appreciate constructive feedback and suggestions for improvement._ @@ -161,9 +171,9 @@ class HelloWorldAgent(Agent): async def run_container_and_two_agents(first_addr, second_addr): first_container = create_tcp_container(addr=first_addr) second_container = create_tcp_container(addr=second_addr) - - first_agent = first_container.include(RepeatingAgent()) - second_agent = second_container.include(HelloWorldAgent()) + + first_agent = first_container.register(RepeatingAgent()) + second_agent = second_container.register(HelloWorldAgent()) async with activate(first_container, second_container) as cl: await second_agent.greet(first_agent.addr) diff --git a/tests/integration_tests/test_distributed_clock.py b/tests/integration_tests/test_distributed_clock.py index 5fb6de2..82d42b9 100644 --- a/tests/integration_tests/test_distributed_clock.py +++ b/tests/integration_tests/test_distributed_clock.py @@ -17,8 +17,8 @@ async def setup_and_run_test_case(connection_type, codec): connection_type, init_addr, repl_addr, codec ) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register( DistributedClockManager( receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] ) diff --git a/tests/integration_tests/test_message_roundtrip.py b/tests/integration_tests/test_message_roundtrip.py index 82477da..41a2945 100644 --- a/tests/integration_tests/test_message_roundtrip.py +++ b/tests/integration_tests/test_message_roundtrip.py @@ -51,8 +51,8 @@ async def setup_and_run_test_case(connection_type, codec): init_target = repl_addr repl_target = init_addr - init_agent = container_1.include(InitiatorAgent(container_1)) - repl_agent = container_2.include(ReplierAgent(container_2)) + init_agent = container_1.register(InitiatorAgent(container_1)) + repl_agent = container_2.register(ReplierAgent(container_2)) repl_agent.target = addr(repl_target, init_agent.aid) init_agent.target = addr(init_target, repl_agent.aid) diff --git a/tests/integration_tests/test_message_roundtrip_mp.py b/tests/integration_tests/test_message_roundtrip_mp.py index 5b7596b..fc0153e 100644 --- a/tests/integration_tests/test_message_roundtrip_mp.py +++ b/tests/integration_tests/test_message_roundtrip_mp.py @@ -36,12 +36,12 @@ async def test_mp_simple_ping_pong_multi_container_tcp(): addr=repl_addr, ) await container_1.as_agent_process( - agent_creator=lambda c: c.include(PingPongAgent(), suggested_aid=aid1) + agent_creator=lambda c: c.register(PingPongAgent(), suggested_aid=aid1) ) await container_2.as_agent_process( - agent_creator=lambda c: c.include(PingPongAgent(), suggested_aid=aid2) + agent_creator=lambda c: c.register(PingPongAgent(), suggested_aid=aid2) ) - agent = container_1.include(PingPongAgent()) + agent = container_1.register(PingPongAgent()) async with activate(container_1, container_2) as cl: await agent.send_message( diff --git a/tests/integration_tests/test_single_container_termination.py b/tests/integration_tests/test_single_container_termination.py index a25a230..5536849 100644 --- a/tests/integration_tests/test_single_container_termination.py +++ b/tests/integration_tests/test_single_container_termination.py @@ -67,8 +67,8 @@ async def test_termination_single_container(): clock = ExternalClock(start_time=1000) c = create_ec_container(clock=clock) - receiver = c.include(Receiver()) - caller = c.include(Caller(receiver.addr, send_response_messages=True)) + receiver = c.register(Receiver()) + caller = c.register(Caller(receiver.addr, send_response_messages=True)) async with activate(c) as c: await asyncio.sleep(0.1) @@ -97,14 +97,14 @@ async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): connection_type, init_addr, repl_addr, codec ) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register( DistributedClockManager( receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] ) ) - receiver = container_ag.include(Receiver()) - caller = container_man.include( + receiver = container_ag.register(Receiver()) + caller = container_man.register( Caller( addr(repl_addr, receiver.aid), send_response_messages=True, @@ -132,14 +132,14 @@ async def distribute_ping_pong_test_timestamp( connection_type, init_addr, repl_addr, codec ) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register( DistributedClockManager( receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] ) ) - receiver = container_ag.include(Receiver()) - caller = container_man.include( + receiver = container_ag.register(Receiver()) + caller = container_man.register( Caller( addr(repl_addr, receiver.aid), send_response_messages=True, @@ -198,14 +198,14 @@ async def distribute_time_test_case(connection_type, codec=None): connection_type, init_addr, repl_addr, codec ) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register( DistributedClockManager( receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] ) ) - receiver = container_ag.include(Receiver()) - caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) + receiver = container_ag.register(Receiver()) + caller = container_ag.register(Caller(addr(repl_addr, receiver.aid))) async with activate(container_man, container_ag) as cl: assert receiver.scheduler.clock.time == 0 @@ -247,14 +247,14 @@ async def send_current_time_test_case(connection_type, codec=None): connection_type, init_addr, repl_addr, codec ) - clock_agent = container_ag.include(DistributedClockAgent()) - clock_manager = container_man.include( + clock_agent = container_ag.register(DistributedClockAgent()) + clock_manager = container_man.register( DistributedClockManager( receiver_clock_addresses=[addr(repl_addr, clock_agent.aid)] ) ) - receiver = container_ag.include(Receiver()) - caller = container_ag.include(Caller(addr(repl_addr, receiver.aid))) + receiver = container_ag.register(Receiver()) + caller = container_ag.register(Caller(addr(repl_addr, receiver.aid))) async with activate(container_man, container_ag) as cl: await tasks_complete_or_sleeping(container_man) diff --git a/tests/unit_tests/container/test_mp.py b/tests/unit_tests/container/test_mp.py index 742be68..1ab00f0 100644 --- a/tests/unit_tests/container/test_mp.py +++ b/tests/unit_tests/container/test_mp.py @@ -65,11 +65,11 @@ async def test_agent_processes_ping_pong(num_sp_agents, num_sp): for i in range(num_sp): await c.as_agent_process( agent_creator=lambda container: [ - container.include(MyAgent(), suggested_aid=f"process_agent{i},{j}") + container.register(MyAgent(), suggested_aid=f"process_agent{i},{j}") for j in range(num_sp_agents) ] ) - agent = c.include(MyAgent()) + agent = c.register(MyAgent()) # WHEN async with activate(c) as c: @@ -92,15 +92,15 @@ async def test_agent_processes_ping_pong_p_to_p(): aid_main_agent = "main_agent" c = create_tcp_container(addr=addr, copy_internal_messages=False) await c.as_agent_process( - agent_creator=lambda container: container.include( + agent_creator=lambda container: container.register( P2PTestAgent(aid_main_agent), suggested_aid="process_agent1" ) ) - main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) + main_agent = c.register(P2PMainAgent(), suggested_aid=aid_main_agent) # WHEN def agent_init(c): - agent = c.include(MyAgent(), suggested_aid="process_agent2") + agent = c.register(MyAgent(), suggested_aid="process_agent2") agent.schedule_instant_message( "Message To Process Agent", receiver_addr=AgentAddress(addr, "process_agent1"), @@ -122,10 +122,10 @@ async def test_async_agent_processes_ping_pong_p_to_p(): addr = ("127.0.0.2", 5811) aid_main_agent = "main_agent" c = create_tcp_container(addr=addr, copy_internal_messages=False) - main_agent = c.include(P2PMainAgent(), suggested_aid=aid_main_agent) + main_agent = c.register(P2PMainAgent(), suggested_aid=aid_main_agent) async def agent_creator(container): - p2pta = container.include( + p2pta = container.register( P2PTestAgent(aid_main_agent), suggested_aid="process_agent1" ) await p2pta.send_message(content="pong", receiver_addr=main_agent.addr) @@ -135,7 +135,7 @@ async def agent_creator(container): # WHEN def agent_init(c): - agent = c.include(MyAgent(), suggested_aid="process_agent2") + agent = c.register(MyAgent(), suggested_aid="process_agent2") agent.schedule_instant_message( "Message To Process Agent", AgentAddress(addr, "process_agent1") ) diff --git a/tests/unit_tests/core/test_agent.py b/tests/unit_tests/core/test_agent.py index 4c82a63..49d57fa 100644 --- a/tests/unit_tests/core/test_agent.py +++ b/tests/unit_tests/core/test_agent.py @@ -18,7 +18,7 @@ def handle_message(self, content, meta: dict[str, Any]): async def test_periodic_facade(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) + agent = c.register(MyAgent()) l = [] async def increase_counter(): @@ -40,8 +40,8 @@ async def increase_counter(): async def test_send_message(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) - agent2 = c.include(MyAgent()) + agent = c.register(MyAgent()) + agent2 = c.register(MyAgent()) async with activate(c) as c: await agent.send_message("", receiver_addr=agent2.addr) @@ -57,8 +57,8 @@ async def test_send_message(): async def test_send_acl_message(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) - agent2 = c.include(MyAgent()) + agent = c.register(MyAgent()) + agent2 = c.register(MyAgent()) async with activate(c) as c: await agent.send_message( @@ -77,8 +77,8 @@ async def test_send_acl_message(): async def test_schedule_message(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) - agent2 = c.include(MyAgent()) + agent = c.register(MyAgent()) + agent2 = c.register(MyAgent()) async with activate(c) as c: await agent.schedule_instant_message("", receiver_addr=agent2.addr) @@ -91,8 +91,8 @@ async def test_schedule_message(): async def test_schedule_acl_message(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) - agent2 = c.include(MyAgent()) + agent = c.register(MyAgent()) + agent2 = c.register(MyAgent()) async with activate(c) as c: await agent.schedule_instant_message( diff --git a/tests/unit_tests/core/test_container.py b/tests/unit_tests/core/test_container.py index 96c0c03..4108cad 100644 --- a/tests/unit_tests/core/test_container.py +++ b/tests/unit_tests/core/test_container.py @@ -20,7 +20,7 @@ async def test_register_aid_pattern_match(): suggested_aid = "agent12" # WHEN - actual_aid = c.register_agent(agent, suggested_aid) + actual_aid = c.register(agent, suggested_aid) # THEN assert actual_aid == "agent0" @@ -35,7 +35,7 @@ async def test_register_aid_success(): suggested_aid = "cagent12" # WHEN - actual_aid = c.register_agent(agent, suggested_aid) + actual_aid = c.register(agent, suggested_aid) # THEN assert actual_aid == suggested_aid @@ -49,7 +49,7 @@ async def test_register_no_suggested(): agent = LooksLikeAgent() # WHEN - actual_aid = c.register_agent(agent) + actual_aid = c.register(agent) # THEN assert actual_aid == "agent0" @@ -64,7 +64,7 @@ async def test_register_pattern_half_match(): suggested_aid = "agentABC" # WHEN - actual_aid = c.register_agent(agent, suggested_aid) + actual_aid = c.register(agent, suggested_aid) # THEN assert actual_aid == "agentABC" @@ -79,8 +79,8 @@ async def test_register_existing(): suggested_aid = "agentABC" # WHEN - actual_aid = c.register_agent(agent, suggested_aid) - actual_aid2 = c.register_agent(agent, suggested_aid) + actual_aid = c.register(agent, suggested_aid) + actual_aid2 = c.register(agent, suggested_aid) # THEN assert actual_aid == "agentABC" @@ -120,7 +120,7 @@ async def test_is_aid_available_but_match(): async def test_is_aid_not_available(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - c.register_agent(LooksLikeAgent(), "abc") + c.register(LooksLikeAgent(), "abc") aid_to_check = "abc" # WHEN @@ -135,7 +135,7 @@ async def test_is_aid_not_available(): async def test_is_aid_not_available_and_match(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - c.register_agent(LooksLikeAgent()) + c.register(LooksLikeAgent()) aid_to_check = "agent0" # WHEN @@ -199,7 +199,7 @@ class Data: @pytest.mark.asyncio async def test_send_message_no_copy(): c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=False) - agent1 = c.include(ExampleAgent()) + agent1 = c.register(ExampleAgent()) message_to_send = Data() await c.send_message(message_to_send, receiver_addr=agent1.addr) @@ -211,7 +211,7 @@ async def test_send_message_no_copy(): @pytest.mark.asyncio async def test_send_message_copy(): c = create_tcp_container(addr=("127.0.0.2", 5555), copy_internal_messages=True) - agent1 = c.include(ExampleAgent()) + agent1 = c.register(ExampleAgent()) message_to_send = Data() await c.send_message(message_to_send, receiver_addr=agent1.addr) diff --git a/tests/unit_tests/core/test_external_scheduling_container.py b/tests/unit_tests/core/test_external_scheduling_container.py index b816aeb..c467481 100644 --- a/tests/unit_tests/core/test_external_scheduling_container.py +++ b/tests/unit_tests/core/test_external_scheduling_container.py @@ -119,7 +119,7 @@ def handle_message(self, content, meta: Dict[str, Any]): @pytest.mark.asyncio async def test_step_with_cond_task(): external_scheduling_container = create_ec_container(addr="external_eid_1") - agent_1 = external_scheduling_container.include(WaitForMessageAgent()) + agent_1 = external_scheduling_container.register(WaitForMessageAgent()) print("Agent init") current_time = 0 @@ -199,7 +199,7 @@ def handle_message(self, content, meta: Dict[str, Any]): @pytest.mark.asyncio async def test_send_internal_messages(): external_scheduling_container = create_ec_container(addr="external_eid_1") - agent_1 = external_scheduling_container.include(SelfSendAgent(final_number=3)) + agent_1 = external_scheduling_container.register(SelfSendAgent(final_number=3)) message = create_acl( content="", receiver_addr=external_scheduling_container.addr, @@ -218,7 +218,7 @@ async def test_send_internal_messages(): @pytest.mark.asyncio async def test_step_with_replying_agent(): external_scheduling_container = create_ec_container(addr="external_eid_1") - reply_agent = external_scheduling_container.include(ReplyAgent()) + reply_agent = external_scheduling_container.register(ReplyAgent()) new_acl_msg = ACLMessage() new_acl_msg.content = "hello you" new_acl_msg.receiver_addr = "external_eid_1" diff --git a/tests/unit_tests/express/test_api.py b/tests/unit_tests/express/test_api.py index 11195e6..da6f2fe 100644 --- a/tests/unit_tests/express/test_api.py +++ b/tests/unit_tests/express/test_api.py @@ -58,8 +58,8 @@ def handle_message(self, content, meta: dict[str, Any]): async def test_activate_api_style_agent(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(MyAgent()) - agent2 = c.include(MyAgent()) + agent = c.register(MyAgent()) + agent2 = c.register(MyAgent()) # WHEN async with activate(c) as c: diff --git a/tests/unit_tests/role_agent_test.py b/tests/unit_tests/role_agent_test.py index 147845b..cc46dee 100644 --- a/tests/unit_tests/role_agent_test.py +++ b/tests/unit_tests/role_agent_test.py @@ -131,7 +131,7 @@ async def test_send_ping_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = c.include(RoleAgent()) + a = c.register(RoleAgent()) a.add_role(PongRole()) agents.append(a) addrs.append(a.addr) @@ -168,7 +168,7 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = c.include(RoleAgent()) + a = c.register(RoleAgent()) a.add_role(PongRole()) agents.append(a) addrs.append(a.addr) @@ -196,7 +196,7 @@ async def test_send_ping_pong_deactivated_pong(num_agents, num_containers): @pytest.mark.asyncio async def test_role_add_remove(): c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(RoleAgent()) + agent = c.register(RoleAgent()) role = SampleRole() agent.add_role(role) @@ -210,7 +210,7 @@ async def test_role_add_remove(): @pytest.mark.asyncio async def test_role_add_remove_context(): c = create_tcp_container(addr=("127.0.0.2", 5555)) - agent = c.include(RoleAgent()) + agent = c.register(RoleAgent()) role = SampleRole() agent.add_role(role) diff --git a/tests/unit_tests/test_agents.py b/tests/unit_tests/test_agents.py index 5e70912..94f0c97 100644 --- a/tests/unit_tests/test_agents.py +++ b/tests/unit_tests/test_agents.py @@ -68,7 +68,7 @@ async def wait_for_pong_replies(self, timeout=1): @pytest.mark.asyncio async def test_init_and_shutdown(): c = create_tcp_container(addr=("127.0.0.1", 5555)) - a = c.include(PingPongAgent()) + a = c.register(PingPongAgent()) async with activate(c) as c: assert a.aid is not None @@ -95,7 +95,7 @@ async def test_send_ping_pong(num_agents, num_containers): addrs = [] for i in range(num_agents): c = containers[i % num_containers] - a = c.include(PingPongAgent()) + a = c.register(PingPongAgent()) agents.append(a) addrs.append(a.addr) From 7e52e25bbdd5f2b30d8a8cc851d53973d164a40f Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 17:54:58 +0200 Subject: [PATCH 09/15] Target SVG size --- docs/source/_static/Logo_mango_ohne_sub.svg | 2 +- docs/source/_static/Logo_mango_ohne_sub_white.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/_static/Logo_mango_ohne_sub.svg b/docs/source/_static/Logo_mango_ohne_sub.svg index 6a7f4aa..b118f61 100644 --- a/docs/source/_static/Logo_mango_ohne_sub.svg +++ b/docs/source/_static/Logo_mango_ohne_sub.svg @@ -1 +1 @@ - + diff --git a/docs/source/_static/Logo_mango_ohne_sub_white.svg b/docs/source/_static/Logo_mango_ohne_sub_white.svg index 915b9c4..1a7ecf5 100644 --- a/docs/source/_static/Logo_mango_ohne_sub_white.svg +++ b/docs/source/_static/Logo_mango_ohne_sub_white.svg @@ -1 +1 @@ - + From 2f1602459c6e6dae348384f510175eea6b4ba960 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 17:58:38 +0200 Subject: [PATCH 10/15] Update Test mango for doctest. --- .github/workflows/test-mango.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index aee10ef..5bcf43d 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -47,7 +47,7 @@ jobs: - name: Doctests run: | source venv/bin/activate - docs/make doctest + make -C docs doctest - name: Test+Coverage run: | source venv/bin/activate From d92e869bdc3a421f48c508cd523cee201b559b9e Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 18:06:05 +0200 Subject: [PATCH 11/15] Update Test mango for doctest. --- .github/workflows/test-mango.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index 5bcf43d..3e3826e 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -32,6 +32,7 @@ jobs: pip install virtualenv virtualenv venv source venv/bin/activate + pip3 install -U sphinx pip3 install -r requirements.txt pip3 install -e . sudo apt update From 6419dab8e1bac7d11bc1e3b9ae5111bcc1f1a017 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 18:07:54 +0200 Subject: [PATCH 12/15] Update Test mango for doctest. --- .github/workflows/test-mango.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index 3e3826e..4c60586 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -33,6 +33,7 @@ jobs: virtualenv venv source venv/bin/activate pip3 install -U sphinx + pip3 install -r docs/requirements.txt pip3 install -r requirements.txt pip3 install -e . sudo apt update From 4d6d84ccf12967740b808aac8eeae359f31e1c28 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Mon, 14 Oct 2024 18:40:11 +0200 Subject: [PATCH 13/15] Fixing some init bugs, and moving some parts to the constructor again if possible. --- mango/container/core.py | 6 + mango/container/mqtt.py | 19 +- mango/container/tcp.py | 13 +- .../test_message_roundtrip.py | 4 +- tests/unit_tests/container/test_tcp.py | 30 +-- tests/unit_tests/core/test_container.py | 25 +- .../test_external_scheduling_container.py | 220 +++++++++--------- 7 files changed, 152 insertions(+), 165 deletions(-) diff --git a/mango/container/core.py b/mango/container/core.py index 07a571c..2c37bdd 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -126,6 +126,12 @@ def register(self, agent: Agent, suggested_aid: str = None): logger.debug("Successfully registered agent;%s", aid) return agent + def _get_aid(self, agent): + for aid, a in self._agents.items(): + if id(a) == id(agent): + return aid + return None + def include(self, agent: A, suggested_aid: str = None) -> A: """Include the agent in the container. Return the agent for convenience. diff --git a/mango/container/mqtt.py b/mango/container/mqtt.py index fdc372e..9c72a6b 100644 --- a/mango/container/mqtt.py +++ b/mango/container/mqtt.py @@ -55,7 +55,6 @@ def __init__( *, client_id: str, broker_addr: tuple | dict | str, - loop: asyncio.AbstractEventLoop, clock: Clock, codec: Codec, inbox_topic: None | str = None, @@ -76,7 +75,6 @@ def __init__( super().__init__( codec=codec, addr=broker_addr, - loop=loop, clock=clock, name=client_id, **kwargs, @@ -92,6 +90,7 @@ def __init__( self.pending_sub_request: None | asyncio.Future = None async def start(self): + self._loop = asyncio.get_event_loop() if not self.client_id: raise ValueError("client_id is required!") if not self.addr: @@ -137,9 +136,7 @@ async def start(self): # callbacks to check for successful connection def on_con(client, userdata, flags, reason_code, properties): logger.info("Connection Callback with the following flags: %s", flags) - asyncio.get_running_loop().call_soon_threadsafe( - connected.set_result, reason_code - ) + self._loop.call_soon_threadsafe(connected.set_result, reason_code) mqtt_messenger.on_connect = on_con @@ -205,9 +202,7 @@ def on_con(client, userdata, flags, reason_code, properties): # set up subscription callback def on_sub(client, userdata, mid, reason_code_list, properties): - asyncio.get_running_loop().call_soon_threadsafe( - subscribed.set_result, True - ) + self._loop.call_soon_threadsafe(subscribed.set_result, True) mqtt_messenger.on_subscribe = on_sub @@ -268,9 +263,7 @@ def on_discon(client, userdata, disconnect_flags, reason_code, properties): self.mqtt_client.on_disconnect = on_discon def on_sub(client, userdata, mid, reason_code_list, properties): - asyncio.get_running_loop().call_soon_threadsafe( - self.pending_sub_request.set_result, 0 - ) + self._loop.call_soon_threadsafe(self.pending_sub_request.set_result, 0) self.mqtt_client.on_subscribe = on_sub @@ -289,9 +282,7 @@ def on_message(client, userdata, message): # update meta dict meta.update(message_meta) # put information to inbox - asyncio.get_running_loop().call_soon_threadsafe( - self.inbox.put_nowait, (0, content, meta) - ) + self._loop.call_soon_threadsafe(self.inbox.put_nowait, (0, content, meta)) self.mqtt_client.on_message = on_message self.mqtt_client.enable_logger(logger) diff --git a/mango/container/tcp.py b/mango/container/tcp.py index e17ba35..076024a 100644 --- a/mango/container/tcp.py +++ b/mango/container/tcp.py @@ -33,11 +33,9 @@ class TCPConnectionPool: def __init__( self, - asyncio_loop, ttl_in_sec: float = 30.0, max_connections_per_target: int = 10, ) -> None: - self._loop = asyncio_loop self._available_connections = {} self._connection_counts = {} self._ttl_in_sec = ttl_in_sec @@ -93,7 +91,7 @@ async def obtain_connection( addr_key, ( ( - await self._loop.create_connection( + await asyncio.get_running_loop().create_connection( lambda: protocol, host, port, @@ -182,18 +180,17 @@ def __init__( **kwargs, ) + self._tcp_connection_pool = None self.server = None # will be set within start self.running = False - - async def start(self): self._tcp_connection_pool = TCPConnectionPool( - asyncio.get_running_loop(), ttl_in_sec=self._kwargs.get(TCP_CONNECTION_TTL, 30), max_connections_per_target=self._kwargs.get( TCP_MAX_CONNECTIONS_PER_TARGET, 10 ), ) + async def start(self): # create a TCP server bound to host and port that uses the # specified protocol self.server = await asyncio.get_running_loop().create_server( @@ -303,7 +300,9 @@ async def shutdown(self): calls shutdown() from super class Container and closes the server """ await super().shutdown() - await self._tcp_connection_pool.shutdown() + + if self._tcp_connection_pool is not None: + await self._tcp_connection_pool.shutdown() if self.server is not None: self.server.close() diff --git a/tests/integration_tests/test_message_roundtrip.py b/tests/integration_tests/test_message_roundtrip.py index 41a2945..b285476 100644 --- a/tests/integration_tests/test_message_roundtrip.py +++ b/tests/integration_tests/test_message_roundtrip.py @@ -79,7 +79,7 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): await self.container.subscribe_for_agent( - aid=self.aid, topic=self.target.addr + aid=self.aid, topic=self.target.protocol_addr ) await asyncio.sleep(0.1) @@ -119,7 +119,7 @@ def handle_message(self, content, meta): async def start(self): if getattr(self.container, "subscribe_for_agent", None): await self.container.subscribe_for_agent( - aid=self.aid, topic=self.target.addr + aid=self.aid, topic=self.target.protocol_addr ) # await "Hello" diff --git a/tests/unit_tests/container/test_tcp.py b/tests/unit_tests/container/test_tcp.py index 7e21a17..6ff0c12 100644 --- a/tests/unit_tests/container/test_tcp.py +++ b/tests/unit_tests/container/test_tcp.py @@ -22,10 +22,8 @@ async def test_connection_pool_obtain_release(): await c2.start() addr = "127.0.0.2", 5556 - connection_pool = TCPConnectionPool(asyncio.get_event_loop()) - raw_prot = ContainerProtocol( - container=c, loop=asyncio.get_event_loop(), codec=c.codec - ) + connection_pool = TCPConnectionPool() + raw_prot = ContainerProtocol(container=c, codec=c.codec) protocol = await connection_pool.obtain_connection(addr[0], addr[1], raw_prot) assert connection_pool._available_connections[addr].qsize() == 0 @@ -49,18 +47,14 @@ async def test_connection_pool_double_obtain_release(): await c2.start() addr = "127.0.0.2", 5556 - connection_pool = TCPConnectionPool(asyncio.get_event_loop()) - raw_prot = ContainerProtocol( - container=c, loop=asyncio.get_event_loop(), codec=c.codec - ) + connection_pool = TCPConnectionPool() + raw_prot = ContainerProtocol(container=c, codec=c.codec) protocol = await connection_pool.obtain_connection(addr[0], addr[1], raw_prot) assert connection_pool._available_connections[addr].qsize() == 0 assert connection_pool._connection_counts[addr] == 1 - raw_prot = ContainerProtocol( - container=c, loop=asyncio.get_event_loop(), codec=c.codec - ) + raw_prot = ContainerProtocol(container=c, codec=c.codec) protocol2 = await connection_pool.obtain_connection(addr[0], addr[1], raw_prot) assert connection_pool._available_connections[addr].qsize() == 0 @@ -92,10 +86,8 @@ async def test_ttl(): await c2.start() await c3.start() - connection_pool = TCPConnectionPool(asyncio.get_event_loop(), ttl_in_sec=0.1) - raw_prot = ContainerProtocol( - container=c, loop=asyncio.get_event_loop(), codec=c.codec - ) + connection_pool = TCPConnectionPool(ttl_in_sec=0.1) + raw_prot = ContainerProtocol(container=c, codec=c.codec) protocol = await connection_pool.obtain_connection(addr[0], addr[1], raw_prot) assert connection_pool._available_connections[addr].qsize() == 0 @@ -134,12 +126,8 @@ async def test_max_connections(): await c2.start() addr = "127.0.0.2", 5556 - connection_pool = TCPConnectionPool( - asyncio.get_event_loop(), max_connections_per_target=1 - ) - raw_prot = ContainerProtocol( - container=c, loop=asyncio.get_event_loop(), codec=c.codec - ) + connection_pool = TCPConnectionPool(max_connections_per_target=1) + raw_prot = ContainerProtocol(container=c, codec=c.codec) protocol = await connection_pool.obtain_connection(addr[0], addr[1], raw_prot) with pytest.raises(asyncio.TimeoutError): diff --git a/tests/unit_tests/core/test_container.py b/tests/unit_tests/core/test_container.py index 4108cad..eb54f1e 100644 --- a/tests/unit_tests/core/test_container.py +++ b/tests/unit_tests/core/test_container.py @@ -20,10 +20,10 @@ async def test_register_aid_pattern_match(): suggested_aid = "agent12" # WHEN - actual_aid = c.register(agent, suggested_aid) + agent_r = c.register(agent, suggested_aid) # THEN - assert actual_aid == "agent0" + assert c._get_aid(agent_r) == "agent0" await c.shutdown() @@ -35,10 +35,10 @@ async def test_register_aid_success(): suggested_aid = "cagent12" # WHEN - actual_aid = c.register(agent, suggested_aid) + agent_r = c.register(agent, suggested_aid) # THEN - assert actual_aid == suggested_aid + assert c._get_aid(agent_r) == suggested_aid await c.shutdown() @@ -49,10 +49,10 @@ async def test_register_no_suggested(): agent = LooksLikeAgent() # WHEN - actual_aid = c.register(agent) + agent_r = c.register(agent) # THEN - assert actual_aid == "agent0" + assert c._get_aid(agent_r) == "agent0" await c.shutdown() @@ -64,10 +64,10 @@ async def test_register_pattern_half_match(): suggested_aid = "agentABC" # WHEN - actual_aid = c.register(agent, suggested_aid) + agent_r = c.register(agent, suggested_aid) # THEN - assert actual_aid == "agentABC" + assert c._get_aid(agent_r) == "agentABC" await c.shutdown() @@ -76,15 +76,16 @@ async def test_register_existing(): # GIVEN c = create_tcp_container(addr=("127.0.0.2", 5555)) agent = LooksLikeAgent() + agent2 = LooksLikeAgent() suggested_aid = "agentABC" # WHEN - actual_aid = c.register(agent, suggested_aid) - actual_aid2 = c.register(agent, suggested_aid) + agent_r = c.register(agent, suggested_aid) + agent_r2 = c.register(agent2, suggested_aid) # THEN - assert actual_aid == "agentABC" - assert actual_aid2 == "agent0" + assert c._get_aid(agent_r) == "agentABC" + assert c._get_aid(agent_r2) == "agent0" await c.shutdown() diff --git a/tests/unit_tests/core/test_external_scheduling_container.py b/tests/unit_tests/core/test_external_scheduling_container.py index c467481..bdb2e5b 100644 --- a/tests/unit_tests/core/test_external_scheduling_container.py +++ b/tests/unit_tests/core/test_external_scheduling_container.py @@ -3,7 +3,7 @@ import pytest -from mango import AgentAddress, create_acl, create_ec_container, sender_addr +from mango import AgentAddress, activate, create_acl, create_ec_container, sender_addr from mango.agent.core import Agent from mango.container.external_coupling import ExternalAgentMessage from mango.messages.message import ACLMessage @@ -39,24 +39,25 @@ async def test_send_msg(): @pytest.mark.asyncio async def test_step(): external_scheduling_container = create_ec_container(addr="external_eid_1234") - await external_scheduling_container.send_message( - content="test", receiver_addr=AgentAddress("eid321", aid="Agent0") - ) - step_output = await external_scheduling_container.step( - simulation_time=12, incoming_messages=[] - ) - assert external_scheduling_container.message_buffer == [] - assert external_scheduling_container.clock.time == 12 - assert 0 < step_output.duration < 0.01 - assert len(step_output.messages) == 1 - external_msg = step_output.messages[0] - assert 0 < external_msg.time < 0.01 - assert external_msg.receiver == "eid321" - decoded_msg = external_scheduling_container.codec.decode(external_msg.message) - assert decoded_msg.content == "test" - assert decoded_msg.meta["receiver_addr"] == "eid321" - assert decoded_msg.meta["receiver_id"] == "Agent0" - await external_scheduling_container.shutdown() + + async with activate(external_scheduling_container) as c: + await external_scheduling_container.send_message( + content="test", receiver_addr=AgentAddress("eid321", aid="Agent0") + ) + step_output = await external_scheduling_container.step( + simulation_time=12, incoming_messages=[] + ) + assert external_scheduling_container.message_buffer == [] + assert external_scheduling_container.clock.time == 12 + assert 0 < step_output.duration < 0.01 + assert len(step_output.messages) == 1 + external_msg = step_output.messages[0] + assert 0 < external_msg.time < 0.01 + assert external_msg.receiver == "eid321" + decoded_msg = external_scheduling_container.codec.decode(external_msg.message) + assert decoded_msg.content == "test" + assert decoded_msg.meta["receiver_addr"] == "eid321" + assert decoded_msg.meta["receiver_id"] == "Agent0" class ReplyAgent(Agent): @@ -124,54 +125,53 @@ async def test_step_with_cond_task(): current_time = 0 - for _ in range(10): - current_time += 1 - # advance time without anything happening - print("starting step") - return_values = await asyncio.wait_for( - external_scheduling_container.step( - simulation_time=current_time, incoming_messages=[] - ), - timeout=1, - ) - - print("One step done") - assert ( - return_values.next_activity == current_time + 1 - and return_values.messages == [] - ) + async with activate(external_scheduling_container) as c: + for _ in range(10): + current_time += 1 + # advance time without anything happening + print("starting step") + return_values = await asyncio.wait_for( + external_scheduling_container.step( + simulation_time=current_time, incoming_messages=[] + ), + timeout=1, + ) - # create and send message in next step - message = create_acl( - content="", - receiver_addr=external_scheduling_container.addr, - receiver_id=agent_1.aid, - sender_addr=external_scheduling_container.addr, - ) - encoded_msg = external_scheduling_container.codec.encode(message) - print("created message") + print("One step done") + assert ( + return_values.next_activity == current_time + 1 + and return_values.messages == [] + ) - # advance time only by 0.5 so that in the next cycle the conditional task will be done - current_time += 0.5 - return_values = await external_scheduling_container.step( - simulation_time=current_time, incoming_messages=[encoded_msg] - ) - print("next step done") + # create and send message in next step + message = create_acl( + content="", + receiver_addr=external_scheduling_container.addr, + receiver_id=agent_1.aid, + sender_addr=external_scheduling_container.addr, + ) + encoded_msg = external_scheduling_container.codec.encode(message) + print("created message") - # the conditional task should still be running and next activity should be in 0.5 seconds - assert ( - return_values.next_activity == current_time + 0.5 - and len(return_values.messages) == 0 - ) - current_time += 0.5 - return_values = await external_scheduling_container.step( - simulation_time=current_time, incoming_messages=[] - ) + # advance time only by 0.5 so that in the next cycle the conditional task will be done + current_time += 0.5 + return_values = await external_scheduling_container.step( + simulation_time=current_time, incoming_messages=[encoded_msg] + ) + print("next step done") - # now everything should be done - assert return_values.next_activity is None and len(return_values.messages) == 0 + # the conditional task should still be running and next activity should be in 0.5 seconds + assert ( + return_values.next_activity == current_time + 0.5 + and len(return_values.messages) == 0 + ) + current_time += 0.5 + return_values = await external_scheduling_container.step( + simulation_time=current_time, incoming_messages=[] + ) - await external_scheduling_container.shutdown() + # now everything should be done + assert return_values.next_activity is None and len(return_values.messages) == 0 class SelfSendAgent(Agent): @@ -200,57 +200,59 @@ def handle_message(self, content, meta: Dict[str, Any]): async def test_send_internal_messages(): external_scheduling_container = create_ec_container(addr="external_eid_1") agent_1 = external_scheduling_container.register(SelfSendAgent(final_number=3)) - message = create_acl( - content="", - receiver_addr=external_scheduling_container.addr, - receiver_id=agent_1.aid, - sender_addr=external_scheduling_container.addr, - ) - encoded_msg = external_scheduling_container.codec.encode(message) - return_values = await external_scheduling_container.step( - simulation_time=1, incoming_messages=[encoded_msg] - ) - assert len(return_values.messages) == 1 - await external_scheduling_container.shutdown() + async with activate(external_scheduling_container) as c: + message = create_acl( + content="", + receiver_addr=external_scheduling_container.addr, + receiver_id=agent_1.aid, + sender_addr=external_scheduling_container.addr, + ) + encoded_msg = external_scheduling_container.codec.encode(message) + return_values = await external_scheduling_container.step( + simulation_time=1, incoming_messages=[encoded_msg] + ) + assert len(return_values.messages) == 1 @pytest.mark.asyncio async def test_step_with_replying_agent(): external_scheduling_container = create_ec_container(addr="external_eid_1") - reply_agent = external_scheduling_container.register(ReplyAgent()) - new_acl_msg = ACLMessage() - new_acl_msg.content = "hello you" - new_acl_msg.receiver_addr = "external_eid_1" - new_acl_msg.receiver_id = reply_agent.aid - new_acl_msg.sender_id = "Agent0" - new_acl_msg.sender_addr = "external_eid_2" - encoded_msg = external_scheduling_container.codec.encode(new_acl_msg) - container_output = await external_scheduling_container.step( - simulation_time=10, incoming_messages=[encoded_msg] - ) - assert ( - len(container_output.messages) == 3 - ), f"output messages: {container_output.messages}" - assert ( - container_output.messages[0].time - < container_output.messages[1].time - < external_scheduling_container.clock.time + 0.1 - ) - assert ( - container_output.messages[2].time - > external_scheduling_container.clock.time + 0.1 - ) # since we had a sleep of 0.1 seconds - assert ( - container_output.next_activity == external_scheduling_container.clock.time + 10 - ) - container_output = await external_scheduling_container.step( - simulation_time=20, incoming_messages=[] - ) - assert len(container_output.messages) == 1 - assert ( - container_output.next_activity == external_scheduling_container.clock.time + 10 - ) - await reply_agent.stop_tasks() - await external_scheduling_container.shutdown() + async with activate(external_scheduling_container) as c: + reply_agent = external_scheduling_container.register(ReplyAgent()) + new_acl_msg = ACLMessage() + new_acl_msg.content = "hello you" + new_acl_msg.receiver_addr = "external_eid_1" + new_acl_msg.receiver_id = reply_agent.aid + new_acl_msg.sender_id = "Agent0" + new_acl_msg.sender_addr = "external_eid_2" + encoded_msg = external_scheduling_container.codec.encode(new_acl_msg) + container_output = await external_scheduling_container.step( + simulation_time=10, incoming_messages=[encoded_msg] + ) + assert ( + len(container_output.messages) == 3 + ), f"output messages: {container_output.messages}" + assert ( + container_output.messages[0].time + < container_output.messages[1].time + < external_scheduling_container.clock.time + 0.1 + ) + assert ( + container_output.messages[2].time + > external_scheduling_container.clock.time + 0.1 + ) # since we had a sleep of 0.1 seconds + assert ( + container_output.next_activity + == external_scheduling_container.clock.time + 10 + ) + container_output = await external_scheduling_container.step( + simulation_time=20, incoming_messages=[] + ) + assert len(container_output.messages) == 1 + assert ( + container_output.next_activity + == external_scheduling_container.clock.time + 10 + ) + await reply_agent.stop_tasks() From 4aef7ac1bc85241f532cc081083c516ff4e8d45a Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Tue, 15 Oct 2024 23:46:17 +0200 Subject: [PATCH 14/15] Update the tutorial to new state. --- docs/source/tutorial.rst | 788 +++++++++++++++++++++++++-------------- mango/__init__.py | 3 +- mango/container/core.py | 9 - mango/express/api.py | 8 +- 4 files changed, 518 insertions(+), 290 deletions(-) diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index f0b01d9..6ea7d95 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -10,8 +10,7 @@ This tutorial gives an overview of the basic functions of mango agents and conta parts building a scenario of two PV plants, operated by their respective agents being directed by a remote controller. -Each part comes with a standalone executable file. Subsequent parts either extend the functionality or simplify -some concept in the previous part. +Subsequent parts either extend the functionality or simplify some concept in the previous part. As a whole, this tutorial covers: - container and agent creation @@ -26,8 +25,6 @@ As a whole, this tutorial covers: 1. Setup and Message Passing ***************************** -Corresponding file: `v1_basic_setup_and_message_passing.py` - For your first mango tutorial, you will learn the fundamentals of creating mango agents and containers as well as making them communicate with each other. @@ -37,101 +34,122 @@ This example covers: - basic message passing - clean shutdown of containers -.. raw:: html - -
- step by step - First, we want to create two simple agents and have the container send a message to one of them. An agent is created by defining a class that inherits from the base Agent class of mango. -Every agent must implement the ``handle_message`` method to which incoming messages are forwarded by the container. +Every agent must implement the :meth:`mango.Agent.handle_message` method to which incoming messages are forwarded by the container. -.. code-block:: python +.. testcode:: from mango import Agent class PVAgent(Agent): - def __init__(self, container): - super().__init__(container) - print(f"Hello I am a PV agent! My id is {self.aid}.") + def __init__(self): + super().__init__() + print("Hello I am a PV agent!") def handle_message(self, content, meta): print(f"Received message with content: {content} and meta {meta}.") + PVAgent() + +.. testoutput:: + + Hello I am a PV agent! + Now we are ready to instantiate our system. mango is fundamentally built on top of asyncio and a lot of its functions are provided as coroutines. This means, practically every mango executable file will implement some variation of this pattern: -.. code-block:: python +.. testcode:: import asyncio async def main(): + print("This will run in the asyncio loop.") # do whatever here - if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main()) + +.. testoutput:: + + This will run in the asyncio loop. -First, we create the container. A container is created via the ``mango.create_container`` coroutine which requires at least -the address of the container as a parameter. +First, we create the container. A tcp container is created via the :meth:`mango.create_tcp_container` function which requires at least +the address of the container as a parameter. Other container types available by using :meth:`mango.create_mqtt_container` and :meth:`mango.create_ec_container`. +For this tutorial we will cover the tcp container. -.. code-block:: python +.. testcode:: + + from mango import create_tcp_container PV_CONTAINER_ADDRESS = ("localhost", 5555) - # defaults to tcp connection - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS) + pv_container = create_tcp_container(addr=PV_CONTAINER_ADDRESS) + + print(pv_container.addr) +.. testoutput:: -Now we can create our agents. Agents always live inside a container and this container must be passed to their constructor. + ('localhost', 5555) -.. code-block:: python +Now we can create our agents. Agents always live inside a container and therefore need to be registered to the container. + +.. testcode:: # agents always live inside a container - pv_agent_0 = PVAgent(pv_container) - pv_agent_1 = PVAgent(pv_container) - -For now, our agents are purely passive entities. To make them do something, we need to send them a message. Messages are -passed by the container via the ``send_message`` function always at least expects some content and a target address. -To send a message directly to an agent, we also need to provide its agent id which is set by the container when the agent -is created. - -.. code-block:: python - - # we can now send a simple message to an agent and observe that it is received: - # Note that as of now agent IDs are set automatically as agent0, agent1, ... in order of instantiation. - await pv_container.send_message( - "Hello, this is a simple message.", - receiver_addr=PV_CONTAINER_ADDRESS, - receiver_id="agent0", - ) + async def main(): + pv_agent_0 = pv_container.register(PVAgent()) + pv_agent_1 = pv_container.register(PVAgent()) -Finally, you should always cleanly shut down your containers before your program terminates. + print(pv_agent_1.addr) -.. code-block:: python + asyncio.run(main()) - # don't forget to properly shut down containers at the end of your program - # otherwise you will get an asyncio.exceptions.CancelledError - await pv_container.shutdown() +.. testoutput:: -This concludes the first part of our tutorial. If you run this code, you should receive the following output: + Hello I am a PV agent! + Hello I am a PV agent! + AgentAddress(protocol_addr=('localhost', 5555), aid='agent1') - | Hello I am a PV agent! My id is agent0. - | Hello I am a PV agent! My id is agent1. - | Received message with content: Hello, this is a simple message. and meta {'network_protocol': 'tcp', 'priority': 0}. +For now, our agents and containers are purely passive entities. First, we need to activate the container to start +the tcp server and its internal asynchronous behavior. In mango this can be done with :meth:`mango.activate` and the `async with` syntax. +Second, we need to send a message from one agent to the other. Messages are passed by the container via the :meth:`mango.Agent.send_message` +function always at least expects some content and a target agent address. To send a message directly to an agent, we also need to provide +its agent id which is set by the container when the agent is created. The address of the container and the aid +is wrapped in the :class:`mango.AgentAddress` class and can be retrieved with :meth:`mango.Agent.addr`. +.. testcode:: -.. raw:: html + from mango import activate + + # agents always live inside a container + async def main(): + pv_agent_0 = pv_container.register(PVAgent()) + pv_agent_1 = pv_container.register(PVAgent()) + + async with activate(pv_container) as c: + # we can now send a simple message to an agent and observe that it is received: + # Note that as of now agent IDs are set automatically as agent0, agent1, ... + # in order of instantiation. + await pv_agent_0.send_message( + "Hello, this is a simple message.", + receiver_addr=pv_agent_1.addr + ) + + asyncio.run(main()) + +.. testoutput:: + + Hello I am a PV agent! + Hello I am a PV agent! + Received message with content: Hello, this is a simple message. and meta {'sender_id': 'agent2', 'sender_addr': ('localhost', 5555), 'receiver_id': 'agent3', 'network_protocol': 'tcp', 'priority': 0}. -
********************************* 2. Messaging between Containers ********************************* -Corresponding file: `v2_inter_container_messaging_and_basic_functionality.py` - In the previous example, you learned how to create mango agents and containers and how to send basic messages between them. In this example, you expand upon this. We introduce a controller agent that asks the current feed_in of our PV agents and subsequently limits the output of both to their minimum. @@ -142,11 +160,6 @@ This example covers: - setting custom agent ids - use of metadata -.. raw:: html - -
- step by step - First, we define our controller Agent. To ensure it can message the pv agents we pass that information directly to it in the constructor. The control agent will send out messages to each pv agent, await their replies and act according to that information. To handle this, we also add some control structures to the @@ -154,26 +167,42 @@ constructor that we will later use to keep track of which agents have already an As an additional feature, we will make it possible to manually set the agent of our agents by. -.. code-block:: python +.. testcode:: + + from mango import Agent, addr class ControllerAgent(Agent): - def __init__(self, container, known_agents, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) + def __init__(self, known_agents): + super().__init__() + self.known_agents = known_agents self.reported_feed_ins = [] self.reported_acks = 0 self.reports_done = None self.acks_done = None -Next, we set up its ``handle_message`` function. The controller needs to distinguish between two message types: + print(ControllerAgent([addr("protocol_addr", "aid")]).known_agents) + +.. testoutput:: + + [AgentAddress(protocol_addr='protocol_addr', aid='aid')] + +Next, we set up its :meth:`mango.Agent.handle_message` function. The controller needs to distinguish between two message types: The replies to feed_in requests and later the acknowledgements that a new maximum feed_in was set by a pv agent. -We use the assign the key ``performative``of the metadata of the message to do this. We set the ``performative`` field to ``inform`` -for feed_in replies and to ``accept_proposal`` for feed_in change acknowledgements. +We assign the key `performative` of the metadata of the message to do this. We set the `performative` entry to `inform` +for feed_in replies and to `accept_proposal` for feed_in change acknowledgements. -.. code-block:: python +.. testcode:: class ControllerAgent(Agent): - """...""" + def __init__(self, known_agents): + super().__init__() + + self.known_agents = known_agents + self.reported_feed_ins = [] + self.reported_acks = 0 + self.reports_done = None + self.acks_done = None def handle_message(self, content, meta): performative = meta['performative'] @@ -198,65 +227,63 @@ for feed_in replies and to ``accept_proposal`` for feed_in change acknowledgemen if self.acks_done is not None: self.acks_done.set_result(True) -We do the same for our PV agents. We will also enable user defined agent ids here. +We do the same for our PV agents. + +.. testcode:: + + from mango import sender_addr -.. code-block:: python + PV_FEED_IN = { + "PV Agent 0": 2.0, + "PV Agent 1": 1.0, + } class PVAgent(Agent): - def __init__(self, container, suggested_aid=None): - super().__init__(container, suggested_aid=suggested_aid) + def __init__(self): + super().__init__() + self.max_feed_in = -1 def handle_message(self, content, meta): performative = meta["performative"] - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] + sender = sender_addr(meta) if performative == Performatives.request: # ask_feed_in message - self.handle_ask_feed_in(sender_addr, sender_id) + self.handle_ask_feed_in(sender) elif performative == Performatives.propose: # set_max_feed_in message - self.handle_set_feed_in_max(content, sender_addr, sender_id) + self.handle_set_feed_in_max(content, sender) else: print(f"{self.aid}: Received an unexpected message with content {content} and meta {meta}") - -When a PV agent receives a request from the controller, it immediately answers. Note two important changes to the first -example here: First, within our message handling methods we can not ``await send_message`` directly -because ``handle_message`` is not a coroutine. Instead, we pass ``send_message`` as a task to the scheduler to be -executed at once via the ``schedule_instant_task`` method. -Second, we set ``meta`` to contain the typing information of our message. - -.. code-block:: python - - class PVAgent(Agent): - """...""" - - def handle_ask_feed_in(self, sender_addr, sender_id): + def handle_ask_feed_in(self, sender): reported_feed_in = PV_FEED_IN[self.aid] # PV_FEED_IN must be defined at the top content = reported_feed_in - meta = {"sender_addr": self.addr, "sender_id": self.aid, - "performative": Performatives.inform} - self.schedule_instant_message( content=content, - receiver_addr=sender_addr, - receiver_id=sender_id, - **meta, + receiver_addr=sender, + performative=Performatives.inform ) - def handle_set_feed_in_max(self, max_feed_in, sender_addr, sender_id): + def handle_set_feed_in_max(self, max_feed_in, sender): self.max_feed_in = float(max_feed_in) print(f"{self.aid}: Limiting my feed_in to {max_feed_in}") + self.schedule_instant_message( content=None, - receiver_addr=sender_addr, - receiver_id=sender_id, + receiver_addr=sender, performative=Performatives.accept_proposal, ) + +When a PV agent receives a request from the controller, it immediately answers. Note two important changes to the first +example here: First, within our message handling methods we can not ``await send_message`` directly +because ``handle_message`` is not a coroutine. Instead, we pass ``send_message`` as a task to the scheduler to be +executed at once via the ``schedule_instant_task`` method. +Second, we set ``meta`` to contain the typing information of our message. + Now both of our agents can handle their respective messages. The last thing to do is make the controller actually perform its active actions. We do this by implementing a ``run`` function with the following control flow: - send a feed_in request to each known pv agent @@ -266,10 +293,40 @@ perform its active actions. We do this by implementing a ``run`` function with t - again, wait for all pv agents to reply - terminate -.. code-block:: python +.. testcode:: class ControllerAgent(Agent): - """...""" + def __init__(self, known_agents): + super().__init__() + + self.known_agents = known_agents + self.reported_feed_ins = [] + self.reported_acks = 0 + self.reports_done = None + self.acks_done = None + + def handle_message(self, content, meta): + performative = meta['performative'] + if performative == Performatives.inform: + # feed_in_reply message + self.handle_feed_in_reply(content) + elif performative == Performatives.accept_proposal: + # set_max_ack message + self.handle_set_max_ack() + else: + print(f"{self.aid}: Received an unexpected message with content {content} and meta {meta}") + + def handle_feed_in_reply(self, feed_in_value): + self.reported_feed_ins.append(float(feed_in_value)) + if len(self.reported_feed_ins) == len(self.known_agents): + if self.reports_done is not None: + self.reports_done.set_result(True) + + def handle_set_max_ack(self): + self.reported_acks += 1 + if self.reported_acks == len(self.known_agents): + if self.acks_done is not None: + self.acks_done.set_result(True) async def run(self): # we define an asyncio future to await replies from all known pv agents: @@ -277,34 +334,30 @@ perform its active actions. We do this by implementing a ``run`` function with t self.acks_done = asyncio.Future() # ask pv agent feed-ins - for addr, aid in self.known_agents: - content = None - meta = {"sender_addr": self.addr, "sender_id": self.aid, - "performative": Performatives.request} + for addr in self.known_agents: self.schedule_instant_message( - content=content, + content=None, receiver_addr=addr, - receiver_id=aid, - **meta, + performative=Performatives.request ) # wait for both pv agents to answer await self.reports_done + # deterministic output + self.reported_feed_ins.sort() + # limit both pv agents to the smaller ones feed-in print(f"{self.aid}: received feed_ins: {self.reported_feed_ins}") min_feed_in = min(self.reported_feed_ins) - for addr, aid in self.known_agents: + for addr in self.known_agents: content = min_feed_in - meta = {"sender_addr": self.addr, "sender_id": self.aid, - "performative": Performatives.propose} self.schedule_instant_message( content=content, receiver_addr=addr, - receiver_id=aid, - **meta, + performative=Performatives.propose ) # wait for both pv agents to acknowledge the change @@ -312,7 +365,9 @@ perform its active actions. We do this by implementing a ``run`` function with t Lastly, we call all relevant instantiations and the run function within our main coroutine: -.. code-block:: python +.. testcode:: + + from mango import create_tcp_container, activate, Performatives PV_CONTAINER_ADDRESS = ("localhost", 5555) CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) @@ -322,49 +377,39 @@ Lastly, we call all relevant instantiations and the run function within our main } async def main(): - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS) - controller_container = await create_container(addr=CONTROLLER_CONTAINER_ADDRESS) + pv_container = create_tcp_container(addr=PV_CONTAINER_ADDRESS) + controller_container = create_tcp_container(addr=CONTROLLER_CONTAINER_ADDRESS) # agents always live inside a container - pv_agent_0 = PVAgent(pv_container, suggested_aid='PV Agent 0') - pv_agent_1 = PVAgent(pv_container, suggested_aid='PV Agent 1') + pv_agent_0 = pv_container.register(PVAgent(), suggested_aid='PV Agent 0') + pv_agent_1 = pv_container.register(PVAgent(), suggested_aid='PV Agent 1') # We pass info of the pv agents addresses to the controller here directly. # In reality, we would use some kind of discovery mechanism for this. known_agents = [ - (PV_CONTAINER_ADDRESS, pv_agent_0.aid), - (PV_CONTAINER_ADDRESS, pv_agent_1.aid), + pv_agent_0.addr, + pv_agent_1.addr, ] - controller_agent = ControllerAgent(controller_container, known_agents, suggested_aid='Controller') + controller_agent = controller_container.register(ControllerAgent(known_agents), suggested_aid='Controller') - # the only active component in this setup - await controller_agent.run() + async with activate(pv_container, controller_container) as cl: + # the only active component in this setup + await controller_agent.run() - # always properly shut down your containers - await pv_container.shutdown() - await controller_container.shutdown() + asyncio.run(main()) - if __name__ == "__main__": - asyncio.run(main()) +.. testoutput:: -This concludes the second part of our tutorial. If you run this code you should receive the following output: + Controller: received feed_ins: [1.0, 2.0] + PV Agent 0: Limiting my feed_in to 1.0 + PV Agent 1: Limiting my feed_in to 1.0 - | Controller: received feed_ins: [2.0, 1.0] - | PV Agent 0: Limiting my feed_in to 1.0 - | PV Agent 1: Limiting my feed_in to 1.0 - - -.. raw:: html - -
******************************************* 3. Using Codecs to simplify Message Types ******************************************* -Corresponding file: `v3_codecs_and_typing.py` - In example 2, you created some basic agent functionality and established inter-container communication. Message types were distinguished by the performative field of the meta information. This approach is tedious and prone to error. A better way is to use dedicated message objects and using their types to distinguish @@ -381,70 +426,76 @@ This example covers: - codec basics - the json_serializable decorator -.. raw:: html - -
- step by step - We want to use the types of custom message objects as the new mechanism for message typing. We define these -as simple data classes. For simple classes like this, we can use the ``json_serializable`` decorator to +as simple data classes. For simple classes like this, we can use the :meth:`mango.json_serializable`` decorator to provide us with the serialization functionality. -.. code-block:: python +.. testcode:: - import mango.messages.codecs as codecs + from mango import json_serializable from dataclasses import dataclass - @codecs.json_serializable + @json_serializable @dataclass class AskFeedInMsg: pass - @codecs.json_serializable + @json_serializable @dataclass class FeedInReplyMsg: feed_in: int - @codecs.json_serializable + @json_serializable @dataclass class SetMaxFeedInMsg: max_feed_in: int - @codecs.json_serializable + @json_serializable @dataclass class MaxFeedInAck: pass Next, we need to create a codec, make our message objects known to it, and pass it to our containers. -.. code-block:: python +.. testcode:: + + from mango import JSON + + PV_CONTAINER_ADDRESS = ("localhost", 5555) + CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) - my_codec = codecs.JSON() + my_codec = JSON() my_codec.add_serializer(*AskFeedInMsg.__serializer__()) my_codec.add_serializer(*SetMaxFeedInMsg.__serializer__()) my_codec.add_serializer(*FeedInReplyMsg.__serializer__()) my_codec.add_serializer(*MaxFeedInAck.__serializer__()) - pv_container = await create_container(addr=PV_CONTAINER_ADDRESS, codec=my_codec) + pv_container = create_tcp_container(addr=PV_CONTAINER_ADDRESS, codec=my_codec) - controller_container = await create_container( + controller_container = create_tcp_container( addr=CONTROLLER_CONTAINER_ADDRESS, codec=my_codec ) Any time the content of a message matches one of these types now the corresponding serialize and deserialize functions are called. Of course, you can also create your own serialization and deserialization functions with -more sophisticated behaviours and pass them to the codec. For more details refer to the ``codecs`` section of +more sophisticated behaviours and pass them to the codec. For more details refer to the :doc:`codecs` section of the documentation. With this, the message handling in our agent classes can be simplified: -.. code-block:: python +.. testcode:: class ControllerAgent(Agent): - """...""" + def __init__(self, known_agents): + super().__init__() + self.known_agents = known_agents + self.reported_feed_ins = [] + self.reported_acks = 0 + self.reports_done = None + self.acks_done = None def handle_message(self, content, meta): if isinstance(content, FeedInReplyMsg): @@ -454,39 +505,125 @@ With this, the message handling in our agent classes can be simplified: else: print(f"{self.aid}: Received a message of unknown type {type(content)}") + def handle_feed_in_reply(self, feed_in_value): + self.reported_feed_ins.append(float(feed_in_value)) + if len(self.reported_feed_ins) == len(self.known_agents): + if self.reports_done is not None: + self.reports_done.set_result(True) + + def handle_set_max_ack(self): + self.reported_acks += 1 + if self.reported_acks == len(self.known_agents): + if self.acks_done is not None: + self.acks_done.set_result(True) + + async def run(self): + # we define an asyncio future to await replies from all known pv agents: + self.reports_done = asyncio.Future() + self.acks_done = asyncio.Future() + + # ask pv agent feed-ins + for addr in self.known_agents: + msg = AskFeedInMsg() + + # alternatively we could call send_acl_message here directly and await it + self.schedule_instant_message( + content=msg, + receiver_addr=addr, + ) + + # wait for both pv agents to answer + await self.reports_done + + # deterministic output + self.reported_feed_ins.sort() + + # limit both pv agents to the smaller ones feed-in + print(f"{self.aid}: received feed_ins: {self.reported_feed_ins}") + min_feed_in = min(self.reported_feed_ins) + + for addr in self.known_agents: + msg = SetMaxFeedInMsg(min_feed_in) + + # alternatively we could call send_acl_message here directly and await it + self.schedule_instant_message( + content=msg, + receiver_addr=addr + ) + + # wait for both pv agents to acknowledge the change + await self.acks_done class PVAgent(Agent): - """...""" + def __init__(self): + super().__init__() + + self.max_feed_in = -1 def handle_message(self, content, meta): - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] + sender = sender_addr(meta) if isinstance(content, AskFeedInMsg): - self.handle_ask_feed_in(sender_addr, sender_id) + self.handle_ask_feed_in(sender) elif isinstance(content, SetMaxFeedInMsg): - self.handle_set_feed_in_max(content.max_feed_in, sender_addr, sender_id) + self.handle_set_feed_in_max(content.max_feed_in, sender) else: print(f"{self.aid}: Received a message of unknown type {type(content)}") + def handle_ask_feed_in(self, sender_addr): + reported_feed_in = PV_FEED_IN[self.aid] # PV_FEED_IN must be defined at the top + msg = FeedInReplyMsg(reported_feed_in) + + self.schedule_instant_message( + content=msg, + receiver_addr=sender_addr + ) -This concludes the third part of our tutorial. If you run the code, -you should receive the same output as in part 2: + def handle_set_feed_in_max(self, max_feed_in, sender_addr): + self.max_feed_in = float(max_feed_in) + print(f"{self.aid}: Limiting my feed_in to {max_feed_in}") + msg = MaxFeedInAck() - | Controller: received feed_ins: [2.0, 1.0] - | PV Agent 0: Limiting my feed_in to 1.0 - | PV Agent 1: Limiting my feed_in to 1.0 + self.schedule_instant_message( + content=msg, + receiver_addr=sender_addr, + ) -.. raw:: html +.. testcode:: -
+ async def main(): + # agents always live inside a container + pv_agent_0 = pv_container.register(PVAgent(), suggested_aid='PV Agent 0') + pv_agent_1 = pv_container.register(PVAgent(), suggested_aid='PV Agent 1') + + # We pass info of the pv agents addresses to the controller here directly. + # In reality, we would use some kind of discovery mechanism for this. + known_agents = [ + pv_agent_0.addr, + pv_agent_1.addr, + ] + + controller_agent = controller_container.register(ControllerAgent(known_agents), + suggested_aid='Controller') + + async with activate(pv_container, controller_container) as cl: + # the only active component in this setup + await controller_agent.run() + + asyncio.run(main()) + +.. testoutput:: + + Controller: received feed_ins: [1.0, 2.0] + PV Agent 0: Limiting my feed_in to 1.0 + PV Agent 1: Limiting my feed_in to 1.0 + +This concludes the third part of our tutorial. ************************* 4. Scheduling and Roles ************************* -Corresponding file: `v4_scheduling_and_roles.py` - In example 3, you restructured your code to use codecs for easier handling of typed message objects. Now it is time to expand the functionality of our controller. In addition to setting the maximum feed_in of the pv agents, the controller should now also periodically check if the pv agents are still reachable. @@ -499,43 +636,65 @@ With the introduction of this task, we know have different responsibilities for responsibilities we can use the role API. The idea of using roles is to divide the functionality of an agent by responsibility in a structured way. -A role is a python object that can be assigned to a RoleAgent. The two main functions each role implements are: +A role is a python object that can be assigned to a RoleAgent. There are several lifecycle functions each role may implement: - __init__ - where you do the initial object setup - - setup - which is called when the role is assigned to an agent + - :meth:`mango.Role.setup` - which is called when the role is assigned to an agent + - :meth:`mango.Role.on_start` - which is called when the container is started + - :meth:`mango.Role.on_ready` - which is called when are activated -This distinction is relevant because only within `setup` the RoleContext (i.e. access to the parent agent and container) exist. -Thus, things like message handlers that require container knowledge are introduced there. +This distinction is relevant because not all features exist after construction with __init__. Most of the time +you want to implement :meth:`mango.Role.on_ready` for actions like message sending, or scheduling, because only +since this point you can be sure that all relevant container are started and the agent the role belongs to has been registered. +However, the setup of the role itself should be done in :meth:`mango.Role.setup`. This example covers: - role API basics - scheduling and periodic tasks -.. raw:: html - -
- step by step - -The key part of defining roles are their ``__init__`` and ``setup`` methods. The first is called to create the role object. -The second is called when the role is assigned to an agent. In our case, the main change is that the previous distinction -of message types within ``handle_message`` is now done by subscribing to the corresponding message type to tell the agent -it should forward these messages to this role. -The ``subscribe_message`` method expects, besides the role and a handle method, a message condition function. +The key part of defining roles are their `__init__`, `setup`, and `on_ready` methods. +The first is called to create the role object. The second is called when the role is assigned to +an agent. While the third is called when all containers are started using :meth:`mango.activate`. +In our case, the main change is that the previous distinction of message types within `handle_message` is now done +by subscribing to the corresponding message type to tell the agent it should forward these messages +to this role. +The :meth:`mango.Role.subscribe_message` method expects, besides the role and a handle method, a message condition function. The idea of the condition function is to allow to define a condition filtering incoming messages. Another idea is that sending messages from the role is now done via its context with the method: -``self.context.send_message``. +`self.context.send_message`. -We first create the ``Ping`` role, which has to periodically send out its messages. +We first create the `Ping` role, which has to periodically send out its messages. We can use mango's scheduling API to handle -this for us via the ``schedule_periodic_tasks`` function. This takes a coroutine to execute and a time +this for us via the :meth:`mango.RoleContext.schedule_periodic_tasks` function. This takes a coroutine to execute and a time interval. Whenever the time interval runs out the coroutine is triggered. With the scheduling API you can also run tasks at specific times. For a full overview we refer to the documentation. -.. code-block:: python +.. testcode:: + + import asyncio + from dataclasses import dataclass + + from mango import sender_addr, Role, RoleAgent, JSON, create_tcp_container, json_serializable, agent_composed_of + + PV_CONTAINER_ADDRESS = ("localhost", 5555) + CONTROLLER_CONTAINER_ADDRESS = ("localhost", 5556) + PV_FEED_IN = { + "PV Agent 0": 2.0, + "PV Agent 1": 1.0, + } + + @json_serializable + @dataclass + class Ping: + ping_id: int - from mango import Role + @json_serializable + @dataclass + class Pong: + pong_id: int class PingRole(Role): def __init__(self, ping_recipients, time_between_pings): + super().__init__() self.ping_recipients = ping_recipients self.time_between_pings = time_between_pings self.ping_counter = 0 @@ -546,20 +705,17 @@ also run tasks at specific times. For a full overview we refer to the documentat self, self.handle_pong, lambda content, meta: isinstance(content, Pong) ) - # this task is automatically executed every "time_between_pings" seconds + def on_ready(self): self.context.schedule_periodic_task(self.send_pings, self.time_between_pings) async def send_pings(self): - for addr, aid in self.ping_recipients: + for addr in self.ping_recipients: ping_id = self.ping_counter msg = Ping(ping_id) - meta = {"sender_addr": self.context.addr, "sender_id": self.context.aid} await self.context.send_message( msg, receiver_addr=addr, - receiver_id=aid, - **meta, ) self.expected_pongs.append(ping_id) self.ping_counter += 1 @@ -574,42 +730,100 @@ also run tasks at specific times. For a full overview we refer to the documentat print( f"Pong {self.context.aid}: Received an unexpected pong with ID: {content.pong_id}" ) + print(Ping(1).ping_id) + print(Pong(1).pong_id) + print(PingRole(["addr"], 1).ping_recipients) + +.. testoutput:: + 1 + 1 + ['addr'] The ControllerRole now covers the former responsibilities of the controller: -.. code-block:: python +.. testcode:: class ControllerRole(Role): - def __init__(self, known_agents): - super().__init__() - self.known_agents = known_agents - self.reported_feed_ins = [] - self.reported_acks = 0 - self.reports_done = None - self.acks_done = None - - def setup(self): - self.context.subscribe_message( - self, - self.handle_feed_in_reply, - lambda content, meta: isinstance(content, FeedInReplyMsg), - ) + def __init__(self, known_agents): + super().__init__() + self.known_agents = known_agents + self.reported_feed_ins = [] + self.reported_acks = 0 + self.reports_done = None + self.acks_done = None - self.context.subscribe_message( - self, - self.handle_set_max_ack, - lambda content, meta: isinstance(content, MaxFeedInAck), - ) + def setup(self): + self.context.subscribe_message( + self, + self.handle_feed_in_reply, + lambda content, meta: isinstance(content, FeedInReplyMsg), + ) + + self.context.subscribe_message( + self, + self.handle_set_max_ack, + lambda content, meta: isinstance(content, MaxFeedInAck), + ) + + def on_ready(self): + self.context.schedule_instant_task(self.run()) + + def handle_feed_in_reply(self, content, meta): + feed_in_value = float(content.feed_in) + + self.reported_feed_ins.append(feed_in_value) + if len(self.reported_feed_ins) == len(self.known_agents): + if self.reports_done is not None: + self.reports_done.set_result(True) + + def handle_set_max_ack(self, content, meta): + self.reported_acks += 1 + if self.reported_acks == len(self.known_agents): + if self.acks_done is not None: + self.acks_done.set_result(True) + + async def run(self): + # we define an asyncio future to await replies from all known pv agents: + self.reports_done = asyncio.Future() + self.acks_done = asyncio.Future() + + # ask pv agent feed-ins + for addr in self.known_agents: + msg = AskFeedInMsg() + + await self.context.send_message( + content=msg, + receiver_addr=addr + ) + + # wait for both pv agents to answer + await self.reports_done + + # limit both pv agents to the smaller ones feed-in + print(f"Controller received feed_ins: {self.reported_feed_ins}") + min_feed_in = min(self.reported_feed_ins) + + for addr in self.known_agents: + msg = SetMaxFeedInMsg(min_feed_in) + + await self.context.send_message( + content=msg, + receiver_addr=addr, + ) + + # wait for both pv agents to acknowledge the change + await self.acks_done - self.context.schedule_instant_task(self.run()) + print(ControllerRole([]).known_agents) -The methods ``handle_feed_in_reply``, ``handle_set_max_ack`` and ``run`` are also part of this role and -remain unchanged. +.. testoutput:: + + [] The ``Pong`` role is associated with the PV Agents and purely reactive. -.. code-block:: python +.. testcode:: class PongRole(Role): def setup(self): @@ -619,27 +833,30 @@ The ``Pong`` role is associated with the PV Agents and purely reactive. def handle_ping(self, content, meta): ping_id = content.ping_id - sender_addr = meta["sender_addr"] - sender_id = meta["sender_id"] answer = Pong(ping_id) print(f"Ping {self.context.aid}: Received a ping with ID: {ping_id}") # message sending from roles is done via the RoleContext - self.context.schedule_message( - answer, - receiver_addr=sender_addr, - receiver_id=sender_id, + self.context.schedule_instant_message( + answer, + receiver_addr=sender_addr(meta) ) + print(type(PongRole())) + +.. testoutput:: + + Since the PV Agent is purely reactive, its other functionality stays basically unchanged and is simply moved to the PVRole. -.. code-block:: python +.. testcode:: class PVRole(Role): def __init__(self): + super().__init__() self.max_feed_in = -1 def setup(self): @@ -655,75 +872,90 @@ unchanged and is simply moved to the PVRole. ) def handle_ask_feed_in(self, content, meta): - """...""" + reported_feed_in = PV_FEED_IN[ + self.context.aid + ] + msg = FeedInReplyMsg(reported_feed_in) + self.context.schedule_instant_message( content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, + receiver_addr=sender_addr(meta) ) - def handle_set_feed_in_max(self, content, meta): - """...""" + max_feed_in = float(content.max_feed_in) + self.max_feed_in = max_feed_in + print(f"{self.context.aid}: Limiting my feed_in to {max_feed_in}") + + msg = MaxFeedInAck() + self.context.schedule_instant_message( content=msg, - receiver_addr=sender_addr, - receiver_id=sender_id, + receiver_addr=sender_addr(meta), ) + print(PVRole().max_feed_in) +.. testoutput:: -The definition of the agent classes itself now simply boils down to assigning it all the roles it has: - -.. code-block:: python - - from mango import RoleAgent + -1 - class PVAgent(RoleAgent): - def __init__(self, container): - super().__init__(container) - self.add_role(PongRole()) - self.add_role(PVRole()) +The definition of the agent classes itself now simply boils down to using the function :meth:`mango.agent_composed_of`. +The following shows the fully rewriten PV/Controller example featuring the newly introduced Ping function. - class ControllerAgent(RoleAgent): - def __init__(self, container, known_agents): - super().__init__(container) - self.add_role(PingRole(known_agents, 2)) - self.add_role(ControllerRole(known_agents)) +.. testcode:: + async def main(): + my_codec = JSON() + my_codec.add_serializer(*AskFeedInMsg.__serializer__()) + my_codec.add_serializer(*SetMaxFeedInMsg.__serializer__()) + my_codec.add_serializer(*FeedInReplyMsg.__serializer__()) + my_codec.add_serializer(*MaxFeedInAck.__serializer__()) -This concludes the last part of our tutorial. -If you want to run the code, you don't need to await the run method of the controller anymore, -since everything now happens automatically within the roles. -In your ``main``, you can replace the line: - -.. code-block:: python - - await controller_agent.run() - -with the following line: - -.. code-block:: python + # dont forget to add our new serializers + my_codec.add_serializer(*Ping.__serializer__()) + my_codec.add_serializer(*Pong.__serializer__()) - await asyncio.sleep(5) + pv_container = create_tcp_container(addr=PV_CONTAINER_ADDRESS, codec=my_codec) -If you then run this code, you should receive the following output: + controller_container = create_tcp_container( + addr=CONTROLLER_CONTAINER_ADDRESS, codec=my_codec + ) - | Ping PV Agent 0: Received a ping with ID: 0 - | Ping PV Agent 1: Received a ping with ID: 1 - | Pong Controller: Received an expected pong with ID: 0 - | Pong Controller: Received an expected pong with ID: 1 - | Controller received feed_ins: [2.0, 1.0] - | PV Agent 0: Limiting my feed_in to 1.0 - | PV Agent 1: Limiting my feed_in to 1.0 - | Ping PV Agent 0: Received a ping with ID: 2 - | Ping PV Agent 1: Received a ping with ID: 3 - | Pong Controller: Received an expected pong with ID: 2 - | Pong Controller: Received an expected pong with ID: 3 - | Ping PV Agent 0: Received a ping with ID: 4 - | Ping PV Agent 1: Received a ping with ID: 5 - | Pong Controller: Received an expected pong with ID: 4 - | Pong Controller: Received an expected pong with ID: 5 + pv_agent_0 = agent_composed_of(PongRole(), PVRole(), + register_in=pv_container, + suggested_aid="PV Agent 0") + pv_agent_1 = agent_composed_of(PongRole(), PVRole(), + register_in=pv_container, + suggested_aid="PV Agent 1") -.. raw:: html + known_agents = [ + pv_agent_0.addr, + pv_agent_1.addr, + ] -
+ controller_agent = agent_composed_of(PingRole(known_agents, 2), ControllerRole(known_agents), + register_in=pv_container, suggested_aid="Controller") + + async with activate(controller_container, pv_container) as cl: + # no more run call since everything now happens automatically within the roles + await asyncio.sleep(5) + + asyncio.run(main()) + +.. testoutput:: + + Ping PV Agent 0: Received a ping with ID: 0 + Ping PV Agent 1: Received a ping with ID: 1 + Pong Controller: Received an expected pong with ID: 0 + Pong Controller: Received an expected pong with ID: 1 + Controller received feed_ins: [2.0, 1.0] + PV Agent 0: Limiting my feed_in to 1.0 + PV Agent 1: Limiting my feed_in to 1.0 + Ping PV Agent 0: Received a ping with ID: 2 + Ping PV Agent 1: Received a ping with ID: 3 + Pong Controller: Received an expected pong with ID: 2 + Pong Controller: Received an expected pong with ID: 3 + Ping PV Agent 0: Received a ping with ID: 4 + Ping PV Agent 1: Received a ping with ID: 5 + Pong Controller: Received an expected pong with ID: 4 + Pong Controller: Received an expected pong with ID: 5 diff --git a/mango/__init__.py b/mango/__init__.py index c9d99e6..e9fa5e2 100644 --- a/mango/__init__.py +++ b/mango/__init__.py @@ -1,4 +1,4 @@ -from .messages.message import create_acl +from .messages.message import create_acl, Performatives from .agent.core import Agent, AgentAddress from .agent.role import Role, RoleAgent, RoleContext from .container.factory import ( @@ -17,3 +17,4 @@ ) from .util.distributed_clock import DistributedClockAgent, DistributedClockManager from .util.clock import ExternalClock +from .messages.codecs import json_serializable, JSON, FastJSON, PROTOBUF diff --git a/mango/container/core.py b/mango/container/core.py index 2c37bdd..25da073 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -46,7 +46,6 @@ def __init__( self._aid_counter: int = 0 # counter for aids self.running: bool = False # True until self.shutdown() is called - self._no_agents_running: asyncio.Future = None # inbox for all incoming messages self.inbox: asyncio.Queue = None @@ -118,8 +117,6 @@ def register(self, agent: Agent, suggested_aid: str = None): :return The agent ID """ - if not self._no_agents_running or self._no_agents_running.done(): - self._no_agents_running = asyncio.Future() aid = self._reserve_aid(suggested_aid) self._agents[aid] = agent agent._do_register(self, aid) @@ -154,8 +151,6 @@ def deregister(self, aid): :return: """ del self._agents[aid] - if len(self._agents) == 0: - self._no_agents_running.set_result(True) @abstractmethod async def send_message( @@ -308,10 +303,6 @@ def dispatch_to_agent_process(self, pid: int, coro_func, *args): async def start(self): self.running: bool = True # True until self.shutdown() is called - self._no_agents_running: asyncio.Future = asyncio.Future() - self._no_agents_running.set_result( - True - ) # signals that currently no agent lives in this container # inbox for all incoming messages self.inbox: asyncio.Queue = asyncio.Queue() diff --git a/mango/express/api.py b/mango/express/api.py index aeb57c7..11d5441 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -210,7 +210,9 @@ class ComposedAgent(RoleAgent): pass -def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAgent: +def agent_composed_of( + *roles: Role, register_in: None | Container = None, suggested_aid: None | str = None +) -> ComposedAgent: """ Create an agent composed of the given `roles`. If a container is provided, the created agent is automatically registered with the container `register_in`. @@ -219,6 +221,8 @@ def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAg :param register_in: container in which the created agent is registered, if provided :type register_in: None | Container + :param suggested_aid: the suggested aid for registration + :type suggested_aid: str :return: the composed agent :rtype: ComposedAgent """ @@ -226,7 +230,7 @@ def agent_composed_of(*roles: Role, register_in: None | Container) -> ComposedAg for role in roles: agent.add_role(role) if register_in is not None: - register_in.register(agent) + register_in.register(agent, suggested_aid=suggested_aid) return agent From 70f74114fd5cf92768df0ae7ba62eee1de802ab5 Mon Sep 17 00:00:00 2001 From: Rico Schrage Date: Wed, 16 Oct 2024 21:01:51 +0200 Subject: [PATCH 15/15] Updating docs, implementing a lot of doctests. --- docs/source/agents-container.rst | 163 +++++++++++++++++++++------ docs/source/codecs.rst | 184 ++++++++----------------------- docs/source/message exchange.rst | 75 +++++++------ docs/source/role-api.rst | 158 ++++++++++++++++++++++---- docs/source/tutorial.rst | 12 +- mango/__init__.py | 8 +- mango/container/core.py | 14 --- mango/express/api.py | 2 +- 8 files changed, 365 insertions(+), 251 deletions(-) diff --git a/docs/source/agents-container.rst b/docs/source/agents-container.rst index e96854d..84adfb7 100644 --- a/docs/source/agents-container.rst +++ b/docs/source/agents-container.rst @@ -12,58 +12,153 @@ serialization and deserialization of messages. Container also help to to speed up message exchange between agents that run on the same physical hardware, as data that is exchanged between such agents will not have to be sent through the network. -In mango, a container is created using the classmethod ``mango.create_container``: +In mango, a container is created using factory methods: + +* :meth:`mango.create_tcp_container` +* :meth:`mango.create_mqtt_container` +* :meth:`mango.create_ec_container` + +Most of the time, the tcp container should be the default choice if you wanna create simulations, which run in real time using +a simple but fast network protocol. .. code-block:: python3 - @classmethod - async def create_container(cls, *, connection_type: str = 'tcp', codec: Codec = None, clock: Clock = None, - addr: Optional[Union[str, Tuple[str, int]]] = None, - proto_msgs_module=None, - **kwargs): + def create_tcp_container( + addr: str | tuple[str, int], + codec: Codec = None, + clock: Clock = None, + copy_internal_messages: bool = False, + **kwargs: dict[str, Any], + ) -> Container: + +The factory methods are asyncio-free, meaning most of the time you can create containers without a running asyncio loop. -The factory method is a coroutine, so it has to be scheduled within a running asyncio loop. A simple container, that uses plain tcp for message exchange can be created as follows: -.. code-block:: python3 +.. testcode:: import asyncio - from mango import create_container + from mango import create_tcp_container - async def get_simple_container(): - container = await create_container(addr=('localhost', 5555)) + def get_simple_container(): + container = create_tcp_container(addr=('localhost', 5555)) return container - simple_container = asyncio.run(get_simple_container())) + print(get_simple_container().addr) + +.. testoutput:: + + ('localhost', 5555) + +The container type depends totally on the factory method you invoke. Every supported type has its own class backing +the functionality. -A container can be parametrized regarding its connection type ('tcp' or 'MQTT') and -regarding the codec that is used for message serialization. The default codec is JSON (see section :doc:`codecs` for more information). It is also possible to -define the clock that an agents scheduler should use (see section scheduling). +define the clock that an agents scheduler should use (see page :doc:`scheduling`). + +Note, that container creation is different from container starting. Before you can work with a container +you will want to register Agents, and then start (or activate) the container. This shall be done using an +asynchronous context manager, which we provide by invoking :meth:`mango.activate`. + +.. testcode:: + + import asyncio + from mango import create_tcp_container, activate + + async def start_container(): + container = create_tcp_container(addr=('localhost', 5555)) + + async with activate(container) as c: + print("The container is activated now!") + await asyncio.sleep(0.1) # activate the container for 0.1 seconds, most of the time you want to include e.g. a condition to await + print("The container is automatically shut down, even on exceptions!") + + asyncio.run(start_container()) + +.. testoutput:: -After a container is created, it is waiting for incoming messages on the given address. -As soon as the container has some agents, it will distribute incoming messages -to the corresponding agents and allow agents to send messages to other agents. + The container is activated now! + The container is automatically shut down, even on exceptions! -At the end of its lifetime, a ``container`` should be shutdown by using the method ``shutdown()``. -It will then shutdown all agents that are still running -in this container and cancel running tasks. +At the end of its lifetime, a ``container`` the container will shutdown. This will be done by the context manager, so no need for the +user to worry about it. This will also shutdown all agents that are still running in this container and cancel running tasks. *************** mango agents *************** mango agents can be implemented by inheriting from the abstract class ``mango.Agent``. -This class provides basic functionality such as to register the agent at the container or -to constantly check the inbox for incoming messages. -Every agent lives in exactly one container and therefore an instance of a container has to be -provided when :py:meth:`__init__()` of an agent is called. -Custom agents that inherit from the ``Agent`` class have to call ``super().__init__(container, suggested_aid: str = None)__`` -on initialization. -This will register the agent at the provided container instance and will assign a unique agent id -(``self.aid``) to the agent. However, it is possible to suggest an aid by setting the variable ``suggested_aid`` to your aid wish. +This class provides basic functionality such as to scheduling convenience methods or to constantly check the inbox for incoming messages. +Every agent can live in exactly one container, to register an agent the method :meth:`mango.Container.register` can be used. This method will assign +the agent a generated agent id (aid) and enables the agent scheduling feature. + +However, it is possible to suggest an aid by setting the parameter ``suggested_aid`` of :meth:`mango.Container.register` to your aid wish. The aid is granted if there is no other agent with this id, and if the aid doesn't interfere with the default aid pattern, otherwise the generated aid will be used. To check if the aid is available beforehand, you can use ``container.is_aid_available``. -It will also create the task to check for incoming messages. + +Note that, custom agents that inherit from the ``Agent`` class have to call ``super().__init__()__`` on initialization. + +.. testcode:: + + from mango import Agent, create_tcp_container + + class MyAgent(Agent): + pass + + async def create_and_register_agent(): + container = create_tcp_container(addr=('localhost', 5555)) + + agent = container.register(MyAgent(), suggested_aid="CustomAgent") + return agent + + print(asyncio.run(create_and_register_agent()).aid) + +.. testoutput:: + + CustomAgent + +Further there are some important lifecycle methods you often want to implement: + +* :meth:`mango.Agent.on_ready` + * Called when all containers have been activated during the activate call, which started the container the agent is registered in. + * At this point all relevant containers have been started and the agent is already registered. This is the correct method for starting to send messages, even to other containers. +* :meth:`mango.Agent.on_register` + * Called when the Agent just has been registered. + * At this point the scheduler is initialized and the agent address is known, but no communication can happen yet. +* :meth:`mango.Agent.on_start` + * Called when the container of the agent has been started during activation. + * At this point internal communication is possible and depending on your setup external communication could be done too. + +Besides the lifecycle, one of the main functions implemented in Agents are message exchange function. For this part read :doc:`/message exchange`. + +********************************* +Express setup of mango simulation +********************************* + +It is not necessary to create the container all by yourself, as you often want to just distribute some agents evenly to a number of containers. This can be done +with an asynchronous context manager created by :meth:`mango.run_with_tcp` (:meth:`mango.run_with_mqtt` for MQTT protocol). This method just expects the number of containers +you want to start and the agents, which shall run in these containers. + +With this method sending a message to an agent in another container looks like this: + +.. testcode:: + + import asyncio + from mango import PrintingAgent, run_with_tcp + + async def run_with_tcp_example(): + agent_tuple = (PrintingAgent(), dict(aid="MyAgent")) + single_agent = PrintingAgent() + + async with run_with_tcp(2, agent_tuple, single_agent) as cl: + # cl is the list of containers, which are created internally + await agent_tuple[0].send_message("Hello, print me!", single_agent.addr) + await asyncio.sleep(0.1) + + asyncio.run(run_with_tcp_example()) + +.. testoutput:: + + Received: Hello, print me! with {'sender_id': 'MyAgent', 'sender_addr': ['127.0.0.1', 5555], 'receiver_id': 'agent0', 'network_protocol': 'tcp', 'priority': 0} *************** agent process @@ -74,12 +169,10 @@ register the agent in a slightly different way. .. code-block:: python3 process_handle = await main_container.as_agent_process( - agent_creator=lambda sub_container: TestAgent( - container, aid_main_agent, suggested_aid=f"process_agent1" - ) + agent_creator=lambda sub_container: sub_container.register(MyAgent(), suggested_aid=f"process_agent1") ) -The process_handle is awaitable and will finish exactly when the process is fully set up. Further, it contains the pid `process_handle.pid`. +The ``process_handle`` is awaitable and will finish exactly when the process is fully set up. Further, it contains the pid ``process_handle.pid``. Note that after the creation, the agent lives in a mirror container in another process. Therefore, it is not possible to interact with the agent directly from the main process. If you want to interact with the agent after the creation, it is possible to @@ -89,6 +182,6 @@ dispatch a task in the agent process using `dispatch_to_agent_process`. main_container.dispatch_to_agent_process( pid, - your_function, # will be called with the mirror container + x as arguments + your_function, # will be called with the mirror container + varargs as arguments ... # varargs, additional arguments you want to pass to your_function ) diff --git a/docs/source/codecs.rst b/docs/source/codecs.rst index 270f716..0f61f6e 100644 --- a/docs/source/codecs.rst +++ b/docs/source/codecs.rst @@ -30,7 +30,7 @@ Quickstart Consider a simple example class we wish to encode as json: -.. code-block:: python3 +.. testcode:: class MyClass: def __init__(self, x, y): @@ -55,25 +55,27 @@ Consider a simple example class we wish to encode as json: If we try to encode an object of ``MyClass`` without adding a serializer we get an SerializationError: -.. code-block:: python3 +.. testcode:: - codec = codecs.JSON() + from mango import JSON, SerializationError - my_object = MyClass("abc", 123) - encoded = codec.encode(my_object) + codec = JSON() -.. code-block:: bash + my_object = MyClass("abc", 123) + try: + encoded = codec.encode(my_object) + except SerializationError as e: + print(e) - python main.py - ... - mango.messages.codecs.SerializationError: No serializer found for type "" +.. testoutput:: + No serializer found for type "" We have to make the type known to the codec to use it: -.. code-block:: python3 +.. testcode:: - codec = codecs.JSON() + codec = JSON() codec.add_serializer(*MyClass.__serializer__()) my_object = MyClass("abc", 123) @@ -83,66 +85,62 @@ We have to make the type known to the codec to use it: print(my_object.x, my_object.y) print(decoded.x, decoded.y) -.. code-block:: bash +.. testoutput:: - python main.py abc 123 abc 123 All that is left to do now is to pass our codec to the container. This is done during container creation in the ``create_container`` method. -.. code-block:: python3 +.. testcode:: + + from mango import Agent, create_tcp_container, activate + import asyncio class SimpleReceivingAgent(Agent): - def __init__(self, container): - super().__init__(container) + def __init__(self): + super().__init__() def handle_message(self, content, meta): - print(f"{self.aid} received a message with content {content} and meta f{meta}") if isinstance(content, MyClass): print(content.x) print(content.y) async def main(): - codec = codecs.JSON() + codec = JSON() codec.add_serializer(*MyClass.__serializer__()) # codecs can be passed directly to the container # if no codec is passed a new instance of JSON() is created - sending_container = await create_container(addr=("localhost", 5556), codec=codec) - receiving_container = await create_container(addr=("localhost", 5555), codec=codec) - receiving_agent = SimpleReceivingAgent(receiving_container) - - # agents can now directly pass content of type MyClass to each other - my_object = MyClass("abc", 123) - await sending_container.send_message( - content=my_object, receiver_addr=("localhost", 5555), receiver_id="agent0" - ) + sending_container = create_tcp_container(addr=("localhost", 5556), codec=codec) + receiving_container = create_tcp_container(addr=("localhost", 5555), codec=codec) + receiving_agent = receiving_container.register(SimpleReceivingAgent()) - await receiving_container.shutdown() - await sending_container.shutdown() + async with activate(sending_container, receiving_container): + # agents can now directly pass content of type MyClass to each other + my_object = MyClass("abc", 123) + await sending_container.send_message( + content=my_object, receiver_addr=receiving_agent.addr + ) + await asyncio.sleep(0.1) + asyncio.run(main()) - if __name__ == "__main__": - asyncio.run(main()) +.. testoutput:: -.. code-block:: bash - - python main.py - agent0 received a message with content <__main__.MyClass object at 0x7f42c930edc0> and meta f{'sender_id': None, 'sender_addr': ['localhost', 5556], 'receiver_id': 'agent0', 'receiver_addr': ['localhost', 5555], 'performative': None, 'conversation_id': None, 'reply_by': None, 'in_reply_to': None, 'protocol': None, 'language': None, 'encoding': None, 'ontology': None, 'reply_with': None, 'network_protocol': 'tcp', 'priority': 0} abc 123 **@json_serializable decorator** In the above example we explicitely defined methods to (de)serialize our class. For simple classes, especially data classes, -we can achieve the same result (for json codecs) via the ``@json_serializable`` decorator. This creates the ``__asdict__``, +we can achieve the same result (for json codecs) via the :meth:`mango.json_serializable`` decorator. This creates the ``__asdict__``, ``__fromdict__`` and ``__serializer__`` functions in the class: -.. code-block:: python3 +.. testcode:: - from mango.messages.codecs import serializable + from mango import json_serializable, JSON @json_serializable class DecoratorData: @@ -151,20 +149,18 @@ we can achieve the same result (for json codecs) via the ``@json_serializable`` self.y = y self.z = z - def main(): - codec = codecs.JSON() - codec.add_serializer(*DecoratorData.__serializer__()) + codec = JSON() + codec.add_serializer(*DecoratorData.__serializer__()) - my_data = DecoratorData(1,2,3) - encoded = codec.encode(my_data) - decoded = codec.decode(encoded) + my_data = DecoratorData(1,2,3) + encoded = codec.encode(my_data) + decoded = codec.decode(encoded) - print(my_data.x, my_data.y, my_data.z) - print(decoded.x, decoded.y, decoded.z) + print(my_data.x, my_data.y, my_data.z) + print(decoded.x, decoded.y, decoded.z) -.. code-block:: bash +.. testoutput:: - python main.py 1 2 3 1 2 3 @@ -185,98 +181,6 @@ protobuf message object and a type id. This is necessary because in general the original type of a protobuf message can not be infered from its serialized form. - The ``ACLMessage`` class is encouraged to be used for fipa compliant agent communication. For ease of use it gets specially handled in the protobuf codec: Its content field may contain any proto object known to the codec and gets encoded with the associated type id just like a non-ACL message would be encoded into the generic message wrapper. - - -Here is an example class implementing a proto serializer for a proto message containing the same fields -as the example class: - -.. code-block:: python3 - - from msg_pb2 import MyOtherMsg - from mango.messages.message import ACLMessage - - class SomeOtherClass: - def __init__(self, x=1, y='abc', z=None) -> None: - self.x = x - self.y = y - if z is None: - self.z = {} - else: - self.z = z - - def __toproto__(self): - msg = MyOtherMsg() - msg.x = self.x - msg.y = self.y - msg.z = str(self.z) - return msg - - @classmethod - def __fromproto__(cls, data): - msg = MyOtherMsg() - msg.ParseFromString(data) - return cls(msg.x, msg.y, eval(msg.z)) - - @classmethod - def __protoserializer__(cls): - return cls, cls.__toproto__, cls.__fromproto__ - - def main(): - codec = codecs.PROTOBUF() - codec.add_serializer(*SomeOtherClass.__protoserializer__()) - - my_object = SomeOtherClass() - decoded = codec.decode(codec.encode(my_object)) - - wrapper = ACLMessage() - wrapper.content = my_object - w_decoded = codec.decode(codec.encode(wrapper)) - - print(my_object.x, my_object.y, my_object.z) - print(decoded.x, decoded.y, decoded.z) - print( - wrapper_decoded.content.x, - wrapper_decoded.content.y, - wrapper_decoded.content.z, - ) - -.. code-block:: bash - - python main.py - 1 2 abc123 {1: 'test', 2: 'data', 3: 123} - 1 2 abc123 {1: 'test', 2: 'data', 3: 123} - 1 2 abc123 {1: 'test', 2: 'data', 3: 123} - - -In case you want to directly pass proto objects as content to the codec (or as content to the containers ``send_message``) you can shorten this -process by making the proto type known to the codec using the ``register_proto_type`` function as in this example: - -.. code-block:: python3 - - from msg_pb2 import MyMsg - - def main(): - codec = codecs.PROTOBUF() - codec.register_proto_type(MyMsg) - - my_obj = MyMsg() - my_obj.content = b"some_bytes" - encoded = codec.encode(my_obj) - decoded = codec.decode(encoded) - - print(my_obj) - print(encoded) - print(decoded) - - -.. code-block:: bash - - python main.py - content: "some_bytes" - - b'\x08\x01\x12\x0c\x12\nsome_bytes' - content: "some_bytes" diff --git a/docs/source/message exchange.rst b/docs/source/message exchange.rst index b11cd11..0ae4cac 100644 --- a/docs/source/message exchange.rst +++ b/docs/source/message exchange.rst @@ -5,7 +5,7 @@ Message exchange ****************** Receiving messages ****************** -Custom agents that inherit from the ``Agent`` class are able to receive messages from +Custom agents that inherit from the :class:`mango.Agent` class are able to receive messages from other agents via the method ``handle_message``. Hence this method has to be overwritten. The structure of this method looks like this: @@ -13,33 +13,35 @@ Hence this method has to be overwritten. The structure of this method looks like @abstractmethod def handle_message(self, content, meta: Dict[str, Any]): - raise NotImplementedError -Once a message arrives at a container, -the container is responsible to deserialize the message and +Once a message arrives at a container, the container is responsible to deserialize the message and to split the content from all meta information. + While the meta information may include e. g. information about the sender of the message or about the performative, the content parameter holds the actual content of the message. -.. - **COMMENT** - The exact structure of the ``ACL-messages`` that are exchanged within - mango is described here ZZZ. **TODO** +There are two entries always present. + +* "sender_addr": protocol address of the sending agent +* "sender_id": aid of the sending agent + +As mango will usually expect an instance of :meth:`mango.AgentAddress` for describing addresses of agents, we +recommend to use :meth:`mango.sender_addr` to retreive the sender information from the meta data. A simple agent, that just prints the content and meta information of incoming messages could look like this: -.. code-block:: python3 +.. testcode:: from mango import Agent class SimpleReceivingAgent(Agent): - def __init__(self, container): - super().__init__(container) + def __init__(self): + super().__init__() def handle_message(self, content, meta): - print(f'{self.aid} received a message with content {content} and' + print(f'{self.aid} received a message with content {content} and ' f'meta {meta}') @@ -51,34 +53,39 @@ Agents are able to send messages to other agents via the container method send_m .. code-block:: python3 - async def send_message(self, content, - receiver_addr: Union[str, Tuple[str, int]], *, - receiver_id: Optional[str] = None, - **kwargs) -> bool: - + async def send_message(self, + content, + receiver_addr: AgentAddress, + **kwargs, + ) -> bool -To send a tcp message, the receiver address and receiver id (the agent id of the receiving agent) -has to be provided. -`content` defines the content of the message. -This will appear as the `content` argument at the receivers handle_message() method. +To send a tcp message, two parameters need to be set, ``content``, which defines the content of the message, and ``receiver_addr``, which describes the destination. The ``receiver_addr`` +needs to be provided as :class:`mango.AgentAddress`. In most cases this can be created using several convenience functions (:meth:`mango.sender_addr`, :meth:`mango.Agent.addr`), if that +is not possible it should be created with :meth:`mango.addr`. If you want to send an ACL-message use the method ``create_acl`` to create the ACL content and send it with the regular ``send_message``-method internally. -The argument ``acl_metadata`` enables to set all meta information of an acl message. -It expects a dictionary with the field name as string as a key and the field value as key. -For example: +The argument ``kwargs`` can be used to put custom key,value pairs in the metadata of the message. For some protocols it might be possible that these metadata is additionally +interpreted internally. -.. code-block:: python3 +With this knowledge, we can now send a message to the ``SimpleReceivingAgent``: + +.. testcode:: + + import asyncio + from mango import run_with_tcp + + async def send_to_receiving(): + receiving_agent = SimpleReceivingAgent() + sending_agent = SimpleReceivingAgent() + + async with run_with_tcp(1, receiving_agent, sending_agent) as cl: + await sending_agent.send_message("Hey!", receiving_agent.addr) + await asyncio.sleep(0.1) - from mango.messages.message import Performatives + asyncio.run(send_to_receiving()) - example_acl_metadata = { - 'performative': Performatives.inform, - 'sender_id': 'agent0', - 'sender_addr': ('localhost', 5555), - 'conversation_id': 'conversation01' - } +.. testoutput:: -The argument ``kwargs`` can be used to set specific configs, if the container is connected via MQTT -to a message broker. + agent0 received a message with content Hey! and meta {'sender_id': 'agent1', 'sender_addr': ('127.0.0.1', 5555), 'receiver_id': 'agent0', 'network_protocol': 'tcp', 'priority': 0} diff --git a/docs/source/role-api.rst b/docs/source/role-api.rst index e8db44c..04336b7 100644 --- a/docs/source/role-api.rst +++ b/docs/source/role-api.rst @@ -1,81 +1,199 @@ ======== Role-API ======== -Besides inheriting from the ``Agent``-class there is another option to integrate features into an agent: the role API. -The idea of using roles is to divide the functionality of an agent by responsibility in a structured way. The target of this API is to increase the reusability and the maintainability of the agents components. To achieve this the role API works with orchestration rather than inheriting to extend the agents features. +Besides inheriting from the :class:`Agent`-class there is another option to integrate features into an agent: the role API. +The idea of using roles is to divide the functionality of an agent by responsibility in a structured way. The objective of this API is to increase the reusability and the maintainability of the agents components. To achieve this the role API works with orchestration rather than inheriting to extend the agents features. *************** The RoleContext *************** -The role context is the API to the environment of the role. It provides functionality to interact with other roles, to send messages to other agents or simply to fetch some meta data. +The role context is the API to the environment of the role. +It provides functionality to interact with other roles, to send messages to other +agents or simply to fetch some meta data. ******** The Role ******** -To implement a role you have to extend the abstract class ``mango.Role``. Concrete instances of implementations can be assigned to the general ``mango.RoleAgent``. +To implement a role you have to extend the abstract class :meth:`mango.Role`. Concrete instances of implementations +can be assigned to the general :class:`mango.RoleAgent` with :meth:`mango.RoleAgent.add_role`. However, the +faster way to create an Agent with a set of roles is to use the API :meth:`mango.agent_composed_of`. + +.. testcode:: + + from mango import RoleAgent, Role, agent_composed_of + + class MyRole(Role): + pass + + # first way + my_role_agent = RoleAgent() + my_role_agent.add_role(MyRole()) + + # second way + my_composed_agent = agent_composed_of(MyRole()) + + print(type(my_role_agent.roles[0])) + print(type(my_composed_agent.roles[0])) + +.. testoutput:: + + + Lifecycle ********* -The first step in the roles life is the instantiation via ``__init__``. This is done by the user itself and can be used to configure the roles behavior. The next step is adding the role to a ``RoleAgent`` using ``add_role``. The role will get notified by this through the method ``setup``. After adding the role the ``RoleContext`` is available, which represents the environment (container, agent, other roles). When the role or agent got removed, or the container shut down, the hook-method ``on_stop`` will be called, so you can do some cleanup or send last messages before the life ends. +The first step in the roles life is the instantiation via ``__init__``. +This is done by the user itself and can be used to configure the roles behavior. + +The next step is adding the role to a RoleAgent using ``add_role`` or ``agent_composed_of``. +The role will get notified by this through the method ``setup``. After adding the role the ``RoleContext`` +is available, which represents the environment (container, agent, other roles). + +The next lifecycle hook-in is :meth:`mango.Role.on_start`, which is called when the container in which +the agent of the role lives is started. After, :meth:`mango.Role.on_ready` is called, when all +containers of the ``activate`` statement have been started. + +When the role or agent got removed, or the container shut down, the hook-method ``on_stop`` will be called, so you can do some cleanup or send last messages before the life ends. + +.. testcode:: + + import asyncio + from mango import Role, agent_composed_of, run_with_tcp + + class LifecycleRole(Role): + def __init__(self): + print("Init") + def setup(self): + print("Setup") + def on_start(self): + print("Start") + def on_ready(self): + print("Ready") + async def on_stop(self): + print("Stop") + + async def show_lifecycle(): + async with run_with_tcp(1, agent_composed_of(LifecycleRole())): + pass + + asyncio.run(show_lifecycle()) + +.. testoutput:: + + Init + Setup + Start + Ready + Stop .. note:: - After a shutdown or removal a role is **not** supposed to be reused! When you want to deactivate a role temporarily use the methods ``activate`` and ``deactivate`` of the RoleContext. + After a shutdown or removal a role is **not** supposed to be reused! If you want to deactivate a role temporarily use the methods ``activate`` and ``deactivate`` of the RoleContext. Sharing Data ************ -There are two possible was to share data between the roles. +There are two possible ways to share data between the roles. -1. Using the data container in the RoleContext (``RoleContext.data``) -2. Creating explicit models using the model API of the RoleContext(``RoleContext.get_or_create_model``) +1. Using the data container in the RoleContext :meth:`mango.RoleContext.data` +2. Creating explicit models using the model API of the RoleContext :meth:`mango.RoleContext.get_or_create_model` The first way is pretty straightforward. For example: -.. code-block:: python3 +.. testcode:: + + from mango import Role, agent_composed_of - ... class MyRole(Role): def setup(self): self.context.data.my_item = "hello" + agent = agent_composed_of(MyRole()) + print(agent.roles[0].context.data.my_item) + +.. testoutput:: + + hello + The stored entry ``my_item`` can be used in every other role of the same agent now. -The second way needs a bit more preparations. First we need to define a model as python class. The class object will be used as key, so every model-type can be stored exactly once. +The second way needs a bit more preparations. First we need to define a model as python class. +The class object will be used as key, so every model-type can be stored exactly once. -.. code-block:: python3 +.. testcode:: class MyModel: def __init__(self): self.my_item = "" - ... + class MyRole(Role): def setup(self): mymodel = self.context.get_or_create_model(MyModel) mymodel.my_item = 'hello' -One advantage of this approach is that a model is subscribable using the method ``RoleContext.subscribe_model``. To make use of this every time the models changed ``RoleContext.update`` has to be called. + agent = agent_composed_of(MyRole()) + print(agent.roles[0].context.get_or_create_model(MyModel).my_item) + +.. testoutput:: + + hello + + +One advantage of this approach is that a model is subscribable using the method :meth:`mango.RoleContext.subscribe_model`. +To make use of this every time the models changed :meth:`mango.RoleContext.update` has to be called. Handle Messages *************** -As in a normal agent implementation, roles can handle incoming messages. To add a message handler you can use ``RoleContext.subscribe_message``. This method expects, besides the role and a handle method, a message condition function. The handle method must have exactly two arguments (excl. ``self``) ``content`` and ``meta``. The condition function must have exactly one argument ``content``. The idea of the condition function is to allow to define a condition filtering incoming messages, so you only handle one type of message per handler. Furthermore you can define a ``priority`` of the message subscription, this will be used to determine the message dispatch order (lower number = earlier execution, default=0). +As in a normal agent implementation, roles can handle incoming messages. +To add a message handler you can use :meth:`mango.RoleContext.subscribe_message`. +This method expects, besides the role and a handle method, a message condition function. +The handle method must have exactly two arguments (excl. ``self``) ``content`` and ``meta``. +The condition function must have exactly one argument ``content``. +The idea of the condition function is to allow to define a condition filtering incoming messages, +so you only handle one type of message per handler. +Furthermore you can define a ``priority`` of the message subscription, this will be used to +determine the message dispatch order (lower number = earlier execution, default=0). + +.. testcode:: + + from mango import Role -.. code-block:: python3 + class Ping: + pass - ... class MyRole(Role): def setup(self): - self.context.subscribe_message(self, self.handle_ping, lambda content: isinstance(content, Ping)) + self.context.subscribe_message(self, + self.handle_ping, + lambda content, meta: isinstance(content, Ping) + ) def handle_ping(self, content, meta): print('Ping received!') + async def show_handle_sub(): + my_composed_agent = agent_composed_of(MyRole()) + async with run_with_tcp(1, my_composed_agent) as cl: + await cl[0].send_message(Ping(), my_composed_agent.addr) + + asyncio.run(show_handle_sub()) + +.. testoutput:: + + Ping received! Deactivate/Activate other Roles ******************************* -Sometimes you might want to deactivate the functionality of a whole role, for example when you entered a new coalition you don't want to accept new coalition invites. It would of course be possible to manage this case with shared data and controlling flags, but this requires a lot of additional code and might lead to errors when implementing it. Furthermore, it increases the complexity of the implemented roles. To tackle this scenario a native deactivation/activation of roles is possible in mango. To deactivate a role the method ``RoleContext.deactivate`` can be used. To activate it again, use ``RoleContext.activate``. When a role is deactivated + +Sometimes you might want to deactivate the functionality of a whole role, for example when +you entered a new coalition you don't want to accept new coalition invites. It would of course +be possible to manage this case with shared data and controlling flags, but this requires a lot +of additional code and might lead to errors when implementing it. Furthermore, it increases the +complexity of the implemented roles. To tackle this scenario a native deactivation/activation of +roles is possible in mango. To deactivate a role the method :meth:`mango.RoleContext.deactivate` +can be used. To activate it again, use :meth:`RoleContext.activate`. When a role is deactivated 1. it is not possible to handle messages anymore 2. the role will not get updates on shared models anymore diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst index 6ea7d95..e3bb8f8 100644 --- a/docs/source/tutorial.rst +++ b/docs/source/tutorial.rst @@ -637,12 +637,12 @@ responsibilities we can use the role API. The idea of using roles is to divide the functionality of an agent by responsibility in a structured way. A role is a python object that can be assigned to a RoleAgent. There are several lifecycle functions each role may implement: - - __init__ - where you do the initial object setup + - ``__init__`` - where you do the initial object setup - :meth:`mango.Role.setup` - which is called when the role is assigned to an agent - :meth:`mango.Role.on_start` - which is called when the container is started - :meth:`mango.Role.on_ready` - which is called when are activated -This distinction is relevant because not all features exist after construction with __init__. Most of the time +This distinction is relevant because not all features exist after construction with ``__init__``. Most of the time you want to implement :meth:`mango.Role.on_ready` for actions like message sending, or scheduling, because only since this point you can be sure that all relevant container are started and the agent the role belongs to has been registered. However, the setup of the role itself should be done in :meth:`mango.Role.setup`. @@ -651,20 +651,20 @@ This example covers: - role API basics - scheduling and periodic tasks -The key part of defining roles are their `__init__`, `setup`, and `on_ready` methods. +The key part of defining roles are their ``__init__``, :meth:`mango.Role.setup`, and :meth:`mango.Role.on_ready` methods. The first is called to create the role object. The second is called when the role is assigned to an agent. While the third is called when all containers are started using :meth:`mango.activate`. In our case, the main change is that the previous distinction of message types within `handle_message` is now done by subscribing to the corresponding message type to tell the agent it should forward these messages to this role. -The :meth:`mango.Role.subscribe_message` method expects, besides the role and a handle method, a message condition function. +The :meth:`mango.RoleContext.subscribe_message` method expects, besides the role and a handle method, a message condition function. The idea of the condition function is to allow to define a condition filtering incoming messages. Another idea is that sending messages from the role is now done via its context with the method: -`self.context.send_message`. +``self.context.send_message```. We first create the `Ping` role, which has to periodically send out its messages. We can use mango's scheduling API to handle -this for us via the :meth:`mango.RoleContext.schedule_periodic_tasks` function. This takes a coroutine to execute and a time +this for us via the :meth:`mango.RoleContext.schedule_periodic_task` function. This takes a coroutine to execute and a time interval. Whenever the time interval runs out the coroutine is triggered. With the scheduling API you can also run tasks at specific times. For a full overview we refer to the documentation. diff --git a/mango/__init__.py b/mango/__init__.py index e9fa5e2..369807e 100644 --- a/mango/__init__.py +++ b/mango/__init__.py @@ -17,4 +17,10 @@ ) from .util.distributed_clock import DistributedClockAgent, DistributedClockManager from .util.clock import ExternalClock -from .messages.codecs import json_serializable, JSON, FastJSON, PROTOBUF +from .messages.codecs import ( + json_serializable, + JSON, + FastJSON, + PROTOBUF, + SerializationError, +) diff --git a/mango/container/core.py b/mango/container/core.py index 25da073..6eeffef 100644 --- a/mango/container/core.py +++ b/mango/container/core.py @@ -129,20 +129,6 @@ def _get_aid(self, agent): return aid return None - def include(self, agent: A, suggested_aid: str = None) -> A: - """Include the agent in the container. Return the agent for - convenience. - - Args: - agent (Agent): the agent to be included - suggested_aid (str, optional): suggested aid for registration - - Returns: - _type_: the agent included - """ - self.register(agent, suggested_aid=suggested_aid) - return agent - def deregister(self, aid): """ Deregister an agent diff --git a/mango/express/api.py b/mango/express/api.py index 11d5441..5f2283f 100644 --- a/mango/express/api.py +++ b/mango/express/api.py @@ -236,7 +236,7 @@ def agent_composed_of( class PrintingAgent(Agent): def handle_message(self, content, meta: dict[str, Any]): - logging.info("Received: %s with %s", content, meta) + print(f"Received: {content} with {meta}") def sender_addr(meta: dict) -> AgentAddress: