diff --git a/.github/workflows/test-mango.yml b/.github/workflows/test-mango.yml index 87d63b3..2743745 100644 --- a/.github/workflows/test-mango.yml +++ b/.github/workflows/test-mango.yml @@ -1,6 +1,6 @@ name: Test mango -on: +on: push: branches: - main @@ -37,8 +37,13 @@ jobs: sudo apt update sudo apt install --assume-yes mosquitto sudo service mosquitto start - pip3 install pytest - pip3 install coverage + pip3 install pytest coverage ruff + - name: Lint with ruff + run: | + # stop the build if there are Python syntax errors or undefined names + source venv/bin/activate + ruff check . + ruff format --check . - name: Test+Coverage run: | source venv/bin/activate diff --git a/examples/distributed_clock/clock_agent.py b/examples/distributed_clock/clock_agent.py index f893b12..f1c4328 100644 --- a/examples/distributed_clock/clock_agent.py +++ b/examples/distributed_clock/clock_agent.py @@ -3,6 +3,7 @@ from typing import TypedDict import numpy as np + from mango import Role, RoleAgent, create_container from mango.messages.message import Performatives from mango.util.clock import ExternalClock diff --git a/examples/distributed_clock/clock_manager.py b/examples/distributed_clock/clock_manager.py index 12923fc..dfabbad 100644 --- a/examples/distributed_clock/clock_manager.py +++ b/examples/distributed_clock/clock_manager.py @@ -4,7 +4,6 @@ from typing import TypedDict import pandas as pd -from serializer import mango_codec_factory from tqdm import tqdm from mango import Role, RoleAgent, create_container @@ -65,7 +64,7 @@ async def clear_market(self): self.demands.append(self.demand) else: logger.info("First opening does not have anything to clear") - price=0 + price = 0 acl_metadata = { "performative": Performatives.inform, "sender_id": self.context.aid, diff --git a/examples/rrule_event.py b/examples/rrule_event.py index 0372274..6c4a6fc 100644 --- a/examples/rrule_event.py +++ b/examples/rrule_event.py @@ -2,6 +2,7 @@ from datetime import datetime from dateutil import rrule + from mango import Agent, create_container from mango.util.clock import ExternalClock diff --git a/mango/modules/mqtt_module.py b/mango/modules/mqtt_module.py index a42cc02..2db2456 100644 --- a/mango/modules/mqtt_module.py +++ b/mango/modules/mqtt_module.py @@ -96,7 +96,9 @@ def conn(self, client, userdata, flags, reason_code, properties): :return: None """ - def on_disconnect(self, client, userdata, disconnect_flags, reason_code, properties): + def on_disconnect( + self, client, userdata, disconnect_flags, reason_code, properties + ): # pylint: disable=unused-argument """ Callback method on broker disconnect on paho mqtt framework diff --git a/mango/util/distributed_clock.py b/mango/util/distributed_clock.py index 14ec066..3990ad9 100644 --- a/mango/util/distributed_clock.py +++ b/mango/util/distributed_clock.py @@ -73,7 +73,7 @@ async def send_current_time(self, time=None): Args: time (number, optional): The current time which is set. Defaults to None. """ - time = time or self._scheduler.clock.time + time = time or self._scheduler.clock.time await self.broadcast(time, add_futures=False) async def wait_for_futures(self): @@ -87,7 +87,7 @@ async def wait_for_futures(self): # waits forever if manager was started first # as answer is never received await fut - + async def wait_all_online(self): """ sends a broadcast to ask for the next event to all expected addresses. @@ -109,14 +109,13 @@ async def wait_all_online(self): else: all_online = True - async def get_next_event(self): - '''Get the next event from the scheduler by requesting all known clock agents''' + """Get the next event from the scheduler by requesting all known clock agents""" self.schedules = [] await self.broadcast("next_event") await asyncio.sleep(0) await self.wait_for_futures() - + # wait for our container too await self.wait_all_done() next_activity = self._scheduler.clock.get_next_activity() @@ -142,11 +141,10 @@ async def distribute_time(self, time=None): Waits until the current container is done. Brodcasts the new time to all the other clock agents. Thn awaits until the work in the other agents is done and their next event is received. - + Args: time (number, optional): The new time which is set. Defaults to None. - Returns: number or None: The time at which the next event happens """ @@ -170,6 +168,7 @@ def handle_message(self, content: float, meta): if not self.stopped.done(): self.stopped.set_result(True) elif content == "next_event": + async def wait_done(): await self.wait_all_done() diff --git a/ruff.toml b/ruff.toml index e3f8b59..a1a8efc 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,7 +1,5 @@ target-version = "py38" -src = ["crawler"] - [lint] # Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. # Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or diff --git a/setup.cfg b/setup.cfg index 6547dc0..6279a4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [tool:pytest] -markers = +markers = mqtt -asyncio_default_fixture_loop_scope = function \ No newline at end of file +asyncio_default_fixture_loop_scope = function diff --git a/tests/integration_tests/test_distributed_clock.py b/tests/integration_tests/test_distributed_clock.py index f7878a7..26c353a 100644 --- a/tests/integration_tests/test_distributed_clock.py +++ b/tests/integration_tests/test_distributed_clock.py @@ -1,11 +1,11 @@ import asyncio import pytest + from mango import create_container from mango.messages.codecs import JSON from mango.util.clock import ExternalClock from mango.util.distributed_clock import DistributedClockAgent, DistributedClockManager -from mango.util.termination_detection import tasks_complete_or_sleeping JSON_CODEC = JSON() diff --git a/tests/integration_tests/test_message_roundtrip.py b/tests/integration_tests/test_message_roundtrip.py index 7d39a90..dc43697 100644 --- a/tests/integration_tests/test_message_roundtrip.py +++ b/tests/integration_tests/test_message_roundtrip.py @@ -1,7 +1,8 @@ import asyncio -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.core import Agent from mango.messages.codecs import JSON, PROTOBUF, FastJSON diff --git a/tests/integration_tests/test_message_roundtrip_mp.py b/tests/integration_tests/test_message_roundtrip_mp.py index 3ca17fe..800fba8 100644 --- a/tests/integration_tests/test_message_roundtrip_mp.py +++ b/tests/integration_tests/test_message_roundtrip_mp.py @@ -1,7 +1,8 @@ import asyncio -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.core import Agent diff --git a/tests/integration_tests/test_single_container_termination.py b/tests/integration_tests/test_single_container_termination.py index 4332586..8458dc1 100644 --- a/tests/integration_tests/test_single_container_termination.py +++ b/tests/integration_tests/test_single_container_termination.py @@ -1,14 +1,23 @@ import asyncio import pytest + from mango import Agent, create_container from mango.util.clock import ExternalClock -from mango.util.termination_detection import tasks_complete_or_sleeping from mango.util.distributed_clock import DistributedClockAgent, DistributedClockManager +from mango.util.termination_detection import tasks_complete_or_sleeping class Caller(Agent): - def __init__(self, container, receiver_addr, receiver_id, send_response_messages=False, max_count=100, schedule_timestamp=False): + def __init__( + self, + container, + receiver_addr, + receiver_id, + send_response_messages=False, + max_count=100, + schedule_timestamp=False, + ): super().__init__(container) self.schedule_timestamp_task( coroutine=self.send_hello_world(receiver_addr, receiver_id), @@ -27,15 +36,18 @@ async def send_hello_world(self, receiver_addr, receiver_id): async def send_ordered(self, meta): await self.send_acl_message( - receiver_addr=meta["sender_addr"], receiver_id=meta["sender_id"], content=self.i + receiver_addr=meta["sender_addr"], + receiver_id=meta["sender_id"], + content=self.i, ) - def handle_message(self, content, meta): self.i += 1 if self.i < self.max_count and self.send_response_messages: if self.schedule_timestamp: - self.schedule_timestamp_task(self.send_ordered(meta), self.current_timestamp+5) + self.schedule_timestamp_task( + self.send_ordered(meta), self.current_timestamp + 5 + ) else: self.schedule_instant_task(self.send_ordered(meta)) elif not self.done.done(): @@ -122,9 +134,15 @@ async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] ) receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller(container_man, repl_addr, receiver.aid, send_response_messages=True, max_count=max_count) + caller = Caller( + container_man, + repl_addr, + receiver.aid, + send_response_messages=True, + max_count=max_count, + ) - clock_man.set_time(clock_man.time+5) + clock_man.set_time(clock_man.time + 5) # we do not have distributed termination detection yet in core assert caller.i < caller.max_count @@ -138,7 +156,10 @@ async def distribute_ping_pong_test(connection_type, codec=None, max_count=100): container_ag.shutdown(), ) -async def distribute_ping_pong_test_timestamp(connection_type, codec=None, max_count=10): + +async def distribute_ping_pong_test_timestamp( + connection_type, codec=None, max_count=10 +): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" @@ -177,12 +198,20 @@ async def distribute_ping_pong_test_timestamp(connection_type, codec=None, max_c container_man, receiver_clock_addresses=[(repl_addr, "clock_agent")] ) receiver = Receiver(container_ag, init_addr, "agent0") - caller = Caller(container_man, repl_addr, receiver.aid, send_response_messages=True, max_count=max_count, schedule_timestamp=True) + caller = Caller( + container_man, + repl_addr, + receiver.aid, + send_response_messages=True, + max_count=max_count, + schedule_timestamp=True, + ) # we do not have distributed termination detection yet in core assert caller.i < caller.max_count import time + tt = 0 if isinstance(clock_man, ExternalClock): for i in range(caller.max_count): @@ -190,7 +219,7 @@ async def distribute_ping_pong_test_timestamp(connection_type, codec=None, max_c t = time.time() await clock_manager.send_current_time() next_event = await clock_manager.get_next_event() - tt += time.time()-t + tt += time.time() - t clock_man.set_time(next_event) @@ -209,21 +238,22 @@ async def distribute_ping_pong_test_timestamp(connection_type, codec=None, max_c async def test_distribute_ping_pong_tcp(): await distribute_ping_pong_test("tcp") + @pytest.mark.asyncio async def test_distribute_ping_pong_mqtt(): await distribute_ping_pong_test("mqtt") + @pytest.mark.asyncio async def test_distribute_ping_pong_ts_tcp(): await distribute_ping_pong_test_timestamp("tcp") + @pytest.mark.asyncio async def test_distribute_ping_pong_ts_mqtt(): await distribute_ping_pong_test_timestamp("mqtt") - - async def distribute_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" @@ -265,7 +295,6 @@ async def distribute_time_test_case(connection_type, codec=None): receiver = Receiver(container_ag, init_addr, "agent0") caller = Caller(container_man, repl_addr, receiver.aid) - assert receiver._scheduler.clock.time == 0 # first synchronize the clock to the receiver next_event = await clock_manager.distribute_time(clock_man.time) @@ -302,6 +331,7 @@ async def distribute_time_test_case(connection_type, codec=None): container_ag.shutdown(), ) + async def send_current_time_test_case(connection_type, codec=None): init_addr = ("localhost", 1555) if connection_type == "tcp" else "c1" repl_addr = ("localhost", 1556) if connection_type == "tcp" else "c2" @@ -394,6 +424,7 @@ async def test_distribute_time_mqtt(): async def test_send_current_time_tcp(): await send_current_time_test_case("tcp") + @pytest.mark.asyncio async def test_send_current_time_mqtt(): await send_current_time_test_case("mqtt") diff --git a/tests/unit_tests/clock/test_external_clock.py b/tests/unit_tests/clock/test_external_clock.py index 388e298..2f389e7 100644 --- a/tests/unit_tests/clock/test_external_clock.py +++ b/tests/unit_tests/clock/test_external_clock.py @@ -2,6 +2,7 @@ import time import pytest + from mango.util.clock import AsyncioClock, ExternalClock from mango.util.scheduling import Scheduler diff --git a/tests/unit_tests/container/test_mp.py b/tests/unit_tests/container/test_mp.py index 7cd3235..4cd511d 100644 --- a/tests/unit_tests/container/test_mp.py +++ b/tests/unit_tests/container/test_mp.py @@ -1,6 +1,7 @@ import asyncio import pytest + from mango import Agent, create_container diff --git a/tests/unit_tests/container/test_tcp.py b/tests/unit_tests/container/test_tcp.py index 133e037..b04ccda 100644 --- a/tests/unit_tests/container/test_tcp.py +++ b/tests/unit_tests/container/test_tcp.py @@ -1,6 +1,7 @@ import asyncio import pytest + from mango import create_container from mango.container.protocol import ContainerProtocol from mango.container.tcp import TCPConnectionPool diff --git a/tests/unit_tests/core/test_agent.py b/tests/unit_tests/core/test_agent.py index 85c408e..df8c436 100644 --- a/tests/unit_tests/core/test_agent.py +++ b/tests/unit_tests/core/test_agent.py @@ -2,6 +2,7 @@ from typing import Any, Dict import pytest + from mango import create_container from mango.agent.core import Agent diff --git a/tests/unit_tests/core/test_container.py b/tests/unit_tests/core/test_container.py index cb4fef9..c36c2b8 100644 --- a/tests/unit_tests/core/test_container.py +++ b/tests/unit_tests/core/test_container.py @@ -1,5 +1,6 @@ -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.core import Agent diff --git a/tests/unit_tests/core/test_external_scheduling_container.py b/tests/unit_tests/core/test_external_scheduling_container.py index befe587..021a4fc 100644 --- a/tests/unit_tests/core/test_external_scheduling_container.py +++ b/tests/unit_tests/core/test_external_scheduling_container.py @@ -1,8 +1,9 @@ import asyncio from typing import Any, Dict -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.core import Agent from mango.container.external_coupling import ExternalAgentMessage from mango.container.factory import EXTERNAL_CONNECTION diff --git a/tests/unit_tests/messages/test_codecs.py b/tests/unit_tests/messages/test_codecs.py index 2225671..31d2a96 100644 --- a/tests/unit_tests/messages/test_codecs.py +++ b/tests/unit_tests/messages/test_codecs.py @@ -2,6 +2,7 @@ from dataclasses import dataclass import pytest + from mango.messages.codecs import ( JSON, PROTOBUF, diff --git a/tests/unit_tests/role_agent_test.py b/tests/unit_tests/role_agent_test.py index 2778ab7..dbfe2de 100644 --- a/tests/unit_tests/role_agent_test.py +++ b/tests/unit_tests/role_agent_test.py @@ -3,8 +3,9 @@ from abc import abstractmethod from typing import Any, Dict -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.role import Role, RoleAgent, RoleContext from mango.util.scheduling import TimestampScheduledTask diff --git a/tests/unit_tests/test_agents.py b/tests/unit_tests/test_agents.py index 595e8d0..6069425 100644 --- a/tests/unit_tests/test_agents.py +++ b/tests/unit_tests/test_agents.py @@ -1,8 +1,9 @@ import asyncio from typing import Any, Dict -import mango.container.factory as container_factory import pytest + +import mango.container.factory as container_factory from mango.agent.core import Agent diff --git a/tests/unit_tests/util/scheduling_test.py b/tests/unit_tests/util/scheduling_test.py index b7497c6..c64baec 100644 --- a/tests/unit_tests/util/scheduling_test.py +++ b/tests/unit_tests/util/scheduling_test.py @@ -4,6 +4,7 @@ import pytest from dateutil import rrule + from mango.util.clock import ExternalClock from mango.util.scheduling import ( ConditionalProcessTask,