Tests: Setup for flaky tests behaviors (#227)

pyproject.toml

@@ -112,6 +112,7 @@ test = [
     "pytest-asyncio~=0.21.0",
     "trove-classifiers>=2023.11.9",
     "trustme~=1.0",
+    "pytest-retry~=1.5",
 ]
 coverage = [
     "coverage~=7.0",

tests/functional_test/test_communication/test_async/test_client/test_tcp.py

@@ -31,6 +31,7 @@ async def readline(loop: asyncio.AbstractEventLoop, sock: Socket) -> bytes:
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures("simulate_no_ssl_module", "use_asyncio_transport")
+@pytest.mark.flaky(retries=3, delay=1)
 class TestAsyncTCPNetworkClient:
     @pytest.fixture
     @staticmethod

tests/functional_test/test_communication/test_async/test_client/test_udp.py

@@ -55,6 +55,7 @@ async def factory() -> DatagramEndpoint:
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures("use_asyncio_transport")
+@pytest.mark.flaky(retries=3, delay=1)
 class TestAsyncUDPNetworkClient:
     @pytest_asyncio.fixture
     @staticmethod

tests/functional_test/test_communication/test_async/test_server/test_tcp.py

@@ -289,6 +289,7 @@ class MyAsyncTCPServer(AsyncTCPNetworkServer[str, str]):
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(retries=3, delay=1)
 class TestAsyncTCPNetworkServer(BaseTestAsyncServer):
     @pytest.fixture(params=["NO_SSL", "USE_SSL"])
     @staticmethod
@@ -401,6 +402,7 @@ async def _wait_client_disconnected(writer: asyncio.StreamWriter, request_handle
         async with asyncio.timeout(1):
             while request_handler.connected_clients:
                 await asyncio.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     @pytest.mark.parametrize("host", [None, ""], ids=repr)
     @pytest.mark.parametrize("log_client_connection", [True, False], ids=lambda p: f"log_client_connection=={p}")
@@ -615,7 +617,6 @@ async def test____serve_forever____several_requests_at_same_time(
         client_address: tuple[Any, ...] = writer.get_extra_info("sockname")
 
         writer.write(b"hello\nworld\n")
-        await asyncio.sleep(0.1)
 
         assert await reader.readline() == b"HELLO\n"
         assert await reader.readline() == b"WORLD\n"
@@ -631,7 +632,6 @@ async def test____serve_forever____several_requests_at_same_time____close_betwee
         request_handler.close_client_after_n_request = 1
 
         writer.write(b"hello\nworld\n")
-        await asyncio.sleep(0.1)
 
         assert await reader.readline() == b"HELLO\n"
         assert await reader.read() == b""
@@ -646,7 +646,6 @@ async def test____serve_forever____save_request_handler_context(
         client_address: tuple[Any, ...] = writer.get_extra_info("sockname")
 
         writer.write(b"__wait__\nhello, world!\n")
-        await asyncio.sleep(0.1)
 
         assert await reader.readline() == b"After wait: hello, world!\n"
         assert request_handler.request_received[client_address] == ["hello, world!"]
@@ -660,7 +659,6 @@ async def test____serve_forever____bad_request(
         client_address: tuple[Any, ...] = writer.get_extra_info("sockname")
 
         writer.write("\u00E9\n".encode("latin-1"))  # StringSerializer does not accept unicode
-        await asyncio.sleep(0.1)
 
         assert await reader.readline() == b"wrong encoding man.\n"
         assert request_handler.request_received[client_address] == []
@@ -715,18 +713,18 @@ async def test____serve_forever____internal_error(
         }
 
         writer.write(b"something\n")
-        await asyncio.sleep(0.1)
 
         if mute_thrown_exception:
             assert await reader.readline() in expected_messages
             writer.write(b"something\n")
-            await asyncio.sleep(0.1)
             assert await reader.readline() in expected_messages
+            await asyncio.sleep(0.1)
             assert len(caplog.records) == 0  # After two attempts
         else:
             with contextlib.suppress(ConnectionError):
                 assert await reader.readline() in expected_messages
                 assert await reader.read() == b""
+            await asyncio.sleep(0.1)
             assert len(caplog.records) == 3
             assert caplog.records[1].exc_info is not None
             assert type(caplog.records[1].exc_info[1]) is RuntimeError
@@ -742,10 +740,10 @@ async def test____serve_forever____unexpected_error_during_process(
         reader, writer = await client_factory()
 
         writer.write(b"__error__\n")
-        await asyncio.sleep(0.1)
-
         with contextlib.suppress(ConnectionError):
             assert await reader.read() == b""
+        await asyncio.sleep(0.1)
+
         assert len(caplog.records) == 3
         assert caplog.records[1].exc_info is not None
         assert type(caplog.records[1].exc_info[1]) is RandomError
@@ -767,9 +765,9 @@ async def test____serve_forever____unexpected_error_during_response_serializatio
             await asyncio.sleep(0.1)
 
         writer.write(b"request\n")
+        assert await reader.read() == b""
         await asyncio.sleep(0.1)
 
-        assert await reader.read() == b""
         assert len(caplog.records) == 1
         assert caplog.records[0].levelno == logging.ERROR
         assert caplog.records[0].message == "RuntimeError: protocol.generate_chunks() crashed (caused by SystemError: CRASH)"
@@ -785,10 +783,9 @@ async def test____serve_forever____os_error(
         reader, writer = await client_factory()
 
         writer.write(b"__os_error__\n")
-        await asyncio.sleep(0.1)
-
         with contextlib.suppress(ConnectionError):
             assert await reader.read() == b""
+        await asyncio.sleep(0.1)
 
         assert len(caplog.records) == 3
         assert caplog.records[1].exc_info is not None
@@ -806,9 +803,9 @@ async def test____serve_forever____use_of_a_closed_client_in_request_handler(
         host, port = writer.get_extra_info("sockname")[:2]
 
         writer.write(b"__closed_client_error__\n")
+        assert await reader.read() == b""
         await asyncio.sleep(0.1)
 
-        assert await reader.read() == b""
         assert len(caplog.records) == 1
         assert caplog.records[0].levelno == logging.WARNING
         assert caplog.records[0].message == f"There have been attempts to do operation on closed client ({host!r}, {port})"
@@ -822,9 +819,9 @@ async def test____serve_forever____connection_error_in_request_handler(
         reader, writer = await client_factory()
 
         writer.write(b"__connection_error__\n")
+        assert await reader.read() == b""
         await asyncio.sleep(0.1)
 
-        assert await reader.read() == b""
         assert len(caplog.records) == 0
 
     async def test____serve_forever____connection_error_in_disconnect_hook(
@@ -853,7 +850,6 @@ async def test____serve_forever____explicitly_closed_by_request_handler(
         reader, writer = await client_factory()
 
         writer.write(b"__close__\n")
-        await asyncio.sleep(0.1)
 
         assert await reader.read() == b""
 
@@ -866,7 +862,6 @@ async def test____serve_forever____request_handler_ask_to_stop_accepting_new_con
         reader, writer = await client_factory()
 
         writer.write(b"__stop_listening__\n")
-        await asyncio.sleep(0.1)
 
         assert await reader.readline() == b"successfully stop listening\n"
 
@@ -923,7 +918,7 @@ async def test____serve_forever____request_handler_is_cancelled(
         reader, writer = await client_factory()
 
         writer.write(b"something\n")
-        await asyncio.sleep(0.1)
+
         with contextlib.suppress(ConnectionError):
             assert await reader.read() == b""
 
@@ -966,6 +961,8 @@ async def test____serve_forever____request_handler_did_not_yield(
                 assert await reader.readline() == b"something\n"
 
             assert await reader.read() == b""
+
+        await asyncio.sleep(0.1)
         assert len(caplog.records) == 0
 
     @pytest.mark.parametrize("request_handler", [InitialHandshakeRequestHandler], indirect=True)

tests/functional_test/test_communication/test_async/test_server/test_udp.py

@@ -208,6 +208,7 @@ class MyAsyncUDPServer(AsyncUDPNetworkServer[str, str]):
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(retries=3, delay=1)
 class TestAsyncUDPNetworkServer(BaseTestAsyncServer):
     @pytest.fixture
     @staticmethod

tests/functional_test/test_communication/test_sync/test_client/test_tcp.py

@@ -31,6 +31,7 @@ def readline(sock: Socket) -> bytes:
 
 
 @pytest.mark.usefixtures("simulate_no_ssl_module")
+@pytest.mark.flaky(retries=3, delay=1)
 class TestTCPNetworkClient:
     @pytest.fixture
     @staticmethod

tests/functional_test/test_communication/test_sync/test_client/test_udp.py

@@ -26,6 +26,7 @@ def bound_udp_socket_factory() -> Socket:
     return bound_udp_socket_factory
 
 
+@pytest.mark.flaky(retries=3, delay=1)
 class TestUDPNetworkClient:
     @pytest.fixture
     @staticmethod

tests/functional_test/test_concurrency/test_sync/test_threaded_client.py

@@ -15,6 +15,11 @@
 ClientType: TypeAlias = AbstractNetworkClient[str, str]
 
 
+pytestmark = [
+    pytest.mark.flaky(retries=3, delay=1),
+]
+
+
 @pytest.fixture
 def executor(client: ClientType) -> Iterator[ThreadPoolExecutor]:
     with ThreadPoolExecutor(max_workers=1) as executor: