From 34b65ecfe1e7c12b5e8c07c507a2c7282dc14a05 Mon Sep 17 00:00:00 2001 From: ekneg54 Date: Tue, 12 Nov 2024 12:03:25 +0100 Subject: [PATCH] speed up exporter tests by introducing asgiref async testing --- .vscode/settings.json | 5 +- pyproject.toml | 12 +++-- tests/unit/metrics/test_exporter.py | 83 +++++++++++++++++++---------- 3 files changed, 66 insertions(+), 34 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4681437a4..3d4b83d9b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "python.testing.pytestEnabled": true, "python.testing.unittestEnabled": false, "python.analysis.importFormat": "absolute", + "python.testing.pytestPath": "pytest", "editor.formatOnSave": true, "editor.rulers": [ 80, @@ -9,7 +10,9 @@ 120 ], "editor.codeActionsOnSave": { - "source.organizeImports": "explicit" + "source.organizeImports": "explicit", + "source.unusedImports": "always", + "source.convertImportFormat": "always" }, "autoDocstring.docstringFormat": "numpy", "files.exclude": { diff --git a/pyproject.toml b/pyproject.toml index 85d38a036..61f54fed8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,10 +56,10 @@ keywords = [ "logdata", ] dependencies = [ - "aiohttp>=3.9.2", # CVE-2024-23334 + "aiohttp>=3.9.2", # CVE-2024-23334 "attrs", - "certifi>=2023.7.22", # CVE-2023-37920 - "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 + "certifi>=2023.7.22", # CVE-2023-37920 + "ciso8601", # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11 "colorama", "confluent-kafka>2", "geoip2", @@ -67,7 +67,7 @@ dependencies = [ "jsonref", "luqum", "more-itertools==8.10.0", - "mysql-connector-python>=9.1.0", # CVE-2024-21272 + "mysql-connector-python>=9.1.0", # CVE-2024-21272 "numpy>=1.26.0", "opensearch-py", "prometheus_client", @@ -84,7 +84,7 @@ dependencies = [ "schedule", "tldextract", "urlextract", - "urllib3>=1.26.17", # CVE-2023-43804 + "urllib3>=1.26.17", # CVE-2023-43804 "uvicorn", "deepdiff", "msgspec", @@ -113,6 +113,8 @@ dev = [ "jinja2", "maturin", "cibuildwheel", + "asgiref", + "pytest-asyncio", ] doc = [ diff --git a/tests/unit/metrics/test_exporter.py b/tests/unit/metrics/test_exporter.py index 4ff498888..67d86ee72 100644 --- a/tests/unit/metrics/test_exporter.py +++ b/tests/unit/metrics/test_exporter.py @@ -2,14 +2,16 @@ # pylint: disable=protected-access # pylint: disable=attribute-defined-outside-init # pylint: disable=line-too-long +import asyncio import os.path from unittest import mock import pytest import requests -from prometheus_client import REGISTRY +from asgiref.testing import ApplicationCommunicator +from prometheus_client import REGISTRY, CollectorRegistry -from logprep.metrics.exporter import PrometheusExporter +from logprep.metrics.exporter import PrometheusExporter, make_patched_asgi_app from logprep.util import http from logprep.util.configuration import MetricsConfig @@ -148,38 +150,63 @@ def test_health_endpoint_calls_updated_functions(self): exporter.server.shut_down() - @pytest.mark.parametrize( - "functions, expected", - [ - ([lambda: True], 200), - ([lambda: True, lambda: True], 200), - ([lambda: False], 503), - ([lambda: False, lambda: False], 503), - ([lambda: False, lambda: True, lambda: True], 503), - ], - ) - def test_health_check_returns_status_code(self, functions, expected): + def test_health_check_returns_body_and_status_code(self): exporter = PrometheusExporter(self.metrics_config) exporter.run(daemon=False) - exporter.update_healthchecks(functions) + exporter.update_healthchecks([lambda: True]) resp = requests.get("http://localhost:8000/health", timeout=0.5) - assert resp.status_code == expected + assert resp.status_code == 200 + assert resp.content.decode() == "OK" exporter.server.shut_down() + +class TestAsgiApp: + """These tests uses the `asgiref.testing.ApplicationCommunicator` to test the ASGI app itself + For more information see: https://dokk.org/documentation/django-channels/2.4.0/topics/testing/ + """ + + def setup_method(self): + self.registry = CollectorRegistry() + self.captured_status = None + self.captured_headers = None + # Setup ASGI scope + self.scope = { + "client": ("127.0.0.1", 32767), + "headers": [], + "http_version": "1.0", + "method": "GET", + "path": "/", + "query_string": b"", + "scheme": "http", + "server": ("127.0.0.1", 80), + "type": "http", + } + self.communicator = None + + def teardown_method(self): + if self.communicator: + asyncio.get_event_loop().run_until_complete(self.communicator.wait()) + + def seed_app(self, app): + self.communicator = ApplicationCommunicator(app, self.scope) + @pytest.mark.parametrize( - "functions, expected", + "functions, expected_status, expected_body", [ - ([lambda: True], "OK"), - ([lambda: True, lambda: True], "OK"), - ([lambda: False], "FAIL"), - ([lambda: False, lambda: False], "FAIL"), - ([lambda: False, lambda: True, lambda: True], "FAIL"), + ([lambda: True], 200, b"OK"), + ([lambda: True, lambda: True], 200, b"OK"), + ([lambda: False], 503, b"FAIL"), + ([lambda: False, lambda: False], 503, b"FAIL"), + ([lambda: False, lambda: True, lambda: True], 503, b"FAIL"), ], ) - def test_health_check_returns_body(self, functions, expected): - exporter = PrometheusExporter(self.metrics_config) - exporter.run(daemon=False) - exporter.update_healthchecks(functions) - resp = requests.get("http://localhost:8000/health", timeout=0.5) - assert resp.content.decode() == expected - exporter.server.shut_down() + @pytest.mark.asyncio + async def test_asgi_app(self, functions, expected_status, expected_body): + app = make_patched_asgi_app(functions) + self.scope["path"] = "/health" + self.seed_app(app) + await self.communicator.send_input({"type": "http.request"}) + event = await self.communicator.receive_output(timeout=1) + assert event["status"] == expected_status + event = await self.communicator.receive_output(timeout=1) + assert expected_body in event["body"]