From 159e811112157110744a789ef845058da771aca7 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Mon, 16 Oct 2023 12:45:33 +0530 Subject: [PATCH] refactor: use utility from openedex_events to reset application state --- CHANGELOG.rst | 8 +++++++ edx_event_bus_redis/__init__.py | 2 +- edx_event_bus_redis/internal/consumer.py | 23 ++++--------------- .../internal/tests/test_consumer.py | 2 +- requirements/base.in | 4 ++-- requirements/base.txt | 7 +++--- requirements/dev.txt | 7 +++--- requirements/doc.txt | 9 ++++---- requirements/quality.txt | 7 +++--- requirements/test.txt | 5 ++-- 10 files changed, 36 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9d4da68..ec3652b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,14 @@ Unreleased * +[0.3.3] - 2023-10-13 +************************************************ + +Added +===== +* Use utility from openedx_events to reset application state before processing + event + [0.3.2] - 2023-09-01 ************************************************ diff --git a/edx_event_bus_redis/__init__.py b/edx_event_bus_redis/__init__.py index dc14858..7d8f35a 100644 --- a/edx_event_bus_redis/__init__.py +++ b/edx_event_bus_redis/__init__.py @@ -5,6 +5,6 @@ from edx_event_bus_redis.internal.consumer import RedisEventConsumer from edx_event_bus_redis.internal.producer import create_producer -__version__ = '0.3.2' +__version__ = '0.3.3' default_app_config = 'edx_event_bus_redis.apps.EdxEventBusRedisConfig' # pylint: disable=invalid-name diff --git a/edx_event_bus_redis/internal/consumer.py b/edx_event_bus_redis/internal/consumer.py index b968695..d55abb7 100644 --- a/edx_event_bus_redis/internal/consumer.py +++ b/edx_event_bus_redis/internal/consumer.py @@ -6,12 +6,11 @@ from typing import Optional from django.conf import settings -from django.db import connection from edx_django_utils.monitoring import record_exception, set_custom_attribute from edx_toggles.toggles import SettingToggle from openedx_events.event_bus import EventBusConsumer from openedx_events.event_bus.avro.deserializer import deserialize_bytes_to_event_data -from openedx_events.tooling import OpenEdxPublicSignal +from openedx_events.tooling import OpenEdxPublicSignal, prepare_for_new_work_cycle from redis.exceptions import ConnectionError as RedisConnectionError from redis.exceptions import ResponseError from walrus import Database @@ -70,21 +69,6 @@ class EventConsumptionException(Exception): """ -def _reconnect_to_db_if_needed(): - """ - Reconnects the db connection if needed. - - This is important because Django only does connection validity/age checks as part of - its request/response cycle, which isn't in effect for the consume-loop. If we don't - force these checks, a broken connection will remain broken indefinitely. For most - consumers, this will cause event processing to fail. - """ - has_connection = bool(connection.connection) - requires_reconnect = has_connection and not connection.is_usable() - if requires_reconnect: - connection.connect() - - class RedisEventConsumer(EventBusConsumer): """ Construct consumer for the given topic and group. The consumer can then @@ -240,8 +224,9 @@ def _consume_indefinitely(self): if isinstance(redis_raw_msg, list): redis_raw_msg = redis_raw_msg[0] msg = RedisMessage.parse(redis_raw_msg, self.full_topic) - # Before processing, make sure our db connection is still active - _reconnect_to_db_if_needed() + # Before processing, try to make sure our application state is cleaned + # up as would happen at the start of a Django request/response cycle. + prepare_for_new_work_cycle() self.emit_signals_from_message(msg) consecutive_errors = 0 diff --git a/edx_event_bus_redis/internal/tests/test_consumer.py b/edx_event_bus_redis/internal/tests/test_consumer.py index f2055d7..4ad72ca 100644 --- a/edx_event_bus_redis/internal/tests/test_consumer.py +++ b/edx_event_bus_redis/internal/tests/test_consumer.py @@ -287,7 +287,7 @@ def test_consecutive_error_limit(self): assert mock_emit.call_args_list == [call(self.normal_message)] * exception_count assert exc_info.value.args == ("Too many consecutive errors, exiting (4 in a row)",) - @patch('edx_event_bus_redis.internal.consumer.connection') + @patch('openedx_events.tooling.connection') @ddt.data( (False, False, False), # no connection, don't reconnect (True, False, True), # connection unusable, reconnect expected diff --git a/requirements/base.in b/requirements/base.in index 021528e..fea8dc7 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,8 +2,8 @@ -c constraints.txt Django # Web application framework -# openedx-events 8.0.0 removes an argument from consumer initialization -openedx-events>=8.0.0 # Events API +# openedx-events 9.5.0 adds utitily to reset app state +openedx-events>=9.5.0 # Events API edx_django_utils edx_toggles diff --git a/requirements/base.txt b/requirements/base.txt index 8810375..90ddb42 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -18,7 +18,7 @@ click==8.1.7 # edx-django-utils code-annotations==1.6.0 # via edx-toggles -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/base.in @@ -39,6 +39,7 @@ edx-django-utils==5.10.1 # via # -r requirements/base.in # edx-toggles + # openedx-events edx-opaque-keys[django]==2.5.1 # via openedx-events edx-toggles==5.1.1 @@ -51,7 +52,7 @@ markupsafe==2.1.5 # via jinja2 newrelic==9.6.0 # via edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.0 # via -r requirements/base.in pbr==6.0.0 # via stevedore @@ -63,7 +64,7 @@ pymongo==3.13.0 # via edx-opaque-keys pynacl==1.5.0 # via edx-django-utils -python-slugify==8.0.3 +python-slugify==8.0.4 # via code-annotations pytz==2024.1 # via django diff --git a/requirements/dev.txt b/requirements/dev.txt index 1757498..18ec859 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -77,7 +77,7 @@ distlib==0.3.8 # via # -r requirements/ci.txt # virtualenv -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/quality.txt @@ -101,6 +101,7 @@ edx-django-utils==5.10.1 # via # -r requirements/quality.txt # edx-toggles + # openedx-events edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.6 @@ -156,7 +157,7 @@ newrelic==9.6.0 # via # -r requirements/quality.txt # edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.0 # via -r requirements/quality.txt packaging==23.2 # via @@ -250,7 +251,7 @@ pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.8.0 # via -r requirements/quality.txt -python-slugify==8.0.3 +python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations diff --git a/requirements/doc.txt b/requirements/doc.txt index e91dcf3..d2b9889 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -55,7 +55,7 @@ cryptography==42.0.2 # via secretstorage ddt==1.7.1 # via -r requirements/test.txt -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -83,6 +83,7 @@ edx-django-utils==5.10.1 # via # -r requirements/test.txt # edx-toggles + # openedx-events edx-opaque-keys[django]==2.5.1 # via # -r requirements/test.txt @@ -114,7 +115,7 @@ iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -jaraco-classes==3.3.0 +jaraco-classes==3.3.1 # via keyring jeepney==0.8.0 # via @@ -143,7 +144,7 @@ newrelic==9.6.0 # edx-django-utils nh3==0.2.15 # via readme-renderer -openedx-events==9.4.0 +openedx-events==9.5.0 # via -r requirements/test.txt packaging==23.2 # via @@ -198,7 +199,7 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt -python-slugify==8.0.3 +python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations diff --git a/requirements/quality.txt b/requirements/quality.txt index 446542c..46da8ef 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -47,7 +47,7 @@ ddt==1.7.1 # via -r requirements/test.txt dill==0.3.8 # via pylint -django==3.2.23 +django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/test.txt @@ -70,6 +70,7 @@ edx-django-utils==5.10.1 # via # -r requirements/test.txt # edx-toggles + # openedx-events edx-lint==5.3.6 # via -r requirements/quality.in edx-opaque-keys[django]==2.5.1 @@ -109,7 +110,7 @@ newrelic==9.6.0 # via # -r requirements/test.txt # edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.0 # via -r requirements/test.txt packaging==23.2 # via @@ -168,7 +169,7 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.8.0 # via -r requirements/test.txt -python-slugify==8.0.3 +python-slugify==8.0.4 # via # -r requirements/test.txt # code-annotations diff --git a/requirements/test.txt b/requirements/test.txt index 85e3604..c0e6e04 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -58,6 +58,7 @@ edx-django-utils==5.10.1 # via # -r requirements/base.txt # edx-toggles + # openedx-events edx-opaque-keys[django]==2.5.1 # via # -r requirements/base.txt @@ -85,7 +86,7 @@ newrelic==9.6.0 # via # -r requirements/base.txt # edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.0 # via -r requirements/base.txt packaging==23.2 # via pytest @@ -119,7 +120,7 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.8.0 # via -r requirements/test.in -python-slugify==8.0.3 +python-slugify==8.0.4 # via # -r requirements/base.txt # code-annotations