From 0fc83c3c64964f0881fb25bd8f48244446a609a3 Mon Sep 17 00:00:00 2001 From: Sara Burns Date: Tue, 4 Jun 2024 16:10:14 -0400 Subject: [PATCH] fix: allow either whitelist or registry.mapping option --- .../management/commands/helpers/queued_sender.py | 2 +- .../management/commands/tests/fixtures/tracking.log | 2 +- .../commands/tests/test_transform_tracking_logs.py | 8 +++++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/event_routing_backends/management/commands/helpers/queued_sender.py b/event_routing_backends/management/commands/helpers/queued_sender.py index 3e60f88c..ad9af36e 100644 --- a/event_routing_backends/management/commands/helpers/queued_sender.py +++ b/event_routing_backends/management/commands/helpers/queued_sender.py @@ -51,7 +51,7 @@ def is_known_event(self, event): """ if "name" in event: for processor in self.engine.processors: - if event["name"] in processor.registry.mapping: + if event["name"] in processor.whitelist or event["name"] in processor.registry.mapping: return True return False diff --git a/event_routing_backends/management/commands/tests/fixtures/tracking.log b/event_routing_backends/management/commands/tests/fixtures/tracking.log index 9954ea42..ad486d77 100644 --- a/event_routing_backends/management/commands/tests/fixtures/tracking.log +++ b/event_routing_backends/management/commands/tests/fixtures/tracking.log @@ -1,6 +1,6 @@ 2023-05-23 13:53:13,461 INFO 20 [tracking] [user None] [ip None] logger.py:41 - {"name": "edx.grades.subsection.grade_calculated", "context": {"course_id": "course-v1:edX+DemoX+Demo_Course", "org_id": "edX", "enterprise_uuid": ""}, "username": "", "session": "", "ip": "", "agent": "", "host": "", "referer": "", "accept_language": "", "event": {"user_id": "6", "course_id": "course-v1:edX+DemoX+Demo_Course", "block_id": "block-v1:edX+DemoX+Demo_Course+type@sequential+block@19a30717eff543078a5d94ae9d6c18a5", "course_version": "6452821a39fb48e67c1a860c", "weighted_total_earned": 3.0, "weighted_total_possible": 3.0, "weighted_graded_earned": 3.0, "weighted_graded_possible": 3.0, "first_attempted": "2023-05-12 20:55:53.096285+00:00", "subtree_edited_timestamp": "2023-05-03 15:47:38.629000+00:00", "event_transaction_id": "45acd5c9-af65-45bd-82fe-313fa5bf9f79", "event_transaction_type": "edx.grades.problem.submitted", "visible_blocks_hash": "5RUTrtbgdsnoSKsFavs24Om8Yp0="}, "time": "2023-05-23T13:53:13.460372+00:00", "event_type": "edx.grades.subsection.grade_calculated", "event_source": "server", "page": null} 2023-05-23 13:53:13,477 INFO 20 [tracking] [user None] [ip None] logger.py:41 - {"name": "edx.grades.course.grade_calculated", "context": {"course_id": "course-v1:edX+DemoX+Demo_Course", "org_id": "edX", "enterprise_uuid": ""}, "username": "", "session": "", "ip": "", "agent": "", "host": "", "referer": "", "accept_language": "", "event": {"user_id": "6", "course_id": "course-v1:edX+DemoX+Demo_Course", "course_version": "6452821a39fb48e67c1a860c", "percent_grade": 0.03, "letter_grade": "", "course_edited_timestamp": "2023-05-03 15:47:38.629000+00:00", "event_transaction_id": "45acd5c9-af65-45bd-82fe-313fa5bf9f79", "event_transaction_type": "edx.grades.problem.submitted", "grading_policy_hash": "2HDb6cWz6xgUQ8b9YhsgN5sukP0="}, "time": "2023-05-23T13:53:13.475692+00:00", "event_type": "edx.grades.course.grade_calculated", "event_source": "server", "page": null} -2023-05-23 13:53:13,495 INFO 20 [tracking] [user None] [ip None] logger.py:41 - {"name": "edx.course.grade.now_failed", "context": "course_id": "course-v1:edX+DemoX+Demo_Course", "org_id": "edX", "enterprise_uuid": ""}, "username": "", "session": "", "ip": "", "agent": "", "host": "", "referer": "", "accept_language": "", "event": {"user_id": "6", "course_id": "course-v1:edX+DemoX+Demo_Course", "event_transaction_id": "45acd5c9-af65-45bd-82fe-313fa5bf9f79", "event_transaction_type": "edx.grades.problem.submitted"}, "time": "2023-05-23T13:53:13.486905+00:00", "event_type": "edx.course.grade.now_failed", "event_source": "server", "page": null} +2023-05-23 13:53:13,495 INFO 20 [tracking] [user None] [ip None] logger.py:41 - {"name": "edx.course.grade.now_failed", "context": {"course_id": "course-v1:edX+DemoX+Demo_Course", "org_id": "edX", "enterprise_uuid": ""}, "username": "", "session": "", "ip": "", "agent": "", "host": "", "referer": "", "accept_language": "", "event": {"user_id": "6", "course_id": "course-v1:edX+DemoX+Demo_Course", "event_transaction_id": "45acd5c9-af65-45bd-82fe-313fa5bf9f79", "event_transaction_type": "edx.grades.problem.submitted"}, "time": "2023-05-23T13:53:13.486905+00:00", "event_type": "edx.course.grade.now_failed", "event_source": "server", "page": null} 2023-05-23 14:12:17,290 INFO 27 [tracking] [user 6] [ip 172.18.0.1] logger.py:41 - {"name": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a/handler/xmodule_handler/problem_check", "context": {"course_id": "course-v1:edX+DemoX+Demo_Course", "course_user_tags": {}, "user_id": 6, "path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a/handler/xmodule_handler/problem_check", "org_id": "edX", "enterprise_uuid": ""}, "username": "bmtcril", "session": "79e015634b37255d8cb5fcdef0a73ad4", "ip": "172.18.0.1", "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0", "host": "local.overhang.io", "referer": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@134df56c516a4a0dbb24dd5facef746e?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view", "accept_language": "en-US,en;q=0.5", "event": "{\"GET\": {}, \"POST\": {\"input_932e6f2ce8274072a355a94560216d1a_2_1\": [\"choice_2\"]}}", "time": "2023-05-23T14:12:17.286772+00:00", "event_type": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a/handler/xmodule_handler/problem_check", "event_source": "server", "page": null} 2023-05-23 14:12:17,308 INFO 7 [tracking] [user 6] [ip 172.18.0.1] logger.py:41 - {"name": "problem_check", "context": {"user_id": 6, "path": "/event", "course_id": "course-v1:edX+DemoX+Demo_Course", "org_id": "edX", "enterprise_uuid": ""}, "username": "bmtcril", "session": "79e015634b37255d8cb5fcdef0a73ad4", "ip": "172.18.0.1", "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0", "host": "local.overhang.io", "referer": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@134df56c516a4a0dbb24dd5facef746e?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view", "accept_language": "en-US,en;q=0.5", "event": "input_932e6f2ce8274072a355a94560216d1a_2_1=choice_2", "time": "2023-05-23T14:12:17.299491+00:00", "event_type": "problem_check", "event_source": "browser", "page": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@134df56c516a4a0dbb24dd5facef746e?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view"} 2023-05-23 14:12:17,388 INFO 27 [tracking] [user 6] [ip 172.18.0.1] logger.py:41 - {"name": "edx.grades.problem.submitted", "context": {"course_id": "course-v1:edX+DemoX+Demo_Course", "course_user_tags": {}, "user_id": 6, "path": "/courses/course-v1:edX+DemoX+Demo_Course/xblock/block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a/handler/xmodule_handler/problem_check", "org_id": "edX", "enterprise_uuid": "", "module": {"display_name": "Perchance to Dream", "usage_key": "block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a"}}, "username": "bmtcril", "session": "79e015634b37255d8cb5fcdef0a73ad4", "ip": "172.18.0.1", "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/113.0", "host": "local.overhang.io", "referer": "http://local.overhang.io/xblock/block-v1:edX+DemoX+Demo_Course+type@vertical+block@134df56c516a4a0dbb24dd5facef746e?show_title=0&show_bookmark_button=0&recheck_access=1&view=student_view", "accept_language": "en-US,en;q=0.5", "event": {"user_id": "6", "course_id": "course-v1:edX+DemoX+Demo_Course", "problem_id": "block-v1:edX+DemoX+Demo_Course+type@problem+block@932e6f2ce8274072a355a94560216d1a", "event_transaction_id": "aee73a98-8bf3-42ca-99a5-ac008eb62c1d", "event_transaction_type": "edx.grades.problem.submitted", "weighted_earned": 1, "weighted_possible": 1}, "time": "2023-05-23T14:12:17.388051+00:00", "event_type": "edx.grades.problem.submitted", "event_source": "server", "page": null} diff --git a/event_routing_backends/management/commands/tests/test_transform_tracking_logs.py b/event_routing_backends/management/commands/tests/test_transform_tracking_logs.py index 3bb47670..c6bf706e 100644 --- a/event_routing_backends/management/commands/tests/test_transform_tracking_logs.py +++ b/event_routing_backends/management/commands/tests/test_transform_tracking_logs.py @@ -76,6 +76,7 @@ def command_options(): "Sending to LRS!" ] }, + "registry_mapping": {"problem_check": 1}, }, # Remote file to LRS dry run no batch size { @@ -94,6 +95,7 @@ def command_options(): "Queued 2 log lines, could not parse 2 log lines, skipped 8 log lines, sent 0 batches.", ] }, + "registry_mapping": {"problem_check": 1}, }, # Remote file to LRS, default batch size { @@ -112,6 +114,7 @@ def command_options(): "Queued 2 log lines, could not parse 2 log lines, skipped 8 log lines, sent 1 batches.", ] }, + "whitelist": ["problem_check"] }, # Local file to remote file { @@ -135,6 +138,7 @@ def command_options(): "Queued 2 log lines, could not parse 2 log lines, skipped 8 log lines, sent 2 batches.", ] }, + "whitelist": ["problem_check"] }, # Remote file dry run { @@ -157,6 +161,7 @@ def command_options(): "Queued 2 log lines, could not parse 2 log lines, skipped 8 log lines, sent 0 batches.", ] }, + "whitelist": ["problem_check"] }, ] @@ -208,7 +213,8 @@ def test_transform_command(command_opts, mock_common_calls, caplog, capsys): mm2 = MagicMock() # Fake a router mapping so some events in the log are actually processed - mm2.registry.mapping = {"problem_check": 1} + mm2.registry.mapping = command_opts.pop("registry_mapping", {}) + mm2.whitelist = command_opts.pop("whitelist", []) # Fake a process response that can be serialized to json mm2.return_value = {"foo": "bar"} tracker.backends["event_transformer"].processors = [mm2]