From 40025a429f30b947ae2a0065786820c478d33748 Mon Sep 17 00:00:00 2001 From: Hakan Date: Thu, 9 May 2024 14:31:51 +0300 Subject: [PATCH 1/4] update config and setup for rabbitmq as celery broker --- docker-compose-dev.yml | 15 +++++++++++++++ docker-compose.yml | 17 ++++++++++++++++- src/etools_datamart/config/settings.py | 5 +++-- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 0fbdc6c7..c01f5957 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -112,3 +112,18 @@ services: redis: image: redis:alpine container_name: datamart_redis + + rabbitmq: + image: rabbitmq:3.12.14-management-alpine + container_name: datamart_rabbitmq + environment: + RABBITMQ_ERLANG_COOKIE: "FORRABBITMQCLUSTERNODES" + RABBITMQ_DEFAULT_USER: "datamart" + RABBITMQ_DEFAULT_PASS: "unicef" + RABBITMQ_DEFAULT_VHOST: "/datamart" + ports: + - "15672:15672" + - "5672:5672" + volumes: + - $PWD/build/rabbitmq/data:/var/lib/rabbitmq + - $PWD/build/rabbitmq/log:/var/log/rabbitmq diff --git a/docker-compose.yml b/docker-compose.yml index 9c819fb4..7af06103 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -99,4 +99,19 @@ services: redis: image: redis:alpine - container_name: datamart_redis \ No newline at end of file + container_name: datamart_redis + + rabbitmq: + image: rabbitmq:3.12.14-management-alpine + container_name: datamart_rabbitmq + environment: + RABBITMQ_ERLANG_COOKIE: "FORRABBITMQCLUSTERNODES" + RABBITMQ_DEFAULT_USER: "datamart" + RABBITMQ_DEFAULT_PASS: "unicef" + RABBITMQ_DEFAULT_VHOST: "/datamart" + ports: + - "15672:15672" + - "5672:5672" + volumes: + - $PWD/build/rabbitmq/data:/var/lib/rabbitmq + - $PWD/build/rabbitmq/log:/var/log/rabbitmq diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index c905f75e..3bd184ca 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -34,8 +34,9 @@ CACHE_URL_LOCK=(str, "redis://127.0.0.1:6379/2?key_prefix=lock"), CACHE_URL_TEMPLATE=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), CELERY_TASK_ALWAYS_EAGER=(bool, False), - CELERY_BROKER_URL=(str, "redis://127.0.0.1:6379/2"), - CELERY_RESULT_BACKEND=(str, "redis://127.0.0.1:6379/3"), + CELERY_BROKER_URL=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), + CELERY_RESULT_BACKEND=(str, "amqp"), + #CELERY_RESULT_DBURI=(str,"amqp://datamart:unicef@rabbitmq:5672/datamart") CSRF_COOKIE_SECURE=(bool, True), EXPLORER_TOKEN=(str, "EXPLORER_DATAMART_TOKEN"), IGNORED_SCHEMAS=(str, ["public", "uat", "frg"]), From 67ab01c47b6533d50fd6d00b0ba9fc3d830581c4 Mon Sep 17 00:00:00 2001 From: Hakan Date: Thu, 9 May 2024 14:40:32 +0300 Subject: [PATCH 2/4] fix linting issue --- src/etools_datamart/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 3bd184ca..9c63b224 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -36,7 +36,7 @@ CELERY_TASK_ALWAYS_EAGER=(bool, False), CELERY_BROKER_URL=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), CELERY_RESULT_BACKEND=(str, "amqp"), - #CELERY_RESULT_DBURI=(str,"amqp://datamart:unicef@rabbitmq:5672/datamart") + # CELERY_RESULT_DBURI=(str,"amqp://datamart:unicef@rabbitmq:5672/datamart") CSRF_COOKIE_SECURE=(bool, True), EXPLORER_TOKEN=(str, "EXPLORER_DATAMART_TOKEN"), IGNORED_SCHEMAS=(str, ["public", "uat", "frg"]), From 0d81a6b7efea71b8d100d4049d941ee2a6163db9 Mon Sep 17 00:00:00 2001 From: Hakan Date: Thu, 9 May 2024 15:30:04 +0300 Subject: [PATCH 3/4] update CELERY_RESULT_BACKEND --- src/etools_datamart/config/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 9c63b224..3dc69529 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -35,8 +35,7 @@ CACHE_URL_TEMPLATE=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), CELERY_TASK_ALWAYS_EAGER=(bool, False), CELERY_BROKER_URL=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), - CELERY_RESULT_BACKEND=(str, "amqp"), - # CELERY_RESULT_DBURI=(str,"amqp://datamart:unicef@rabbitmq:5672/datamart") + CELERY_RESULT_BACKEND=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), CSRF_COOKIE_SECURE=(bool, True), EXPLORER_TOKEN=(str, "EXPLORER_DATAMART_TOKEN"), IGNORED_SCHEMAS=(str, ["public", "uat", "frg"]), From 769cabb1a7d272407824603cda7decf8cfa89a37 Mon Sep 17 00:00:00 2001 From: Hakan Date: Thu, 9 May 2024 16:34:45 +0300 Subject: [PATCH 4/4] revert CELERY_RESULT_BACKEND to use Redis --- src/etools_datamart/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/config/settings.py b/src/etools_datamart/config/settings.py index 3dc69529..b6453c59 100644 --- a/src/etools_datamart/config/settings.py +++ b/src/etools_datamart/config/settings.py @@ -35,7 +35,7 @@ CACHE_URL_TEMPLATE=(str, "redis://127.0.0.1:6379/2?key_prefix=template"), CELERY_TASK_ALWAYS_EAGER=(bool, False), CELERY_BROKER_URL=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), - CELERY_RESULT_BACKEND=(str, "amqp://datamart:unicef@rabbitmq:5672/datamart"), + CELERY_RESULT_BACKEND=(str, "redis://127.0.0.1:6379/3"), CSRF_COOKIE_SECURE=(bool, True), EXPLORER_TOKEN=(str, "EXPLORER_DATAMART_TOKEN"), IGNORED_SCHEMAS=(str, ["public", "uat", "frg"]),