diff --git a/changelog.d/20240229_171657_arbrandes_celery_concurrency.md b/changelog.d/20240229_171657_arbrandes_celery_concurrency.md new file mode 100644 index 0000000000..48cae42098 --- /dev/null +++ b/changelog.d/20240229_171657_arbrandes_celery_concurrency.md @@ -0,0 +1 @@ +- [Feature] Allow the user to configure Celery worker concurrency. (by @arbrandes) diff --git a/docs/configuration.rst b/docs/configuration.rst index d12620ee2f..4ce5515b3f 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -149,6 +149,11 @@ This defines the version that will be pulled from just the Open edX platform git By default, there are 2 `uwsgi worker processes `__ to serve requests for the LMS and the CMS. However, each worker requires upwards of 500 Mb of RAM. You should reduce this value to 1 if your computer/server does not have enough memory. +- ``OPENEDX_LMS_CELERY_WORKERS`` (default: ``"0"``) +- ``OPENEDX_CMS_CELERY_WORKERS`` (default: ``"0"``) + +This sets the number of Celery workers at startup. Defaults to the number of CPUs detected on the system (represented by "0"). + - ``OPENEDX_CELERY_REDIS_DB`` (default: ``0``) - ``OPENEDX_CACHE_REDIS_DB`` (default: ``1``) diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index c5205859e9..1b315d5f1a 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -54,7 +54,9 @@ OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" OPENEDX_CACHE_REDIS_DB: 1 OPENEDX_CELERY_REDIS_DB: 0 +OPENEDX_CMS_CELERY_WORKERS: 0 OPENEDX_CMS_UWSGI_WORKERS: 2 +OPENEDX_LMS_CELERY_WORKERS: 0 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index a48740baf3..83ea627200 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -141,7 +141,7 @@ spec: containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] + args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--concurrency={{ OPENEDX_CMS_CELERY_WORKERS }}", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] env: - name: SERVICE_VARIANT value: cms @@ -250,7 +250,7 @@ spec: containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] + args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--concurrency={{ OPENEDX_LMS_CELERY_WORKERS }}", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] env: - name: SERVICE_VARIANT value: lms diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 5580cbae97..f6f758cdbb 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -158,7 +158,7 @@ services: environment: SERVICE_VARIANT: lms DJANGO_SETTINGS_MODULE: lms.envs.tutor.production - command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default + command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --concurrency={{ OPENEDX_LMS_CELERY_WORKERS }} --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro @@ -177,7 +177,7 @@ services: environment: SERVICE_VARIANT: cms DJANGO_SETTINGS_MODULE: cms.envs.tutor.production - command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default + command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --concurrency={{ OPENEDX_CMS_CELERY_WORKERS }} --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro