From 03828771d4cee20fb892ae9e656ee4600369dbf6 Mon Sep 17 00:00:00 2001 From: "Adolfo R. Brandes" Date: Thu, 29 Feb 2024 17:16:01 -0300 Subject: [PATCH] feat: Celery worker concurrency setting This allows the user to configure how many Celery workers are spawned independently of how many CPUs there are in the system. The default is to spawn as many workers as there are CPUs, which in some cases can consume too many resources. (The setting should be particularly useful to people running Tutor for development on Linux machines, where reducing the concurrency to "1" can reduce RAM usage significantly.) --- ...20240229_171657_arbrandes_celery_concurrency.md | 1 + docs/configuration.rst | 5 +++++ docs/dev.rst | 14 ++++++++++++++ tutor/templates/config/defaults.yml | 2 ++ tutor/templates/k8s/deployments.yml | 4 ++-- tutor/templates/local/docker-compose.yml | 4 ++-- 6 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 changelog.d/20240229_171657_arbrandes_celery_concurrency.md diff --git a/changelog.d/20240229_171657_arbrandes_celery_concurrency.md b/changelog.d/20240229_171657_arbrandes_celery_concurrency.md new file mode 100644 index 0000000000..48cae42098 --- /dev/null +++ b/changelog.d/20240229_171657_arbrandes_celery_concurrency.md @@ -0,0 +1 @@ +- [Feature] Allow the user to configure Celery worker concurrency. (by @arbrandes) diff --git a/docs/configuration.rst b/docs/configuration.rst index d1af01f52a..76062a0f28 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -149,6 +149,11 @@ This defines the version that will be pulled from just the Open edX platform git By default, there are 2 `uwsgi worker processes `__ to serve requests for the LMS and the CMS. However, each worker requires upwards of 500 Mb of RAM. You should reduce this value to 1 if your computer/server does not have enough memory. +- ``OPENEDX_LMS_CELERY_WORKERS`` (default: ``"0"``) +- ``OPENEDX_CMS_CELERY_WORKERS`` (default: ``"0"``) + +This sets the number of Celery workers at startup. Defaults to the number of CPUs detected on the system (represented by "0"). If your system's RAM is limited, you may need to set this to a low integer value in order to conserve it. (On development deployments, it is suggested to use "1" for both.) + - ``OPENEDX_CELERY_REDIS_DB`` (default: ``0``) - ``OPENEDX_CACHE_REDIS_DB`` (default: ``1``) diff --git a/docs/dev.rst b/docs/dev.rst index 6b1f1975bf..bc4b56d6d7 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -60,6 +60,20 @@ Now, use the ``tutor dev ...`` command-line interface to manage the development Read more about bind-mounts :ref:`below `. +Conserving system resources +--------------------------- + +When developing, you'll usually want the development environment to take up as little system resources as possible. To do so, we recommend setting the following Tutor configuration and relaunching your environment:: + + tutor config save \ + --set OPENEDX_CMS_CELERY_WORKERS=1 \ + --set OPENEDX_LMS_CELERY_WORKERS=1 \ + --set OPENEDX_CMS_UWSGI_WORKERS=1 \ + --set OPENEDX_LMS_UWSGI_WORKERS=1 \ + --set ELASTICSEARCH_HEAP_SIZE=100m + +This has been known to reduce RAM usage by more than 70%, specially if you're running Docker natively and have a lot of CPU cores. + Stopping the platform --------------------- diff --git a/tutor/templates/config/defaults.yml b/tutor/templates/config/defaults.yml index d0d3a31a70..74107e6649 100644 --- a/tutor/templates/config/defaults.yml +++ b/tutor/templates/config/defaults.yml @@ -54,7 +54,9 @@ OPENEDX_AWS_ACCESS_KEY: "" OPENEDX_AWS_SECRET_ACCESS_KEY: "" OPENEDX_CACHE_REDIS_DB: 1 OPENEDX_CELERY_REDIS_DB: 0 +OPENEDX_CMS_CELERY_WORKERS: 0 OPENEDX_CMS_UWSGI_WORKERS: 2 +OPENEDX_LMS_CELERY_WORKERS: 0 OPENEDX_LMS_UWSGI_WORKERS: 2 OPENEDX_MYSQL_DATABASE: "openedx" OPENEDX_MYSQL_USERNAME: "openedx" diff --git a/tutor/templates/k8s/deployments.yml b/tutor/templates/k8s/deployments.yml index a48740baf3..83ea627200 100644 --- a/tutor/templates/k8s/deployments.yml +++ b/tutor/templates/k8s/deployments.yml @@ -141,7 +141,7 @@ spec: containers: - name: cms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] + args: ["celery", "--app=cms.celery", "worker", "--loglevel=info", "--hostname=edx.cms.core.default.%%h", "--concurrency={{ OPENEDX_CMS_CELERY_WORKERS }}", "--max-tasks-per-child", "100", "--exclude-queues=edx.lms.core.default"] env: - name: SERVICE_VARIANT value: cms @@ -250,7 +250,7 @@ spec: containers: - name: lms-worker image: {{ DOCKER_IMAGE_OPENEDX }} - args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] + args: ["celery", "--app=lms.celery", "worker", "--loglevel=info", "--hostname=edx.lms.core.default.%%h", "--concurrency={{ OPENEDX_LMS_CELERY_WORKERS }}", "--max-tasks-per-child=100", "--exclude-queues=edx.cms.core.default"] env: - name: SERVICE_VARIANT value: lms diff --git a/tutor/templates/local/docker-compose.yml b/tutor/templates/local/docker-compose.yml index 5580cbae97..f6f758cdbb 100644 --- a/tutor/templates/local/docker-compose.yml +++ b/tutor/templates/local/docker-compose.yml @@ -158,7 +158,7 @@ services: environment: SERVICE_VARIANT: lms DJANGO_SETTINGS_MODULE: lms.envs.tutor.production - command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default + command: celery --app=lms.celery worker --loglevel=info --hostname=edx.lms.core.default.%%h --concurrency={{ OPENEDX_LMS_CELERY_WORKERS }} --max-tasks-per-child=100 --exclude-queues=edx.cms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro @@ -177,7 +177,7 @@ services: environment: SERVICE_VARIANT: cms DJANGO_SETTINGS_MODULE: cms.envs.tutor.production - command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default + command: celery --app=cms.celery worker --loglevel=info --hostname=edx.cms.core.default.%%h --concurrency={{ OPENEDX_CMS_CELERY_WORKERS }} --max-tasks-per-child 100 --exclude-queues=edx.lms.core.default restart: unless-stopped volumes: - ../apps/openedx/settings/lms:/openedx/edx-platform/lms/envs/tutor:ro