feat: allow to pass extra parameters for celery workers

.github/workflows/test.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-         python-version: ['3.8', '3.12']
+         python-version: ['3.12']
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}

README.md

@@ -79,6 +79,24 @@ CELERY_CMS_EXPLICIT_QUEUES:
     queue: edx.cms.core.high
 ```
 
+### Custom parameters
+
+Each deployment can be configured to run with different paramaters to override the defaults, the setting `extra_param`
+is a list that can be used to pass custom parameters to the Celery workers. e.g changing the Celery's pool parameter
+for the high_mem lms worker deployment:
+
+```python
+@CELERY_WORKERS_CONFIG.add()
+def _add_celery_workers_config(workers_config):
+    # Adding LMS extra queues
+    workers_config["lms"]["high_mem"]["extra_params"] = {
+      "--pool=gevent",
+      "--concurrency=100",
+    }
+
+    return workers_config
+```
+
 ### Autoscaling
 
 As an alternative to the CPU/memory based autoscaling offered by the plugin [tutor-contrib-pod-autoscaling](https://github.com/eduNEXT/tutor-contrib-pod-autoscaling),

pyproject.toml

@@ -7,7 +7,7 @@ name = "tutor-contrib-celery"
 dynamic = ["version"]
 description = "A Tutor plugin to manage our opinionated Open edX operations"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = { text = "AGPLv3" }
 authors = [
     { name = "eduNEXT" }
@@ -18,14 +18,13 @@ classifiers = [
     "License :: OSI Approved :: GNU Affero General Public License v3",
     "Operating System :: OS Independent",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "tutor>=18.0.0,<19.0.0"
+    "tutor>=18.2,<19.0.0"
 ]
 
 optional-dependencies = { dev = ["python-semantic-release", "pylint", "black"]}

tutorcelery/hooks.py

@@ -20,6 +20,7 @@ class CELERY_WORKERS_ATTRS_TYPE(TypedDict):
     max_replicas: NotRequired[int]
     list_length: NotRequired[int]
     enable_keda: bool
+    extra_params: NotRequired[list[str]]
 
 
 CELERY_WORKERS_CONFIG: Filter[dict[str, dict[str, CELERY_WORKERS_ATTRS_TYPE]], []] = (

tutorcelery/patches/k8s-deployments

@@ -31,6 +31,9 @@ spec:
             - "--hostname=edx.{{service}}.core.{{variant}}.%%h"
             - "--max-tasks-per-child=100"
             - "--queues=edx.{{service}}.core.{{variant}}"
+            {% for param in config.get("extra_params", []) %}
+            - "{{param}}"
+            {% endfor %}
           env:
           - name: SERVICE_VARIANT
             value: {{service}}

tutorcelery/plugin.py

@@ -80,6 +80,14 @@ def is_celery_multiqueue(service: str) -> bool:
     return True
 
 
+@hooks.Actions.PROJECT_ROOT_READY.add()
+def configure_default_workers(root: str) -> None:
+    if is_celery_multiqueue("lms"):
+        hooks.Filters.LMS_WORKER_COMMAND.add_items(["--queues=edx.lms.core.default"])
+    if is_celery_multiqueue("cms"):
+        hooks.Filters.CMS_WORKER_COMMAND.add_items(["--queues=edx.cms.core.default"])
+
+
 hooks.Filters.CONFIG_DEFAULTS.add_items(
     [
         # Add your new settings that have default values here.