From 0d9a0a801822207e862259a66e4fa72b0d0b9d7f Mon Sep 17 00:00:00 2001
From: Cristhian Garcia <crisgarta8@gmail.com>
Date: Wed, 11 Sep 2024 16:51:05 -0500
Subject: [PATCH] feat: allow to pass extra parameters for celery workers

(cherry picked from commit c858d5d0a7e285f050eda6057b214a5313691b0d)

docs: add section with extra parameters

docs: add section with extra parameters

refactor!: use new filter to define celery command

(cherry picked from commit 53fafef821a33979f24b867228487f812eb8524c)
---
 .github/workflows/test.yml          |  2 +-
 README.md                           | 18 ++++++++++++++++++
 pyproject.toml                      |  5 ++---
 tutorcelery/hooks.py                |  1 +
 tutorcelery/patches/k8s-deployments |  3 +++
 tutorcelery/patches/k8s-override    | 26 --------------------------
 tutorcelery/plugin.py               |  8 ++++++++
 7 files changed, 33 insertions(+), 30 deletions(-)
 delete mode 100644 tutorcelery/patches/k8s-override

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 9a62a00..e5330dc 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-         python-version: ['3.8', '3.12']
+         python-version: ['3.12']
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
diff --git a/README.md b/README.md
index 9d79653..f3ab30b 100644
--- a/README.md
+++ b/README.md
@@ -79,6 +79,24 @@ CELERY_CMS_EXPLICIT_QUEUES:
     queue: edx.cms.core.high
 ```
 
+### Custom parameters
+
+Each deployment can be configured to run with different paramaters to override the defaults, the setting `extra_param`
+is a list that can be used to pass custom parameters to the Celery workers. e.g changing the Celery's pool parameter
+for the high_mem lms worker deployment:
+
+```python
+@CELERY_WORKERS_CONFIG.add()
+def _add_celery_workers_config(workers_config):
+    # Adding LMS extra queues
+    workers_config["lms"]["high_mem"]["extra_params"] = {
+      "--pool=gevent",
+      "--concurrency=100",
+    }
+
+    return workers_config
+```
+
 ### Autoscaling
 
 As an alternative to the CPU/memory based autoscaling offered by the plugin [tutor-contrib-pod-autoscaling](https://github.com/eduNEXT/tutor-contrib-pod-autoscaling),
diff --git a/pyproject.toml b/pyproject.toml
index f6cedcd..73afbf1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,7 +7,7 @@ name = "tutor-contrib-celery"
 dynamic = ["version"]
 description = "A Tutor plugin to manage our opinionated Open edX operations"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 license = { text = "AGPLv3" }
 authors = [
     { name = "eduNEXT" }
@@ -18,14 +18,13 @@ classifiers = [
     "License :: OSI Approved :: GNU Affero General Public License v3",
     "Operating System :: OS Independent",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
 dependencies = [
-    "tutor>=18.0.0,<19.0.0"
+    "tutor>=18.2,<19.0.0"
 ]
 
 optional-dependencies = { dev = ["python-semantic-release", "pylint", "black"]}
diff --git a/tutorcelery/hooks.py b/tutorcelery/hooks.py
index eb46d8d..931a2ab 100644
--- a/tutorcelery/hooks.py
+++ b/tutorcelery/hooks.py
@@ -20,6 +20,7 @@ class CELERY_WORKERS_ATTRS_TYPE(TypedDict):
     max_replicas: NotRequired[int]
     list_length: NotRequired[int]
     enable_keda: bool
+    extra_params: NotRequired[list[str]]
 
 
 CELERY_WORKERS_CONFIG: Filter[dict[str, dict[str, CELERY_WORKERS_ATTRS_TYPE]], []] = (
diff --git a/tutorcelery/patches/k8s-deployments b/tutorcelery/patches/k8s-deployments
index c0d2559..7b33cb7 100644
--- a/tutorcelery/patches/k8s-deployments
+++ b/tutorcelery/patches/k8s-deployments
@@ -31,6 +31,9 @@ spec:
             - "--hostname=edx.{{service}}.core.{{variant}}.%%h"
             - "--max-tasks-per-child=100"
             - "--queues=edx.{{service}}.core.{{variant}}"
+            {% for param in config.get("extra_params", []) %}
+            - "{{param}}"
+            {% endfor %}
           env:
           - name: SERVICE_VARIANT
             value: {{service}}
diff --git a/tutorcelery/patches/k8s-override b/tutorcelery/patches/k8s-override
deleted file mode 100644
index 3d3eda8..0000000
--- a/tutorcelery/patches/k8s-override
+++ /dev/null
@@ -1,26 +0,0 @@
-{% for service in ["lms", "cms"] %}
-{% set exclude = "lms" if service == "cms" else "cms" %}
-{% set service_variants = iter_celery_workers_config().get(service) %}
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: {{service}}-worker
-spec:
-  template:
-    spec:
-      containers:
-        - name: {{service}}-worker
-          args:
-            - "celery"
-            - "--app={{service}}.celery"
-            - "worker"
-            - "--loglevel=info"
-            - "--hostname=edx.{{service}}.core.default.%%h"
-            - "--max-tasks-per-child=100"
-            {% if is_celery_multiqueue(service) -%}
-            - "--queues=edx.{{service}}.core.default"
-            {% else -%}
-            - "--exclude-queues=edx.{{exclude}}.core.default"
-            {% endif -%}
-{% endfor %}
diff --git a/tutorcelery/plugin.py b/tutorcelery/plugin.py
index afcdbd3..6000d37 100644
--- a/tutorcelery/plugin.py
+++ b/tutorcelery/plugin.py
@@ -80,6 +80,14 @@ def is_celery_multiqueue(service: str) -> bool:
     return True
 
 
+@hooks.Actions.PROJECT_ROOT_READY.add()
+def configure_default_workers(root: str) -> None:
+    if is_celery_multiqueue("lms"):
+        hooks.Filters.LMS_WORKER_COMMAND.add_items(["--queues=edx.lms.core.default"])
+    if is_celery_multiqueue("cms"):
+        hooks.Filters.CMS_WORKER_COMMAND.add_items(["--queues=edx.cms.core.default"])
+
+
 hooks.Filters.CONFIG_DEFAULTS.add_items(
     [
         # Add your new settings that have default values here.