chore(celery): Cleanup config and async query specifications (apache#…

…25314)
preset-io · Oct 31, 2023 · 1e37f0b · 1e37f0b
1 parent 2177a50
commit 1e37f0b
Show file tree

Hide file tree

Showing 10 changed files with 62 additions and 88 deletions.
diff --git a/docs/docs/installation/alerts-reports.mdx b/docs/docs/installation/alerts-reports.mdx
@@ -104,30 +104,27 @@ REDIS_HOST = "superset_cache"
 REDIS_PORT = "6379"
 
 class CeleryConfig:
-    broker_url = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
-    imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
-    result_backend = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
+    broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
+    imports = (
+        "superset.sql_lab",
+        "superset.tasks.scheduler",
+    )
+    result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
     worker_prefetch_multiplier = 10
     task_acks_late = True
     task_annotations = {
-        'sql_lab.get_sql_results': {
-            'rate_limit': '100/s',
-        },
-        'email_reports.send': {
-            'rate_limit': '1/s',
-            'time_limit': 600,
-            'soft_time_limit': 600,
-            'ignore_result': True,
+        "sql_lab.get_sql_results": {
+            "rate_limit": "100/s",
         },
     }
     beat_schedule = {
-        'reports.scheduler': {
-            'task': 'reports.scheduler',
-            'schedule': crontab(minute='*', hour='*'),
+        "reports.scheduler": {
+            "task": "reports.scheduler",
+            "schedule": crontab(minute="*", hour="*"),
         },
-        'reports.prune_log': {
-            'task': 'reports.prune_log',
-            'schedule': crontab(minute=0, hour=0),
+        "reports.prune_log": {
+            "task": "reports.prune_log",
+            "schedule": crontab(minute=0, hour=0),
         },
     }
 CELERY_CONFIG = CeleryConfig

diff --git a/docs/docs/installation/async-queries-celery.mdx b/docs/docs/installation/async-queries-celery.mdx
@@ -23,30 +23,17 @@ and web server processes should have the same configuration.
 
 ```python
 class CeleryConfig(object):
-    broker_url = 'redis://localhost:6379/0'
+    broker_url = "redis://localhost:6379/0"
     imports = (
-        'superset.sql_lab',
-        'superset.tasks',
+        "superset.sql_lab",
+        "superset.tasks.scheduler",
     )
-    result_backend = 'redis://localhost:6379/0'
-    worker_log_level = 'DEBUG'
+    result_backend = "redis://localhost:6379/0"
     worker_prefetch_multiplier = 10
     task_acks_late = True
     task_annotations = {
-        'sql_lab.get_sql_results': {
-            'rate_limit': '100/s',
-        },
-        'email_reports.send': {
-            'rate_limit': '1/s',
-            'time_limit': 120,
-            'soft_time_limit': 150,
-            'ignore_result': True,
-        },
-    }
-    beat_schedule = {
-        'email_reports.schedule_hourly': {
-            'task': 'email_reports.schedule_hourly',
-            'schedule': crontab(minute=1, hour='*'),
+        "sql_lab.get_sql_results": {
+            "rate_limit": "100/s",
         },
     }
 

diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx
@@ -114,7 +114,10 @@ from s3cache.s3cache import S3Cache
 
 class CeleryConfig(object):
     broker_url = "redis://localhost:6379/0"
-    imports = ("superset.sql_lab", "superset.tasks", "superset.tasks.thumbnails")
+    imports = (
+        "superset.sql_lab",
+        "superset.tasks.thumbnails",
+    )
     result_backend = "redis://localhost:6379/0"
     worker_prefetch_multiplier = 10
     task_acks_late = True

diff --git a/docs/docs/installation/running-on-kubernetes.mdx b/docs/docs/installation/running-on-kubernetes.mdx
@@ -367,37 +367,35 @@ configOverrides:
   celery_conf: |
     from celery.schedules import crontab
 
-    class CeleryConfig(object):
+    class CeleryConfig:
       broker_url = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
-      imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
+      imports = (
+          "superset.sql_lab",
+          "superset.tasks.cache",
+          "superset.tasks.scheduler",
+      )
       result_backend = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
       task_annotations = {
-          'sql_lab.get_sql_results': {
-              'rate_limit': '100/s',
-          },
-          'email_reports.send': {
-              'rate_limit': '1/s',
-              'time_limit': 600,
-              'soft_time_limit': 600,
-              'ignore_result': True,
+          "sql_lab.get_sql_results": {
+              "rate_limit": "100/s",
           },
       }
       beat_schedule = {
-          'reports.scheduler': {
-              'task': 'reports.scheduler',
-              'schedule': crontab(minute='*', hour='*'),
+          "reports.scheduler": {
+              "task": "reports.scheduler",
+              "schedule": crontab(minute="*", hour="*"),
           },
-          'reports.prune_log': {
-              'task': 'reports.prune_log',
+          "reports.prune_log": {
+              "task": "reports.prune_log",
               'schedule': crontab(minute=0, hour=0),
           },
           'cache-warmup-hourly': {
-              'task': 'cache-warmup',
-              'schedule': crontab(minute='*/30', hour='*'),
-              'kwargs': {
-                  'strategy_name': 'top_n_dashboards',
-                  'top_n': 10,
-                  'since': '7 days ago',
+              "task": "cache-warmup",
+              "schedule": crontab(minute="*/30", hour="*"),
+              "kwargs": {
+                  "strategy_name": "top_n_dashboards",
+                  "top_n": 10,
+                  "since": "7 days ago",
               },
           }
       }

diff --git a/helm/superset/Chart.yaml b/helm/superset/Chart.yaml
@@ -29,7 +29,7 @@ maintainers:
   - name: craig-rueda
     email: [email protected]
     url: https://github.com/craig-rueda
-version: 0.10.13
+version: 0.10.14
 dependencies:
   - name: postgresql
     version: 12.1.6

diff --git a/helm/superset/README.md b/helm/superset/README.md
@@ -23,7 +23,7 @@ NOTE: This file is generated by helm-docs: https://github.com/norwoodj/helm-docs
 
 # superset
 
-![Version: 0.10.13](https://img.shields.io/badge/Version-0.10.13-informational?style=flat-square)
+![Version: 0.10.14](https://img.shields.io/badge/Version-0.10.14-informational?style=flat-square)
 
 Apache Superset is a modern, enterprise-ready business intelligence web application
 

diff --git a/helm/superset/templates/_helpers.tpl b/helm/superset/templates/_helpers.tpl
@@ -84,15 +84,14 @@ SQLALCHEMY_DATABASE_URI = f"postgresql+psycopg2://{env('DB_USER')}:{env('DB_PASS
 SQLALCHEMY_TRACK_MODIFICATIONS = True
 SECRET_KEY = env('SECRET_KEY', 'thisISaSECRET_1234')
 
-class CeleryConfig(object):
-  CELERY_IMPORTS = ('superset.sql_lab', )
-  CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
+class CeleryConfig:
+  imports  = ("superset.sql_lab", )
   {{- if .Values.supersetNode.connections.redis_password }}
-  BROKER_URL = f"redis://:{env('REDIS_PASSWORD')}@{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
-  CELERY_RESULT_BACKEND = f"redis://:{env('REDIS_PASSWORD')}@{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+  broker_url = f"redis://:{env('REDIS_PASSWORD')}@{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+  result_backend = f"redis://:{env('REDIS_PASSWORD')}@{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
   {{- else }}
-  BROKER_URL = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
-  CELERY_RESULT_BACKEND = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+  broker_url = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
+  result_backend = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0"
   {{- end }}
 
 CELERY_CONFIG = CeleryConfig

diff --git a/superset/config.py b/superset/config.py
@@ -940,24 +940,16 @@ class D3Format(TypedDict, total=False):
 
 class CeleryConfig:  # pylint: disable=too-few-public-methods
     broker_url = "sqla+sqlite:///celerydb.sqlite"
-    imports = ("superset.sql_lab",)
+    imports = ("superset.sql_lab", "superset.tasks.scheduler")
     result_backend = "db+sqlite:///celery_results.sqlite"
     worker_prefetch_multiplier = 1
     task_acks_late = False
     task_annotations = {
-        "sql_lab.get_sql_results": {"rate_limit": "100/s"},
-        "email_reports.send": {
-            "rate_limit": "1/s",
-            "time_limit": int(timedelta(seconds=120).total_seconds()),
-            "soft_time_limit": int(timedelta(seconds=150).total_seconds()),
-            "ignore_result": True,
+        "sql_lab.get_sql_results": {
+            "rate_limit": "100/s",
         },
     }
     beat_schedule = {
-        "email_reports.schedule_hourly": {
-            "task": "email_reports.schedule_hourly",
-            "schedule": crontab(minute=1, hour="*"),
-        },
         "reports.scheduler": {
             "task": "reports.scheduler",
             "schedule": crontab(minute="*", hour="*"),
@@ -1558,7 +1550,7 @@ def EMAIL_HEADER_MUTATOR(  # pylint: disable=invalid-name,unused-argument
 ) = None
 GLOBAL_ASYNC_QUERIES_JWT_COOKIE_DOMAIN = None
 GLOBAL_ASYNC_QUERIES_JWT_SECRET = "test-secret-change-me"
-GLOBAL_ASYNC_QUERIES_TRANSPORT = "polling"
+GLOBAL_ASYNC_QUERIES_TRANSPORT: Literal["polling", "ws"] = "polling"
 GLOBAL_ASYNC_QUERIES_POLLING_DELAY = int(
     timedelta(milliseconds=500).total_seconds() * 1000
 )

diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py
@@ -133,11 +133,10 @@ def GET_FEATURE_FLAGS_FUNC(ff):
 
 
 class CeleryConfig:
-    BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
-    CELERY_IMPORTS = ("superset.sql_lab",)
-    CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
-    CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}}
-    CONCURRENCY = 1
+    broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
+    imports = ("superset.sql_lab",)
+    result_backend = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}"
+    concurrency = 1
 
 
 CELERY_CONFIG = CeleryConfig

diff --git a/tests/integration_tests/superset_test_config_thumbnails.py b/tests/integration_tests/superset_test_config_thumbnails.py
@@ -62,10 +62,9 @@ def GET_FEATURE_FLAGS_FUNC(ff):
 
 
 class CeleryConfig:
-    BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
-    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks.thumbnails")
-    CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}}
-    CONCURRENCY = 1
+    broker_url = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}"
+    imports = ("superset.sql_lab", "superset.tasks.thumbnails")
+    concurrency = 1
 
 
 CELERY_CONFIG = CeleryConfig