From 22c7c76a7524f3d3f0d19c23d1c07ffcd11a1a34 Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Wed, 4 May 2022 19:05:01 +0200 Subject: [PATCH 01/29] [#305] Added abstract models. `models` module replaced by a `models` package containing an `abstract` module (for abstract models) and a `generic` module (for the default models, previously in the `models` module) --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/models/__init__.py | 3 + .../{models.py => models/abstract.py} | 59 ++------------- django_celery_results/models/generic.py | 73 +++++++++++++++++++ 3 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 django_celery_results/models/__init__.py rename django_celery_results/{models.py => models/abstract.py} (82%) create mode 100644 django_celery_results/models/generic.py diff --git a/django_celery_results/models/__init__.py b/django_celery_results/models/__init__.py new file mode 100644 index 00000000..4165e265 --- /dev/null +++ b/django_celery_results/models/__init__.py @@ -0,0 +1,3 @@ +from .generic import ChordCounter, GroupResult, TaskResult + +__ALL__ = [ChordCounter, GroupResult, TaskResult] diff --git a/django_celery_results/models.py b/django_celery_results/models/abstract.py similarity index 82% rename from django_celery_results/models.py rename to django_celery_results/models/abstract.py index db8986e2..2efaea08 100644 --- a/django_celery_results/models.py +++ b/django_celery_results/models/abstract.py @@ -1,10 +1,6 @@ -"""Database models.""" - -import json +"""Abstract models.""" from celery import states -from celery.result import GroupResult as CeleryGroupResult -from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ @@ -15,8 +11,8 @@ TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) -class TaskResult(models.Model): - """Task result/status.""" +class AbstractTaskResult(models.Model): + """Abstract Task result/status.""" task_id = models.CharField( max_length=getattr( @@ -98,8 +94,8 @@ class TaskResult(models.Model): class Meta: """Table information.""" + abstract = True ordering = ['-date_done'] - verbose_name = _('task result') verbose_name_plural = _('task results') @@ -135,49 +131,8 @@ def __str__(self): return ''.format(self) -class ChordCounter(models.Model): - """Chord synchronisation.""" - - group_id = models.CharField( - max_length=getattr( - settings, - "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", - 255), - unique=True, - verbose_name=_("Group ID"), - help_text=_("Celery ID for the Chord header group"), - ) - sub_tasks = models.TextField( - help_text=_( - "JSON serialized list of task result tuples. " - "use .group_result() to decode" - ) - ) - count = models.PositiveIntegerField( - help_text=_( - "Starts at len(chord header) and decrements after each task is " - "finished" - ) - ) - - def group_result(self, app=None): - """Return the :class:`celery.result.GroupResult` of self. - - Arguments: - app (celery.app.base.Celery): app instance to create the - :class:`celery.result.GroupResult` with. - - """ - return CeleryGroupResult( - self.group_id, - [result_from_tuple(r, app=app) - for r in json.loads(self.sub_tasks)], - app=app - ) - - -class GroupResult(models.Model): - """Task Group result/status.""" +class AbstractGroupResult(models.Model): + """Abstract Task Group result/status.""" group_id = models.CharField( max_length=getattr( @@ -230,8 +185,8 @@ def __str__(self): class Meta: """Table information.""" + abstract = True ordering = ['-date_done'] - verbose_name = _('group result') verbose_name_plural = _('group results') diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py new file mode 100644 index 00000000..74faf614 --- /dev/null +++ b/django_celery_results/models/generic.py @@ -0,0 +1,73 @@ +"""Database models.""" + +import json + +from celery.result import GroupResult as CeleryGroupResult +from celery.result import result_from_tuple +from django.conf import settings +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from django_celery_results.models.abstract import ( + AbstractGroupResult, + AbstractTaskResult +) + + +class TaskResult(AbstractTaskResult): + """Task result/status.""" + + class Meta(AbstractTaskResult.Meta): + """Table information.""" + + abstract = False + + +class ChordCounter(models.Model): + """Chord synchronisation.""" + + group_id = models.CharField( + max_length=getattr( + settings, + "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", + 255), + unique=True, + verbose_name=_("Group ID"), + help_text=_("Celery ID for the Chord header group"), + ) + sub_tasks = models.TextField( + help_text=_( + "JSON serialized list of task result tuples. " + "use .group_result() to decode" + ) + ) + count = models.PositiveIntegerField( + help_text=_( + "Starts at len(chord header) and decrements after each task is " + "finished" + ) + ) + + def group_result(self, app=None): + """Return the GroupResult of self. + + Arguments: + --------- + app (Celery): app instance to create the GroupResult with. + + """ + return CeleryGroupResult( + self.group_id, + [result_from_tuple(r, app=app) + for r in json.loads(self.sub_tasks)], + app=app + ) + + +class GroupResult(AbstractGroupResult): + """Task Group result/status.""" + + class Meta(AbstractGroupResult.Meta): + """Table information.""" + + abstract = False From c49ad2efef06ba61cd7a762faea4c409d12b0f81 Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Wed, 4 May 2022 18:20:21 +0200 Subject: [PATCH 02/29] [#305] `ChordCounter` moved from `abstract` module to `generic` module. Added some minor changes. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/models/abstract.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 2efaea08..319a5de4 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -186,9 +186,9 @@ class Meta: """Table information.""" abstract = True - ordering = ['-date_done'] verbose_name = _('group result') verbose_name_plural = _('group results') + ordering = ['-date_done'] # Explicit names to solve https://code.djangoproject.com/ticket/33483 indexes = [ From 25496f8a6982652c604d8ebd6938d7076c22d180 Mon Sep 17 00:00:00 2001 From: "jose.padin" Date: Thu, 12 May 2022 18:38:25 +0200 Subject: [PATCH 03/29] Issue 305: abstract models * Fixed import bug --- django_celery_results/models/generic.py | 2 +- django_celery_results/{ => models}/managers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename django_celery_results/{ => models}/managers.py (99%) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 74faf614..09b9dfdb 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -10,7 +10,7 @@ from django_celery_results.models.abstract import ( AbstractGroupResult, - AbstractTaskResult + AbstractTaskResult, ) diff --git a/django_celery_results/managers.py b/django_celery_results/models/managers.py similarity index 99% rename from django_celery_results/managers.py rename to django_celery_results/models/managers.py index 1caa124b..812374f1 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/models/managers.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import connections, models, router, transaction -from .utils import now +from ..utils import now W_ISOLATION_REP = """ Polling results with transaction isolation level 'repeatable-read' From 97661fd36b213ca89343adaf3cd7985b95a94f2e Mon Sep 17 00:00:00 2001 From: Asif Saif Uddin Date: Tue, 7 Jun 2022 16:44:30 +0600 Subject: [PATCH 04/29] Update django_celery_results/models/__init__.py Co-authored-by: Allex --- django_celery_results/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/models/__init__.py b/django_celery_results/models/__init__.py index 4165e265..5e478755 100644 --- a/django_celery_results/models/__init__.py +++ b/django_celery_results/models/__init__.py @@ -1,3 +1,3 @@ from .generic import ChordCounter, GroupResult, TaskResult -__ALL__ = [ChordCounter, GroupResult, TaskResult] +__all__ = ["ChordCounter", "GroupResult", "TaskResult"] From a5554d701c922f89d786594d779e4246e97f39d5 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 10 Aug 2022 08:44:30 +0200 Subject: [PATCH 05/29] [#305]: Improving abstract models implementation. Added a `helpers` module into `models` containing the functions `taskresult_model()` and `groupresult_model()`. * `taskresult_model()`: will try to find the custom model using a dotted path defined under the constant `CELERY_RESULTS_TASKRESULT_MODEL` in the settings of the user's project * `groupresult_model()` will try to do the same using under the constant `CELERY_RESULTS_GROUPRESULT_MODEL`. By default if these attributes are not found `django-celery-results` will use the default models (`models.TaskResult` and `models.GroupResult`). Updated database backend in order to use custom models for `TaskResult and `GroupResult` it they're present. Instead to import explicitely the `TaskResult` and the `GroupResult` (default models from `django-celery-results`) we make use of the model helpers to load the right classes, the custom ones if they're present otherwise we use the default ones. Getting data from `task_kwargs` to extend the `task_properties` and be able to store them into the database (using the custom models). First of all we need a way to get data from `task_kwargs` (or somewhere else) just before a `task_result` record is created, evaluate that data and find the right values that will be used to fill the new fields defined in the custom model. So for this purpose we added a settings module to `django-celery-results` which will hold default settings, the first setting that will contain is a function in charge to get a callback from the settings of the user project. This callback will be feeded by the task `task_kwargs`, which will be intercepted in `DatabaseBackend._get_extended_properties` just before the `task_kwargs` are encoded by `encode_content()` method and send it to the `store_result` method from the object manager of `TaskModel` (Custom/Default one). To end, we must to extend the arguments of the `store_result` method from the `TaskResult` Manager adding `extra_fields` argument that will make us able to send extra data to the custom model, when it's defined. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- .gitignore | 1 + django_celery_results/admin.py | 5 +- django_celery_results/backends/database.py | 15 ++++-- .../{models => }/managers.py | 8 ++-- django_celery_results/models/abstract.py | 2 +- django_celery_results/models/generic.py | 6 +++ django_celery_results/models/helpers.py | 47 +++++++++++++++++++ django_celery_results/settings.py | 16 +++++++ 8 files changed, 91 insertions(+), 9 deletions(-) rename django_celery_results/{models => }/managers.py (97%) create mode 100644 django_celery_results/models/helpers.py create mode 100644 django_celery_results/settings.py diff --git a/.gitignore b/.gitignore index 42bed400..6c0f8773 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ cover/ .cache/ htmlcov/ coverage.xml +.vscode diff --git a/django_celery_results/admin.py b/django_celery_results/admin.py index de5172af..e0654aec 100644 --- a/django_celery_results/admin.py +++ b/django_celery_results/admin.py @@ -10,7 +10,10 @@ ALLOW_EDITS = False pass -from .models import GroupResult, TaskResult +from .models.helpers import taskresult_model, groupresult_model + +GroupResult = groupresult_model() +TaskResult = taskresult_model() class TaskResultAdmin(admin.ModelAdmin): diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 4a3548bf..018e5ca0 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -13,8 +13,8 @@ from kombu.exceptions import DecodeError from ..models import ChordCounter -from ..models import GroupResult as GroupResultModel -from ..models import TaskResult +from ..models.helpers import groupresult_model, taskresult_model +from ..settings import extend_task_props_callback EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -30,8 +30,8 @@ class DatabaseBackend(BaseDictBackend): """The Django database backend, using models to store task state.""" - TaskModel = TaskResult - GroupModel = GroupResultModel + TaskModel = taskresult_model() + GroupModel = groupresult_model() subpolling_interval = 0.5 def exception_safe_to_retry(self, exc): @@ -80,6 +80,13 @@ def _get_extended_properties(self, request, traceback): # task protocol 1 task_kwargs = getattr(request, 'kwargs', None) + # TODO: We assuming that task protocol 1 could be always in use. :/ + extra_fields = extend_task_props_callback( + getattr(request, 'kwargs', None) + ) + if extra_fields: + extended_props.update({"extra_fields": extra_fields}) + # Encode input arguments if task_args is not None: _, _, task_args = self.encode_content(task_args) diff --git a/django_celery_results/models/managers.py b/django_celery_results/managers.py similarity index 97% rename from django_celery_results/models/managers.py rename to django_celery_results/managers.py index 812374f1..b38a15e8 100644 --- a/django_celery_results/models/managers.py +++ b/django_celery_results/managers.py @@ -8,7 +8,7 @@ from django.conf import settings from django.db import connections, models, router, transaction -from ..utils import now +from .utils import now W_ISOLATION_REP = """ Polling results with transaction isolation level 'repeatable-read' @@ -119,7 +119,7 @@ def store_result(self, content_type, content_encoding, traceback=None, meta=None, periodic_task_name=None, task_name=None, task_args=None, task_kwargs=None, - worker=None, using=None, **kwargs): + worker=None, using=None, extra_fields=None, **kwargs): """Store the result and status of a task. Arguments: @@ -140,6 +140,7 @@ def store_result(self, content_type, content_encoding, exception (only passed if the task failed). meta (str): Serialized result meta data (this contains e.g. children). + extra_fields (dict, optional): Extra (model) fields to store. Keyword Arguments: exception_retry_count (int): How many times to retry by @@ -159,7 +160,8 @@ def store_result(self, content_type, content_encoding, 'task_name': task_name, 'task_args': task_args, 'task_kwargs': task_kwargs, - 'worker': worker + 'worker': worker, + **extra_fields } if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 319a5de4..99aeb65e 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -5,7 +5,7 @@ from django.db import models from django.utils.translation import gettext_lazy as _ -from . import managers +from .. import managers ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 09b9dfdb..f1ec7e01 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -21,6 +21,7 @@ class Meta(AbstractTaskResult.Meta): """Table information.""" abstract = False + app_label = "django_celery_results" class ChordCounter(models.Model): @@ -48,6 +49,10 @@ class ChordCounter(models.Model): ) ) + class Meta: + app_label = "django_celery_results" + + def group_result(self, app=None): """Return the GroupResult of self. @@ -71,3 +76,4 @@ class Meta(AbstractGroupResult.Meta): """Table information.""" abstract = False + app_label = "django_celery_results" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py new file mode 100644 index 00000000..668ea244 --- /dev/null +++ b/django_celery_results/models/helpers.py @@ -0,0 +1,47 @@ +from django.apps import apps +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured + +from .generic import TaskResult, GroupResult + +def taskresult_model(): + """Return the TaskResult model that is active in this project.""" + if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): + return TaskResult + + try: + return apps.get_model( + settings.CELERY_RESULTS_TASKRESULT_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_TASKRESULT_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_TASKRESULT_MODEL refers to model " + f"'{settings.CELERY_RESULTS_TASKRESULT_MODEL}' that has not " + "been installed" + ) + +def groupresult_model(): + """Return the GroupResult model that is active in this project.""" + if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): + return GroupResult + + try: + return apps.get_model( + settings.CELERY_RESULTS_GROUPRESULT_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_GROUPRESULT_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_GROUPRESULT_MODEL refers to model " + f"'{settings.CELERY_RESULTS_GROUPRESULT_MODEL}' that has not " + "been installed" + ) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py new file mode 100644 index 00000000..5f4eb61d --- /dev/null +++ b/django_celery_results/settings.py @@ -0,0 +1,16 @@ +from django.conf import settings + + +def get_callback_function(settings_name, default=None): + """Return the callback function for the given settings name.""" + + callback = getattr(settings, settings_name, None) + if callback is None: + return default + + if callable(callback): + return callback + +extend_task_props_callback = get_callback_function( + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" +) \ No newline at end of file From dbeb9a16be1402db5da0ef4f9254370d66b05deb Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 10 Aug 2022 21:27:41 +0200 Subject: [PATCH 06/29] [#305]: `extend_task_props_callback` relocated. `extend_task_props_callback` moved from `_get_extended_properties` to `_store_result`. Suggested by @AllesVeldman. `extend_task_props_callback` will get the `request` object as first parameter and a copy of `task_props` (avoiding potential manipulation of the original `task_props`) as second paramenter. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/backends/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 018e5ca0..ab888846 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -80,7 +80,7 @@ def _get_extended_properties(self, request, traceback): # task protocol 1 task_kwargs = getattr(request, 'kwargs', None) - # TODO: We assuming that task protocol 1 could be always in use. :/ + # TODO: We assume that task protocol 1 could be always in use. :/ extra_fields = extend_task_props_callback( getattr(request, 'kwargs', None) ) @@ -151,6 +151,8 @@ def _store_result( task_props.update( self._get_extended_properties(request, traceback) ) + task_props.update( + extend_task_props_callback(request, dict(task_props))) if status == states.STARTED: task_props['date_started'] = Now() From 7565b3669905dd9b24c165801846efb77da2111e Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 15 Aug 2022 20:42:13 +0200 Subject: [PATCH 07/29] [#305]: Added a default callable to `get_callback_function` `get_callback_function()` gets a default callback as an arg returning explicitely an empty dict. `get_callback_function()` raises an `ImproperlyConfigured` exception when the callback is not callable. --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/settings.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 5f4eb61d..a9889a13 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,16 +1,19 @@ from django.conf import settings +from django.core.exceptions import ImproperlyConfigured def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" callback = getattr(settings, settings_name, None) - if callback is None: + if not callback: return default - if callable(callback): - return callback + if not callable(callback): + raise ImproperlyConfigured(f"{settings_name} must be callable.") + + return callback extend_task_props_callback = get_callback_function( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict ) \ No newline at end of file From 1dfa0fc1288952b716c4d3d289701261967c10e3 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 09:12:57 +0200 Subject: [PATCH 08/29] Added newline to the end of --- django_celery_results/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index a9889a13..5f10baae 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -4,7 +4,7 @@ def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" - + callback = getattr(settings, settings_name, None) if not callback: return default @@ -14,6 +14,7 @@ def get_callback_function(settings_name, default=None): return callback + extend_task_props_callback = get_callback_function( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict -) \ No newline at end of file +) From 14928be075d18adf00c970bcd13adefb80993559 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 16:23:18 +0200 Subject: [PATCH 09/29] [#305] Added a sanity check to `task_props_extension` Added `get_task_props_extension` to `settings` module which will raise an `ImproperlyConfigured` when the task_props_extension doesn't complies with the Mapping protocol. `DatabaseBackend` will make use of `get_task_props_extension` to update a potential custom model with the custom properties --- Resolves celery/django-celery-results#305 Fixes celery/django-celery-results#314 --- django_celery_results/backends/database.py | 10 ++++------ django_celery_results/settings.py | 19 ++++++++++++++++++- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index ab888846..9b8ddb97 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -1,5 +1,6 @@ import binascii import json +from typing import Mapping from celery import maybe_signature, states from celery.backends.base import BaseDictBackend, get_current_task @@ -14,7 +15,7 @@ from ..models import ChordCounter from ..models.helpers import groupresult_model, taskresult_model -from ..settings import extend_task_props_callback +from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -148,11 +149,8 @@ def _store_result( 'using': using, } - task_props.update( - self._get_extended_properties(request, traceback) - ) - task_props.update( - extend_task_props_callback(request, dict(task_props))) + task_props.update(self._get_extended_properties(request, traceback)) + task_props.update(get_task_props_extension(request, dict(task_props))) if status == states.STARTED: task_props['date_started'] = Now() diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 5f10baae..11cb524b 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,5 +1,6 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from collections.abc import Mapping def get_callback_function(settings_name, default=None): @@ -16,5 +17,21 @@ def get_callback_function(settings_name, default=None): extend_task_props_callback = get_callback_function( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK", dict + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK" ) + + +def get_task_props_extension(request, task_props): + """Extend the task properties with custom properties to fill custom models.""" + + task_props_extension = extend_task_props_callback(request, task_props) or {} + if task_props_extension is None: + return {} + + if not isinstance(task_props_extension, Mapping): + raise ImproperlyConfigured( + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " + "instance." + ) + + return task_props_extension From b70c44f4a8d5d58dd705e8c5f8b89e403c17ac0e Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 16 Aug 2022 20:22:25 +0200 Subject: [PATCH 10/29] Fixed a NoneType error when the callback is not defined in project settings. --- django_celery_results/settings.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 11cb524b..c41a2469 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -5,7 +5,6 @@ def get_callback_function(settings_name, default=None): """Return the callback function for the given settings name.""" - callback = getattr(settings, settings_name, None) if not callback: return default @@ -22,12 +21,11 @@ def get_callback_function(settings_name, default=None): def get_task_props_extension(request, task_props): - """Extend the task properties with custom properties to fill custom models.""" - - task_props_extension = extend_task_props_callback(request, task_props) or {} - if task_props_extension is None: + """Extend the task properties with custom props to fill custom models.""" + if not extend_task_props_callback: return {} + task_props_extension = extend_task_props_callback(request, task_props) or {} if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " From cedccdf88b92c40dee4ad3c061802c5e5f377581 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 13 Oct 2022 13:22:35 +0200 Subject: [PATCH 11/29] [#305] Added documentation about this feature. A new page added to the documentation where we explain how to use this feature. -- issue: celery/django-celery-results#305 pull-request: celery/django-celery-results#314 --- docs/extending_task_results.rst | 43 +++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 44 insertions(+) create mode 100644 docs/extending_task_results.rst diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst new file mode 100644 index 00000000..145ff404 --- /dev/null +++ b/docs/extending_task_results.rst @@ -0,0 +1,43 @@ +Extending Task Results +====================== + +There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from an tenant. + +To extend the Task Results model follow the next steps: + +#. Create a custom model that inherits from the abstract base class `django_celery_results.models.abstract.AbstractTaskResult`: + + .. code-block:: python + + from django_celery_results.models.abstract import AbstractTaskResult + + class TaskResult(AbstractTaskResult): + tenant = models.ForeignKey(Tenant, on_delete=models.CASCADE, null=True) + +#. Tell Django to use the custom `TaskResult` model by setting the `CELERY_RESULTS_TASKRESULT_MODEL` constant to the path of the custom model. + + .. code-block:: python + + CELERY_RESULTS_TASKRESULT_MODEL = 'myapp.TaskResult' + +#. Write a function in your Django project's :file:`settings.py` that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in the your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. + + .. code-block:: python + + def extend_task_props_callback(request, task_properties): + """Extend task props with custom data from task_kwargs.""" + task_kwargs = getattr(request, "kwargs", None) + + return {"tenant_id": task_kwargs.get("tenant_id", None)} + +#. To let :pypi:`django-celery-results` call this function internally you've to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py` with the function that you've just created. + + .. code-block:: python + + CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK = extend_task_props_callback + +#. Finally make sure that you're passing the additional information to the celery task when you're calling it. + + .. code-block:: python + + task.apply_async(kwargs={"tenant_id": tenant.id}) \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index ad00baf2..65441516 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -11,6 +11,7 @@ Contents :maxdepth: 1 getting_started + extending_task_results injecting_metadata copyright From 0fc0b4895262ac984de27dec271386e6751a9fa4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 17 Aug 2022 06:03:54 +0000 Subject: [PATCH 12/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/admin.py | 2 +- django_celery_results/models/helpers.py | 3 ++- django_celery_results/settings.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/django_celery_results/admin.py b/django_celery_results/admin.py index e0654aec..aff82117 100644 --- a/django_celery_results/admin.py +++ b/django_celery_results/admin.py @@ -10,7 +10,7 @@ ALLOW_EDITS = False pass -from .models.helpers import taskresult_model, groupresult_model +from .models.helpers import groupresult_model, taskresult_model GroupResult = groupresult_model() TaskResult = taskresult_model() diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 668ea244..a9818070 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,7 +2,8 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import TaskResult, GroupResult +from .generic import GroupResult, TaskResult + def taskresult_model(): """Return the TaskResult model that is active in this project.""" diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index c41a2469..6b246438 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -1,6 +1,7 @@ +from collections.abc import Mapping + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from collections.abc import Mapping def get_callback_function(settings_name, default=None): From 242edabb8785f88bf81db2210ce8dbac6d2791f7 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 17 Aug 2022 17:02:38 +0200 Subject: [PATCH 13/29] Fixed a "wrong" description for the `ImproperlyConfigured` exception in `django_celery_results.settings` Fixed a "wrong" description for the `ImproperlyConfigured` exception raised when `task_props_extension` doesn't complies with the Mapping protocol. At this point `task_props_extension` is just a dict that inherits from Mapping, not an explicit instance. Thanks @AllexVeldman Co-authored-by: Allex --- django_celery_results/settings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 6b246438..05689fe5 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -29,8 +29,7 @@ def get_task_props_extension(request, task_props): task_props_extension = extend_task_props_callback(request, task_props) or {} if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( - "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping " - "instance." + "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping." ) return task_props_extension From 7ddefa85c3b83f361bf665706450f9ec63727b93 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 13 Oct 2022 19:51:00 +0200 Subject: [PATCH 14/29] [#305] Fixed some pre-commit failures --- django_celery_results/backends/database.py | 1 - django_celery_results/models/generic.py | 1 - django_celery_results/models/helpers.py | 5 +++-- django_celery_results/settings.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 9b8ddb97..eb0abdb5 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -1,6 +1,5 @@ import binascii import json -from typing import Mapping from celery import maybe_signature, states from celery.backends.base import BaseDictBackend, get_current_task diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index f1ec7e01..582983eb 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -52,7 +52,6 @@ class ChordCounter(models.Model): class Meta: app_label = "django_celery_results" - def group_result(self, app=None): """Return the GroupResult of self. diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index a9818070..fb64fd84 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -9,7 +9,7 @@ def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): return TaskResult - + try: return apps.get_model( settings.CELERY_RESULTS_TASKRESULT_MODEL @@ -26,11 +26,12 @@ def taskresult_model(): "been installed" ) + def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): return GroupResult - + try: return apps.get_model( settings.CELERY_RESULTS_GROUPRESULT_MODEL diff --git a/django_celery_results/settings.py b/django_celery_results/settings.py index 05689fe5..022b52a0 100644 --- a/django_celery_results/settings.py +++ b/django_celery_results/settings.py @@ -26,7 +26,7 @@ def get_task_props_extension(request, task_props): if not extend_task_props_callback: return {} - task_props_extension = extend_task_props_callback(request, task_props) or {} + task_props_extension = extend_task_props_callback(request, task_props) or {} # noqa E501 if not isinstance(task_props_extension, Mapping): raise ImproperlyConfigured( "CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK must return a Mapping." From a5681a65d46914a8980864732c7c35c680e52ade Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:01 +0200 Subject: [PATCH 15/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index 145ff404..bc8df9be 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -1,7 +1,7 @@ Extending Task Results ====================== -There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from an tenant. +There are situations where you want to extend the Task Results with additional information that will make you able to retrieve information that was important at execution time of the task but not part of the task result itself. For example if you use :pypi:`django-celery-results` to track the task results from a tenant. To extend the Task Results model follow the next steps: From 48f278a5b1baed353ddefd3113659871fb0004f5 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:35 +0200 Subject: [PATCH 16/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index bc8df9be..85f19ed4 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -30,7 +30,7 @@ To extend the Task Results model follow the next steps: return {"tenant_id": task_kwargs.get("tenant_id", None)} -#. To let :pypi:`django-celery-results` call this function internally you've to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py` with the function that you've just created. +#. To let :pypi:`django-celery-results` call this function, you'll have to set the `CELERY_RESULTS_EXTEND_TASK_PROPS_CALLBACK` constant in your Django project's :file:`settings.py`. .. code-block:: python From fb5ea91174345aab8613a8686be78f30d65c3591 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 17 Oct 2022 19:03:59 +0200 Subject: [PATCH 17/29] Update docs/extending_task_results.rst Co-authored-by: Allex --- docs/extending_task_results.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/extending_task_results.rst b/docs/extending_task_results.rst index 85f19ed4..167422b9 100644 --- a/docs/extending_task_results.rst +++ b/docs/extending_task_results.rst @@ -20,7 +20,7 @@ To extend the Task Results model follow the next steps: CELERY_RESULTS_TASKRESULT_MODEL = 'myapp.TaskResult' -#. Write a function in your Django project's :file:`settings.py` that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in the your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. +#. Write a function that will consume a `request` and `task_properties` as positional arguments and will return a dictionary with the additional information that you want to store in your custom `TaskResult` model. The keys of this dictionary will be the fields of the custom model and the values the data you can retrieve from the `request` and/or `task_properties`. .. code-block:: python From 10a5c3aa7e0c951aa0f8483793cce3dc635874fc Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Mon, 21 Oct 2024 21:10:09 +0200 Subject: [PATCH 18/29] feat(models): add `AbstractChordCounter` and update `ChordCounter` - Added new abstract model `AbstractChordCounter` in `abstract.py` for Chord synchronization, including fields for `group_id`, `sub_tasks`, and `count`. - Updated `ChordCounter` model in `generic.py` to inherit from `AbstractChordCounter`. - Moved `group_result` method from `ChordCounter` to `AbstractChordCounter`. - Added helper function `chordcounter_model` in `helpers.py` to return the active `ChordCounter` model. - Updated import statements in `database.py` to use `chordcounter_model` helper function. - Added `ChordCounterModel` attribute to `DatabaseBackend` class, and updated `create` and `get` methods to use `ChordCounterModel` instead of `ChordCounter`. Relates to: #305 --- django_celery_results/backends/database.py | 10 ++--- django_celery_results/models/abstract.py | 51 ++++++++++++++++++++++ django_celery_results/models/generic.py | 47 ++------------------ django_celery_results/models/helpers.py | 25 ++++++++++- 4 files changed, 84 insertions(+), 49 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index eb0abdb5..8492a0a8 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,8 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models import ChordCounter -from ..models.helpers import groupresult_model, taskresult_model +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -32,6 +31,7 @@ class DatabaseBackend(BaseDictBackend): TaskModel = taskresult_model() GroupModel = groupresult_model() + ChordCounterModel = chordcounter_model() subpolling_interval = 0.5 def exception_safe_to_retry(self, exc): @@ -246,7 +246,7 @@ def apply_chord(self, header_result_args, body, **kwargs): results = [r.as_tuple() for r in header_result] chord_size = body.get("chord_size", None) or len(results) data = json.dumps(results) - ChordCounter.objects.create( + self.ChordCounterModel.objects.create( group_id=header_result.id, sub_tasks=data, count=chord_size ) @@ -263,10 +263,10 @@ def on_chord_part_return(self, request, state, result, **kwargs): # SELECT FOR UPDATE is not supported on all databases try: chord_counter = ( - ChordCounter.objects.select_for_update() + self.ChordCounterModel.objects.select_for_update() .get(group_id=gid) ) - except ChordCounter.DoesNotExist: + except self.ChordCounterModel.DoesNotExist: logger.warning("Can't find ChordCounter for Group %s", gid) return chord_counter.count -= 1 diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 99aeb65e..fe48423e 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -1,11 +1,15 @@ """Abstract models.""" +import json + from celery import states +from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from .. import managers +from ..models.helpers import groupresult_model ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) @@ -131,6 +135,53 @@ def __str__(self): return ''.format(self) +class AbstractChordCounter(models.Model): + """Abstract Chord synchronisation.""" + + group_id = models.CharField( + max_length=getattr( + settings, + "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", + 255 + ), + unique=True, + verbose_name=_("Group ID"), + help_text=_("Celery ID for the Chord header group"), + ) + sub_tasks = models.TextField( + help_text=_( + "JSON serialized list of task result tuples. " + "use .group_result() to decode" + ) + ) + count = models.PositiveIntegerField( + help_text=_( + "Starts at len(chord header) and decrements after each task is " + "finished" + ) + ) + + class Meta: + """Table information.""" + + abstract = True + + def group_result(self, app=None): + """Return the GroupResult of self. + + Arguments: + --------- + app (Celery): app instance to create the GroupResult with. + + """ + return groupresult_model()( + self.group_id, + [result_from_tuple(r, app=app) + for r in json.loads(self.sub_tasks)], + app=app + ) + + class AbstractGroupResult(models.Model): """Abstract Task Group result/status.""" diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 582983eb..e5b5da7e 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -2,13 +2,10 @@ import json -from celery.result import GroupResult as CeleryGroupResult -from celery.result import result_from_tuple -from django.conf import settings -from django.db import models from django.utils.translation import gettext_lazy as _ from django_celery_results.models.abstract import ( + AbstractChordCounter, AbstractGroupResult, AbstractTaskResult, ) @@ -24,49 +21,13 @@ class Meta(AbstractTaskResult.Meta): app_label = "django_celery_results" -class ChordCounter(models.Model): +class ChordCounter(AbstractChordCounter): """Chord synchronisation.""" - group_id = models.CharField( - max_length=getattr( - settings, - "DJANGO_CELERY_RESULTS_TASK_ID_MAX_LENGTH", - 255), - unique=True, - verbose_name=_("Group ID"), - help_text=_("Celery ID for the Chord header group"), - ) - sub_tasks = models.TextField( - help_text=_( - "JSON serialized list of task result tuples. " - "use .group_result() to decode" - ) - ) - count = models.PositiveIntegerField( - help_text=_( - "Starts at len(chord header) and decrements after each task is " - "finished" - ) - ) - - class Meta: + class Meta(AbstractChordCounter.Meta): + abstract = False app_label = "django_celery_results" - def group_result(self, app=None): - """Return the GroupResult of self. - - Arguments: - --------- - app (Celery): app instance to create the GroupResult with. - - """ - return CeleryGroupResult( - self.group_id, - [result_from_tuple(r, app=app) - for r in json.loads(self.sub_tasks)], - app=app - ) - class GroupResult(AbstractGroupResult): """Task Group result/status.""" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index fb64fd84..0d9da60a 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,7 +2,7 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import GroupResult, TaskResult +from .generic import ChordCounter, GroupResult, TaskResult def taskresult_model(): @@ -27,6 +27,29 @@ def taskresult_model(): ) +def chordcounter_model(): + """Return the ChordCounter model that is active in this project.""" + + if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): + return ChordCounter + + try: + return apps.get_model( + settings.CELERY_RESULTS_CHORDCOUNTER_MODEL + ) + except ValueError: + raise ImproperlyConfigured( + "CELERY_RESULTS_CHORDCOUNTER_MODEL must be of the form " + "'app_label.model_name'" + ) + except LookupError: + raise ImproperlyConfigured( + "CELERY_RESULTS_CHORDCOUNTER_MODEL refers to model " + f"'{settings.CELERY_RESULTS_CHORDCOUNTER_MODEL}' that has not " + "been installed" + ) + + def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): From d6b0d19ad1f8f3d49ed110af07b2a6e52846edbc Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 29 Oct 2024 09:40:47 +0100 Subject: [PATCH 19/29] fix: refactor helper functions to avoid circular dependencies --- django_celery_results/models/generic.py | 4 ++-- django_celery_results/models/helpers.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index e5b5da7e..25c0744c 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -1,7 +1,5 @@ """Database models.""" -import json - from django.utils.translation import gettext_lazy as _ from django_celery_results.models.abstract import ( @@ -25,6 +23,8 @@ class ChordCounter(AbstractChordCounter): """Chord synchronisation.""" class Meta(AbstractChordCounter.Meta): + """Table information.""" + abstract = False app_label = "django_celery_results" diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 0d9da60a..13387643 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,12 +2,12 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from .generic import ChordCounter, GroupResult, TaskResult - def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): + from .generic import TaskResult + return TaskResult try: @@ -31,6 +31,8 @@ def chordcounter_model(): """Return the ChordCounter model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): + from .generic import ChordCounter + return ChordCounter try: @@ -53,6 +55,8 @@ def chordcounter_model(): def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): + from .generic import GroupResult + return GroupResult try: From 946ff07ca7701d41e7b03f9354131e5067a13a2a Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 29 Oct 2024 10:05:27 +0100 Subject: [PATCH 20/29] fix: undefined name 'ChordCounter' and minor fixes --- django_celery_results/backends/database.py | 12 +++++++++--- django_celery_results/models/generic.py | 2 -- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index 8492a0a8..efc3a27b 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) @@ -81,7 +85,7 @@ def _get_extended_properties(self, request, traceback): task_kwargs = getattr(request, 'kwargs', None) # TODO: We assume that task protocol 1 could be always in use. :/ - extra_fields = extend_task_props_callback( + extra_fields = get_task_props_extension( getattr(request, 'kwargs', None) ) if extra_fields: @@ -256,7 +260,9 @@ def on_chord_part_return(self, request, state, result, **kwargs): if not gid or not tid: return call_callback = False - with transaction.atomic(using=router.db_for_write(ChordCounter)): + with transaction.atomic( + using=router.db_for_write(self.ChordCounterModel) + ): # We need to know if `count` hits 0. # wrap the update in a transaction # with a `select_for_update` lock to prevent race conditions. diff --git a/django_celery_results/models/generic.py b/django_celery_results/models/generic.py index 25c0744c..7b626dae 100644 --- a/django_celery_results/models/generic.py +++ b/django_celery_results/models/generic.py @@ -1,7 +1,5 @@ """Database models.""" -from django.utils.translation import gettext_lazy as _ - from django_celery_results.models.abstract import ( AbstractChordCounter, AbstractGroupResult, From 39081b3c4bc3fcce5d9ad38d9fea6b7b313685a7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:05:49 +0000 Subject: [PATCH 21/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/backends/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index efc3a27b..b41a99b8 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,11 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import ( - chordcounter_model, - groupresult_model, - taskresult_model, -) +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From cbdb7a47648feac198e800fe4beab982b96d6595 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 31 Oct 2024 16:04:23 +0100 Subject: [PATCH 22/29] fix: 'TypeError' introduced in previous commit --- django_celery_results/managers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_celery_results/managers.py b/django_celery_results/managers.py index b38a15e8..58c00c9c 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/managers.py @@ -161,7 +161,7 @@ def store_result(self, content_type, content_encoding, 'task_args': task_args, 'task_kwargs': task_kwargs, 'worker': worker, - **extra_fields + 'extra_fields': extra_fields } if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] From 29cc821a94d9788d62dd26a3ef18f3be8fc42a08 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Thu, 31 Oct 2024 16:29:21 +0100 Subject: [PATCH 23/29] fix: include 'extra_fields' conditionally --- django_celery_results/managers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/django_celery_results/managers.py b/django_celery_results/managers.py index 58c00c9c..274b85cf 100644 --- a/django_celery_results/managers.py +++ b/django_celery_results/managers.py @@ -161,8 +161,11 @@ def store_result(self, content_type, content_encoding, 'task_args': task_args, 'task_kwargs': task_kwargs, 'worker': worker, - 'extra_fields': extra_fields } + + if extra_fields is not None: + fields.update(extra_fields) + if 'date_started' in kwargs: fields['date_started'] = kwargs['date_started'] From c5081793556a9ff2d8287bf9da2b67771316d3bb Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 1 Nov 2024 20:28:11 +0100 Subject: [PATCH 24/29] fix: 'get_task_props_extensions()' missing 1 required argument --- django_celery_results/backends/database.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index b41a99b8..dc04f822 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -82,6 +82,7 @@ def _get_extended_properties(self, request, traceback): # TODO: We assume that task protocol 1 could be always in use. :/ extra_fields = get_task_props_extension( + request, getattr(request, 'kwargs', None) ) if extra_fields: From 101a24fa24a508298a2e39d1d276dab618fe48f4 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Tue, 5 Nov 2024 21:13:38 +0100 Subject: [PATCH 25/29] fix: TypeError introducedn on prev commit on 'AbstractChordCounter' --- django_celery_results/backends/database.py | 6 +++++- django_celery_results/models/abstract.py | 5 ++--- django_celery_results/models/helpers.py | 8 ++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index dc04f822..d088bfad 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index fe48423e..4b43a9e0 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -3,13 +3,12 @@ import json from celery import states -from celery.result import result_from_tuple +from celery.result import CeleryGroupResult, result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ from .. import managers -from ..models.helpers import groupresult_model ALL_STATES = sorted(states.ALL_STATES) TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) @@ -174,7 +173,7 @@ def group_result(self, app=None): app (Celery): app instance to create the GroupResult with. """ - return groupresult_model()( + return CeleryGroupResult( self.group_id, [result_from_tuple(r, app=app) for r in json.loads(self.sub_tasks)], diff --git a/django_celery_results/models/helpers.py b/django_celery_results/models/helpers.py index 13387643..0d9da60a 100644 --- a/django_celery_results/models/helpers.py +++ b/django_celery_results/models/helpers.py @@ -2,12 +2,12 @@ from django.conf import settings from django.core.exceptions import ImproperlyConfigured +from .generic import ChordCounter, GroupResult, TaskResult + def taskresult_model(): """Return the TaskResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_TASKRESULT_MODEL'): - from .generic import TaskResult - return TaskResult try: @@ -31,8 +31,6 @@ def chordcounter_model(): """Return the ChordCounter model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_CHORDCOUNTER_MODEL'): - from .generic import ChordCounter - return ChordCounter try: @@ -55,8 +53,6 @@ def chordcounter_model(): def groupresult_model(): """Return the GroupResult model that is active in this project.""" if not hasattr(settings, 'CELERY_RESULTS_GROUPRESULT_MODEL'): - from .generic import GroupResult - return GroupResult try: From af843ba26441c828cc29c8e8a087484a0f1b9549 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 5 Nov 2024 20:15:19 +0000 Subject: [PATCH 26/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- django_celery_results/backends/database.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index d088bfad..dc04f822 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,11 +12,7 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import ( - chordcounter_model, - groupresult_model, - taskresult_model, -) +from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From 90310b838b2e129248de8c0213ffbfa01aa96984 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Wed, 6 Nov 2024 21:20:11 +0100 Subject: [PATCH 27/29] fix: ImportError introduced in previous commit in 'abstract.py' --- django_celery_results/models/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/django_celery_results/models/abstract.py b/django_celery_results/models/abstract.py index 4b43a9e0..be3b595c 100644 --- a/django_celery_results/models/abstract.py +++ b/django_celery_results/models/abstract.py @@ -3,7 +3,8 @@ import json from celery import states -from celery.result import CeleryGroupResult, result_from_tuple +from celery.result import GroupResult as CeleryGroupResult +from celery.result import result_from_tuple from django.conf import settings from django.db import models from django.utils.translation import gettext_lazy as _ From e2a2bc02c44860a6d973f70142f4a4f521fd4cc7 Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 8 Nov 2024 07:46:03 +0100 Subject: [PATCH 28/29] style: Reformat import statements for better readability in 'database.py' --- django_celery_results/backends/database.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/django_celery_results/backends/database.py b/django_celery_results/backends/database.py index dc04f822..d088bfad 100644 --- a/django_celery_results/backends/database.py +++ b/django_celery_results/backends/database.py @@ -12,7 +12,11 @@ from django.db.utils import InterfaceError from kombu.exceptions import DecodeError -from ..models.helpers import chordcounter_model, groupresult_model, taskresult_model +from ..models.helpers import ( + chordcounter_model, + groupresult_model, + taskresult_model, +) from ..settings import get_task_props_extension EXCEPTIONS_TO_CATCH = (InterfaceError,) From af5a2266e73b010ea69a4b926ede952802b541fe Mon Sep 17 00:00:00 2001 From: Diego Castro Date: Fri, 8 Nov 2024 08:00:00 +0100 Subject: [PATCH 29/29] fix: Update configuration for isort and black to enforce line length and multi-line output This commit addresses an issue where the pre-commit hook merges import formatted with isort's multiline output mode 3 back into a single line, resulting in a flake8 E508 violation in CI due to lines exceeding 79 characters --- setup.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.cfg b/setup.cfg index 26f1ac65..8086949d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,3 +18,8 @@ match-dir = [^migrations] [isort] profile=black +line_length=79 +multi_line_output=3 + +[black] +line-length = 79