From ffa550b12e684472d166c6c44e9a9788fa4e3bad Mon Sep 17 00:00:00 2001 From: Selwin Ong Date: Mon, 16 Sep 2024 10:50:07 +0700 Subject: [PATCH 01/15] rqworker command can now run --- django_rq/management/commands/rqworker.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/django_rq/management/commands/rqworker.py b/django_rq/management/commands/rqworker.py index 2e61442d..e8b0d2dc 100644 --- a/django_rq/management/commands/rqworker.py +++ b/django_rq/management/commands/rqworker.py @@ -2,7 +2,6 @@ import sys from redis.exceptions import ConnectionError -from rq import Connection from rq.logutils import setup_loghandlers from django.core.management.base import BaseCommand @@ -84,21 +83,20 @@ def handle(self, *args, **options): 'queue_class': options['queue_class'], 'job_class': options['job_class'], 'name': options['name'], - 'default_worker_ttl': options['worker_ttl'], + 'worker_ttl': options['worker_ttl'], 'serializer': options['serializer'] } w = get_worker(*args, **worker_kwargs) # Call Connection context manager to push the redis connection into LocalStack # without this, jobs using RQ's get_current_job() will fail - with Connection(w.connection): - # Close any opened DB connection before any fork - reset_db_connections() + # Close any opened DB connection before any fork + reset_db_connections() - w.work( - burst=options.get('burst', False), with_scheduler=options.get('with_scheduler', False), - logging_level=level, max_jobs=options['max_jobs'] - ) + w.work( + burst=options.get('burst', False), with_scheduler=options.get('with_scheduler', False), + logging_level=level, max_jobs=options['max_jobs'] + ) except ConnectionError as e: self.stderr.write(str(e)) sys.exit(1) From b02a7245d1c355f6a676db4fdf9966809456f576 Mon Sep 17 00:00:00 2001 From: Selwin Ong Date: Tue, 1 Oct 2024 18:39:37 +0700 Subject: [PATCH 02/15] Make job_detail.html display better --- django_rq/templates/django_rq/job_detail.html | 140 +++++++++++------- 1 file changed, 88 insertions(+), 52 deletions(-) diff --git a/django_rq/templates/django_rq/job_detail.html b/django_rq/templates/django_rq/job_detail.html index e10a2ec3..c920cad3 100644 --- a/django_rq/templates/django_rq/job_detail.html +++ b/django_rq/templates/django_rq/job_detail.html @@ -37,30 +37,38 @@
- -
{{ job.origin }}
+
+ +
{{ job.origin }}
+
- -
{{ job.timeout }}
+
+ +
{{ job.timeout }}
+
- -
{{ job.result_ttl }}
+
+ +
{{ job.result_ttl }}
+
{% if job.created_at %}
- -
{{ job.created_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
+ +
{{ job.created_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
{% endif %} @@ -68,8 +76,10 @@ {% if job.enqueued_at %}
- -
{{ job.enqueued_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
+ +
{{ job.enqueued_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
{% endif %} @@ -77,8 +87,10 @@ {% if job.started_at %}
- -
{{ job.started_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
+ +
{{ job.started_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
{% endif %} @@ -86,30 +98,38 @@ {% if job.ended_at %}
- -
{{ job.ended_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
+ +
{{ job.ended_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
{% endif %}
- -
{{ job.get_status }}
+
+ +
{{ job.get_status }}
+
- -
{{ job|show_func_name }}
+
+ +
{{ job|show_func_name }}
+
- -
{{ job.meta }}
+
+ +
{{ job.meta }}
+
@@ -117,17 +137,19 @@
- {% if data_is_valid %} - {% if job.args %} -
    - {% for arg in job.args %} -
  • {{ arg|force_escape }}
    • - {% endfor %} -
+
+ {% if data_is_valid %} + {% if job.args %} +
    + {% for arg in job.args %} +
  • {{ arg|force_escape }}
    • + {% endfor %} +
+ {% endif %} + {% else %} + Unpickling Error {% endif %} - {% else %} - Unpickling Error - {% endif %} +
@@ -136,17 +158,19 @@
- {% if data_is_valid %} - {% if job.kwargs %} -
    - {% for key, value in job.kwargs|items %} -
  • {{ key }}: {{ value|force_escape }}
    • - {% endfor %} -
+
+ {% if data_is_valid %} + {% if job.kwargs %} +
    + {% for key, value in job.kwargs|items %} +
  • {{ key }}: {{ value|force_escape }}
    • + {% endfor %} +
+ {% endif %} + {% else %} + Unpickling Error {% endif %} - {% else %} - Unpickling Error - {% endif %} +
@@ -175,8 +199,10 @@ {% if job.legacy_result %}
- -
{{ job.result }}
+
+ +
{{ job.result }}
+
{% endif %} @@ -193,13 +219,13 @@ {% endif %} {% if job.is_failed %} -
+
{% csrf_token %} {% endif %} {% if not job.is_queued and not job.is_failed %} -
+
{% csrf_token %} @@ -216,29 +242,39 @@

Result {{ result.id }}

- -
{{ result.type.name }}
+
+ +
+ {{ result.type.name }} +
+
- -
{{ result.created_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
+ +
{{ result.created_at|to_localtime|date:"Y-m-d, H:i:s" }}
+
{% if result.type.value == 1 %}
- - 
{{ result.return_value }}
+
+ + 
{{ result.return_value }}
+
{% elif result.type.value == 2 %}
- - 
{{ result.exc_string }}
+
+ + 
{{ result.exc_string }}
+
{% endif %} From 7df99c298335aa6e98383eab400500e9ec86ded9 Mon Sep 17 00:00:00 2001 From: Selwin Ong Date: Tue, 1 Oct 2024 21:25:39 +0700 Subject: [PATCH 03/15] Properly show currently running executions --- .../django_rq/started_job_registry.html | 140 ++++++++++++++++++ django_rq/utils.py | 15 ++ django_rq/views.py | 7 +- 3 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 django_rq/templates/django_rq/started_job_registry.html diff --git a/django_rq/templates/django_rq/started_job_registry.html b/django_rq/templates/django_rq/started_job_registry.html new file mode 100644 index 00000000..71e9bbe8 --- /dev/null +++ b/django_rq/templates/django_rq/started_job_registry.html @@ -0,0 +1,140 @@ +{% extends "admin/base_site.html" %} + +{% load static jquery_path django_rq %} + +{% block title %}{{ job_status }} Job Executions in {{ queue.name }} {{ block.super }}{% endblock %} + +{% block extrastyle %} + {{ block.super }} + +{% endblock %} + +{% block extrahead %} + {{ block.super }} + + + + +{% endblock %} + + +{% block breadcrumbs %} +
+ Home › + Django RQ › + {{ queue.name }} +
+{% endblock %} + +{% block content_title %}

Job Executions in {{ queue.name }}

{% endblock %} + +{% block content %} + +
+ +
+
+
+ {% csrf_token %} +
+ + +
+
+ + + + + + + + + + {% block extra_columns %} + {% endblock extra_columns %} + + + + {% for execution in executions %} + + + + + + + + {% block extra_columns_values %} + {% endblock extra_columns_values %} + + {% endfor %} + +
+
+ +
+
+ 		+
ID
+
+ 		+
Created
+
+ 		+
Last Heartbeat
+
+ 		+
Enqueued
+
+ 		+
Callable
+
+
+ + + + {{ execution.id }} + + + {% if execution.created_at %} + {{ execution.created_at|to_localtime|date:"Y-m-d, H:i:s" }} + {% endif %} + + {% if execution.last_heartbeat %} + {{ execution.last_heartbeat|to_localtime|date:"Y-m-d, H:i:s" }} + {% endif %} + + {% if execution.job.enqueued_at %} + {{ execution.job.enqueued_at|to_localtime|date:"Y-m-d, H:i:s" }} + {% endif %} + {{ execution.job|show_func_name }}
+
+

+ {% for p in page_range %} + {% if p == page %} + {{ p }} + {% elif forloop.last %} + {{ p }} + {% else %} + {{ p }} + {% endif %} + {% endfor %} + {{ num_jobs }} jobs +

+
+
+
+
+ +{% endblock %} diff --git a/django_rq/utils.py b/django_rq/utils.py index 106c6db8..828176eb 100644 --- a/django_rq/utils.py +++ b/django_rq/utils.py @@ -4,6 +4,7 @@ from django.db import connections from redis.sentinel import SentinelConnectionPool from rq.command import send_stop_job_command +from rq.executions import Execution from rq.job import Job from rq.registry import ( DeferredJobRegistry, @@ -154,6 +155,20 @@ def get_jobs( return valid_jobs +def get_executions(queue, composite_keys: List[str]) -> List[Execution]: + """Fetch executions in bulk from Redis. + 1. If execution data is not present in Redis, discard the result + """ + executions = [] + for key in composite_keys: + job_id, id = key.split(':') + try: + executions.append(Execution.fetch(id=id, job_id=job_id, connection=queue.connection)) + except ValueError: + pass + return executions + + def stop_jobs(queue, job_ids): job_ids = job_ids if isinstance(job_ids, (list, tuple)) else [job_ids] stopped_job_ids = [] diff --git a/django_rq/views.py b/django_rq/views.py index 1affe006..4f29d2ac 100644 --- a/django_rq/views.py +++ b/django_rq/views.py @@ -26,7 +26,7 @@ from .queues import get_queue_by_index, get_scheduler_by_index from .settings import API_TOKEN, QUEUES_MAP -from .utils import get_jobs, get_scheduler_statistics, get_statistics, stop_jobs +from .utils import get_executions, get_jobs, get_scheduler_statistics, get_statistics, stop_jobs @never_cache @@ -204,6 +204,7 @@ def started_jobs(request, queue_index): num_jobs = len(registry) page = int(request.GET.get('page', 1)) jobs = [] + executions = [] if num_jobs > 0: last_page = int(ceil(num_jobs / items_per_page)) @@ -211,6 +212,7 @@ def started_jobs(request, queue_index): offset = items_per_page * (page - 1) job_ids = registry.get_job_ids(offset, offset + items_per_page - 1) jobs = get_jobs(queue, job_ids, registry) + executions = get_executions(queue, job_ids) else: page_range = [] @@ -224,8 +226,9 @@ def started_jobs(request, queue_index): 'page': page, 'page_range': page_range, 'job_status': 'Started', + 'executions': executions, } - return render(request, 'django_rq/jobs.html', context_data) + return render(request, 'django_rq/started_job_registry.html', context_data) @never_cache From 0981474d8b79c68f1a9da4cbab426c9d00599455 Mon Sep 17 00:00:00 2001 From: Selwin Ong Date: Thu, 3 Oct 2024 10:49:17 +0700 Subject: [PATCH 04/15] Minor job detail cleanup --- django_rq/management/commands/rqworker.py | 2 -- django_rq/templates/django_rq/job_detail.html | 17 +++++++++-------- .../django_rq/started_job_registry.html | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/django_rq/management/commands/rqworker.py b/django_rq/management/commands/rqworker.py index d968b3bb..c973af55 100644 --- a/django_rq/management/commands/rqworker.py +++ b/django_rq/management/commands/rqworker.py @@ -88,8 +88,6 @@ def handle(self, *args, **options): } w = get_worker(*args, **worker_kwargs) - # Call Connection context manager to push the redis connection into LocalStack - # without this, jobs using RQ's get_current_job() will fail # Close any opened DB connection before any fork reset_db_connections() diff --git a/django_rq/templates/django_rq/job_detail.html b/django_rq/templates/django_rq/job_detail.html index c920cad3..68ef0594 100644 --- a/django_rq/templates/django_rq/job_detail.html +++ b/django_rq/templates/django_rq/job_detail.html @@ -135,9 +135,10 @@
- -
-
+
+ +
+ {% if data_is_valid %} {% if job.args %}