Skip to content

Commit

Permalink
Roll back multi-queue support for Django.
Browse files Browse the repository at this point in the history 
Reverts #230
  • Loading branch information
@coleifer
coleifer committed Mar 12, 2018
1 parent f85efe0 commit b652b39
Show file tree
Hide file tree
Showing 10 changed files with 144 additions and 779 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ python:
- "3.6"
services:
- redis
install: "pip install -r test_requirements.txt"
install: "pip install redis"
script: "python runtests.py"
notifications:
email: true
126 changes: 34 additions & 92 deletions docs/django.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,40 +45,38 @@ options with their default values:
# settings.py
HUEY = {
'my-app': { # name of the huey queue (just use your app name)
'result_store': True, # Store return values of tasks.
'events': True, # Consumer emits events allowing real-time monitoring.
'store_none': False, # If a task returns None, do not save to results.
'always_eager': settings.DEBUG, # If DEBUG=True, run synchronously.
'store_errors': True, # Store error info if task throws exception.
'blocking': False, # Poll the queue rather than do blocking pop.
'backend_class': 'huey.RedisHuey', # Use path to redis huey by default,
'default': False, # Indicates the default app. Only useful if you have several huey instances configured.
'connection': {
'host': 'localhost',
'port': 6379,
'db': 0,
'connection_pool': None, # Definitely you should use pooling!
# ... tons of other options, see redis-py for details.
# huey-specific connection parameters.
'read_timeout': 1, # If not polling (blocking pop), use timeout.
'max_errors': 1000, # Only store the 1000 most recent errors.
'url': None, # Allow Redis config via a DSN.
},
'consumer': {
'workers': 1,
'worker_type': 'thread',
'initial_delay': 0.1, # Smallest polling interval, same as -d.
'backoff': 1.15, # Exponential backoff using this rate, -b.
'max_delay': 10.0, # Max possible polling interval, -m.
'utc': True, # Treat ETAs and schedules as UTC datetimes.
'scheduler_interval': 1, # Check schedule every second, -s.
'periodic': True, # Enable crontab feature.
'check_worker_health': True, # Enable worker health checks.
'health_check_interval': 1, # Check worker health every second.
},
}
'name': settings.DATABASES['default']['NAME'], # Use db name for huey.
'result_store': True, # Store return values of tasks.
'events': True, # Consumer emits events allowing real-time monitoring.
'store_none': False, # If a task returns None, do not save to results.
'always_eager': settings.DEBUG, # If DEBUG=True, run synchronously.
'store_errors': True, # Store error info if task throws exception.
'blocking': False, # Poll the queue rather than do blocking pop.
'backend_class': 'huey.RedisHuey', # Use path to redis huey by default,
'connection': {
'host': 'localhost',
'port': 6379,
'db': 0,
'connection_pool': None, # Definitely you should use pooling!
# ... tons of other options, see redis-py for details.
# huey-specific connection parameters.
'read_timeout': 1, # If not polling (blocking pop), use timeout.
'max_errors': 1000, # Only store the 1000 most recent errors.
'url': None, # Allow Redis config via a DSN.
},
'consumer': {
'workers': 1,
'worker_type': 'thread',
'initial_delay': 0.1, # Smallest polling interval, same as -d.
'backoff': 1.15, # Exponential backoff using this rate, -b.
'max_delay': 10.0, # Max possible polling interval, -m.
'utc': True, # Treat ETAs and schedules as UTC datetimes.
'scheduler_interval': 1, # Check schedule every second, -s.
'periodic': True, # Enable crontab feature.
'check_worker_health': True, # Enable worker health checks.
'health_check_interval': 1, # Check worker health every second.
},
}
Alternatively, you can simply set ``settings.HUEY`` to a :py:class:`Huey`
Expand Down Expand Up @@ -132,11 +130,6 @@ listed here.
specify tens or hundreds of workers since they are extremely lightweight
compared to threads/processes. *See note below on using gevent/greenlet*.

``-qu``, ``--queue``
Indicate the huey queue you want to listen on. For example "-qu my-app".
You only need this option if you configured several huey instances in
your settings.

.. note::
Due to a conflict with Django's base option list, the "verbose" option is
set using ``-V`` or ``--huey-verbose``. When enabled, huey logs at the
Expand Down Expand Up @@ -245,9 +238,8 @@ This section contains example ``HUEY`` configurations.
# Redis running locally with four worker threads.
HUEY = {
'my-app': {
'consumer': {'workers': 4, 'worker_type': 'thread'},
}
'name': 'my-app',
'consumer': {'workers': 4, 'worker_type': 'thread'},
}
Expand Down Expand Up @@ -285,53 +277,3 @@ Alternatively, you can just assign a :py:class:`Huey` instance to the ``HUEY`` s
from huey import RedisHuey
HUEY = RedisHuey('my-app')
Several queues
^^^^^^^^^^^^^^^^^^^^^^
You can even use huey to distribute your tasks to several queues which are processed independently.

.. code-block:: python
# settings.py
HUEY = {
'first_queue': {
'default': True,
'consumer': {
'worker_type': 'thread'
'workers': 2,
},
},
'second_queue': {
'consumer': {
'worker_type': 'thread'
'workers': 2,
},
}
}
# tasks.py
@task(queue='first-queue')
def count_beans(number):
print('-- counted %s beans --' % number)
return 'Counted %s beans' % number
@periodic_task(crontab(minute='*/5')) # If no queue is given, the default queue is used.
def every_five_mins():
print('Every five minutes this will be printed by the consumer')
@task(queue='second-queue', retries=3, retry_delay=10)
def try_thrice():
if random.randint(1, 3) == 1:
print('OK')
else:
print('About to fail, will retry in 10 seconds')
raise Exception('Crap something went wrong')
As soon as you have configured your queues, you can start workers for each queue:

.. code-block:: console
// Create worker for each queue.
$ ./manage.py run_huey --queue first_queue
$ ./manage.py run_huey --queue second_queue
96 changes: 86 additions & 10 deletions huey/contrib/djhuey/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,98 @@
from django.conf import settings
from django.db import close_old_connections

from huey.contrib.djhuey.configuration import HueySettingsReader

configuration_message = """
Configuring Huey for use with Django
====================================
_huey_settings = getattr(settings, 'HUEY', None)
Huey was designed to be simple to configure in the general case. For that
reason, huey will "just work" with no configuration at all provided you have
Redis installed and running locally.
_django_huey = HueySettingsReader(_huey_settings)
_django_huey.start()
On the other hand, you can configure huey manually using the following
setting structure.
HUEY = _django_huey.huey
hueys = _django_huey.hueys
The following example uses Redis on localhost, and will run four worker
processes:
consumer = _django_huey.consumer
consumers = _django_huey.consumers
HUEY = {
'name': 'my-app',
'connection': {'host': 'localhost', 'port': 6379},
'consumer': {
'workers': 4,
'worker_type': 'process', # "thread" or "greenlet" are other options
},
}
task = _django_huey.task
periodic_task = _django_huey.periodic_task
If you would like to configure Huey's logger using Django's integrated logging
settings, the logger used by consumer is named "huey.consumer".
Alternatively you can simply assign `settings.HUEY` to an actual `Huey`
object instance:
from huey import RedisHuey
HUEY = RedisHuey('my-app')
"""


default_backend_path = 'huey.RedisHuey'

def default_queue_name():
try:
return settings.DATABASE_NAME
except AttributeError:
try:
return settings.DATABASES['default']['NAME']
except KeyError:
return 'huey'


def get_backend(import_path=default_backend_path):
module_path, class_name = import_path.rsplit('.', 1)
module = import_module(module_path)
return getattr(module, class_name)


def config_error(msg):
print(configuration_message)
print('\n\n')
print(msg)
sys.exit(1)


HUEY = getattr(settings, 'HUEY', None)
if HUEY is None:
try:
RedisHuey = get_backend(default_backend_path)
except ImportError:
config_error('Error: Huey could not import the redis backend. '
'Install `redis-py`.')
else:
HUEY = RedisHuey(default_queue_name())

if isinstance(HUEY, dict):
huey_config = HUEY.copy() # Operate on a copy.
name = huey_config.pop('name', default_queue_name())
backend_path = huey_config.pop('backend_class', default_backend_path)
conn_kwargs = huey_config.pop('connection', {})
try:
del huey_config['consumer'] # Don't need consumer opts here.
except KeyError:
pass
if 'always_eager' not in huey_config:
huey_config['always_eager'] = settings.DEBUG
huey_config.update(conn_kwargs)

try:
backend_cls = get_backend(backend_path)
except (ValueError, ImportError, AttributeError):
config_error('Error: could not import Huey backend:\n%s' % traceback.format_exc())

HUEY = backend_cls(name, **huey_config)

task = HUEY.task
periodic_task = HUEY.periodic_task


def close_db(fn):
Expand Down
Loading

0 comments on commit b652b39

Please sign in to comment.