diff --git a/.circleci/config.yml b/.circleci/config.yml index ed43953..74dae0f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,8 @@ references: run: name: Install Poetry command: | - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python + # Need to use version < 1.2.0 in order to support Python 3.6 + curl -sSL https://install.python-poetry.org | python3 - --version 1.1.15 restore-dependencies-cache: &restore-dependencies-cache restore_cache: keys: @@ -14,7 +15,6 @@ references: run: name: Install Dependencies command: | - source $HOME/.poetry/env poetry install poetry run pip install "django~=<< parameters.django-version >>.0" save-dependencies-cache: &save-dependencies-cache @@ -65,7 +65,6 @@ jobs: - run: name: Run Tests command: | - source $HOME/.poetry/env poetry run ./runtests lint: @@ -82,7 +81,6 @@ jobs: - run: name: Run Flake8 command: | - source $HOME/.poetry/env poetry run flake8 type-check: @@ -99,7 +97,6 @@ jobs: - run: name: Run Mypy command: | - source $HOME/.poetry/env poetry run ./script/type-check deploy: @@ -108,11 +105,10 @@ jobs: version: "3.7" steps: - checkout + - *install-poetry - run: name: Push to PyPI command: | - curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python - source $HOME/.poetry/env poetry publish \ --build \ --username "${PYPI_USERNAME}" \ diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..3be4f9e --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,21 @@ +# Development + +Dependencies are managed with [poetry](https://python-poetry.org/): `poetry install` + +Tests are run with `./runtests` + +## Releasing + +CI handles releasing to PyPI. +Releases on GitHub are created manually. + +Here's how to do a release: + + - Get all the desired changes into `master` + - Wait for CI to pass that + - Add a bump commit (see previous "Declare vX.Y.Z" commits; `poetry version` may be useful here) + - Push that commit on master + - Create a tag of that version number (`git tag v$(poetry version --short)`) + - Push the tag (`git push --tags`) + - CI will build & deploy that release + - Create a Release on GitHub, ideally with a summary of changes diff --git a/README.md b/README.md index abce973..b7d2acf 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,19 @@ backends are great candidates for community contributions. ## Basic Usage +Start by adding `django_lightweight_queue` to your `INSTALLED_APPS`: + +```python +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + ..., + "django_lightweight_queue", +] +``` + +After that, define your task in any file you want: + ```python import time from django_lightweight_queue import task @@ -56,7 +69,7 @@ LIGHTWEIGHT_QUEUE_REDIS_PORT = 12345 and then running: ``` -$ python manage.py queue_runner --config=special.py +$ python manage.py queue_runner --extra-settings=special.py ``` will result in the runner to use the settings from the specified configuration @@ -67,12 +80,57 @@ present in the specified file are inherited from the global configuration. There are four built-in backends: -| Backend | Type | Description | -| -------------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Synchronous | Development | Executes the task inline, without any actual queuing. | -| Redis | Production | Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. | -| Reliable Redis | Production | Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. | -| Debug Web | Debugging | Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. | +### Synchronous (Development backend) + +`django_lightweight_queue.backends.synchronous.SynchronousBackend` + +Executes the task inline, without any actual queuing. + +### Redis (Production backend) + +`django_lightweight_queue.backends.redis.RedisBackend` + +Executes tasks at-most-once using [Redis][redis] for storage of the enqueued tasks. + +### Reliable Redis (Production backend) + +`django_lightweight_queue.backends.reliable_redis.ReliableRedisBackend` + +Executes tasks at-least-once using [Redis][redis] for storage of the enqueued tasks (subject to Redis consistency). Does not guarantee the task _completes_. + +### Debug Web (Debug backend) + +`django_lightweight_queue.backends.debug_web.DebugWebBackend` + +Instead of running jobs it prints the url to a view that can be used to run a task in a transaction which will be rolled back. This is useful for debugging and optimising tasks. + +Use this to append the appropriate URLs to the bottom of your root `urls.py`: + +```python +from django.conf import settings +from django.urls import path, include + +urlpatterns = [ + ... +] + +if settings.DEBUG: + urlpatterns += [ + path( + "", + include( + "django_lightweight_queue.urls", namespace="django-lightweight-queue" + ), + ) + ] +``` + +This backend may require an extra setting if your debug site is not on localhost: + +```python +# defaults to http://localhost:8000 +LIGHTWEIGHT_QUEUE_SITE_URL = "http://example.com:8000" +``` [redis]: https://redis.io/ @@ -91,10 +149,13 @@ part of a pool: $ python manage.py queue_runner --machine 2 --of 4 ``` -Alternatively a runner can be told explicitly which configuration to use: +Alternatively a runner can be told explicitly how to behave by having +extra settings loaded (any `LIGHTWEIGHT_QUEUE_*` constants found in the file +will replace equivalent django settings) and being configured to run exactly as +the settings describe: ``` -$ python manage.py queue_runner --exact-configuration --config=special.py +$ python manage.py queue_runner --exact-configuration --extra-settings=special.py ``` When using `--exact-configuration` the number of workers is configured exactly, @@ -130,7 +191,7 @@ $ python manage.py queue_runner --machine 3 --of 3 will result in one worker for `queue1` on the current machine, while: ``` -$ python manage.py queue_runner --exact-configuration --config=special.py +$ python manage.py queue_runner --exact-configuration --extra-settings=special.py ``` will result in two workers on the current machine. diff --git a/django_lightweight_queue/cron_scheduler.py b/django_lightweight_queue/cron_scheduler.py index f1f8221..d974813 100644 --- a/django_lightweight_queue/cron_scheduler.py +++ b/django_lightweight_queue/cron_scheduler.py @@ -144,7 +144,7 @@ def get_matcher(minval, maxval, t): # No module, move on. continue - app_cron_config: List[CronConfig] = mod.CONFIG # type: ignore[attr-defined] + app_cron_config: List[CronConfig] = mod.CONFIG for row in app_cron_config: row['min_matcher'] = get_matcher(0, 59, row.get('minutes')) row['hour_matcher'] = get_matcher(0, 23, row.get('hours')) diff --git a/django_lightweight_queue/management/commands/queue_configuration.py b/django_lightweight_queue/management/commands/queue_configuration.py index 787ce98..a57eae9 100644 --- a/django_lightweight_queue/management/commands/queue_configuration.py +++ b/django_lightweight_queue/management/commands/queue_configuration.py @@ -1,26 +1,46 @@ +import warnings from typing import Any from django.core.management.base import BaseCommand, CommandParser -from ...utils import get_backend, get_queue_counts, load_extra_config +from ...utils import get_backend, get_queue_counts, load_extra_settings +from ...constants import SETTING_NAME_PREFIX from ...app_settings import app_settings from ...cron_scheduler import get_cron_config class Command(BaseCommand): def add_arguments(self, parser: CommandParser) -> None: - parser.add_argument( + extra_settings_group = parser.add_mutually_exclusive_group() + extra_settings_group.add_argument( '--config', action='store', default=None, - help="The path to an additional django-style config file to load", + help="The path to an additional django-style config file to load " + "(this spelling is deprecated in favour of '--extra-settings')", + ) + extra_settings_group.add_argument( + '--extra-settings', + action='store', + default=None, + help="The path to an additional django-style settings file to load. " + f"{SETTING_NAME_PREFIX}* settings discovered in this file will " + "override those from the default Django settings.", ) def handle(self, **options: Any) -> None: - # Configuration overrides - extra_config = options['config'] + extra_config = options.pop('config') if extra_config is not None: - load_extra_config(extra_config) + warnings.warn( + "Use of '--config' is deprecated in favour of '--extra-settings'.", + category=DeprecationWarning, + ) + options['extra_settings'] = extra_config + + # Configuration overrides + extra_settings = options['extra_settings'] + if extra_settings is not None: + load_extra_settings(extra_settings) print("django-lightweight-queue") print("========================") diff --git a/django_lightweight_queue/management/commands/queue_runner.py b/django_lightweight_queue/management/commands/queue_runner.py index c4a5706..6e4fa06 100644 --- a/django_lightweight_queue/management/commands/queue_runner.py +++ b/django_lightweight_queue/management/commands/queue_runner.py @@ -1,3 +1,4 @@ +import warnings from typing import Any, Dict, Optional import daemonize @@ -10,8 +11,14 @@ ) from ...types import QueueName -from ...utils import get_logger, get_backend, get_middleware, load_extra_config +from ...utils import ( + get_logger, + get_backend, + get_middleware, + load_extra_settings, +) from ...runner import runner +from ...constants import SETTING_NAME_PREFIX from ...machine_types import Machine, PooledMachine, DirectlyConfiguredMachine @@ -51,23 +58,42 @@ def add_arguments(self, parser: CommandParser) -> None: default=None, help="Only run the given queue, useful for local debugging", ) - parser.add_argument( + extra_settings_group = parser.add_mutually_exclusive_group() + extra_settings_group.add_argument( '--config', action='store', default=None, - help="The path to an additional django-style config file to load", + help="The path to an additional django-style config file to load " + "(this spelling is deprecated in favour of '--extra-settings')", + ) + extra_settings_group.add_argument( + '--extra-settings', + action='store', + default=None, + help="The path to an additional django-style settings file to load. " + f"{SETTING_NAME_PREFIX}* settings discovered in this file will " + "override those from the default Django settings.", ) parser.add_argument( '--exact-configuration', action='store_true', help="Run queues on this machine exactly as specified. Requires the" - " use of the '--config' option in addition. It is an error to" - " use this option together with either '--machine' or '--of'.", + " use of the '--extra-settings' option in addition. It is an" + " error to use this option together with either '--machine' or" + " '--of'.", ) def validate_and_normalise(self, options: Dict[str, Any]) -> None: + extra_config = options.pop('config') + if extra_config is not None: + warnings.warn( + "Use of '--config' is deprecated in favour of '--extra-settings'.", + category=DeprecationWarning, + ) + options['extra_settings'] = extra_config + if options['exact_configuration']: - if not options['config']: + if not options['extra_settings']: raise CommandError( "Must provide a value for '--config' when using " "'--exact-configuration'.", @@ -110,9 +136,9 @@ def touch_filename(name: str) -> Optional[str]: return None # Configuration overrides - extra_config = options['config'] + extra_config = options['extra_settings'] if extra_config is not None: - load_extra_config(extra_config) + load_extra_settings(extra_config) logger.info("Starting queue master") diff --git a/django_lightweight_queue/runner.py b/django_lightweight_queue/runner.py index 11855b8..8cd6556 100644 --- a/django_lightweight_queue/runner.py +++ b/django_lightweight_queue/runner.py @@ -43,7 +43,7 @@ def runner( # Note: we deliberately configure our handling of SIGTERM _after_ the # startup processes have happened; this ensures that the startup processes - # (which could take a long time) are naturally interupted by the signal. + # (which could take a long time) are naturally interrupted by the signal. def handle_term(signum: int, stack: object) -> None: nonlocal running logger.debug("Caught TERM signal") diff --git a/django_lightweight_queue/urls.py b/django_lightweight_queue/urls.py index f6facb1..f138b39 100644 --- a/django_lightweight_queue/urls.py +++ b/django_lightweight_queue/urls.py @@ -1,9 +1,9 @@ -from django.conf.urls import url +from django.urls import path from . import views app_name = 'django_lightweight_queue' urlpatterns = ( - url(r'^debug/django-lightweight-queue/debug-run$', views.debug_run, name='debug-run'), + path(r'debug/django-lightweight-queue/debug-run', views.debug_run, name='debug-run'), ) diff --git a/django_lightweight_queue/utils.py b/django_lightweight_queue/utils.py index 1edb6a4..c9731be 100644 --- a/django_lightweight_queue/utils.py +++ b/django_lightweight_queue/utils.py @@ -33,7 +33,7 @@ FIVE_SECONDS = datetime.timedelta(seconds=5) -def load_extra_config(file_path: str) -> None: +def load_extra_settings(file_path: str) -> None: # Based on https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly spec = importlib.util.spec_from_file_location('extra_settings', file_path) extra_settings = importlib.util.module_from_spec(spec) # type: ignore[arg-type] diff --git a/django_lightweight_queue/worker.py b/django_lightweight_queue/worker.py index 565c31e..9465c96 100644 --- a/django_lightweight_queue/worker.py +++ b/django_lightweight_queue/worker.py @@ -159,6 +159,7 @@ def configure_cancellation(self, timeout: Optional[int], sigkill_on_stop: bool) signal.signal(signal.SIGUSR2, self._handle_sigusr2) if timeout is not None: + signal.signal(signal.SIGALRM, self._handle_alarm) # alarm(3) takes whole seconds alarm_duration = int(math.ceil(timeout)) signal.alarm(alarm_duration) @@ -166,6 +167,17 @@ def configure_cancellation(self, timeout: Optional[int], sigkill_on_stop: bool) # Cancel any scheduled alarms signal.alarm(0) + def _handle_alarm(self, signal_number: int, frame: object) -> None: + # Log for observability + self.log(logging.ERROR, "Alarm received: job has timed out") + + # Disconnect ourselves then re-signal so that Python does what it + # normally would. We could raise an exception here, however raising + # exceptions from signal handlers is generally discouraged. + signal.signal(signal.SIGALRM, signal.SIG_DFL) + # TODO(python-upgrade): use signal.raise_signal on Python 3.8+ + os.kill(os.getpid(), signal.SIGALRM) + def set_process_title(self, *titles: str) -> None: set_process_title(self.name, *titles) diff --git a/poetry.lock b/poetry.lock index cfb26d3..6ca6c62 100644 --- a/poetry.lock +++ b/poetry.lock @@ -58,7 +58,7 @@ dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "im [[package]] name = "django" -version = "3.2.13" +version = "3.2.15" description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." category = "main" optional = false @@ -70,8 +70,8 @@ pytz = "*" sqlparse = ">=0.2.2" [package.extras] -argon2 = ["argon2-cffi (>=19.1.0)"] bcrypt = ["bcrypt"] +argon2 = ["argon2-cffi (>=19.1.0)"] [[package]] name = "fakeredis" @@ -285,21 +285,22 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.910" +version = "0.940" description = "Optional static typing for Python" category = "dev" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" [package.dependencies] -mypy-extensions = ">=0.4.3,<0.5.0" -toml = "*" -typed-ast = {version = ">=1.4.0,<1.5.0", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.7.4" +mypy-extensions = ">=0.4.3" +tomli = ">=1.1.0" +typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} +typing-extensions = ">=3.10" [package.extras] +reports = ["lxml"] +python2 = ["typed-ast (>=1.4.0,<2)"] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<1.5.0)"] [[package]] name = "mypy-extensions" @@ -432,12 +433,12 @@ docs = ["sphinx", "zope.component", "sybil", "twisted", "mock", "django (<2)", " test = ["pytest (>=3.6)", "pytest-cov", "pytest-django", "zope.component", "sybil", "twisted", "mock", "django (<2)", "django"] [[package]] -name = "toml" -version = "0.10.2" -description = "Python Library for Tom's Obvious, Minimal Language" +name = "tomli" +version = "1.2.3" +description = "A lil' TOML parser" category = "dev" optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +python-versions = ">=3.6" [[package]] name = "tqdm" @@ -514,7 +515,7 @@ redis = ["redis"] [metadata] lock-version = "1.1" python-versions = ">=3.6.1,<4" -content-hash = "df908ca4d0c2f3bd0c8c405c9cbe759316de0f603e4009210b403a4c0268831d" +content-hash = "1c476679a210884146b4b5c417c6e15ea46f1d5105c9cb97666e3cfa2aa55dd2" [metadata.files] asgiref = [ @@ -538,8 +539,8 @@ deprecated = [ {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, ] django = [ - {file = "Django-3.2.13-py3-none-any.whl", hash = "sha256:b896ca61edc079eb6bbaa15cf6071eb69d6aac08cce5211583cfb41515644fdf"}, - {file = "Django-3.2.13.tar.gz", hash = "sha256:6d93497a0a9bf6ba0e0b1a29cccdc40efbfc76297255b1309b3a884a688ec4b6"}, + {file = "Django-3.2.15-py3-none-any.whl", hash = "sha256:115baf5049d5cf4163e43492cdc7139c306ed6d451e7d3571fe9612903903713"}, + {file = "Django-3.2.15.tar.gz", hash = "sha256:f71934b1a822f14a86c9ac9634053689279cd04ae69cb6ade4a59471b886582b"}, ] fakeredis = [ {file = "fakeredis-1.7.1-py3-none-any.whl", hash = "sha256:be3668e50f6b57d5fc4abfd27f9f655bed07a2c5aecfc8b15d0aad59f997c1ba"}, @@ -606,29 +607,29 @@ mccabe = [ {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, ] mypy = [ - {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, - {file = "mypy-0.910-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb"}, - {file = "mypy-0.910-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9"}, - {file = "mypy-0.910-cp35-cp35m-win_amd64.whl", hash = "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e"}, - {file = "mypy-0.910-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921"}, - {file = "mypy-0.910-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6"}, - {file = "mypy-0.910-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212"}, - {file = "mypy-0.910-cp36-cp36m-win_amd64.whl", hash = "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885"}, - {file = "mypy-0.910-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0"}, - {file = "mypy-0.910-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de"}, - {file = "mypy-0.910-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703"}, - {file = "mypy-0.910-cp37-cp37m-win_amd64.whl", hash = "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a"}, - {file = "mypy-0.910-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504"}, - {file = "mypy-0.910-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9"}, - {file = "mypy-0.910-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072"}, - {file = "mypy-0.910-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811"}, - {file = "mypy-0.910-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e"}, - {file = "mypy-0.910-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b"}, - {file = "mypy-0.910-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2"}, - {file = "mypy-0.910-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97"}, - {file = "mypy-0.910-cp39-cp39-win_amd64.whl", hash = "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8"}, - {file = "mypy-0.910-py3-none-any.whl", hash = "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"}, - {file = "mypy-0.910.tar.gz", hash = "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150"}, + {file = "mypy-0.940-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0fdc9191a49c77ab5fa0439915d405e80a1118b163ab03cd2a530f346b12566a"}, + {file = "mypy-0.940-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1903c92ff8642d521b4627e51a67e49f5be5aedb1fb03465b3aae4c3338ec491"}, + {file = "mypy-0.940-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:471af97c35a32061883b0f8a3305ac17947fd42ce962ca9e2b0639eb9141492f"}, + {file = "mypy-0.940-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:13677cb8b050f03b5bb2e8bf7b2668cd918b001d56c2435082bbfc9d5f730f42"}, + {file = "mypy-0.940-cp310-cp310-win_amd64.whl", hash = "sha256:2efd76893fb8327eca7e942e21b373e6f3c5c083ff860fb1e82ddd0462d662bd"}, + {file = "mypy-0.940-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8fe1bfab792e4300f80013edaf9949b34e4c056a7b2531b5ef3a0fb9d598ae2"}, + {file = "mypy-0.940-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2dba92f58610d116f68ec1221fb2de2a346d081d17b24a784624389b17a4b3f9"}, + {file = "mypy-0.940-cp36-cp36m-win_amd64.whl", hash = "sha256:712affcc456de637e774448c73e21c84dfa5a70bcda34e9b0be4fb898a9e8e07"}, + {file = "mypy-0.940-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8aaf18d0f8bc3ffba56d32a85971dfbd371a5be5036da41ac16aefec440eff17"}, + {file = "mypy-0.940-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:51be997c1922e2b7be514a5215d1e1799a40832c0a0dee325ba8794f2c48818f"}, + {file = "mypy-0.940-cp37-cp37m-win_amd64.whl", hash = "sha256:628f5513268ebbc563750af672ccba5eef7f92d2d90154233edd498dfb98ca4e"}, + {file = "mypy-0.940-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:68038d514ae59d5b2f326be502a359160158d886bd153fc2489dbf7a03c44c96"}, + {file = "mypy-0.940-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b2fa5f2d597478ccfe1f274f8da2f50ea1e63da5a7ae2342c5b3b2f3e57ec340"}, + {file = "mypy-0.940-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b1a116c451b41e35afc09618f454b5c2704ba7a4e36f9ff65014fef26bb6075b"}, + {file = "mypy-0.940-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:1f66f2309cdbb07e95e60e83fb4a8272095bd4ea6ee58bf9a70d5fb304ec3e3f"}, + {file = "mypy-0.940-cp38-cp38-win_amd64.whl", hash = "sha256:3ac14949677ae9cb1adc498c423b194ad4d25b13322f6fe889fb72b664c79121"}, + {file = "mypy-0.940-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:6eab2bcc2b9489b7df87d7c20743b66d13254ad4d6430e1dfe1a655d51f0933d"}, + {file = "mypy-0.940-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0b52778a018559a256c819ee31b2e21e10b31ddca8705624317253d6d08dbc35"}, + {file = "mypy-0.940-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d9d7647505bf427bc7931e8baf6cacf9be97e78a397724511f20ddec2a850752"}, + {file = "mypy-0.940-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a0e5657ccaedeb5fdfda59918cc98fc6d8a8e83041bc0cec347a2ab6915f9998"}, + {file = "mypy-0.940-cp39-cp39-win_amd64.whl", hash = "sha256:83f66190e3c32603217105913fbfe0a3ef154ab6bbc7ef2c989f5b2957b55840"}, + {file = "mypy-0.940-py3-none-any.whl", hash = "sha256:a168da06eccf51875fdff5f305a47f021f23f300e2b89768abdac24538b1f8ec"}, + {file = "mypy-0.940.tar.gz", hash = "sha256:71bec3d2782d0b1fecef7b1c436253544d81c1c0e9ca58190aed9befd8f081c5"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, @@ -682,9 +683,9 @@ testfixtures = [ {file = "testfixtures-6.18.3-py2.py3-none-any.whl", hash = "sha256:6ddb7f56a123e1a9339f130a200359092bd0a6455e31838d6c477e8729bb7763"}, {file = "testfixtures-6.18.3.tar.gz", hash = "sha256:2600100ae96ffd082334b378e355550fef8b4a529a6fa4c34f47130905c7426d"}, ] -toml = [ - {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, - {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +tomli = [ + {file = "tomli-1.2.3-py3-none-any.whl", hash = "sha256:e3069e4be3ead9668e21cb9b074cd948f7b3113fd9c8bba083f48247aab8b11c"}, + {file = "tomli-1.2.3.tar.gz", hash = "sha256:05b6166bff487dc068d322585c7ea4ef78deed501cc124060e0f238e89a9231f"}, ] tqdm = [ {file = "tqdm-4.62.3-py2.py3-none-any.whl", hash = "sha256:8dd278a422499cd6b727e6ae4061c40b48fce8b76d1ccbf5d34fca9b7f925b0c"}, diff --git a/pyproject.toml b/pyproject.toml index 0c3996d..408dead 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "django-lightweight-queue" -version = "4.5.1" +version = "4.9.0" description = "Lightweight & modular queue and cron system for Django" authors = ["Thread Engineering "] license = "BSD-3-Clause" @@ -34,7 +34,7 @@ freezegun = "^1.1.0" # Linting tools flake8 = "^4.0.0" isort = "^5.10" -mypy = "^0.910" +mypy = "^0.940" # Flake 8 plugins flake8-bugbear = "^21.11.29" diff --git a/setup.cfg b/setup.cfg index 21daa1b..8d473d1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,3 +76,5 @@ strict_equality = True scripts_are_modules = True warn_unused_configs = True + +enable_error_code = ignore-without-code diff --git a/tests/_demo_extra_config.py b/tests/_demo_extra_settings.py similarity index 68% rename from tests/_demo_extra_config.py rename to tests/_demo_extra_settings.py index 67779ba..eb0277c 100644 --- a/tests/_demo_extra_config.py +++ b/tests/_demo_extra_settings.py @@ -1,7 +1,7 @@ from django_lightweight_queue.types import QueueName LIGHTWEIGHT_QUEUE_BACKEND_OVERRIDES = { - QueueName('test-queue'): 'tests.test_extra_config.TestBackend', + QueueName('test-queue'): 'tests.test_extra_settings.TestBackend', } LIGHTWEIGHT_QUEUE_REDIS_PASSWORD = 'a very bad password' diff --git a/tests/_demo_extra_config_falsey.py b/tests/_demo_extra_settings_falsey.py similarity index 100% rename from tests/_demo_extra_config_falsey.py rename to tests/_demo_extra_settings_falsey.py diff --git a/tests/_demo_extra_config_unexpected.py b/tests/_demo_extra_settings_unexpected.py similarity index 100% rename from tests/_demo_extra_config_unexpected.py rename to tests/_demo_extra_settings_unexpected.py diff --git a/tests/settings.py b/tests/settings.py index aaed1d9..9bedbcd 100644 --- a/tests/settings.py +++ b/tests/settings.py @@ -5,3 +5,7 @@ INSTALLED_APPS = [ 'django_lightweight_queue', ] + +ROOT_URLCONF = 'tests.urls' + +SITE_URL = 'http://localhost:8000' diff --git a/tests/test_debug_web_backend.py b/tests/test_debug_web_backend.py new file mode 100644 index 0000000..e10d450 --- /dev/null +++ b/tests/test_debug_web_backend.py @@ -0,0 +1,45 @@ +import io +import unittest +import contextlib +import urllib.parse +from unittest import mock + +from django.http import QueryDict +from django.urls import resolve + +from django_lightweight_queue import task +from django_lightweight_queue.job import Job +from django_lightweight_queue.types import QueueName +from django_lightweight_queue.backends.debug_web import DebugWebBackend + + +@task() +def demo_task() -> None: + pass + + +class DebugWebBackendTests(unittest.TestCase): + def test_enqueue_prints_valid_url(self) -> None: + backend = DebugWebBackend() + + job = Job('tests.test_debug_web_backend.demo_task', ('positional',), {'keyword': '&arg='}) + + with mock.patch('tests.test_debug_web_backend.demo_task') as demo_task_mock: + with contextlib.redirect_stdout(io.StringIO()) as mock_stdout: + backend.enqueue(job, QueueName('test-queue')) + + url = mock_stdout.getvalue().strip() + parse_result = urllib.parse.urlparse(url) + + match = resolve(parse_result.path) + self.assertIsNotNone(match, f"Failed to match {parse_result.path}") + + query = QueryDict(parse_result.query) + + self.assertEqual( + {'job': [job.to_json()]}, + dict(query), + "Wrong query arguments printed", + ) + + demo_task_mock.assert_not_called() diff --git a/tests/test_extra_config.py b/tests/test_extra_settings.py similarity index 77% rename from tests/test_extra_config.py rename to tests/test_extra_settings.py index fc739d4..98f8edb 100644 --- a/tests/test_extra_config.py +++ b/tests/test_extra_settings.py @@ -7,7 +7,7 @@ from django_lightweight_queue import app_settings from django_lightweight_queue.job import Job from django_lightweight_queue.types import QueueName, WorkerNumber -from django_lightweight_queue.utils import get_backend, load_extra_config +from django_lightweight_queue.utils import get_backend, load_extra_settings from django_lightweight_queue.backends.base import BaseBackend TESTS_DIR = Path(__file__).parent @@ -24,7 +24,7 @@ def length(self, queue: QueueName) -> int: pass -class ExtraConfigTests(SimpleTestCase): +class ExtraSettingsTests(SimpleTestCase): def setUp(self) -> None: get_backend.cache_clear() @@ -42,8 +42,8 @@ def tearDown(self) -> None: self._settings_patch.stop() super().tearDown() - def test_updates_configuration(self) -> None: - load_extra_config(str(TESTS_DIR / '_demo_extra_config.py')) + def test_updates_settings(self) -> None: + load_extra_settings(str(TESTS_DIR / '_demo_extra_settings.py')) backend = get_backend('test-queue') self.assertIsInstance(backend, TestBackend) @@ -52,17 +52,17 @@ def test_updates_configuration(self) -> None: def test_warns_about_unexpected_settings(self) -> None: with self.assertWarnsRegex(Warning, r'Ignoring unexpected setting.+\bNOT_REDIS_PASSWORD\b'): - load_extra_config(str(TESTS_DIR / '_demo_extra_config_unexpected.py')) + load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_unexpected.py')) self.assertEqual('expected', self.settings.REDIS_PASSWORD) - def test_updates_configuration_with_falsey_values(self) -> None: - load_extra_config(str(TESTS_DIR / '_demo_extra_config.py')) - load_extra_config(str(TESTS_DIR / '_demo_extra_config_falsey.py')) + def test_updates_settings_with_falsey_values(self) -> None: + load_extra_settings(str(TESTS_DIR / '_demo_extra_settings.py')) + load_extra_settings(str(TESTS_DIR / '_demo_extra_settings_falsey.py')) self.assertIsNone(self.settings.REDIS_PASSWORD) self.assertFalse(self.settings.ATOMIC_JOBS) def test_rejects_missing_file(self) -> None: with self.assertRaises(FileNotFoundError): - load_extra_config(str(TESTS_DIR / '_no_such_file.py')) + load_extra_settings(str(TESTS_DIR / '_no_such_file.py')) diff --git a/tests/test_task.py b/tests/test_task.py index 742f949..d96f55d 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -45,7 +45,7 @@ def setUp(self) -> None: with mock.patch('redis.StrictRedis', fakeredis.FakeStrictRedis): self.backend = RedisBackend() - # Mock get_backend. Unfortunately due to the nameing of the 'task' + # Mock get_backend. Unfortunately due to the naming of the 'task' # decorator class being the same as its containing module and it being # exposed as the symbol at django_lightweight_queue.task, we cannot mock # this in the normal way. Instead we mock get_path (which get_backend @@ -55,12 +55,12 @@ def mocked_get_path(path: str) -> Any: return lambda: self.backend return get_path(path) - patch = mock.patch( + get_path_patch = mock.patch( 'django_lightweight_queue.utils.get_path', side_effect=mocked_get_path, ) - patch.start() - self.addCleanup(patch.stop) + get_path_patch.start() + self.addCleanup(get_path_patch.stop) def tearDown(self) -> None: super().tearDown() diff --git a/tests/urls.py b/tests/urls.py new file mode 100644 index 0000000..4111a45 --- /dev/null +++ b/tests/urls.py @@ -0,0 +1,5 @@ +from django.urls import path, include + +urlpatterns = [ + path('', include('django_lightweight_queue.urls', namespace='django-lightweight-queue')), +]