From c9b369a4283d3a18dabcf853e4a12ae670ba1832 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Tue, 11 Sep 2018 10:03:11 -0400 Subject: [PATCH 1/3] add test for `es_update --start` and fix regression in `--start flag` #39 es_update start flag broken --- .../management/commands/es_update.py | 2 +- django_elastic_migrations/models.py | 9 ++++++- tests/test_management_commands.py | 25 +++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/django_elastic_migrations/management/commands/es_update.py b/django_elastic_migrations/management/commands/es_update.py index 69b6985..56d7ffa 100644 --- a/django_elastic_migrations/management/commands/es_update.py +++ b/django_elastic_migrations/management/commands/es_update.py @@ -47,7 +47,7 @@ def add_arguments(self, parser): "--start", dest="start_date", help="The start date for indexing. Can be any dateutil-parsable string;" - " YYYY-MM-DDTHH:MM:SS is recommended to avoid confusion", + "YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS is recommended to avoid confusion", ) def handle(self, *args, **options): diff --git a/django_elastic_migrations/models.py b/django_elastic_migrations/models.py index 2ec61b2..6b6f9d2 100644 --- a/django_elastic_migrations/models.py +++ b/django_elastic_migrations/models.py @@ -965,7 +965,14 @@ def __init__(self, *args, **kwargs): super(UpdateIndexAction, self).__init__(*args, **kwargs) if self.task_kwargs == '{}' and self.self_kwargs: # retain a history of how this command was called - self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) + try: + self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) + except TypeError as e: + if 'Object of type datetime is not JSON serializable' in str(e): + for key, val in self.self_kwargs.items(): + if isinstance(val, datetime.date): + self.self_kwargs[key] = str(val) + self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) self._batch_num = 0 self._expected_remaining = 0 diff --git a/tests/test_management_commands.py b/tests/test_management_commands.py index 97d7430..7f3753b 100644 --- a/tests/test_management_commands.py +++ b/tests/test_management_commands.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) import logging +from datetime import datetime, timedelta from unittest import skip from django.contrib.humanize.templatetags.humanize import ordinal @@ -17,6 +18,10 @@ log = logging.getLogger(__file__) +def days_ago(d): + return datetime.now() - timedelta(days=d) + + # noinspection PyUnresolvedReferences class CommonDEMTestUtilsMixin(object): @@ -227,6 +232,26 @@ def test_newer_flag(self): actual_num=actual_num_docs )) + def test_start_flag(self): + self.check_basic_setup_and_get_models() + + num_docs = MovieSearchIndex.get_num_docs() + self.assertEqual(num_docs, 0) + + Movie.objects.all().update(last_modified=days_ago(3)) + call_command('es_update', 'movies', '--start={}'.format(days_ago(2).isoformat())) + num_docs = MovieSearchIndex.get_num_docs() + # we had last modified set to three days ago, and we only updated last two days, + # so we should not have any. + self.assertEqual(num_docs, 0) + + Movie.objects.all().update(last_modified=days_ago(1)) + call_command('es_update', 'movies', '--start={}'.format(days_ago(2).isoformat())) + num_docs = MovieSearchIndex.get_num_docs() + # we had last modified set to one days ago, and we only updated last two days, + # so we should have two + self.assertEqual(num_docs, 2) + @skip("Skipped multiprocessing tests until SQLLite can be integrated into test setup") # @tag('multiprocessing') From 958ea7f2c42836d77e3cb26a185a6d3647e06dbc Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Tue, 11 Sep 2018 10:09:05 -0400 Subject: [PATCH 2/3] bump 0.7.5 --> 0.7.6 #39 es_update start flag broken --- CHANGELOG.rst | 10 ++++++---- django_elastic_migrations/__init__.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1c7b3a9..8616494 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,15 +1,17 @@ Changelog --------- +0.7.6 (2018-09-11) +~~~~~~~~~~~~~~~~~~ +* fix `#36 es_update --start flag broken `_ + 0.7.5 (2018-08-20) ~~~~~~~~~~~~~~~~~~ -* fix `#35 open multiprocessing log in context handler -`_ +* fix `#35 open multiprocessing log in context handler `_ 0.7.4 (2018-08-15) ~~~~~~~~~~~~~~~~~~ -* fix `#33 error when nothing to resume using --resume -`_ +* fix `#33 error when nothing to resume using --resume `_ 0.7.3 (2018-08-14) ~~~~~~~~~~~~~~~~~~ diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index c3717d1..f1aa71a 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -10,7 +10,7 @@ from django_elastic_migrations.utils import loading from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -__version__ = '0.7.5' +__version__ = '0.7.6' default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig' # pylint: disable=invalid-name From bed2a153197ce16972a6ec1db5519e53e63be947 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Tue, 11 Sep 2018 10:36:46 -0400 Subject: [PATCH 3/3] simplify coercion from `datetime.date` to `str` in `es_update --start` #39 es_update start flag broken --- django_elastic_migrations/models.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/django_elastic_migrations/models.py b/django_elastic_migrations/models.py index 6b6f9d2..6cda607 100644 --- a/django_elastic_migrations/models.py +++ b/django_elastic_migrations/models.py @@ -957,7 +957,10 @@ def __init__(self, *args, **kwargs): actual_val = kwargs.pop(kwarg_name, default_val) setattr(self, kwarg_name, actual_val) if actual_val != default_val: - self.self_kwargs[kwarg_name] = actual_val + if kwarg_name == 'start_date': + self.self_kwargs['start_date'] = str(actual_val) + else: + self.self_kwargs[kwarg_name] = actual_val if NewerModeMixin.MODE_NAME in kwargs: self.self_kwargs[NewerModeMixin.MODE_NAME] = True @@ -965,14 +968,7 @@ def __init__(self, *args, **kwargs): super(UpdateIndexAction, self).__init__(*args, **kwargs) if self.task_kwargs == '{}' and self.self_kwargs: # retain a history of how this command was called - try: - self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) - except TypeError as e: - if 'Object of type datetime is not JSON serializable' in str(e): - for key, val in self.self_kwargs.items(): - if isinstance(val, datetime.date): - self.self_kwargs[key] = str(val) - self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) + self.task_kwargs = json.dumps(self.self_kwargs, sort_keys=True) self._batch_num = 0 self._expected_remaining = 0