From 3bfcbca4c006402a1a14570fd83c55a87f00da1c Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Mon, 10 Jul 2023 16:27:36 -0400 Subject: [PATCH 1/9] elasticsearch-dsl-py < 7.0 mn/#3-support-latest-elasticsearch-dsl-6 --- requirements/base.in | 2 +- requirements/dev.txt | 27 +++++++++++++-------------- requirements/quality.txt | 12 ++++++------ requirements/test.txt | 8 +++----- requirements/travis.txt | 4 ++-- setup.py | 8 +------- 6 files changed, 26 insertions(+), 35 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 5e661e0..4b25d24 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,6 @@ # Core requirements for using this application Django # Web application framework -elasticsearch-dsl==6.1.0 # For interacting with Elaticsearch +elasticsearch-dsl<7.0.0 # For interacting with Elaticsearch texttable # Printing tabular data from management commands multiprocessing-logging # for logging in ./manage.py es_update --workers \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 505a850..09ea31c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,11 +6,11 @@ # asgiref==3.7.2 # via django -astroid==2.15.5 +astroid==2.15.6 # via # pylint # pylint-celery -backports-functools-lru-cache==1.6.5 +backports-functools-lru-cache==1.6.6 # via caniusepython3 bleach==6.0.0 # via readme-renderer @@ -22,9 +22,9 @@ certifi==2023.5.7 # via requests chardet==5.1.0 # via diff-cover -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.4 # via # click-log # code-annotations @@ -42,7 +42,7 @@ distlib==0.3.6 # via # caniusepython3 # virtualenv -django==4.2.2 +django==4.2.3 # via -r requirements/base.in docutils==0.20.1 # via readme-renderer @@ -52,7 +52,7 @@ edx-lint==5.3.4 # -r requirements/quality.in elasticsearch==6.8.2 # via elasticsearch-dsl -elasticsearch-dsl==6.1.0 +elasticsearch-dsl==6.4.0 # via -r requirements/base.in filelock==3.12.2 # via @@ -60,17 +60,15 @@ filelock==3.12.2 # virtualenv idna==3.4 # via requests -importlib-metadata==6.7.0 +importlib-metadata==6.8.0 # via # keyring # twine -ipaddress==1.0.23 - # via elasticsearch-dsl isort==5.12.0 # via # -r requirements/quality.in # pylint -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jinja2==3.1.2 # via @@ -99,11 +97,11 @@ packaging==23.1 # tox pbr==5.11.1 # via stevedore -pip-tools==6.13.0 +pip-tools==6.14.0 # via -r requirements/dev.in pkginfo==1.9.6 # via twine -platformdirs==3.8.0 +platformdirs==3.8.1 # via # pylint # virtualenv @@ -177,6 +175,7 @@ texttable==1.6.7 tomli==2.0.1 # via # build + # pip-tools # pylint # pyproject-hooks # tox @@ -193,7 +192,7 @@ tox-pyenv==1.1.0 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/dev.in -typing-extensions==4.7.0 +typing-extensions==4.7.1 # via # asgiref # astroid @@ -213,7 +212,7 @@ wheel==0.40.0 # pip-tools wrapt==1.15.0 # via astroid -zipp==3.15.0 +zipp==3.16.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/quality.txt b/requirements/quality.txt index 640d9db..76cf470 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,19 +4,19 @@ # # pip-compile --output-file=requirements/quality.txt requirements/quality.in # -astroid==2.15.5 +astroid==2.15.6 # via # pylint # pylint-celery -backports-functools-lru-cache==1.6.5 +backports-functools-lru-cache==1.6.6 # via caniusepython3 caniusepython3==7.3.0 # via -r requirements/quality.in certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests -click==8.1.3 +click==8.1.4 # via # click-log # code-annotations @@ -49,7 +49,7 @@ packaging==23.1 # via caniusepython3 pbr==5.11.1 # via stevedore -platformdirs==3.8.0 +platformdirs==3.8.1 # via pylint pycodestyle==2.10.0 # via -r requirements/quality.in @@ -87,7 +87,7 @@ tomli==2.0.1 # via pylint tomlkit==0.11.8 # via pylint -typing-extensions==4.7.0 +typing-extensions==4.7.1 # via # astroid # pylint diff --git a/requirements/test.txt b/requirements/test.txt index 9446c25..3f3f592 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,7 +8,7 @@ asgiref==3.7.2 # via django certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests codecov==2.1.13 # via -r requirements/test.in @@ -27,13 +27,11 @@ filelock==3.12.2 # virtualenv idna==3.4 # via requests -ipaddress==1.0.23 - # via elasticsearch-dsl multiprocessing-logging==0.3.4 # via -r requirements/base.in packaging==23.1 # via tox -platformdirs==3.8.0 +platformdirs==3.8.1 # via virtualenv pluggy==1.2.0 # via tox @@ -60,7 +58,7 @@ tox==3.28.0 # tox-battery tox-battery==0.6.1 # via -r requirements/test.in -typing-extensions==4.7.0 +typing-extensions==4.7.1 # via asgiref urllib3==2.0.3 # via diff --git a/requirements/travis.txt b/requirements/travis.txt index 3456fbf..f96a9a6 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -6,7 +6,7 @@ # certifi==2023.5.7 # via requests -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via requests codecov==2.1.13 # via -r requirements/travis.in @@ -22,7 +22,7 @@ idna==3.4 # via requests packaging==23.1 # via tox -platformdirs==3.8.0 +platformdirs==3.8.1 # via virtualenv pluggy==1.2.0 # via tox diff --git a/setup.py b/setup.py index 9bdec25..d7d0852 100755 --- a/setup.py +++ b/setup.py @@ -49,8 +49,7 @@ def get_version(*file_paths): license='MIT', include_package_data=True, install_requires=[ - # TBD: GH issue #3 includes support for elasticsearch-dsl>=6.2.0 - "Django>=1.8", "elasticsearch-dsl>=6.0.0,<6.2.0", "texttable>=1.2.1", + "Django>=1.8", "elasticsearch-dsl>=6.4.0<7.0.0", "texttable>=1.2.1", "multiprocessing-logging>=0.2.6" ], zip_safe=False, @@ -59,14 +58,9 @@ def get_version(*file_paths): classifiers=[ 'Development Status :: 4 - Beta', 'Framework :: Django', - 'Framework :: Django :: 1.11', - 'Framework :: Django :: 2.0', - 'Framework :: Django :: 2.1', - 'Framework :: Django :: 2.2', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', ], ) From 2459e023055385c1818c977f46f736717bf9eb93 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Mon, 10 Jul 2023 16:31:23 -0400 Subject: [PATCH 2/9] typofix mn/#3-support-latest-elasticsearch-dsl-6 --- django_elastic_migrations/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elastic_migrations/models.py b/django_elastic_migrations/models.py index f2fa569..a5d38ea 100644 --- a/django_elastic_migrations/models.py +++ b/django_elastic_migrations/models.py @@ -30,7 +30,7 @@ class Index(models.Model): """ name = models.CharField(verbose_name="Index Name", max_length=32, unique=True) - # Django convention is to use '+' for related name when you don't need the + # Django's convention is to use '+' for related name when you don't need the # reverse relation. in this case, we already have IndexVersion pointing # back to Index, so we don't need that reverse name. # See https://docs.djangoproject.com/en/2.0/ref/models/fields/#django.db.models.ForeignKey.related_name From 6b7e1c791feafbb48cac607f44d86946d5b6c92f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Mon, 10 Jul 2023 16:34:06 -0400 Subject: [PATCH 3/9] update dev env for es 6.4 mn/#3-support-latest-elasticsearch-dsl-6 --- Makefile | 28 +++++++++++++++++++++ docker-compose.yml | 63 +++++++++++++++++++++++++++------------------- setup.py | 2 +- 3 files changed, 66 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 99c3adc..5d1eb4d 100644 --- a/Makefile +++ b/Makefile @@ -74,3 +74,31 @@ selfcheck: ## check that the Makefile is well-formed pylintrc: ## check that the Makefile is well-formed edx_lint write pylintrc + +nuke: + @echo "Removing containers, networks, and data... ๐Ÿงจ" + @echo + docker compose down -v + +es-health-local-api: + @echo "Assuming you've got docker up, getting health of elasticsearch... ๐Ÿ“„" + @echo + curl 'localhost:9200/_cat/health?format=json&pretty=true' + +es-list-indexes-local-api: + @echo "Assuming you've got docker up, getting elasticsearch indexes via API call... ๐Ÿ“„" + curl 'localhost:9200/_cat/indices?format=json&pretty=true' + +es-print-java-env: + @echo "printing runtime value of ES_JAVA_OPTS" + docker-compose run --rm elastic printenv ES_JAVA_OPTS + +up: + @echo "Bringing up elastic 6 in docker ๐Ÿš€" + docker compose up -d --remove-orphans + +down: + @echo "Destroying local containers, but keeping data volumes intact... ๐Ÿ˜ด" + docker compose down + + diff --git a/docker-compose.yml b/docker-compose.yml index 5cf86fe..73cdd70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,8 @@ version: '2.1' services: elastic: - image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1 + # arm64v8 is for Apple M1 + image: webhippie/elasticsearch:6.4 ports: - 9200:9200 - 9300:9300 @@ -23,31 +24,41 @@ services: interval: 10s timeout: 10s retries: 10 - db: - image: postgres - restart: always - ports: - - "5432:5432" - environment: - POSTGRES_USER: "pguser" - POSTGRES_PASSWORD: "pgpass" - POSTGRES_DB: "pgdb" + volumes: + # for normal elasticsearch: + # - elasticsearch:/usr/share/elasticsearch/data + # for webhippie/elasticsearch: + - elasticsearch:/var/lib/elasticsearch -# uncomment to get kibana, which can be useful for analyzing index contents -# this is currently not a required part of testing infrastructure, just left here for convenience -# kibana: -# image: docker.elastic.co/kibana/kibana:6.0.1 -# depends_on: -# elastic: -# condition: service_healthy -# links: -# - elastic + # commented out because right now we're not using postgres for tests +# comment back in if we need it for multiprocessing tests +# db: +# image: postgres +# restart: always # ports: -# - 5601:5601 +# - "5432:5432" # environment: -# - "ELASTICSEARCH_URL=http://elastic:9200" -# healthcheck: -# test: ["CMD", "curl", "-f", "http://localhost:5601"] -# interval: 10s -# timeout: 10s -# retries: 10 \ No newline at end of file +# POSTGRES_USER: "pguser" +# POSTGRES_PASSWORD: "pgpass" +# POSTGRES_DB: "pgdb" + +# uncomment to get kibana, which can be useful for analyzing index contents +# this is currently not a required part of testing infrastructure, just left here for convenience + kibana: + image: docker.elastic.co/kibana/kibana:6.4.3 + depends_on: + elastic: + condition: service_healthy + links: + - elastic + ports: + - 5601:5601 + environment: + - "ELASTICSEARCH_URL=http://elastic:9200" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:5601"] + interval: 10s + timeout: 10s + retries: 10 +volumes: + elasticsearch: \ No newline at end of file diff --git a/setup.py b/setup.py index d7d0852..74e8e9f 100755 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ def get_version(*file_paths): license='MIT', include_package_data=True, install_requires=[ - "Django>=1.8", "elasticsearch-dsl>=6.4.0<7.0.0", "texttable>=1.2.1", + "Django>=1.8", "elasticsearch-dsl>=6.4.0,<7.0.0", "texttable>=1.2.1", "multiprocessing-logging>=0.2.6" ], zip_safe=False, From 22c96d56841a97f982e7361492cb03a3b830b74d Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Tue, 11 Jul 2023 12:39:28 -0400 Subject: [PATCH 4/9] fix #96 es_list doesn't list all indexes mn/#3-support-latest-elasticsearch-dsl-6 --- django_elastic_migrations/management/commands/es_list.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/django_elastic_migrations/management/commands/es_list.py b/django_elastic_migrations/management/commands/es_list.py index c205300..df59e04 100644 --- a/django_elastic_migrations/management/commands/es_list.py +++ b/django_elastic_migrations/management/commands/es_list.py @@ -64,7 +64,6 @@ def handle(self, *args, **options): for dem_index in indexes: dem_index_model = dem_index.get_index_model() index_versions = dem_index_model.get_available_versions_with_prefix() - row = None if index_versions: for index_version in index_versions: num_docs = DEMIndexManager.get_es_index_doc_count(index_version.name) @@ -74,9 +73,9 @@ def handle(self, *args, **options): index_version.is_active or 0, num_docs, index_version.tag) + rows.append(row) else: row = EsListRow(dem_index.get_base_name(), "", False, False, 0, "Current (not created)") - if row: rows.append(row) except AttributeError: raise FirstMigrationNotRunError() From bcac9d8fa8e3c455baf9fe787d4f07f84a574125 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Tue, 11 Jul 2023 12:39:36 -0400 Subject: [PATCH 5/9] fix typo mn/#3-support-latest-elasticsearch-dsl-6 --- django_elastic_migrations/management/commands/es.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elastic_migrations/management/commands/es.py b/django_elastic_migrations/management/commands/es.py index 130247c..72fb77b 100644 --- a/django_elastic_migrations/management/commands/es.py +++ b/django_elastic_migrations/management/commands/es.py @@ -122,7 +122,7 @@ def get_index_specifying_options(cls, options, require_one_include_list=None): if apply_all and indexes: logger.warning( "Received --all along with index names: '{indexes}'." - "Noramlly you would not specify names of indexes " + "Normally you would not specify names of indexes " "with --all, since --all covers all the indexes. " "The --all has been canceled; operating on just '{indexes}'." "To clear *all* the indexes, just use --all.".format( From 572d6b12946c71c81730967ec140df89daf201e0 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 12 Jul 2023 14:28:24 -0400 Subject: [PATCH 6/9] all tests passing locally * fixes failing test in elasticsearch-dsl-py 6.4 relative to 6.1: `test_newer_flag` mn/#3-support-latest-elasticsearch-dsl-6 --- django_elastic_migrations/indexes.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/django_elastic_migrations/indexes.py b/django_elastic_migrations/indexes.py index 4360a14..1e44e35 100644 --- a/django_elastic_migrations/indexes.py +++ b/django_elastic_migrations/indexes.py @@ -7,7 +7,7 @@ from django.db import ProgrammingError from elasticsearch import TransportError from elasticsearch.helpers import expand_action, bulk -from elasticsearch_dsl import Index as ESIndex, DocType as ESDocType, Q as ESQ, Search +from elasticsearch_dsl import Index as ESIndex, Document as ESDocument, Q as ESQ, Search from django_elastic_migrations import es_client, environment_prefix, es_test_prefix, dem_index_paths, get_logger, codebase_id from django_elastic_migrations.exceptions import DEMIndexNotFound, DEMDocTypeRequiresGetReindexIterator, \ @@ -478,7 +478,7 @@ def __getattribute__(self, item): return None -class DEMDocType(ESDocType): +class DEMDocType(ESDocument): """ Django users subclass DEMDocType instead of Elasticsearch's DocType to use Django Elastic Migrations. All documentation from their class @@ -771,6 +771,24 @@ def batched_bulk_index(cls, queryset=None, workers=0, last_updated_datetime=None num_failures += result_failures return num_successes, num_failures + def _get_index(self, index=None, required=True): + """ + We have chosen to override this method to allow for the use of + a dynamic index at runtime. This is necessary to support the + key use case of Django Elastic Migrations: being able to index + into a new index version while the old index version is still + being used by the application. + + A future version of DEM may instead elect to use a different + technique to supply this information. The inheritance hierarchy + for elasticsearch-dsl-py is not very friendly to overriding. + """ + if index is None: + index = self.get_dem_index().get_es_index_name() + if index is None: + index = super(DEMDocType, self)._get_index(index, required) + return index + class DEMIndex(ESIndex): """ From e06a0827d93d873d942b84971d059e6f99f996e3 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 12 Jul 2023 17:06:23 -0400 Subject: [PATCH 7/9] rename DEMDocType to DEMDocument mn/#3-support-latest-elasticsearch-dsl-6 --- CHANGELOG.rst | 9 ++++ README.rst | 11 ++-- django_elastic_migrations/exceptions.py | 12 ++--- django_elastic_migrations/indexes.py | 54 ++++++++++--------- .../management/commands/es_update.py | 2 +- tests/search.py | 12 ++--- 6 files changed, 56 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3f5fc2c..c352240 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Changelog --------- +0.10.0 (2023-07-12) +^^^^^^^^^^^^^^^^^^ +* support elasticsearch-dsl 6.4 - closes `#3 support elasticsearch-dsl-py v6.4` +* fix `#90 Convert to GitHub Actions` +* fix `#96 es_list doesn't list all indexes` +* open up django to anything > 1.10 +* rename DEMDocType to DEMDocument, following elasticsearch-dsl-py + * DEMDocType will be removed in 1.X + 0.9.0 (2018-11-27) ^^^^^^^^^^^^^^^^^^ * added postgres to docker-compose and travis, and started using postgres for all tests instead of sqlite diff --git a/README.rst b/README.rst index 46424ae..d98ff27 100644 --- a/README.rst +++ b/README.rst @@ -32,12 +32,11 @@ Django Elastic Migrations adapts these tools into a Django app which also: * Implements concurrent bulk indexing powered by python ``multiprocessing`` * Gives Django test hooks for Elasticsearch * Records a history of all actions that change Elasticsearch indexes -* Supports AWS Elasticsearch 6.0, 6.1 (6.2 TBD; see `#3 support elasticsearch-dsl 6.2`_) +* Supports elasticsearch-dsl-py>=6.4 (support for 7.10.2 is in progress) * Enables having two or more servers share the same Elasticsearch cluster .. _elasticsearch-py: https://github.com/elastic/elasticsearch-py .. _elasticsearch-dsl-py: https://github.com/elastic/elasticsearch-dsl-py -.. _#3 support elasticsearch-dsl 6.2: https://github.com/HBS-HBX/django-elastic-migrations/issues/3 Models @@ -123,7 +122,7 @@ Installation #. Create an ``DEMIndex``: :: - from django_elastic_migrations.indexes import DEMIndex, DEMDocType + from django_elastic_migrations.indexes import DEMIndex, DEMDocument from .models import Movie from elasticsearch_dsl import Text @@ -131,7 +130,7 @@ Installation @MoviesIndex.doc_type - class MovieSearchDoc(DEMDocType): + class MovieSearchDoc(DEMDocument): text = TEXT_COMPLEX_ENGLISH_NGRAM_METAPHONE @classmethod @@ -292,11 +291,11 @@ Tuning Bulk Indexing Parameters ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ``/.manage.py es_update`` will divide the result of -``DEMDocType.get_queryset()`` into batches of size ``DocType.BATCH_SIZE``. +``DEMDocument.get_queryset()`` into batches of size ``DocType.BATCH_SIZE``. Override this number to change the batch size. There are many configurable paramters to Elasticsearch's `bulk updater `_. -To provide a custom value, override ``DEMDocType.get_bulk_indexing_kwargs()`` +To provide a custom value, override ``DEMDocument.get_bulk_indexing_kwargs()`` and return the kwargs you would like to customize. Development diff --git a/django_elastic_migrations/exceptions.py b/django_elastic_migrations/exceptions.py index 7073312..bf993c8 100644 --- a/django_elastic_migrations/exceptions.py +++ b/django_elastic_migrations/exceptions.py @@ -63,22 +63,22 @@ class IndexNamePropertyCannotBeSet(DjangoElasticMigrationsException): """ -class DEMDocTypeRequiresGetReindexIterator(DjangoElasticMigrationsException): +class DEMDocumentRequiresGetReindexIterator(DjangoElasticMigrationsException): """ - Raised when ./manage.py es_update tries to call DEMDocType.get_reindex_iterator() + Raised when ./manage.py es_update tries to call DEMDocument.get_reindex_iterator() on a subclass, but the subclass has not implemented this. """ message = ("To run ./manage.py es_update my_index, my_index needs to " - "implement DEMDocType.get_reindex_iterator(self, last_updated_datetime=None)") + "implement DEMDocument.get_reindex_iterator(self, last_updated_datetime=None)") -class DEMDocTypeRequiresGetQueryset(DjangoElasticMigrationsException): +class DEMDocumentRequiresGetQueryset(DjangoElasticMigrationsException): """ - Raised when ./manage.py es_update tries to call DEMDocType.get_queryset() + Raised when ./manage.py es_update tries to call DEMDocument.get_queryset() on a subclass, but the subclass has not implemented this. """ message = ("To run ./manage.py es_update my_index, my_index needs to " - "implement DEMDocType.get_queryset()") + "implement DEMDocument.get_queryset()") class DEMIndexVersionCodebaseMismatchError(DjangoElasticMigrationsException): diff --git a/django_elastic_migrations/indexes.py b/django_elastic_migrations/indexes.py index 1e44e35..b7711a6 100644 --- a/django_elastic_migrations/indexes.py +++ b/django_elastic_migrations/indexes.py @@ -10,8 +10,8 @@ from elasticsearch_dsl import Index as ESIndex, Document as ESDocument, Q as ESQ, Search from django_elastic_migrations import es_client, environment_prefix, es_test_prefix, dem_index_paths, get_logger, codebase_id -from django_elastic_migrations.exceptions import DEMIndexNotFound, DEMDocTypeRequiresGetReindexIterator, \ - IllegalDEMIndexState, NoActiveIndexVersion, DEMDocTypeRequiresGetQueryset +from django_elastic_migrations.exceptions import DEMIndexNotFound, DEMDocumentRequiresGetReindexIterator, \ + IllegalDEMIndexState, NoActiveIndexVersion, DEMDocumentRequiresGetQueryset from django_elastic_migrations.utils.es_utils import get_index_hash_and_json from django_elastic_migrations.utils.loading import import_module_element from django_elastic_migrations.utils.multiprocessing_utils import DjangoMultiProcess, USE_ALL_WORKERS @@ -273,7 +273,7 @@ def reinitialize_esindex_instances(cls): After our django-elastic-migrations app is ready to talk to the DB and find out the names of the indexes, go through and reinitialize each ES Index subclass instance - as well as their associated ES DocType subclasses + as well as their associated ES Document subclasses with the appropriate index names """ for index_base_name, instance in cls.instances.items(): @@ -442,7 +442,7 @@ def _start_action_for_indexes(cls, action, index_name, exact_mode=False): raise DEMIndexNotFound() -class _DEMDocTypeIndexHandler(object): +class _DEMDocumentIndexHandler(object): """ Internally, Elasticsearch-dsl-py uses a string stored in the DocType to determine which index to write to. This class is @@ -478,12 +478,12 @@ def __getattribute__(self, item): return None -class DEMDocType(ESDocument): +class DEMDocument(ESDocument): """ - Django users subclass DEMDocType instead of Elasticsearch's DocType + Django users should subclass DEMDocument instead of elasticsearch-dsl-py's Document to use Django Elastic Migrations. All documentation from their class applies here. - https://elasticsearch-dsl.readthedocs.io/en/latest/api.html#elasticsearch_dsl.DocType + https://elasticsearch-dsl.readthedocs.io/en/latest/api.html#elasticsearch_dsl.Document Change from Elasticsearch: we manage the doc type's index name to make it the activated version of the index by default. @@ -507,29 +507,29 @@ class DEMDocType(ESDocument): MAX_RETRIES = 5 def __init__(self, *args, **kwargs): - super(DEMDocType, self).__init__(*args, **kwargs) + super(DEMDocument, self).__init__(*args, **kwargs) # super.__init__ creates the self._doc_type property that we # modify here - self._doc_type = _DEMDocTypeIndexHandler( + self._doc_type = _DEMDocumentIndexHandler( getattr(self, '_doc_type', None)) @classmethod def get_reindex_iterator(cls, queryset): """ Django users override this method. It must return an iterator - or generator of instantiated DEMDocType subclasses, ready + or generator of instantiated DEMDocument subclasses, ready for inserting into Elasticsearch. For example: - class UsersDocType(DEMDocType) + class UsersDocument(DEMDocument) @classmethod def get_reindex_iterator(cls, queryset): return [cls.getDocForUser(u) for user in queryset] :param queryset: queryset of objects to index; result of get_queryset() - :return: iterator / generator of *DEMDocType instances* + :return: iterator / generator of *DEMDocument instances* """ - raise DEMDocTypeRequiresGetReindexIterator() + raise DEMDocumentRequiresGetReindexIterator() @classmethod def get_queryset(cls, last_updated_datetime=None): @@ -539,7 +539,7 @@ def get_queryset(cls, last_updated_datetime=None): :param last_updated_datetime: :return: """ - raise DEMDocTypeRequiresGetQueryset() + raise DEMDocumentRequiresGetQueryset() @classmethod def get_db_objects_by_id(cls, ids): @@ -547,7 +547,7 @@ def get_db_objects_by_id(cls, ids): @classmethod def get_dem_index(cls): - # TODO: what should happen if DEMDocType instance has no active version? + # TODO: what should happen if DEMDocument instance has no active version? # currently, if this exact version is not found, it will return None return DEMIndexManager.get_dem_index(cls._doc_type.index, exact_mode=True) @@ -686,7 +686,7 @@ def bulk_index(cls, reindex_iterator): """ Execute Elasticsearch's bulk indexing helper, passing in the result of cls.get_bulk_indexing_kwargs() - :param reindex_iterator: an iterator of DocType instances from cls.get_reindex_iterator() + :param reindex_iterator: an iterator of Document instances from cls.get_reindex_iterator() :return: (num_success, num_failed) """ kwargs = cls.get_bulk_indexing_kwargs() @@ -786,10 +786,14 @@ def _get_index(self, index=None, required=True): if index is None: index = self.get_dem_index().get_es_index_name() if index is None: - index = super(DEMDocType, self)._get_index(index, required) + index = super(DEMDocument, self)._get_index(index, required) return index +# until we can support only elasticsearch 7.x, we need to support both DocType and Document +DEMDocType = DEMDocument + + class DEMIndex(ESIndex): """ Django users subclass DEMIndex instead of elasticsearch-dsl-py's Index @@ -881,25 +885,25 @@ def delete(self, **kwargs): if index_version: index_version.delete() - def doc_type(self, doc_type=None) -> DEMDocType: + def doc_type(self, doc_type=None) -> DEMDocument: """ Overrides elasticsearch_dsl.Index.doc_type() and called during DEMIndexManager.initialize(). - Associates a DEMDocType with this DEMIndex, so that commands + Associates a DEMDocument with this DEMIndex, so that commands directed to this DEMIndex always go to the active index version by querying the DEMIndexManager. - Sets the active index version name into the DEMDocType, so that - DEMDocType.search() queries the active index version. + Sets the active index version name into the DEMDocument, so that + DEMDocument.search() queries the active index version. - :returns DEMDocType associated with this DEMIndex (if any) + :returns DEMDocument associated with this DEMIndex (if any) """ if doc_type: self.__doc_type = doc_type active_version_name = self.get_active_version_index_name() - # set the active index version name into the DEMDocType, - # so DEMDocType.search() is directed to the active index in elasticsearch + # set the active index version name into the DEMDocument, + # so DEMDocument.search() is directed to the active index in elasticsearch if active_version_name and self.__doc_type._doc_type: self.__doc_type._doc_type.index = active_version_name @@ -918,7 +922,7 @@ def doc_type(self, doc_type=None) -> DEMDocType: our_tag = codebase_id msg = ( "DEMIndex.doc_type received a request to use an elasticsearch index whose exact " - "schema / DEMDocType was not accessible in this codebase. " + "schema / DEMDocument was not accessible in this codebase. " "This may lead to undefined behavior (for example if this codebase searches or indexes " "a field that has changed in the requested index, it may not return correctly). " "\n - requested index: {version_name} " diff --git a/django_elastic_migrations/management/commands/es_update.py b/django_elastic_migrations/management/commands/es_update.py index 8482846..9c4a7dc 100644 --- a/django_elastic_migrations/management/commands/es_update.py +++ b/django_elastic_migrations/management/commands/es_update.py @@ -15,7 +15,7 @@ def add_arguments(self, parser): parser.add_argument( '--resume', action='store_true', help=("Only update documents that have changed since " - "the last ./manage.py es_update. NOTE: DEMDocType subclass " + "the last ./manage.py es_update. NOTE: DEMDocument subclass " "needs to implement this.") ) parser.add_argument( diff --git a/tests/search.py b/tests/search.py index 2a35f1d..3b4f443 100644 --- a/tests/search.py +++ b/tests/search.py @@ -5,7 +5,7 @@ from django.conf import settings from elasticsearch_dsl import Text, Q, analyzer, token_filter, tokenizer -from django_elastic_migrations.indexes import DEMIndex, DEMDocType, DEMIndexManager +from django_elastic_migrations.indexes import DEMIndex, DEMDocument, DEMIndexManager from tests.models import Movie basic_analyzer = analyzer( @@ -16,7 +16,7 @@ alternate_textfield = Text(analyzer=basic_analyzer, search_analyzer=basic_analyzer) -class GenericDocType(DEMDocType): +class GenericDocument(DEMDocument): full_text = Text(required=True) full_text_boosted = Text(required=True) @@ -85,7 +85,7 @@ def get_full_text_search_query(cls, search_query): @MovieSearchIndex.doc_type -class MovieSearchDoc(GenericDocType): +class MovieSearchDoc(GenericDocument): @classmethod def get_queryset(cls, last_updated_datetime=None): @@ -107,7 +107,7 @@ def get_model_full_text_boosted(cls, model): return model.title -class DefaultNewSearchDocTypeMixin(GenericDocType): +class DefaultNewSearchDocTypeMixin(GenericDocument): """ Used by get_new_search_index() in the case that doc_type_mixin is not supplied """ @@ -133,7 +133,7 @@ def get_new_search_index(name, doc_type_mixin=None, create_and_activate=True, de :param cls: parameters of GenericDocType to override :param doc_type_mixin: a class to mix in to the generated doctype :param create_and_activate: if True, call DEMIndexManager.initialize(True, True) before returning - :return: DEMIndex, DEMDocType + :return: DEMIndex, DEMDocument """ if name == "movies": raise ValueError("Don't use the movies index for testing; it will interfere with the fixture") @@ -146,7 +146,7 @@ def get_new_search_index(name, doc_type_mixin=None, create_and_activate=True, de my_new_index = DEMIndex(name) my_new_index.settings(**settings.ELASTICSEARCH_INDEX_SETTINGS) - class MyNewSearchDocType(doc_type_mixin, GenericDocType): + class MyNewSearchDocType(doc_type_mixin, GenericDocument): pass my_new_index.doc_type(MyNewSearchDocType) From 30afbc4f400a9bf59c95ecaac74daa84aaf054b2 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 12 Jul 2023 17:06:38 -0400 Subject: [PATCH 8/9] bump actions to cache@v3 mn/#3-support-latest-elasticsearch-dsl-6 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0e9be7..37e0697 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache Pip Dependencies - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements/test.txt') }} From b12fd8c672e079ba79a916b78ce3df35f3c572d3 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 13 Jul 2023 09:12:54 -0400 Subject: [PATCH 9/9] bump version number to 0.10.0 mn/#3-support-latest-elasticsearch-dsl-6 --- django_elastic_migrations/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index 26ccb77..640d570 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -8,7 +8,7 @@ from django_elastic_migrations.utils import loading from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -__version__ = '0.9.0' +__version__ = '0.10.0' default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig' # pylint: disable=invalid-name