From d0f247db94e3523b37bebc8a256553c2d157be76 Mon Sep 17 00:00:00 2001 From: qasimgulzar Date: Fri, 5 Jul 2024 13:57:48 +0500 Subject: [PATCH] feat: added meilisearch dependency and fixt index creation feat: integrate django_meili --- .../apps/course_metadata/models.py | 30 ++++++++++ .../management/commands/update_index.py | 2 +- course_discovery/settings/base.py | 18 ++++++ requirements/production.in | 1 + requirements/production.txt | 59 +++++++++---------- 5 files changed, 77 insertions(+), 33 deletions(-) diff --git a/course_discovery/apps/course_metadata/models.py b/course_discovery/apps/course_metadata/models.py index 4921c6a35d..b068236fd7 100644 --- a/course_discovery/apps/course_metadata/models.py +++ b/course_discovery/apps/course_metadata/models.py @@ -23,6 +23,7 @@ from django_elasticsearch_dsl.registries import registry from django_extensions.db.fields import AutoSlugField from django_extensions.db.models import TimeStampedModel +from django_meili.models import IndexMixin from edx_django_utils.cache import RequestCache, get_cache_key from elasticsearch.exceptions import RequestError from elasticsearch_dsl.query import Q as ESDSLQ @@ -4604,3 +4605,32 @@ class RestrictedCourseRun(DraftModelMixin, TimeStampedModel): def __str__(self): return f"{self.course_run.key}: <{self.restriction_type}>" + +if settings.MEILISEARCH_ENABLED: + class CourseProxy(IndexMixin, Course): + class MeiliMeta: + index_name = "Course" + filterable_fields = ("title",) + searchable_fields = ("uuid", "title") + displayed_fields = ("uuid", "title") + index_name = "course" + + class Meta: + proxy = True + + + class PersonProxy(IndexMixin, Person): + class MeiliMeta: + index_name = "person" + + class Meta: + proxy = True + + + class CourseRunProxy(IndexMixin, CourseRun): + class MeiliMeta: + index_name = "course_run" + primary_key = "uuid" + + class Meta: + proxy = True diff --git a/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py b/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py index 3a5e0f2dfe..f2b2acc45b 100644 --- a/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py +++ b/course_discovery/apps/edx_elasticsearch_dsl_extensions/management/commands/update_index.py @@ -98,8 +98,8 @@ def _update(self, models, options): for document in registry.get_documents(models): # pylint: disable=protected-access index = document._index - record_count = self.get_record_count(document) alias, new_index_name = self.prepare_backend_index(index) + record_count = self.get_record_count(document) alias_mappings.append(AliasMapper(document, index, new_index_name, alias, record_count)) # Set the alias (from settings) to the timestamped catalog. run_attempts = 0 diff --git a/course_discovery/settings/base.py b/course_discovery/settings/base.py index aad79e5f81..e4c3216b8a 100644 --- a/course_discovery/settings/base.py +++ b/course_discovery/settings/base.py @@ -68,6 +68,7 @@ 'nested_admin', 'openedx_events', 'multi_email_field', + 'django_meili', ] ALGOLIA = { @@ -787,3 +788,20 @@ ) ENABLE_COURSE_REVIEWS_ADMIN = False + +################### Studio Search (beta), using Meilisearch ################### +MEILISEARCH_ENABLED = False +if MEILISEARCH_ENABLED: + MEILISEARCH = { + 'HTTPS': False, # Whether HTTPS is enabled for the meilisearch server + 'HOST': 'meilisearch', # The host for the meilisearch server + 'MASTER_KEY': 'masterKey', + # The master key for meilisearch. See https://www.meilisearch.com/docs/learn/security/basic_security for more detail + 'PORT': 7700, # The port for the meilisearch server + 'TIMEOUT': None, # The timeout to wait for when using sync meilisearch server + 'CLIENT_AGENTS': None, # The client agents for the meilisearch server + 'DEBUG': DEBUG, # Whether to throw exceptions on failed creation of documents + 'SYNC': False, + # Whether to execute operations to meilisearch in a synchronous manner (waiting for each rather than letting the task queue operate) + 'OFFLINE': False, # Whether to make any http requests for the application. + } diff --git a/requirements/production.in b/requirements/production.in index 09ff870897..c3ac1d30af 100644 --- a/requirements/production.in +++ b/requirements/production.in @@ -12,3 +12,4 @@ newrelic python-memcached pymemcache PyYAML +django-meili==0.0.6 diff --git a/requirements/production.txt b/requirements/production.txt index 9ebd52bae0..c119bc1f14 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --output-file=requirements/production.txt requirements/production.in @@ -19,6 +19,8 @@ algoliasearch-django==1.7.3 # -r requirements/base.in amqp==5.2.0 # via kombu +annotated-types==0.7.0 + # via pydantic asgiref==3.8.1 # via # django @@ -26,10 +28,6 @@ asgiref==3.8.1 # django-countries asn1crypto==1.5.1 # via snowflake-connector-python -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==23.2.0 # via # aiohttp @@ -37,22 +35,15 @@ attrs==23.2.0 # zeep backoff==2.2.1 # via -r requirements/base.in -backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # celery - # django - # djangorestframework - # kombu beautifulsoup4==4.12.3 # via # -r requirements/base.in # taxonomy-connector billiard==4.2.0 # via celery -boto3==1.34.136 +boto3==1.34.139 # via django-ses -botocore==1.34.136 +botocore==1.34.139 # via # boto3 # s3transfer @@ -64,11 +55,13 @@ cairocffi==1.4.0 # cairosvg cairosvg==2.7.1 # via -r requirements/base.in +camel-converter[pydantic]==3.1.2 + # via meilisearch celery==5.4.0 # via # -c requirements/constraints.txt # taxonomy-connector -certifi==2024.6.2 +certifi==2024.7.4 # via # -r requirements/production.in # elasticsearch @@ -100,7 +93,7 @@ click-repl==0.3.0 # via celery code-annotations==1.8.0 # via edx-toggles -contentful==2.1.1 +contentful==2.2.0 # via -r requirements/base.in cryptography==42.0.8 # via @@ -136,6 +129,7 @@ django==4.2.13 # django-filter # django-guardian # django-localflavor + # django-meili # django-model-utils # django-multi-email-field # django-multiselectfield @@ -215,11 +209,13 @@ django-libsass==0.9 # via -r requirements/base.in django-localflavor==4.0 # via -r requirements/base.in +django-meili==0.0.6 + # via -r requirements/production.in django-model-utils==4.5.1 # via taxonomy-connector django-multi-email-field==0.7.0 # via -r requirements/base.in -django-multiselectfield==0.1.12 +django-multiselectfield==0.1.13 # via -r requirements/base.in django-nested-admin==4.0.2 # via -r requirements/base.in @@ -237,7 +233,7 @@ django-ses==4.1.0 # taxonomy-connector django-simple-history==3.7.0 # via -r requirements/base.in -django-solo==2.2.0 +django-solo==2.3.0 # via # -r requirements/base.in # taxonomy-connector @@ -365,9 +361,9 @@ gevent==24.2.1 # via -r requirements/production.in google-api-core==2.19.1 # via google-api-python-client -google-api-python-client==2.135.0 +google-api-python-client==2.136.0 # via -r requirements/base.in -google-auth==2.30.0 +google-auth==2.31.0 # via # google-api-core # google-api-python-client @@ -403,9 +399,6 @@ importlib-metadata==6.11.0 # via # -c requirements/common_constraints.txt # -r requirements/base.in - # markdown -importlib-resources==6.4.0 - # via pycountry inflection==0.5.1 # via drf-yasg isodate==0.6.1 @@ -433,6 +426,8 @@ markdown==3.6 # via -r requirements/base.in markupsafe==2.1.5 # via jinja2 +meilisearch==0.31.3 + # via django-meili more-itertools==10.3.0 # via simple-salesforce multidict==6.0.5 @@ -500,6 +495,10 @@ pycountry==24.6.1 # via -r requirements/base.in pycparser==2.22 # via cffi +pydantic==2.8.2 + # via camel-converter +pydantic-core==2.20.1 + # via pydantic pyjwt[crypto]==2.8.0 # via # drf-jwt @@ -568,6 +567,7 @@ requests==2.32.3 # edx-drf-extensions # edx-rest-api-client # google-api-core + # meilisearch # openai # requests-file # requests-oauthlib @@ -646,16 +646,14 @@ tqdm==4.66.4 # via openai typing-extensions==4.12.2 # via - # asgiref # django-countries # edx-opaque-keys - # kombu + # pydantic + # pydantic-core # simple-salesforce # snowflake-connector-python tzdata==2024.1 - # via - # backports-zoneinfo - # celery + # via celery unicodecsv==0.14.1 # via -r requirements/base.in uritemplate==4.1.1 @@ -667,7 +665,6 @@ urllib3==1.26.19 # botocore # elasticsearch # requests - # snowflake-connector-python vine==5.1.0 # via # amqp @@ -688,9 +685,7 @@ yarl==1.9.4 zeep==4.2.1 # via simple-salesforce zipp==3.19.2 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata zope-event==5.0 # via gevent zope-interface==6.4.post2