From af95c3d9d9bfe12268a08f3dfa6acf271a689c0c Mon Sep 17 00:00:00 2001 From: Kyle Villegas <86266231+kylevillegas93@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:16:30 -0500 Subject: [PATCH] NO-REF: Remove Elastic Search Manager Ancestor (#435) --- processes/cluster.py | 14 ++++++++------ processes/core.py | 5 ++--- tests/unit/processes/test_core_process.py | 4 ---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/processes/cluster.py b/processes/cluster.py index 377c8666e0..c04c1f0f78 100644 --- a/processes/cluster.py +++ b/processes/cluster.py @@ -4,7 +4,7 @@ from typing import Optional from .core import CoreProcess -from managers import SFRRecordManager, KMeansManager, SFRElasticRecordManager +from managers import SFRRecordManager, KMeansManager, SFRElasticRecordManager, ElasticsearchManager from model import Record, Work from logger import createLog @@ -27,10 +27,12 @@ def __init__(self, *args): self.createSession() self.createRedisClient() + + self.elastic_search_manager = ElasticsearchManager() - self.createElasticConnection() - self.createElasticSearchIngestPipeline() - self.createElasticSearchIndex() + self.elastic_search_manager.createElasticConnection() + self.elastic_search_manager.createElasticSearchIngestPipeline() + self.elastic_search_manager.createElasticSearchIndex() def runProcess(self): try: @@ -135,7 +137,7 @@ def update_cluster_status(self, record_ids: list[str], cluster_status: bool=True ) def update_elastic_search(self, works_to_index: list, word_ids_to_delete: set): - self.deleteWorkRecords(word_ids_to_delete) + self.elastic_search_manager.deleteWorkRecords(word_ids_to_delete) self.index_works_in_elastic_search(works_to_index) def delete_stale_works(self, work_ids: set[str]): @@ -234,7 +236,7 @@ def index_works_in_elastic_search(self, works: Work): elastic_manager.getCreateWork() work_documents.append(elastic_manager.work) - self.saveWorkRecords(work_documents) + self.elastic_search_manager.saveWorkRecords(work_documents) def titles_overlap(self, tokenized_record_title: set, tokenized_matched_record_title: set): if len(tokenized_record_title) == 1 and not tokenized_record_title <= tokenized_matched_record_title: diff --git a/processes/core.py b/processes/core.py index 72e146822b..0811926c2a 100644 --- a/processes/core.py +++ b/processes/core.py @@ -1,4 +1,4 @@ -from managers import DBManager, RabbitMQManager, RedisManager, ElasticsearchManager, S3Manager +from managers import DBManager, RabbitMQManager, RedisManager, S3Manager from model import Record from static.manager import StaticManager @@ -8,8 +8,7 @@ logger = createLog(__name__) -class CoreProcess(DBManager, RabbitMQManager, RedisManager, StaticManager, - ElasticsearchManager, S3Manager): +class CoreProcess(DBManager, RabbitMQManager, RedisManager, StaticManager, S3Manager): def __init__(self, process, customFile, ingestPeriod, singleRecord, batchSize=500): super(CoreProcess, self).__init__() self.process = process diff --git a/tests/unit/processes/test_core_process.py b/tests/unit/processes/test_core_process.py index b0cb9eb5d3..da37ac61ce 100644 --- a/tests/unit/processes/test_core_process.py +++ b/tests/unit/processes/test_core_process.py @@ -34,10 +34,6 @@ def test_coreProcess_initial_values(self, coreInstance): assert coreInstance.port == 'test_psql_port' assert coreInstance.db == 'test_psql_name' - # Properties set by the ElasticSearchManager - assert coreInstance.index == 'test_es_index' - assert coreInstance.client is None - # Properties set by the RabbitMQManager assert coreInstance.rabbitHost == 'test_rbmq_host' assert coreInstance.rabbitPort == 'test_rbmq_port'