NO-REF: Refactoring statics (#448)

constants/get_constants.py

@@ -0,0 +1,17 @@
+from glob import glob
+import json
+import re
+
+
+def get_constants():
+    constants = {}
+
+    for path in glob('./constants/*.json'):
+        file_name = re.search(r'\/([a-z0-9]+)\.json', path).group(1)
+
+        with open(path, 'r') as constants_file:
+            file_constants = json.load(constants_file)
+
+            constants[file_name] = file_constants
+
+    return constants

mappings/base_mapping.py

@@ -7,11 +7,11 @@
 
 
 class BaseMapping(RecordMapping):
-    def __init__(self, source, statics):
+    def __init__(self, source, constants):
         self.mapping = {}
         self.source = source
         self.record = None
-        self.staticValues = statics
+        self.constants = constants
 
         self.formatter = CustomFormatter()
 

mappings/csv.py

@@ -4,8 +4,8 @@
 logger = create_log(__name__)
 
 class CSVMapping(BaseMapping):
-    def __init__(self, source, statics):
-        super().__init__(source, statics)
+    def __init__(self, source, constants):
+        super().__init__(source, constants)
 
     def applyMapping(self):
         newRecord = self.initEmptyRecord()

mappings/doab.py

@@ -4,8 +4,8 @@
 
 class DOABMapping(XMLMapping):
     DOI_REGEX = r'doabooks.org\/handle\/([0-9]+\.[0-9]+\.[0-9]+\/[0-9]+)'
-    def __init__(self, source, namespace, statics):
-        super(DOABMapping, self).__init__(source, namespace, statics)
+    def __init__(self, source, namespace, constants):
+        super(DOABMapping, self).__init__(source, namespace, constants)
         self.mapping = self.createMapping()
 
     def createMapping(self):

mappings/gutenberg.py

@@ -4,8 +4,8 @@
 from mappings.xml import XMLMapping
 
 class GutenbergMapping(XMLMapping):
-    def __init__(self, source, namespace, statics):
-        super(GutenbergMapping, self).__init__(source, namespace, statics)
+    def __init__(self, source, namespace, constants):
+        super(GutenbergMapping, self).__init__(source, namespace, constants)
         self.mapping = self.createMapping()
 
     def createMapping(self):
@@ -69,7 +69,7 @@ def applyFormatting(self):
         # Parse contributors to set proper roles
         for i, contributor in enumerate(self.record.contributors):
             contribComponents = contributor.split('|')
-            lcRelation = self.staticValues['lc']['relators'].get(contribComponents[-1], 'Contributor')
+            lcRelation = self.constants['lc']['relators'].get(contribComponents[-1], 'Contributor')
             self.record.contributors[i] = '{}|{}|{}|{}'.format(
                 *contribComponents[:-1], lcRelation
             )

mappings/hathitrust.py

@@ -5,8 +5,8 @@
 
 
 class HathiMapping(CSVMapping):
-    def __init__(self, source, statics):
-        super(HathiMapping, self).__init__(source, statics)
+    def __init__(self, source, constants):
+        super(HathiMapping, self).__init__(source, constants)
         self.mapping = self.createMapping()
 
     def createMapping(self):
@@ -67,15 +67,15 @@ def applyFormatting(self):
         self.record.contributors = self.record.contributors or []
         for i, contributor in enumerate(self.record.contributors):
             contribElements = contributor.lower().split('|')
-            fullContribName = self.staticValues['hathitrust']['sourceCodes'].get(contribElements[0], contribElements[0])
+            fullContribName = self.constants['hathitrust']['sourceCodes'].get(contribElements[0], contribElements[0])
             self.record.contributors[i] = contributor.replace(contribElements[0], fullContribName)
 
         # Parse rights codes
         rightsElements = self.record.rights.split('|') if self.record.rights else [''] * 5
-        rightsMetadata = self.staticValues['hathitrust']['rightsValues'].get(rightsElements[1], {'license': 'und', 'statement': 'und'}) 
+        rightsMetadata = self.constants['hathitrust']['rightsValues'].get(rightsElements[1], {'license': 'und', 'statement': 'und'}) 
         self.record.rights = 'hathitrust|{}|{}|{}|{}'.format(
             rightsMetadata['license'],
-            self.staticValues['hathitrust']['rightsReasons'].get(rightsElements[2], rightsElements[2]),
+            self.constants['hathitrust']['rightsReasons'].get(rightsElements[2], rightsElements[2]),
             rightsMetadata['statement'],
             rightsElements[4]
         )
@@ -102,4 +102,4 @@ def applyFormatting(self):
 
         # Parse spatial (pub place) codes
         self.record.spatial = self.record.spatial or ''
-        self.record.spatial = self.staticValues['marc']['countryCodes'].get(self.record.spatial.strip(), 'xx')
+        self.record.spatial = self.constants['marc']['countryCodes'].get(self.record.spatial.strip(), 'xx')

mappings/json.py

@@ -2,8 +2,8 @@
 
 
 class JSONMapping(BaseMapping):
-    def __init__(self, source, statics):
-        super().__init__(source, statics)
+    def __init__(self, source, constants):
+        super().__init__(source, constants)
 
     def applyMapping(self):
         self.record = self.initEmptyRecord()

mappings/marc.py

@@ -7,8 +7,8 @@
 
 
 class MARCMapping(BaseMapping):
-    def __init__(self, source, statics):
-        super().__init__(source, statics)
+    def __init__(self, source, constants):
+        super().__init__(source, constants)
 
     def applyMapping(self):
         newRecord = self.initEmptyRecord()

mappings/nypl.py

@@ -3,8 +3,8 @@
 from .sql import SQLMapping
 
 class NYPLMapping(SQLMapping):
-    def __init__(self, source, bibItems, statics, locationCodes):
-        super().__init__(source, statics)
+    def __init__(self, source, bibItems, constants, locationCodes):
+        super().__init__(source, constants)
         self.mapping = self.createMapping()
         self.locationCodes = locationCodes
         self.bibItems = bibItems
@@ -119,7 +119,7 @@ def applyFormatting(self):
         # Parse contributors to set proper roles
         for i, contributor in enumerate(self.record.contributors):
             contribComponents = contributor.split('|')
-            lcRelation = self.staticValues['lc']['relators'].get(contribComponents[-1], 'Contributor')
+            lcRelation = self.constants['lc']['relators'].get(contribComponents[-1], 'Contributor')
             self.record.contributors[i] = '{}|{}|{}|{}'.format(
                 *contribComponents[:-1], lcRelation
             )

mappings/oclcCatalog.py

@@ -16,8 +16,8 @@ class CatalogMapping(XMLMapping):
         'hathitrust': r'catalog.hathitrust.org\/api\/volumes\/([a-z]{3,6}\/[a-zA-Z0-9]+)\.html'
     }
 
-    def __init__(self, source, namespace, statics):
-        super(CatalogMapping, self).__init__(source, namespace, statics)
+    def __init__(self, source, namespace, constants):
+        super(CatalogMapping, self).__init__(source, namespace, constants)
         self.mapping = self.createMapping()
 
     def createMapping(self):

mappings/sql.py

@@ -5,8 +5,8 @@
 
 
 class SQLMapping(BaseMapping):
-    def __init__(self, source, statics):
-        super().__init__(source, statics)
+    def __init__(self, source, constants):
+        super().__init__(source, constants)
 
     def applyMapping(self):
         newRecord = self.initEmptyRecord()

mappings/xml.py

@@ -8,8 +8,8 @@
 
 
 class XMLMapping(BaseMapping):
-    def __init__(self, source, namespace, statics):
-        super().__init__(source, statics)
+    def __init__(self, source, namespace, constants):
+        super().__init__(source, constants)
         self.namespace = namespace
 
     def applyMapping(self):

processes/cluster.py

@@ -3,12 +3,12 @@
 from sqlalchemy.exc import DataError
 from typing import Optional
 
+from constants.get_constants import get_constants
 from .core import CoreProcess
 from managers import SFRRecordManager, KMeansManager, SFRElasticRecordManager, ElasticsearchManager, RedisManager
 from model import Record, Work
 from logger import create_log
 
-
 logger = create_log(__name__)
 
 
@@ -35,6 +35,8 @@ def __init__(self, *args):
         self.elastic_search_manager.createElasticSearchIngestPipeline()
         self.elastic_search_manager.createElasticSearchIndex()
 
+        self.constants = get_constants()
+
     def runProcess(self):
         try:
             if self.process == 'daily':
@@ -219,7 +221,7 @@ def get_matched_records(self, identifiers: list[str], already_matched_record_ids
         return matched_records
 
     def create_work_from_editions(self, editions, records):
-        record_manager = SFRRecordManager(self.session, self.statics['iso639'])
+        record_manager = SFRRecordManager(self.session, self.constants['iso639'])
 
         work_data = record_manager.buildWork(records, editions)
 

processes/core.py

@@ -1,14 +1,13 @@
 from managers import DBManager, S3Manager
 from model import Record
-from static.manager import StaticManager
 
 from logger import create_log
 
 
 logger = create_log(__name__)
 
 
-class CoreProcess(DBManager, StaticManager, S3Manager):
+class CoreProcess(DBManager, S3Manager):
     def __init__(self, process, customFile, ingestPeriod, singleRecord, batchSize=500):
         super(CoreProcess, self).__init__()
         self.process = process

processes/ingest/doab.py

@@ -4,6 +4,7 @@
 import os
 import requests
 
+from constants.get_constants import get_constants
 from ..core import CoreProcess
 from logger import create_log
 from mappings.doab import DOABMapping
@@ -44,6 +45,8 @@ def __init__(self, *args):
         self.rabbitmq_manager.createRabbitConnection()
         self.rabbitmq_manager.createOrConnectQueue(self.fileQueue, self.fileRoute)
 
+        self.constants = get_constants()
+
     def runProcess(self):
         if self.process == 'daily':
             self.importOAIRecords()
@@ -61,7 +64,7 @@ def runProcess(self):
 
     def parseDOABRecord(self, oaiRec):
         try:
-            doabRec = DOABMapping(oaiRec, self.OAI_NAMESPACES, self.statics)
+            doabRec = DOABMapping(oaiRec, self.OAI_NAMESPACES, self.constants)
             doabRec.applyMapping()
         except MappingError as e:
             raise DOABError(e.message)

processes/ingest/gutenberg.py

@@ -4,6 +4,7 @@
 import os
 import re
 
+from constants.get_constants import get_constants
 from ..core import CoreProcess
 from managers import GutenbergManager, RabbitMQManager
 from mappings.gutenberg import GutenbergMapping
@@ -41,6 +42,8 @@ def __init__(self, *args):
 
         self.s3Bucket = os.environ['FILE_BUCKET']
 
+        self.constants = get_constants()
+
     def runProcess(self):
         if self.process == 'daily':
             self.importRDFRecords()
@@ -89,7 +92,7 @@ def processGutenbergBatch(self, dataFiles):
             gutenbergRec = GutenbergMapping(
                 gutenbergRDF,
                 self.GUTENBERG_NAMESPACES,
-                self.statics
+                self.constants
             )
 
             gutenbergRec.applyMapping()

processes/ingest/hathi_trust.py

@@ -6,6 +6,7 @@
 import requests
 from requests.exceptions import ReadTimeout, HTTPError
 
+from constants.get_constants import get_constants
 from ..core import CoreProcess
 from mappings.hathitrust import HathiMapping
 from logger import create_log
@@ -23,6 +24,8 @@ def __init__(self, *args):
         self.generateEngine()
         self.createSession()
 
+        self.constants = get_constants()
+
     def runProcess(self):
         if self.process == 'daily':
             date_time = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=24)
@@ -47,7 +50,7 @@ def importFromSpecificFile(self, file_path):
         self.readHathiFile(hathiReader)
 
     def parseHathiDataRow(self, data_row):
-        hathiRec = HathiMapping(data_row, self.statics)
+        hathiRec = HathiMapping(data_row, self.constants)
         hathiRec.applyMapping()
         self.addDCDWToUpdateList(hathiRec)
 

processes/local_development/seed_local_data.py

@@ -4,6 +4,7 @@
 import os
 import requests
 
+from constants.get_constants import get_constants
 from ..core import CoreProcess
 from logger import create_log
 from managers import RedisManager
@@ -18,6 +19,7 @@ def __init__(self, *args):
         super(SeedLocalDataProcess, self).__init__(*args[:4])
 
         self.redis_manager = RedisManager()
+        self.constants = get_constants()
 
     def runProcess(self):
         try:
@@ -86,7 +88,7 @@ def import_from_hathi_trust_data_file(self):
                 book_right = book[2]
 
                 if book_right not in in_copyright_statuses:
-                    hathi_record = HathiMapping(book, self.statics)
+                    hathi_record = HathiMapping(book, self.constants)
                     hathi_record.applyMapping()
 
                     self.addDCDWToUpdateList(hathi_record)

services/sources/nypl_bib_service.py

@@ -3,9 +3,9 @@
 import requests
 from typing import Optional
 
+from constants.get_constants import get_constants
 from logger import create_log
 from managers.db import DBManager
-from static.manager import StaticManager
 from managers.nyplApi import NyplApiManager
 from mappings.nypl import NYPLMapping
 from .source_service import SourceService
@@ -32,7 +32,7 @@ def __init__(self):
         self.location_codes = self.load_location_codes()
         self.cce_api = os.environ['BARDO_CCE_API']
 
-        self.static_manager = StaticManager()
+        self.constants = get_constants()
 
     def get_records(
         self,
@@ -79,7 +79,7 @@ def parse_nypl_bib(self, bib) -> Optional[NYPLMapping]:
             if self.is_pd_research_bib(dict(bib)):
                 bib_items = self.fetch_bib_items(dict(bib))
 
-                nypl_record = NYPLMapping(bib, bib_items, self.static_manager.statics, self.location_codes)
+                nypl_record = NYPLMapping(bib, bib_items, self.constants, self.location_codes)
                 nypl_record.applyMapping()
 
                 return nypl_record