SFR-2217 Fixing MUSE Mapping

managers/muse.py

@@ -68,22 +68,22 @@ def identifyReadableVersions(self):
 
     def addReadableLinks(self):
         if self.pdfDownloadURL:
-            self.record.addHasPartLink(
+            self.record.add_has_part_link(
                 self.pdfDownloadURL,
                 'application/pdf',
                 json.dumps({'download': True, 'reader': False, 'catalog': False})
             )
 
         if self.epubURL:
             self.s3EpubPath = 'epubs/muse/{}.epub'.format(self.museID)
-            self.record.addHasPartLink(
+            self.record.add_has_part_link(
                 self.constructS3Link(self.s3EpubPath),
                 'application/epub+zip',
                 json.dumps({'download': True, 'reader': False, 'catalog': False})
             )
 
             self.epubReadPath = 'epubs/muse/{}/manifest.json'.format(self.museID)
-            self.record.addHasPartLink(
+            self.record.add_has_part_link(
                 self.constructS3Link(self.epubReadPath),
                 'application/webpub+json',
                 json.dumps({'download': False, 'reader': True, 'catalog': False})
@@ -92,7 +92,7 @@ def addReadableLinks(self):
         self.constructWebpubManifest()
         if self.pdfWebpubManifest:
             self.s3PDFReadPath = 'manifests/muse/{}.json'.format(self.museID)
-            self.record.addHasPartLink(
+            self.record.add_has_part_link(
                 self.createManifestInS3(self.s3PDFReadPath),
                 'application/webpub+json',
                 json.dumps({'reader': True, 'download': False, 'catalog': False})

mappings/muse.py

@@ -1,5 +1,8 @@
 from mappings.marc import MARCMapping
 
+DEFAULT_PUBLISHER = 'John Hopkins University Press||'
+
+
 class MUSEMapping(MARCMapping):
     def __init__(self, source):
         super(MUSEMapping, self).__init__(source, {})
@@ -68,54 +71,69 @@ def applyFormatting(self):
         # Take first title as they are in order of preference
         self.record.title = self.record.title[0]
 
-        # Extract language code from 008 fixed data field
-        self.record.languages = [self.extractLanguage(l) for l in self.record.languages]
+        self.record.identifiers = [self.cleanup_identifier(id) for id in self.record.identifiers]
+
+        self.record.languages = [
+            extracted_langauge
+            for language in self.record.languages
+            if (extracted_langauge := self.extract_language(language))
+        ]
 
-        # Extract publication date from 008 fixed field if 264 field is missing
         if len(self.record.dates) < 1:
-            pubDate = self.source['008'].data[11:15]
-            self.record.dates.append('{}|publication_date'.format(pubDate))
-
-        # If publisher missing, assume JHU
+            publication_date = self.source['008'].data[11:15]
+            self.record.dates.append('{}|publication_date'.format(publication_date))
+
         if len(self.record.publisher) < 1:
-            self.record.publisher.append('John Hopkins University Press||')
+            self.record.publisher.append(DEFAULT_PUBLISHER)
 
-        # Clean up subjects to remove spots for missing subheadings
-        self.record.subjects = [
-            self.cleanUpSubjectHead(s)
-            for s in self.record.subjects
-        ]
+        self.record.subjects = [self.clean_up_subject_head(subject) for subject in self.record.subjects]
 
-        # Add Rights statement
         self.record.rights = '{}|{}||{}|'.format(
-            'muse', 'https://creativecommons.org/licenses/by-nc/4.0/',
+            'muse', 
+            'https://creativecommons.org/licenses/by-nc/4.0/',
             'Creative Commons Attribution-NonCommercial 4.0 International'
         )
 
-    def cleanUpSubjectHead(self, subject):
-        subjectStr, *subjectMeta = subject.split('|')
-        subjectParts = subjectStr.split('--')
+    def clean_up_subject_head(self, subject):
+        subject_str, *subject_metadata = subject.split('|')
+        subject_parts = subject_str.split('--')
 
-        outParts = []
+        out_parts = []
 
-        for part in subjectParts:
-            cleanPart = part.strip(' .')
+        for part in subject_parts:
+            clean_parts = part.strip(' .')
 
-            if cleanPart == '': continue
+            if clean_parts == '': continue
 
-            outParts.append(cleanPart)
+            out_parts.append(clean_parts)
 
-        cleanSubject = ' -- '.join([p for p in outParts])
+        cleaned_subject = ' -- '.join([part for part in out_parts])
 
-        return '|'.join([cleanSubject] + subjectMeta)
+        return '|'.join([cleaned_subject] + subject_metadata)
 
-    def extractLanguage(self, language):
-        _, _, marcData, *_ = language.split('|')
-        return '||{}'.format(marcData[35:38])
+    def extract_language(self, language):
+        _, _, marc_data, *_ = language.split('|')
+        marc_data = marc_data.split(' ')
 
-    def addHasPartLink(self, url, mediaType, flags):
-        lastItemNo = int(self.record.has_part[-1][0])
+        # MARC data example: 100607s2011 mdu o 00 0 eng d
+        if len(marc_data) >= 7:
+            return f'||{marc_data[5]}'
+
+        return None
+
+    def add_has_part_link(self, url, media_type, flags):
+        last_item_no = int(self.record.has_part[-1][0])
 
         self.record.has_part.append(
-            '{}|{}|muse|{}|{}'.format(lastItemNo, url, mediaType, flags)
+            '{}|{}|muse|{}|{}'.format(last_item_no, url, media_type, flags)
         )
+
+    def cleanup_identifier(self, identifier):
+        oclc_number_prefix = '(OCoLC)'
+        id, id_type = identifier.split('|')
+        id = id.strip()
+
+        if id.startswith(oclc_number_prefix):
+            return f'{id[len(oclc_number_prefix):]}|{id_type}'
+
+        return f'{id}|{id_type}'

tests/unit/test_muse_manager.py

@@ -116,7 +116,7 @@ def test_addReadableLinks_pdf(self, testManager, mocker):
 
         testManager.addReadableLinks()
 
-        testManager.record.addHasPartLink.assert_called_once_with(
+        testManager.record.add_has_part_link.assert_called_once_with(
             'testPDFURL', 'application/pdf', '{"download": true, "reader": false, "catalog": false}'
         )
         mockConstruct.assert_called_once()
@@ -130,7 +130,7 @@ def test_addReadableLinks_epub(self, testManager, mocker):
 
         testManager.addReadableLinks()
 
-        testManager.record.addHasPartLink.assert_has_calls([
+        testManager.record.add_has_part_link.assert_has_calls([
             mocker.call('epubDownloadURL', 'application/epub+zip', '{"download": true, "reader": false, "catalog": false}'),
             mocker.call('epubReadURL', 'application/webpub+json', '{"download": false, "reader": true, "catalog": false}')
         ])
@@ -149,7 +149,7 @@ def test_addReadableLinks_manifest(self, testManager, mocker):
 
         testManager.addReadableLinks()
 
-        testManager.record.addHasPartLink.assert_called_once_with(
+        testManager.record.add_has_part_link.assert_called_once_with(
             'webpubReadURL', 'application/webpub+json', '{"reader": true, "download": false, "catalog": false}'
         )
         mockS3.assert_called_once_with('manifests/muse/1.json')

tests/unit/test_muse_mapping.py

@@ -1,68 +1,76 @@
 import pytest
 
 from mappings.muse import MUSEMapping
+from model import Record
 
+test_source = { 
+    '008': type('data-object', (object,), { 'data' : 'testingdate2000pla' })
+}
 
-class TestMUSEMapping:
-    @pytest.fixture
-    def testRecord(self, mocker):
-        mockRecord = mocker.MagicMock()
-        mockRecord.identifiers = ['1|muse', '2|test', '3|other']
-        mockRecord.title = ['Main Title', 'Secondary Title']
-        mockRecord.subjects = ['subj1', 'subj2', 'subj3']
-        mockRecord.has_part = ['1|testURL|muse|testType|testFlags']
-        mockRecord.languages = ['||lang1', '||lang2']
-        mockRecord.dates = []
+def test_create_mapping():
+    muse_mapping = MUSEMapping(test_source)
+
+    assert set([
+        'identifiers', 'authors', 'title', 'alternative', 'has_version',
+        'publisher', 'spatial', 'dates', 'languages', 'extent',
+        'table_of_contents', 'abstract', 'subjects', 'contributors',
+        'is_part_of', 'has_part'
+    ]).issubset(set(muse_mapping.mapping.keys()))
+    assert muse_mapping.mapping['is_part_of'] == ('490', '{a}|{v}|volume')
 
-        return mockRecord
 
-    @pytest.fixture
-    def testMapping(self, testRecord, mocker):
-        class TestMapping(MUSEMapping):
-            def __init__(self):
-                self.mapping = None
-                self.record = testRecord
-                self.source = {'008': mocker.MagicMock(data='testingdate2000pla')}
-
-        return TestMapping()
+def test_apply_formatting():
+    muse_mapping = MUSEMapping(test_source)
+    record = Record()
+    record.identifiers = ['1|muse', '2|test', '3|other', '(OCoLC)123|oclc']
+    record.title = ['Main Title', 'Secondary Title']
+    record.subjects = ['subj1', 'subj2', 'subj3']
+    record.has_part = ['1|testURL|muse|testType|testFlags']
+    record.languages = ['||100607s2011 mdu o 00 0 lng1 d', '||100607s2011 mdu o 00 0 lng2 d']
+    record.publisher = []
+    record.dates = []
+    muse_mapping.record = record
 
-    def test_createMapping(self, testMapping):
-        recordMapping = testMapping.createMapping()
+    muse_mapping.applyFormatting()
 
-        assert list(recordMapping.keys()) == [
-            'identifiers', 'authors', 'title', 'alternative', 'has_version',
-            'publisher', 'spatial', 'dates', 'languages', 'extent',
-            'table_of_contents', 'abstract', 'subjects', 'contributors',
-            'is_part_of', 'has_part'
-        ]
-        assert recordMapping['is_part_of'] == ('490', '{a}|{v}|volume')
+    assert muse_mapping.record.source == 'muse'
+    assert muse_mapping.record.source_id == '1'
+    assert muse_mapping.record.title == 'Main Title'
+    assert muse_mapping.record.identifiers == ['1|muse', '2|test', '3|other', '123|oclc']
+    assert muse_mapping.record.subjects == ['subj1', 'subj2', 'subj3']
+    assert muse_mapping.record.languages == ['||lng1', '||lng2']
+    assert muse_mapping.record.dates[0] == '2000|publication_date'
 
-    def test_applyFormatting(self, testMapping, mocker):
 
-        mockCleanSubject = mocker.patch.object(MUSEMapping, 'cleanUpSubjectHead')
-        mockCleanSubject.side_effect = [1, 2, 3]
+def test_clean_up_subject_head():
+    muse_mapping = MUSEMapping(test_source)
+
+    cleaned_subject = muse_mapping.clean_up_subject_head('first -- second. -- -- |||')
 
-        mockExtractLanguage = mocker.patch.object(MUSEMapping, 'extractLanguage')
-        mockExtractLanguage.side_effect = ['lng1', 'lng2']
+    assert cleaned_subject == 'first -- second|||'
 
-        testMapping.applyFormatting()
 
-        assert testMapping.record.source == 'muse'
-        assert testMapping.record.source_id == '1'
-        assert testMapping.record.title == 'Main Title'
-        assert testMapping.record.subjects == [1, 2, 3]
-        assert testMapping.record.languages == ['lng1', 'lng2']
-        assert testMapping.record.dates[0] == '2000|publication_date'
+def test_extract_language():
+    muse_mapping = MUSEMapping(test_source)
 
-    def test_cleanUpSubjectHead(self, testMapping):
-        cleanSubject = testMapping.cleanUpSubjectHead('first -- second. -- -- |||')
+    extracted_language = muse_mapping.extract_language('||100607s2011 mdu o 00 0 eng d')
 
-        assert cleanSubject == 'first -- second|||'
+    assert extracted_language == '||eng'
 
-    def test_extractLanguage(self, testMapping):
-        assert testMapping.extractLanguage('||100607s2011    mdu     o      00 0 eng d  z  ') == '||eng'
 
-    def test_addHasPartLink(self, testMapping):
-        testMapping.addHasPartLink('newURL', 'pdf+json', 'pdfFlags')
+def test_cleanup_identifier():
+    muse_mapping = MUSEMapping(test_source)
 
-        assert testMapping.record.has_part[1] == '1|newURL|muse|pdf+json|pdfFlags'
+    cleaned_identifier = muse_mapping.cleanup_identifier('(OCoLC)1223|oclc')
+
+    assert cleaned_identifier == '1223|oclc'
+
+
+def test_add_has_part_link():
+    muse_mapping = MUSEMapping(test_source)
+    muse_mapping.record = Record()
+    muse_mapping.record.has_part = ['1|test_url|muse|epub|flags']
+
+    muse_mapping.add_has_part_link('newURL', 'pdf+json', 'pdfFlags')
+
+    assert muse_mapping.record.has_part[1] == '1|newURL|muse|pdf+json|pdfFlags'