SFR-2280: Limit Hathi Records Ingested

processes/ingest/hathi_trust.py

@@ -1,5 +1,5 @@
 import csv
-from datetime import datetime
+from datetime import datetime, timedelta, timezone
 from io import BytesIO
 import gzip
 import os
@@ -72,7 +72,15 @@ def importFromHathiTrustDataFile(self, fullDump=False):
         )
 
         for hathiFile in fileJSON:
+            print(hathiFile)
             if hathiFile['full'] == fullDump:
+                if hathiFile['full'] == False:
+                    start_date_time = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(hours=24)
+                    hathi_date_modified = datetime.strptime(hathiFile['modified'],self.returnHathiDateFormat(hathiFile['modified'])).replace(tzinfo=None)
+                    if hathi_date_modified > start_date_time:
+                        self.importFromHathiFile(hathiFile['url'])
+                        break       
+
                 self.importFromHathiFile(hathiFile['url'])
                 break
 
@@ -99,23 +107,14 @@ def importFromHathiFile(self, hathiURL):
             self.readHathiFile(hathiTSV)
 
     def readHathiFile(self, hathiTSV):
-        processed_record_count = 0
-
-        while True:
-            if self.ingest_limit and processed_record_count >= self.ingest_limit:
+        for number_of_books_ingested, book in enumerate(hathiTSV):
+            if self.ingest_limit and number_of_books_ingested > self.ingest_limit:
                 break
 
-            try:
-                row = next(hathiTSV)
-                logger.info('Reading row')
-            except csv.Error as e:
-                logger.warning('Unable to read TSV row')
-                logger.debug(e)
-                continue
-            except StopIteration:
-                logger.info('Reached end of TSV file')
-                break
+            book_right = book[2]
+            book_date_updated = book[14]
+            start_date_time = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=1) 
+            hathi_date_modified = datetime.strptime(book[14],'%Y-%m-%d %H:%M:%S')
 
-            if row is not None and row[2] not in self.HATHI_RIGHTS_SKIPS:
-                self.parseHathiDataRow(row)
-                processed_record_count += 1
+            if book_right is not None and book_right not in self.HATHI_RIGHTS_SKIPS and hathi_date_modified > start_date_time:
+                self.parseHathiDataRow(book)

tests/unit/processes/ingest/test_hathi_trust_process.py

@@ -30,19 +30,26 @@ def __init__(self, process, customFile, ingestPeriod):
     @pytest.fixture
     def hathiFilesData(self):
         return [
-            {'created': '2020-01-01T00:00:00-0000', 'url': 'hathiUrl1', 'full': False},
+            {'created': '2020-01-01T00:00:00-0000', 'url': 'hathiUrl1', 'full': False, 'modified': '2024-10-28 11:00:00 Z'},
             {'created': '2019-01-01T00:00:00-0000', 'url': 'hathiUrl2', 'full': True},
             {'created': '2018-01-01T00:00:00-0000', 'url': 'hathiUrl3', 'full': False}
         ]
 
     @pytest.fixture
     def hathiTSV(self):
         def tsvIter():
-            for i in range(1000):
+            for i in range(200):
                 rightsStmt = 'ic' if i % 3 == 0 else 'pd'
                 yield [i, 'hathi', rightsStmt]
-
-            raise csv.Error
+
+        return tsvIter()
+
+    @pytest.fixture
+    def hathiTSV2(self):
+        def tsvIter():
+            for i in range(200):
+                rightsStmt = 'ic' if i % 3 == 0 else 'pd'
+                yield [i, 'hathi', rightsStmt, '', '', '', '', '', '', '', '', '', '', '', '2024-10-28 12:00:00']
 
         return tsvIter()
 
@@ -180,9 +187,9 @@ def test_importFromHathiFile_error(self, testInstance, mocker):
 
         assert testInstance.importFromHathiFile('badURL') == None
 
-    def test_readHathiFile(self, testInstance, hathiTSV, mocker):
+    def test_readHathiFile(self, testInstance, hathiTSV2, mocker):
         mockParseRow = mocker.patch.object(HathiTrustProcess, 'parseHathiDataRow')
 
-        testInstance.readHathiFile(hathiTSV)
+        testInstance.readHathiFile(hathiTSV2)
 
-        assert mockParseRow.call_count == 666
+        assert mockParseRow.call_count == 133