Linting and fixups

backend/btrixcloud/migrations/migration_0042_page_filenames.py

@@ -25,20 +25,14 @@ async def migrate_up(self):
         """
         pages_mdb = self.mdb["pages"]
 
-        crawl_ids_to_update = set()
-
         if self.page_ops is None:
             print(
                 "Unable to add filename to pages, missing page_ops",
                 flush=True,
             )
             return
 
-        async for page_raw in pages_mdb.find({"filename": None}):
-            crawl_id = page_raw.get("crawl_id")
-            if crawl_id:
-                crawl_ids_to_update.add(crawl_id)
-
+        crawl_ids_to_update = await pages_mdb.distinct("crawl_id", {"filename": None})
         for crawl_id in crawl_ids_to_update:
             try:
                 await self.page_ops.add_crawl_wacz_filename_to_pages(crawl_id)

backend/btrixcloud/pages.py

@@ -105,29 +105,34 @@ async def add_crawl_wacz_filename_to_pages(self, crawl_id: str):
         """Add WACZ filename to existing pages in crawl if not already set"""
         try:
             crawl = await self.crawl_ops.get_crawl_out(crawl_id)
-            for wacz_file in crawl.resources:
+            if not crawl.resources:
+                return
 
+            for wacz_file in crawl.resources:
+                # Strip oid directory from filename
                 filename = wacz_file.name
                 name_parts = wacz_file.name.split("/")
                 if name_parts and len(name_parts) > 1:
                     filename = name_parts[-1]
 
-                wacz_page_ids = []
+                page_ids_to_update = []
 
                 stream = await self.storage_ops.sync_stream_wacz_pages([wacz_file])
                 for page_dict in stream:
                     if not page_dict.get("url"):
                         continue
 
-                    if page_dict.get("filename"):
-                        continue
-
-                    if page_dict.get("id"):
-                        wacz_page_ids.append(page_dict["id"])
+                    page_id = page_dict.get("id")
+                    if page_id:
+                        try:
+                            page_ids_to_update.append(UUID(page_id))
+                        # pylint: disable=broad-exception-caught
+                        except Exception:
+                            continue
 
                 # Update pages in batch per-filename
                 await self.pages.update_many(
-                    {"_id": {"$in": wacz_page_ids}},
+                    {"_id": {"$in": page_ids_to_update}},
                     {"$set": {"filename": filename}},
                 )
         # pylint: disable=broad-exception-caught, raise-missing-from