Consolidate list page endpoints + better QA sorting + optimize pages …

…fix (#2417)

- consolidate list_pages() and list_replay_query_pages() into
list_pages()
- to keep backwards compatibility, add <crawl>/pagesSearch that does not
include page totals, keep <crawl>/pages with page total (slower)
- qa frontend: add default 'Crawl Order' sort order, to better show
pages in QA view
- bgjob: account for parallelism in bgjobs, add logging if succeeded
mismatches parallelism
- QA sorting: default to 'crawl order' by default to get better results.
- Optimize pages job: also cover crawls that may not have any pages but have pages listed in done stats
- Bgjobs: give custom op jobs more memory

backend/btrixcloud/basecrawls.py

@@ -171,14 +171,15 @@ async def get_crawl_out(
                 res["collections"] = await self.colls.get_collection_names(coll_ids)
 
             if res.get("version", 1) == 2:
-                res["initialPages"] = await self.page_ops.list_replay_query_pages(
+                res["initialPages"], _ = await self.page_ops.list_pages(
                     crawl_ids=[crawlid], is_seed=True, page_size=25
                 )
 
                 oid = res.get("oid")
                 if oid:
                     res["pagesQueryUrl"] = (
-                        get_origin(headers) + f"/api/orgs/{oid}/crawls/{crawlid}/pages"
+                        get_origin(headers)
+                        + f"/api/orgs/{oid}/crawls/{crawlid}/pagesSearch"
                     )
 
         crawl = CrawlOutWithResources.from_dict(res)

backend/btrixcloud/colls.py

@@ -42,7 +42,6 @@
     OrgPublicCollections,
     PublicOrgDetails,
     CollAccessType,
-    PageOut,
     UpdateCollHomeUrl,
     User,
     ImageFile,
@@ -346,8 +345,7 @@ async def get_collection_out(
                 await self.get_collection_crawl_resources(coll_id)
             )
 
-            initial_pages: List[PageOut] = await self.page_ops.list_replay_query_pages(
-                coll_id,
+            initial_pages, _ = await self.page_ops.list_pages(
                 crawl_ids=crawl_ids,
                 page_size=25,
             )

backend/btrixcloud/crawlmanager.py

@@ -198,6 +198,7 @@ async def _run_bg_job_with_ops_classes(
             "job_type": job_type,
             "backend_image": os.environ.get("BACKEND_IMAGE", ""),
             "pull_policy": os.environ.get("BACKEND_IMAGE_PULL_POLICY", ""),
+            "larger_resources": True,
             **kwargs,
         }
         if oid:

backend/btrixcloud/operator/bgjobs.py

@@ -38,7 +38,12 @@ async def finalize_background_job(self, data: MCDecoratorSyncData) -> dict:
         job_id: str = labels.get("job_id") or metadata.get("name")
 
         status = data.object["status"]
-        success = status.get("succeeded") == 1
+        spec = data.object["spec"]
+        success = status.get("succeeded") == spec.get("parallelism")
+        if not success:
+            print(
+                "Succeeded: {status.get('succeeded')}, Num Pods: {spec.get('parallelism')}"
+            )
         completion_time = status.get("completionTime")
 
         finalized = True