normalize URLs

reworkd · May 20, 2024 · 5c4f1d4 · 5c4f1d4
1 parent 7c12def
commit 5c4f1d4
Showing 1 changed file with 5 additions and 4 deletions.
diff --git a/harambe/core.py b/harambe/core.py
@@ -35,8 +35,7 @@ class AsyncScraper(Protocol):
     Note that scrapers in harambe should be functions, not classes.
     """
 
-    async def scrape(self, sdk: "SDK", url: URL, context: Context) -> None:
-        ...
+    async def scrape(self, sdk: "SDK", url: URL, context: Context) -> None: ...
 
 
 class SDK:
@@ -105,7 +104,8 @@ async def enqueue(self, *urls: URL, context: Optional[Context] = None) -> None:
         context["__url"] = self.page.url
 
         for url in urls:
-            await self._notify_observers("on_queue_url", url, context)
+            normalized_url = normalize_url(url, self.page.url) if hasattr(self.page, "url") else url
+            await self._notify_observers("on_queue_url", normalized_url, context)
 
     async def paginate(
         self,
@@ -337,7 +337,8 @@ async def run_from_file(
                 if setup:
                     await setup(sdk)
                 await page.goto(listing["url"])
-                await scraper(sdk,
+                await scraper(
+                    sdk,
                     listing["url"],
                     listing["context"],
                 )