Avoid browser fingerprinting

griptape/drivers/web_scraper/markdownify_web_scraper_driver.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import re
+from random import randint
 from typing import Any, Optional
 
 from attrs import Factory, define, field
@@ -40,6 +41,7 @@ class MarkdownifyWebScraperDriver(BaseWebScraperDriver):
 
     def scrape_url(self, url: str) -> TextArtifact:
         sync_playwright = import_optional_dependency("playwright.sync_api").sync_playwright
+        fake_useragent = import_optional_dependency("fake_useragent")
         bs4 = import_optional_dependency("bs4")
         markdownify = import_optional_dependency("markdownify")
 
@@ -54,7 +56,43 @@ def convert_a(self, el: Any, text: str, convert_as_inline: Any) -> str:
                 return text
 
         with sync_playwright() as p, p.chromium.launch(headless=True) as browser:
-            page = browser.new_page()
+            # Randomize user agent to help prevent fingerprinting
+            user_agent = fake_useragent.UserAgent().random
+
+            # Randomize viewport size to help prevent fingerprinting
+            viewport = {"width": randint(1024, 1920), "height": randint(768, 1080)}
+
+            context = browser.new_context(user_agent=user_agent, viewport=viewport)
+
+            # Disable WebRTC to prevent IP leaks
+            context.add_init_script("""
+            Object.defineProperty(navigator, 'mediaDevices', {
+                value: {
+                    getUserMedia: () => Promise.reject(new Error('Not allowed')),
+                },
+                configurable: True,
+            });
+            """)
+
+            # Prevent canvas fingerprinting
+            context.add_init_script("""
+            HTMLCanvasElement.prototype.toDataURL = () => "data:image/png;base64,spoofedData";
+            HTMLCanvasElement.prototype.getImageData = function(sx, sy, sw, sh) {
+                const data = CanvasRenderingContext2D.prototype.getImageData.call(this, sx, sy, sw, sh);
+                for (let i = 0; i < data.data.length; i += 4) data.data[i] ^= 0xFF; // Invert colors
+                return data;
+            };
+            """)
+
+            # Add random plugins to prevent fingerprinting
+            context.add_init_script(f"""
+            Object.defineProperty(navigator, 'plugins', {{
+                get: () => {self._random_js_plugin_array(user_agent)},
+                configurable: True,
+            }});
+            """)
+
+            page = context.new_page()
 
             def skip_loading_images(route: Any) -> Any:
                 if route.request.resource_type == "image":
@@ -101,3 +139,41 @@ def skip_loading_images(route: Any) -> Any:
             text = re.sub(r"\n\n+", "\n\n", text)
 
             return TextArtifact(text)
+
+    def _random_js_plugin_array(self, user_agent: str) -> str:
+        faker = import_optional_dependency("faker")
+        fake = faker.Faker()
+        num_plugins = randint(0, 5)
+        extension = self._get_os_extension(user_agent)
+        plugins = []
+        for _ in range(num_plugins):
+            plugins.append(
+                "".join(
+                    [
+                        "{",
+                        ", ".join(
+                            [
+                                f'{k}: "{v}"'
+                                for k, v in {
+                                    "name": f"{fake.word().capitalize()} Plugin",
+                                    "description": f"{fake.catch_phrase()} Description",
+                                    "filename": f"{fake.file_name(extension=extension)}",
+                                }.items()
+                            ]
+                        ),
+                        "}",
+                    ]
+                )
+            )
+
+        return f"[{', '.join(plugins)}]"
+
+    def _get_os_extension(self, user_agent: str) -> str:
+        if "Windows" in user_agent:
+            return "dll"
+        elif "Macintosh" in user_agent:
+            return "dylib"
+        elif "Linux" in user_agent:
+            return "so"
+        else:
+            return "plugin"  # Default fallback

pyproject.toml

@@ -45,6 +45,8 @@ psycopg2-binary = { version = "^2.9.9", optional = true }
 google-generativeai = { version = "^0.7.2", optional = true }
 trafilatura = {version = "^1.6", optional = true}
 playwright = {version = "^1.42", optional = true}
+fake-useragent = {version = "^1.5.1", optional = true}
+faker = {version = "^27.4.0", optional = true}
 beautifulsoup4 = {version = "^4.12.3", optional = true}
 markdownify = {version = "^0.11.6", optional = true}
 voyageai = {version = "^0.2.1", optional = true}
@@ -108,7 +110,7 @@ drivers-embedding-cohere = ["cohere"]
 drivers-embedding-ollama = ["ollama"]
 
 drivers-web-scraper-trafilatura = ["trafilatura"]
-drivers-web-scraper-markdownify = ["playwright", "beautifulsoup4", "markdownify"]
+drivers-web-scraper-markdownify = ["playwright", "fake-useragent", "faker", "beautifulsoup4", "markdownify"]
 
 drivers-web-search-duckduckgo = ["duckduckgo-search"]
 
@@ -179,6 +181,8 @@ all = [
     "google-generativeai",
     "trafilatura",
     "playwright",
+    "fake-useragent",
+    "faker",
     "beautifulsoup4",
     "markdownify",
     "voyageai",

tests/unit/drivers/web_scraper/test_markdownify_web_scraper_driver.py

@@ -12,7 +12,7 @@ def mock_playwright(self, mocker):
 
     @pytest.fixture(autouse=True)
     def mock_content(self, mock_playwright):
-        mock_content = mock_playwright.__enter__.return_value.chromium.launch.return_value.__enter__.return_value.new_page.return_value.content
+        mock_content = mock_playwright.__enter__.return_value.chromium.launch.return_value.__enter__.return_value.new_context.return_value.new_page.return_value.content
         mock_content.return_value = '<html><a href="foobar.com">foobar</a></html>'
         return mock_content