Merge pull request #4061 from seleniumbase/a-few-fixes-for-chrome-142-and-more

A few fixes for Chrome 142 and more

Author: mdmintz

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.44.2"
+__version__ = "4.44.3"

seleniumbase/core/browser_launcher.py

@@ -293,7 +293,17 @@ def extend_driver(
         )
     if hasattr(driver, "proxy"):
         driver.set_wire_proxy = DM.set_wire_proxy
+    completed_loads = []
+    for ext_dir in sb_config._ext_dirs:
+        with suppress(Exception):
+            if ext_dir not in completed_loads:
+                completed_loads.append(ext_dir)
+                if not use_uc and os.path.exists(os.path.abspath(ext_dir)):
+                    driver.webextension.install(os.path.abspath(ext_dir))
     if proxy_auth:
+        with suppress(Exception):
+            if not use_uc and os.path.exists(proxy_helper.PROXY_DIR_PATH):
+                driver.webextension.install(proxy_helper.PROXY_DIR_PATH)
         # Proxy needs a moment to load in Manifest V3
         if use_uc:
             time.sleep(0.14)
@@ -2087,6 +2097,7 @@ def _add_chrome_proxy_extension(
     """Implementation of https://stackoverflow.com/a/35293284/7058266
     for https://stackoverflow.com/q/12848327/7058266
     (Run Selenium on a proxy server that requires authentication.)"""
+    zip_it = False
     args = " ".join(sys.argv)
     bypass_list = proxy_bypass_list
     if (
@@ -2467,13 +2478,27 @@ def _set_chrome_options(
         extension_zip_list = extension_zip.split(",")
         for extension_zip_item in extension_zip_list:
             abs_path = os.path.abspath(extension_zip_item)
-            chrome_options.add_extension(abs_path)
+            if os.path.exists(abs_path):
+                try:
+                    abs_path_dir = os.path.join(
+                        DOWNLOADS_FOLDER, abs_path.split(".")[0]
+                    )
+                    _unzip_to_new_folder(abs_path, abs_path_dir)
+                    chrome_options = add_chrome_ext_dir(
+                        chrome_options, abs_path_dir
+                    )
+                    sb_config._ext_dirs.append(abs_path_dir)
+                except Exception:
+                    with suppress(Exception):
+                        chrome_options.add_extension(abs_path)
     if extension_dir:
         # load-extension input can be a comma-separated list
         abs_path = (
             ",".join(os.path.abspath(p) for p in extension_dir.split(","))
         )
         chrome_options = add_chrome_ext_dir(chrome_options, abs_path)
+        for p in extension_dir.split(","):
+            sb_config._ext_dirs.append(os.path.abspath(p))
     if (
         page_load_strategy
         and page_load_strategy.lower() in ["eager", "none"]
@@ -2508,37 +2533,32 @@ def _set_chrome_options(
     if (settings.DISABLE_CSP_ON_CHROME or disable_csp) and not headless:
         # Headless Chrome does not support extensions, which are required
         # for disabling the Content Security Policy on Chrome.
-        if is_using_uc(undetectable, browser_name):
-            disable_csp_zip = DISABLE_CSP_ZIP_PATH
-            disable_csp_dir = os.path.join(DOWNLOADS_FOLDER, "disable_csp")
-            _unzip_to_new_folder(disable_csp_zip, disable_csp_dir)
-            chrome_options = add_chrome_ext_dir(
-                chrome_options, disable_csp_dir
-            )
-        else:
-            chrome_options = _add_chrome_disable_csp_extension(chrome_options)
+        disable_csp_zip = DISABLE_CSP_ZIP_PATH
+        disable_csp_dir = os.path.join(DOWNLOADS_FOLDER, "disable_csp")
+        _unzip_to_new_folder(disable_csp_zip, disable_csp_dir)
+        chrome_options = add_chrome_ext_dir(
+            chrome_options, disable_csp_dir
+        )
+        sb_config._ext_dirs.append(disable_csp_dir)
     if ad_block_on and not headless:
         # Headless Chrome does not support extensions.
-        if is_using_uc(undetectable, browser_name):
-            ad_block_zip = AD_BLOCK_ZIP_PATH
-            ad_block_dir = os.path.join(DOWNLOADS_FOLDER, "ad_block")
-            _unzip_to_new_folder(ad_block_zip, ad_block_dir)
-            chrome_options = add_chrome_ext_dir(chrome_options, ad_block_dir)
-        else:
-            chrome_options = _add_chrome_ad_block_extension(chrome_options)
+        ad_block_zip = AD_BLOCK_ZIP_PATH
+        ad_block_dir = os.path.join(DOWNLOADS_FOLDER, "ad_block")
+        _unzip_to_new_folder(ad_block_zip, ad_block_dir)
+        chrome_options = add_chrome_ext_dir(chrome_options, ad_block_dir)
+        sb_config._ext_dirs.append(ad_block_dir)
     if recorder_ext and not headless:
-        if is_using_uc(undetectable, browser_name):
-            recorder_zip = RECORDER_ZIP_PATH
-            recorder_dir = os.path.join(DOWNLOADS_FOLDER, "recorder")
-            _unzip_to_new_folder(recorder_zip, recorder_dir)
-            chrome_options = add_chrome_ext_dir(chrome_options, recorder_dir)
-        else:
-            chrome_options = _add_chrome_recorder_extension(chrome_options)
+        recorder_zip = RECORDER_ZIP_PATH
+        recorder_dir = os.path.join(DOWNLOADS_FOLDER, "recorder")
+        _unzip_to_new_folder(recorder_zip, recorder_dir)
+        chrome_options = add_chrome_ext_dir(chrome_options, recorder_dir)
+        sb_config._ext_dirs.append(recorder_dir)
     if chromium_arg and "sbase" in chromium_arg:
         sbase_ext_zip = SBASE_EXT_ZIP_PATH
         sbase_ext_dir = os.path.join(DOWNLOADS_FOLDER, "sbase_ext")
         _unzip_to_new_folder(sbase_ext_zip, sbase_ext_dir)
         chrome_options = add_chrome_ext_dir(chrome_options, sbase_ext_dir)
+        sb_config._ext_dirs.append(sbase_ext_dir)
     if proxy_string:
         if proxy_auth:
             zip_it = True
@@ -2724,6 +2744,10 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-features=%s" % d_f_string)
     if proxy_auth:
         chrome_options.add_argument("--test-type")
+    if proxy_auth or sb_config._ext_dirs:
+        if not is_using_uc(undetectable, browser_name):
+            chrome_options.enable_webextensions = True
+            chrome_options.enable_bidi = True
     if (
         is_using_uc(undetectable, browser_name)
         and (
@@ -2988,6 +3012,7 @@ def get_driver(
     device_pixel_ratio=None,
     browser=None,  # A duplicate of browser_name to avoid confusion
 ):
+    sb_config._ext_dirs = []
     driver_dir = DRIVER_DIR
     if (
         hasattr(sb_config, "binary_location")

seleniumbase/fixtures/base_case.py

@@ -4694,7 +4694,7 @@ def save_cookies(self, name="cookies.txt"):
         if not os.path.exists(file_path):
             os.makedirs(file_path)
         cookies_file_path = os.path.join(file_path, name)
-        cookies_file = open(cookies_file_path, "w+", encoding="utf-8")
+        cookies_file = open(cookies_file_path, mode="w+", encoding="utf-8")
         cookies_file.writelines(json_cookies)
         cookies_file.close()
 
@@ -5751,7 +5751,7 @@ def __process_recorded_actions(self):
         extra_file_name = "__init__.py"
         extra_file_path = os.path.join(recordings_folder, extra_file_name)
         if not os.path.exists(extra_file_path):
-            out_file = open(extra_file_path, "w+", "utf-8")
+            out_file = open(extra_file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             sys.stdout.write("\nCreated recordings%s__init__.py" % os.sep)
@@ -5799,7 +5799,7 @@ def __process_recorded_actions(self):
         extra_file_name = "pytest.ini"
         extra_file_path = os.path.join(recordings_folder, extra_file_name)
         if not os.path.exists(extra_file_path):
-            out_file = open(extra_file_path, "w+", "utf-8")
+            out_file = open(extra_file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             sys.stdout.write("\nCreated recordings%spytest.ini" % os.sep)
@@ -5820,7 +5820,7 @@ def __process_recorded_actions(self):
         extra_file_name = "setup.cfg"
         extra_file_path = os.path.join(recordings_folder, extra_file_name)
         if not os.path.exists(extra_file_path):
-            out_file = open(extra_file_path, "w+", "utf-8")
+            out_file = open(extra_file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             sys.stdout.write("\nCreated recordings%ssetup.cfg" % os.sep)
@@ -5838,7 +5838,7 @@ def __process_recorded_actions(self):
         elif context_filename:
             file_name = context_filename
         file_path = os.path.join(recordings_folder, file_name)
-        out_file = open(file_path, "w+", "utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines("\r\n".join(data))
         out_file.close()
         rec_message = ">>> RECORDING SAVED as: "
@@ -5940,7 +5940,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
             file_name = sb_config.behave_scenario.filename.replace(".", "_")
         file_name = file_name.split("/")[-1].split("\\")[-1] + "_rec.feature"
         file_path = os.path.join(features_folder, file_name)
-        out_file = open(file_path, "w+", "utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines("\r\n".join(data))
         out_file.close()
 
@@ -5978,7 +5978,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
         file_name = "__init__.py"
         file_path = os.path.join(features_folder, file_name)
         if not os.path.exists(file_path):
-            out_file = open(file_path, "w+", "utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             print("Created recordings/features/__init__.py")
@@ -5991,7 +5991,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
         file_name = "behave.ini"
         file_path = os.path.join(features_folder, file_name)
         if not os.path.exists(file_path):
-            out_file = open(file_path, "w+", "utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             print("Created recordings/features/behave.ini")
@@ -6030,7 +6030,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
         file_name = "environment.py"
         file_path = os.path.join(features_folder, file_name)
         if not os.path.exists(file_path):
-            out_file = open(file_path, "w+", "utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             print("Created recordings/features/environment.py")
@@ -6040,7 +6040,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
         file_name = "__init__.py"
         file_path = os.path.join(steps_folder, file_name)
         if not os.path.exists(file_path):
-            out_file = open(file_path, "w+", "utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             print("Created recordings/features/steps/__init__.py")
@@ -6051,7 +6051,7 @@ def __process_recorded_behave_actions(self, srt_actions, colorama):
         file_name = "imported.py"
         file_path = os.path.join(steps_folder, file_name)
         if not os.path.exists(file_path):
-            out_file = open(file_path, "w+", "utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines("\r\n".join(data))
             out_file.close()
             print("Created recordings/features/steps/imported.py")
@@ -11120,7 +11120,7 @@ def __process_visual_baseline_logs(self):
             return  # Skip the rest when deferred visual asserts are used
         the_html = visual_helper.get_sbs_html()
         file_path = os.path.join(test_logpath, constants.SideBySide.HTML_FILE)
-        out_file = open(file_path, "w+", encoding="utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines(the_html)
         out_file.close()
 
@@ -11280,16 +11280,16 @@ def check_window(
             self.save_screenshot(
                 baseline_png, visual_baseline_path, selector="body"
             )
-            out_file = open(page_url_file, "w+", encoding="utf-8")
+            out_file = open(page_url_file, mode="w+", encoding="utf-8")
             out_file.writelines(page_url)
             out_file.close()
-            out_file = open(level_1_file, "w+", encoding="utf-8")
+            out_file = open(level_1_file, mode="w+", encoding="utf-8")
             out_file.writelines(json.dumps(level_1))
             out_file.close()
-            out_file = open(level_2_file, "w+", encoding="utf-8")
+            out_file = open(level_2_file, mode="w+", encoding="utf-8")
             out_file.writelines(json.dumps(level_2))
             out_file.close()
-            out_file = open(level_3_file, "w+", encoding="utf-8")
+            out_file = open(level_3_file, mode="w+", encoding="utf-8")
             out_file.writelines(json.dumps(level_3))
             out_file.close()
 
@@ -11428,7 +11428,7 @@ def check_window(
             alpha_n_d_name = "".join([x if x.isalnum() else "_" for x in name])
             side_by_side_name = "side_by_side_%s.html" % alpha_n_d_name
             file_path = os.path.join(test_logpath, side_by_side_name)
-            out_file = open(file_path, "w+", encoding="utf-8")
+            out_file = open(file_path, mode="w+", encoding="utf-8")
             out_file.writelines(the_html)
             out_file.close()
 
@@ -12120,7 +12120,7 @@ def save_presentation(
             with suppress(Exception):
                 os.makedirs(saved_presentations_folder)
         file_path = os.path.join(saved_presentations_folder, filename)
-        out_file = open(file_path, "w+", encoding="utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines(the_html)
         out_file.close()
         if self._output_file_saves:
@@ -12815,7 +12815,7 @@ def save_chart(self, chart_name=None, filename=None, folder=None):
             with suppress(Exception):
                 os.makedirs(saved_charts_folder)
         file_path = os.path.join(saved_charts_folder, filename)
-        out_file = open(file_path, "w+", encoding="utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines(the_html)
         out_file.close()
         if self._output_file_saves:
@@ -16378,7 +16378,7 @@ def __process_dashboard(self, has_exception, init=False):
                     dash_pie = json.dumps(sb_config._saved_dashboard_pie)
                     dash_pie_loc = constants.Dashboard.DASH_PIE
                     pie_path = os.path.join(abs_path, dash_pie_loc)
-                    pie_file = open(pie_path, "w+", encoding="utf-8")
+                    pie_file = open(pie_path, mode="w+", encoding="utf-8")
                     pie_file.writelines(dash_pie)
                     pie_file.close()
         DASH_PIE_PNG_1 = constants.Dashboard.get_dash_pie_1()
@@ -16538,7 +16538,7 @@ def __process_dashboard(self, has_exception, init=False):
         )
         abs_path = os.path.abspath(".")
         file_path = os.path.join(abs_path, "dashboard.html")
-        out_file = open(file_path, "w+", encoding="utf-8")
+        out_file = open(file_path, mode="w+", encoding="utf-8")
         out_file.writelines(the_html)
         out_file.close()
         sb_config._dash_html = the_html
@@ -16551,7 +16551,7 @@ def __process_dashboard(self, has_exception, init=False):
             dash_json = json.dumps((_results, _display_id, _rt, _tlp, d_stats))
             dash_json_loc = constants.Dashboard.DASH_JSON
             dash_jsonpath = os.path.join(abs_path, dash_json_loc)
-            dash_json_file = open(dash_jsonpath, "w+", encoding="utf-8")
+            dash_json_file = open(dash_jsonpath, mode="w+", encoding="utf-8")
             dash_json_file.writelines(dash_json)
             dash_json_file.close()
 

seleniumbase/undetected/cdp.py

@@ -1,9 +1,7 @@
-import fasteners
 import json
 import logging
 import requests
-from seleniumbase.fixtures import constants
-from seleniumbase.fixtures import shared_utils
+import websockets
 
 log = logging.getLogger(__name__)
 
@@ -107,15 +105,6 @@ def tab_close_last_opened(self):
             return resp.json()
 
     async def send(self, method, params):
-        pip_find_lock = fasteners.InterProcessLock(
-            constants.PipInstall.FINDLOCK
-        )
-        with pip_find_lock:
-            try:
-                import websockets
-            except Exception:
-                shared_utils.pip_install("websockets")
-                import websockets
         self._reqid += 1
         async with websockets.connect(self.wsurl) as ws:
             await ws.send(

seleniumbase/undetected/cdp_driver/browser.py

@@ -350,8 +350,7 @@ async def get(
             if _cdp_geolocation:
                 await connection.send(cdp.page.navigate("about:blank"))
                 await connection.set_geolocation(_cdp_geolocation)
-            # This part isn't needed now, but may be needed later
-            """
+            # (The code below is for the Chrome 142 extension fix)
             if (
                 hasattr(sb_config, "_cdp_proxy")
                 and "@" in sb_config._cdp_proxy
@@ -363,7 +362,6 @@ async def get(
                 proxy_pass = username_and_password.split(":")[1]
                 await connection.set_auth(proxy_user, proxy_pass, self.tabs[0])
                 time.sleep(0.25)
-            """
             if "auth" in kwargs and kwargs["auth"] and ":" in kwargs["auth"]:
                 username_and_password = kwargs["auth"]
                 proxy_user = username_and_password.split(":")[0]
@@ -375,12 +373,12 @@ async def get(
                 cdp.page.navigate(url)
             )
             if _cdp_recorder:
-                pass  # (The code below was for the Chrome 137 extension fix)
-                '''from seleniumbase.js_code.recorder_js import recorder_js
+                # (The code below is for the Chrome 142 extension fix)
+                from seleniumbase.js_code.recorder_js import recorder_js
                 recorder_code = (
                     """window.onload = function() { %s };""" % recorder_js
                 )
-                await connection.send(cdp.runtime.evaluate(recorder_code))'''
+                await connection.send(cdp.runtime.evaluate(recorder_code))
             # Update the frame_id on the tab
             connection.frame_id = frame_id
             connection.browser = self