From 128948cd4e359082eed6b9f4ea23962869259285 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 27 May 2025 19:49:20 -0400 Subject: [PATCH 1/3] Fix extensions on Chrome 137+ --- seleniumbase/console_scripts/sb_mkrec.py | 5 +- seleniumbase/core/browser_launcher.py | 52 +++++++++++++++--- seleniumbase/extensions/recorder.zip | Bin 11918 -> 11813 bytes seleniumbase/undetected/cdp_driver/browser.py | 31 +++++++++++ .../undetected/cdp_driver/cdp_util.py | 22 ++++++-- seleniumbase/undetected/cdp_driver/config.py | 3 +- .../undetected/cdp_driver/connection.py | 30 ++++++++++ 7 files changed, 125 insertions(+), 18 deletions(-) diff --git a/seleniumbase/console_scripts/sb_mkrec.py b/seleniumbase/console_scripts/sb_mkrec.py index 01ca9cfe431..7f72eb7fde7 100644 --- a/seleniumbase/console_scripts/sb_mkrec.py +++ b/seleniumbase/console_scripts/sb_mkrec.py @@ -192,8 +192,9 @@ def main(): elif "'" not in start_page: used_sp = "'%s'" % start_page data.append( - " self.uc_open_with_disconnect(\n" - " %s\n" + " self.activate_cdp_mode(\n" + " %s,\n" + " recorder=True,\n" " )" % used_sp ) else: diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index eb30a46053e..963042aab12 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -108,7 +108,9 @@ def make_driver_executable_if_not(driver_path): shared_utils.make_executable(driver_path) -def extend_driver(driver, proxy_auth=False, use_uc=True): +def extend_driver( + driver, proxy_auth=False, use_uc=True, recorder_ext=False +): # Extend the driver with new methods driver.default_find_element = driver.find_element driver.default_find_elements = driver.find_elements @@ -234,6 +236,14 @@ def extend_driver(driver, proxy_auth=False, use_uc=True): driver.switch_to_tab = DM.switch_to_tab driver.switch_to_frame = DM.switch_to_frame driver.reset_window_size = DM.reset_window_size + if recorder_ext: + from seleniumbase.js_code.recorder_js import recorder_js + recorder_code = ( + """window.onload = function() { %s };""" % recorder_js + ) + driver.execute_cdp_cmd( + "Page.addScriptToEvaluateOnNewDocument", {"source": recorder_code} + ) if hasattr(driver, "proxy"): driver.set_wire_proxy = DM.set_wire_proxy if proxy_auth: @@ -2542,6 +2552,7 @@ def _set_chrome_options( included_disabled_features.append("PrivacySandboxSettings4") included_disabled_features.append("SidePanelPinning") included_disabled_features.append("UserAgentClientHint") + included_disabled_features.append("DisableLoadExtensionCommandLineSwitch") for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -2819,6 +2830,8 @@ def get_driver( if headless2 and browser_name == constants.Browser.FIREFOX: headless2 = False # Only for Chromium headless = True + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() if ( is_using_uc(undetectable, browser_name) and binary_location @@ -3014,6 +3027,8 @@ def get_driver( proxy_pass = None proxy_scheme = "http" if proxy_string: + # (The code below was for the Chrome 137 extension fix) + # sb_config._cdp_proxy = proxy_string username_and_password = None if "@" in proxy_string: # Format => username:password@hostname:port @@ -4450,6 +4465,9 @@ def get_local_driver( included_disabled_features.append("PrivacySandboxSettings4") included_disabled_features.append("SidePanelPinning") included_disabled_features.append("UserAgentClientHint") + included_disabled_features.append( + "DisableLoadExtensionCommandLineSwitch" + ) for item in extra_disabled_features: if item not in included_disabled_features: included_disabled_features.append(item) @@ -5328,7 +5346,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) if not auto_upgrade_chromedriver: raise # Not an obvious fix. else: @@ -5580,11 +5600,15 @@ def get_local_driver( 'Emulation.setDeviceMetricsOverride', set_device_metrics_override ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) else: # Running headless on Linux (and not using --uc) try: driver = webdriver.Chrome(options=chrome_options) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception as e: if not hasattr(e, "msg"): raise @@ -5606,7 +5630,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) mcv = None # Major Chrome Version if "Current browser version is " in e.msg: line = e.msg.split("Current browser version is ")[1] @@ -5649,7 +5675,9 @@ def get_local_driver( service=service, options=chrome_options, ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) # Use the virtual display on Linux during headless errors logging.debug( "\nWarning: Chrome failed to launch in" @@ -5667,7 +5695,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception as original_exception: if use_uc: raise @@ -5677,7 +5707,9 @@ def get_local_driver( driver = webdriver.Chrome( service=service, options=chrome_options ) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) if user_data_dir: print("\nUnable to set user_data_dir while starting Chrome!\n") raise @@ -5704,7 +5736,9 @@ def get_local_driver( ) try: driver = webdriver.Chrome(service=service) - return extend_driver(driver, proxy_auth, use_uc) + return extend_driver( + driver, proxy_auth, use_uc, recorder_ext + ) except Exception: raise original_exception else: diff --git a/seleniumbase/extensions/recorder.zip b/seleniumbase/extensions/recorder.zip index 257fcd562d839a5d9143d5c89e05f025d5479803..79a4870afae38ca68ffee75279d76252681100fb 100644 GIT binary patch delta 1668 zcmeB+T^cjNgr&IKBx7&mL>=TxCO=?vSmJfm({YDJU9^%J|TK?LR=7|xj>9>vx=q0sY8y|v!i?x zv?r83&}Hn6<6kD$eg4*)IcqiQi?^~|%SlzTOIVpCDRy*9t%Bn8e_c2JwE9?ho2ndP zySL$B#o6>@f^1&V-KE^?9{6SN{V=O+7jIHe!JT0FTb5O3Z{Nvx^iEx>E&7ye`;%(> z$8IXOPk(C?u$U5<{6DEyzSSAa@nf2d()3> zIxCm@_1~Jml0SE4F^M}>AHP;;@$KQS<@K+4CZ%25tH1Z(kv9P)avU?vZ7(biXKvf| zqWRrxduD@2CKG#YXA6AwZ&vv=H|r;3fHymbqE4E$4lwwnm>3uWyqSTa$N_|)P@gQo zsmlWkYjB)5uoQD2Ox)zj<}vYSTz2T>hsug9zl}8?On$>@&!G>D4G`XJ%*84K=0Ww8 zaMgpfT)e!$KVtGeIYkyrZI?NdAIn)mVqH<*8-;r>7#bi!MsBT0xs`s zT%Q7Lsx%2M((LXYe>FOy*kIiNYK(t6(3oBosD9pAE| zQCFyb8xyP4?O&l59(E2=(G97W<+Ku%qg-PoHA4kzvnjh zGvmF+Qo%XPH%TOTE@ql(wkt)v+JsMZ@B4Gob~bl;tX3ND-`)F7-Is5&Jt!H?y;SQR1WZPtSV1Hs zP<#QCk@RG5Js}=gl)#hH!um2yYro3me=LeDocB&mo4iWR2a+x>>PSx(Rd<4< zjf*5-m%UDE?0W2Y4i zmjRbNh}~oNF z3bo}JCrqB8Ed$RrAk*ZR?70#F${aA$q)|+pH2IyjIwE6%g2Y5eNv2(nfgv+DF+H_d zADsH0QcHK@WT+v+KxCXq}%c{Wcq GkUaoR*=lP5 delta 1737 zcmZ1)(-%9zgyrVzlC;T*Op+4=O;}ca1+p7hG$-F-)B-l?gHV` z3T_5QmamKq3@jo*Rc}Ll^KTpQ?EM|ivGa-}v-AzagM7KSoEwrPer@y)-?HV5N>1*n z2`^Y&C;t5wm^rCZv*yL#T^$#c<&S=6S$jab|JqNXzsg1;qSK!37QFvR#WSo+X!47b zds4NZ+*soLd74CQW!}1anJKq#=>IKCaNu$Sy~^Bz96v6O6o0q>jw!PDe|iP3 zN^DfL?~$Ik=Hb!6#t9)4->_+?WW6rt$XHkI`Yfl)`gdt%NKi7zskN@(r`MZbZr}Km zbFJ9ka;3PG{XhSmpFA%2B!v-3!b&ir$>QVeb0Z zjt}ofJ>0zZ!mA@D%f9|n2&kF)fm8S%?~Rj79?4#*%zj+aywpSZSfyOq_b@{PW%C<7 zKI>mp-b>$@J)t@9*}8+-I&}xS#5S1Uxa#BEdMUfGlPRRN_Bj9f49+!u5_?0opDr^y z{!w!6g84E^ea7>4>-|rgbR=K->IeM*Z+4EX@~zAhfN_+_!oU#V%?ykx4j=?Y?c_Ba zx;(H5g(k3K?t@>x&lH)kNIXp2b#j!PBA0rq#Kj(9^nPNPJV91s@-j|EjtpR8Oa*cIqsZnzBH8~?#Dg&kF)XB5tO*vpm zYV&n@1!gc0X5!=`WhdT(I-O8hLKL5TP+3y-Xs>*7D>N;lrNi1uXY&pj2(-MnnfzZ_ zv_9sK@J;CzT@~MN^*J}JdLnYR#(G|F@r;yzinBBox_j0*G>OfvO6A+c|KM0y!pSY| zS~oWMDNSuNmMpxm`1uY|U9QE=jxUS8`aP7s?>6@m!cQG`=0J3J0bhac1Py1Bwq|>XDw@pen=ziwanb zaVLhauQBqN7(I2q+~j{OiYx+-Qa2|%s`)_Dz(loJNc>M!6Q6umU1YMXx*kkMda|~L zz~m}*3uG1c$gjpcXPK9P<~TSGTiShO1ciQ3zSF7mAF&oG;OnxaR4|o0KW}x5~ zpx|10C3ubjSuyz~P(o0dVe%;@T}0*qN;rTb3ONdSQT)QE!oZN5n3tKBT3n)+Rh*v} p8p6xK4vSA%fXIUk0mUJt6hIG<1Cy`lDxf5SNqX{Zo76yI1^_Oa=-~hW diff --git a/seleniumbase/undetected/cdp_driver/browser.py b/seleniumbase/undetected/cdp_driver/browser.py index 1a991f84f57..3d99d0222ff 100644 --- a/seleniumbase/undetected/cdp_driver/browser.py +++ b/seleniumbase/undetected/cdp_driver/browser.py @@ -292,6 +292,7 @@ async def get( _cdp_locale = None _cdp_platform = None _cdp_geolocation = None + _cdp_recorder = None if ( hasattr(sb_config, "_cdp_timezone") and sb_config._cdp_timezone ): @@ -330,6 +331,8 @@ async def get( _cdp_geolocation = kwargs["geolocation"] elif "geoloc" in kwargs: _cdp_geolocation = kwargs["geoloc"] + if "recorder" in kwargs: + _cdp_recorder = kwargs["recorder"] if _cdp_timezone: await connection.send(cdp.page.navigate("about:blank")) await connection.set_timezone(_cdp_timezone) @@ -344,9 +347,37 @@ async def get( await connection.send(cdp.page.navigate("about:blank")) await connection.set_geolocation(_cdp_geolocation) # Use the tab to navigate to new url + if ( + hasattr(sb_config, "_cdp_proxy") + and "@" in sb_config._cdp_proxy + and sb_config._cdp_proxy + and "auth" not in kwargs + ): + username_and_password = sb_config._cdp_proxy.split("@")[0] + proxy_user = username_and_password.split(":")[0] + proxy_pass = username_and_password.split(":")[1] + await connection.set_auth( + proxy_user, proxy_pass, self.tabs[0] + ) + time.sleep(0.25) + elif "auth" in kwargs and kwargs["auth"] and ":" in kwargs["auth"]: + username_and_password = kwargs["auth"] + proxy_user = username_and_password.split(":")[0] + proxy_pass = username_and_password.split(":")[1] + await connection.set_auth( + proxy_user, proxy_pass, self.tabs[0] + ) + time.sleep(0.25) frame_id, loader_id, *_ = await connection.send( cdp.page.navigate(url) ) + if _cdp_recorder: + pass # (The code below was for the Chrome 137 extension fix) + '''from seleniumbase.js_code.recorder_js import recorder_js + recorder_code = ( + """window.onload = function() { %s };""" % recorder_js + ) + await connection.send(cdp.runtime.evaluate(recorder_code))''' # Update the frame_id on the tab connection.frame_id = frame_id connection.browser = self diff --git a/seleniumbase/undetected/cdp_driver/cdp_util.py b/seleniumbase/undetected/cdp_driver/cdp_util.py index fda6ce9451f..945f45871d3 100644 --- a/seleniumbase/undetected/cdp_driver/cdp_util.py +++ b/seleniumbase/undetected/cdp_driver/cdp_util.py @@ -360,8 +360,10 @@ async def start( except Exception: time.sleep(0.15) driver = await Browser.create(config) - if proxy and "@" in str(proxy): - time.sleep(0.15) + if proxy: + sb_config._cdp_proxy = proxy + if "@" in str(proxy): + time.sleep(0.15) if lang: sb_config._cdp_locale = lang elif "locale" in kwargs: @@ -402,10 +404,14 @@ async def start_async(*args, **kwargs) -> Browser: binary_location = None if "browser_executable_path" in kwargs: binary_location = kwargs["browser_executable_path"] + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() else: binary_location = detect_b_ver.get_binary_location("google-chrome") - if binary_location and not os.path.exists(binary_location): - binary_location = None + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() + if not os.path.exists(binary_location): + binary_location = None if ( shared_utils.is_chrome_130_or_newer(binary_location) and "user_data_dir" in kwargs @@ -438,10 +444,14 @@ def start_sync(*args, **kwargs) -> Browser: binary_location = None if "browser_executable_path" in kwargs: binary_location = kwargs["browser_executable_path"] + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() else: binary_location = detect_b_ver.get_binary_location("google-chrome") - if binary_location and not os.path.exists(binary_location): - binary_location = None + if binary_location and isinstance(binary_location, str): + binary_location = binary_location.strip() + if not os.path.exists(binary_location): + binary_location = None if ( shared_utils.is_chrome_130_or_newer(binary_location) and "user_data_dir" in kwargs diff --git a/seleniumbase/undetected/cdp_driver/config.py b/seleniumbase/undetected/cdp_driver/config.py index b11228ea59d..e701bb4a801 100644 --- a/seleniumbase/undetected/cdp_driver/config.py +++ b/seleniumbase/undetected/cdp_driver/config.py @@ -144,7 +144,8 @@ def __init__( "--disable-features=IsolateOrigins,site-per-process,Translate," "InsecureDownloadWarnings,DownloadBubble,DownloadBubbleV2," "OptimizationTargetPrediction,OptimizationGuideModelDownloading," - "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4", + "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4," + "DisableLoadExtensionCommandLineSwitch", ] @property diff --git a/seleniumbase/undetected/cdp_driver/connection.py b/seleniumbase/undetected/cdp_driver/connection.py index 8bcdc82ff36..ecb7f13b345 100644 --- a/seleniumbase/undetected/cdp_driver/connection.py +++ b/seleniumbase/undetected/cdp_driver/connection.py @@ -378,6 +378,36 @@ async def set_geolocation(self, geolocation: Optional[tuple] = None): accuracy=100, )) + async def set_auth(self, username, password, tab): + async def auth_challenge_handler(event: cdp.fetch.AuthRequired): + await tab.send( + cdp.fetch.continue_with_auth( + request_id=event.request_id, + auth_challenge_response=cdp.fetch.AuthChallengeResponse( + response="ProvideCredentials", + username=username, + password=password, + ), + ) + ) + + async def req_paused(event: cdp.fetch.RequestPaused): + await tab.send( + cdp.fetch.continue_request(request_id=event.request_id) + ) + + tab.add_handler( + cdp.fetch.RequestPaused, + lambda event: asyncio.create_task(req_paused(event)), + ) + + tab.add_handler( + cdp.fetch.AuthRequired, + lambda event: asyncio.create_task(auth_challenge_handler(event)), + ) + + await tab.send(cdp.fetch.enable(handle_auth_requests=True)) + def __getattr__(self, item): """:meta private:""" try: From b1d58a0d3a82fd6de9e25c7b14457057709413d9 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 27 May 2025 19:49:50 -0400 Subject: [PATCH 2/3] Refresh Python dependencies --- requirements.txt | 3 ++- setup.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 58faf54d3f8..f144a905b20 100755 --- a/requirements.txt +++ b/requirements.txt @@ -60,7 +60,8 @@ pytest-metadata==3.1.1 pytest-ordering==0.6 pytest-rerunfailures==14.0;python_version<"3.9" pytest-rerunfailures==15.1;python_version>="3.9" -pytest-xdist==3.6.1 +pytest-xdist==3.6.1;python_version<"3.9" +pytest-xdist==3.7.0;python_version>="3.9" parameterized==0.9.0 behave==1.2.6 soupsieve==2.7 diff --git a/setup.py b/setup.py index 76ea8d31143..899bdbaded7 100755 --- a/setup.py +++ b/setup.py @@ -211,7 +211,8 @@ "pytest-ordering==0.6", 'pytest-rerunfailures==14.0;python_version<"3.9"', 'pytest-rerunfailures==15.1;python_version>="3.9"', - 'pytest-xdist==3.6.1', + 'pytest-xdist==3.6.1;python_version<"3.9"', + 'pytest-xdist==3.7.0;python_version>="3.9"', 'parameterized==0.9.0', "behave==1.2.6", 'soupsieve==2.7', @@ -268,7 +269,7 @@ 'pdfminer.six==20250324;python_version<"3.9"', 'pdfminer.six==20250506;python_version>="3.9"', 'cryptography==39.0.2;python_version<"3.9"', - 'cryptography==45.0.2;python_version>="3.9"', + 'cryptography==45.0.3;python_version>="3.9"', 'cffi==1.17.1', "pycparser==2.22", ], From 7d9ea59407466fda77fe4a247615b0f079ca6e2a Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 27 May 2025 19:49:58 -0400 Subject: [PATCH 3/3] Version 4.39.1 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 5173bc46dcb..9958ccb6fb9 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.39.0" +__version__ = "4.39.1"