From dbf82861126231637c3fae2d4bca7c9f0f3f63b7 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 24 Oct 2024 09:58:54 -0400 Subject: [PATCH 1/4] Update UC Mode --- seleniumbase/fixtures/base_case.py | 136 ++++++++++++++++--------- seleniumbase/plugins/driver_manager.py | 17 +++- seleniumbase/plugins/sb_manager.py | 24 ----- 3 files changed, 100 insertions(+), 77 deletions(-) diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index 4007c3fe606..2d37b533135 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -4090,6 +4090,88 @@ def get_new_driver( "Browser: {%s} is not a valid browser option. " "Valid options = {%s}" % (browser, valid_browsers) ) + # Fix Chrome-130 issues by creating a user-data-dir in advance + if ( + undetectable + and ( + not user_data_dir + or not os.path.exists(user_data_dir) + or not any(os.scandir(user_data_dir)) + ) + and self.browser == "chrome" + ): + import tempfile + if not user_data_dir: + user_data_dir = os.path.normpath(tempfile.mkdtemp()) + self.user_data_dir = user_data_dir + sb_config.user_data_dir = user_data_dir + try: + decoy_driver = browser_launcher.get_driver( + browser_name=browser_name, + headless=headless, + locale_code=locale_code, + use_grid=use_grid, + protocol=protocol, + servername=servername, + port=port, + proxy_string=proxy_string, + proxy_bypass_list=proxy_bypass_list, + proxy_pac_url=proxy_pac_url, + multi_proxy=multi_proxy, + user_agent=user_agent, + cap_file=cap_file, + cap_string=cap_string, + recorder_ext=recorder_ext, + disable_cookies=disable_cookies, + disable_js=disable_js, + disable_csp=disable_csp, + enable_ws=enable_ws, + enable_sync=enable_sync, + use_auto_ext=use_auto_ext, + undetectable=undetectable, + uc_cdp_events=uc_cdp_events, + uc_subprocess=uc_subprocess, + log_cdp_events=log_cdp_events, + no_sandbox=no_sandbox, + disable_gpu=disable_gpu, + headless1=headless1, + headless2=True, + incognito=incognito, + guest_mode=guest_mode, + dark_mode=dark_mode, + devtools=devtools, + remote_debug=remote_debug, + enable_3d_apis=enable_3d_apis, + swiftshader=swiftshader, + ad_block_on=ad_block_on, + host_resolver_rules=host_resolver_rules, + block_images=block_images, + do_not_track=do_not_track, + chromium_arg=chromium_arg, + firefox_arg=firefox_arg, + firefox_pref=firefox_pref, + user_data_dir=user_data_dir, + extension_zip=extension_zip, + extension_dir=extension_dir, + disable_features=disable_features, + binary_location=binary_location, + driver_version=driver_version, + page_load_strategy=page_load_strategy, + use_wire=use_wire, + external_pdf=external_pdf, + test_id=test_id, + mobile_emulator=is_mobile, + device_width=d_width, + device_height=d_height, + device_pixel_ratio=d_p_r, + browser=browser_name, + ) + time.sleep(0.555) + except Exception: + pass + finally: + with suppress(Exception): + decoy_driver.quit() # Launch a web browser new_driver = browser_launcher.get_driver( browser_name=browser_name, @@ -4435,7 +4517,12 @@ def load_cookies(self, name="cookies.txt"): """Loads the page cookies from the "saved_cookies" folder.""" cookies = self.get_saved_cookies(name) self.wait_for_ready_state_complete() + origin = self.get_origin() + trim_origin = origin.split("://")[-1] for cookie in cookies: + if "domain" in cookie: + if cookie["domain"] not in origin: + cookie["domain"] = trim_origin if "expiry" in cookie: del cookie["expiry"] self.driver.add_cookie(cookie) @@ -4656,30 +4743,6 @@ def activate_cdp_mode(self, url=None): if hasattr(self.driver, "_is_using_uc") and self.driver._is_using_uc: self.driver.uc_open_with_cdp_mode(url) else: - # Fix Chrome-130 issues by creating a user-data-dir in advance - if ( - ( - not self.user_data_dir - or not os.path.exists(self.user_data_dir) - ) - and self.browser == "chrome" - ): - import tempfile - user_data_dir = os.path.normpath(tempfile.mkdtemp()) - self.user_data_dir = user_data_dir - sb_config.user_data_dir = user_data_dir - try: - driver = self.get_new_driver( - user_data_dir=user_data_dir, - undetectable=True, - headless2=True, - ) - time.sleep(0.555) - except Exception: - pass - finally: - with suppress(Exception): - driver.quit() self.get_new_driver(undetectable=True) self.driver.uc_open_with_cdp_mode(url) self.cdp = self.driver.cdp @@ -14941,31 +15004,6 @@ def setUp(self, masterqa_mode=False): self.__js_start_time = int(time.time() * 1000.0) else: # Launch WebDriver for both pytest and pynose - - # Fix Chrome-130 issues by creating a user-data-dir in advance - if ( - self.undetectable - and ( - not self.user_data_dir - or not os.path.exists(self.user_data_dir) - ) - and self.browser == "chrome" - ): - import tempfile - user_data_dir = os.path.normpath(tempfile.mkdtemp()) - self.user_data_dir = user_data_dir - sb_config.user_data_dir = user_data_dir - try: - driver = self.get_new_driver( - user_data_dir=user_data_dir, - headless2=True, - ) - time.sleep(0.555) - except Exception: - pass - finally: - with suppress(Exception): - driver.quit() self.driver = self.get_new_driver( browser=self.browser, headless=self.headless, diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py index 64f9250ecd0..192e93f71d4 100644 --- a/seleniumbase/plugins/driver_manager.py +++ b/seleniumbase/plugins/driver_manager.py @@ -792,12 +792,21 @@ def Driver( from seleniumbase.core import browser_launcher # Fix Chrome-130 issues by creating a user-data-dir in advance - if undetectable and not user_data_dir and browser == "chrome": + if ( + undetectable + and ( + not user_data_dir + or not os.path.exists(user_data_dir) + or not any(os.scandir(user_data_dir)) + ) + and browser == "chrome" + ): import tempfile import time - user_data_dir = ( - os.path.normpath(tempfile.mkdtemp()) - ) + if not user_data_dir: + user_data_dir = ( + os.path.normpath(tempfile.mkdtemp()) + ) try: decoy_driver = browser_launcher.get_driver( browser_name=browser_name, diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 3f00d3f7f84..a7ba66bffb1 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -1212,30 +1212,6 @@ def SB( if not sb_config.multi_proxy: proxy_helper.remove_proxy_zip_if_present() start_time = time.time() - saved_headless2 = headless2 - - # Fix Chrome-130 issues by creating a user-data-dir in advance - if undetectable and not user_data_dir and browser == "chrome": - import tempfile - user_data_dir = ( - os.path.normpath(tempfile.mkdtemp()) - ) - sb.user_data_dir = user_data_dir - sb_config.user_data_dir = user_data_dir - try: - decoy = sb - decoy.headless2 = True - decoy.setUp() - decoy.sleep(0.555) - except Exception: - pass - finally: - try: - decoy.tearDown() - except Exception: - pass - sb.headless2 = saved_headless2 - sb.setUp() test_passed = True # This can change later teardown_exception = None From 3258db07f84f592dad60ae42cfa51a8ff9c6d16c Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 24 Oct 2024 10:00:22 -0400 Subject: [PATCH 2/4] Include CDP Mode in packages --- pyproject.toml | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 328dd55b152..336999d3b49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,7 @@ packages = [ "seleniumbase.resources", "seleniumbase.translate", "seleniumbase.undetected", + "seleniumbase.undetected.cdp_driver", "seleniumbase.utilities", "seleniumbase.utilities.selenium_grid", "seleniumbase.utilities.selenium_ide", diff --git a/setup.py b/setup.py index 969e32c4ed0..5f7f3c7dcfd 100755 --- a/setup.py +++ b/setup.py @@ -317,6 +317,7 @@ "seleniumbase.resources", "seleniumbase.translate", "seleniumbase.undetected", + "seleniumbase.undetected.cdp_driver", "seleniumbase.utilities", "seleniumbase.utilities.selenium_grid", "seleniumbase.utilities.selenium_ide", From 101d2af4d5b09e70fb66c1f5b0278caab178a733 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 24 Oct 2024 10:02:33 -0400 Subject: [PATCH 3/4] Update examples for CDP Mode --- examples/cdp_mode/raw_async.py | 2 +- examples/cdp_mode/raw_cdp.py | 2 +- examples/cdp_mode/raw_cdp_with_sb.py | 2 +- examples/verify_undetected.py | 18 ++++++------------ 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/examples/cdp_mode/raw_async.py b/examples/cdp_mode/raw_async.py index 865210de939..779afd3d9e2 100644 --- a/examples/cdp_mode/raw_async.py +++ b/examples/cdp_mode/raw_async.py @@ -38,7 +38,7 @@ async def main(): sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") sb.sleep(2) - sb.remove_element("msm-cookie-banner") + sb.remove_elements("msm-cookie-banner") sb.sleep(1) sb.press_keys('input[data-test-id*="search"]', "Amsterdam") sb.sleep(2) diff --git a/examples/cdp_mode/raw_cdp.py b/examples/cdp_mode/raw_cdp.py index 30998c88661..1d70cb66e1e 100644 --- a/examples/cdp_mode/raw_cdp.py +++ b/examples/cdp_mode/raw_cdp.py @@ -17,7 +17,7 @@ def main(): sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") sb.sleep(2) - sb.remove_element("msm-cookie-banner") + sb.remove_elements("msm-cookie-banner") sb.sleep(1) location = "Amsterdam" sb.press_keys('input[data-test-id*="search"]', location) diff --git a/examples/cdp_mode/raw_cdp_with_sb.py b/examples/cdp_mode/raw_cdp_with_sb.py index 7e40e38c721..7ed71eabe0b 100644 --- a/examples/cdp_mode/raw_cdp_with_sb.py +++ b/examples/cdp_mode/raw_cdp_with_sb.py @@ -10,7 +10,7 @@ sb.internalize_links() # Don't open links in a new tab sb.click("#link_header_nav_experiences") sb.sleep(2) - sb.remove_element("msm-cookie-banner") + sb.remove_elements("msm-cookie-banner") sb.sleep(1) location = "Amsterdam" sb.press_keys('input[data-test-id*="search"]', location) diff --git a/examples/verify_undetected.py b/examples/verify_undetected.py index 7fca3a78016..96a88633fc1 100644 --- a/examples/verify_undetected.py +++ b/examples/verify_undetected.py @@ -2,7 +2,6 @@ Some sites use scripts to detect Selenium, and then block you. To evade detection, add --uc as a pytest command-line option.""" from seleniumbase import BaseCase -from seleniumbase import DriverContext BaseCase.main(__name__, __file__, "--uc", "-s") @@ -10,14 +9,9 @@ class UndetectedTest(BaseCase): def test_browser_is_undetected(self): url = "https://gitlab.com/users/sign_in" if not self.undetectable: - with DriverContext(uc=True) as driver: - driver.uc_activate_cdp_mode(url) - driver.uc_gui_click_captcha() - driver.assert_text("Username", '[for="user_login"]', timeout=3) - print("\n Success! Website did not detect Selenium! ") - else: - self.activate_cdp_mode(url) - self.uc_gui_click_captcha() - self.assert_text("Username", '[for="user_login"]', timeout=3) - self.post_message("SeleniumBase wasn't detected", duration=4) - self._print("\n Success! Website did not detect Selenium! ") + self.get_new_driver(undetectable=True) + self.activate_cdp_mode(url) + self.uc_gui_click_captcha() + self.assert_text("Username", '[for="user_login"]', timeout=3) + self.post_message("SeleniumBase wasn't detected", duration=4) + self._print("\n Success! Website did not detect Selenium! ") From e88de457f240ec376dc3b69567621c63bbbc80a1 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Thu, 24 Oct 2024 10:03:19 -0400 Subject: [PATCH 4/4] Version 4.32.2 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index d2c74c338d1..eac1b105033 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.32.1" +__version__ = "4.32.2"