Merge pull request #3220 from seleniumbase/cdp-mode-patch-2

CDP Mode - Patch 2

Author: mdmintz

examples/cdp_mode/raw_async.py

@@ -38,7 +38,7 @@ async def main():
     sb.internalize_links()  # Don't open links in a new tab
     sb.click("#link_header_nav_experiences")
     sb.sleep(2)
-    sb.remove_element("msm-cookie-banner")
+    sb.remove_elements("msm-cookie-banner")
     sb.sleep(1)
     sb.press_keys('input[data-test-id*="search"]', "Amsterdam")
     sb.sleep(2)

examples/cdp_mode/raw_cdp.py

@@ -17,7 +17,7 @@ def main():
     sb.internalize_links()  # Don't open links in a new tab
     sb.click("#link_header_nav_experiences")
     sb.sleep(2)
-    sb.remove_element("msm-cookie-banner")
+    sb.remove_elements("msm-cookie-banner")
     sb.sleep(1)
     location = "Amsterdam"
     sb.press_keys('input[data-test-id*="search"]', location)

examples/cdp_mode/raw_cdp_with_sb.py

@@ -10,7 +10,7 @@
     sb.internalize_links()  # Don't open links in a new tab
     sb.click("#link_header_nav_experiences")
     sb.sleep(2)
-    sb.remove_element("msm-cookie-banner")
+    sb.remove_elements("msm-cookie-banner")
     sb.sleep(1)
     location = "Amsterdam"
     sb.press_keys('input[data-test-id*="search"]', location)

examples/verify_undetected.py

@@ -2,22 +2,16 @@
 Some sites use scripts to detect Selenium, and then block you.
 To evade detection, add --uc as a pytest command-line option."""
 from seleniumbase import BaseCase
-from seleniumbase import DriverContext
 BaseCase.main(__name__, __file__, "--uc", "-s")
 
 
 class UndetectedTest(BaseCase):
     def test_browser_is_undetected(self):
         url = "https://gitlab.com/users/sign_in"
         if not self.undetectable:
-            with DriverContext(uc=True) as driver:
-                driver.uc_activate_cdp_mode(url)
-                driver.uc_gui_click_captcha()
-                driver.assert_text("Username", '[for="user_login"]', timeout=3)
-                print("\n Success! Website did not detect Selenium! ")
-        else:
-            self.activate_cdp_mode(url)
-            self.uc_gui_click_captcha()
-            self.assert_text("Username", '[for="user_login"]', timeout=3)
-            self.post_message("SeleniumBase wasn't detected", duration=4)
-            self._print("\n Success! Website did not detect Selenium! ")
+            self.get_new_driver(undetectable=True)
+        self.activate_cdp_mode(url)
+        self.uc_gui_click_captcha()
+        self.assert_text("Username", '[for="user_login"]', timeout=3)
+        self.post_message("SeleniumBase wasn't detected", duration=4)
+        self._print("\n Success! Website did not detect Selenium! ")

pyproject.toml

@@ -49,6 +49,7 @@ packages = [
     "seleniumbase.resources",
     "seleniumbase.translate",
     "seleniumbase.undetected",
+    "seleniumbase.undetected.cdp_driver",
     "seleniumbase.utilities",
     "seleniumbase.utilities.selenium_grid",
     "seleniumbase.utilities.selenium_ide",

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.32.1"
+__version__ = "4.32.2"

seleniumbase/fixtures/base_case.py

@@ -4090,6 +4090,88 @@ def get_new_driver(
                 "Browser: {%s} is not a valid browser option. "
                 "Valid options = {%s}" % (browser, valid_browsers)
             )
+        # Fix Chrome-130 issues by creating a user-data-dir in advance
+        if (
+            undetectable
+            and (
+                not user_data_dir
+                or not os.path.exists(user_data_dir)
+                or not any(os.scandir(user_data_dir))
+            )
+            and self.browser == "chrome"
+        ):
+            import tempfile
+            if not user_data_dir:
+                user_data_dir = os.path.normpath(tempfile.mkdtemp())
+            self.user_data_dir = user_data_dir
+            sb_config.user_data_dir = user_data_dir
+            try:
+                decoy_driver = browser_launcher.get_driver(
+                    browser_name=browser_name,
+                    headless=headless,
+                    locale_code=locale_code,
+                    use_grid=use_grid,
+                    protocol=protocol,
+                    servername=servername,
+                    port=port,
+                    proxy_string=proxy_string,
+                    proxy_bypass_list=proxy_bypass_list,
+                    proxy_pac_url=proxy_pac_url,
+                    multi_proxy=multi_proxy,
+                    user_agent=user_agent,
+                    cap_file=cap_file,
+                    cap_string=cap_string,
+                    recorder_ext=recorder_ext,
+                    disable_cookies=disable_cookies,
+                    disable_js=disable_js,
+                    disable_csp=disable_csp,
+                    enable_ws=enable_ws,
+                    enable_sync=enable_sync,
+                    use_auto_ext=use_auto_ext,
+                    undetectable=undetectable,
+                    uc_cdp_events=uc_cdp_events,
+                    uc_subprocess=uc_subprocess,
+                    log_cdp_events=log_cdp_events,
+                    no_sandbox=no_sandbox,
+                    disable_gpu=disable_gpu,
+                    headless1=headless1,
+                    headless2=True,
+                    incognito=incognito,
+                    guest_mode=guest_mode,
+                    dark_mode=dark_mode,
+                    devtools=devtools,
+                    remote_debug=remote_debug,
+                    enable_3d_apis=enable_3d_apis,
+                    swiftshader=swiftshader,
+                    ad_block_on=ad_block_on,
+                    host_resolver_rules=host_resolver_rules,
+                    block_images=block_images,
+                    do_not_track=do_not_track,
+                    chromium_arg=chromium_arg,
+                    firefox_arg=firefox_arg,
+                    firefox_pref=firefox_pref,
+                    user_data_dir=user_data_dir,
+                    extension_zip=extension_zip,
+                    extension_dir=extension_dir,
+                    disable_features=disable_features,
+                    binary_location=binary_location,
+                    driver_version=driver_version,
+                    page_load_strategy=page_load_strategy,
+                    use_wire=use_wire,
+                    external_pdf=external_pdf,
+                    test_id=test_id,
+                    mobile_emulator=is_mobile,
+                    device_width=d_width,
+                    device_height=d_height,
+                    device_pixel_ratio=d_p_r,
+                    browser=browser_name,
+                )
+                time.sleep(0.555)
+            except Exception:
+                pass
+            finally:
+                with suppress(Exception):
+                    decoy_driver.quit()
         # Launch a web browser
         new_driver = browser_launcher.get_driver(
             browser_name=browser_name,
@@ -4435,7 +4517,12 @@ def load_cookies(self, name="cookies.txt"):
         """Loads the page cookies from the "saved_cookies" folder."""
         cookies = self.get_saved_cookies(name)
         self.wait_for_ready_state_complete()
+        origin = self.get_origin()
+        trim_origin = origin.split("://")[-1]
         for cookie in cookies:
+            if "domain" in cookie:
+                if cookie["domain"] not in origin:
+                    cookie["domain"] = trim_origin
             if "expiry" in cookie:
                 del cookie["expiry"]
             self.driver.add_cookie(cookie)
@@ -4656,30 +4743,6 @@ def activate_cdp_mode(self, url=None):
         if hasattr(self.driver, "_is_using_uc") and self.driver._is_using_uc:
             self.driver.uc_open_with_cdp_mode(url)
         else:
-            # Fix Chrome-130 issues by creating a user-data-dir in advance
-            if (
-                (
-                    not self.user_data_dir
-                    or not os.path.exists(self.user_data_dir)
-                )
-                and self.browser == "chrome"
-            ):
-                import tempfile
-                user_data_dir = os.path.normpath(tempfile.mkdtemp())
-                self.user_data_dir = user_data_dir
-                sb_config.user_data_dir = user_data_dir
-                try:
-                    driver = self.get_new_driver(
-                        user_data_dir=user_data_dir,
-                        undetectable=True,
-                        headless2=True,
-                    )
-                    time.sleep(0.555)
-                except Exception:
-                    pass
-                finally:
-                    with suppress(Exception):
-                        driver.quit()
             self.get_new_driver(undetectable=True)
             self.driver.uc_open_with_cdp_mode(url)
         self.cdp = self.driver.cdp
@@ -14941,31 +15004,6 @@ def setUp(self, masterqa_mode=False):
                         self.__js_start_time = int(time.time() * 1000.0)
         else:
             # Launch WebDriver for both pytest and pynose
-
-            # Fix Chrome-130 issues by creating a user-data-dir in advance
-            if (
-                self.undetectable
-                and (
-                    not self.user_data_dir
-                    or not os.path.exists(self.user_data_dir)
-                )
-                and self.browser == "chrome"
-            ):
-                import tempfile
-                user_data_dir = os.path.normpath(tempfile.mkdtemp())
-                self.user_data_dir = user_data_dir
-                sb_config.user_data_dir = user_data_dir
-                try:
-                    driver = self.get_new_driver(
-                        user_data_dir=user_data_dir,
-                        headless2=True,
-                    )
-                    time.sleep(0.555)
-                except Exception:
-                    pass
-                finally:
-                    with suppress(Exception):
-                        driver.quit()
             self.driver = self.get_new_driver(
                 browser=self.browser,
                 headless=self.headless,

seleniumbase/plugins/driver_manager.py

@@ -792,12 +792,21 @@ def Driver(
     from seleniumbase.core import browser_launcher
 
     # Fix Chrome-130 issues by creating a user-data-dir in advance
-    if undetectable and not user_data_dir and browser == "chrome":
+    if (
+        undetectable
+        and (
+            not user_data_dir
+            or not os.path.exists(user_data_dir)
+            or not any(os.scandir(user_data_dir))
+        )
+        and browser == "chrome"
+    ):
         import tempfile
         import time
-        user_data_dir = (
-            os.path.normpath(tempfile.mkdtemp())
-        )
+        if not user_data_dir:
+            user_data_dir = (
+                os.path.normpath(tempfile.mkdtemp())
+            )
         try:
             decoy_driver = browser_launcher.get_driver(
                 browser_name=browser_name,

seleniumbase/plugins/sb_manager.py

@@ -1212,30 +1212,6 @@ def SB(
         if not sb_config.multi_proxy:
             proxy_helper.remove_proxy_zip_if_present()
     start_time = time.time()
-    saved_headless2 = headless2
-
-    # Fix Chrome-130 issues by creating a user-data-dir in advance
-    if undetectable and not user_data_dir and browser == "chrome":
-        import tempfile
-        user_data_dir = (
-            os.path.normpath(tempfile.mkdtemp())
-        )
-        sb.user_data_dir = user_data_dir
-        sb_config.user_data_dir = user_data_dir
-        try:
-            decoy = sb
-            decoy.headless2 = True
-            decoy.setUp()
-            decoy.sleep(0.555)
-        except Exception:
-            pass
-        finally:
-            try:
-                decoy.tearDown()
-            except Exception:
-                pass
-            sb.headless2 = saved_headless2
-
     sb.setUp()
     test_passed = True  # This can change later
     teardown_exception = None

setup.py

@@ -317,6 +317,7 @@
         "seleniumbase.resources",
         "seleniumbase.translate",
         "seleniumbase.undetected",
+        "seleniumbase.undetected.cdp_driver",
         "seleniumbase.utilities",
         "seleniumbase.utilities.selenium_grid",
         "seleniumbase.utilities.selenium_ide",