Chromium overhaul with CDP Mode updates

examples/cdp_mode/ReadMe.md

@@ -499,15 +499,16 @@ sb.click_captcha()
 sb.gui_press_key(key)
 sb.gui_press_keys(keys)
 sb.gui_write(text)
-sb.gui_click_x_y(x, y, timeframe=0.25)
-sb.gui_click_element(selector, timeframe=0.25)
-sb.gui_click_with_offset(selector, x, y, timeframe=0.25, center=False)
+sb.gui_click_x_y(x, y, timeframe=0.27)
+sb.gui_click_element(selector, timeframe=0.27)
+sb.gui_click_with_offset(selector, x, y, timeframe=0.27, center=False)
 sb.gui_click_captcha()
 sb.gui_drag_drop_points(x1, y1, x2, y2, timeframe=0.35)
 sb.gui_drag_and_drop(drag_selector, drop_selector, timeframe=0.35)
 sb.gui_click_and_hold(selector, timeframe=0.35)
-sb.gui_hover_x_y(x, y)
-sb.gui_hover_element(selector)
+sb.gui_move_to_element(selector, timeframe=0.27)
+sb.gui_hover_x_y(x, y, timeframe=0.27)
+sb.gui_hover_element(selector, timeframe=0.27)
 sb.gui_hover_and_click(hover_selector, click_selector)
 sb.hover_element(selector)
 sb.hover_and_click(hover_selector, click_selector)
@@ -731,7 +732,7 @@ element.click()
 element.click_with_offset(x, y, center=False)
 element.flash(duration=0.5, color="EE4488")
 element.focus()
-element.gui_click(timeframe=0.25)
+element.gui_click(timeframe=0.27)
 element.highlight_overlay()
 element.is_in_viewport()
 element.mouse_click()

examples/cdp_mode/raw_priceline.py

@@ -1,4 +1,5 @@
-"""Priceline does a lot of A/B testing. Selectors change frequently."""
+"""An example of bypassing PerimeterX detection on Priceline.
+(PyAutoGUI is installed at runtime if it's not installed.)"""
 from seleniumbase import SB
 
 with SB(uc=True, test=True, locale="en", guest=True, pls="none") as sb:
@@ -8,10 +9,11 @@
     input_selector = 'input[name="endLocation"]'
     if not sb.is_element_present(input_selector):
         input_selector = "div.location-input input"
+    sb.gui_hover_element(input_selector)
     sb.mouse_click(input_selector)
-    sb.sleep(0.5)
     location = "Portland, OR"
     selection = "Oregon, United States"  # (Dropdown option)
+    sb.gui_hover_element(input_selector)
     sb.press_keys(input_selector, location)
     sb.sleep(0.5)
     sb.click(selection)

examples/cdp_mode/raw_southwest.py

@@ -1,3 +1,5 @@
+"""An example of bypassing bot-detection on Southwest.com.
+(PyAutoGUI is installed at runtime if it's not installed.)"""
 from seleniumbase import SB
 
 with SB(uc=True, test=True, locale="en", ad_block=True) as sb:

help_docs/cdp_mode_methods.md

@@ -143,15 +143,16 @@ sb.click_captcha()
 sb.gui_press_key(key)
 sb.gui_press_keys(keys)
 sb.gui_write(text)
-sb.gui_click_x_y(x, y, timeframe=0.25)
-sb.gui_click_element(selector, timeframe=0.25)
-sb.gui_click_with_offset(selector, x, y, timeframe=0.25, center=False)
+sb.gui_click_x_y(x, y, timeframe=0.27)
+sb.gui_click_element(selector, timeframe=0.27)
+sb.gui_click_with_offset(selector, x, y, timeframe=0.27, center=False)
 sb.gui_click_captcha()
 sb.gui_drag_drop_points(x1, y1, x2, y2, timeframe=0.35)
 sb.gui_drag_and_drop(drag_selector, drop_selector, timeframe=0.35)
 sb.gui_click_and_hold(selector, timeframe=0.35)
-sb.gui_hover_x_y(x, y)
-sb.gui_hover_element(selector)
+sb.gui_move_to_element(selector, timeframe=0.27)
+sb.gui_hover_x_y(x, y, timeframe=0.27)
+sb.gui_hover_element(selector, timeframe=0.27)
 sb.gui_hover_and_click(hover_selector, click_selector)
 sb.hover_element(selector)
 sb.hover_and_click(hover_selector, click_selector)
@@ -375,7 +376,7 @@ element.click()
 element.click_with_offset(x, y, center=False)
 element.flash(duration=0.5, color="EE4488")
 element.focus()
-element.gui_click(timeframe=0.25)
+element.gui_click(timeframe=0.27)
 element.highlight_overlay()
 element.is_in_viewport()
 element.mouse_click()

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.49.13"
+__version__ = "4.49.14"

seleniumbase/console_scripts/sb_install.py

@@ -734,6 +734,26 @@ def main(override=None, intel_for_uc=None, force_uc=None):
                 platform_code = "Win"
             file_name = "chrome-win.zip"
         revision = get_chromium_latest_revision(platform_code)
+        arg_join = " ".join(sys.argv)
+        if "--revision" in arg_join:
+            rev_str = None
+            if "--revision=" in arg_join:
+                rev_str = arg_join.split("--revision=")[1].split(" ")[0]
+            elif "--revision " in arg_join:
+                rev_str = arg_join.split("--revision ")[1].split(" ")[0]
+            if rev_str:
+                if rev_str.startswith('"') and rev_str.endswith('"'):
+                    rev_str = rev_str[1:-1]
+                elif rev_str.startswith("'") and rev_str.endswith("'"):
+                    rev_str = rev_str[1:-1]
+                with suppress(Exception):
+                    if int(rev_str) > 1000000:
+                        revision = rev_str
+                    else:
+                        print(
+                            "Invalid revision number! Defaulting to %s!"
+                            % revision
+                        )
         msg = c2 + "Chromium revision to download" + cr
         p_version = c3 + revision + cr
         log_d("\n*** %s = %s" % (msg, p_version))

seleniumbase/core/browser_launcher.py

@@ -851,8 +851,9 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs):
     cdp.gui_drag_drop_points = CDPM.gui_drag_drop_points
     cdp.gui_drag_and_drop = CDPM.gui_drag_and_drop
     cdp.gui_click_and_hold = CDPM.gui_click_and_hold
-    cdp.gui_hover_x_y = CDPM.gui_hover_x_y
+    cdp.gui_move_to_element = CDPM.gui_move_to_element
     cdp.gui_hover_element = CDPM.gui_hover_element
+    cdp.gui_hover_x_y = CDPM.gui_hover_x_y
     cdp.gui_hover_and_click = CDPM.gui_hover_and_click
     cdp.hover_element = CDPM.hover_element
     cdp.hover_and_click = CDPM.hover_and_click
@@ -2763,6 +2764,7 @@ def _set_chrome_options(
         chrome_options.add_argument("--disable-3d-apis")
     if headless or headless2 or is_using_uc(undetectable, browser_name):
         chrome_options.add_argument("--disable-renderer-backgrounding")
+    chrome_options.add_argument("--disable-background-networking")
     chrome_options.add_argument("--disable-backgrounding-occluded-windows")
     chrome_options.add_argument("--disable-client-side-phishing-detection")
     chrome_options.add_argument("--disable-device-discovery-notifications")
@@ -2779,7 +2781,6 @@ def _set_chrome_options(
     chrome_options.add_argument("--disable-domain-reliability")
     chrome_options.add_argument("--disable-breakpad")
     included_disabled_features = []
-    included_disabled_features.append("OptimizationHints")
     included_disabled_features.append("OptimizationHintsFetching")
     included_disabled_features.append("Translate")
     included_disabled_features.append("ComponentUpdater")
@@ -2800,14 +2801,19 @@ def _set_chrome_options(
     included_disabled_features.append("UnifiedWebBluetooth")
     included_disabled_features.append("WebAuthentication")
     included_disabled_features.append("PasskeyAuth")
+    included_disabled_features.append("MediaRouter")
+    included_disabled_features.append("DialMediaRouteProvider")
+    included_disabled_features.append("WebRtcHideLocalIpsWithMdns")
+    if is_using_uc(undetectable, browser_name):
+        included_disabled_features.append("IsolateOrigins")
+        included_disabled_features.append("site-per-process")
     for item in extra_disabled_features:
         if item not in included_disabled_features:
             included_disabled_features.append(item)
     d_f_string = ",".join(included_disabled_features)
     chrome_options.add_argument("--disable-features=%s" % d_f_string)
     chrome_options.add_argument("--enable-unsafe-extension-debugging")
-    if proxy_string:
-        chrome_options.add_argument("--test-type")
+    chrome_options.add_argument("--test-type")
     if proxy_auth or sb_config._ext_dirs:
         if not is_using_uc(undetectable, browser_name):
             chrome_options.add_argument("--remote-debugging-pipe")
@@ -2825,13 +2831,12 @@ def _set_chrome_options(
         chrome_options.add_argument("--animation-duration-scale=0")
         chrome_options.add_argument("--wm-window-animations-disabled")
         chrome_options.add_argument("--enable-privacy-sandbox-ads-apis")
+        chrome_options.add_argument("--disable-auto-reload")
         chrome_options.add_argument("--disable-background-timer-throttling")
         # Prevent new tabs opened by Selenium from being blocked:
         chrome_options.add_argument("--disable-popup-blocking")
         # Skip remaining options that trigger anti-bot services
         return chrome_options
-    if not proxy_string:
-        chrome_options.add_argument("--test-type")
     chrome_options.add_argument("--log-level=3")
     chrome_options.add_argument("--no-first-run")
     chrome_options.add_argument("--allow-insecure-localhost")
@@ -4803,7 +4808,6 @@ def get_local_driver(
         edge_options.add_argument("--disable-domain-reliability")
         edge_options.add_argument("--disable-breakpad")
         included_disabled_features = []
-        included_disabled_features.append("OptimizationHints")
         included_disabled_features.append("OptimizationHintsFetching")
         included_disabled_features.append("Translate")
         included_disabled_features.append("ComponentUpdater")

seleniumbase/core/sb_cdp.py

@@ -1,4 +1,4 @@
-"""Add CDP methods to extend the driver"""
+"""The CDP Mode API (sync format), which wraps the async format."""
 import asyncio
 import fasteners
 import mycdp
@@ -2035,7 +2035,7 @@ def gui_write(self, text):
         self.__slow_mode_pause_if_set()
         self.loop.run_until_complete(self.page.sleep(0.025))
 
-    def __gui_click_x_y(self, x, y, timeframe=0.25, uc_lock=False):
+    def __gui_click_x_y(self, x, y, timeframe=0.27, uc_lock=False):
         self.__install_pyautogui_if_missing()
         import pyautogui
         pyautogui = self.__get_configured_pyautogui(pyautogui)
@@ -2065,7 +2065,7 @@ def __gui_click_x_y(self, x, y, timeframe=0.25, uc_lock=False):
                 print(" <DEBUG> pyautogui.click(%s, %s)" % (x, y))
             pyautogui.click(x=x, y=y)
 
-    def gui_click_x_y(self, x, y, timeframe=0.25):
+    def gui_click_x_y(self, x, y, timeframe=0.27):
         gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
         with gui_lock:  # Prevent issues with multiple processes
             self.__make_sure_pyautogui_lock_is_writable()
@@ -2097,7 +2097,7 @@ def gui_click_x_y(self, x, y, timeframe=0.25):
             self.bring_active_window_to_front()
             self.__gui_click_x_y(x, y, timeframe=timeframe, uc_lock=False)
 
-    def gui_click_element(self, selector, timeframe=0.25):
+    def gui_click_element(self, selector, timeframe=0.27):
         self.__slow_mode_pause_if_set()
         x, y = self.get_gui_element_center(selector)
         self.__add_light_pause()
@@ -2106,7 +2106,7 @@ def gui_click_element(self, selector, timeframe=0.25):
         self.loop.run_until_complete(self.page.wait(0.2))
 
     def gui_click_with_offset(
-        self, selector, x, y, timeframe=0.25, center=False
+        self, selector, x, y, timeframe=0.27, center=False
     ):
         """Click an element at an {X,Y}-offset location.
         {0,0} is the top-left corner of the element.
@@ -2577,7 +2577,7 @@ def __click_captcha(self, use_cdp=False):
             return True
         return False
 
-    def __gui_drag_drop(self, x1, y1, x2, y2, timeframe=0.25, uc_lock=False):
+    def __gui_drag_drop(self, x1, y1, x2, y2, timeframe=0.27, uc_lock=False):
         self.__install_pyautogui_if_missing()
         import pyautogui
         pyautogui = self.__get_configured_pyautogui(pyautogui)
@@ -2672,7 +2672,11 @@ def gui_click_and_hold(self, selector, timeframe=0.35):
         self.__add_light_pause()
         self.gui_drag_drop_points(x, y, x, y, timeframe=timeframe)
 
-    def __gui_hover_x_y(self, x, y, timeframe=0.25, uc_lock=False):
+    def gui_move_to_element(self, selector, timeframe=0.27):
+        """Same as gui_hover_element()"""
+        return self.gui_hover_element(selector, timeframe=timeframe)
+
+    def __gui_hover_x_y(self, x, y, timeframe=0.27, uc_lock=False):
         self.__install_pyautogui_if_missing()
         import pyautogui
         pyautogui = self.__get_configured_pyautogui(pyautogui)
@@ -2697,7 +2701,7 @@ def __gui_hover_x_y(self, x, y, timeframe=0.25, uc_lock=False):
             pyautogui.moveTo(x, y, timeframe, pyautogui.easeOutQuad)
             time.sleep(0.056)
 
-    def gui_hover_x_y(self, x, y, timeframe=0.25):
+    def gui_hover_x_y(self, x, y, timeframe=0.27):
         gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
         with gui_lock:  # Prevent issues with multiple processes
             self.__install_pyautogui_if_missing()
@@ -2748,7 +2752,12 @@ def gui_hover_x_y(self, x, y, timeframe=0.25):
             self.bring_active_window_to_front()
             self.__gui_hover_x_y(x, y, timeframe=timeframe, uc_lock=False)
 
-    def gui_hover_element(self, selector, timeframe=0.25):
+    def gui_hover_element(self, selector, timeframe=0.27):
+        try:
+            self.__verify_pyautogui_has_a_headed_browser()
+        except Exception:
+            self.hover_element(selector, timeframe=timeframe)
+            return
         self.__slow_mode_pause_if_set()
         element_rect = self.get_gui_element_rect(selector)
         width = element_rect["width"]
@@ -2760,7 +2769,7 @@ def gui_hover_element(self, selector, timeframe=0.25):
             self.__slow_mode_pause_if_set()
         self.loop.run_until_complete(self.page.wait(0.1))
 
-    def hover_element(self, selector, timeframe=0.25):
+    def hover_element(self, selector, timeframe=0.27):
         element = self.select(selector)
         gui_lock = FileLock(constants.MultiBrowser.PYAUTOGUILOCK)
         with gui_lock:

seleniumbase/fixtures/base_case.py

@@ -5151,6 +5151,14 @@ def activate_cdp_mode(self, url=None, **kwargs):
             self.gui_drag_and_drop = self.cdp.gui_drag_and_drop
         if hasattr(self.cdp, "gui_drag_drop_points"):
             self.gui_drag_drop_points = self.cdp.gui_drag_drop_points
+        if hasattr(self.cdp, "gui_hover_and_click"):
+            self.gui_hover_and_click = self.cdp.gui_hover_and_click
+        if hasattr(self.cdp, "gui_hover_element"):
+            self.gui_hover_element = self.cdp.gui_hover_element
+        if hasattr(self.cdp, "gui_hover_x_y"):
+            self.gui_hover_x_y = self.cdp.gui_hover_x_y
+        if hasattr(self.cdp, "gui_move_to_element"):
+            self.gui_move_to_element = self.cdp.gui_move_to_element
         if hasattr(self.cdp, "highlight_overlay"):
             self.highlight_overlay = self.cdp.highlight_overlay
         if hasattr(self.cdp, "js_dumps"):

seleniumbase/undetected/cdp_driver/config.py

@@ -250,6 +250,7 @@ def __init__(
             "--disable-ipc-flooding-protection",
             "--disable-background-timer-throttling",
             "--disable-search-engine-choice-screen",
+            "--disable-background-networking",
             "--disable-backgrounding-occluded-windows",
             "--disable-client-side-phishing-detection",
             "--disable-device-discovery-notifications",
@@ -314,9 +315,9 @@ def __call__(self):
             "SidePanelPinning,UserAgentClientHint,PrivacySandboxSettings4,"
             "OptimizationHintsFetching,InterestFeedContentSuggestions,"
             "Bluetooth,WebBluetooth,UnifiedWebBluetooth,ComponentUpdater,"
-            "DisableLoadExtensionCommandLineSwitch,"
-            "OmniboxUIFeedback,OmniboxPopupShortcut,"
-            "WebAuthentication,PasskeyAuth"
+            "DisableLoadExtensionCommandLineSwitch,WebAuthentication,"
+            "OmniboxUIFeedback,OmniboxPopupShortcut,PasskeyAuth,"
+            "MediaRouter,DialMediaRouteProvider,WebRtcHideLocalIpsWithMdns"
         ]
         if self.expert:
             args += [