From 660ee46c2260300b73d85292e776d3add7528863 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 26 Aug 2025 22:09:54 -0400 Subject: [PATCH 1/5] Update CDP Mode --- examples/cdp_mode/ReadMe.md | 1 + seleniumbase/core/browser_launcher.py | 1 + seleniumbase/core/sb_cdp.py | 146 ++++++++++++++++++++++++++ 3 files changed, 148 insertions(+) diff --git a/examples/cdp_mode/ReadMe.md b/examples/cdp_mode/ReadMe.md index a21fb1857f2..d62ba379dac 100644 --- a/examples/cdp_mode/ReadMe.md +++ b/examples/cdp_mode/ReadMe.md @@ -465,6 +465,7 @@ sb.cdp.gui_press_keys(keys) sb.cdp.gui_write(text) sb.cdp.gui_click_x_y(x, y, timeframe=0.25) sb.cdp.gui_click_element(selector, timeframe=0.25) +sb.cdp.gui_click_captcha() sb.cdp.gui_drag_drop_points(x1, y1, x2, y2, timeframe=0.35) sb.cdp.gui_drag_and_drop(drag_selector, drop_selector, timeframe=0.35) sb.cdp.gui_click_and_hold(selector, timeframe=0.35) diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 07481de1e2c..6c5fbd1c2dc 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -762,6 +762,7 @@ def uc_open_with_cdp_mode(driver, url=None, **kwargs): cdp.gui_write = CDPM.gui_write cdp.gui_click_x_y = CDPM.gui_click_x_y cdp.gui_click_element = CDPM.gui_click_element + cdp.gui_click_captcha = CDPM.gui_click_captcha cdp.gui_drag_drop_points = CDPM.gui_drag_drop_points cdp.gui_drag_and_drop = CDPM.gui_drag_and_drop cdp.gui_click_and_hold = CDPM.gui_click_and_hold diff --git a/seleniumbase/core/sb_cdp.py b/seleniumbase/core/sb_cdp.py index d42cf67ba4c..a6f39c2d810 100644 --- a/seleniumbase/core/sb_cdp.py +++ b/seleniumbase/core/sb_cdp.py @@ -1629,6 +1629,152 @@ def gui_click_element(self, selector, timeframe=0.25): self.__slow_mode_pause_if_set() self.loop.run_until_complete(self.page.wait()) + def _on_a_cf_turnstile_page(self): + source = self.get_page_source() + if ( + 'data-callback="onCaptchaSuccess"' in source + or "/challenge-platform/scripts/" in source + or 'id="challenge-widget-' in source + or "cf-turnstile-" in source + ): + return True + return False + + def gui_click_captcha(self): + if not self._on_a_cf_turnstile_page(): + return + selector = None + if ( + self.is_element_present('[name*="cf-turnstile-"]') + and self.is_element_present("#challenge-form div > div") + ): + selector = "#challenge-form div > div" + elif ( + self.is_element_present('[name*="cf-turnstile-"]') + and self.is_element_present( + '[style="display: grid;"] div div' + ) + ): + selector = '[style="display: grid;"] div div' + elif ( + self.is_element_present('[name*="cf-turnstile-"]') + and self.is_element_present("[class*=spacer] + div div") + ): + selector = '[class*=spacer] + div div' + elif ( + self.is_element_present('[name*="cf-turnstile-"]') + and self.is_element_present("div.spacer div") + ): + selector = "div.spacer div" + elif ( + self.is_element_present('script[src*="challenges.c"]') + and self.is_element_present( + '[data-testid*="challenge-"] div' + ) + ): + selector = '[data-testid*="challenge-"] div' + elif self.is_element_present( + "div#turnstile-widget div:not([class])" + ): + selector = "div#turnstile-widget div:not([class])" + elif self.is_element_present( + 'form div:not([class]):has(input[name*="cf-turn"])' + ): + selector = 'form div:not([class]):has(input[name*="cf-turn"])' + elif ( + self.is_element_present('[src*="/turnstile/"]') + and self.is_element_present("form div:not(:has(*))") + ): + selector = "form div:not(:has(*))" + elif ( + self.is_element_present('[src*="/turnstile/"]') + and self.is_element_present( + "body > div#check > div:not([class])" + ) + ): + selector = "body > div#check > div:not([class])" + elif self.is_element_present(".cf-turnstile-wrapper"): + selector = ".cf-turnstile-wrapper" + elif self.is_element_present('[class="cf-turnstile"]'): + selector = '[class="cf-turnstile"]' + elif self.is_element_present( + '[data-callback="onCaptchaSuccess"]' + ): + selector = '[data-callback="onCaptchaSuccess"]' + else: + return + if not selector: + return + if ( + self.is_element_present("form") + and ( + self.is_element_present('form[class*="center"]') + or self.is_element_present('form[class*="right"]') + or self.is_element_present('form div[class*="center"]') + or self.is_element_present('form div[class*="right"]') + ) + ): + script = ( + """var $elements = document.querySelectorAll( + 'form[class], form div[class]'); + var index = 0, length = $elements.length; + for(; index < length; index++){ + the_class = $elements[index].getAttribute('class'); + new_class = the_class.replaceAll('center', 'left'); + new_class = new_class.replaceAll('right', 'left'); + $elements[index].setAttribute('class', new_class);}""" + ) + with suppress(Exception): + self.loop.run_until_complete(self.page.evaluate(script)) + self.loop.run_until_complete(self.page.wait()) + elif ( + self.is_element_present("form") + and ( + self.is_element_present('form div[style*="center"]') + or self.is_element_present('form div[style*="right"]') + ) + ): + script = ( + """var $elements = document.querySelectorAll( + 'form[style], form div[style]'); + var index = 0, length = $elements.length; + for(; index < length; index++){ + the_style = $elements[index].getAttribute('style'); + new_style = the_style.replaceAll('center', 'left'); + new_style = new_style.replaceAll('right', 'left'); + $elements[index].setAttribute('style', new_style);}""" + ) + with suppress(Exception): + self.loop.run_until_complete(self.page.evaluate(script)) + self.loop.run_until_complete(self.page.wait()) + elif ( + self.is_element_present("form") + and self.is_element_present( + 'form [id*="turnstile"] > div:not([class])' + ) + ): + script = ( + """var $elements = document.querySelectorAll( + 'form [id*="turnstile"]'); + var index = 0, length = $elements.length; + for(; index < length; index++){ + $elements[index].setAttribute('align', 'left');}""" + ) + with suppress(Exception): + self.loop.run_until_complete(self.page.evaluate(script)) + self.loop.run_until_complete(self.page.wait()) + with suppress(Exception): + element_rect = self.get_gui_element_rect(selector, timeout=1) + e_x = element_rect["x"] + e_y = element_rect["y"] + x = e_x + 32 + if not shared_utils.is_windows(): + y = e_y + 32 + else: + y = e_y + 22 + sb_config._saved_cf_x_y = (x, y) + self.gui_click_x_y(x, y) + def __gui_drag_drop(self, x1, y1, x2, y2, timeframe=0.25, uc_lock=False): self.__install_pyautogui_if_missing() import pyautogui From 8d9d51c55d33b56bf3602f638e2b0f5e86e93eae Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 26 Aug 2025 22:11:28 -0400 Subject: [PATCH 2/5] Update CDP Mode examples --- examples/cdp_mode/raw_cdp_turnstile.py | 8 ++++++++ examples/cdp_mode/raw_cf_clearance.py | 18 ++++++++++++++++++ examples/cdp_mode/raw_consecutive_c.py | 4 ++-- examples/cdp_mode/raw_gitlab.py | 2 +- examples/cdp_mode/raw_glassdoor.py | 2 +- examples/cdp_mode/raw_indeed.py | 2 +- 6 files changed, 31 insertions(+), 5 deletions(-) create mode 100644 examples/cdp_mode/raw_cdp_turnstile.py create mode 100644 examples/cdp_mode/raw_cf_clearance.py diff --git a/examples/cdp_mode/raw_cdp_turnstile.py b/examples/cdp_mode/raw_cdp_turnstile.py new file mode 100644 index 00000000000..6a5aab576a0 --- /dev/null +++ b/examples/cdp_mode/raw_cdp_turnstile.py @@ -0,0 +1,8 @@ +from seleniumbase import sb_cdp + +url = "https://seleniumbase.io/apps/turnstile" +sb = sb_cdp.Chrome(url) +sb.gui_click_captcha() +sb.assert_element("img#captcha-success") +sb.sleep(2) +sb.driver.stop() diff --git a/examples/cdp_mode/raw_cf_clearance.py b/examples/cdp_mode/raw_cf_clearance.py new file mode 100644 index 00000000000..f099f5cf464 --- /dev/null +++ b/examples/cdp_mode/raw_cf_clearance.py @@ -0,0 +1,18 @@ +from seleniumbase import sb_cdp + +url = "https://gitlab.com/users/sign_in" +sb = sb_cdp.Chrome(url) +sb.sleep(2.2) +sb.gui_click_captcha() +sb.sleep(2) +cf_cookie = None +all_cookies = sb.get_all_cookies() +for cookie in all_cookies: + if cookie.name == 'cf_clearance': + cf_cookie = cookie + break +if cf_cookie: + print("cf_clearance cookie: %s" % cf_cookie.value) +else: + print("Didn't find the cf_clearance cookie!") +sb.driver.stop() diff --git a/examples/cdp_mode/raw_consecutive_c.py b/examples/cdp_mode/raw_consecutive_c.py index 5ac1b7aa5ec..378949de1d0 100644 --- a/examples/cdp_mode/raw_consecutive_c.py +++ b/examples/cdp_mode/raw_consecutive_c.py @@ -4,8 +4,8 @@ with SB(uc=True, test=True) as sb: url = "https://sms-man.com/login" sb.activate_cdp_mode(url) - sb.sleep(2) + sb.sleep(2.2) sb.uc_gui_click_captcha() - sb.sleep(2) + sb.sleep(2.2) sb.uc_gui_click_captcha() sb.sleep(2) diff --git a/examples/cdp_mode/raw_gitlab.py b/examples/cdp_mode/raw_gitlab.py index 77f4d2e3d6f..3d645752cb2 100644 --- a/examples/cdp_mode/raw_gitlab.py +++ b/examples/cdp_mode/raw_gitlab.py @@ -3,7 +3,7 @@ with SB(uc=True, test=True, locale="en") as sb: url = "https://gitlab.com/users/sign_in" sb.activate_cdp_mode(url) - sb.sleep(1) + sb.sleep(2.2) sb.uc_gui_click_captcha() sb.assert_text("Username", '[for="user_login"]', timeout=3) sb.assert_element('label[for="user_login"]') diff --git a/examples/cdp_mode/raw_glassdoor.py b/examples/cdp_mode/raw_glassdoor.py index 6fc08ade69f..0dfd2086a2e 100644 --- a/examples/cdp_mode/raw_glassdoor.py +++ b/examples/cdp_mode/raw_glassdoor.py @@ -3,7 +3,7 @@ with SB(uc=True, test=True, ad_block=True) as sb: url = "https://www.glassdoor.com/Reviews/index.htm" sb.activate_cdp_mode(url) - sb.sleep(2) + sb.sleep(2.2) sb.uc_gui_click_captcha() sb.highlight('[data-test="global-nav-glassdoor-logo"]') sb.highlight('[data-test="site-header-companies"]') diff --git a/examples/cdp_mode/raw_indeed.py b/examples/cdp_mode/raw_indeed.py index 27b26744e94..e25f1e00451 100644 --- a/examples/cdp_mode/raw_indeed.py +++ b/examples/cdp_mode/raw_indeed.py @@ -3,7 +3,7 @@ with SB(uc=True, test=True) as sb: url = "https://www.indeed.com/companies/search" sb.activate_cdp_mode(url) - sb.sleep(2) + sb.sleep(2.2) sb.uc_gui_click_captcha() sb.sleep(1) company = "NASA Jet Propulsion Laboratory" From aa56109630f025c44bcb9c1316dbc76c83d29545 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 26 Aug 2025 22:11:58 -0400 Subject: [PATCH 3/5] Refresh Python dependencies --- mkdocs_build/requirements.txt | 2 +- requirements.txt | 8 ++++---- setup.py | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index b266ba7f79a..e892f4d8ea7 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -14,7 +14,7 @@ pathspec==0.12.1 Babel==2.17.0 paginate==0.5.7 mkdocs==1.6.1 -mkdocs-material==9.6.17 +mkdocs-material==9.6.18 mkdocs-exclude-search==0.6.6 mkdocs-simple-hooks==0.1.5 mkdocs-material-extensions==1.3.1 diff --git a/requirements.txt b/requirements.txt index a244ce43091..883faea17cd 100755 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ fasteners>=0.20 mycdp>=1.2.0 pynose>=1.5.4 platformdirs>=4.3.6;python_version<"3.9" -platformdirs>=4.3.8;python_version>="3.9" +platformdirs>=4.4.0;python_version>="3.9" typing-extensions>=4.13.2 sbvirtualdisplay>=1.4.0 MarkupSafe==2.1.5;python_version<"3.9" @@ -35,7 +35,7 @@ chardet==5.2.0 charset-normalizer>=3.4.3,<4 urllib3>=1.26.20,<2;python_version<"3.10" urllib3>=1.26.20,<2.6.0;python_version>="3.10" -requests==2.32.4 +requests>=2.32.5,<2.33 sniffio==1.3.1 h11==0.16.0 outcome==1.3.0.post0 @@ -66,7 +66,7 @@ pytest-xdist==3.8.0;python_version>="3.9" parameterized==0.9.0 behave==1.2.6 soupsieve==2.7 -beautifulsoup4==4.13.4 +beautifulsoup4>=4.13.5,<4.14 pyotp==2.9.0 python-xlib==0.33;platform_system=="Linux" markdown-it-py==3.0.0;python_version<"3.10" @@ -78,7 +78,7 @@ rich>=14.1.0,<15 # ("pip install -r requirements.txt" also installs this, but "pip install -e ." won't.) coverage>=7.6.1;python_version<"3.9" -coverage>=7.10.4;python_version>="3.9" +coverage>=7.10.5;python_version>="3.9" pytest-cov>=5.0.0;python_version<"3.9" pytest-cov>=6.2.1;python_version>="3.9" flake8==5.0.4;python_version<"3.9" diff --git a/setup.py b/setup.py index 967e93587fa..7398e60925a 100755 --- a/setup.py +++ b/setup.py @@ -163,7 +163,7 @@ "mycdp>=1.2.0", "pynose>=1.5.4", 'platformdirs>=4.3.6;python_version<"3.9"', - 'platformdirs>=4.3.8;python_version>="3.9"', + 'platformdirs>=4.4.0;python_version>="3.9"', 'typing-extensions>=4.13.2', "sbvirtualdisplay>=1.4.0", 'MarkupSafe==2.1.5;python_version<"3.9"', @@ -183,7 +183,7 @@ 'charset-normalizer>=3.4.3,<4', 'urllib3>=1.26.20,<2;python_version<"3.10"', 'urllib3>=1.26.20,<2.6.0;python_version>="3.10"', - 'requests==2.32.4', + 'requests>=2.32.5,<2.33', 'sniffio==1.3.1', 'h11==0.16.0', 'outcome==1.3.0.post0', @@ -214,7 +214,7 @@ 'parameterized==0.9.0', "behave==1.2.6", # Newer ones had issues 'soupsieve==2.7', - "beautifulsoup4==4.13.4", + "beautifulsoup4>=4.13.5,<4.14", 'pyotp==2.9.0', 'python-xlib==0.33;platform_system=="Linux"', 'markdown-it-py==3.0.0;python_version<"3.10"', @@ -235,7 +235,7 @@ # Usage: coverage run -m pytest; coverage html; coverage report "coverage": [ 'coverage>=7.6.1;python_version<"3.9"', - 'coverage>=7.10.4;python_version>="3.9"', + 'coverage>=7.10.5;python_version>="3.9"', 'pytest-cov>=5.0.0;python_version<"3.9"', 'pytest-cov>=6.2.1;python_version>="3.9"', ], From 54d0ab43337ccfe75a8b3059d7d42b39ebb65486 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 26 Aug 2025 22:12:14 -0400 Subject: [PATCH 4/5] Version 4.41.2 --- seleniumbase/__version__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 53f8d5fff0b..d8c30a552e8 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.41.1" +__version__ = "4.41.2" From 1bd12ba7831099e20e215798b77632a76eeea132 Mon Sep 17 00:00:00 2001 From: Michael Mintz Date: Tue, 26 Aug 2025 22:21:58 -0400 Subject: [PATCH 5/5] Refresh a Python dependency --- requirements.txt | 3 ++- setup.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 883faea17cd..acca77078d9 100755 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,8 @@ chardet==5.2.0 charset-normalizer>=3.4.3,<4 urllib3>=1.26.20,<2;python_version<"3.10" urllib3>=1.26.20,<2.6.0;python_version>="3.10" -requests>=2.32.5,<2.33 +requests==2.32.4;python_version<"3.9" +requests>=2.32.5,<2.33;python_version>="3.9" sniffio==1.3.1 h11==0.16.0 outcome==1.3.0.post0 diff --git a/setup.py b/setup.py index 7398e60925a..1d222563bf9 100755 --- a/setup.py +++ b/setup.py @@ -183,7 +183,8 @@ 'charset-normalizer>=3.4.3,<4', 'urllib3>=1.26.20,<2;python_version<"3.10"', 'urllib3>=1.26.20,<2.6.0;python_version>="3.10"', - 'requests>=2.32.5,<2.33', + 'requests==2.32.4;python_version<"3.9"', + 'requests>=2.32.5,<2.33;python_version>="3.9"', 'sniffio==1.3.1', 'h11==0.16.0', 'outcome==1.3.0.post0',