diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index 6da14587b76..26ce7786fb3 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -11,7 +11,7 @@ pipdeptree>=2.12.0 bleach>=6.0.0 docutils>=0.20.1 python-dateutil>=2.8.2 -tqdm>=4.66.0 +tqdm>=4.66.1 nltk>=3.8.1 joblib>=1.3.2 livereload==2.6.3 @@ -21,7 +21,7 @@ Jinja2==3.1.2 click==8.1.6 ghp-import==2.1.0 lunr==0.6.2 -tornado==6.3.2 +tornado==6.3.3 watchdog==3.0.0 cairocffi==1.6.1 cairosvg==2.7.1 diff --git a/requirements.txt b/requirements.txt index 04941be2731..6cc3e1a9e58 100755 --- a/requirements.txt +++ b/requirements.txt @@ -70,7 +70,7 @@ pygments==2.14.0;python_version<"3.7" pygments==2.16.1;python_version>="3.7" pyreadline3==3.4.1;platform_system=="Windows" tabcompleter==1.2.1 -pdbp==1.4.4 +pdbp==1.4.5 colorama==0.4.5;python_version<"3.7" colorama==0.4.6;python_version>="3.7" exceptiongroup==1.1.2;python_version>="3.7" diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 29d7bd19870..13489a2bf99 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.17.2" +__version__ = "4.17.3" diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index de10e48e216..d5f02e4edb9 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -96,16 +96,35 @@ def make_driver_executable_if_not(driver_path): make_executable(driver_path) -def requests_get(url): +def requests_get(url, proxy_string=None): import requests + protocol = "http" + if proxy_string: + if proxy_string.endswith(":443"): + protocol = "https" + elif "socks4" in proxy_string: + protocol = "socks4" + elif "socks5" in proxy_string: + protocol = "socks5" response = None - try: - response = requests.get(url) - except Exception: - # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED - url = url.replace("https://", "http://") - response = requests.get(url) + if proxy_string: + proxies = {protocol: proxy_string} + try: + response = requests.get(url, proxies=proxies) + except Exception: + # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED + url = url.replace("https://", "http://") + time.sleep(0.04) + response = requests.get(url, proxies=proxies) + else: + try: + response = requests.get(url) + except Exception: + # Prevent SSLCertVerificationError / CERTIFICATE_VERIFY_FAILED + url = url.replace("https://", "http://") + time.sleep(0.04) + response = requests.get(url) return response @@ -178,6 +197,7 @@ def find_edgedriver_version_to_use(use_version, driver_version): def has_cf(text): if ( "Just a moment..." in text + or "403 Forbidden" in text or 'id="challenge-error-text"' in text or 'action="/?__cf_chl_f_tk' in text or 'id="challenge-form"' in text @@ -187,10 +207,10 @@ def has_cf(text): return False -def uc_special_open_if_cf(driver, url): +def uc_special_open_if_cf(driver, url, proxy_string=None): if ( (url.startswith("http:") or url.startswith("https:")) - and has_cf(requests_get(url).text) + and has_cf(requests_get(url, proxy_string).text) ): with driver: time.sleep(0.25) @@ -3405,7 +3425,9 @@ def get_local_driver( ) driver.open = driver.get # Save copy of original if uc_activated: - driver.get = lambda url: uc_special_open_if_cf(driver, url) + driver.get = lambda url: uc_special_open_if_cf( + driver, url, proxy_string + ) driver.uc_open = lambda url: uc_open(driver, url) driver.uc_open_with_tab = ( lambda url: uc_open_with_tab(driver, url) diff --git a/setup.py b/setup.py index 9bccc0fa614..740934bfa90 100755 --- a/setup.py +++ b/setup.py @@ -204,7 +204,7 @@ 'pygments==2.16.1;python_version>="3.7"', 'pyreadline3==3.4.1;platform_system=="Windows"', "tabcompleter==1.2.1", - "pdbp==1.4.4", + "pdbp==1.4.5", 'colorama==0.4.5;python_version<"3.7"', 'colorama==0.4.6;python_version>="3.7"', 'exceptiongroup==1.1.2;python_version>="3.7"',