From e6a08584c05579e5ff0dedf6ca8f6d7ae57dcdc5 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Fri, 16 Feb 2024 04:35:37 +0000 Subject: [PATCH 01/32] Update README.md thanks @kimboslice99 --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index b5208fc..38526fc 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,17 @@ response = requests.post(url, headers=headers, json=data) print(response.text) ``` +Example PowerShell request: +```ps1 +$body = @{ + cmd = "request.get" + url = "http://www.google.com/" + maxTimeout = 60000 +} | ConvertTo-Json + +irm -UseBasicParsing 'http://localhost:8191/v1' -Headers @{"Content-Type"="application/json"} -Method Post -Body $body +``` + ### Commands #### + `sessions.create` From 227bd7ac72ad390de52dd12fc5cd7e81498beb2e Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 17 Feb 2024 00:50:14 +0000 Subject: [PATCH 02/32] Update Chrome downloads (#1070) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- src/build_package.py | 2 +- src/undetected_chromedriver/patcher.py | 2 +- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7a18f2..b0eef1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.14 (2024/02/17) + +* Update Chrome downloads. Thanks @opemvbs + ## v3.3.13 (2024/01/07) * Fix too many open files error diff --git a/Dockerfile b/Dockerfile index 791e144..97d21ac 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.13 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.13 +# docker build -t ngosang/flaresolverr:3.3.14 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.13 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.14 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.13 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.13 +# docker buildx build -t ngosang/flaresolverr:3.3.14 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14 diff --git a/package.json b/package.json index 531f810..a551f80 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.13", + "version": "3.3.14", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" diff --git a/src/build_package.py b/src/build_package.py index db85a30..5462a7b 100644 --- a/src/build_package.py +++ b/src/build_package.py @@ -25,7 +25,7 @@ def clean_files(): def download_chromium(): # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/ - revision = "1140001" if os.name == 'nt' else '1140000' + revision = "1250005" if os.name == 'nt' else '1250006' arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64' dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux' dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome') diff --git a/src/undetected_chromedriver/patcher.py b/src/undetected_chromedriver/patcher.py index 78b7617..4114e56 100644 --- a/src/undetected_chromedriver/patcher.py +++ b/src/undetected_chromedriver/patcher.py @@ -290,7 +290,7 @@ class Patcher(object): download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name) else: zip_name = zip_name.replace("_", "-", 1) - download_url = "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/%s/%s/%s" + download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s" download_url %= (self.version_full.vstring, self.platform_name, zip_name) logger.debug("downloading from %s" % download_url) From 3ca6d08f4160273380b81eb9949ee72897c8bdeb Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 17 Feb 2024 01:15:32 +0000 Subject: [PATCH 03/32] Hotfix for Linux build - some Chrome files no longer exist (#1071) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- src/build_package.py | 3 +-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0eef1a..4c69c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.14-hotfix (2024/02/17) + +* Hotfix for Linux build - some Chrome files no longer exist + ## v3.3.14 (2024/02/17) * Update Chrome downloads. Thanks @opemvbs diff --git a/Dockerfile b/Dockerfile index 97d21ac..f615072 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.14 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14 +# docker build -t ngosang/flaresolverr:3.3.14-hotfix . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14-hotfix # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14 +# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14-hotfix diff --git a/package.json b/package.json index a551f80..8dfe3fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.14", + "version": "3.3.14-hotfix", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" diff --git a/src/build_package.py b/src/build_package.py index 5462a7b..94b6416 100644 --- a/src/build_package.py +++ b/src/build_package.py @@ -59,8 +59,7 @@ def download_chromium(): # Give executable permissions for *nix # file * | grep executable | cut -d: -f1 print("Giving executable permissions...") - execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'nacl_helper', - 'nacl_helper_bootstrap', 'nacl_irt_x86_64.nexe', 'xdg-mime', 'xdg-settings'] + execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'xdg-mime', 'xdg-settings'] for exec_file in execs: exec_path = os.path.join(chrome_path, exec_file) os.chmod(exec_path, 0o755) From d21a332519de6f9dfc6bdc875d112930e7f475ca Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 17 Feb 2024 05:53:45 +0000 Subject: [PATCH 04/32] Hotfix 2 - bad Chromium build, instances failed to terminate (#1072) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- src/build_package.py | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c69c29..3a974e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.14-hotfix2 (2024/02/17) + +* Hotfix 2 - bad Chromium build, instances failed to terminate + ## v3.3.14-hotfix (2024/02/17) * Hotfix for Linux build - some Chrome files no longer exist diff --git a/Dockerfile b/Dockerfile index f615072..0ffd16e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.14-hotfix . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14-hotfix +# docker build -t ngosang/flaresolverr:3.3.14-hotfix2 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14-hotfix2 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix2 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14-hotfix +# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix2 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14-hotfix2 diff --git a/package.json b/package.json index 8dfe3fd..0cac948 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.14-hotfix", + "version": "3.3.14-hotfix2", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" diff --git a/src/build_package.py b/src/build_package.py index 94b6416..5ff4956 100644 --- a/src/build_package.py +++ b/src/build_package.py @@ -25,7 +25,7 @@ def clean_files(): def download_chromium(): # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/ - revision = "1250005" if os.name == 'nt' else '1250006' + revision = "1260008" if os.name == 'nt' else '1260015' arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64' dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux' dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome') From 043f18b231b4f409080b2b5c4421ce0f4cac7dec Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 17 Feb 2024 19:28:06 +0000 Subject: [PATCH 05/32] Bump UC version to 3.5.5 --- src/undetected_chromedriver/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/undetected_chromedriver/__init__.py b/src/undetected_chromedriver/__init__.py index 9bd9823..755f7a4 100644 --- a/src/undetected_chromedriver/__init__.py +++ b/src/undetected_chromedriver/__init__.py @@ -17,7 +17,7 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam) from __future__ import annotations -__version__ = "3.5.4" +__version__ = "3.5.5" import json import logging From 800866d03390c70c14c4caa0955080284ca3b604 Mon Sep 17 00:00:00 2001 From: Tadas Gedgaudas <83356979+tadasgedgaudas@users.noreply.github.com> Date: Wed, 21 Feb 2024 01:41:02 +0200 Subject: [PATCH 06/32] Fix looping challenges. #1036 (#1065) Co-authored-by: Tadas Gedgaudas Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> --- src/flaresolverr_service.py | 28 ++++++++++++++++++++++------ src/utils.py | 2 ++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 6cef3e4..0987fed 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -287,20 +287,35 @@ def click_verify(driver: WebDriver): time.sleep(2) +def get_correct_window(driver: WebDriver) -> WebDriver: + if len(driver.window_handles) > 1: + for window_handle in driver.window_handles: + driver.switch_to.window(window_handle) + current_url = driver.current_url + if not current_url.startswith("devtools://devtools"): + return driver + return driver + + +def access_page(driver: WebDriver, url: str) -> None: + driver.get(url) + driver.start_session() + driver.start_session() # required to bypass Cloudflare + + def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT: res = ChallengeResolutionT({}) res.status = STATUS_OK res.message = "" + # navigate to the page logging.debug(f'Navigating to... {req.url}') - driver.get(req.url) - driver.start_session() # required to bypass Cloudflare if method == 'POST': _post_request(req, driver) else: - driver.get(req.url) - driver.start_session() # required to bypass Cloudflare + access_page(driver, req.url) + driver = get_correct_window(driver) # set cookies if required if req.cookies is not None and len(req.cookies) > 0: @@ -312,8 +327,8 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge if method == 'POST': _post_request(req, driver) else: - driver.get(req.url) - driver.start_session() # required to bypass Cloudflare + access_page(driver, req.url) + driver = get_correct_window(driver) # wait for the page if utils.get_config_log_html(): @@ -433,4 +448,5 @@ def _post_request(req: V1RequestBase, driver: WebDriver): """ driver.get("data:text/html;charset=utf-8," + html_content) + driver.start_session() driver.start_session() # required to bypass Cloudflare diff --git a/src/utils.py b/src/utils.py index 3672511..b812387 100644 --- a/src/utils.py +++ b/src/utils.py @@ -164,6 +164,8 @@ def get_webdriver(proxy: dict = None) -> WebDriver: # For normal headless mode: # options.add_argument('--headless') + options.add_argument("--auto-open-devtools-for-tabs") + # if we are inside the Docker container, we avoid downloading the driver driver_exe_path = None version_main = None From aaf29be8e169c170de4df9502a516b93e94c9648 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Tue, 20 Feb 2024 23:48:23 +0000 Subject: [PATCH 07/32] Bump version 3.3.15 (#1088) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a974e1..b215861 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.15 (2024/02/20) + +* Fix looping challenges + ## v3.3.14-hotfix2 (2024/02/17) * Hotfix 2 - bad Chromium build, instances failed to terminate diff --git a/Dockerfile b/Dockerfile index 0ffd16e..2bbacda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.14-hotfix2 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.14-hotfix2 +# docker build -t ngosang/flaresolverr:3.3.15 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.15 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix2 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.15 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.14-hotfix2 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.14-hotfix2 +# docker buildx build -t ngosang/flaresolverr:3.3.15 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.15 diff --git a/package.json b/package.json index 0cac948..fcfa5c4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.14-hotfix2", + "version": "3.3.15", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" From c5a5f6d65e95540c4f70dd8ae932297c7b50ce4a Mon Sep 17 00:00:00 2001 From: Xewdy Date: Wed, 21 Feb 2024 15:14:25 -0600 Subject: [PATCH 08/32] Add platform specifiers to dependencies (#877) --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 212df01..fc002d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ prometheus-client==0.17.1 requests==2.31.0 certifi==2023.7.22 websockets==11.0.3 -# only required for linux -xvfbwrapper==0.2.9 +# only required for linux and macos +xvfbwrapper==0.2.9; platform_system != "Windows" # only required for windows -pefile==2023.2.7 +pefile==2023.2.7; platform_system == "Windows" From 0bc7a4498c0c19aaf051458c6e3333cc127b348b Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Fri, 23 Feb 2024 05:01:00 +0000 Subject: [PATCH 09/32] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 38526fc..a88e69e 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ curl -L -X POST 'http://localhost:8191/v1' \ -H 'Content-Type: application/json' \ --data-raw '{ "cmd": "request.get", - "url":"http://www.google.com/", + "url": "http://www.google.com/", "maxTimeout": 60000 }' ``` From afdc1c7a8e6cbb693fb790a0cdcb3650015c2ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl?= Date: Wed, 28 Feb 2024 03:45:04 +0100 Subject: [PATCH 10/32] Add FreeBSD support (#1054) --- README.md | 7 +++ src/undetected_chromedriver/__init__.py | 4 +- src/undetected_chromedriver/patcher.py | 84 ++++++++++++++++++------- src/utils.py | 10 ++- 4 files changed, 78 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index a88e69e..be86f32 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,13 @@ This is the recommended way for Windows users. * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies. * Run `python src/flaresolverr.py` command to start FlareSolverr. +### From source code (FreeBSD/TrueNAS CORE) + +* Run `pkg install chromium python39 py39-pip xorg-vfbserver` command to install the required dependencies. +* Clone this repository and open a shell in that path. +* Run `python3.9 -m pip install -r requirements.txt` command to install FlareSolverr dependencies. +* Run `python3.9 src/flaresolverr.py` command to start FlareSolverr. + ### Systemd service We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables. diff --git a/src/undetected_chromedriver/__init__.py b/src/undetected_chromedriver/__init__.py index 755f7a4..1382758 100644 --- a/src/undetected_chromedriver/__init__.py +++ b/src/undetected_chromedriver/__init__.py @@ -451,8 +451,10 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): options.binary_location, *options.arguments ) else: - startupinfo = subprocess.STARTUPINFO() + startupinfo = None if os.name == 'nt' and windows_headless: + # STARTUPINFO() is Windows only + startupinfo = subprocess.STARTUPINFO() startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW browser = subprocess.Popen( [options.binary_location, *options.arguments], diff --git a/src/undetected_chromedriver/patcher.py b/src/undetected_chromedriver/patcher.py index 4114e56..a6eeab2 100644 --- a/src/undetected_chromedriver/patcher.py +++ b/src/undetected_chromedriver/patcher.py @@ -21,7 +21,7 @@ from multiprocessing import Lock logger = logging.getLogger(__name__) -IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2")) +IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2", "freebsd")) class Patcher(object): @@ -80,9 +80,14 @@ class Patcher(object): os.makedirs(self.data_path, exist_ok=True) if not executable_path: - self.executable_path = os.path.join( - self.data_path, "_".join([prefix, self.exe_name]) - ) + if sys.platform.startswith("freebsd"): + self.executable_path = os.path.join( + self.data_path, self.exe_name + ) + else: + self.executable_path = os.path.join( + self.data_path, "_".join([prefix, self.exe_name]) + ) if not IS_POSIX: if executable_path: @@ -127,6 +132,9 @@ class Patcher(object): else: self.platform_name = "mac-x64" self.exe_name %= "" + if self.platform.startswith("freebsd"): + self.platform_name = "freebsd" + self.exe_name %= "" def auto(self, executable_path=None, force=False, version_main=None, _=None): """ @@ -166,26 +174,56 @@ class Patcher(object): if force is True: self.force = force - try: - os.unlink(self.executable_path) - except PermissionError: - if self.force: - self.force_kill_instances(self.executable_path) - return self.auto(force=not self.force) - try: - if self.is_binary_patched(): - # assumes already running AND patched - return True - except PermissionError: - pass - # return False - except FileNotFoundError: - pass - release = self.fetch_release_number() - self.version_main = release.version[0] - self.version_full = release - self.unzip_package(self.fetch_package()) + if self.platform_name == "freebsd": + chromedriver_path = shutil.which("chromedriver") + + if not os.path.isfile(chromedriver_path) or not os.access(chromedriver_path, os.X_OK): + logging.error("Chromedriver not installed!") + return + + version_path = os.path.join(os.path.dirname(self.executable_path), "version.txt") + + process = os.popen(f'"{chromedriver_path}" --version') + chromedriver_version = process.read().split(' ')[1].split(' ')[0] + process.close() + + current_version = None + if os.path.isfile(version_path) or os.access(version_path, os.X_OK): + with open(version_path, 'r') as f: + current_version = f.read() + + if current_version != chromedriver_version: + logging.info("Copying chromedriver executable...") + shutil.copy(chromedriver_path, self.executable_path) + os.chmod(self.executable_path, 0o755) + + with open(version_path, 'w') as f: + f.write(chromedriver_version) + + logging.info("Chromedriver executable copied!") + else: + try: + os.unlink(self.executable_path) + except PermissionError: + if self.force: + self.force_kill_instances(self.executable_path) + return self.auto(force=not self.force) + try: + if self.is_binary_patched(): + # assumes already running AND patched + return True + except PermissionError: + pass + # return False + except FileNotFoundError: + pass + + release = self.fetch_release_number() + self.version_main = release.version[0] + self.version_full = release + self.unzip_package(self.fetch_package()) + return self.patch() def driver_binary_in_use(self, path: str = None) -> bool: diff --git a/src/utils.py b/src/utils.py index b812387..e8b39db 100644 --- a/src/utils.py +++ b/src/utils.py @@ -5,6 +5,7 @@ import re import shutil import urllib.parse import tempfile +import sys from selenium.webdriver.chrome.webdriver import WebDriver import undetected_chromedriver as uc @@ -182,9 +183,12 @@ def get_webdriver(proxy: dict = None) -> WebDriver: # downloads and patches the chromedriver # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time - driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path, - driver_executable_path=driver_exe_path, version_main=version_main, - windows_headless=windows_headless, headless=windows_headless) + try: + driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path, + driver_executable_path=driver_exe_path, version_main=version_main, + windows_headless=windows_headless, headless=windows_headless) + except Exception as e: + logging.error("Error starting Chrome: %s" % e) # save the patched driver to avoid re-downloads if driver_exe_path is None: From 35c7bff3c873034bbfb5696fb814dd85a38548e8 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Wed, 28 Feb 2024 21:36:38 +0100 Subject: [PATCH 11/32] Use headless configuration properly (#1104) --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index e8b39db..9d02d58 100644 --- a/src/utils.py +++ b/src/utils.py @@ -186,7 +186,7 @@ def get_webdriver(proxy: dict = None) -> WebDriver: try: driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path, driver_executable_path=driver_exe_path, version_main=version_main, - windows_headless=windows_headless, headless=windows_headless) + windows_headless=windows_headless, headless=get_config_headless()) except Exception as e: logging.error("Error starting Chrome: %s" % e) From fd773e59097c1aca4e08edc07c1ed30164f88bd3 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Wed, 28 Feb 2024 21:38:53 +0000 Subject: [PATCH 12/32] Bump version 3.3.16 (#1105) --- CHANGELOG.md | 6 ++++++ Dockerfile | 10 +++++----- package.json | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b215861..8535429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v3.3.16 (2024/02/28) + +* Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo +* Add FreeBSD support. Thanks @Asthowen +* Use headless configuration properly. Thanks @hashworks + ## v3.3.15 (2024/02/20) * Fix looping challenges diff --git a/Dockerfile b/Dockerfile index 2bbacda..fde140c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.15 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.15 +# docker build -t ngosang/flaresolverr:3.3.16 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.16 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.15 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.15 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.15 +# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.16 diff --git a/package.json b/package.json index fcfa5c4..7d08a19 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.15", + "version": "3.3.16", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" From 20cd2944a790eeb7d3d1b5b87f5f0b17fbf1c628 Mon Sep 17 00:00:00 2001 From: francisco-lafe <90066435+francisco-lafe@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:05:11 -0300 Subject: [PATCH 13/32] Update README.md (#1127) Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be86f32..12f0589 100644 --- a/README.md +++ b/README.md @@ -276,8 +276,8 @@ This is the same as `request.get` but it takes one more param: Environment variables are set differently depending on the operating system. Some examples: * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command. -* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell. -* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell. +* Linux: Run `export LOG_LEVEL=debug` and then run `flaresolverr` in the same shell. +* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then run `flaresolverr.exe` in the same shell. ## Prometheus exporter From 8dea0ed0174fa13e496fdd0950716e72772463e9 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Tue, 9 Apr 2024 20:27:42 +0100 Subject: [PATCH 14/32] Fix file descriptor leak in service on quit(). resolves #983 credit: @zkulis - https://github.com/ultrafunkamsterdam/undetected-chromedriver/pull/1812 --- src/undetected_chromedriver/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/undetected_chromedriver/__init__.py b/src/undetected_chromedriver/__init__.py index 1382758..08381c5 100644 --- a/src/undetected_chromedriver/__init__.py +++ b/src/undetected_chromedriver/__init__.py @@ -771,7 +771,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): def quit(self): try: + self.service.stop() self.service.process.kill() + self.command_executor.close() self.service.process.wait(5) logger.debug("webdriver process ended") except (AttributeError, RuntimeError, OSError): From 38166dfaa02e1d9320bb143cba1d9103d8fe34a0 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Tue, 9 Apr 2024 20:31:21 +0100 Subject: [PATCH 15/32] Bump version 3.3.17 (#1147) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8535429..76a798e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.17 (2024/04/09) + +* Fix file descriptor leak in service on quit(). Thanks @zkulis + ## v3.3.16 (2024/02/28) * Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo diff --git a/Dockerfile b/Dockerfile index fde140c..7c1c7ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.16 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.16 +# docker build -t ngosang/flaresolverr:3.3.17 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.17 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.17 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.16 +# docker buildx build -t ngosang/flaresolverr:3.3.17 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.17 diff --git a/package.json b/package.json index 7d08a19..19ca51b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.16", + "version": "3.3.17", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" From 0bb8de144f8267b603a0b16d686a5565f264e19e Mon Sep 17 00:00:00 2001 From: Ross Patterson Date: Wed, 17 Apr 2024 21:07:06 -0700 Subject: [PATCH 16/32] Add Compose V2 command to readme (#1154) Co-authored-by: root@library.moodysalon.net Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 12f0589..e0b30aa 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,8 @@ Supported architectures are: | ARM32 | linux/arm/v7 | | ARM64 | linux/arm64 | -We provide a `docker-compose.yml` configuration file. Clone this repository and execute `docker-compose up -d` to start +We provide a `docker-compose.yml` configuration file. Clone this repository and execute +`docker-compose up -d` _(Compose V1)_ or `docker compose up -d` _(Compose V2)_ to start the container. If you prefer the `docker cli` execute the following command. From b811412699f633b58771e9cd182e612734b60cf5 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 20 Apr 2024 03:41:53 +0100 Subject: [PATCH 17/32] Fix LANG ENV for Linux. #1036 --- src/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils.py b/src/utils.py index 9d02d58..79ccf87 100644 --- a/src/utils.py +++ b/src/utils.py @@ -139,7 +139,7 @@ def get_webdriver(proxy: dict = None) -> WebDriver: language = os.environ.get('LANG', None) if language is not None: - options.add_argument('--lang=%s' % language) + options.add_argument('--accept-lang=%s' % language) # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910 if USER_AGENT is not None: From c304da2964ad33de24a85fa0d2dd550ad51e6210 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Mon, 22 Apr 2024 23:30:52 +0100 Subject: [PATCH 18/32] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e0b30aa..8f97158 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,8 @@ This is the recommended way for Windows users. * Install [Python 3.11](https://www.python.org/downloads/). * Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser. -* (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package. +* (Only in Linux) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package. +* (Only in macOS) Install [XQuartz](https://www.xquartz.org/) package. * Clone this repository and open a shell in that path. * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies. * Run `python src/flaresolverr.py` command to start FlareSolverr. From 5a2c61601e756438241df3b0b1eb7cb1807233fc Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Mon, 20 May 2024 00:52:55 +0100 Subject: [PATCH 19/32] Fix Chrome v124+ not closing on Windows. resolves #1161 (#1193) --- CHANGELOG.md | 5 +++++ Dockerfile | 10 +++++----- package.json | 2 +- src/flaresolverr_service.py | 1 + src/sessions.py | 1 + src/undetected_chromedriver/__init__.py | 9 --------- src/undetected_chromedriver/devtool.py | 1 + src/utils.py | 1 + 8 files changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76a798e..cde1582 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v3.3.18 (2024/05/20) + +* Fix LANG ENV for Linux +* Fix Chrome v124+ not closing on Windows. Thanks @RileyXX + ## v3.3.17 (2024/04/09) * Fix file descriptor leak in service on quit(). Thanks @zkulis diff --git a/Dockerfile b/Dockerfile index 7c1c7ca..2cb2be8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.17 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.17 +# docker build -t ngosang/flaresolverr:3.3.18 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.18 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.17 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.18 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.17 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.17 +# docker buildx build -t ngosang/flaresolverr:3.3.18 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.18 diff --git a/package.json b/package.json index 19ca51b..6b2460a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.17", + "version": "3.3.18", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 0987fed..a92e5c9 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -245,6 +245,7 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT: raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n')) finally: if not req.session and driver is not None: + driver.close() driver.quit() logging.debug('A used instance of webdriver has been destroyed') diff --git a/src/sessions.py b/src/sessions.py index 1a635e0..dab0cdc 100644 --- a/src/sessions.py +++ b/src/sessions.py @@ -66,6 +66,7 @@ class SessionsStorage: return False session = self.sessions.pop(session_id) + session.driver.close() session.driver.quit() return True diff --git a/src/undetected_chromedriver/__init__.py b/src/undetected_chromedriver/__init__.py index 08381c5..4e7fa1a 100644 --- a/src/undetected_chromedriver/__init__.py +++ b/src/undetected_chromedriver/__init__.py @@ -788,15 +788,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver): logger.debug("gracefully closed browser") except Exception as e: # noqa pass - # Force kill Chrome process in Windows - # https://github.com/FlareSolverr/FlareSolverr/issues/772 - if os.name == 'nt': - try: - subprocess.call(['taskkill', '/f', '/pid', str(self.browser_pid)], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL) - except Exception: - pass if ( hasattr(self, "keep_user_data_dir") and hasattr(self, "user_data_dir") diff --git a/src/undetected_chromedriver/devtool.py b/src/undetected_chromedriver/devtool.py index 30e7c08..7322ad5 100644 --- a/src/undetected_chromedriver/devtool.py +++ b/src/undetected_chromedriver/devtool.py @@ -187,4 +187,5 @@ def test(): time.sleep(10) + driver.close() driver.quit() diff --git a/src/utils.py b/src/utils.py index 79ccf87..7c6d4f3 100644 --- a/src/utils.py +++ b/src/utils.py @@ -314,6 +314,7 @@ def get_user_agent(driver=None) -> str: raise Exception("Error getting browser User-Agent. " + str(e)) finally: if driver is not None: + driver.close() driver.quit() From 6c1d78cb847c8dd6fb50889161a53b81ef0bd98d Mon Sep 17 00:00:00 2001 From: 21hsmw <153914136+21hsmw@users.noreply.github.com> Date: Fri, 24 May 2024 16:33:46 +0000 Subject: [PATCH 20/32] Fix occasional headless issue on Linux when set to "false" (#1199) * Fix occasional headless issue on Linux when set to "false" - Add a variable containing the current platform - Check if the platform is "nt" (Windows) before closing the driver * Update CHANGELOG.md --------- Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> --- CHANGELOG.md | 3 +++ Dockerfile | 10 +++++----- package.json | 2 +- src/flaresolverr.py | 3 +++ src/flaresolverr_service.py | 3 ++- src/sessions.py | 3 ++- src/undetected_chromedriver/devtool.py | 4 +++- src/utils.py | 11 ++++++++++- 8 files changed, 29 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cde1582..3a26f5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## v3.3.19 (2024/05/23) +* Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw + ## v3.3.18 (2024/05/20) * Fix LANG ENV for Linux diff --git a/Dockerfile b/Dockerfile index 2cb2be8..699db27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.18 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.18 +# docker build -t ngosang/flaresolverr:3.3.19 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.19 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.18 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.19 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.18 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.18 +# docker buildx build -t ngosang/flaresolverr:3.3.19 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.19 diff --git a/package.json b/package.json index 6b2460a..6cfb1f7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.18", + "version": "3.3.19", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" diff --git a/src/flaresolverr.py b/src/flaresolverr.py index 9e5f8de..3596fe1 100644 --- a/src/flaresolverr.py +++ b/src/flaresolverr.py @@ -101,6 +101,9 @@ if __name__ == "__main__": logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}') logging.debug('Debug log enabled') + # Get current OS for global variable + utils.get_current_platform() + # test browser installation flaresolverr_service.test_browser_installation() diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index a92e5c9..e702526 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -245,7 +245,8 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT: raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n')) finally: if not req.session and driver is not None: - driver.close() + if utils.PLATFORM_VERSION == "nt": + driver.close() driver.quit() logging.debug('A used instance of webdriver has been destroyed') diff --git a/src/sessions.py b/src/sessions.py index dab0cdc..30bb3c1 100644 --- a/src/sessions.py +++ b/src/sessions.py @@ -66,7 +66,8 @@ class SessionsStorage: return False session = self.sessions.pop(session_id) - session.driver.close() + if utils.PLATFORM_VERSION == "nt": + session.driver.close() session.driver.quit() return True diff --git a/src/undetected_chromedriver/devtool.py b/src/undetected_chromedriver/devtool.py index 7322ad5..915d417 100644 --- a/src/undetected_chromedriver/devtool.py +++ b/src/undetected_chromedriver/devtool.py @@ -2,6 +2,7 @@ import asyncio from collections.abc import Mapping from collections.abc import Sequence from functools import wraps +import os import logging import threading import time @@ -187,5 +188,6 @@ def test(): time.sleep(10) - driver.close() + if os.name == "nt": + driver.close() driver.quit() diff --git a/src/utils.py b/src/utils.py index 7c6d4f3..a5bd1ef 100644 --- a/src/utils.py +++ b/src/utils.py @@ -11,6 +11,7 @@ from selenium.webdriver.chrome.webdriver import WebDriver import undetected_chromedriver as uc FLARESOLVERR_VERSION = None +PLATFORM_VERSION = None CHROME_EXE_PATH = None CHROME_MAJOR_VERSION = None USER_AGENT = None @@ -38,6 +39,13 @@ def get_flaresolverr_version() -> str: FLARESOLVERR_VERSION = json.loads(f.read())['version'] return FLARESOLVERR_VERSION +def get_current_platform() -> str: + global PLATFORM_VERSION + if PLATFORM_VERSION is not None: + return PLATFORM_VERSION + PLATFORM_VERSION = os.name + return PLATFORM_VERSION + def create_proxy_extension(proxy: dict) -> str: parsed_url = urllib.parse.urlparse(proxy['url']) @@ -314,7 +322,8 @@ def get_user_agent(driver=None) -> str: raise Exception("Error getting browser User-Agent. " + str(e)) finally: if driver is not None: - driver.close() + if PLATFORM_VERSION == "nt": + driver.close() driver.quit() From 984368edb5a32af4299cd0e0a0e41bac4f2894e9 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 15 Jun 2024 05:41:45 +0100 Subject: [PATCH 21/32] maxTimeout should always be int. resolves #1212 --- src/flaresolverr_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index e702526..6d70aa2 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -119,7 +119,7 @@ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase: logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.") # set default values - if req.maxTimeout is None or req.maxTimeout < 1: + if req.maxTimeout is None or int(req.maxTimeout) < 1: req.maxTimeout = 60000 # execute the command @@ -220,7 +220,7 @@ def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase: def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT: - timeout = req.maxTimeout / 1000 + timeout = int(req.maxTimeout) / 1000 driver = None try: if req.session: From 2ecf88895b851eacec75afaa82f08532fe2dc8a6 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 15 Jun 2024 08:37:42 +0100 Subject: [PATCH 22/32] Check not running in Docker before logging version_main error --- src/undetected_chromedriver/patcher.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/undetected_chromedriver/patcher.py b/src/undetected_chromedriver/patcher.py index a6eeab2..e8a0e96 100644 --- a/src/undetected_chromedriver/patcher.py +++ b/src/undetected_chromedriver/patcher.py @@ -68,8 +68,10 @@ class Patcher(object): # check if version_main_int is less than or equal to e.g 114 self.is_old_chromedriver = version_main and version_main_int <= 114 except (ValueError,TypeError): - # If the conversion fails, print an error message - print("version_main cannot be converted to an integer") + # Check not running inside Docker + if not os.path.exists("/app/chromedriver"): + # If the conversion fails, log an error message + logging.info("version_main cannot be converted to an integer") # Set self.is_old_chromedriver to False if the conversion fails self.is_old_chromedriver = False From a2c0e4348e048b656e8b259e51d6055d694cd85b Mon Sep 17 00:00:00 2001 From: tenettow Date: Fri, 21 Jun 2024 22:07:03 +0100 Subject: [PATCH 23/32] Update Cloudflare challenge and checkbox selectors (#1224) --- src/flaresolverr_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 6d70aa2..8bb5ac4 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -40,7 +40,7 @@ CHALLENGE_TITLES = [ ] CHALLENGE_SELECTORS = [ # Cloudflare - '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', + '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands 'td.info #js_info', # Fairlane / pararius.com @@ -258,7 +258,7 @@ def click_verify(driver: WebDriver): driver.switch_to.frame(iframe) checkbox = driver.find_element( by=By.XPATH, - value='//*[@id="challenge-stage"]/div/label/input', + value='//*[@id="content"]/div/div/label/input', ) if checkbox: actions = ActionChains(driver) From 5a1f25cd524f7938d82f2e5aedd5753da8c48db6 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Fri, 21 Jun 2024 22:21:37 +0100 Subject: [PATCH 24/32] Bump version 3.3.20 (#1229) --- CHANGELOG.md | 5 +++++ Dockerfile | 10 +++++----- package.json | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a26f5f..77a0bb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## v3.3.20 (2024/06/21) +* maxTimeout should always be int +* Check not running in Docker before logging version_main error +* Update Cloudflare challenge and checkbox selectors. Thanks @tenettow & @21hsmw + ## v3.3.19 (2024/05/23) * Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw diff --git a/Dockerfile b/Dockerfile index 699db27..5c2d74b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.19 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.19 +# docker build -t ngosang/flaresolverr:3.3.20 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.20 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.19 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.20 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.19 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.19 +# docker buildx build -t ngosang/flaresolverr:3.3.20 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.20 diff --git a/package.json b/package.json index 6cfb1f7..9315fd1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.19", + "version": "3.3.20", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" From e96161c87349e971093a99c321cf86b5a844a2d3 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Tue, 25 Jun 2024 22:32:06 +0100 Subject: [PATCH 25/32] Add challenge selector to catch reloading page on non-English systems. resolves #1237 --- src/flaresolverr_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 8bb5ac4..82f68df 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -40,7 +40,7 @@ CHALLENGE_TITLES = [ ] CHALLENGE_SELECTORS = [ # Cloudflare - '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', + '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring', # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands 'td.info #js_info', # Fairlane / pararius.com From a8644532a1d14b7326bb1042d631793d07328a12 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Wed, 26 Jun 2024 04:04:59 +0300 Subject: [PATCH 26/32] Escape values for generated form used in `request.post` (#1236) and build docker images for PRs --- .github/workflows/release-docker.yml | 65 +++++++++++++++++----------- src/flaresolverr_service.py | 7 +-- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 357f906..88d5259 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -4,50 +4,63 @@ on: push: tags: - 'v*.*.*' + pull_request: + branches: + - master + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: build-docker-images: runs-on: ubuntu-22.04 steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Downcase repo + - name: Checkout + uses: actions/checkout@v4 + + - name: Downcase repo run: echo REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV - - - name: Docker meta + + - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v3 + uses: docker/metadata-action@v5 with: - images: ${{ env.REPOSITORY }},ghcr.io/${{ env.REPOSITORY }} - tag-sha: false - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to DockerHub - uses: docker/login-action@v2 + images: | + ${{ env.REPOSITORY }},enable=${{ github.event_name != 'pull_request' }} + ghcr.io/${{ env.REPOSITORY }} + tags: | + type=semver,pattern={{version}},prefix=v + type=ref,event=pr + flavor: | + latest=auto + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + if: github.event_name != 'pull_request' with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.repository_owner }} password: ${{ secrets.GH_PAT }} - - - name: Build and push - uses: docker/build-push-action@v3 + + - name: Build and push + uses: docker/build-push-action@v5 with: context: . file: ./Dockerfile platforms: linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 82f68df..cfc2088 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -3,7 +3,8 @@ import platform import sys import time from datetime import timedelta -from urllib.parse import unquote +from html import escape +from urllib.parse import unquote, quote from func_timeout import FunctionTimedOut, func_timeout from selenium.common import TimeoutException @@ -439,7 +440,7 @@ def _post_request(req: V1RequestBase, driver: WebDriver): value = unquote(parts[1]) except Exception: value = parts[1] - post_form += f'
' + post_form += f'
' post_form += '' html_content = f""" @@ -449,6 +450,6 @@ def _post_request(req: V1RequestBase, driver: WebDriver): """ - driver.get("data:text/html;charset=utf-8," + html_content) + driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content)) driver.start_session() driver.start_session() # required to bypass Cloudflare From 031177bbdb551835de946b79accbade39e21c6a4 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Wed, 26 Jun 2024 02:14:25 +0100 Subject: [PATCH 27/32] Bump version 3.3.21 (#1240) --- CHANGELOG.md | 4 ++++ Dockerfile | 10 +++++----- package.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 77a0bb8..a02e8d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v3.3.21 (2024/06/26) +* Add challenge selector to catch reloading page on non-English systems +* Escape values for generated form used in request.post. Thanks @mynameisbogdan + ## v3.3.20 (2024/06/21) * maxTimeout should always be int * Check not running in Docker before logging version_main error diff --git a/Dockerfile b/Dockerfile index 5c2d74b..3d8ea35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"] # Local build -# docker build -t ngosang/flaresolverr:3.3.20 . -# docker run -p 8191:8191 ngosang/flaresolverr:3.3.20 +# docker build -t ngosang/flaresolverr:3.3.21 . +# docker run -p 8191:8191 ngosang/flaresolverr:3.3.21 # Multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.20 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . +# docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 . # add --push to publish in DockerHub # Test multi-arch build # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes # docker buildx create --use -# docker buildx build -t ngosang/flaresolverr:3.3.20 --platform linux/arm/v7 --load . -# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.20 +# docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/arm/v7 --load . +# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.21 diff --git a/package.json b/package.json index 9315fd1..525abd8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flaresolverr", - "version": "3.3.20", + "version": "3.3.21", "description": "Proxy server to bypass Cloudflare protection", "author": "Diego Heras (ngosang / ngosang@hotmail.es)", "license": "MIT" From dd7eaee2e3bfa529fbc8d88434cec468eac2958c Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Fri, 12 Jul 2024 17:11:40 +0100 Subject: [PATCH 28/32] Bump requirements resolves Dependabot alerts --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fc002d1..26a6f98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,8 @@ selenium==4.15.2 func-timeout==4.3.5 prometheus-client==0.17.1 # required by undetected_chromedriver -requests==2.31.0 -certifi==2023.7.22 +requests==2.32.0 +certifi==2024.07.04 websockets==11.0.3 # only required for linux and macos xvfbwrapper==0.2.9; platform_system != "Windows" From 3d9bc5627bb9ce0521cac61e7adb713302d3b9dc Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 20 Jul 2024 14:21:34 +0100 Subject: [PATCH 29/32] Change to GITHUB_TOKEN for GHRC login --- .github/workflows/release-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 88d5259..ebe302a 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -53,7 +53,7 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GH_PAT }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v5 From 0f8f0bec259f706ee42c0ad1b2ed62a0d1a42585 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Sat, 20 Jul 2024 19:41:49 +0100 Subject: [PATCH 30/32] revert and bump action version --- .github/workflows/release-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index ebe302a..4f1b652 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -53,10 +53,10 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GH_PAT }} - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: ./Dockerfile From eb680efc9059ef08907aaec58340d73325048aeb Mon Sep 17 00:00:00 2001 From: Bogdan Date: Sat, 20 Jul 2024 22:08:40 +0300 Subject: [PATCH 31/32] Don't build docker images for PRs from forks (#1281) --- .github/workflows/release-docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 4f1b652..71fe5a1 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -14,6 +14,7 @@ concurrency: jobs: build-docker-images: + if: ${{ !github.event.pull_request.head.repo.fork }} runs-on: ubuntu-22.04 steps: - name: Checkout From a798561338f6bcbe4b9b3f5dad9b8a4fd3b16ab3 Mon Sep 17 00:00:00 2001 From: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com> Date: Tue, 30 Jul 2024 02:38:13 +0100 Subject: [PATCH 32/32] Bump requests version *.0 was yanked --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 26a6f98..d633aeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ selenium==4.15.2 func-timeout==4.3.5 prometheus-client==0.17.1 # required by undetected_chromedriver -requests==2.32.0 +requests==2.32.3 certifi==2024.07.04 websockets==11.0.3 # only required for linux and macos