From 313fb2c14b9e9c28e01f322adbab3e0ebfc752b0 Mon Sep 17 00:00:00 2001 From: ngosang Date: Sun, 23 Jul 2023 19:45:00 +0200 Subject: [PATCH] Add support for proxy authentication username/password. Thanks @jacobprice808 --- CHANGELOG.md | 2 +- README.md | 8 ++--- src/tests.py | 36 +++++++++++++++++++++ src/utils.py | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 127 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb81644..b642655 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## v3.2.2 (2023/07/16) -* Workaround for updated 'verify your are human' check +* Workaround for updated 'verify you are human' check ## v3.2.1 (2023/06/10) diff --git a/README.md b/README.md index 87eda6e..496b128 100644 --- a/README.md +++ b/README.md @@ -110,10 +110,10 @@ cookies for the browser to use. This also speeds up the requests since it won't have to launch a new browser instance for every request. -| Parameter | Notes | -|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. | -| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. | +| Parameter | Notes | +|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. | +| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. Eg: `"proxy": {"url": "http://127.0.0.1:8888", username": "testuser", "password": "testpass"}` | #### + `sessions.list` diff --git a/src/tests.py b/src/tests.py index b446786..9575db2 100644 --- a/src/tests.py +++ b/src/tests.py @@ -335,6 +335,42 @@ class TestFlareSolverr(unittest.TestCase): self.assertGreater(len(solution.cookies), 0) self.assertIn("Chrome/", solution.userAgent) + def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self): + """ + To configure TinyProxy in local: + * sudo vim /etc/tinyproxy/tinyproxy.conf + * edit => LogFile "/tmp/tinyproxy.log" + * edit => Syslog Off + * add => BasicAuth testuser testpass + * sudo tinyproxy -d + * sudo tail -f /tmp/tinyproxy.log + """ + res = self.app.post_json('/v1', { + "cmd": "request.get", + "url": self.google_url, + "proxy": { + "url": self.proxy_url, + "username": "testuser", + "password": "testpass" + } + }) + self.assertEqual(res.status_code, 200) + + body = V1ResponseBase(res.json) + self.assertEqual(STATUS_OK, body.status) + self.assertEqual("Challenge not detected!", body.message) + self.assertGreater(body.startTimestamp, 10000) + self.assertGreaterEqual(body.endTimestamp, body.startTimestamp) + self.assertEqual(utils.get_flaresolverr_version(), body.version) + + solution = body.solution + self.assertIn(self.google_url, solution.url) + self.assertEqual(solution.status, 200) + self.assertIs(len(solution.headers), 0) + self.assertIn("Google", solution.response) + self.assertGreater(len(solution.cookies), 0) + self.assertIn("Chrome/", solution.userAgent) + def test_v1_endpoint_request_get_proxy_socks_param(self): """ To configure Dante in local: diff --git a/src/utils.py b/src/utils.py index f2ae5c9..2517970 100644 --- a/src/utils.py +++ b/src/utils.py @@ -3,6 +3,8 @@ import logging import os import re import shutil +import urllib.parse +import tempfile from selenium.webdriver.chrome.webdriver import WebDriver import undetected_chromedriver as uc @@ -36,6 +38,80 @@ def get_flaresolverr_version() -> str: return FLARESOLVERR_VERSION +def create_proxy_extension(proxy: dict) -> str: + parsed_url = urllib.parse.urlparse(proxy['url']) + scheme = parsed_url.scheme + host = parsed_url.hostname + port = parsed_url.port + username = proxy['username'] + password = proxy['password'] + manifest_json = """ + { + "version": "1.0.0", + "manifest_version": 2, + "name": "Chrome Proxy", + "permissions": [ + "proxy", + "tabs", + "unlimitedStorage", + "storage", + "", + "webRequest", + "webRequestBlocking" + ], + "background": {"scripts": ["background.js"]}, + "minimum_chrome_version": "76.0.0" + } + """ + + background_js = """ + var config = { + mode: "fixed_servers", + rules: { + singleProxy: { + scheme: "%s", + host: "%s", + port: %d + }, + bypassList: ["localhost"] + } + }; + + chrome.proxy.settings.set({value: config, scope: "regular"}, function() {}); + + function callbackFn(details) { + return { + authCredentials: { + username: "%s", + password: "%s" + } + }; + } + + chrome.webRequest.onAuthRequired.addListener( + callbackFn, + { urls: [""] }, + ['blocking'] + ); + """ % ( + scheme, + host, + port, + username, + password + ) + + proxy_extension_dir = tempfile.mkdtemp() + + with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f: + f.write(manifest_json) + + with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f: + f.write(background_js) + + return proxy_extension_dir + + def get_webdriver(proxy: dict = None) -> WebDriver: global PATCHED_DRIVER_PATH logging.debug('Launching web browser...') @@ -59,11 +135,15 @@ def get_webdriver(proxy: dict = None) -> WebDriver: # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069 # https://peter.sh/experiments/chromium-command-line-switches/#use-gl options.add_argument('--use-gl=swiftshader') - # workaround for updated 'verify your are human' check + # workaround for updated 'verify you are human' check # https://github.com/FlareSolverr/FlareSolverr/issues/811 options.add_argument('--auto-open-devtools-for-tabs') - if proxy and 'url' in proxy: + proxy_extension_dir = None + if proxy and all(key in proxy for key in ['url', 'username', 'password']): + proxy_extension_dir = create_proxy_extension(proxy) + options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir)) + elif proxy and 'url' in proxy: proxy_url = proxy['url'] logging.debug("Using webdriver proxy: %s", proxy_url) options.add_argument('--proxy-server=%s' % proxy_url) @@ -107,6 +187,10 @@ def get_webdriver(proxy: dict = None) -> WebDriver: if PATCHED_DRIVER_PATH != driver.patcher.executable_path: shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH) + # clean up proxy extension directory + if proxy_extension_dir is not None: + shutil.rmtree(proxy_extension_dir) + # selenium vanilla # options = webdriver.ChromeOptions() # options.add_argument('--no-sandbox')