github/FlareSolverr
1
1
Fork 0
mirror of https://github.com/FlareSolverr/FlareSolverr.git synced 2025-06-10 05:17:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

proxy fix, user data folder fix

Browse Source
This commit is contained in:
AlexFozor 2024-07-31 16:28:05 +02:00
parent b8639bc611
commit 99708041b5
5 changed files with 45 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -187,7 +187,7 @@ session. When you no longer need to use a session you should make sure to close
| maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
| cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. |
| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
> **Warning**
> If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.

3
src/flaresolverr.py
View File

@ -114,6 +114,9 @@ if __name__ == "__main__":
prometheus_plugin.setup()
app.install(prometheus_plugin.prometheus_plugin)
webdriver_data = utils.get_webdriver_data_path()
utils.remove_all_subfolders(webdriver_data)
# start webserver
# default server 'wsgiref' does not support concurrent requests
# https://github.com/FlareSolverr/FlareSolverr/issues/680

6
src/flaresolverr_service.py
View File

@ -218,6 +218,7 @@ def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
timeout = int(req.maxTimeout) / 1000
driver = None
user_data_path = None
try:
if req.session:
session_id = req.session
@ -232,7 +233,8 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
driver = session.driver
else:
driver = utils.get_webdriver(req.proxy)
user_data_path = utils.get_user_data_path()
driver = utils.get_webdriver(req.proxy, user_data_path)
logging.debug('New instance of webdriver has been created to perform the request')
return func_timeout(timeout, _evil_logic, (req, driver, method))
except FunctionTimedOut:
@ -245,6 +247,8 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
driver.close()
driver.quit()
logging.debug('A used instance of webdriver has been destroyed')
if user_data_path:
utils.remove_user_data(user_data_path)
def click_verify(driver: ChromiumPage) -> DataPacket:
try:

6
src/sessions.py
View File

@ -45,7 +45,7 @@ class SessionsStorage:
if self.exists(session_id):
return self.sessions[session_id], False
driver = utils.get_webdriver(proxy)
driver = utils.get_webdriver(proxy, utils.get_user_data_path(session_id))
created_at = datetime.now()
session = Session(session_id, driver, created_at)
@ -69,6 +69,10 @@ class SessionsStorage:
if utils.PLATFORM_VERSION == "nt":
session.driver.close()
session.driver.quit()
session_data = utils.get_user_data_path(session_id)
utils.remove_user_data(session_data)
return True
def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:

36
src/utils.py
View File

@ -7,6 +7,10 @@ import urllib.parse
import tempfile
import sys
from uuid import uuid1
from pathlib import Path
from tempfile import gettempdir
from DrissionPage import ChromiumPage, ChromiumOptions
# from DrissionPage.common import Settings
@ -120,8 +124,25 @@ def create_proxy_extension(proxy: dict) -> str:
return proxy_extension_dir
def get_webdriver_data_path() -> Path:
return Path(gettempdir()) / 'FlareSolverr'
def get_webdriver(proxy: dict = None) -> ChromiumPage:
def get_user_data_path(user_name: str = None) -> Path:
user_name = user_name or str(uuid1())
return get_webdriver_data_path() / user_name
def remove_user_data(user_data_path: Path):
if user_data_path.exists():
shutil.rmtree(user_data_path)
def remove_all_subfolders(parent_folder: str):
for item in os.listdir(parent_folder):
item_path = os.path.join(parent_folder, item)
if os.path.isdir(item_path):
shutil.rmtree(item_path)
def get_webdriver(proxy: dict = None, user_data_path: str = None) -> ChromiumPage:
global CHROME_EXE_PATH, USER_AGENT
logging.debug('Launching web browser...')
@ -156,7 +177,7 @@ def get_webdriver(proxy: dict = None) -> ChromiumPage:
proxy_extension_dir = None
if proxy and all(key in proxy for key in ['url', 'username', 'password']):
proxy_extension_dir = create_proxy_extension(proxy)
options.set_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
options.add_extension(proxy_extension_dir)
elif proxy and 'url' in proxy:
proxy_url = proxy['url']
logging.debug("Using webdriver proxy: %s", proxy_url)
@ -171,15 +192,17 @@ def get_webdriver(proxy: dict = None) -> ChromiumPage:
options.headless(windows_headless)
options.set_argument("--auto-open-devtools-for-tabs")
if user_data_path:
options.set_user_data_path(user_data_path)
if CHROME_EXE_PATH is not None:
options.set_paths(browser_path=CHROME_EXE_PATH)
driver = ChromiumPage(addr_or_opts=options)
# clean up proxy extension directory
if proxy_extension_dir is not None:
shutil.rmtree(proxy_extension_dir)
driver = ChromiumPage(addr_or_opts=options)
return driver
@ -319,8 +342,9 @@ def get_user_agent(driver=None) -> str:
return USER_AGENT
try:
user_path = get_user_data_path("DefaultUserAgent")
if driver is None:
driver = get_webdriver()
driver = get_webdriver(user_data_path=user_path)
USER_AGENT = driver.user_agent
# Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
@ -332,6 +356,8 @@ def get_user_agent(driver=None) -> str:
if PLATFORM_VERSION == "nt":
driver.close()
driver.quit()
if user_path:
remove_user_data(user_path)
def start_xvfb_display():