diff --git a/requirements.txt b/requirements.txt index 069dd92..8db3c70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ websockets==11.0.3 xvfbwrapper==0.2.9; platform_system != "Windows" # only required for windows pefile==2023.2.7; platform_system == "Windows" + +selenium_fetch \ No newline at end of file diff --git a/src/dtos.py b/src/dtos.py index 1e9aace..bce4b19 100644 --- a/src/dtos.py +++ b/src/dtos.py @@ -19,6 +19,7 @@ class ChallengeResolutionT: status: str = None message: str = None result: ChallengeResolutionResultT = None + response: str = None def __init__(self, _dict): self.__dict__.update(_dict) @@ -39,7 +40,9 @@ class V1RequestBase(object): # V1Request url: str = None + contentType: str = None postData: str = None + headers: dict = None returnOnlyCookies: bool = None download: bool = None # deprecated v2.0.0, not used returnRawHtml: bool = None # deprecated v2.0.0, not used diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index a469bea..c353fe4 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -1,3 +1,4 @@ +import json import logging import platform import sys @@ -9,12 +10,13 @@ from urllib.parse import unquote, quote from func_timeout import FunctionTimedOut, func_timeout from selenium.common import TimeoutException from selenium.webdriver.chrome.webdriver import WebDriver +from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.expected_conditions import ( presence_of_element_located, staleness_of, title_is) -from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.support.wait import WebDriverWait +from selenium_fetch import fetch, Options import utils from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT, @@ -148,6 +150,10 @@ def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase: raise Exception("Request parameter 'url' is mandatory in 'request.get' command.") if req.postData is not None: raise Exception("Cannot use 'postBody' when sending a GET request.") + if req.contentType is not None: + raise Exception("Cannot use 'contentType' when sending a GET request.") + if req.headers is not None: + raise Exception("Cannot use 'headers' when sending a GET request.") if req.returnRawHtml is not None: logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.") if req.download is not None: @@ -165,6 +171,8 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase: # do some validations if req.postData is None: raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.") + if req.contentType is None: + raise Exception("Request parameter 'contentType' is mandatory in 'request.post' command.") if req.returnRawHtml is not None: logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.") if req.download is not None: @@ -178,6 +186,23 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase: return res +def _cmd_request_postJSON(req: V1RequestBase) -> V1ResponseBase: + # do some validations + if req.postData is None: + raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.") + if req.returnRawHtml is not None: + logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.") + if req.download is not None: + logging.warning("Request parameter 'download' was removed in FlareSolverr v2.") + + challenge_res = _resolve_challenge(req, 'POSTJSON') + res = V1ResponseBase({}) + res.status = challenge_res.status + res.message = challenge_res.message + res.solution = challenge_res.result + return res + + def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase: logging.debug("Creating new session...") @@ -291,7 +316,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge # navigate to the page logging.debug(f'Navigating to... {req.url}') if method == 'POST': - _post_request(req, driver) + res.response = _post_request(req, driver) else: driver.get(req.url) @@ -303,7 +328,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge driver.add_cookie(cookie) # reload the page if method == 'POST': - _post_request(req, driver) + res.response = _post_request(req, driver) else: driver.get(req.url) @@ -397,31 +422,148 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge def _post_request(req: V1RequestBase, driver: WebDriver): - post_form = f'
' - html_content = f""" - - - - {post_form} - - - """ - driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content)) + import logging + import traceback + import time + + try: + content_type = getattr(req, 'contentType', 'application/x-www-form-urlencoded') + headers = getattr(req, 'headers', {}) + + if content_type == 'application/json': + + if not req.postData: + raise Exception("postData is empty for JSON request") + + try: + if isinstance(req.postData, str): + post_data = json.loads(req.postData) + else: + post_data = req.postData + except json.JSONDecodeError as e: + logging.error(f"JSON parsing failed: {e}") + raise Exception(f"Invalid JSON in postData: {e}") + + try: + driver.get(req.url) + + time.sleep(2) + + page_source = driver.page_source.lower() + if any(term in page_source for term in + ['cloudflare', 'checking your browser', 'ddos protection', 'please wait']): + logging.info("Protection detected, waiting for bypass...") + time.sleep(5) + + except Exception as e: + logging.warning(f"Could not load target page directly: {e}") + + json_data_str = json.dumps(post_data) + escaped_json = json_data_str.replace("'", "\\'") + + headers_js_lines = ["xhr.setRequestHeader('Content-Type', 'application/json');"] + + if headers: + for header_name, header_value in headers.items(): + if header_name.lower() != 'content-type': + escaped_value = str(header_value).replace("'", "\\'") + headers_js_lines.append(f"xhr.setRequestHeader('{header_name}', '{escaped_value}');") + + headers_js = '\n '.join(headers_js_lines) + + script = f""" + var xhr = new XMLHttpRequest(); + xhr.open('POST', '{req.url}', false); + {headers_js} + + try {{ + xhr.send('{escaped_json}'); + return {{ + status: xhr.status, + statusText: xhr.statusText, + responseText: xhr.responseText, + success: true + }}; + }} catch (error) {{ + return {{ + status: 0, + statusText: error.message, + responseText: '', + success: false, + error: error.message + }}; + }} + """ + + try: + result = driver.execute_script(script) + + if result and result.get('success'): + response_text = result.get('responseText', '') + status_code = result.get('status', 0) + + logging.info(f"POST request completed with status: {status_code}") + + return response_text + else: + error_msg = result.get('statusText', 'Unknown error') if result else 'Script execution failed' + error_detail = result.get('error', '') if result else '' + logging.error(f"XHR request failed: {error_msg} - {error_detail}") + raise Exception(f"POST request failed: {error_msg}") + + except Exception as script_error: + logging.error(f"Script execution error: {script_error}") + raise + + elif content_type == 'application/x-www-form-urlencoded': + + headers_meta = "" + if headers: + for header_name, header_value in headers.items(): + if header_name.lower() != 'content-type': + headers_meta += f'' + + post_form = f'' + html_content = f""" + + + + {headers_meta} + + + {post_form} + + + """ + driver.get(f"data:text/html;charset=utf-8,{html_content}") + return "Success" + + else: + raise Exception( + f"Request parameter 'contentType' = '{content_type}' is invalid. Supported: 'application/json', 'application/x-www-form-urlencoded'") + + except Exception as e: + logging.error(f"ERROR in _post_request: {e}") + logging.error(f"Error type: {type(e)}") + logging.error(f"Traceback: {traceback.format_exc()}") + raise \ No newline at end of file