Feat: added headers and the application/json content type for post requests

requirements.txt

@@ -11,3 +11,5 @@ websockets==11.0.3
 xvfbwrapper==0.2.9; platform_system != "Windows"
 # only required for windows
 pefile==2023.2.7; platform_system == "Windows"
+
+selenium_fetch

src/dtos.py

@@ -19,6 +19,7 @@ class ChallengeResolutionT:
     status: str = None
     message: str = None
     result: ChallengeResolutionResultT = None
+    response: str = None
 
     def __init__(self, _dict):
         self.__dict__.update(_dict)
@@ -39,7 +40,9 @@ class V1RequestBase(object):
 
     # V1Request
     url: str = None
+    contentType: str = None
     postData: str = None
+    headers: dict = None
     returnOnlyCookies: bool = None
     download: bool = None   # deprecated v2.0.0, not used
     returnRawHtml: bool = None  # deprecated v2.0.0, not used

src/flaresolverr_service.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import platform
 import sys
@@ -9,12 +10,13 @@ from urllib.parse import unquote, quote
 from func_timeout import FunctionTimedOut, func_timeout
 from selenium.common import TimeoutException
 from selenium.webdriver.chrome.webdriver import WebDriver
+from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support.expected_conditions import (
     presence_of_element_located, staleness_of, title_is)
-from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.support.wait import WebDriverWait
+from selenium_fetch import fetch, Options
 
 import utils
 from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
@@ -148,6 +150,10 @@ def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase:
         raise Exception("Request parameter 'url' is mandatory in 'request.get' command.")
     if req.postData is not None:
         raise Exception("Cannot use 'postBody' when sending a GET request.")
+    if req.contentType is not None:
+        raise Exception("Cannot use 'contentType' when sending a GET request.")
+    if req.headers is not None:
+        raise Exception("Cannot use 'headers' when sending a GET request.")
     if req.returnRawHtml is not None:
         logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
     if req.download is not None:
@@ -165,6 +171,8 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
     # do some validations
     if req.postData is None:
         raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
+    if req.contentType is None:
+        raise Exception("Request parameter 'contentType' is mandatory in 'request.post' command.")
     if req.returnRawHtml is not None:
         logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
     if req.download is not None:
@@ -178,6 +186,23 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
     return res
 
 
+def _cmd_request_postJSON(req: V1RequestBase) -> V1ResponseBase:
+    # do some validations
+    if req.postData is None:
+        raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
+    if req.returnRawHtml is not None:
+        logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    if req.download is not None:
+        logging.warning("Request parameter 'download' was removed in FlareSolverr v2.")
+
+    challenge_res = _resolve_challenge(req, 'POSTJSON')
+    res = V1ResponseBase({})
+    res.status = challenge_res.status
+    res.message = challenge_res.message
+    res.solution = challenge_res.result
+    return res
+
+
 def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
     logging.debug("Creating new session...")
 
@@ -291,7 +316,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
     # navigate to the page
     logging.debug(f'Navigating to... {req.url}')
     if method == 'POST':
-        _post_request(req, driver)
+        res.response = _post_request(req, driver)
     else:
         driver.get(req.url)
 
@@ -303,7 +328,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
             driver.add_cookie(cookie)
         # reload the page
         if method == 'POST':
-            _post_request(req, driver)
+            res.response = _post_request(req, driver)
         else:
             driver.get(req.url)
 
@@ -397,31 +422,148 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
 
 def _post_request(req: V1RequestBase, driver: WebDriver):
-    post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
+    import logging
-    query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
+    import traceback
-    pairs = query_string.split('&')
+    import time
-    for pair in pairs:
+
-        parts = pair.split('=')
+    try:
-        # noinspection PyBroadException
+        content_type = getattr(req, 'contentType', 'application/x-www-form-urlencoded')
-        try:
+        headers = getattr(req, 'headers', {})
-            name = unquote(parts[0])
+
-        except Exception:
+        if content_type == 'application/json':
-            name = parts[0]
+
-        if name == 'submit':
+            if not req.postData:
-            continue
+                raise Exception("postData is empty for JSON request")
-        # noinspection PyBroadException
+
-        try:
+            try:
-            value = unquote(parts[1])
+                if isinstance(req.postData, str):
-        except Exception:
+                    post_data = json.loads(req.postData)
-            value = parts[1]
+                else:
-        post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
+                    post_data = req.postData
-    post_form += '</form>'
+            except json.JSONDecodeError as e:
-    html_content = f"""
+                logging.error(f"JSON parsing failed: {e}")
-        <!DOCTYPE html>
+                raise Exception(f"Invalid JSON in postData: {e}")
-        <html>
+
-        <body>
+            try:
-            {post_form}
+                driver.get(req.url)
-            <script>document.getElementById('hackForm').submit();</script>
+
-        </body>
+                time.sleep(2)
-        </html>"""
+
-    driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
+                page_source = driver.page_source.lower()
+                if any(term in page_source for term in
+                       ['cloudflare', 'checking your browser', 'ddos protection', 'please wait']):
+                    logging.info("Protection detected, waiting for bypass...")
+                    time.sleep(5)
+
+            except Exception as e:
+                logging.warning(f"Could not load target page directly: {e}")
+
+            json_data_str = json.dumps(post_data)
+            escaped_json = json_data_str.replace("'", "\\'")
+
+            headers_js_lines = ["xhr.setRequestHeader('Content-Type', 'application/json');"]
+
+            if headers:
+                for header_name, header_value in headers.items():
+                    if header_name.lower() != 'content-type':
+                        escaped_value = str(header_value).replace("'", "\\'")
+                        headers_js_lines.append(f"xhr.setRequestHeader('{header_name}', '{escaped_value}');")
+
+            headers_js = '\n            '.join(headers_js_lines)
+
+            script = f"""
+            var xhr = new XMLHttpRequest();
+            xhr.open('POST', '{req.url}', false);
+            {headers_js}
+
+            try {{
+                xhr.send('{escaped_json}');
+                return {{
+                    status: xhr.status,
+                    statusText: xhr.statusText,
+                    responseText: xhr.responseText,
+                    success: true
+                }};
+            }} catch (error) {{
+                return {{
+                    status: 0,
+                    statusText: error.message,
+                    responseText: '',
+                    success: false,
+                    error: error.message
+                }};
+            }}
+            """
+
+            try:
+                result = driver.execute_script(script)
+
+                if result and result.get('success'):
+                    response_text = result.get('responseText', '')
+                    status_code = result.get('status', 0)
+
+                    logging.info(f"POST request completed with status: {status_code}")
+
+                    return response_text
+                else:
+                    error_msg = result.get('statusText', 'Unknown error') if result else 'Script execution failed'
+                    error_detail = result.get('error', '') if result else ''
+                    logging.error(f"XHR request failed: {error_msg} - {error_detail}")
+                    raise Exception(f"POST request failed: {error_msg}")
+
+            except Exception as script_error:
+                logging.error(f"Script execution error: {script_error}")
+                raise
+
+        elif content_type == 'application/x-www-form-urlencoded':
+
+            headers_meta = ""
+            if headers:
+                for header_name, header_value in headers.items():
+                    if header_name.lower() != 'content-type':
+                        headers_meta += f'<meta http-equiv="{header_name}" content="{header_value}">'
+
+            post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
+            query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
+            pairs = query_string.split('&')
+
+            for pair in pairs:
+                if '=' not in pair:
+                    continue
+                parts = pair.split('=', 1)
+                try:
+                    name = unquote(parts[0])
+                except:
+                    name = parts[0]
+                if name == 'submit':
+                    continue
+                try:
+                    value = unquote(parts[1]) if len(parts) > 1 else ''
+                except:
+                    value = parts[1] if len(parts) > 1 else ''
+                post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
+
+            post_form += '</form>'
+            html_content = f"""
+                    <!DOCTYPE html>
+                    <html>
+                    <head>
+                        {headers_meta}
+                    </head>
+                    <body>
+                        {post_form}
+                        <script>document.getElementById('hackForm').submit();</script>
+                    </body>
+                    </html>"""
+            driver.get(f"data:text/html;charset=utf-8,{html_content}")
+            return "Success"
+
+        else:
+            raise Exception(
+                f"Request parameter 'contentType' = '{content_type}' is invalid. Supported: 'application/json', 'application/x-www-form-urlencoded'")
+
+    except Exception as e:
+        logging.error(f"ERROR in _post_request: {e}")
+        logging.error(f"Error type: {type(e)}")
+        logging.error(f"Traceback: {traceback.format_exc()}")
+        raise