mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-07-24 19:10:06 +00:00
Refactor POST request handling to streamline logic and add contentType
validation. Remove obsolete request.postJSON
logic and integrate selenium_fetch
. Update requirements.txt
with selenium_fetch
dependency and enhance DTOs to support contentType
and response
fields.
This commit is contained in:
parent
48383a459d
commit
668e0aeca1
@ -11,3 +11,5 @@ websockets==11.0.3
|
||||
xvfbwrapper==0.2.9; platform_system != "Windows"
|
||||
# only required for windows
|
||||
pefile==2023.2.7; platform_system == "Windows"
|
||||
|
||||
selenium_fetch
|
||||
|
@ -19,6 +19,7 @@ class ChallengeResolutionT:
|
||||
status: str = None
|
||||
message: str = None
|
||||
result: ChallengeResolutionResultT = None
|
||||
response: str = None
|
||||
|
||||
def __init__(self, _dict):
|
||||
self.__dict__.update(_dict)
|
||||
@ -39,6 +40,7 @@ class V1RequestBase(object):
|
||||
|
||||
# V1Request
|
||||
url: str = None
|
||||
contentType: str = None
|
||||
postData: str = None
|
||||
returnOnlyCookies: bool = None
|
||||
download: bool = None # deprecated v2.0.0, not used
|
||||
|
@ -1,8 +1,8 @@
|
||||
import json
|
||||
import logging
|
||||
import platform
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from html import escape
|
||||
from urllib.parse import unquote, quote
|
||||
@ -10,13 +10,13 @@ from urllib.parse import unquote, quote
|
||||
from func_timeout import FunctionTimedOut, func_timeout
|
||||
from selenium.common import TimeoutException
|
||||
from selenium.webdriver.chrome.webdriver import WebDriver
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.webdriver.support.expected_conditions import (
|
||||
presence_of_element_located, staleness_of, title_is)
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
from selenium.webdriver.support.wait import WebDriverWait
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium_fetch import fetch, Options
|
||||
|
||||
import utils
|
||||
from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
|
||||
@ -138,8 +138,6 @@ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
|
||||
res = _cmd_request_get(req)
|
||||
elif req.cmd == 'request.post':
|
||||
res = _cmd_request_post(req)
|
||||
elif req.cmd == 'request.postJSON':
|
||||
res = _cmd_request_postJSON(req)
|
||||
else:
|
||||
raise Exception(f"Request parameter 'cmd' = '{req.cmd}' is invalid.")
|
||||
|
||||
@ -152,6 +150,8 @@ def _cmd_request_get(req: V1RequestBase) -> V1ResponseBase:
|
||||
raise Exception("Request parameter 'url' is mandatory in 'request.get' command.")
|
||||
if req.postData is not None:
|
||||
raise Exception("Cannot use 'postBody' when sending a GET request.")
|
||||
if req.contentType is not None:
|
||||
raise Exception("Cannot use 'contentType' when sending a GET request.")
|
||||
if req.returnRawHtml is not None:
|
||||
logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
||||
if req.download is not None:
|
||||
@ -169,6 +169,8 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
|
||||
# do some validations
|
||||
if req.postData is None:
|
||||
raise Exception("Request parameter 'postData' is mandatory in 'request.post' command.")
|
||||
if req.contentType is None:
|
||||
raise Exception("Request parameter 'contentType' is mandatory in 'request.post' command.")
|
||||
if req.returnRawHtml is not None:
|
||||
logging.warning("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
||||
if req.download is not None:
|
||||
@ -312,9 +314,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
||||
# navigate to the page
|
||||
logging.debug(f'Navigating to... {req.url}')
|
||||
if method == 'POST':
|
||||
_post_request(req, driver)
|
||||
elif method == 'POSTJSON':
|
||||
_post_request_json(req, driver)
|
||||
res.response = _post_request(req, driver)
|
||||
else:
|
||||
driver.get(req.url)
|
||||
|
||||
@ -326,7 +326,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
||||
driver.add_cookie(cookie)
|
||||
# reload the page
|
||||
if method == 'POST':
|
||||
_post_request(req, driver)
|
||||
res.response = _post_request(req, driver)
|
||||
else:
|
||||
driver.get(req.url)
|
||||
|
||||
@ -420,172 +420,42 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
||||
|
||||
|
||||
def _post_request(req: V1RequestBase, driver: WebDriver):
|
||||
post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
|
||||
query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
|
||||
pairs = query_string.split('&')
|
||||
for pair in pairs:
|
||||
parts = pair.split('=')
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
name = unquote(parts[0])
|
||||
except Exception:
|
||||
name = parts[0]
|
||||
if name == 'submit':
|
||||
continue
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
value = unquote(parts[1])
|
||||
except Exception:
|
||||
value = parts[1]
|
||||
post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
|
||||
post_form += '</form>'
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
{post_form}
|
||||
<script>document.getElementById('hackForm').submit();</script>
|
||||
</body>
|
||||
</html>"""
|
||||
driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
|
||||
|
||||
def _post_request_json(req: V1RequestBase, driver: WebDriver):
|
||||
try:
|
||||
# Step 2a: JSON Payload Preparation
|
||||
# Convert Python dictionary to JSON string.
|
||||
json_payload_str_raw = json.dumps(req.postData)
|
||||
|
||||
# Step 2b: HTML Document Generation
|
||||
# Construct a minimal HTML5 document with embedded JavaScript.
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>POST Request Executor</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Executing POST Request...</h1>
|
||||
<div id="status-message">Request initiated...</div>
|
||||
|
||||
<script>
|
||||
// JavaScript implementation
|
||||
(async function() {{
|
||||
const url = '{req.url}';
|
||||
// The JSON payload string is directly embedded here.
|
||||
// Note: It's wrapped in single quotes in the JS.
|
||||
const jsonPayloadStr = {json_payload_str_raw};
|
||||
const statusDiv = document.getElementById('status-message');
|
||||
|
||||
console.log('Request URL:', url);
|
||||
console.log('JSON Payload (raw string for JS):', jsonPayloadStr);
|
||||
|
||||
try {{
|
||||
// Parse the JSON string to ensure it's valid before sending
|
||||
// This also correctly unescapes characters for the actual HTTP body
|
||||
const jsonData = JSON.parse(jsonPayloadStr);
|
||||
console.log('JSON Payload (parsed JS object):', jsonData);
|
||||
|
||||
statusDiv.textContent = 'Sending request to ' + url + '...';
|
||||
|
||||
// fetch API Call
|
||||
const response = await fetch(url, {{
|
||||
method: 'POST',
|
||||
headers: {{
|
||||
'Content-Type': 'application/json',
|
||||
// Add any other headers if necessary, e.g., 'Accept': 'application/json'
|
||||
}},
|
||||
body: JSON.stringify(jsonData) // Send the parsed and re-stringified JSON object
|
||||
}});
|
||||
|
||||
statusDiv.textContent = 'Waiting for response... Status: ' + response.status;
|
||||
const responseText = await response.text(); // Get response as text
|
||||
|
||||
if (!response.ok) {{
|
||||
// If response is not ok, throw an error to be caught by the catch block
|
||||
throw new Error(`HTTP error! Status: ${{response.status}} - ${{response.statusText}}. Response: ${{responseText.substring(0, 500)}}`);
|
||||
}}
|
||||
|
||||
// Success Display
|
||||
// Displaying a snippet of the response.
|
||||
const responseSnippet = responseText.substring(0, 200);
|
||||
statusDiv.textContent = `Success! Status: ${{response.status}}. Response Snippet: ${{responseSnippet}}`;
|
||||
console.log('Full Response:', responseText);
|
||||
|
||||
}} catch (error) {{
|
||||
// Error Handling
|
||||
console.error('Fetch Error:', error);
|
||||
statusDiv.textContent = 'Error: ' + error.message;
|
||||
}}
|
||||
}})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
# Step 2c: WebDriver Execution
|
||||
# URL-encode the HTML content for the data URI.
|
||||
# `safe=''` ensures that characters like '/', '?', '&', '=', ':' are also encoded.
|
||||
# However, for data URIs, common characters in HTML are generally fine.
|
||||
# The primary concern is characters that would break the URI structure itself.
|
||||
# Using `quote` without `safe=''` is usually sufficient for `data:text/html`.
|
||||
# Let's be more specific with `safe` if issues arise, but default should be fine.
|
||||
encoded_html_content = quote(html_content)
|
||||
data_uri = f"data:text/html,{encoded_html_content}"
|
||||
|
||||
# Load the data URI into the WebDriver.
|
||||
driver.get(data_uri)
|
||||
|
||||
# Wait for the JavaScript to update the status message.
|
||||
# This indicates that the fetch operation has likely completed (or failed).
|
||||
# We wait for "Success!", "Error:", or "HTTP error!" to appear in the div.
|
||||
# Increased timeout for potentially slow network requests.
|
||||
wait = WebDriverWait(driver, 30) # 30 seconds timeout
|
||||
try:
|
||||
status_element = wait.until(
|
||||
EC.presence_of_element_located((By.ID, "status-message"))
|
||||
)
|
||||
|
||||
# Wait until text contains one of the terminal keywords
|
||||
wait.until(
|
||||
lambda d: "Success!" in status_element.text or \
|
||||
"Error:" in status_element.text or \
|
||||
"HTTP error!" in status_element.text or \
|
||||
"Request initiated..." not in status_element.text # fallback if it never changes from initial
|
||||
)
|
||||
|
||||
# A small explicit sleep to allow final JS updates to the DOM if any race condition.
|
||||
time.sleep(1)
|
||||
|
||||
final_status_text = status_element.text
|
||||
except Exception as e:
|
||||
print(f"Timeout or error waiting for status message: {e}")
|
||||
# Try to get current status message anyway or logs
|
||||
if req.contentType == 'application/x-www-form-urlencoded':
|
||||
post_form = f'<form id="hackForm" action="{req.url}" method="POST">'
|
||||
query_string = req.postData if req.postData[0] != '?' else req.postData[1:]
|
||||
pairs = query_string.split('&')
|
||||
for pair in pairs:
|
||||
parts = pair.split('=')
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
final_status_text = driver.find_element(By.ID, "status-message").text
|
||||
except:
|
||||
final_status_text = "Error: Could not retrieve final status message."
|
||||
|
||||
# Retrieve browser console logs for debugging (optional but good for complex cases)
|
||||
try:
|
||||
browser_logs = driver.get_log('browser')
|
||||
if browser_logs:
|
||||
print("Browser Console Logs:")
|
||||
for entry in browser_logs:
|
||||
print(f" [{entry['level']}] {entry['message']}")
|
||||
except Exception as e:
|
||||
print(f"Could not retrieve browser logs: {e} (This might be normal if not supported by driver/config)")
|
||||
|
||||
return final_status_text
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
error_msg = f"JSON Encoding Error: {e}"
|
||||
print(error_msg)
|
||||
return error_msg
|
||||
except Exception as e:
|
||||
# Catch any other exceptions during the process.
|
||||
error_msg = f"An unexpected error occurred in solve_post_json: {e}"
|
||||
print(error_msg)
|
||||
# It might be useful to also get the current page source for debugging
|
||||
# print("Current Page Source on Error:\n", driver.page_source)
|
||||
return error_msg
|
||||
name = unquote(parts[0])
|
||||
except Exception:
|
||||
name = parts[0]
|
||||
if name == 'submit':
|
||||
continue
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
value = unquote(parts[1])
|
||||
except Exception:
|
||||
value = parts[1]
|
||||
post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
|
||||
post_form += '</form>'
|
||||
html_content = f"""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
{post_form}
|
||||
<script>document.getElementById('hackForm').submit();</script>
|
||||
</body>
|
||||
</html>"""
|
||||
driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
|
||||
return "Success"
|
||||
elif req.contentType == 'application/json':
|
||||
post_data = json.loads(unquote(req.postData))
|
||||
options = Options(method="POST", body=post_data)
|
||||
logging.debug(f"Request => POST /v1 options: {utils.object_to_dict(options)}")
|
||||
response = fetch(driver, req.url, options)
|
||||
logging.debug(f"Response => POST /v1 response: {utils.object_to_dict(response)}")
|
||||
return response.text
|
||||
else:
|
||||
raise Exception(f"Request parameter 'contentType' = '{req.contentType}' is invalid.")
|
||||
|
Loading…
x
Reference in New Issue
Block a user