github/FlareSolverr
1
1
Fork 0
mirror of https://github.com/FlareSolverr/FlareSolverr.git synced 2025-06-08 04:25:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Code clean up, remove returnRawHtml, download, headers params

Browse Source
This commit is contained in:
ngosang 2021-10-17 20:43:36 +02:00
parent a0e897067a
commit a5b3e08e1f
6 changed files with 173 additions and 208 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -86,10 +86,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
--data-raw '{ --data-raw '{
"cmd": "request.get", "cmd": "request.get",
"url":"http://www.google.com/", "url":"http://www.google.com/",
"maxTimeout": 60000, "maxTimeout": 60000
"headers": {
"X-Test": "Testing 123..."
}
}' }'
``` ```
@ -140,11 +137,9 @@ Parameter | Notes
|--|--| |--|--|
url | Mandatory url | Mandatory
session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
headers | Optional. To specify user headers.
maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format. cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
returnRawHtml | Optional, default false. The response data will be returned without JS processing. This is useful for JSON or plain text content.
Example response from running the `curl` above: Example response from running the `curl` above:
@ -211,16 +206,7 @@ This is the same as `request.get` but it takes one more param:
Parameter | Notes Parameter | Notes
|--|--| |--|--|
postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request. postData | Must be a string. If you want to POST a form with format `application/x-www-form-urlencoded`.
### Download small files
If you need to access an image/pdf or small file, you should pass the `download` parameter to `request.get` setting it
to `true`. Rather than access the html and return text it will return the buffer **base64** encoded which you will be
able to decode and save the image/pdf.
This method isn't recommended for videos or anything larger. As that should be streamed back to the client and at the
moment there is nothing setup to do so. If this is something you need feel free to create an issue and/or submit a PR.
## Environment variables ## Environment variables

36
src/controllers/v1.ts
View File

@ -15,11 +15,11 @@ interface V1Routes {
export interface V1RequestBase { export interface V1RequestBase {
cmd: string cmd: string
cookies?: SetCookie[], cookies?: SetCookie[],
headers?: Headers
maxTimeout?: number maxTimeout?: number
proxy?: any// TODO: use interface not any proxy?: any// TODO: use interface not any
session: string session: string
userAgent?: string // deprecated, not used headers?: Headers // deprecated v2, not used
userAgent?: string // deprecated v2, not used
} }
interface V1RequestSession extends V1RequestBase { interface V1RequestSession extends V1RequestBase {
@ -29,9 +29,9 @@ export interface V1Request extends V1RequestBase {
url: string url: string
method?: HttpMethod method?: HttpMethod
postData?: string postData?: string
download?: boolean
returnOnlyCookies?: boolean returnOnlyCookies?: boolean
returnRawHtml?: boolean download?: boolean // deprecated v2, not used
returnRawHtml?: boolean // deprecated v2, not used
} }
export interface V1ResponseBase { export interface V1ResponseBase {
@ -59,7 +59,6 @@ export const routes: V1Routes = {
const options: SessionCreateOptions = { const options: SessionCreateOptions = {
oneTimeSession: false, oneTimeSession: false,
cookies: params.cookies, cookies: params.cookies,
headers: params.headers,
maxTimeout: params.maxTimeout, maxTimeout: params.maxTimeout,
proxy: params.proxy proxy: params.proxy
} }
@ -87,12 +86,15 @@ export const routes: V1Routes = {
}, },
'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => { 'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
params.method = 'GET' params.method = 'GET'
if (params.userAgent) {
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
}
if (params.postData) { if (params.postData) {
throw Error('Cannot use "postBody" when sending a GET request.') throw Error('Cannot use "postBody" when sending a GET request.')
} }
if (params.returnRawHtml) {
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
}
if (params.download) {
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
}
const result: ChallengeResolutionT = await browserRequest(params) const result: ChallengeResolutionT = await browserRequest(params)
response.status = result.status; response.status = result.status;
@ -104,12 +106,15 @@ export const routes: V1Routes = {
}, },
'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => { 'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
params.method = 'POST' params.method = 'POST'
if (params.userAgent) {
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
}
if (!params.postData) { if (!params.postData) {
throw Error('Must send param "postBody" when sending a POST request.') throw Error('Must send param "postBody" when sending a POST request.')
} }
if (params.returnRawHtml) {
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
}
if (params.download) {
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
}
const result: ChallengeResolutionT = await browserRequest(params) const result: ChallengeResolutionT = await browserRequest(params)
response.status = result.status; response.status = result.status;
@ -133,8 +138,15 @@ export async function controllerV1(req: Request, res: Response): Promise<void> {
try { try {
const params: V1RequestBase = req.body const params: V1RequestBase = req.body
if (!params.cmd) { if (!params.cmd) {
throw Error("Parameter 'cmd' is mandatory.") throw Error("Request parameter 'cmd' is mandatory.")
} }
if (params.headers) {
log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
}
if (params.userAgent) {
log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
}
const route = routes[params.cmd] const route = routes[params.cmd]
if (route) { if (route) {
await route(params, response) await route(params, response)

22
src/services/sessions.ts
View File

@ -2,22 +2,17 @@ import {v1 as UUIDv1} from 'uuid'
import * as os from 'os' import * as os from 'os'
import * as path from 'path' import * as path from 'path'
import * as fs from 'fs' import * as fs from 'fs'
import {LaunchOptions, Headers, SetCookie, Browser} from 'puppeteer' import {LaunchOptions, SetCookie, Browser} from 'puppeteer'
import log from './log' import log from './log'
import {deleteFolderRecursive, sleep, removeEmptyFields} from './utils' import {deleteFolderRecursive, sleep} from './utils'
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
interface SessionPageDefaults {
headers?: Headers
}
export interface SessionsCacheItem { export interface SessionsCacheItem {
sessionId: string sessionId: string
browser: Browser browser: Browser
userDataDir?: string userDataDir?: string
defaults: SessionPageDefaults
} }
interface SessionsCache { interface SessionsCache {
@ -27,7 +22,6 @@ interface SessionsCache {
export interface SessionCreateOptions { export interface SessionCreateOptions {
oneTimeSession: boolean oneTimeSession: boolean
cookies?: SetCookie[], cookies?: SetCookie[],
headers?: Headers
maxTimeout?: number maxTimeout?: number
proxy?: any// TODO: use interface not any proxy?: any// TODO: use interface not any
} }
@ -79,6 +73,8 @@ export async function testWebBrowserInstallation(): Promise<void> {
export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> { export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
const sessionId = session || UUIDv1() const sessionId = session || UUIDv1()
// todo: cookies can't be set in the session, you need to open the page first
// todo: these args are only supported in chrome // todo: these args are only supported in chrome
let args = [ let args = [
'--no-sandbox', '--no-sandbox',
@ -126,18 +122,14 @@ export async function create(session: string, options: SessionCreateOptions): Pr
} }
} }
if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) } if (!browser) {
throw Error(`Failed to launch browser 3 times in a row.`)
if (options.cookies) {
const page = await browser.newPage()
await page.setCookie(...options.cookies)
} }
sessionCache[sessionId] = { sessionCache[sessionId] = {
sessionId: sessionId, sessionId: sessionId,
browser: browser, browser: browser,
userDataDir: puppeteerOptions.userDataDir, userDataDir: puppeteerOptions.userDataDir
defaults: removeEmptyFields(options) // todo: review
} }
return sessionCache[sessionId] return sessionCache[sessionId]

239
src/services/solver.ts
View File

@ -1,8 +1,8 @@
import {Response, Headers, Page, Browser} from 'puppeteer' import {Response, Headers, Page} from 'puppeteer'
const Timeout = require('await-timeout'); const Timeout = require('await-timeout');
import log from './log' import log from './log'
import {SessionsCacheItem} from "./sessions"; import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
import {V1Request} from "../controllers/v1"; import {V1Request} from "../controllers/v1";
import cloudflareProvider from '../providers/cloudflare'; import cloudflareProvider from '../providers/cloudflare';
@ -23,22 +23,11 @@ export interface ChallengeResolutionT {
result: ChallengeResolutionResultT result: ChallengeResolutionResultT
} }
// interface OverrideResolvers { async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
// method?: (request: Request) => HttpMethod,
// postData?: (request: Request) => string,
// headers?: (request: Request) => Headers
// }
//
// type OverridesProps =
// 'method' |
// 'postData' |
// 'headers'
async function resolveChallengeWithTimeout(params: V1Request, page: Page) {
const maxTimeout = params.maxTimeout || 60000 const maxTimeout = params.maxTimeout || 60000
const timer = new Timeout(); const timer = new Timeout();
try { try {
const promise = resolveChallenge(params, page); const promise = resolveChallenge(params, session);
return await Promise.race([ return await Promise.race([
promise, promise,
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`) timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
@ -48,167 +37,107 @@ async function resolveChallengeWithTimeout(params: V1Request, page: Page) {
} }
} }
async function resolveChallenge({ url, proxy, download, returnOnlyCookies, returnRawHtml }: V1Request, async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
page: Page): Promise<ChallengeResolutionT | void> {
let status = 'ok'
let message = ''
if (proxy) {
log.debug("Apply proxy");
if (proxy.username)
await page.authenticate({ username: proxy.username, password: proxy.password });
}
log.debug(`Navigating to... ${url}`)
let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
log.html(await page.content())
// Detect protection services and solve challenges
try { try {
response = await cloudflareProvider(url, page, response); let status = 'ok'
} catch (e) { let message = ''
status = "error";
message = "Cloudflare " + e.toString();
}
const payload: ChallengeResolutionT = { const page: Page = await session.browser.newPage()
status,
message, // the user-agent is changed just for linux arm build
result: { await page.setUserAgent(sessions.getUserAgent())
url: page.url(),
status: response.status(), // todo: review
headers: response.headers(), if (params.proxy) {
response: null, log.debug("Apply proxy");
cookies: await page.cookies(), if (params.proxy.username) {
userAgent: await page.evaluate(() => navigator.userAgent) await page.authenticate({
username: params.proxy.username,
password: params.proxy.password
});
}
} }
}
if (returnOnlyCookies) { log.debug(`Navigating to... ${params.url}`)
payload.result.headers = null; let response: Response = await page.goto(params.url, { waitUntil: 'domcontentloaded' })
payload.result.userAgent = null;
} else {
if (download) {
// for some reason we get an error unless we reload the page
// has something to do with a stale buffer and this is the quickest
// fix since I am short on time
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
payload.result.response = (await response.buffer()).toString('base64')
// todo: review this functionality // set cookies
// } else if (returnRawHtml) { if (params.cookies) {
// payload.result.response = await response.text() for (const cookie of params.cookies) {
// the other fields in the cookie can cause issues
await page.setCookie({
"name": cookie.name,
"value": cookie.value
})
}
// reload the page
response = await page.goto(params.url, { waitUntil: 'domcontentloaded' })
}
// log html in debug mode
log.html(await page.content())
// Detect protection services and solve challenges
try {
response = await cloudflareProvider(params.url, page, response);
} catch (e) {
status = "error";
message = "Cloudflare " + e.toString();
}
const payload: ChallengeResolutionT = {
status,
message,
result: {
url: page.url(),
status: response.status(),
headers: response.headers(),
response: null,
cookies: await page.cookies(),
userAgent: sessions.getUserAgent()
}
}
if (params.returnOnlyCookies) {
payload.result.headers = null;
payload.result.userAgent = null;
} else { } else {
payload.result.response = await page.content() payload.result.response = await page.content()
} }
// make sure the page is closed because if it isn't and error will be thrown
// when a user uses a temporary session, the browser make be quit before
// the page is properly closed.
await page.close()
return payload
} catch (e) {
log.error("Unexpected error: " + e);
throw e;
} }
// Add final url in result
payload.result.url = page.url();
// make sure the page is closed because if it isn't and error will be thrown
// when a user uses a temporary session, the browser make be quit before
// the page is properly closed.
await page.close()
return payload
}
function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: V1Request): V1Request {
const copy = { ...defaults, ...params }
// custom merging logic
copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
return copy
}
async function setupPage(params: V1Request, browser: Browser): Promise<Page> {
const page = await browser.newPage()
// merge session defaults with params
const { method, postData, headers, cookies } = params
// the user-agent is changed just for linux arm build
await page.setUserAgent(sessions.getUserAgent())
// todo: redo all functionality
// let overrideResolvers: OverrideResolvers = {}
//
// if (method !== 'GET') {
// log.debug(`Setting method to ${method}`)
// overrideResolvers.method = request => method
// }
//
// if (postData) {
// log.debug(`Setting body data to ${postData}`)
// overrideResolvers.postData = request => postData
// }
//
// if (headers) {
// log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
// overrideResolvers.headers = request => Object.assign(request.headers(), headers)
// }
//
// if (cookies) {
// log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
// await page.setCookie(...cookies)
// }
//
// // if any keys have been set on the object
// if (Object.keys(overrideResolvers).length > 0) {
// let callbackRunOnce = false
// const callback = (request: Request) => {
//
// // avoid loading resources to speed up page load
// if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
// request.abort()
// return
// }
//
// if (callbackRunOnce || !request.isNavigationRequest()) {
// request.continue()
// return
// }
//
// callbackRunOnce = true
// const overrides: Overrides = {}
//
// Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
// // @ts-ignore
// overrides[key] = overrideResolvers[key](request)
// });
//
// log.debug(`Overrides: ${JSON.stringify(overrides)}`)
// request.continue(overrides)
// }
//
// await page.setRequestInterception(true)
// page.on('request', callback)
// }
return page
} }
export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> { export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
const oneTimeSession = params.session === undefined; const oneTimeSession = params.session === undefined;
const options: SessionCreateOptions = {
oneTimeSession: oneTimeSession,
cookies: params.cookies,
maxTimeout: params.maxTimeout,
proxy: params.proxy
}
const session: SessionsCacheItem = oneTimeSession const session: SessionsCacheItem = oneTimeSession
? await sessions.create(null, { ? await sessions.create(null, options)
oneTimeSession: true
})
: sessions.get(params.session) : sessions.get(params.session)
if (!session) { if (!session) {
throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.') throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
} }
params = mergeSessionWithParams(session, params)
try { try {
const page = await setupPage(params, session.browser) // const page = await setupPage(params, session.browser)
return await resolveChallengeWithTimeout(params, page) return await resolveChallengeWithTimeout(params, session)
} catch (error) { } catch (error) {
throw Error("Unable to process browser request. Error: " + error) throw Error("Unable to process browser request. Error: " + error)
} finally { } finally {

10
src/services/utils.ts
View File

@ -19,13 +19,3 @@ export function deleteFolderRecursive(path: string) {
fs.rmdirSync(path) fs.rmdirSync(path)
} }
} }
export const removeEmptyFields = (o: Record<string, any>): typeof o => {
const r: typeof o = {}
for (const k in o) {
if (o[k] !== undefined) {
r[k] = o[k]
}
}
return r
}

56
src/tests/app.test.ts
View File

@ -7,6 +7,7 @@ import {testWebBrowserInstallation} from "../services/sessions";
const request = require("supertest"); const request = require("supertest");
const app = require("../app"); const app = require("../app");
const version: string = require('../../package.json').version const version: string = require('../../package.json').version
const googleUrl = "https://www.google.com"; const googleUrl = "https://www.google.com";
const cfUrl = "https://pirateiro.com/torrents/?search=s"; const cfUrl = "https://pirateiro.com/torrents/?search=s";
const cfCaptchaUrl = "https://idope.se" const cfCaptchaUrl = "https://idope.se"
@ -136,6 +137,60 @@ describe("Test '/v1' path", () => {
expect(apiResponse.solution.url).toContain(cfCaptchaUrl) expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
}); });
test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"cookies": [
{
"name": "testcookie1",
"value": "testvalue1"
},
{
"name": "testcookie2",
"value": "testvalue2"
}
]
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "testcookie1";
})[0].value
expect(cookie1).toBe("testvalue1")
const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "testcookie2";
})[0].value
expect(cookie2).toBe("testvalue2")
});
test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"returnOnlyCookies": true
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
expect(solution.headers).toBe(null)
expect(solution.response).toBe(null)
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toBe(null)
});
test("Cmd 'request.get' should return timeout", async () => { test("Cmd 'request.get' should return timeout", async () => {
const payload = { const payload = {
"cmd": "request.get", "cmd": "request.get",
@ -306,4 +361,5 @@ describe("Test '/v1' path", () => {
expect(cfCookie2.length).toBeGreaterThan(30) expect(cfCookie2.length).toBeGreaterThan(30)
expect(cfCookie2).toBe(cfCookie) expect(cfCookie2).toBe(cfCookie)
}); });
}); });