mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-06-09 12:57:12 +00:00
Refactor the app to use Express server and Jest for tests
This commit is contained in:
parent
0459f2642d
commit
744de4d158
@ -69,7 +69,7 @@ This is the recommended way for macOS users and for developers.
|
|||||||
* Install [NodeJS](https://nodejs.org/).
|
* Install [NodeJS](https://nodejs.org/).
|
||||||
* Clone this repository and open a shell in that path.
|
* Clone this repository and open a shell in that path.
|
||||||
* Run `PUPPETEER_PRODUCT=firefox npm install` command to install FlareSolverr dependencies.
|
* Run `PUPPETEER_PRODUCT=firefox npm install` command to install FlareSolverr dependencies.
|
||||||
* Run `node node_modules/puppeteer/install.js` to install Firefox.
|
* Run `PUPPETEER_PRODUCT=firefox node node_modules/puppeteer/install.js` to install Firefox.
|
||||||
* Run `npm run build` command to compile TypeScript code.
|
* Run `npm run build` command to compile TypeScript code.
|
||||||
* Run `npm start` command to start FlareSolverr.
|
* Run `npm start` command to start FlareSolverr.
|
||||||
|
|
||||||
|
@ -26,7 +26,7 @@ const version = 'v' + require('./package.json').version;
|
|||||||
fsZipName: 'windows-x64',
|
fsZipName: 'windows-x64',
|
||||||
fsLicenseName: 'LICENSE.txt'
|
fsLicenseName: 'LICENSE.txt'
|
||||||
}
|
}
|
||||||
// TODO: this is working but changes are required in session.ts to find chrome path
|
// TODO: this is working but changes are required in sessions.ts to find chrome path
|
||||||
// {
|
// {
|
||||||
// platform: 'mac',
|
// platform: 'mac',
|
||||||
// version: 756035,
|
// version: 756035,
|
||||||
|
5
jest.config.js
Normal file
5
jest.config.js
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
module.exports = {
|
||||||
|
transform: {
|
||||||
|
'^.+\\.(ts|tsx)$': 'ts-jest'
|
||||||
|
}
|
||||||
|
}
|
8370
package-lock.json
generated
8370
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
15
package.json
15
package.json
@ -3,10 +3,11 @@
|
|||||||
"version": "1.2.9",
|
"version": "1.2.9",
|
||||||
"description": "Proxy server to bypass Cloudflare protection.",
|
"description": "Proxy server to bypass Cloudflare protection.",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node ./dist/index.js",
|
"start": "node ./dist/server.js",
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"dev": "nodemon -e ts --exec ts-node src/index.ts",
|
"dev": "nodemon -e ts --exec ts-node src/server.ts",
|
||||||
"package": "node build-binaries.js"
|
"package": "node build-binaries.js",
|
||||||
|
"test": "jest --runInBand"
|
||||||
},
|
},
|
||||||
"author": "Diego Heras (ngosang)",
|
"author": "Diego Heras (ngosang)",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
@ -19,7 +20,9 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"await-timeout": "^1.1.1",
|
"await-timeout": "^1.1.1",
|
||||||
|
"body-parser": "^1.19.0",
|
||||||
"console-log-level": "^1.4.1",
|
"console-log-level": "^1.4.1",
|
||||||
|
"express": "^4.17.1",
|
||||||
"got": "^11.8.2",
|
"got": "^11.8.2",
|
||||||
"hcaptcha-solver": "^1.0.2",
|
"hcaptcha-solver": "^1.0.2",
|
||||||
"puppeteer": "^3.3.0",
|
"puppeteer": "^3.3.0",
|
||||||
@ -27,12 +30,18 @@
|
|||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/await-timeout": "^0.3.1",
|
"@types/await-timeout": "^0.3.1",
|
||||||
|
"@types/body-parser": "^1.19.1",
|
||||||
|
"@types/express": "^4.17.13",
|
||||||
|
"@types/jest": "^27.0.2",
|
||||||
"@types/node": "^14.17.27",
|
"@types/node": "^14.17.27",
|
||||||
"@types/puppeteer": "^3.0.6",
|
"@types/puppeteer": "^3.0.6",
|
||||||
|
"@types/supertest": "^2.0.11",
|
||||||
"@types/uuid": "^8.3.1",
|
"@types/uuid": "^8.3.1",
|
||||||
"archiver": "^5.3.0",
|
"archiver": "^5.3.0",
|
||||||
"nodemon": "^2.0.13",
|
"nodemon": "^2.0.13",
|
||||||
"pkg": "^5.3.3",
|
"pkg": "^5.3.3",
|
||||||
|
"supertest": "^6.1.6",
|
||||||
|
"ts-jest": "^27.0.7",
|
||||||
"ts-node": "^10.3.0",
|
"ts-node": "^10.3.0",
|
||||||
"typescript": "^4.4.4"
|
"typescript": "^4.4.4"
|
||||||
}
|
}
|
||||||
|
56
src/app.ts
Normal file
56
src/app.ts
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
import log from './services/log'
|
||||||
|
import {Request, Response} from 'express';
|
||||||
|
import {getUserAgent} from "./services/sessions";
|
||||||
|
import {controllerV1} from "./controllers/v1";
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const app = express();
|
||||||
|
const bodyParser = require('body-parser');
|
||||||
|
const version: string = require('../package.json').version
|
||||||
|
|
||||||
|
// Convert request objects to JSON
|
||||||
|
app.use(bodyParser.json({
|
||||||
|
limit: '50mb',
|
||||||
|
verify(req: Request, res: Response, buf: any) {
|
||||||
|
req.body = buf;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
// *********************************************************************************************************************
|
||||||
|
// Routes
|
||||||
|
|
||||||
|
// show welcome message
|
||||||
|
app.get("/", ( req: Request, res: Response ) => {
|
||||||
|
log.info(`Incoming request: /`);
|
||||||
|
res.send({
|
||||||
|
"msg": "FlareSolverr is ready!",
|
||||||
|
"version": version,
|
||||||
|
"userAgent": getUserAgent()
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// health endpoint. this endpoint is special because it doesn't print traces
|
||||||
|
app.get("/health", ( req: Request, res: Response ) => {
|
||||||
|
res.send({
|
||||||
|
"status": "ok"
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// controller v1
|
||||||
|
app.post("/v1", async( req: Request, res: Response ) => {
|
||||||
|
// count the request for the log prefix
|
||||||
|
log.incRequests()
|
||||||
|
|
||||||
|
const params = req.body;
|
||||||
|
log.info(`Incoming request: /v1 Params: ${JSON.stringify(params)}`);
|
||||||
|
await controllerV1(req, res);
|
||||||
|
});
|
||||||
|
|
||||||
|
// *********************************************************************************************************************
|
||||||
|
|
||||||
|
// Unknown paths or verbs
|
||||||
|
app.use(function (req : Request, res : Response) {
|
||||||
|
res.status(404).send({"error": "Unknown resource or HTTP verb"})
|
||||||
|
})
|
||||||
|
|
||||||
|
module.exports = app;
|
@ -1,5 +1,5 @@
|
|||||||
import got from 'got'
|
import got from 'got'
|
||||||
import { sleep } from '../utils'
|
import { sleep } from '../services/utils'
|
||||||
|
|
||||||
/*
|
/*
|
||||||
This method uses the captcha-harvester project:
|
This method uses the captcha-harvester project:
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
import log from "../log";
|
import log from "../services/log";
|
||||||
|
|
||||||
export enum CaptchaType {
|
export enum CaptchaType {
|
||||||
re = 'reCaptcha',
|
re = 'reCaptcha',
|
||||||
|
155
src/controllers/v1.ts
Normal file
155
src/controllers/v1.ts
Normal file
@ -0,0 +1,155 @@
|
|||||||
|
// todo: avoid puppeter objects
|
||||||
|
import {SetCookie, Headers, HttpMethod} from 'puppeteer'
|
||||||
|
import {Request, Response} from 'express';
|
||||||
|
|
||||||
|
import log from '../services/log'
|
||||||
|
import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
|
||||||
|
import {SessionCreateOptions} from "../services/sessions";
|
||||||
|
const sessions = require('../services/sessions')
|
||||||
|
const version: string = require('../../package.json').version
|
||||||
|
|
||||||
|
interface V1Routes {
|
||||||
|
[key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1RequestBase {
|
||||||
|
cmd: string
|
||||||
|
cookies?: SetCookie[],
|
||||||
|
headers?: Headers
|
||||||
|
maxTimeout?: number
|
||||||
|
proxy?: any// TODO: use interface not any
|
||||||
|
session: string
|
||||||
|
userAgent?: string // deprecated, not used
|
||||||
|
}
|
||||||
|
|
||||||
|
interface V1RequestSession extends V1RequestBase {
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1Request extends V1RequestBase {
|
||||||
|
url: string
|
||||||
|
method?: HttpMethod
|
||||||
|
postData?: string
|
||||||
|
download?: boolean
|
||||||
|
returnOnlyCookies?: boolean
|
||||||
|
returnRawHtml?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseBase {
|
||||||
|
status: string
|
||||||
|
message: string
|
||||||
|
startTimestamp: number
|
||||||
|
endTimestamp: number
|
||||||
|
version: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSolution extends V1ResponseBase {
|
||||||
|
solution: ChallengeResolutionResultT
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSession extends V1ResponseBase {
|
||||||
|
session: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSessions extends V1ResponseBase {
|
||||||
|
sessions: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export const routes: V1Routes = {
|
||||||
|
'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
|
||||||
|
const options: SessionCreateOptions = {
|
||||||
|
oneTimeSession: false,
|
||||||
|
cookies: params.cookies,
|
||||||
|
headers: params.headers,
|
||||||
|
maxTimeout: params.maxTimeout,
|
||||||
|
proxy: params.proxy
|
||||||
|
}
|
||||||
|
const { sessionId, browser } = await sessions.create(params.session, options)
|
||||||
|
if (browser) {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "Session created successfully.";
|
||||||
|
response.session = sessionId
|
||||||
|
} else {
|
||||||
|
throw Error('Error creating session.')
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "";
|
||||||
|
response.sessions = sessions.list();
|
||||||
|
},
|
||||||
|
'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
|
||||||
|
if (await sessions.destroy(params.session)) {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "The session has been removed.";
|
||||||
|
} else {
|
||||||
|
throw Error('This session does not exist.')
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
||||||
|
params.method = 'GET'
|
||||||
|
if (params.userAgent) {
|
||||||
|
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
|
||||||
|
}
|
||||||
|
if (params.postData) {
|
||||||
|
throw Error('Cannot use "postBody" when sending a GET request.')
|
||||||
|
}
|
||||||
|
const result: ChallengeResolutionT = await browserRequest(params)
|
||||||
|
|
||||||
|
response.status = result.status;
|
||||||
|
response.message = result.message;
|
||||||
|
response.solution = result.result;
|
||||||
|
if (response.message) {
|
||||||
|
log.info(response.message)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
||||||
|
params.method = 'POST'
|
||||||
|
if (params.userAgent) {
|
||||||
|
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
|
||||||
|
}
|
||||||
|
if (!params.postData) {
|
||||||
|
throw Error('Must send param "postBody" when sending a POST request.')
|
||||||
|
}
|
||||||
|
const result: ChallengeResolutionT = await browserRequest(params)
|
||||||
|
|
||||||
|
response.status = result.status;
|
||||||
|
response.message = result.message;
|
||||||
|
response.solution = result.result;
|
||||||
|
if (response.message) {
|
||||||
|
log.info(response.message)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function controllerV1(req: Request, res: Response): Promise<void> {
|
||||||
|
const response: V1ResponseBase = {
|
||||||
|
status: null,
|
||||||
|
message: null,
|
||||||
|
startTimestamp: Date.now(),
|
||||||
|
endTimestamp: 0,
|
||||||
|
version: version
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const params: V1RequestBase = req.body
|
||||||
|
if (!params.cmd) {
|
||||||
|
throw Error("Parameter 'cmd' is mandatory.")
|
||||||
|
}
|
||||||
|
const route = routes[params.cmd]
|
||||||
|
if (route) {
|
||||||
|
await route(params, response)
|
||||||
|
|
||||||
|
} else {
|
||||||
|
throw Error(`The command '${params.cmd}' is invalid.`)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500)
|
||||||
|
response.status = "error";
|
||||||
|
response.message = e.toString();
|
||||||
|
log.error(response.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
response.endTimestamp = Date.now()
|
||||||
|
log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
|
||||||
|
res.send(response)
|
||||||
|
}
|
194
src/index.ts
194
src/index.ts
@ -1,194 +0,0 @@
|
|||||||
const fs = require('fs');
|
|
||||||
const os = require('os');
|
|
||||||
const path = require('path');
|
|
||||||
const process = require('process')
|
|
||||||
import log from './log'
|
|
||||||
import { createServer, IncomingMessage, ServerResponse } from 'http';
|
|
||||||
import { RequestContext } from './types'
|
|
||||||
import Router, { BaseAPICall } from './routes'
|
|
||||||
import getCaptchaSolver from "./captcha";
|
|
||||||
import sessions from "./session";
|
|
||||||
import {v1 as UUIDv1} from "uuid";
|
|
||||||
|
|
||||||
const version: string = "v" + require('../package.json').version
|
|
||||||
const serverPort: number = Number(process.env.PORT) || 8191
|
|
||||||
const serverHost: string = process.env.HOST || '0.0.0.0'
|
|
||||||
let webBrowserUserAgent: string = ""
|
|
||||||
|
|
||||||
function validateEnvironmentVariables() {
|
|
||||||
// ip and port variables are validated by nodejs
|
|
||||||
if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
|
|
||||||
log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
|
|
||||||
log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
|
|
||||||
log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
getCaptchaSolver();
|
|
||||||
} catch (e) {
|
|
||||||
log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testWebBrowserInstallation() {
|
|
||||||
const sessionId = UUIDv1()
|
|
||||||
log.debug("Testing web browser installation...")
|
|
||||||
const session = await sessions.create(sessionId, {
|
|
||||||
oneTimeSession: true
|
|
||||||
})
|
|
||||||
const page = await session.browser.newPage()
|
|
||||||
await page.goto("https://www.google.com")
|
|
||||||
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
|
|
||||||
|
|
||||||
// replace Linux ARM user-agent because it's detected
|
|
||||||
if (webBrowserUserAgent.toLocaleLowerCase().includes('linux arm')) {
|
|
||||||
webBrowserUserAgent = webBrowserUserAgent.replace(/linux arm[^;]+;/i, 'Linux x86_64;')
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
|
|
||||||
await page.close()
|
|
||||||
await sessions.destroy(sessionId)
|
|
||||||
log.debug("Test successful")
|
|
||||||
}
|
|
||||||
|
|
||||||
function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
|
|
||||||
log.error(errorMsg)
|
|
||||||
const response = {
|
|
||||||
status: 'error',
|
|
||||||
message: errorMsg,
|
|
||||||
startTimestamp,
|
|
||||||
endTimestamp: Date.now(),
|
|
||||||
version
|
|
||||||
}
|
|
||||||
res.writeHead(500, {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
res.write(JSON.stringify(response))
|
|
||||||
res.end()
|
|
||||||
}
|
|
||||||
|
|
||||||
function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
|
|
||||||
const endTimestamp = Date.now()
|
|
||||||
log.info(`Response in ${(endTimestamp - startTimestamp) / 1000} s`)
|
|
||||||
if (successMsg) { log.info(successMsg) }
|
|
||||||
|
|
||||||
const response = Object.assign({
|
|
||||||
status: 'ok',
|
|
||||||
message: successMsg || '',
|
|
||||||
startTimestamp,
|
|
||||||
endTimestamp,
|
|
||||||
version
|
|
||||||
}, extendedProperties || {})
|
|
||||||
res.writeHead(200, {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
res.write(JSON.stringify(response))
|
|
||||||
res.end()
|
|
||||||
}
|
|
||||||
|
|
||||||
function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
|
|
||||||
log.info(`Params: ${JSON.stringify(params)}`)
|
|
||||||
|
|
||||||
if (ctx.req.method !== 'POST') {
|
|
||||||
ctx.errorResponse('Only the POST method is allowed')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ctx.req.url !== '/v1') {
|
|
||||||
ctx.errorResponse('Only /v1 endpoint is allowed')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!params.cmd) {
|
|
||||||
ctx.errorResponse("Parameter 'cmd' is mandatory")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// init
|
|
||||||
log.info(`FlareSolverr ${version}`);
|
|
||||||
log.debug('Debug log enabled');
|
|
||||||
|
|
||||||
process.on('SIGTERM', () => {
|
|
||||||
// Capture signal on Docker Stop #158
|
|
||||||
log.info("Process interrupted")
|
|
||||||
process.exit(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
validateEnvironmentVariables();
|
|
||||||
|
|
||||||
testWebBrowserInstallation()
|
|
||||||
.catch(e => {
|
|
||||||
log.error("Error starting web browser.", e);
|
|
||||||
process.exit(1);
|
|
||||||
})
|
|
||||||
.then(r =>
|
|
||||||
createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
||||||
const startTimestamp = Date.now()
|
|
||||||
|
|
||||||
// health endpoint. this endpoint is special because it doesn't print traces
|
|
||||||
if (req.url == '/health') {
|
|
||||||
res.writeHead(200, {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
res.write(JSON.stringify({"status": "ok"}))
|
|
||||||
res.end()
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// count the request for the log prefix
|
|
||||||
log.incRequests()
|
|
||||||
log.info(`Incoming request: ${req.method} ${req.url}`)
|
|
||||||
|
|
||||||
// show welcome message
|
|
||||||
if (req.url == '/') {
|
|
||||||
const extendedProperties = {"userAgent": webBrowserUserAgent};
|
|
||||||
successResponse("FlareSolverr is ready!", extendedProperties, res, startTimestamp);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// get request body
|
|
||||||
const bodyParts: any[] = []
|
|
||||||
req.on('data', chunk => {
|
|
||||||
bodyParts.push(chunk)
|
|
||||||
}).on('end', () => {
|
|
||||||
// parse params
|
|
||||||
const body = Buffer.concat(bodyParts).toString()
|
|
||||||
let params: BaseAPICall = null
|
|
||||||
try {
|
|
||||||
params = JSON.parse(body)
|
|
||||||
} catch (err) {
|
|
||||||
errorResponse('Body must be in JSON format', res, startTimestamp)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const ctx: RequestContext = {
|
|
||||||
req,
|
|
||||||
res,
|
|
||||||
startTimestamp,
|
|
||||||
errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
|
|
||||||
successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
// validate params
|
|
||||||
if (!validateIncomingRequest(ctx, params)) { return }
|
|
||||||
|
|
||||||
// process request
|
|
||||||
Router(ctx, params, webBrowserUserAgent).catch(e => {
|
|
||||||
console.error(e)
|
|
||||||
ctx.errorResponse(e.message)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}).listen(serverPort, serverHost, () => {
|
|
||||||
log.info(`Listening on http://${serverHost}:${serverPort}`);
|
|
||||||
})
|
|
||||||
)
|
|
@ -1,7 +1,8 @@
|
|||||||
import log from "../log";
|
|
||||||
import getCaptchaSolver, {CaptchaType} from "../captcha";
|
|
||||||
import {Page, Response} from 'puppeteer'
|
import {Page, Response} from 'puppeteer'
|
||||||
|
|
||||||
|
import log from "../services/log";
|
||||||
|
import getCaptchaSolver, {CaptchaType} from "../captcha";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This class contains the logic to solve protections provided by CloudFlare
|
* This class contains the logic to solve protections provided by CloudFlare
|
||||||
**/
|
**/
|
||||||
|
307
src/routes.ts
307
src/routes.ts
@ -1,307 +0,0 @@
|
|||||||
import { v1 as UUIDv1 } from 'uuid'
|
|
||||||
import {SetCookie, Request, Response, Headers, HttpMethod, Overrides, Page, Browser} from 'puppeteer'
|
|
||||||
const Timeout = require('await-timeout');
|
|
||||||
|
|
||||||
import log from './log'
|
|
||||||
import sessions, { SessionsCacheItem } from './session'
|
|
||||||
import { RequestContext } from './types'
|
|
||||||
import cloudflareProvider from './providers/cloudflare';
|
|
||||||
|
|
||||||
export interface BaseAPICall {
|
|
||||||
cmd: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BaseSessionsAPICall extends BaseAPICall {
|
|
||||||
session?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionsCreateAPICall extends BaseSessionsAPICall {
|
|
||||||
cookies?: SetCookie[],
|
|
||||||
headers?: Headers
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BaseRequestAPICall extends BaseAPICall {
|
|
||||||
url: string
|
|
||||||
method?: HttpMethod
|
|
||||||
postData?: string
|
|
||||||
session?: string
|
|
||||||
userAgent?: string // deprecated, not used
|
|
||||||
maxTimeout?: number
|
|
||||||
cookies?: SetCookie[],
|
|
||||||
headers?: Headers
|
|
||||||
proxy?: any, // TODO: use interface not any
|
|
||||||
download?: boolean
|
|
||||||
returnOnlyCookies?: boolean
|
|
||||||
returnRawHtml?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
interface Routes {
|
|
||||||
[key: string]: (ctx: RequestContext, params: BaseAPICall, webBrowserUserAgent: string) => void | Promise<void>
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChallengeResolutionResultT {
|
|
||||||
url: string
|
|
||||||
status: number,
|
|
||||||
headers?: Headers,
|
|
||||||
response: string,
|
|
||||||
cookies: object[]
|
|
||||||
userAgent: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChallengeResolutionT {
|
|
||||||
status?: string
|
|
||||||
message: string
|
|
||||||
result: ChallengeResolutionResultT
|
|
||||||
}
|
|
||||||
|
|
||||||
interface OverrideResolvers {
|
|
||||||
method?: (request: Request) => HttpMethod,
|
|
||||||
postData?: (request: Request) => string,
|
|
||||||
headers?: (request: Request) => Headers
|
|
||||||
}
|
|
||||||
|
|
||||||
type OverridesProps =
|
|
||||||
'method' |
|
|
||||||
'postData' |
|
|
||||||
'headers'
|
|
||||||
|
|
||||||
async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Page) {
|
|
||||||
const maxTimeout = params.maxTimeout || 60000
|
|
||||||
const timer = new Timeout();
|
|
||||||
try {
|
|
||||||
const promise = resolveChallenge(ctx, params, page);
|
|
||||||
return await Promise.race([
|
|
||||||
promise,
|
|
||||||
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
|
|
||||||
]);
|
|
||||||
} finally {
|
|
||||||
timer.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveChallenge(ctx: RequestContext,
|
|
||||||
{ url, proxy, download, returnOnlyCookies, returnRawHtml }: BaseRequestAPICall,
|
|
||||||
page: Page): Promise<ChallengeResolutionT | void> {
|
|
||||||
|
|
||||||
let status = 'ok'
|
|
||||||
let message = ''
|
|
||||||
|
|
||||||
if (proxy) {
|
|
||||||
log.debug("Apply proxy");
|
|
||||||
if (proxy.username)
|
|
||||||
await page.authenticate({ username: proxy.username, password: proxy.password });
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug(`Navigating to... ${url}`)
|
|
||||||
let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
|
||||||
log.html(await page.content())
|
|
||||||
|
|
||||||
// Detect protection services and solve challenges
|
|
||||||
try {
|
|
||||||
response = await cloudflareProvider(url, page, response);
|
|
||||||
} catch (e) {
|
|
||||||
status = "error";
|
|
||||||
message = "Cloudflare " + e.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload: ChallengeResolutionT = {
|
|
||||||
status,
|
|
||||||
message,
|
|
||||||
result: {
|
|
||||||
url: page.url(),
|
|
||||||
status: response.status(),
|
|
||||||
headers: response.headers(),
|
|
||||||
response: null,
|
|
||||||
cookies: await page.cookies(),
|
|
||||||
userAgent: await page.evaluate(() => navigator.userAgent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (returnOnlyCookies) {
|
|
||||||
payload.result.headers = null;
|
|
||||||
payload.result.userAgent = null;
|
|
||||||
} else {
|
|
||||||
if (download) {
|
|
||||||
// for some reason we get an error unless we reload the page
|
|
||||||
// has something to do with a stale buffer and this is the quickest
|
|
||||||
// fix since I am short on time
|
|
||||||
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
|
||||||
payload.result.response = (await response.buffer()).toString('base64')
|
|
||||||
|
|
||||||
// todo: review this functionality
|
|
||||||
// } else if (returnRawHtml) {
|
|
||||||
// payload.result.response = await response.text()
|
|
||||||
} else {
|
|
||||||
payload.result.response = await page.content()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add final url in result
|
|
||||||
payload.result.url = page.url();
|
|
||||||
|
|
||||||
// make sure the page is closed because if it isn't and error will be thrown
|
|
||||||
// when a user uses a temporary session, the browser make be quit before
|
|
||||||
// the page is properly closed.
|
|
||||||
await page.close()
|
|
||||||
|
|
||||||
return payload
|
|
||||||
}
|
|
||||||
|
|
||||||
function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseRequestAPICall): BaseRequestAPICall {
|
|
||||||
const copy = { ...defaults, ...params }
|
|
||||||
|
|
||||||
// custom merging logic
|
|
||||||
copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
|
|
||||||
|
|
||||||
return copy
|
|
||||||
}
|
|
||||||
|
|
||||||
async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Browser,
|
|
||||||
webBrowserUserAgent: string): Promise<Page> {
|
|
||||||
const page = await browser.newPage()
|
|
||||||
|
|
||||||
// merge session defaults with params
|
|
||||||
const { method, postData, headers, cookies } = params
|
|
||||||
|
|
||||||
// the user-agent is changed just for linux arm build
|
|
||||||
await page.setUserAgent(webBrowserUserAgent)
|
|
||||||
|
|
||||||
// todo: redo all functionality
|
|
||||||
|
|
||||||
// let overrideResolvers: OverrideResolvers = {}
|
|
||||||
//
|
|
||||||
// if (method !== 'GET') {
|
|
||||||
// log.debug(`Setting method to ${method}`)
|
|
||||||
// overrideResolvers.method = request => method
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (postData) {
|
|
||||||
// log.debug(`Setting body data to ${postData}`)
|
|
||||||
// overrideResolvers.postData = request => postData
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (headers) {
|
|
||||||
// log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
|
|
||||||
// overrideResolvers.headers = request => Object.assign(request.headers(), headers)
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (cookies) {
|
|
||||||
// log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
|
|
||||||
// await page.setCookie(...cookies)
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// // if any keys have been set on the object
|
|
||||||
// if (Object.keys(overrideResolvers).length > 0) {
|
|
||||||
// let callbackRunOnce = false
|
|
||||||
// const callback = (request: Request) => {
|
|
||||||
//
|
|
||||||
// // avoid loading resources to speed up page load
|
|
||||||
// if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
|
|
||||||
// request.abort()
|
|
||||||
// return
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// if (callbackRunOnce || !request.isNavigationRequest()) {
|
|
||||||
// request.continue()
|
|
||||||
// return
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// callbackRunOnce = true
|
|
||||||
// const overrides: Overrides = {}
|
|
||||||
//
|
|
||||||
// Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
|
|
||||||
// // @ts-ignore
|
|
||||||
// overrides[key] = overrideResolvers[key](request)
|
|
||||||
// });
|
|
||||||
//
|
|
||||||
// log.debug(`Overrides: ${JSON.stringify(overrides)}`)
|
|
||||||
// request.continue(overrides)
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// await page.setRequestInterception(true)
|
|
||||||
// page.on('request', callback)
|
|
||||||
// }
|
|
||||||
|
|
||||||
return page
|
|
||||||
}
|
|
||||||
|
|
||||||
const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall, webBrowserUserAgent: string) => {
|
|
||||||
const oneTimeSession = params.session === undefined
|
|
||||||
const sessionId = params.session || UUIDv1()
|
|
||||||
const session = oneTimeSession
|
|
||||||
? await sessions.create(sessionId, {
|
|
||||||
oneTimeSession
|
|
||||||
})
|
|
||||||
: sessions.get(sessionId)
|
|
||||||
|
|
||||||
if (session === false) {
|
|
||||||
return ctx.errorResponse('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
|
|
||||||
}
|
|
||||||
|
|
||||||
params = mergeSessionWithParams(session, params)
|
|
||||||
|
|
||||||
try {
|
|
||||||
const page = await setupPage(ctx, params, session.browser, webBrowserUserAgent)
|
|
||||||
const data = await resolveChallengeWithTimeout(ctx, params, page)
|
|
||||||
|
|
||||||
if (data) {
|
|
||||||
const { status } = data
|
|
||||||
delete data.status
|
|
||||||
ctx.successResponse(data.message, {
|
|
||||||
...(oneTimeSession ? {} : { session: sessionId }),
|
|
||||||
...(status ? { status } : {}),
|
|
||||||
solution: data.result
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
log.error(error)
|
|
||||||
return ctx.errorResponse("Unable to process browser request. Error: " + error)
|
|
||||||
} finally {
|
|
||||||
if (oneTimeSession) {
|
|
||||||
await sessions.destroy(sessionId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const routes: Routes = {
|
|
||||||
'sessions.create': async (ctx, { session, ...options }: SessionsCreateAPICall) => {
|
|
||||||
session = session || UUIDv1()
|
|
||||||
const { browser } = await sessions.create(session, options)
|
|
||||||
if (browser) { ctx.successResponse('Session created successfully.', { session }) }
|
|
||||||
},
|
|
||||||
'sessions.list': (ctx) => {
|
|
||||||
ctx.successResponse(null, { sessions: sessions.list() })
|
|
||||||
},
|
|
||||||
'sessions.destroy': async (ctx, { session }: BaseSessionsAPICall) => {
|
|
||||||
if (await sessions.destroy(session)) { return ctx.successResponse('The session has been removed.') }
|
|
||||||
ctx.errorResponse('This session does not exist.')
|
|
||||||
},
|
|
||||||
'request.get': async (ctx, params: BaseRequestAPICall, webBrowserUserAgent: string) => {
|
|
||||||
params.method = 'GET'
|
|
||||||
if (params.userAgent) {
|
|
||||||
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
|
|
||||||
}
|
|
||||||
if (params.postData) {
|
|
||||||
return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
|
|
||||||
}
|
|
||||||
await browserRequest(ctx, params, webBrowserUserAgent)
|
|
||||||
},
|
|
||||||
'request.post': async (ctx, params: BaseRequestAPICall, webBrowserUserAgent: string) => {
|
|
||||||
params.method = 'POST'
|
|
||||||
if (params.userAgent) {
|
|
||||||
log.warn('Request parameter "userAgent" was removed in FlareSolverr v2.')
|
|
||||||
}
|
|
||||||
if (!params.postData) {
|
|
||||||
return ctx.errorResponse('Must send param "postBody" when sending a POST request.')
|
|
||||||
}
|
|
||||||
await browserRequest(ctx, params, webBrowserUserAgent)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
export default async function Router(ctx: RequestContext, params: BaseAPICall, webBrowserUserAgent: string): Promise<void> {
|
|
||||||
const route = routes[params.cmd]
|
|
||||||
if (route) { return await route(ctx, params, webBrowserUserAgent) }
|
|
||||||
return ctx.errorResponse(`The command '${params.cmd}' is invalid.`)
|
|
||||||
}
|
|
49
src/server.ts
Normal file
49
src/server.ts
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
import log from './services/log'
|
||||||
|
import {testWebBrowserInstallation} from "./services/sessions";
|
||||||
|
|
||||||
|
const app = require("./app");
|
||||||
|
const version: string = require('../package.json').version
|
||||||
|
const serverPort: number = Number(process.env.PORT) || 8191
|
||||||
|
const serverHost: string = process.env.HOST || '0.0.0.0'
|
||||||
|
|
||||||
|
function validateEnvironmentVariables() {
|
||||||
|
// ip and port variables are validated by nodejs
|
||||||
|
if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
|
||||||
|
log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
|
||||||
|
log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
|
||||||
|
log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
// todo: fix resolvers
|
||||||
|
// try {
|
||||||
|
// getCaptchaSolver();
|
||||||
|
// } catch (e) {
|
||||||
|
// log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
|
||||||
|
// process.exit(1);
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init
|
||||||
|
log.info(`FlareSolverr ${version}`);
|
||||||
|
log.debug('Debug log enabled');
|
||||||
|
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
// Capture signal on Docker Stop #158
|
||||||
|
log.info("Process interrupted")
|
||||||
|
process.exit(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
validateEnvironmentVariables();
|
||||||
|
|
||||||
|
testWebBrowserInstallation().then(() => {
|
||||||
|
// Start server
|
||||||
|
app.listen(serverPort, serverHost, () => {
|
||||||
|
log.info(`FlareSolverr v${version} listening on http://${serverHost}:${serverPort}`);
|
||||||
|
})
|
||||||
|
})
|
177
src/services/sessions.ts
Normal file
177
src/services/sessions.ts
Normal file
@ -0,0 +1,177 @@
|
|||||||
|
import {v1 as UUIDv1} from 'uuid'
|
||||||
|
import * as os from 'os'
|
||||||
|
import * as path from 'path'
|
||||||
|
import * as fs from 'fs'
|
||||||
|
import {LaunchOptions, Headers, SetCookie, Browser} from 'puppeteer'
|
||||||
|
|
||||||
|
import log from './log'
|
||||||
|
import {deleteFolderRecursive, sleep, removeEmptyFields} from './utils'
|
||||||
|
|
||||||
|
const puppeteer = require('puppeteer');
|
||||||
|
|
||||||
|
interface SessionPageDefaults {
|
||||||
|
headers?: Headers
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SessionsCacheItem {
|
||||||
|
sessionId: string
|
||||||
|
browser: Browser
|
||||||
|
userDataDir?: string
|
||||||
|
defaults: SessionPageDefaults
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SessionsCache {
|
||||||
|
[key: string]: SessionsCacheItem
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SessionCreateOptions {
|
||||||
|
oneTimeSession: boolean
|
||||||
|
cookies?: SetCookie[],
|
||||||
|
headers?: Headers
|
||||||
|
maxTimeout?: number
|
||||||
|
proxy?: any// TODO: use interface not any
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionCache: SessionsCache = {}
|
||||||
|
let webBrowserUserAgent: string;
|
||||||
|
|
||||||
|
|
||||||
|
function userDataDirFromId(id: string): string {
|
||||||
|
return path.join(os.tmpdir(), `/puppeteer_profile_${id}`)
|
||||||
|
}
|
||||||
|
|
||||||
|
function prepareBrowserProfile(id: string): string {
|
||||||
|
// TODO: maybe pass SessionCreateOptions for loading later?
|
||||||
|
const userDataDir = userDataDirFromId(id)
|
||||||
|
|
||||||
|
if (!fs.existsSync(userDataDir)) {
|
||||||
|
fs.mkdirSync(userDataDir, { recursive: true })
|
||||||
|
}
|
||||||
|
|
||||||
|
return userDataDir
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getUserAgent() {
|
||||||
|
return webBrowserUserAgent
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function testWebBrowserInstallation(): Promise<void> {
|
||||||
|
log.info("Testing web browser installation...")
|
||||||
|
const session = await create(null, {
|
||||||
|
oneTimeSession: true
|
||||||
|
})
|
||||||
|
const page = await session.browser.newPage()
|
||||||
|
await page.goto("https://www.google.com")
|
||||||
|
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
|
||||||
|
|
||||||
|
// replace Linux ARM user-agent because it's detected
|
||||||
|
if (webBrowserUserAgent.toLocaleLowerCase().includes('linux arm')) {
|
||||||
|
webBrowserUserAgent = webBrowserUserAgent.replace(/linux arm[^;]+;/i, 'Linux x86_64;')
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
|
||||||
|
await page.close()
|
||||||
|
await destroy(session.sessionId)
|
||||||
|
|
||||||
|
log.info("Test successful")
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
|
||||||
|
const sessionId = session || UUIDv1()
|
||||||
|
|
||||||
|
// todo: these args are only supported in chrome
|
||||||
|
let args = [
|
||||||
|
'--no-sandbox',
|
||||||
|
'--disable-setuid-sandbox',
|
||||||
|
'--disable-dev-shm-usage' // issue #45
|
||||||
|
];
|
||||||
|
if (options.proxy && options.proxy.url) {
|
||||||
|
args.push(`--proxy-server=${options.proxy.url}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const puppeteerOptions: LaunchOptions = {
|
||||||
|
product: 'firefox',
|
||||||
|
headless: process.env.HEADLESS !== 'false',
|
||||||
|
args
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!options.oneTimeSession) {
|
||||||
|
log.debug('Creating userDataDir for session.')
|
||||||
|
puppeteerOptions.userDataDir = prepareBrowserProfile(sessionId)
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo: fix native package with firefox
|
||||||
|
// if we are running inside executable binary, change browser path
|
||||||
|
if (typeof (process as any).pkg !== 'undefined') {
|
||||||
|
const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
|
||||||
|
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug('Launching web browser...')
|
||||||
|
|
||||||
|
// TODO: maybe access env variable?
|
||||||
|
// TODO: sometimes browser instances are created and not connected to correctly.
|
||||||
|
// how do we handle/quit those instances inside Docker?
|
||||||
|
let launchTries = 3
|
||||||
|
let browser: Browser;
|
||||||
|
|
||||||
|
while (0 <= launchTries--) {
|
||||||
|
try {
|
||||||
|
browser = await puppeteer.launch(puppeteerOptions)
|
||||||
|
break
|
||||||
|
} catch (e) {
|
||||||
|
if (e.message !== 'Failed to launch the browser process!')
|
||||||
|
throw e
|
||||||
|
log.warn('Failed to open browser, trying again...')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
|
||||||
|
|
||||||
|
if (options.cookies) {
|
||||||
|
const page = await browser.newPage()
|
||||||
|
await page.setCookie(...options.cookies)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionCache[sessionId] = {
|
||||||
|
sessionId: sessionId,
|
||||||
|
browser: browser,
|
||||||
|
userDataDir: puppeteerOptions.userDataDir,
|
||||||
|
defaults: removeEmptyFields(options) // todo: review
|
||||||
|
}
|
||||||
|
|
||||||
|
return sessionCache[sessionId]
|
||||||
|
}
|
||||||
|
|
||||||
|
export function list(): string[] {
|
||||||
|
return Object.keys(sessionCache)
|
||||||
|
}
|
||||||
|
|
||||||
|
// todo: create a sessions.close that doesn't rm the userDataDir
|
||||||
|
|
||||||
|
export async function destroy(id: string): Promise<boolean>{
|
||||||
|
if (id && sessionCache.hasOwnProperty(id)) {
|
||||||
|
const { browser, userDataDir } = sessionCache[id]
|
||||||
|
if (browser) {
|
||||||
|
await browser.close()
|
||||||
|
delete sessionCache[id]
|
||||||
|
if (userDataDir) {
|
||||||
|
const userDataDirPath = userDataDirFromId(id)
|
||||||
|
try {
|
||||||
|
// for some reason this keeps an error from being thrown in Windows, figures
|
||||||
|
await sleep(5000)
|
||||||
|
deleteFolderRecursive(userDataDirPath)
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e)
|
||||||
|
throw Error(`Error deleting browser session folder. ${e.message}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
export function get(id: string): SessionsCacheItem {
|
||||||
|
return sessionCache[id]
|
||||||
|
}
|
219
src/services/solver.ts
Normal file
219
src/services/solver.ts
Normal file
@ -0,0 +1,219 @@
|
|||||||
|
import {Response, Headers, Page, Browser} from 'puppeteer'
|
||||||
|
const Timeout = require('await-timeout');
|
||||||
|
|
||||||
|
import log from './log'
|
||||||
|
import {SessionsCacheItem} from "./sessions";
|
||||||
|
import {V1Request} from "../controllers/v1";
|
||||||
|
import cloudflareProvider from '../providers/cloudflare';
|
||||||
|
|
||||||
|
const sessions = require('./sessions')
|
||||||
|
|
||||||
|
export interface ChallengeResolutionResultT {
|
||||||
|
url: string
|
||||||
|
status: number,
|
||||||
|
headers?: Headers,
|
||||||
|
response: string,
|
||||||
|
cookies: object[]
|
||||||
|
userAgent: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChallengeResolutionT {
|
||||||
|
status?: string
|
||||||
|
message: string
|
||||||
|
result: ChallengeResolutionResultT
|
||||||
|
}
|
||||||
|
|
||||||
|
// interface OverrideResolvers {
|
||||||
|
// method?: (request: Request) => HttpMethod,
|
||||||
|
// postData?: (request: Request) => string,
|
||||||
|
// headers?: (request: Request) => Headers
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// type OverridesProps =
|
||||||
|
// 'method' |
|
||||||
|
// 'postData' |
|
||||||
|
// 'headers'
|
||||||
|
|
||||||
|
async function resolveChallengeWithTimeout(params: V1Request, page: Page) {
|
||||||
|
const maxTimeout = params.maxTimeout || 60000
|
||||||
|
const timer = new Timeout();
|
||||||
|
try {
|
||||||
|
const promise = resolveChallenge(params, page);
|
||||||
|
return await Promise.race([
|
||||||
|
promise,
|
||||||
|
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
timer.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveChallenge({ url, proxy, download, returnOnlyCookies, returnRawHtml }: V1Request,
|
||||||
|
page: Page): Promise<ChallengeResolutionT | void> {
|
||||||
|
|
||||||
|
let status = 'ok'
|
||||||
|
let message = ''
|
||||||
|
|
||||||
|
if (proxy) {
|
||||||
|
log.debug("Apply proxy");
|
||||||
|
if (proxy.username)
|
||||||
|
await page.authenticate({ username: proxy.username, password: proxy.password });
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug(`Navigating to... ${url}`)
|
||||||
|
let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
||||||
|
log.html(await page.content())
|
||||||
|
|
||||||
|
// Detect protection services and solve challenges
|
||||||
|
try {
|
||||||
|
response = await cloudflareProvider(url, page, response);
|
||||||
|
} catch (e) {
|
||||||
|
status = "error";
|
||||||
|
message = "Cloudflare " + e.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload: ChallengeResolutionT = {
|
||||||
|
status,
|
||||||
|
message,
|
||||||
|
result: {
|
||||||
|
url: page.url(),
|
||||||
|
status: response.status(),
|
||||||
|
headers: response.headers(),
|
||||||
|
response: null,
|
||||||
|
cookies: await page.cookies(),
|
||||||
|
userAgent: await page.evaluate(() => navigator.userAgent)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (returnOnlyCookies) {
|
||||||
|
payload.result.headers = null;
|
||||||
|
payload.result.userAgent = null;
|
||||||
|
} else {
|
||||||
|
if (download) {
|
||||||
|
// for some reason we get an error unless we reload the page
|
||||||
|
// has something to do with a stale buffer and this is the quickest
|
||||||
|
// fix since I am short on time
|
||||||
|
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
||||||
|
payload.result.response = (await response.buffer()).toString('base64')
|
||||||
|
|
||||||
|
// todo: review this functionality
|
||||||
|
// } else if (returnRawHtml) {
|
||||||
|
// payload.result.response = await response.text()
|
||||||
|
} else {
|
||||||
|
payload.result.response = await page.content()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add final url in result
|
||||||
|
payload.result.url = page.url();
|
||||||
|
|
||||||
|
// make sure the page is closed because if it isn't and error will be thrown
|
||||||
|
// when a user uses a temporary session, the browser make be quit before
|
||||||
|
// the page is properly closed.
|
||||||
|
await page.close()
|
||||||
|
|
||||||
|
return payload
|
||||||
|
}
|
||||||
|
|
||||||
|
function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: V1Request): V1Request {
|
||||||
|
const copy = { ...defaults, ...params }
|
||||||
|
|
||||||
|
// custom merging logic
|
||||||
|
copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
|
||||||
|
|
||||||
|
return copy
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setupPage(params: V1Request, browser: Browser): Promise<Page> {
|
||||||
|
const page = await browser.newPage()
|
||||||
|
|
||||||
|
// merge session defaults with params
|
||||||
|
const { method, postData, headers, cookies } = params
|
||||||
|
|
||||||
|
// the user-agent is changed just for linux arm build
|
||||||
|
await page.setUserAgent(sessions.getUserAgent())
|
||||||
|
|
||||||
|
// todo: redo all functionality
|
||||||
|
|
||||||
|
// let overrideResolvers: OverrideResolvers = {}
|
||||||
|
//
|
||||||
|
// if (method !== 'GET') {
|
||||||
|
// log.debug(`Setting method to ${method}`)
|
||||||
|
// overrideResolvers.method = request => method
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// if (postData) {
|
||||||
|
// log.debug(`Setting body data to ${postData}`)
|
||||||
|
// overrideResolvers.postData = request => postData
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// if (headers) {
|
||||||
|
// log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
|
||||||
|
// overrideResolvers.headers = request => Object.assign(request.headers(), headers)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// if (cookies) {
|
||||||
|
// log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
|
||||||
|
// await page.setCookie(...cookies)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// // if any keys have been set on the object
|
||||||
|
// if (Object.keys(overrideResolvers).length > 0) {
|
||||||
|
// let callbackRunOnce = false
|
||||||
|
// const callback = (request: Request) => {
|
||||||
|
//
|
||||||
|
// // avoid loading resources to speed up page load
|
||||||
|
// if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
|
||||||
|
// request.abort()
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// if (callbackRunOnce || !request.isNavigationRequest()) {
|
||||||
|
// request.continue()
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// callbackRunOnce = true
|
||||||
|
// const overrides: Overrides = {}
|
||||||
|
//
|
||||||
|
// Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
|
||||||
|
// // @ts-ignore
|
||||||
|
// overrides[key] = overrideResolvers[key](request)
|
||||||
|
// });
|
||||||
|
//
|
||||||
|
// log.debug(`Overrides: ${JSON.stringify(overrides)}`)
|
||||||
|
// request.continue(overrides)
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// await page.setRequestInterception(true)
|
||||||
|
// page.on('request', callback)
|
||||||
|
// }
|
||||||
|
|
||||||
|
return page
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
|
||||||
|
const oneTimeSession = params.session === undefined;
|
||||||
|
const session: SessionsCacheItem = oneTimeSession
|
||||||
|
? await sessions.create(null, {
|
||||||
|
oneTimeSession: true
|
||||||
|
})
|
||||||
|
: sessions.get(params.session)
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
|
||||||
|
}
|
||||||
|
|
||||||
|
params = mergeSessionWithParams(session, params)
|
||||||
|
|
||||||
|
try {
|
||||||
|
const page = await setupPage(params, session.browser)
|
||||||
|
return await resolveChallengeWithTimeout(params, page)
|
||||||
|
} catch (error) {
|
||||||
|
throw Error("Unable to process browser request. Error: " + error)
|
||||||
|
} finally {
|
||||||
|
if (oneTimeSession) {
|
||||||
|
await sessions.destroy(session.sessionId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
142
src/session.ts
142
src/session.ts
@ -1,142 +0,0 @@
|
|||||||
import * as os from 'os'
|
|
||||||
import * as path from 'path'
|
|
||||||
import * as fs from 'fs'
|
|
||||||
|
|
||||||
import log from './log'
|
|
||||||
import { deleteFolderRecursive, sleep, removeEmptyFields } from './utils'
|
|
||||||
import {LaunchOptions, Headers, SetCookie, Browser} from 'puppeteer'
|
|
||||||
const puppeteer = require('puppeteer');
|
|
||||||
|
|
||||||
interface SessionPageDefaults {
|
|
||||||
headers?: Headers
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SessionsCacheItem {
|
|
||||||
browser: Browser
|
|
||||||
userDataDir?: string
|
|
||||||
defaults: SessionPageDefaults
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionsCache {
|
|
||||||
[key: string]: SessionsCacheItem
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionCreateOptions {
|
|
||||||
oneTimeSession?: boolean
|
|
||||||
cookies?: SetCookie[]
|
|
||||||
headers?: Headers,
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
const sessionCache: SessionsCache = {}
|
|
||||||
|
|
||||||
function userDataDirFromId(id: string): string {
|
|
||||||
return path.join(os.tmpdir(), `/puppeteer_profile_${id}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
function prepareBrowserProfile(id: string): string {
|
|
||||||
// TODO: maybe pass SessionCreateOptions for loading later?
|
|
||||||
const userDataDir = userDataDirFromId(id)
|
|
||||||
|
|
||||||
if (!fs.existsSync(userDataDir)) {
|
|
||||||
fs.mkdirSync(userDataDir, { recursive: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
return userDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
export default {
|
|
||||||
create: async (id: string, { cookies, oneTimeSession, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
|
|
||||||
let args = [
|
|
||||||
'--no-sandbox',
|
|
||||||
'--disable-setuid-sandbox',
|
|
||||||
'--disable-dev-shm-usage' // issue #45
|
|
||||||
];
|
|
||||||
if (proxy && proxy.url) {
|
|
||||||
args.push(`--proxy-server=${proxy.url}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const puppeteerOptions: LaunchOptions = {
|
|
||||||
product: 'firefox',
|
|
||||||
headless: process.env.HEADLESS !== 'false',
|
|
||||||
args
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!oneTimeSession) {
|
|
||||||
log.debug('Creating userDataDir for session.')
|
|
||||||
puppeteerOptions.userDataDir = prepareBrowserProfile(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
// todo: fix native package with firefox
|
|
||||||
// if we are running inside executable binary, change browser path
|
|
||||||
if (typeof (process as any).pkg !== 'undefined') {
|
|
||||||
const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
|
|
||||||
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Launching web browser...')
|
|
||||||
|
|
||||||
// TODO: maybe access env variable?
|
|
||||||
// TODO: sometimes browser instances are created and not connected to correctly.
|
|
||||||
// how do we handle/quit those instances inside Docker?
|
|
||||||
let launchTries = 3
|
|
||||||
let browser: Browser;
|
|
||||||
|
|
||||||
while (0 <= launchTries--) {
|
|
||||||
try {
|
|
||||||
browser = await puppeteer.launch(puppeteerOptions)
|
|
||||||
break
|
|
||||||
} catch (e) {
|
|
||||||
if (e.message !== 'Failed to launch the browser process!')
|
|
||||||
throw e
|
|
||||||
log.warn('Failed to open browser, trying again...')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
|
|
||||||
|
|
||||||
if (cookies) {
|
|
||||||
const page = await browser.newPage()
|
|
||||||
await page.setCookie(...cookies)
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionCache[id] = {
|
|
||||||
browser: browser,
|
|
||||||
userDataDir: puppeteerOptions.userDataDir,
|
|
||||||
defaults: removeEmptyFields({
|
|
||||||
headers,
|
|
||||||
maxTimeout
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return sessionCache[id]
|
|
||||||
},
|
|
||||||
|
|
||||||
list: (): string[] => Object.keys(sessionCache),
|
|
||||||
|
|
||||||
// TODO: create a sessions.close that doesn't rm the userDataDir
|
|
||||||
|
|
||||||
destroy: async (id: string): Promise<boolean> => {
|
|
||||||
const { browser, userDataDir } = sessionCache[id]
|
|
||||||
if (browser) {
|
|
||||||
await browser.close()
|
|
||||||
delete sessionCache[id]
|
|
||||||
if (userDataDir) {
|
|
||||||
const userDataDirPath = userDataDirFromId(id)
|
|
||||||
try {
|
|
||||||
// for some reason this keeps an error from being thrown in Windows, figures
|
|
||||||
await sleep(5000)
|
|
||||||
deleteFolderRecursive(userDataDirPath)
|
|
||||||
} catch (e) {
|
|
||||||
console.error(e)
|
|
||||||
throw Error(`Error deleting browser session folder. ${e.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
},
|
|
||||||
|
|
||||||
get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
|
|
||||||
}
|
|
290
src/tests/app.test.ts
Normal file
290
src/tests/app.test.ts
Normal file
@ -0,0 +1,290 @@
|
|||||||
|
// noinspection DuplicatedCode
|
||||||
|
|
||||||
|
import {Response} from "superagent";
|
||||||
|
import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
|
||||||
|
import {testWebBrowserInstallation} from "../services/sessions";
|
||||||
|
|
||||||
|
const request = require("supertest");
|
||||||
|
const app = require("../app");
|
||||||
|
const version: string = require('../../package.json').version
|
||||||
|
const googleUrl = "https://www.google.com";
|
||||||
|
const cfUrl = "https://pirateiro.com/torrents/?search=s";
|
||||||
|
|
||||||
|
describe("Test '/' path", () => {
|
||||||
|
test("GET method should return OK ", async () => {
|
||||||
|
// Init session
|
||||||
|
await testWebBrowserInstallation();
|
||||||
|
|
||||||
|
const response: Response = await request(app).get("/");
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.body.msg).toBe("FlareSolverr is ready!");
|
||||||
|
expect(response.body.version).toBe(version);
|
||||||
|
expect(response.body.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("POST method should fail", async () => {
|
||||||
|
const response: Response = await request(app).post("/");
|
||||||
|
expect(response.statusCode).toBe(404);
|
||||||
|
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/health' path", () => {
|
||||||
|
test("GET method should return OK", async () => {
|
||||||
|
const response: Response = await request(app).get("/health");
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.body.status).toBe("ok");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/wrong' path", () => {
|
||||||
|
test("GET method should fail", async () => {
|
||||||
|
const response: Response = await request(app).get("/wrong");
|
||||||
|
expect(response.statusCode).toBe(404);
|
||||||
|
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/v1' path", () => {
|
||||||
|
jest.setTimeout(70000);
|
||||||
|
|
||||||
|
test("Cmd 'request.bad' should fail", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.bad",
|
||||||
|
"url": googleUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.response).toContain("<!DOCTYPE html>")
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with Cloudflare", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": cfUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(cfUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.response).toContain("<!DOCTYPE html>")
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
|
||||||
|
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie.length).toBeGreaterThan(30)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return timeout", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"maxTimeout": 10
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Error: Maximum timeout reached. maxTimeout=10 (ms)");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should accept deprecated params", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"userAgent": "Test User-Agent" // was removed in v2, not used
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.create' should return OK", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSession = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("Session created successfully.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.session.length).toBe(36);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.create' should return OK with session", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.create",
|
||||||
|
"session": "2bc6bb20-2f56-11ec-9543-test"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSession = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("Session created successfully.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.list' should return OK", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.list"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSessions = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.sessions.length).toBeGreaterThan(0)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.destroy' should return OK", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
const apiResponse0: V1ResponseSession = response0.body;
|
||||||
|
const sessionId0 = apiResponse0.session
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.destroy",
|
||||||
|
"session": sessionId0
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("The session has been removed.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.destroy' should fail", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.destroy",
|
||||||
|
"session": "bad-session"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: This session does not exist.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should use session", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
const apiResponse0: V1ResponseSession = response0.body;
|
||||||
|
const sessionId0 = apiResponse0.session
|
||||||
|
|
||||||
|
// first request should solve the challenge
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": cfUrl,
|
||||||
|
"session": sessionId0
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie.length).toBeGreaterThan(30)
|
||||||
|
|
||||||
|
// second request should have the same cookie
|
||||||
|
const response2: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response2.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse2: V1ResponseSolution = response2.body;
|
||||||
|
expect(apiResponse2.status).toBe("ok");
|
||||||
|
const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie2.length).toBeGreaterThan(30)
|
||||||
|
expect(cfCookie2).toBe(cfCookie)
|
||||||
|
});
|
||||||
|
});
|
@ -1,9 +0,0 @@
|
|||||||
import { IncomingMessage, ServerResponse } from 'http';
|
|
||||||
|
|
||||||
export interface RequestContext {
|
|
||||||
req: IncomingMessage
|
|
||||||
res: ServerResponse
|
|
||||||
startTimestamp: number
|
|
||||||
errorResponse: (msg: string) => void,
|
|
||||||
successResponse: (msg: string, extendedProperties?: object) => void
|
|
||||||
}
|
|
Loading…
x
Reference in New Issue
Block a user