mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-06-09 21:07:14 +00:00
Prepare for version 3.0, remove JS code
This commit is contained in:
ngosang
parent
345628e3e4
commit
383025032b
@ -1,7 +0,0 @@
|
|||||||
.git/
|
|
||||||
.github/
|
|
||||||
.idea/
|
|
||||||
bin/
|
|
||||||
dist/
|
|
||||||
node_modules/
|
|
||||||
resources/
|
|
||||||
15
.eslintrc.js
15
.eslintrc.js
@ -1,15 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
env: {
|
|
||||||
browser: true,
|
|
||||||
commonjs: true,
|
|
||||||
es2020: true
|
|
||||||
},
|
|
||||||
extends: [
|
|
||||||
'standard'
|
|
||||||
],
|
|
||||||
parserOptions: {
|
|
||||||
ecmaVersion: 11
|
|
||||||
},
|
|
||||||
rules: {
|
|
||||||
}
|
|
||||||
}
|
|
||||||
29
Dockerfile
29
Dockerfile
@ -1,29 +0,0 @@
|
|||||||
FROM node:16-alpine3.15
|
|
||||||
|
|
||||||
# Install the web browser (package firefox-esr is available too)
|
|
||||||
RUN apk update && \
|
|
||||||
apk add --no-cache firefox dumb-init && \
|
|
||||||
rm -Rf /var/cache
|
|
||||||
|
|
||||||
# Copy FlareSolverr code
|
|
||||||
USER node
|
|
||||||
RUN mkdir -p /home/node/flaresolverr
|
|
||||||
WORKDIR /home/node/flaresolverr
|
|
||||||
COPY --chown=node:node package.json package-lock.json tsconfig.json install.js ./
|
|
||||||
COPY --chown=node:node src ./src/
|
|
||||||
|
|
||||||
# Install package. Skip installing the browser, we will use the installed package.
|
|
||||||
ENV PUPPETEER_PRODUCT=firefox \
|
|
||||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
|
||||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
|
|
||||||
RUN npm install && \
|
|
||||||
npm run build && \
|
|
||||||
npm prune --production && \
|
|
||||||
rm -rf /home/node/.npm
|
|
||||||
|
|
||||||
EXPOSE 8191
|
|
||||||
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
|
||||||
CMD ["node", "./dist/server.js"]
|
|
||||||
|
|
||||||
# docker build -t flaresolverr:custom .
|
|
||||||
# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom
|
|
||||||
@ -1,90 +0,0 @@
|
|||||||
const fs = require('fs')
|
|
||||||
const path = require('path')
|
|
||||||
const { execSync } = require('child_process')
|
|
||||||
const archiver = require('archiver')
|
|
||||||
const https = require('https')
|
|
||||||
const puppeteer = require('puppeteer')
|
|
||||||
const version = 'v' + require('./package.json').version;
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
const builds = [
|
|
||||||
{
|
|
||||||
platform: 'linux',
|
|
||||||
firefoxFolder: 'firefox',
|
|
||||||
fsExec: 'flaresolverr-linux',
|
|
||||||
fsZipExec: 'flaresolverr',
|
|
||||||
fsZipName: 'linux-x64',
|
|
||||||
fsLicenseName: 'LICENSE'
|
|
||||||
},
|
|
||||||
{
|
|
||||||
platform: 'win64',
|
|
||||||
firefoxFolder: 'firefox',
|
|
||||||
fsExec: 'flaresolverr-win.exe',
|
|
||||||
fsZipExec: 'flaresolverr.exe',
|
|
||||||
fsZipName: 'windows-x64',
|
|
||||||
fsLicenseName: 'LICENSE.txt'
|
|
||||||
}
|
|
||||||
// todo: this has to be build in macOS (hdiutil is required). changes required in sessions.ts too
|
|
||||||
// {
|
|
||||||
// platform: 'mac',
|
|
||||||
// firefoxFolder: 'firefox',
|
|
||||||
// fsExec: 'flaresolverr-macos',
|
|
||||||
// fsZipExec: 'flaresolverr',
|
|
||||||
// fsZipName: 'macos',
|
|
||||||
// fsLicenseName: 'LICENSE'
|
|
||||||
// }
|
|
||||||
]
|
|
||||||
|
|
||||||
// generate executables
|
|
||||||
console.log('Generating executables...')
|
|
||||||
if (fs.existsSync('bin')) {
|
|
||||||
fs.rmSync('bin', { recursive: true })
|
|
||||||
}
|
|
||||||
execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
|
|
||||||
// execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
|
|
||||||
|
|
||||||
// Puppeteer does not allow to download Firefox revisions, just the last Nightly
|
|
||||||
// We this script we can download any version
|
|
||||||
const revision = '94.0a1';
|
|
||||||
const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
|
|
||||||
|
|
||||||
// download firefox and zip together
|
|
||||||
for (const os of builds) {
|
|
||||||
console.log('Building ' + os.fsZipName + ' artifact')
|
|
||||||
|
|
||||||
// download firefox
|
|
||||||
console.log(`Downloading firefox ${revision} for ${os.platform} ...`)
|
|
||||||
const f = puppeteer.createBrowserFetcher({
|
|
||||||
product: 'firefox',
|
|
||||||
platform: os.platform,
|
|
||||||
host: downloadHost,
|
|
||||||
path: path.join(__dirname, 'bin', 'puppeteer')
|
|
||||||
})
|
|
||||||
await f.download(revision)
|
|
||||||
|
|
||||||
// compress in zip
|
|
||||||
console.log('Compressing zip file...')
|
|
||||||
const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
|
|
||||||
const output = fs.createWriteStream(zipName)
|
|
||||||
const archive = archiver('zip')
|
|
||||||
|
|
||||||
output.on('close', function () {
|
|
||||||
console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
|
|
||||||
})
|
|
||||||
|
|
||||||
archive.on('error', function (err) {
|
|
||||||
throw err
|
|
||||||
})
|
|
||||||
|
|
||||||
archive.pipe(output)
|
|
||||||
|
|
||||||
archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
|
|
||||||
archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
|
|
||||||
archive.directory('bin/puppeteer/' + os.platform + '-' + revision + '/' + os.firefoxFolder, 'flaresolverr/firefox')
|
|
||||||
if (os.platform === 'linux') {
|
|
||||||
archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
|
|
||||||
}
|
|
||||||
|
|
||||||
await archive.finalize()
|
|
||||||
}
|
|
||||||
})()
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
---
|
|
||||||
version: "2.1"
|
|
||||||
services:
|
|
||||||
flaresolverr:
|
|
||||||
# DockerHub mirror flaresolverr/flaresolverr:latest
|
|
||||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
|
||||||
container_name: flaresolverr
|
|
||||||
environment:
|
|
||||||
- LOG_LEVEL=${LOG_LEVEL:-info}
|
|
||||||
- LOG_HTML=${LOG_HTML:-false}
|
|
||||||
- CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
|
|
||||||
- TZ=Europe/London
|
|
||||||
ports:
|
|
||||||
- "${PORT:-8191}:8191"
|
|
||||||
restart: unless-stopped
|
|
||||||
@ -1,19 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=FlareSolverr
|
|
||||||
After=network.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
SyslogIdentifier=flaresolverr
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
Type=simple
|
|
||||||
User=flaresolverr
|
|
||||||
Group=flaresolverr
|
|
||||||
Environment="LOG_LEVEL=info"
|
|
||||||
Environment="CAPTCHA_SOLVER=none"
|
|
||||||
WorkingDirectory=/opt/flaresolverr
|
|
||||||
ExecStart=/opt/flaresolverr/flaresolverr
|
|
||||||
TimeoutStopSec=30
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
40
install.js
40
install.js
@ -1,40 +0,0 @@
|
|||||||
const fs = require('fs');
|
|
||||||
const puppeteer = require('puppeteer');
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
|
|
||||||
// Puppeteer does not allow to download Firefox revisions, just the last Nightly
|
|
||||||
// We this script we can download any version
|
|
||||||
const revision = '94.0a1';
|
|
||||||
const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
|
|
||||||
|
|
||||||
// skip installation (for Dockerfile)
|
|
||||||
if (process.env.PUPPETEER_EXECUTABLE_PATH) {
|
|
||||||
console.log('Skipping Firefox installation because the environment variable "PUPPETEER_EXECUTABLE_PATH" is set.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// check if Firefox is already installed
|
|
||||||
const f = puppeteer.createBrowserFetcher({
|
|
||||||
product: 'firefox',
|
|
||||||
host: downloadHost
|
|
||||||
})
|
|
||||||
if (fs.existsSync(f._getFolderPath(revision))) {
|
|
||||||
console.log(`Firefox ${revision} already installed...`)
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Installing firefox ${revision} ...`)
|
|
||||||
const downloadPath = f._downloadsFolder;
|
|
||||||
console.log(`Download path: ${downloadPath}`)
|
|
||||||
if (fs.existsSync(downloadPath)) {
|
|
||||||
console.log(`Removing previous downloads...`)
|
|
||||||
fs.rmSync(downloadPath, { recursive: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
console.log(`Downloading firefox ${revision} ...`)
|
|
||||||
await f.download(revision)
|
|
||||||
|
|
||||||
console.log('Installation complete...')
|
|
||||||
|
|
||||||
})()
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
module.exports = {
|
|
||||||
// A list of paths to directories that Jest should use to search for files in
|
|
||||||
roots: [
|
|
||||||
"./src/"
|
|
||||||
],
|
|
||||||
// Compile Typescript
|
|
||||||
transform: {
|
|
||||||
'^.+\\.(ts|tsx)$': 'ts-jest'
|
|
||||||
},
|
|
||||||
// Default value for FlareSolverr maxTimeout is 60000
|
|
||||||
testTimeout: 70000
|
|
||||||
}
|
|
||||||
11811
package-lock.json
generated
11811
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
46
package.json
46
package.json
@ -1,46 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "flaresolverr",
|
|
||||||
"version": "2.2.10",
|
|
||||||
"description": "Proxy server to bypass Cloudflare protection.",
|
|
||||||
"scripts": {
|
|
||||||
"install": "node install.js",
|
|
||||||
"start": "tsc && node ./dist/server.js",
|
|
||||||
"build": "tsc",
|
|
||||||
"dev": "nodemon -e ts --exec ts-node src/server.ts",
|
|
||||||
"package": "tsc && node build-binaries.js",
|
|
||||||
"test": "jest --runInBand"
|
|
||||||
},
|
|
||||||
"author": "Diego Heras (ngosang)",
|
|
||||||
"license": "MIT",
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "https://github.com/ngosang/FlareSolverr"
|
|
||||||
},
|
|
||||||
"bin": {
|
|
||||||
"flaresolverr": "dist/server.js"
|
|
||||||
},
|
|
||||||
"dependencies": {
|
|
||||||
"await-timeout": "^1.1.1",
|
|
||||||
"body-parser": "^1.20.0",
|
|
||||||
"console-log-level": "^1.4.1",
|
|
||||||
"express": "^4.18.1",
|
|
||||||
"puppeteer": "^13.7.0",
|
|
||||||
"uuid": "^8.3.2"
|
|
||||||
},
|
|
||||||
"devDependencies": {
|
|
||||||
"@types/await-timeout": "^0.3.1",
|
|
||||||
"@types/body-parser": "^1.19.2",
|
|
||||||
"@types/express": "^4.17.13",
|
|
||||||
"@types/jest": "^28.1.6",
|
|
||||||
"@types/node": "^18.6.2",
|
|
||||||
"@types/supertest": "^2.0.12",
|
|
||||||
"@types/uuid": "^8.3.4",
|
|
||||||
"archiver": "^5.3.1",
|
|
||||||
"nodemon": "^2.0.19",
|
|
||||||
"pkg": "^5.8.0",
|
|
||||||
"supertest": "^6.2.4",
|
|
||||||
"ts-jest": "^28.0.7",
|
|
||||||
"ts-node": "^10.9.1",
|
|
||||||
"typescript": "^4.7.4"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
83
src/app.ts
83
src/app.ts
@ -1,83 +0,0 @@
|
|||||||
import log from './services/log'
|
|
||||||
import {NextFunction, Request, Response} from 'express';
|
|
||||||
import {getUserAgent} from "./services/sessions";
|
|
||||||
import {controllerV1} from "./controllers/v1";
|
|
||||||
|
|
||||||
const express = require('express');
|
|
||||||
const app = express();
|
|
||||||
const bodyParser = require('body-parser');
|
|
||||||
const version: string = 'v' + require('../package.json').version
|
|
||||||
|
|
||||||
// Convert request objects to JSON
|
|
||||||
app.use(bodyParser.json({
|
|
||||||
limit: '50mb',
|
|
||||||
verify(req: Request, res: Response, buf: any) {
|
|
||||||
req.body = buf;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Access log
|
|
||||||
app.use(function(req: Request, res: Response, next: NextFunction) {
|
|
||||||
if (req.url != '/health') {
|
|
||||||
// count the request for the log prefix
|
|
||||||
log.incRequests()
|
|
||||||
// build access message
|
|
||||||
let body = "";
|
|
||||||
if (req.method == 'POST' && req.body) {
|
|
||||||
body += " body: "
|
|
||||||
try {
|
|
||||||
body += JSON.stringify(req.body)
|
|
||||||
} catch(e) {
|
|
||||||
body += req.body
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.info(`Incoming request => ${req.method} ${req.url}${body}`);
|
|
||||||
}
|
|
||||||
next();
|
|
||||||
});
|
|
||||||
|
|
||||||
// *********************************************************************************************************************
|
|
||||||
// Routes
|
|
||||||
|
|
||||||
// Show welcome message
|
|
||||||
app.get("/", ( req: Request, res: Response ) => {
|
|
||||||
res.send({
|
|
||||||
"msg": "FlareSolverr is ready!",
|
|
||||||
"version": version,
|
|
||||||
"userAgent": getUserAgent()
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Health endpoint. this endpoint is special because it doesn't print traces
|
|
||||||
app.get("/health", ( req: Request, res: Response ) => {
|
|
||||||
res.send({
|
|
||||||
"status": "ok"
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
// Controller v1
|
|
||||||
app.post("/v1", async( req: Request, res: Response ) => {
|
|
||||||
await controllerV1(req, res);
|
|
||||||
});
|
|
||||||
|
|
||||||
// *********************************************************************************************************************
|
|
||||||
|
|
||||||
// Unknown paths or verbs
|
|
||||||
app.use(function (req : Request, res : Response) {
|
|
||||||
res.status(404)
|
|
||||||
.send({"error": "Unknown resource or HTTP verb"})
|
|
||||||
})
|
|
||||||
|
|
||||||
// Errors
|
|
||||||
app.use(function (err: any, req: Request, res: Response, next: NextFunction) {
|
|
||||||
if (err) {
|
|
||||||
let msg = 'Invalid request: ' + err;
|
|
||||||
msg = msg.replace("\n", "").replace("\r", "")
|
|
||||||
log.error(msg)
|
|
||||||
res.send({"error": msg})
|
|
||||||
} else {
|
|
||||||
next()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
module.exports = app;
|
|
||||||
@ -1,41 +0,0 @@
|
|||||||
import log from "../services/log";
|
|
||||||
|
|
||||||
export enum CaptchaType {
|
|
||||||
re = 'reCaptcha',
|
|
||||||
h = 'hCaptcha'
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SolverOptions {
|
|
||||||
url: string
|
|
||||||
sitekey: string
|
|
||||||
type: CaptchaType
|
|
||||||
}
|
|
||||||
|
|
||||||
export type Solver = (options: SolverOptions) => Promise<string>
|
|
||||||
|
|
||||||
const captchaSolvers: { [key: string]: Solver } = {}
|
|
||||||
|
|
||||||
export default (): Solver => {
|
|
||||||
const method = process.env.CAPTCHA_SOLVER
|
|
||||||
|
|
||||||
if (!method || method.toLowerCase() == 'none') {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!(method in captchaSolvers)) {
|
|
||||||
try {
|
|
||||||
captchaSolvers[method] = require('./' + method).default as Solver
|
|
||||||
} catch (e) {
|
|
||||||
if (e.code === 'MODULE_NOT_FOUND') {
|
|
||||||
throw Error(`The solver '${method}' is not a valid captcha solving method.`)
|
|
||||||
} else {
|
|
||||||
console.error(e)
|
|
||||||
throw Error(`An error occurred loading the solver '${method}'.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info(`Using '${method}' to solve the captcha.`);
|
|
||||||
|
|
||||||
return captchaSolvers[method]
|
|
||||||
}
|
|
||||||
@ -1,178 +0,0 @@
|
|||||||
import {Request, Response} from 'express';
|
|
||||||
import {Protocol} from "devtools-protocol";
|
|
||||||
|
|
||||||
import log from '../services/log'
|
|
||||||
import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
|
|
||||||
import {SessionCreateOptions} from "../services/sessions";
|
|
||||||
const sessions = require('../services/sessions')
|
|
||||||
const version: string = 'v' + require('../../package.json').version
|
|
||||||
|
|
||||||
interface V1Routes {
|
|
||||||
[key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface Proxy {
|
|
||||||
url?: string
|
|
||||||
username?: string
|
|
||||||
password?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1RequestBase {
|
|
||||||
cmd: string
|
|
||||||
cookies?: Protocol.Network.CookieParam[],
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: Proxy
|
|
||||||
session: string
|
|
||||||
headers?: Record<string, string> // deprecated v2, not used
|
|
||||||
userAgent?: string // deprecated v2, not used
|
|
||||||
}
|
|
||||||
|
|
||||||
interface V1RequestSession extends V1RequestBase {
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1Request extends V1RequestBase {
|
|
||||||
url: string
|
|
||||||
method?: string
|
|
||||||
postData?: string
|
|
||||||
returnOnlyCookies?: boolean
|
|
||||||
download?: boolean // deprecated v2, not used
|
|
||||||
returnRawHtml?: boolean // deprecated v2, not used
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1ResponseBase {
|
|
||||||
status: string
|
|
||||||
message: string
|
|
||||||
startTimestamp: number
|
|
||||||
endTimestamp: number
|
|
||||||
version: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1ResponseSolution extends V1ResponseBase {
|
|
||||||
solution: ChallengeResolutionResultT
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1ResponseSession extends V1ResponseBase {
|
|
||||||
session: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface V1ResponseSessions extends V1ResponseBase {
|
|
||||||
sessions: string[]
|
|
||||||
}
|
|
||||||
|
|
||||||
export const routes: V1Routes = {
|
|
||||||
'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
|
|
||||||
const options: SessionCreateOptions = {
|
|
||||||
oneTimeSession: false,
|
|
||||||
cookies: params.cookies,
|
|
||||||
maxTimeout: params.maxTimeout,
|
|
||||||
proxy: params.proxy
|
|
||||||
}
|
|
||||||
const { sessionId, browser } = await sessions.create(params.session, options)
|
|
||||||
if (browser) {
|
|
||||||
response.status = "ok";
|
|
||||||
response.message = "Session created successfully.";
|
|
||||||
response.session = sessionId
|
|
||||||
} else {
|
|
||||||
throw Error('Error creating session.')
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
|
|
||||||
response.status = "ok";
|
|
||||||
response.message = "";
|
|
||||||
response.sessions = sessions.list();
|
|
||||||
},
|
|
||||||
'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
|
|
||||||
if (await sessions.destroy(params.session)) {
|
|
||||||
response.status = "ok";
|
|
||||||
response.message = "The session has been removed.";
|
|
||||||
} else {
|
|
||||||
throw Error('This session does not exist.')
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
|
||||||
params.method = 'GET'
|
|
||||||
if (params.postData) {
|
|
||||||
throw Error('Cannot use "postBody" when sending a GET request.')
|
|
||||||
}
|
|
||||||
if (params.returnRawHtml) {
|
|
||||||
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
if (params.download) {
|
|
||||||
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
const result: ChallengeResolutionT = await browserRequest(params)
|
|
||||||
|
|
||||||
response.status = result.status;
|
|
||||||
response.message = result.message;
|
|
||||||
response.solution = result.result;
|
|
||||||
if (response.message) {
|
|
||||||
log.info(response.message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
|
||||||
params.method = 'POST'
|
|
||||||
if (!params.postData) {
|
|
||||||
throw Error('Must send param "postBody" when sending a POST request.')
|
|
||||||
}
|
|
||||||
if (params.returnRawHtml) {
|
|
||||||
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
if (params.download) {
|
|
||||||
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
const result: ChallengeResolutionT = await browserRequest(params)
|
|
||||||
|
|
||||||
response.status = result.status;
|
|
||||||
response.message = result.message;
|
|
||||||
response.solution = result.result;
|
|
||||||
if (response.message) {
|
|
||||||
log.info(response.message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function controllerV1(req: Request, res: Response): Promise<void> {
|
|
||||||
const response: V1ResponseBase = {
|
|
||||||
status: null,
|
|
||||||
message: null,
|
|
||||||
startTimestamp: Date.now(),
|
|
||||||
endTimestamp: 0,
|
|
||||||
version: version
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const params: V1RequestBase = req.body
|
|
||||||
// do some validations
|
|
||||||
if (!params.cmd) {
|
|
||||||
throw Error("Request parameter 'cmd' is mandatory.")
|
|
||||||
}
|
|
||||||
if (params.headers) {
|
|
||||||
log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
if (params.userAgent) {
|
|
||||||
log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
|
|
||||||
}
|
|
||||||
|
|
||||||
// set default values
|
|
||||||
if (!params.maxTimeout || params.maxTimeout < 1) {
|
|
||||||
params.maxTimeout = 60000;
|
|
||||||
}
|
|
||||||
|
|
||||||
// execute the command
|
|
||||||
const route = routes[params.cmd]
|
|
||||||
if (route) {
|
|
||||||
await route(params, response)
|
|
||||||
} else {
|
|
||||||
throw Error(`The command '${params.cmd}' is invalid.`)
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
res.status(500)
|
|
||||||
response.status = "error";
|
|
||||||
response.message = e.toString();
|
|
||||||
log.error(response.message)
|
|
||||||
}
|
|
||||||
|
|
||||||
response.endTimestamp = Date.now()
|
|
||||||
log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
|
|
||||||
res.send(response)
|
|
||||||
}
|
|
||||||
@ -1,152 +0,0 @@
|
|||||||
import {Page, HTTPResponse} from 'puppeteer'
|
|
||||||
|
|
||||||
import log from "../services/log";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* This class contains the logic to solve protections provided by CloudFlare
|
|
||||||
**/
|
|
||||||
|
|
||||||
const BAN_SELECTORS: string[] = [
|
|
||||||
'div.main-wrapper div.header.section h1 span.code-label span' // CloudFlare
|
|
||||||
];
|
|
||||||
const CHALLENGE_SELECTORS: string[] = [
|
|
||||||
// todo: deprecate '#trk_jschal_js', '#cf-please-wait'
|
|
||||||
'#cf-challenge-running', '#trk_jschal_js', '#cf-please-wait', // CloudFlare
|
|
||||||
'#link-ddg', // DDoS-GUARD
|
|
||||||
'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
|
|
||||||
];
|
|
||||||
const CHALLENGE_TITLES: string[] = [
|
|
||||||
'DDOS-GUARD'
|
|
||||||
];
|
|
||||||
const CAPTCHA_SELECTORS: string[] = [
|
|
||||||
// todo: deprecate 'input[name="cf_captcha_kind"]'
|
|
||||||
'#cf-challenge-hcaptcha-wrapper', '#cf-norobot-container', 'input[name="cf_captcha_kind"]'
|
|
||||||
];
|
|
||||||
|
|
||||||
export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
|
|
||||||
|
|
||||||
// look for challenge and return fast if not detected
|
|
||||||
let cfDetected = response.headers().server &&
|
|
||||||
(response.headers().server.startsWith('cloudflare') || response.headers().server.startsWith('ddos-guard'));
|
|
||||||
if (cfDetected) {
|
|
||||||
if (response.status() == 403 || response.status() == 503) {
|
|
||||||
cfDetected = true; // Defected CloudFlare and DDoS-GUARD
|
|
||||||
} else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
|
|
||||||
response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
|
|
||||||
cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
|
|
||||||
} else {
|
|
||||||
cfDetected = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cfDetected) {
|
|
||||||
log.info('Cloudflare detected');
|
|
||||||
} else {
|
|
||||||
log.info('Cloudflare not detected');
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (await findAnySelector(page, BAN_SELECTORS)) {
|
|
||||||
throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
|
|
||||||
}
|
|
||||||
|
|
||||||
// find Cloudflare selectors
|
|
||||||
let selectorFound = false;
|
|
||||||
let selector: string = await findChallenge(page)
|
|
||||||
if (selector) {
|
|
||||||
selectorFound = true;
|
|
||||||
log.debug(`Javascript challenge element '${selector}' detected.`)
|
|
||||||
log.debug('Waiting for Cloudflare challenge...')
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
try {
|
|
||||||
|
|
||||||
selector = await findChallenge(page)
|
|
||||||
if (!selector) {
|
|
||||||
// solved!
|
|
||||||
log.debug('Challenge element not found')
|
|
||||||
break
|
|
||||||
|
|
||||||
} else {
|
|
||||||
log.debug(`Javascript challenge element '${selector}' detected.`)
|
|
||||||
|
|
||||||
// check for CAPTCHA challenge
|
|
||||||
if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
|
|
||||||
// captcha detected
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.debug('Found challenge element again')
|
|
||||||
|
|
||||||
} catch (error)
|
|
||||||
{
|
|
||||||
log.debug("Unexpected error: " + error);
|
|
||||||
if (!error.toString().includes("Execution context was destroyed")) {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Waiting for Cloudflare challenge...')
|
|
||||||
await page.waitForTimeout(1000)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Validating HTML code...')
|
|
||||||
} else {
|
|
||||||
log.debug(`No challenge element detected.`)
|
|
||||||
}
|
|
||||||
|
|
||||||
// check for CAPTCHA challenge
|
|
||||||
if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
|
|
||||||
log.info('CAPTCHA challenge detected');
|
|
||||||
throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
|
|
||||||
|
|
||||||
// const captchaSolver = getCaptchaSolver()
|
|
||||||
// if (captchaSolver) {
|
|
||||||
// // to-do: get the params
|
|
||||||
// log.info('Waiting to receive captcha token to bypass challenge...')
|
|
||||||
// const token = await captchaSolver({
|
|
||||||
// url,
|
|
||||||
// sitekey,
|
|
||||||
// type: captchaType
|
|
||||||
// })
|
|
||||||
// log.debug(`Token received: ${token}`);
|
|
||||||
// // to-do: send the token
|
|
||||||
// }
|
|
||||||
// } else {
|
|
||||||
// throw new Error('Captcha detected but no automatic solver is configured.');
|
|
||||||
// }
|
|
||||||
} else {
|
|
||||||
if (!selectorFound)
|
|
||||||
{
|
|
||||||
throw new Error('No challenge selectors found, unable to proceed.')
|
|
||||||
} else {
|
|
||||||
log.info('Challenge solved');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function findChallenge(page: Page): Promise<string|null> {
|
|
||||||
const selector = await findAnySelector(page, CHALLENGE_SELECTORS);
|
|
||||||
|
|
||||||
if (selector == null) {
|
|
||||||
const title = await page.title();
|
|
||||||
|
|
||||||
if (CHALLENGE_TITLES.includes(title)) {
|
|
||||||
return `:title[${title}]`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return selector;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function findAnySelector(page: Page, selectors: string[]) {
|
|
||||||
for (const selector of selectors) {
|
|
||||||
const cfChallengeElem = await page.$(selector)
|
|
||||||
if (cfChallengeElem) {
|
|
||||||
return selector;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
@ -1,63 +0,0 @@
|
|||||||
import log from './services/log'
|
|
||||||
import {testWebBrowserInstallation} from "./services/sessions";
|
|
||||||
|
|
||||||
const app = require("./app");
|
|
||||||
const version: string = 'v' + require('../package.json').version
|
|
||||||
const serverPort: number = Number(process.env.PORT) || 8191
|
|
||||||
const serverHost: string = process.env.HOST || '0.0.0.0'
|
|
||||||
|
|
||||||
function validateEnvironmentVariables() {
|
|
||||||
// ip and port variables are validated by nodejs
|
|
||||||
if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
|
|
||||||
log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
|
|
||||||
log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
|
|
||||||
log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
// todo: fix resolvers
|
|
||||||
// try {
|
|
||||||
// getCaptchaSolver();
|
|
||||||
// } catch (e) {
|
|
||||||
// log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
|
|
||||||
// process.exit(1);
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Init
|
|
||||||
log.info(`FlareSolverr ${version}`);
|
|
||||||
log.debug('Debug log enabled');
|
|
||||||
|
|
||||||
process.on('SIGTERM', () => {
|
|
||||||
// Capture signal on Docker Stop #158
|
|
||||||
log.info("Process interrupted")
|
|
||||||
process.exit(0)
|
|
||||||
})
|
|
||||||
|
|
||||||
process.on('uncaughtException', function(err) {
|
|
||||||
// Avoid crashing in NodeJS 17 due to UnhandledPromiseRejectionWarning: Unhandled promise rejection.
|
|
||||||
log.error(err)
|
|
||||||
})
|
|
||||||
|
|
||||||
validateEnvironmentVariables();
|
|
||||||
|
|
||||||
testWebBrowserInstallation().then(() => {
|
|
||||||
// Start server
|
|
||||||
app.listen(serverPort, serverHost, () => {
|
|
||||||
log.info(`Listening on http://${serverHost}:${serverPort}`);
|
|
||||||
})
|
|
||||||
}).catch(function(e) {
|
|
||||||
log.error(e);
|
|
||||||
const msg: string = "" + e;
|
|
||||||
if (msg.includes('while trying to connect to the browser!')) {
|
|
||||||
log.error(`It seems that the system is too slow to run FlareSolverr.
|
|
||||||
If you are running with Docker, try to remove CPU limits in the container.
|
|
||||||
If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
|
|
||||||
}
|
|
||||||
process.exit(1);
|
|
||||||
})
|
|
||||||
@ -1,41 +0,0 @@
|
|||||||
let requests = 0
|
|
||||||
|
|
||||||
const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
|
|
||||||
|
|
||||||
function toIsoString(date: Date) {
|
|
||||||
// this function fixes Date.toISOString() adding timezone
|
|
||||||
let tzo = -date.getTimezoneOffset(),
|
|
||||||
dif = tzo >= 0 ? '+' : '-',
|
|
||||||
pad = function(num: number) {
|
|
||||||
let norm = Math.floor(Math.abs(num));
|
|
||||||
return (norm < 10 ? '0' : '') + norm;
|
|
||||||
};
|
|
||||||
|
|
||||||
return date.getFullYear() +
|
|
||||||
'-' + pad(date.getMonth() + 1) +
|
|
||||||
'-' + pad(date.getDate()) +
|
|
||||||
'T' + pad(date.getHours()) +
|
|
||||||
':' + pad(date.getMinutes()) +
|
|
||||||
':' + pad(date.getSeconds()) +
|
|
||||||
dif + pad(tzo / 60) +
|
|
||||||
':' + pad(tzo % 60);
|
|
||||||
}
|
|
||||||
|
|
||||||
export default {
|
|
||||||
incRequests: () => {
|
|
||||||
requests++
|
|
||||||
},
|
|
||||||
html(html: string) {
|
|
||||||
if (LOG_HTML) {
|
|
||||||
this.debug(html)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
...require('console-log-level')(
|
|
||||||
{level: process.env.LOG_LEVEL || 'info',
|
|
||||||
prefix(level: string) {
|
|
||||||
const req = (requests > 0) ? ` REQ-${requests}` : '';
|
|
||||||
return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
|
||||||
@ -1,194 +0,0 @@
|
|||||||
import {v1 as UUIDv1} from 'uuid'
|
|
||||||
import * as path from 'path'
|
|
||||||
import {Browser} from 'puppeteer'
|
|
||||||
import {Protocol} from "devtools-protocol";
|
|
||||||
|
|
||||||
import log from './log'
|
|
||||||
import {Proxy} from "../controllers/v1";
|
|
||||||
|
|
||||||
const os = require('os');
|
|
||||||
const fs = require('fs');
|
|
||||||
const puppeteer = require('puppeteer');
|
|
||||||
|
|
||||||
export interface SessionsCacheItem {
|
|
||||||
sessionId: string
|
|
||||||
browser: Browser
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionsCache {
|
|
||||||
[key: string]: SessionsCacheItem
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SessionCreateOptions {
|
|
||||||
oneTimeSession: boolean
|
|
||||||
cookies?: Protocol.Network.CookieParam[],
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: Proxy
|
|
||||||
}
|
|
||||||
|
|
||||||
const sessionCache: SessionsCache = {}
|
|
||||||
let webBrowserUserAgent: string;
|
|
||||||
|
|
||||||
function buildExtraPrefsFirefox(proxy: Proxy): object {
|
|
||||||
// Default configurations are defined here
|
|
||||||
// https://github.com/puppeteer/puppeteer/blob/v3.3.0/src/Launcher.ts#L481
|
|
||||||
const extraPrefsFirefox = {
|
|
||||||
// Disable newtabpage
|
|
||||||
"browser.newtabpage.enabled": false,
|
|
||||||
"browser.startup.homepage": "about:blank",
|
|
||||||
|
|
||||||
// Do not warn when closing all open tabs
|
|
||||||
"browser.tabs.warnOnClose": false,
|
|
||||||
|
|
||||||
// Disable telemetry
|
|
||||||
"toolkit.telemetry.reportingpolicy.firstRun": false,
|
|
||||||
|
|
||||||
// Disable first-run welcome page
|
|
||||||
"startup.homepage_welcome_url": "about:blank",
|
|
||||||
"startup.homepage_welcome_url.additional": "",
|
|
||||||
|
|
||||||
// Detected !
|
|
||||||
// // Disable images to speed up load
|
|
||||||
// "permissions.default.image": 2,
|
|
||||||
|
|
||||||
// Limit content processes to 1
|
|
||||||
"dom.ipc.processCount": 1
|
|
||||||
}
|
|
||||||
|
|
||||||
// proxy.url format => http://<host>:<port>
|
|
||||||
if (proxy && proxy.url) {
|
|
||||||
log.debug(`Using proxy: ${proxy.url}`)
|
|
||||||
const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
|
|
||||||
const port = parseInt(portStr);
|
|
||||||
if (!host || !portStr || !port) {
|
|
||||||
throw new Error("Proxy configuration is invalid! Use the format: protocol://ip:port")
|
|
||||||
}
|
|
||||||
|
|
||||||
const proxyPrefs = {
|
|
||||||
"network.proxy.type": 1,
|
|
||||||
"network.proxy.share_proxy_settings": true
|
|
||||||
}
|
|
||||||
if (proxy.url.indexOf("socks") != -1) {
|
|
||||||
// SOCKSv4 & SOCKSv5
|
|
||||||
Object.assign(proxyPrefs, {
|
|
||||||
"network.proxy.socks": host,
|
|
||||||
"network.proxy.socks_port": port,
|
|
||||||
"network.proxy.socks_remote_dns": true
|
|
||||||
});
|
|
||||||
if (proxy.url.indexOf("socks4") != -1) {
|
|
||||||
Object.assign(proxyPrefs, {
|
|
||||||
"network.proxy.socks_version": 4
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
Object.assign(proxyPrefs, {
|
|
||||||
"network.proxy.socks_version": 5
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// HTTP
|
|
||||||
Object.assign(proxyPrefs, {
|
|
||||||
"network.proxy.ftp": host,
|
|
||||||
"network.proxy.ftp_port": port,
|
|
||||||
"network.proxy.http": host,
|
|
||||||
"network.proxy.http_port": port,
|
|
||||||
"network.proxy.ssl": host,
|
|
||||||
"network.proxy.ssl_port": port
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// merge objects
|
|
||||||
Object.assign(extraPrefsFirefox, proxyPrefs);
|
|
||||||
}
|
|
||||||
|
|
||||||
return extraPrefsFirefox;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getUserAgent() {
|
|
||||||
return webBrowserUserAgent
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function testWebBrowserInstallation(): Promise<void> {
|
|
||||||
log.info("Testing web browser installation...")
|
|
||||||
|
|
||||||
// check user home dir. this dir will be used by Firefox
|
|
||||||
const homeDir = os.homedir();
|
|
||||||
fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
|
|
||||||
log.debug("FlareSolverr user home directory is OK: " + homeDir)
|
|
||||||
|
|
||||||
// test web browser
|
|
||||||
const testUrl = process.env.TEST_URL || "https://www.google.com";
|
|
||||||
log.debug("Test URL: " + testUrl)
|
|
||||||
const session = await create(null, {
|
|
||||||
oneTimeSession: true
|
|
||||||
})
|
|
||||||
const page = await session.browser.newPage()
|
|
||||||
const pageTimeout = Number(process.env.BROWSER_TIMEOUT) || 40000
|
|
||||||
await page.goto(testUrl, {waitUntil: 'domcontentloaded', timeout: pageTimeout})
|
|
||||||
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
|
|
||||||
|
|
||||||
// replace Linux ARM user-agent because it's detected
|
|
||||||
if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
|
|
||||||
webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
|
|
||||||
await page.close()
|
|
||||||
await destroy(session.sessionId)
|
|
||||||
|
|
||||||
log.info("Test successful")
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
|
|
||||||
log.debug('Creating new session...')
|
|
||||||
|
|
||||||
const sessionId = session || UUIDv1()
|
|
||||||
|
|
||||||
// NOTE: cookies can't be set in the session, you need to open the page first
|
|
||||||
|
|
||||||
const puppeteerOptions: any = {
|
|
||||||
product: 'firefox',
|
|
||||||
headless: process.env.HEADLESS !== 'false',
|
|
||||||
timeout: Number(process.env.BROWSER_TIMEOUT) || 40000
|
|
||||||
}
|
|
||||||
|
|
||||||
puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)
|
|
||||||
|
|
||||||
// if we are running inside executable binary, change browser path
|
|
||||||
if (typeof (process as any).pkg !== 'undefined') {
|
|
||||||
const exe = process.platform === "win32" ? 'firefox.exe' : 'firefox';
|
|
||||||
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'firefox', exe)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Launching web browser...')
|
|
||||||
let browser: Browser = await puppeteer.launch(puppeteerOptions)
|
|
||||||
if (!browser) {
|
|
||||||
throw Error(`Failed to launch web browser.`)
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionCache[sessionId] = {
|
|
||||||
sessionId: sessionId,
|
|
||||||
browser: browser
|
|
||||||
}
|
|
||||||
|
|
||||||
return sessionCache[sessionId]
|
|
||||||
}
|
|
||||||
|
|
||||||
export function list(): string[] {
|
|
||||||
return Object.keys(sessionCache)
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function destroy(id: string): Promise<boolean>{
|
|
||||||
if (id && sessionCache.hasOwnProperty(id)) {
|
|
||||||
const { browser } = sessionCache[id]
|
|
||||||
if (browser) {
|
|
||||||
await browser.close()
|
|
||||||
delete sessionCache[id]
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
export function get(id: string): SessionsCacheItem {
|
|
||||||
return sessionCache[id]
|
|
||||||
}
|
|
||||||
@ -1,206 +0,0 @@
|
|||||||
import {Page, HTTPResponse} from 'puppeteer'
|
|
||||||
const Timeout = require('await-timeout');
|
|
||||||
|
|
||||||
import log from './log'
|
|
||||||
import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
|
|
||||||
import {V1Request} from "../controllers/v1";
|
|
||||||
import cloudflareProvider from '../providers/cloudflare';
|
|
||||||
|
|
||||||
const sessions = require('./sessions')
|
|
||||||
|
|
||||||
export interface ChallengeResolutionResultT {
|
|
||||||
url: string
|
|
||||||
status: number,
|
|
||||||
headers?: Record<string, string>,
|
|
||||||
response: string,
|
|
||||||
cookies: object[]
|
|
||||||
userAgent: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ChallengeResolutionT {
|
|
||||||
status?: string
|
|
||||||
message: string
|
|
||||||
result: ChallengeResolutionResultT
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
|
|
||||||
const timer = new Timeout();
|
|
||||||
try {
|
|
||||||
const promise = resolveChallenge(params, session);
|
|
||||||
return await Promise.race([
|
|
||||||
promise,
|
|
||||||
timer.set(params.maxTimeout, `Maximum timeout reached. maxTimeout=${params.maxTimeout} (ms)`)
|
|
||||||
]);
|
|
||||||
} finally {
|
|
||||||
timer.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
|
|
||||||
try {
|
|
||||||
let status = 'ok'
|
|
||||||
let message = ''
|
|
||||||
|
|
||||||
const page: Page = await session.browser.newPage()
|
|
||||||
|
|
||||||
// the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
|
|
||||||
// the user can set a really high maxTimeout if he wants to
|
|
||||||
await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
|
|
||||||
|
|
||||||
// the user-agent is changed just for linux arm build
|
|
||||||
await page.setUserAgent(sessions.getUserAgent())
|
|
||||||
|
|
||||||
// set the proxy
|
|
||||||
if (params.proxy) {
|
|
||||||
log.debug(`Using proxy: ${params.proxy.url}`);
|
|
||||||
// todo: credentials are not working
|
|
||||||
// if (params.proxy.username) {
|
|
||||||
// await page.authenticate({
|
|
||||||
// username: params.proxy.username,
|
|
||||||
// password: params.proxy.password
|
|
||||||
// });
|
|
||||||
// }
|
|
||||||
}
|
|
||||||
|
|
||||||
// go to the page
|
|
||||||
log.debug(`Navigating to... ${params.url}`)
|
|
||||||
let response: HTTPResponse = await gotoPage(params, page);
|
|
||||||
|
|
||||||
// set cookies
|
|
||||||
if (params.cookies) {
|
|
||||||
for (const cookie of params.cookies) {
|
|
||||||
// the other fields in the cookie can cause issues
|
|
||||||
await page.setCookie({
|
|
||||||
"name": cookie.name,
|
|
||||||
"value": cookie.value
|
|
||||||
})
|
|
||||||
}
|
|
||||||
// reload the page
|
|
||||||
response = await gotoPage(params, page);
|
|
||||||
}
|
|
||||||
|
|
||||||
// log html in debug mode
|
|
||||||
log.html(await page.content())
|
|
||||||
|
|
||||||
// detect protection services and solve challenges
|
|
||||||
try {
|
|
||||||
response = await cloudflareProvider(params.url, page, response);
|
|
||||||
|
|
||||||
// is response is ok
|
|
||||||
// reload the page to be sure we get the real page
|
|
||||||
log.debug("Reloading the page")
|
|
||||||
try {
|
|
||||||
response = await gotoPage(params, page, params.method);
|
|
||||||
} catch (e) {
|
|
||||||
log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
|
|
||||||
}
|
|
||||||
|
|
||||||
} catch (e) {
|
|
||||||
status = "error";
|
|
||||||
message = "Cloudflare " + e.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload: ChallengeResolutionT = {
|
|
||||||
status,
|
|
||||||
message,
|
|
||||||
result: {
|
|
||||||
url: page.url(),
|
|
||||||
status: response.status(),
|
|
||||||
headers: response.headers(),
|
|
||||||
response: null,
|
|
||||||
cookies: await page.cookies(),
|
|
||||||
userAgent: sessions.getUserAgent()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (params.returnOnlyCookies) {
|
|
||||||
payload.result.headers = null;
|
|
||||||
payload.result.userAgent = null;
|
|
||||||
} else {
|
|
||||||
payload.result.response = await page.content()
|
|
||||||
}
|
|
||||||
|
|
||||||
// make sure the page is closed because if it isn't and error will be thrown
|
|
||||||
// when a user uses a temporary session, the browser make be quit before
|
|
||||||
// the page is properly closed.
|
|
||||||
await page.close()
|
|
||||||
|
|
||||||
return payload
|
|
||||||
} catch (e) {
|
|
||||||
log.error("Unexpected error: " + e);
|
|
||||||
throw e;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function gotoPage(params: V1Request, page: Page, method: string = 'GET'): Promise<HTTPResponse> {
|
|
||||||
let pageTimeout = params.maxTimeout / 3;
|
|
||||||
let response: HTTPResponse
|
|
||||||
|
|
||||||
try {
|
|
||||||
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
|
|
||||||
} catch (e) {
|
|
||||||
// retry
|
|
||||||
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
|
|
||||||
}
|
|
||||||
|
|
||||||
if (method == 'POST') {
|
|
||||||
// post hack, it only works with utf-8 encoding
|
|
||||||
|
|
||||||
let postForm = `<form id="hackForm" action="${params.url}" method="POST">`;
|
|
||||||
let queryString = params.postData;
|
|
||||||
let pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
|
|
||||||
for (let i = 0; i < pairs.length; i++) {
|
|
||||||
let pair = pairs[i].split('=');
|
|
||||||
let name; try { name = decodeURIComponent(pair[0]) } catch { name = pair[0] }
|
|
||||||
if (name == 'submit') continue;
|
|
||||||
let value; try { value = decodeURIComponent(pair[1] || '') } catch { value = pair[1] || '' }
|
|
||||||
postForm += `<input type="text" name="${name}" value="${value}"><br>`;
|
|
||||||
}
|
|
||||||
postForm += `</form>`;
|
|
||||||
|
|
||||||
await page.setContent(`
|
|
||||||
<!DOCTYPE html>
|
|
||||||
<html>
|
|
||||||
<body>
|
|
||||||
${postForm}
|
|
||||||
<script>document.getElementById('hackForm').submit();</script>
|
|
||||||
</body>
|
|
||||||
</html>`
|
|
||||||
);
|
|
||||||
await page.waitForTimeout(2000)
|
|
||||||
try {
|
|
||||||
await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
|
|
||||||
} catch (e) {}
|
|
||||||
|
|
||||||
}
|
|
||||||
return response
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
|
|
||||||
const oneTimeSession = params.session === undefined;
|
|
||||||
|
|
||||||
const options: SessionCreateOptions = {
|
|
||||||
oneTimeSession: oneTimeSession,
|
|
||||||
cookies: params.cookies,
|
|
||||||
maxTimeout: params.maxTimeout,
|
|
||||||
proxy: params.proxy
|
|
||||||
}
|
|
||||||
|
|
||||||
const session: SessionsCacheItem = oneTimeSession
|
|
||||||
? await sessions.create(null, options)
|
|
||||||
: sessions.get(params.session)
|
|
||||||
|
|
||||||
if (!session) {
|
|
||||||
throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
return await resolveChallengeWithTimeout(params, session)
|
|
||||||
} catch (error) {
|
|
||||||
throw Error("Unable to process browser request. " + error)
|
|
||||||
} finally {
|
|
||||||
if (oneTimeSession) {
|
|
||||||
await sessions.destroy(session.sessionId)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,625 +0,0 @@
|
|||||||
// noinspection DuplicatedCode
|
|
||||||
|
|
||||||
import {Response} from "superagent";
|
|
||||||
import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
|
|
||||||
|
|
||||||
const request = require("supertest");
|
|
||||||
const app = require("../app");
|
|
||||||
const sessions = require('../services/sessions');
|
|
||||||
const version: string = 'v' + require('../../package.json').version
|
|
||||||
|
|
||||||
const proxyUrl = "http://127.0.0.1:8888"
|
|
||||||
const proxySocksUrl = "socks5://127.0.0.1:1080"
|
|
||||||
const googleUrl = "https://www.google.com";
|
|
||||||
const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
|
|
||||||
const cfUrl = "https://nowsecure.nl";
|
|
||||||
const cfCaptchaUrl = "https://idope.se"
|
|
||||||
const cfBlockedUrl = "https://avistaz.to/api/v1/jackett/torrents?in=1&type=0&search="
|
|
||||||
const ddgUrl = "https://anidex.info/";
|
|
||||||
const ccfUrl = "https://www.muziekfabriek.org";
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
|
||||||
// Init session
|
|
||||||
await sessions.testWebBrowserInstallation();
|
|
||||||
});
|
|
||||||
|
|
||||||
afterEach(async () => {
|
|
||||||
// Clean sessions
|
|
||||||
const sessionList = sessions.list();
|
|
||||||
for (const session of sessionList) {
|
|
||||||
await sessions.destroy(session);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("Test '/' path", () => {
|
|
||||||
test("GET method should return OK ", async () => {
|
|
||||||
const response: Response = await request(app).get("/");
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
expect(response.body.msg).toBe("FlareSolverr is ready!");
|
|
||||||
expect(response.body.version).toBe(version);
|
|
||||||
expect(response.body.userAgent).toContain("Firefox/")
|
|
||||||
});
|
|
||||||
|
|
||||||
test("POST method should fail", async () => {
|
|
||||||
const response: Response = await request(app).post("/");
|
|
||||||
expect(response.statusCode).toBe(404);
|
|
||||||
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("Test '/health' path", () => {
|
|
||||||
test("GET method should return OK", async () => {
|
|
||||||
const response: Response = await request(app).get("/health");
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
expect(response.body.status).toBe("ok");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("Test '/wrong' path", () => {
|
|
||||||
test("GET method should fail", async () => {
|
|
||||||
const response: Response = await request(app).get("/wrong");
|
|
||||||
expect(response.statusCode).toBe(404);
|
|
||||||
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("Test '/v1' path", () => {
|
|
||||||
test("Cmd 'request.bad' should fail", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.bad",
|
|
||||||
"url": googleUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.response).toContain("<!DOCTYPE html>")
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with Cloudflare JS", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": cfUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(cfUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.response).toContain("<!DOCTYPE html>")
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
|
|
||||||
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "cf_clearance";
|
|
||||||
})[0].value
|
|
||||||
expect(cfCookie.length).toBeGreaterThan(30)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return fail with Cloudflare CAPTCHA", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": cfCaptchaUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Cloudflare Error: FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn't always appear, you may have better luck with the next request.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
// solution is filled but not useful
|
|
||||||
expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.post' should return fail with Cloudflare Blocked", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.post",
|
|
||||||
"url": cfBlockedUrl,
|
|
||||||
"postData": "test1=test2"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Cloudflare Error: Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
// solution is filled but not useful
|
|
||||||
expect(apiResponse.solution.url).toContain(cfBlockedUrl)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": ddgUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(ddgUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.response).toContain("<!DOCTYPE html>")
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
|
|
||||||
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "__ddg1_";
|
|
||||||
})[0].value
|
|
||||||
expect(cfCookie.length).toBeGreaterThan(10)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": ccfUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(ccfUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.response).toContain("<html><head>")
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
|
|
||||||
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "ct_anti_ddos_key";
|
|
||||||
})[0].value
|
|
||||||
expect(cfCookie.length).toBeGreaterThan(10)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"cookies": [
|
|
||||||
{
|
|
||||||
"name": "testcookie1",
|
|
||||||
"value": "testvalue1"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "testcookie2",
|
|
||||||
"value": "testvalue2"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
|
|
||||||
const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "testcookie1";
|
|
||||||
})[0].value
|
|
||||||
expect(cookie1).toBe("testvalue1")
|
|
||||||
const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "testcookie2";
|
|
||||||
})[0].value
|
|
||||||
expect(cookie2).toBe("testvalue2")
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"returnOnlyCookies": true
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(solution.headers).toBe(null)
|
|
||||||
expect(solution.response).toBe(null)
|
|
||||||
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.userAgent).toBe(null)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
|
|
||||||
/*
|
|
||||||
To configure TinyProxy in local:
|
|
||||||
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
|
||||||
* edit => LogFile "/tmp/tinyproxy.log"
|
|
||||||
* edit => Syslog Off
|
|
||||||
* sudo tinyproxy -d
|
|
||||||
* sudo tail -f /tmp/tinyproxy.log
|
|
||||||
*/
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"proxy": {
|
|
||||||
"url": proxyUrl
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
});
|
|
||||||
|
|
||||||
// todo: credentials are not working
|
|
||||||
test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
|
|
||||||
/*
|
|
||||||
To configure TinyProxy in local:
|
|
||||||
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
|
||||||
* edit => LogFile "/tmp/tinyproxy.log"
|
|
||||||
* edit => Syslog Off
|
|
||||||
* add => BasicAuth testuser testpass
|
|
||||||
* sudo tinyproxy -d
|
|
||||||
* sudo tail -f /tmp/tinyproxy.log
|
|
||||||
*/
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"proxy": {
|
|
||||||
"url": proxyUrl,
|
|
||||||
"username": "testuser",
|
|
||||||
"password": "testpass"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toContain(200)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
|
|
||||||
/*
|
|
||||||
To configure Dante in local:
|
|
||||||
* https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
|
|
||||||
* sudo vim /etc/sockd.conf
|
|
||||||
* sudo systemctl restart sockd.service
|
|
||||||
* curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
|
|
||||||
*/
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"proxy": {
|
|
||||||
"url": proxySocksUrl
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"proxy": {
|
|
||||||
"url": "http://127.0.0.1:43210"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return fail with timeout", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"maxTimeout": 10
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=10 (ms)");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should return fail with bad domain", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": "https://www.google.combad"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_UNKNOWN_HOST at https://www.google.combad");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should accept deprecated params", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": googleUrl,
|
|
||||||
"userAgent": "Test User-Agent" // was removed in v2, not used
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(googleUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.post' should return OK with no Cloudflare", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.post",
|
|
||||||
"url": postUrl + '/post',
|
|
||||||
"postData": "param1=value1¶m2=value2"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
|
|
||||||
const solution = apiResponse.solution;
|
|
||||||
expect(solution.url).toContain(postUrl)
|
|
||||||
expect(solution.status).toBe(200);
|
|
||||||
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
|
||||||
expect(solution.response).toContain(" I hope you have a lovely day!")
|
|
||||||
expect(Object.keys(solution.cookies).length).toBe(0)
|
|
||||||
expect(solution.userAgent).toContain("Firefox/")
|
|
||||||
|
|
||||||
// check that we sent the date
|
|
||||||
const payload2 = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": postUrl
|
|
||||||
}
|
|
||||||
const response2: Response = await request(app).post("/v1").send(payload2);
|
|
||||||
expect(response2.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse2: V1ResponseSolution = response2.body;
|
|
||||||
expect(apiResponse2.status).toBe("ok");
|
|
||||||
|
|
||||||
const solution2 = apiResponse2.solution;
|
|
||||||
expect(solution2.status).toBe(200);
|
|
||||||
expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' '))
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.post' should fail without 'postData' param", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.post",
|
|
||||||
"url": googleUrl
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'sessions.create' should return OK", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "sessions.create"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSession = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("Session created successfully.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
expect(apiResponse.session.length).toBe(36);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'sessions.create' should return OK with session", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"session": "2bc6bb20-2f56-11ec-9543-test"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSession = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("Session created successfully.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'sessions.list' should return OK", async () => {
|
|
||||||
// create one session for testing
|
|
||||||
const payload0 = {
|
|
||||||
"cmd": "sessions.create"
|
|
||||||
}
|
|
||||||
const response0: Response = await request(app).post("/v1").send(payload0);
|
|
||||||
expect(response0.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const payload = {
|
|
||||||
"cmd": "sessions.list"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSessions = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
expect(apiResponse.sessions.length).toBeGreaterThan(0)
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'sessions.destroy' should return OK", async () => {
|
|
||||||
// create one session for testing
|
|
||||||
const payload0 = {
|
|
||||||
"cmd": "sessions.create"
|
|
||||||
}
|
|
||||||
const response0: Response = await request(app).post("/v1").send(payload0);
|
|
||||||
expect(response0.statusCode).toBe(200);
|
|
||||||
const apiResponse0: V1ResponseSession = response0.body;
|
|
||||||
const sessionId0 = apiResponse0.session
|
|
||||||
|
|
||||||
const payload = {
|
|
||||||
"cmd": "sessions.destroy",
|
|
||||||
"session": sessionId0
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
expect(apiResponse.message).toBe("The session has been removed.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'sessions.destroy' should fail", async () => {
|
|
||||||
const payload = {
|
|
||||||
"cmd": "sessions.destroy",
|
|
||||||
"session": "bad-session"
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(500);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseBase = response.body;
|
|
||||||
expect(apiResponse.status).toBe("error");
|
|
||||||
expect(apiResponse.message).toBe("Error: This session does not exist.");
|
|
||||||
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
|
||||||
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
|
||||||
expect(apiResponse.version).toBe(version);
|
|
||||||
});
|
|
||||||
|
|
||||||
test("Cmd 'request.get' should use session", async () => {
|
|
||||||
// create one session for testing
|
|
||||||
const payload0 = {
|
|
||||||
"cmd": "sessions.create"
|
|
||||||
}
|
|
||||||
const response0: Response = await request(app).post("/v1").send(payload0);
|
|
||||||
expect(response0.statusCode).toBe(200);
|
|
||||||
const apiResponse0: V1ResponseSession = response0.body;
|
|
||||||
const sessionId0 = apiResponse0.session
|
|
||||||
|
|
||||||
// first request should solve the challenge
|
|
||||||
const payload = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": cfUrl,
|
|
||||||
"session": sessionId0
|
|
||||||
}
|
|
||||||
const response: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse: V1ResponseSolution = response.body;
|
|
||||||
expect(apiResponse.status).toBe("ok");
|
|
||||||
const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "cf_clearance";
|
|
||||||
})[0].value
|
|
||||||
expect(cfCookie.length).toBeGreaterThan(30)
|
|
||||||
|
|
||||||
// second request should have the same cookie
|
|
||||||
const response2: Response = await request(app).post("/v1").send(payload);
|
|
||||||
expect(response2.statusCode).toBe(200);
|
|
||||||
|
|
||||||
const apiResponse2: V1ResponseSolution = response2.body;
|
|
||||||
expect(apiResponse2.status).toBe("ok");
|
|
||||||
const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
|
|
||||||
return cookie.name == "cf_clearance";
|
|
||||||
})[0].value
|
|
||||||
expect(cfCookie2.length).toBeGreaterThan(30)
|
|
||||||
expect(cfCookie2).toBe(cfCookie)
|
|
||||||
});
|
|
||||||
|
|
||||||
});
|
|
||||||
@ -1,21 +0,0 @@
|
|||||||
{
|
|
||||||
"compilerOptions": {
|
|
||||||
"moduleResolution": "node",
|
|
||||||
"target": "es2017",
|
|
||||||
"noImplicitAny": true,
|
|
||||||
"removeComments": true,
|
|
||||||
"preserveConstEnums": true,
|
|
||||||
"allowSyntheticDefaultImports": true,
|
|
||||||
"resolveJsonModule": true,
|
|
||||||
"lib": [
|
|
||||||
"es2015", "dom"
|
|
||||||
],
|
|
||||||
"module": "commonjs",
|
|
||||||
"outDir": "dist",
|
|
||||||
"sourceMap": true
|
|
||||||
},
|
|
||||||
"include": [
|
|
||||||
"src", "node_modules/@types/puppeteer/index.d.ts"
|
|
||||||
],
|
|
||||||
"exclude": ["node_modules"]
|
|
||||||
}
|
|
||||||
Loading…
x
Reference in New Issue
Block a user