github/FlareSolverr
1
1
Fork 0
mirror of https://github.com/FlareSolverr/FlareSolverr.git synced 2025-06-08 12:35:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite request timeout handling (maxTimeout) resolves #42

Browse Source
This commit is contained in:
ngosang 2020-12-20 01:43:47 +01:00
parent d2b680520d
commit a23fa0983f
3 changed files with 31 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
package-lock.json generated
View File

@ -74,6 +74,12 @@
"defer-to-connect": "^2.0.0"
}
},
"@types/await-timeout": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/@types/await-timeout/-/await-timeout-0.3.1.tgz",
"integrity": "sha512-H5PzROT4KuP7XQDua13Iw8did//OCKAZ/3TL15DjvMzDonrk4HvhH1+tLko96f2guU6XaD3AoqRa49ZOwbwNig==",
"dev": true
},
"@types/cacheable-request": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
@ -288,6 +294,11 @@
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"await-timeout": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/await-timeout/-/await-timeout-1.1.1.tgz",
"integrity": "sha512-gsDXAS6XVc4Jt+7S92MPX6Noq69bdeXUPEaXd8dk3+yVr629LTDLxNt4j1ycBbrU+AStK2PhKIyNIM+xzWMVOQ=="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",

2
package.json
View File

@ -20,6 +20,7 @@
"url": "https://github.com/ngosang/FlareSolverr"
},
"dependencies": {
"await-timeout": "^1.1.1",
"console-log-level": "^1.4.1",
"got": "^11.5.1",
"hcaptcha-solver": "^1.0.2",
@ -29,6 +30,7 @@
"uuid": "^8.2.0"
},
"devDependencies": {
"@types/await-timeout": "^0.3.1",
"@types/node": "^14.0.23",
"@types/puppeteer": "^3.0.1",
"@types/uuid": "^8.0.0",

28
src/routes.ts
View File

@ -6,6 +6,7 @@ import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'pupp
import { TimeoutError } from 'puppeteer/Errors'
import getCaptchaSolver, { CaptchaType } from './captcha'
import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
const Timeout = require('await-timeout');
export interface BaseAPICall {
cmd: string
@ -119,9 +120,22 @@ async function interceptResponse(page: Puppeteer.Page, callback: (payload: Chall
});
}
async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Puppeteer.Page) {
const maxTimeout = params.maxTimeout || 60000
const timer = new Timeout();
try {
const promise = resolveChallenge(ctx, params, page);
return await Promise.race([
promise,
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
]);
} finally {
timer.clear();
}
}
async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
maxTimeout = maxTimeout || 60000
let status = 'ok'
let message = ''
@ -162,8 +176,7 @@ async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, d
});
}
// TODO: find out why these pages hang sometimes
while (Date.now() - ctx.startTimestamp < maxTimeout) {
while (true) {
await page.waitFor(1000)
try {
// catch exception timeout in waitForNavigation
@ -188,11 +201,6 @@ async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, d
log.html(await page.content())
}
if (Date.now() - ctx.startTimestamp >= maxTimeout) {
ctx.errorResponse(`Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
return
}
log.debug('Validating HTML code...')
break
} else {
@ -418,7 +426,7 @@ const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) =
try {
const page = await setupPage(ctx, params, session.browser)
const data = await resolveChallenge(ctx, params, page)
const data = await resolveChallengeWithTimeout(ctx, params, page)
if (data) {
const { status } = data