diff --git a/README.md b/README.md index 3bc4bfa..d8dfd40 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ This is the same as `request.get` but it takes one more param: Parameter | Notes |--|--| -postData | Must be a string. If you want to POST a form with format `application/x-www-form-urlencoded`. +postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `postData": "a=b&c=d"` ## Environment variables diff --git a/src/providers/cloudflare.ts b/src/providers/cloudflare.ts index 5f15668..51e97b6 100644 --- a/src/providers/cloudflare.ts +++ b/src/providers/cloudflare.ts @@ -120,16 +120,6 @@ export default async function resolveChallenge(url: string, page: Page, response { throw new Error('No challenge selectors found, unable to proceed.') } else { - // reload the page to make sure we get the real response - // do not use page.reload() to avoid #162 #143 - response = await page.goto(url, { waitUntil: 'domcontentloaded' }) - - await page.content() - // log.info(response.headers()) - // while (response.headers() == null) { - // await page.waitFor(1000) - // } - log.info('Challenge solved.'); } } diff --git a/src/services/solver.ts b/src/services/solver.ts index c4cb688..c2acfd1 100644 --- a/src/services/solver.ts +++ b/src/services/solver.ts @@ -59,7 +59,7 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem): } log.debug(`Navigating to... ${params.url}`) - let response: Response = await page.goto(params.url, { waitUntil: 'domcontentloaded' }) + let response: Response = await gotoPage(params, page); // set cookies if (params.cookies) { @@ -71,13 +71,13 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem): }) } // reload the page - response = await page.goto(params.url, { waitUntil: 'domcontentloaded' }) + response = await gotoPage(params, page); } // log html in debug mode log.html(await page.content()) - // Detect protection services and solve challenges + // detect protection services and solve challenges try { response = await cloudflareProvider(params.url, page, response); } catch (e) { @@ -85,6 +85,9 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem): message = "Cloudflare " + e.toString(); } + // reload the page to be sure we get the real page + response = await gotoPage(params, page); + const payload: ChallengeResolutionT = { status, message, @@ -117,6 +120,55 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem): } } +async function gotoPage(params: V1Request, page: Page): Promise { + const response = await page.goto(params.url, { waitUntil: 'domcontentloaded' }); + if (params.method == 'POST') { + // post hack + await page.setContent( + ` + + + + + + + ` + ); + await page.waitFor(2000); + } + return response +} + export async function browserRequest(params: V1Request): Promise { const oneTimeSession = params.session === undefined; @@ -136,7 +188,6 @@ export async function browserRequest(params: V1Request): Promise { + // Init session + await testWebBrowserInstallation(); +}); + describe("Test '/' path", () => { test("GET method should return OK ", async () => { - // Init session - await testWebBrowserInstallation(); - const response: Response = await request(app).get("/"); expect(response.statusCode).toBe(200); expect(response.body.msg).toBe("FlareSolverr is ready!"); @@ -62,7 +65,7 @@ describe("Test '/v1' path", () => { expect(apiResponse.status).toBe("error"); expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid."); expect(apiResponse.startTimestamp).toBeGreaterThan(1000); - expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp); + expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp); expect(apiResponse.version).toBe(version); }); @@ -226,6 +229,62 @@ describe("Test '/v1' path", () => { expect(solution.userAgent).toContain("Firefox/") }); + test("Cmd 'request.post' should return OK with no Cloudflare", async () => { + const payload = { + "cmd": "request.post", + "url": postUrl + '/post', + "postData": "param1=value1¶m2=value2" + } + const response: Response = await request(app).post("/v1").send(payload); + expect(response.statusCode).toBe(200); + + const apiResponse: V1ResponseSolution = response.body; + expect(apiResponse.status).toBe("ok"); + expect(apiResponse.message).toBe(""); + expect(apiResponse.startTimestamp).toBeGreaterThan(1000); + expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp); + expect(apiResponse.version).toBe(version); + + const solution = apiResponse.solution; + expect(solution.url).toContain(postUrl) + expect(solution.status).toBe(200); + expect(Object.keys(solution.headers).length).toBeGreaterThan(0) + expect(solution.response).toContain(" I hope you have a lovely day!") + expect(Object.keys(solution.cookies).length).toBe(0) + expect(solution.userAgent).toContain("Firefox/") + + // check that we sent the date + const payload2 = { + "cmd": "request.get", + "url": postUrl + } + const response2: Response = await request(app).post("/v1").send(payload2); + expect(response2.statusCode).toBe(200); + + const apiResponse2: V1ResponseSolution = response2.body; + expect(apiResponse2.status).toBe("ok"); + + const solution2 = apiResponse2.solution; + expect(solution2.status).toBe(200); + expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' ')) + }); + + test("Cmd 'request.post' should fail without 'postData' param", async () => { + const payload = { + "cmd": "request.post", + "url": googleUrl + } + const response: Response = await request(app).post("/v1").send(payload); + expect(response.statusCode).toBe(500); + + const apiResponse: V1ResponseBase = response.body; + expect(apiResponse.status).toBe("error"); + expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request."); + expect(apiResponse.startTimestamp).toBeGreaterThan(1000); + expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp); + expect(apiResponse.version).toBe(version); + }); + test("Cmd 'sessions.create' should return OK", async () => { const payload = { "cmd": "sessions.create"