From 5dd563e0038756c5b88db75918f6ee1c393fa23f Mon Sep 17 00:00:00 2001 From: ngosang Date: Sat, 16 Oct 2021 19:16:25 +0200 Subject: [PATCH] Replace Chrome web browser with Firefox --- Dockerfile | 14 +++--- README.md | 8 ++-- build-binaries.js | 1 + src/index.ts | 26 +++++------ src/routes.ts | 112 ++++++++++++++++++++++++---------------------- src/session.ts | 7 +-- 6 files changed, 84 insertions(+), 84 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6dcf9d2..0ec22a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,13 @@ -FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11 +FROM --platform=${TARGETPLATFORM:-linux/amd64} node:14-alpine3.14 # Print build information ARG TARGETPLATFORM ARG BUILDPLATFORM RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n" -# Install Chromium, dumb-init and remove all locales but en-US -RUN apk add --no-cache chromium dumb-init && \ - find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete +# Install the web browser +RUN apk update && \ + apk add --no-cache firefox-esr dumb-init # Copy FlareSolverr code USER node @@ -16,10 +16,10 @@ WORKDIR /home/node/flaresolverr COPY --chown=node:node package.json package-lock.json tsconfig.json ./ COPY --chown=node:node src ./src/ -# Install package. Skip installing Chrome, we will use the installed package. -ENV PUPPETEER_PRODUCT=chrome \ +# Install package. Skip installing the browser, we will use the installed package. +ENV PUPPETEER_PRODUCT=firefox \ PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \ - PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser + PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox RUN npm install && \ npm run build && \ npm prune --production && \ diff --git a/README.md b/README.md index 1e2dce3..a03f76b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ FlareSolverr is a proxy server to bypass Cloudflare protection. FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources. When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the [stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth) -to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge +to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to bypass Cloudflare using other HTTP clients. @@ -60,7 +60,7 @@ docker run -d \ This is the recommended way for Windows users. * Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux. -* Extract the zip file. FlareSolverr executable and chrome folder must be in the same directory. +* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory. * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration. ### From source code @@ -68,8 +68,8 @@ This is the recommended way for Windows users. This is the recommended way for macOS users and for developers. * Install [NodeJS](https://nodejs.org/). * Clone this repository and open a shell in that path. -* Run `npm install` command to install FlareSolverr dependencies. -* Run `node node_modules/puppeteer/install.js` to install Chromium. +* Run `PUPPETEER_PRODUCT=firefox npm install` command to install FlareSolverr dependencies. +* Run `node node_modules/puppeteer/install.js` to install Firefox. * Run `npm run build` command to compile TypeScript code. * Run `npm start` command to start FlareSolverr. diff --git a/build-binaries.js b/build-binaries.js index 3e00f70..e3d2a31 100644 --- a/build-binaries.js +++ b/build-binaries.js @@ -5,6 +5,7 @@ const archiver = require('archiver') const puppeteer = require('puppeteer') const version = 'v' + require('./package.json').version; +// todo: package firefox (async () => { const builds = [ { diff --git a/src/index.ts b/src/index.ts index 62064d0..bcbe3b0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,6 +13,7 @@ import {v1 as UUIDv1} from "uuid"; const version: string = "v" + require('../package.json').version const serverPort: number = Number(process.env.PORT) || 8191 const serverHost: string = process.env.HOST || '0.0.0.0' +let webBrowserUserAgent: string = "" function validateEnvironmentVariables() { // ip and port variables are validated by nodejs @@ -36,24 +37,16 @@ function validateEnvironmentVariables() { } } -async function testChromeInstallation() { +async function testWebBrowserInstallation() { const sessionId = UUIDv1() - // create a temporary file for testing - log.debug("Testing Chrome installation...") - const fileContent = `flaresolverr_${version}` - const filePath = path.join(os.tmpdir(), `flaresolverr_${sessionId}.txt`) - const fileUrl = `file://${filePath}` - fs.writeFileSync(filePath, fileContent) - // launch the browser + log.debug("Testing web browser installation...") const session = await sessions.create(sessionId, { oneTimeSession: true }) const page = await session.browser.newPage() - const response = await page.goto(fileUrl, { waitUntil: 'domcontentloaded' }) - const responseBody = (await response.buffer()).toString().trim() - if (responseBody != fileContent) { - throw new Error("The response body does not match!") - } + await page.goto("https://www.google.com") + webBrowserUserAgent = await page.evaluate(() => navigator.userAgent) + log.info("FlareSolverr User-Agent: " + webBrowserUserAgent) await page.close() await sessions.destroy(sessionId) log.debug("Test successful") @@ -127,9 +120,9 @@ process.on('SIGTERM', () => { validateEnvironmentVariables(); -testChromeInstallation() +testWebBrowserInstallation() .catch(e => { - log.error("Error starting Chrome browser.", e); + log.error("Error starting web browser.", e); process.exit(1); }) .then(r => @@ -152,7 +145,8 @@ testChromeInstallation() // show welcome message if (req.url == '/') { - successResponse("FlareSolverr is ready!", null, res, startTimestamp); + const extendedProperties = {"userAgent": webBrowserUserAgent}; + successResponse("FlareSolverr is ready!", extendedProperties, res, startTimestamp); return; } diff --git a/src/routes.ts b/src/routes.ts index 3cb0041..a385bd4 100644 --- a/src/routes.ts +++ b/src/routes.ts @@ -129,8 +129,10 @@ async function resolveChallenge(ctx: RequestContext, // fix since I am short on time response = await page.goto(url, { waitUntil: 'domcontentloaded' }) payload.result.response = (await response.buffer()).toString('base64') - } else if (returnRawHtml) { - payload.result.response = await response.text() + + // todo: review this functionality + // } else if (returnRawHtml) { + // payload.result.response = await response.text() } else { payload.result.response = await page.content() } @@ -162,59 +164,61 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse // merge session defaults with params const { method, postData, headers, cookies } = params - let overrideResolvers: OverrideResolvers = {} + // todo: redo all functionality - if (method !== 'GET') { - log.debug(`Setting method to ${method}`) - overrideResolvers.method = request => method - } - - if (postData) { - log.debug(`Setting body data to ${postData}`) - overrideResolvers.postData = request => postData - } - - if (headers) { - log.debug(`Adding custom headers: ${JSON.stringify(headers)}`) - overrideResolvers.headers = request => Object.assign(request.headers(), headers) - } - - if (cookies) { - log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`) - await page.setCookie(...cookies) - } - - // if any keys have been set on the object - if (Object.keys(overrideResolvers).length > 0) { - let callbackRunOnce = false - const callback = (request: Request) => { - - // avoid loading resources to speed up page load - if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') { - request.abort() - return - } - - if (callbackRunOnce || !request.isNavigationRequest()) { - request.continue() - return - } - - callbackRunOnce = true - const overrides: Overrides = {} - - Object.keys(overrideResolvers).forEach((key: OverridesProps) => { - // @ts-ignore - overrides[key] = overrideResolvers[key](request) - }); - - log.debug(`Overrides: ${JSON.stringify(overrides)}`) - request.continue(overrides) - } - - await page.setRequestInterception(true) - page.on('request', callback) - } + // let overrideResolvers: OverrideResolvers = {} + // + // if (method !== 'GET') { + // log.debug(`Setting method to ${method}`) + // overrideResolvers.method = request => method + // } + // + // if (postData) { + // log.debug(`Setting body data to ${postData}`) + // overrideResolvers.postData = request => postData + // } + // + // if (headers) { + // log.debug(`Adding custom headers: ${JSON.stringify(headers)}`) + // overrideResolvers.headers = request => Object.assign(request.headers(), headers) + // } + // + // if (cookies) { + // log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`) + // await page.setCookie(...cookies) + // } + // + // // if any keys have been set on the object + // if (Object.keys(overrideResolvers).length > 0) { + // let callbackRunOnce = false + // const callback = (request: Request) => { + // + // // avoid loading resources to speed up page load + // if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') { + // request.abort() + // return + // } + // + // if (callbackRunOnce || !request.isNavigationRequest()) { + // request.continue() + // return + // } + // + // callbackRunOnce = true + // const overrides: Overrides = {} + // + // Object.keys(overrideResolvers).forEach((key: OverridesProps) => { + // // @ts-ignore + // overrides[key] = overrideResolvers[key](request) + // }); + // + // log.debug(`Overrides: ${JSON.stringify(overrides)}`) + // request.continue(overrides) + // } + // + // await page.setRequestInterception(true) + // page.on('request', callback) + // } return page } diff --git a/src/session.ts b/src/session.ts index a42d135..967583d 100644 --- a/src/session.ts +++ b/src/session.ts @@ -32,7 +32,7 @@ interface SessionCreateOptions { const sessionCache: SessionsCache = {} function userDataDirFromId(id: string): string { - return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`) + return path.join(os.tmpdir(), `/puppeteer_profile_${id}`) } function prepareBrowserProfile(id: string): string { @@ -58,7 +58,7 @@ export default { } const puppeteerOptions: LaunchOptions = { - product: 'chrome', + product: 'firefox', headless: process.env.HEADLESS !== 'false', args } @@ -68,7 +68,8 @@ export default { puppeteerOptions.userDataDir = prepareBrowserProfile(id) } - // if we are running inside executable binary, change chrome path + // todo: fix native package with firefox + // if we are running inside executable binary, change browser path if (typeof (process as any).pkg !== 'undefined') { const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome'; puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)