mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-06-08 04:25:25 +00:00
Replace Chrome web browser with Firefox
This commit is contained in:
ngosang
parent
78c10d6b24
commit
5dd563e003
14
Dockerfile
14
Dockerfile
@ -1,13 +1,13 @@
|
||||
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11
|
||||
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:14-alpine3.14
|
||||
|
||||
# Print build information
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
|
||||
|
||||
# Install Chromium, dumb-init and remove all locales but en-US
|
||||
RUN apk add --no-cache chromium dumb-init && \
|
||||
find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete
|
||||
# Install the web browser
|
||||
RUN apk update && \
|
||||
apk add --no-cache firefox-esr dumb-init
|
||||
|
||||
# Copy FlareSolverr code
|
||||
USER node
|
||||
@ -16,10 +16,10 @@ WORKDIR /home/node/flaresolverr
|
||||
COPY --chown=node:node package.json package-lock.json tsconfig.json ./
|
||||
COPY --chown=node:node src ./src/
|
||||
|
||||
# Install package. Skip installing Chrome, we will use the installed package.
|
||||
ENV PUPPETEER_PRODUCT=chrome \
|
||||
# Install package. Skip installing the browser, we will use the installed package.
|
||||
ENV PUPPETEER_PRODUCT=firefox \
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
|
||||
RUN npm install && \
|
||||
npm run build && \
|
||||
npm prune --production && \
|
||||
|
||||
@ -15,7 +15,7 @@ FlareSolverr is a proxy server to bypass Cloudflare protection.
|
||||
FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
|
||||
When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
|
||||
[stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
|
||||
to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
|
||||
to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge
|
||||
is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
|
||||
bypass Cloudflare using other HTTP clients.
|
||||
|
||||
@ -60,7 +60,7 @@ docker run -d \
|
||||
|
||||
This is the recommended way for Windows users.
|
||||
* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
|
||||
* Extract the zip file. FlareSolverr executable and chrome folder must be in the same directory.
|
||||
* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
|
||||
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
||||
|
||||
### From source code
|
||||
@ -68,8 +68,8 @@ This is the recommended way for Windows users.
|
||||
This is the recommended way for macOS users and for developers.
|
||||
* Install [NodeJS](https://nodejs.org/).
|
||||
* Clone this repository and open a shell in that path.
|
||||
* Run `npm install` command to install FlareSolverr dependencies.
|
||||
* Run `node node_modules/puppeteer/install.js` to install Chromium.
|
||||
* Run `PUPPETEER_PRODUCT=firefox npm install` command to install FlareSolverr dependencies.
|
||||
* Run `node node_modules/puppeteer/install.js` to install Firefox.
|
||||
* Run `npm run build` command to compile TypeScript code.
|
||||
* Run `npm start` command to start FlareSolverr.
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ const archiver = require('archiver')
|
||||
const puppeteer = require('puppeteer')
|
||||
const version = 'v' + require('./package.json').version;
|
||||
|
||||
// todo: package firefox
|
||||
(async () => {
|
||||
const builds = [
|
||||
{
|
||||
|
||||
26
src/index.ts
26
src/index.ts
@ -13,6 +13,7 @@ import {v1 as UUIDv1} from "uuid";
|
||||
const version: string = "v" + require('../package.json').version
|
||||
const serverPort: number = Number(process.env.PORT) || 8191
|
||||
const serverHost: string = process.env.HOST || '0.0.0.0'
|
||||
let webBrowserUserAgent: string = ""
|
||||
|
||||
function validateEnvironmentVariables() {
|
||||
// ip and port variables are validated by nodejs
|
||||
@ -36,24 +37,16 @@ function validateEnvironmentVariables() {
|
||||
}
|
||||
}
|
||||
|
||||
async function testChromeInstallation() {
|
||||
async function testWebBrowserInstallation() {
|
||||
const sessionId = UUIDv1()
|
||||
// create a temporary file for testing
|
||||
log.debug("Testing Chrome installation...")
|
||||
const fileContent = `flaresolverr_${version}`
|
||||
const filePath = path.join(os.tmpdir(), `flaresolverr_${sessionId}.txt`)
|
||||
const fileUrl = `file://${filePath}`
|
||||
fs.writeFileSync(filePath, fileContent)
|
||||
// launch the browser
|
||||
log.debug("Testing web browser installation...")
|
||||
const session = await sessions.create(sessionId, {
|
||||
oneTimeSession: true
|
||||
})
|
||||
const page = await session.browser.newPage()
|
||||
const response = await page.goto(fileUrl, { waitUntil: 'domcontentloaded' })
|
||||
const responseBody = (await response.buffer()).toString().trim()
|
||||
if (responseBody != fileContent) {
|
||||
throw new Error("The response body does not match!")
|
||||
}
|
||||
await page.goto("https://www.google.com")
|
||||
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
|
||||
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
|
||||
await page.close()
|
||||
await sessions.destroy(sessionId)
|
||||
log.debug("Test successful")
|
||||
@ -127,9 +120,9 @@ process.on('SIGTERM', () => {
|
||||
|
||||
validateEnvironmentVariables();
|
||||
|
||||
testChromeInstallation()
|
||||
testWebBrowserInstallation()
|
||||
.catch(e => {
|
||||
log.error("Error starting Chrome browser.", e);
|
||||
log.error("Error starting web browser.", e);
|
||||
process.exit(1);
|
||||
})
|
||||
.then(r =>
|
||||
@ -152,7 +145,8 @@ testChromeInstallation()
|
||||
|
||||
// show welcome message
|
||||
if (req.url == '/') {
|
||||
successResponse("FlareSolverr is ready!", null, res, startTimestamp);
|
||||
const extendedProperties = {"userAgent": webBrowserUserAgent};
|
||||
successResponse("FlareSolverr is ready!", extendedProperties, res, startTimestamp);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
112
src/routes.ts
112
src/routes.ts
@ -129,8 +129,10 @@ async function resolveChallenge(ctx: RequestContext,
|
||||
// fix since I am short on time
|
||||
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
||||
payload.result.response = (await response.buffer()).toString('base64')
|
||||
} else if (returnRawHtml) {
|
||||
payload.result.response = await response.text()
|
||||
|
||||
// todo: review this functionality
|
||||
// } else if (returnRawHtml) {
|
||||
// payload.result.response = await response.text()
|
||||
} else {
|
||||
payload.result.response = await page.content()
|
||||
}
|
||||
@ -162,59 +164,61 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
|
||||
// merge session defaults with params
|
||||
const { method, postData, headers, cookies } = params
|
||||
|
||||
let overrideResolvers: OverrideResolvers = {}
|
||||
// todo: redo all functionality
|
||||
|
||||
if (method !== 'GET') {
|
||||
log.debug(`Setting method to ${method}`)
|
||||
overrideResolvers.method = request => method
|
||||
}
|
||||
|
||||
if (postData) {
|
||||
log.debug(`Setting body data to ${postData}`)
|
||||
overrideResolvers.postData = request => postData
|
||||
}
|
||||
|
||||
if (headers) {
|
||||
log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
|
||||
overrideResolvers.headers = request => Object.assign(request.headers(), headers)
|
||||
}
|
||||
|
||||
if (cookies) {
|
||||
log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
|
||||
await page.setCookie(...cookies)
|
||||
}
|
||||
|
||||
// if any keys have been set on the object
|
||||
if (Object.keys(overrideResolvers).length > 0) {
|
||||
let callbackRunOnce = false
|
||||
const callback = (request: Request) => {
|
||||
|
||||
// avoid loading resources to speed up page load
|
||||
if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
|
||||
request.abort()
|
||||
return
|
||||
}
|
||||
|
||||
if (callbackRunOnce || !request.isNavigationRequest()) {
|
||||
request.continue()
|
||||
return
|
||||
}
|
||||
|
||||
callbackRunOnce = true
|
||||
const overrides: Overrides = {}
|
||||
|
||||
Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
|
||||
// @ts-ignore
|
||||
overrides[key] = overrideResolvers[key](request)
|
||||
});
|
||||
|
||||
log.debug(`Overrides: ${JSON.stringify(overrides)}`)
|
||||
request.continue(overrides)
|
||||
}
|
||||
|
||||
await page.setRequestInterception(true)
|
||||
page.on('request', callback)
|
||||
}
|
||||
// let overrideResolvers: OverrideResolvers = {}
|
||||
//
|
||||
// if (method !== 'GET') {
|
||||
// log.debug(`Setting method to ${method}`)
|
||||
// overrideResolvers.method = request => method
|
||||
// }
|
||||
//
|
||||
// if (postData) {
|
||||
// log.debug(`Setting body data to ${postData}`)
|
||||
// overrideResolvers.postData = request => postData
|
||||
// }
|
||||
//
|
||||
// if (headers) {
|
||||
// log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
|
||||
// overrideResolvers.headers = request => Object.assign(request.headers(), headers)
|
||||
// }
|
||||
//
|
||||
// if (cookies) {
|
||||
// log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
|
||||
// await page.setCookie(...cookies)
|
||||
// }
|
||||
//
|
||||
// // if any keys have been set on the object
|
||||
// if (Object.keys(overrideResolvers).length > 0) {
|
||||
// let callbackRunOnce = false
|
||||
// const callback = (request: Request) => {
|
||||
//
|
||||
// // avoid loading resources to speed up page load
|
||||
// if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
|
||||
// request.abort()
|
||||
// return
|
||||
// }
|
||||
//
|
||||
// if (callbackRunOnce || !request.isNavigationRequest()) {
|
||||
// request.continue()
|
||||
// return
|
||||
// }
|
||||
//
|
||||
// callbackRunOnce = true
|
||||
// const overrides: Overrides = {}
|
||||
//
|
||||
// Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
|
||||
// // @ts-ignore
|
||||
// overrides[key] = overrideResolvers[key](request)
|
||||
// });
|
||||
//
|
||||
// log.debug(`Overrides: ${JSON.stringify(overrides)}`)
|
||||
// request.continue(overrides)
|
||||
// }
|
||||
//
|
||||
// await page.setRequestInterception(true)
|
||||
// page.on('request', callback)
|
||||
// }
|
||||
|
||||
return page
|
||||
}
|
||||
|
||||
@ -32,7 +32,7 @@ interface SessionCreateOptions {
|
||||
const sessionCache: SessionsCache = {}
|
||||
|
||||
function userDataDirFromId(id: string): string {
|
||||
return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`)
|
||||
return path.join(os.tmpdir(), `/puppeteer_profile_${id}`)
|
||||
}
|
||||
|
||||
function prepareBrowserProfile(id: string): string {
|
||||
@ -58,7 +58,7 @@ export default {
|
||||
}
|
||||
|
||||
const puppeteerOptions: LaunchOptions = {
|
||||
product: 'chrome',
|
||||
product: 'firefox',
|
||||
headless: process.env.HEADLESS !== 'false',
|
||||
args
|
||||
}
|
||||
@ -68,7 +68,8 @@ export default {
|
||||
puppeteerOptions.userDataDir = prepareBrowserProfile(id)
|
||||
}
|
||||
|
||||
// if we are running inside executable binary, change chrome path
|
||||
// todo: fix native package with firefox
|
||||
// if we are running inside executable binary, change browser path
|
||||
if (typeof (process as any).pkg !== 'undefined') {
|
||||
const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
|
||||
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user