mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-06-07 20:15:24 +00:00
Compare commits
No commits in common. "master" and "v3.0.1" have entirely different histories.
17
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
17
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@ -8,13 +8,6 @@ body:
|
|||||||
options:
|
options:
|
||||||
- label: I have checked the README
|
- label: I have checked the README
|
||||||
required: true
|
required: true
|
||||||
- type: checkboxes
|
|
||||||
attributes:
|
|
||||||
label: Have you followed our Troubleshooting?
|
|
||||||
description: Please follow our <a href="https://github.com/FlareSolverr/FlareSolverr/wiki/Troubleshooting">Troubleshooting</a>.
|
|
||||||
options:
|
|
||||||
- label: I have followed your Troubleshooting
|
|
||||||
required: true
|
|
||||||
- type: checkboxes
|
- type: checkboxes
|
||||||
attributes:
|
attributes:
|
||||||
label: Is there already an issue for your problem?
|
label: Is there already an issue for your problem?
|
||||||
@ -29,13 +22,6 @@ body:
|
|||||||
options:
|
options:
|
||||||
- label: I have read the Discussions
|
- label: I have read the Discussions
|
||||||
required: true
|
required: true
|
||||||
- type: input
|
|
||||||
attributes:
|
|
||||||
label: Have you ACTUALLY checked all these?
|
|
||||||
description: Please do not waste our time and yours; these checks are there for a reason, it is not just so you can tick boxes for fun. If you type <b>YES</b> and it is clear you did not or have put in no effort, your issue will be closed and locked without comment. If you type <b>NO</b> but still open this issue, you will be permanently blocked for timewasting.
|
|
||||||
placeholder: YES or NO
|
|
||||||
validations:
|
|
||||||
required: true
|
|
||||||
- type: textarea
|
- type: textarea
|
||||||
attributes:
|
attributes:
|
||||||
label: Environment
|
label: Environment
|
||||||
@ -46,8 +32,7 @@ body:
|
|||||||
- Operating system:
|
- Operating system:
|
||||||
- Are you using Docker: [yes/no]
|
- Are you using Docker: [yes/no]
|
||||||
- FlareSolverr User-Agent (see log traces or / endpoint):
|
- FlareSolverr User-Agent (see log traces or / endpoint):
|
||||||
- Are you using a VPN: [yes/no]
|
- Are you using a proxy or VPN: [yes/no]
|
||||||
- Are you using a Proxy: [yes/no]
|
|
||||||
- Are you using Captcha Solver: [yes/no]
|
- Are you using Captcha Solver: [yes/no]
|
||||||
- If using captcha solver, which one:
|
- If using captcha solver, which one:
|
||||||
- URL to test this issue:
|
- URL to test this issue:
|
||||||
|
|||||||
2
.github/workflows/autotag.yml
vendored
2
.github/workflows/autotag.yml
vendored
@ -6,7 +6,7 @@ on:
|
|||||||
- "master"
|
- "master"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
tag-release:
|
build:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
-
|
-
|
||||||
|
|||||||
70
.github/workflows/release-docker.yml
vendored
70
.github/workflows/release-docker.yml
vendored
@ -4,64 +4,50 @@ on:
|
|||||||
push:
|
push:
|
||||||
tags:
|
tags:
|
||||||
- 'v*.*.*'
|
- 'v*.*.*'
|
||||||
pull_request:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build-docker-images:
|
build:
|
||||||
if: ${{ !github.event.pull_request.head.repo.fork }}
|
runs-on: ubuntu-latest
|
||||||
runs-on: ubuntu-22.04
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
-
|
||||||
uses: actions/checkout@v4
|
name: Checkout
|
||||||
|
uses: actions/checkout@v3
|
||||||
- name: Downcase repo
|
-
|
||||||
|
name: Downcase repo
|
||||||
run: echo REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
|
run: echo REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
|
||||||
|
-
|
||||||
- name: Docker meta
|
name: Docker meta
|
||||||
id: docker_meta
|
id: docker_meta
|
||||||
uses: docker/metadata-action@v5
|
uses: crazy-max/ghaction-docker-meta@v3
|
||||||
with:
|
with:
|
||||||
images: |
|
images: ${{ env.REPOSITORY }},ghcr.io/${{ env.REPOSITORY }}
|
||||||
${{ env.REPOSITORY }},enable=${{ github.event_name != 'pull_request' }}
|
tag-sha: false
|
||||||
ghcr.io/${{ env.REPOSITORY }}
|
-
|
||||||
tags: |
|
name: Set up QEMU
|
||||||
type=semver,pattern={{version}},prefix=v
|
uses: docker/setup-qemu-action@v2
|
||||||
type=ref,event=pr
|
-
|
||||||
flavor: |
|
name: Set up Docker Buildx
|
||||||
latest=auto
|
uses: docker/setup-buildx-action@v2
|
||||||
|
-
|
||||||
- name: Set up QEMU
|
name: Login to DockerHub
|
||||||
uses: docker/setup-qemu-action@v3
|
uses: docker/login-action@v2
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
|
|
||||||
- name: Login to DockerHub
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
if: github.event_name != 'pull_request'
|
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
|
-
|
||||||
- name: Login to GitHub Container Registry
|
name: Login to GitHub Container Registry
|
||||||
uses: docker/login-action@v3
|
uses: docker/login-action@v2
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
password: ${{ secrets.GH_PAT }}
|
password: ${{ secrets.GH_PAT }}
|
||||||
|
-
|
||||||
- name: Build and push
|
name: Build and push
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@v3
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
file: ./Dockerfile
|
file: ./Dockerfile
|
||||||
platforms: linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8
|
platforms: linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8
|
||||||
push: true
|
push: ${{ github.event_name != 'pull_request' }}
|
||||||
tags: ${{ steps.docker_meta.outputs.tags }}
|
tags: ${{ steps.docker_meta.outputs.tags }}
|
||||||
labels: ${{ steps.docker_meta.outputs.labels }}
|
labels: ${{ steps.docker_meta.outputs.labels }}
|
||||||
|
|||||||
70
.github/workflows/release.yml
vendored
70
.github/workflows/release.yml
vendored
@ -6,15 +6,26 @@ on:
|
|||||||
- 'v*.*.*'
|
- 'v*.*.*'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
create-release:
|
build:
|
||||||
name: Create release
|
name: Create release
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v3
|
uses: actions/checkout@v3
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
|
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
|
||||||
|
|
||||||
|
- name: Setup Node
|
||||||
|
uses: actions/setup-node@v3
|
||||||
|
with:
|
||||||
|
node-version: '16'
|
||||||
|
|
||||||
|
- name: Build artifacts
|
||||||
|
run: |
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
npm run package
|
||||||
|
|
||||||
- name: Build changelog
|
- name: Build changelog
|
||||||
id: github_changelog
|
id: github_changelog
|
||||||
run: |
|
run: |
|
||||||
@ -36,60 +47,9 @@ jobs:
|
|||||||
draft: false
|
draft: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|
||||||
build-linux-package:
|
|
||||||
name: Build Linux binary
|
|
||||||
needs: create-release
|
|
||||||
runs-on: ubuntu-22.04
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
with:
|
|
||||||
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
|
|
||||||
|
|
||||||
- name: Setup Python
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
|
|
||||||
- name: Build artifacts
|
|
||||||
run: |
|
|
||||||
python -m pip install -r requirements.txt
|
|
||||||
python -m pip install pyinstaller==5.13.0
|
|
||||||
cd src
|
|
||||||
python build_package.py
|
|
||||||
|
|
||||||
- name: Upload release artifacts
|
- name: Upload release artifacts
|
||||||
uses: alexellis/upload-assets@0.4.0
|
uses: alexellis/upload-assets@0.2.2
|
||||||
env:
|
env:
|
||||||
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
||||||
with:
|
with:
|
||||||
asset_paths: '["./dist/flaresolverr_*"]'
|
asset_paths: '["./bin/*.zip"]'
|
||||||
|
|
||||||
build-windows-package:
|
|
||||||
name: Build Windows binary
|
|
||||||
needs: create-release
|
|
||||||
runs-on: windows-2022
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v3
|
|
||||||
with:
|
|
||||||
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
|
|
||||||
|
|
||||||
- name: Setup Python
|
|
||||||
uses: actions/setup-python@v4
|
|
||||||
with:
|
|
||||||
python-version: '3.11'
|
|
||||||
|
|
||||||
- name: Build artifacts
|
|
||||||
run: |
|
|
||||||
python -m pip install -r requirements.txt
|
|
||||||
python -m pip install pyinstaller==5.13.0
|
|
||||||
cd src
|
|
||||||
python build_package.py
|
|
||||||
|
|
||||||
- name: Upload release artifacts
|
|
||||||
uses: alexellis/upload-assets@0.4.0
|
|
||||||
env:
|
|
||||||
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
|
||||||
with:
|
|
||||||
asset_paths: '["./dist/flaresolverr_*"]'
|
|
||||||
|
|||||||
4
.gitignore
vendored
4
.gitignore
vendored
@ -25,7 +25,6 @@ __pycache__/
|
|||||||
build/
|
build/
|
||||||
develop-eggs/
|
develop-eggs/
|
||||||
dist/
|
dist/
|
||||||
dist_chrome/
|
|
||||||
downloads/
|
downloads/
|
||||||
eggs/
|
eggs/
|
||||||
.eggs/
|
.eggs/
|
||||||
@ -124,6 +123,3 @@ venv.bak/
|
|||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.dmypy.json
|
.dmypy.json
|
||||||
dmypy.json
|
dmypy.json
|
||||||
|
|
||||||
# node
|
|
||||||
node_modules/
|
|
||||||
185
CHANGELOG.md
185
CHANGELOG.md
@ -1,189 +1,6 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
## v3.3.24 (2025/06/04)
|
## v3.1.0 (upcoming)
|
||||||
* Remove hidden character
|
|
||||||
|
|
||||||
## v3.3.23 (2025/06/04)
|
|
||||||
* Update base image to bookworm. Thanks @rwjack
|
|
||||||
|
|
||||||
## v3.3.22 (2025/06/03)
|
|
||||||
* Disable search engine choice screen
|
|
||||||
* Fix headless=false stalling. Thanks @MAKMED1337
|
|
||||||
* Change from click to keys. Thanks @sh4dowb
|
|
||||||
* Don't open devtools
|
|
||||||
* Bump Chromium to v137 for build
|
|
||||||
* Bump requirements
|
|
||||||
|
|
||||||
## v3.3.21 (2024/06/26)
|
|
||||||
* Add challenge selector to catch reloading page on non-English systems
|
|
||||||
* Escape values for generated form used in request.post. Thanks @mynameisbogdan
|
|
||||||
|
|
||||||
## v3.3.20 (2024/06/21)
|
|
||||||
* maxTimeout should always be int
|
|
||||||
* Check not running in Docker before logging version_main error
|
|
||||||
* Update Cloudflare challenge and checkbox selectors. Thanks @tenettow & @21hsmw
|
|
||||||
|
|
||||||
## v3.3.19 (2024/05/23)
|
|
||||||
* Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw
|
|
||||||
|
|
||||||
## v3.3.18 (2024/05/20)
|
|
||||||
|
|
||||||
* Fix LANG ENV for Linux
|
|
||||||
* Fix Chrome v124+ not closing on Windows. Thanks @RileyXX
|
|
||||||
|
|
||||||
## v3.3.17 (2024/04/09)
|
|
||||||
|
|
||||||
* Fix file descriptor leak in service on quit(). Thanks @zkulis
|
|
||||||
|
|
||||||
## v3.3.16 (2024/02/28)
|
|
||||||
|
|
||||||
* Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo
|
|
||||||
* Add FreeBSD support. Thanks @Asthowen
|
|
||||||
* Use headless configuration properly. Thanks @hashworks
|
|
||||||
|
|
||||||
## v3.3.15 (2024/02/20)
|
|
||||||
|
|
||||||
* Fix looping challenges
|
|
||||||
|
|
||||||
## v3.3.14-hotfix2 (2024/02/17)
|
|
||||||
|
|
||||||
* Hotfix 2 - bad Chromium build, instances failed to terminate
|
|
||||||
|
|
||||||
## v3.3.14-hotfix (2024/02/17)
|
|
||||||
|
|
||||||
* Hotfix for Linux build - some Chrome files no longer exist
|
|
||||||
|
|
||||||
## v3.3.14 (2024/02/17)
|
|
||||||
|
|
||||||
* Update Chrome downloads. Thanks @opemvbs
|
|
||||||
|
|
||||||
## v3.3.13 (2024/01/07)
|
|
||||||
|
|
||||||
* Fix too many open files error
|
|
||||||
|
|
||||||
## v3.3.12 (2023/12/15)
|
|
||||||
|
|
||||||
* Fix looping challenges and invalid cookies
|
|
||||||
|
|
||||||
## v3.3.11 (2023/12/11)
|
|
||||||
|
|
||||||
* Update UC 3.5.4 & Selenium 4.15.2. Thanks @txtsd
|
|
||||||
|
|
||||||
## v3.3.10 (2023/11/14)
|
|
||||||
|
|
||||||
* Add LANG ENV - resolves issues with YGGtorrent
|
|
||||||
|
|
||||||
## v3.3.9 (2023/11/13)
|
|
||||||
|
|
||||||
* Fix for Docker build, capture TypeError
|
|
||||||
|
|
||||||
## v3.3.8 (2023/11/13)
|
|
||||||
|
|
||||||
* Fix headless=true for Chrome 117+. Thanks @NabiKAZ
|
|
||||||
* Support running Chrome 119 from source. Thanks @koleg and @Chris7X
|
|
||||||
* Fix "OSError: [WinError 6] The handle is invalid" on exit. Thanks @enesgorkemgenc
|
|
||||||
|
|
||||||
## v3.3.7 (2023/11/05)
|
|
||||||
|
|
||||||
* Bump to rebuild. Thanks @JoachimDorchies
|
|
||||||
|
|
||||||
## v3.3.6 (2023/09/15)
|
|
||||||
|
|
||||||
* Update checkbox selector, again
|
|
||||||
|
|
||||||
## v3.3.5 (2023/09/13)
|
|
||||||
|
|
||||||
* Change checkbox selector, support languages other than English
|
|
||||||
|
|
||||||
## v3.3.4 (2023/09/02)
|
|
||||||
|
|
||||||
* Update checkbox selector
|
|
||||||
|
|
||||||
## v3.3.3 (2023/08/31)
|
|
||||||
|
|
||||||
* Update undetected_chromedriver to v3.5.3
|
|
||||||
|
|
||||||
## v3.3.2 (2023/08/03)
|
|
||||||
|
|
||||||
* Fix URL domain in Prometheus exporter
|
|
||||||
|
|
||||||
## v3.3.1 (2023/08/03)
|
|
||||||
|
|
||||||
* Fix for Cloudflare verify checkbox
|
|
||||||
* Fix HEADLESS=false in Windows binary
|
|
||||||
* Fix Prometheus exporter for management and health endpoints
|
|
||||||
* Remove misleading stack trace when the verify checkbox is not found
|
|
||||||
* Revert "Update base Docker image to Debian Bookworm" #849
|
|
||||||
* Revert "Install Chromium 115 from Debian testing" #849
|
|
||||||
|
|
||||||
## v3.3.0 (2023/08/02)
|
|
||||||
|
|
||||||
* Fix for new Cloudflare detection. Thanks @cedric-bour for #845
|
|
||||||
* Add support for proxy authentication username/password. Thanks @jacobprice808 for #807
|
|
||||||
* Implement Prometheus metrics
|
|
||||||
* Fix Chromium Driver for Chrome / Chromium version > 114
|
|
||||||
* Use Chromium 115 in binary packages (Windows and Linux)
|
|
||||||
* Install Chromium 115 from Debian testing (Docker)
|
|
||||||
* Update base Docker image to Debian Bookworm
|
|
||||||
* Update Selenium 4.11.2
|
|
||||||
* Update pyinstaller 5.13.0
|
|
||||||
* Add more traces in build_package.py
|
|
||||||
|
|
||||||
## v3.2.2 (2023/07/16)
|
|
||||||
|
|
||||||
* Workaround for updated 'verify you are human' check
|
|
||||||
|
|
||||||
## v3.2.1 (2023/06/10)
|
|
||||||
|
|
||||||
* Kill dead Chrome processes in Windows
|
|
||||||
* Fix Chrome GL erros in ASUSTOR NAS
|
|
||||||
|
|
||||||
## v3.2.0 (2023/05/23)
|
|
||||||
|
|
||||||
* Support "proxy" param in requests and sessions
|
|
||||||
* Support "cookies" param in requests
|
|
||||||
* Fix Chromium exec permissions in Linux package
|
|
||||||
* Update Python dependencies
|
|
||||||
|
|
||||||
## v3.1.2 (2023/04/02)
|
|
||||||
|
|
||||||
* Fix headless mode in macOS
|
|
||||||
* Remove redundant artifact from Windows binary package
|
|
||||||
* Bump Selenium dependency
|
|
||||||
|
|
||||||
## v3.1.1 (2023/03/25)
|
|
||||||
|
|
||||||
* Distribute binary executables in compressed package
|
|
||||||
* Add icon for binary executable
|
|
||||||
* Include information about supported architectures in the readme
|
|
||||||
* Check Python version on start
|
|
||||||
|
|
||||||
## v3.1.0 (2023/03/20)
|
|
||||||
|
|
||||||
* Build binaries for Linux x64 and Windows x64
|
|
||||||
* Sessions with auto-creation on fetch request and TTL
|
|
||||||
* Fix error trace: Crash Reports/pending No such file or directory
|
|
||||||
* Fix Waitress server error with asyncore_use_poll=true
|
|
||||||
* Attempt to fix Docker ARM32 build
|
|
||||||
* Print platform information on start up
|
|
||||||
* Add Fairlane challenge selector
|
|
||||||
* Update DDOS-GUARD title
|
|
||||||
* Update dependencies
|
|
||||||
|
|
||||||
## v3.0.4 (2023/03/07)
|
|
||||||
|
|
||||||
* Click on the Cloudflare's 'Verify you are human' button if necessary
|
|
||||||
|
|
||||||
## v3.0.3 (2023/03/06)
|
|
||||||
|
|
||||||
* Update undetected_chromedriver version to 3.4.6
|
|
||||||
|
|
||||||
## v3.0.2 (2023/01/08)
|
|
||||||
|
|
||||||
* Detect Cloudflare blocked access
|
|
||||||
* Check Chrome / Chromium web browser is installed correctly
|
|
||||||
|
|
||||||
## v3.0.1 (2023/01/06)
|
|
||||||
|
|
||||||
* Kill Chromium processes properly to avoid defunct/zombie processes
|
* Kill Chromium processes properly to avoid defunct/zombie processes
|
||||||
* Update undetected-chromedriver
|
* Update undetected-chromedriver
|
||||||
|
|||||||
21
Dockerfile
21
Dockerfile
@ -1,4 +1,4 @@
|
|||||||
FROM python:3.11-slim-bookworm as builder
|
FROM python:3.11-slim-bullseye as builder
|
||||||
|
|
||||||
# Build dummy packages to skip installing them and their dependencies
|
# Build dummy packages to skip installing them and their dependencies
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
@ -12,7 +12,7 @@ RUN apt-get update \
|
|||||||
&& equivs-build adwaita-icon-theme \
|
&& equivs-build adwaita-icon-theme \
|
||||||
&& mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
|
&& mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
|
||||||
|
|
||||||
FROM python:3.11-slim-bookworm
|
FROM python:3.11-slim-bullseye
|
||||||
|
|
||||||
# Copy dummy packages
|
# Copy dummy packages
|
||||||
COPY --from=builder /*.deb /
|
COPY --from=builder /*.deb /
|
||||||
@ -30,7 +30,7 @@ RUN dpkg -i /libgl1-mesa-dri.deb \
|
|||||||
# Install dependencies
|
# Install dependencies
|
||||||
&& apt-get update \
|
&& apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
|
&& apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
|
||||||
procps curl vim xauth \
|
procps curl vim \
|
||||||
# Remove temporary files and hardware decoding libraries
|
# Remove temporary files and hardware decoding libraries
|
||||||
&& rm -rf /var/lib/apt/lists/* \
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
&& rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
|
&& rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
|
||||||
@ -48,13 +48,10 @@ RUN pip install -r requirements.txt \
|
|||||||
|
|
||||||
USER flaresolverr
|
USER flaresolverr
|
||||||
|
|
||||||
RUN mkdir -p "/app/.config/chromium/Crash Reports/pending"
|
|
||||||
|
|
||||||
COPY src .
|
COPY src .
|
||||||
COPY package.json ../
|
COPY package.json ../
|
||||||
|
|
||||||
EXPOSE 8191
|
EXPOSE 8191
|
||||||
EXPOSE 8192
|
|
||||||
|
|
||||||
# dumb-init avoids zombie chromium processes
|
# dumb-init avoids zombie chromium processes
|
||||||
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||||
@ -62,17 +59,11 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
|||||||
CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
|
CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
|
||||||
|
|
||||||
# Local build
|
# Local build
|
||||||
# docker build -t ngosang/flaresolverr:3.3.24 .
|
# docker build -t ngosang/flaresolverr:3.0.0 .
|
||||||
# docker run -p 8191:8191 ngosang/flaresolverr:3.3.24
|
# docker run -p 8191:8191 ngosang/flaresolverr:3.0.0
|
||||||
|
|
||||||
# Multi-arch build
|
# Multi-arch build
|
||||||
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
||||||
# docker buildx create --use
|
# docker buildx create --use
|
||||||
# docker buildx build -t ngosang/flaresolverr:3.3.24 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
|
# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
|
||||||
# add --push to publish in DockerHub
|
# add --push to publish in DockerHub
|
||||||
|
|
||||||
# Test multi-arch build
|
|
||||||
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
|
||||||
# docker buildx create --use
|
|
||||||
# docker buildx build -t ngosang/flaresolverr:3.3.24 --platform linux/arm/v7 --load .
|
|
||||||
# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.24
|
|
||||||
|
|||||||
147
README.md
147
README.md
@ -45,8 +45,7 @@ Supported architectures are:
|
|||||||
| ARM32 | linux/arm/v7 |
|
| ARM32 | linux/arm/v7 |
|
||||||
| ARM64 | linux/arm64 |
|
| ARM64 | linux/arm64 |
|
||||||
|
|
||||||
We provide a `docker-compose.yml` configuration file. Clone this repository and execute
|
We provide a `docker-compose.yml` configuration file. Clone this repository and execute `docker-compose up -d` to start
|
||||||
`docker-compose up -d` _(Compose V1)_ or `docker compose up -d` _(Compose V2)_ to start
|
|
||||||
the container.
|
the container.
|
||||||
|
|
||||||
If you prefer the `docker cli` execute the following command.
|
If you prefer the `docker cli` execute the following command.
|
||||||
@ -65,76 +64,38 @@ Remember to restart the Docker daemon and the container after the update.
|
|||||||
|
|
||||||
### Precompiled binaries
|
### Precompiled binaries
|
||||||
|
|
||||||
> **Warning**
|
|
||||||
> Precompiled binaries are only available for x64 architecture. For other architectures see Docker images.
|
|
||||||
|
|
||||||
This is the recommended way for Windows users.
|
This is the recommended way for Windows users.
|
||||||
* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
|
* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
|
||||||
|
* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
|
||||||
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
||||||
|
|
||||||
### From source code
|
### From source code
|
||||||
|
|
||||||
> **Warning**
|
This is the recommended way for macOS users and for developers.
|
||||||
> Installing from source code only works for x64 architecture. For other architectures see Docker images.
|
* Install [Python 3.10](https://www.python.org/downloads/).
|
||||||
|
* Install [Chrome](https://www.google.com/intl/en_us/chrome/) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) web browser.
|
||||||
* Install [Python 3.11](https://www.python.org/downloads/).
|
* (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
|
||||||
* Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
|
|
||||||
* (Only in Linux) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
|
|
||||||
* (Only in macOS) Install [XQuartz](https://www.xquartz.org/) package.
|
|
||||||
* Clone this repository and open a shell in that path.
|
* Clone this repository and open a shell in that path.
|
||||||
* Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
|
* Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
|
||||||
* Run `python src/flaresolverr.py` command to start FlareSolverr.
|
* Run `python src/flaresolverr.py` command to start FlareSolverr.
|
||||||
|
|
||||||
### From source code (FreeBSD/TrueNAS CORE)
|
|
||||||
|
|
||||||
* Run `pkg install chromium python311 py311-pip xorg-vfbserver` command to install the required dependencies.
|
|
||||||
* Clone this repository and open a shell in that path.
|
|
||||||
* Run `python3.11 -m pip install -r requirements.txt` command to install FlareSolverr dependencies.
|
|
||||||
* Run `python3.11 src/flaresolverr.py` command to start FlareSolverr.
|
|
||||||
|
|
||||||
### Systemd service
|
### Systemd service
|
||||||
|
|
||||||
We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
|
We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
Example Bash request:
|
Example request:
|
||||||
```bash
|
```bash
|
||||||
curl -L -X POST 'http://localhost:8191/v1' \
|
curl -L -X POST 'http://localhost:8191/v1' \
|
||||||
-H 'Content-Type: application/json' \
|
-H 'Content-Type: application/json' \
|
||||||
--data-raw '{
|
--data-raw '{
|
||||||
"cmd": "request.get",
|
"cmd": "request.get",
|
||||||
"url": "http://www.google.com/",
|
"url":"http://www.google.com/",
|
||||||
"maxTimeout": 60000
|
"maxTimeout": 60000
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
Example Python request:
|
|
||||||
```py
|
|
||||||
import requests
|
|
||||||
|
|
||||||
url = "http://localhost:8191/v1"
|
|
||||||
headers = {"Content-Type": "application/json"}
|
|
||||||
data = {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": "http://www.google.com/",
|
|
||||||
"maxTimeout": 60000
|
|
||||||
}
|
|
||||||
response = requests.post(url, headers=headers, json=data)
|
|
||||||
print(response.text)
|
|
||||||
```
|
|
||||||
|
|
||||||
Example PowerShell request:
|
|
||||||
```ps1
|
|
||||||
$body = @{
|
|
||||||
cmd = "request.get"
|
|
||||||
url = "http://www.google.com/"
|
|
||||||
maxTimeout = 60000
|
|
||||||
} | ConvertTo-Json
|
|
||||||
|
|
||||||
irm -UseBasicParsing 'http://localhost:8191/v1' -Headers @{"Content-Type"="application/json"} -Method Post -Body $body
|
|
||||||
```
|
|
||||||
|
|
||||||
### Commands
|
### Commands
|
||||||
|
|
||||||
#### + `sessions.create`
|
#### + `sessions.create`
|
||||||
@ -145,10 +106,10 @@ cookies for the browser to use.
|
|||||||
|
|
||||||
This also speeds up the requests since it won't have to launch a new browser instance for every request.
|
This also speeds up the requests since it won't have to launch a new browser instance for every request.
|
||||||
|
|
||||||
| Parameter | Notes |
|
| Parameter | Notes |
|
||||||
|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. |
|
| session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned. |
|
||||||
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. Eg: `"proxy": {"url": "http://127.0.0.1:8888", "username": "testuser", "password": "testpass"}` |
|
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. |
|
||||||
|
|
||||||
#### + `sessions.list`
|
#### + `sessions.list`
|
||||||
|
|
||||||
@ -179,18 +140,16 @@ session. When you no longer need to use a session you should make sure to close
|
|||||||
|
|
||||||
#### + `request.get`
|
#### + `request.get`
|
||||||
|
|
||||||
| Parameter | Notes |
|
| Parameter | Notes |
|
||||||
|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| url | Mandatory |
|
| url | Mandatory |
|
||||||
| session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
|
| session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
|
||||||
| session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes. |
|
| maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
|
||||||
| maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
|
| cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format. |
|
||||||
| cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. |
|
| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
|
||||||
| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
|
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
|
||||||
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
|
|
||||||
|
|
||||||
> **Warning**
|
:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
|
||||||
> If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
|
|
||||||
|
|
||||||
Example response from running the `curl` above:
|
Example response from running the `curl` above:
|
||||||
|
|
||||||
@ -261,63 +220,33 @@ This is the same as `request.get` but it takes one more param:
|
|||||||
|
|
||||||
## Environment variables
|
## Environment variables
|
||||||
|
|
||||||
| Name | Default | Notes |
|
| Name | Default | Notes |
|
||||||
|--------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|-----------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. |
|
| LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information. |
|
||||||
| LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. |
|
| LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level. |
|
||||||
| CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. |
|
| CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. |
|
||||||
| TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. |
|
| TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. |
|
||||||
| LANG | none | Language used in the web browser. Example: `LANG=en_GB`. |
|
| HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. |
|
||||||
| HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. |
|
| BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
|
||||||
| BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
|
| TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. |
|
||||||
| TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country. |
|
| PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. |
|
||||||
| PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. |
|
| HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. |
|
||||||
| HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. |
|
|
||||||
| PROMETHEUS_ENABLED | false | Enable Prometheus exporter. See the Prometheus section below. |
|
|
||||||
| PROMETHEUS_PORT | 8192 | Listening port for Prometheus exporter. See the Prometheus section below. |
|
|
||||||
|
|
||||||
Environment variables are set differently depending on the operating system. Some examples:
|
Environment variables are set differently depending on the operating system. Some examples:
|
||||||
* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
|
* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
|
||||||
* Linux: Run `export LOG_LEVEL=debug` and then run `flaresolverr` in the same shell.
|
* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
|
||||||
* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then run `flaresolverr.exe` in the same shell.
|
* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
|
||||||
|
|
||||||
## Prometheus exporter
|
|
||||||
|
|
||||||
The Prometheus exporter for FlareSolverr is disabled by default. It can be enabled with the environment variable `PROMETHEUS_ENABLED`. If you are using Docker make sure you expose the `PROMETHEUS_PORT`.
|
|
||||||
|
|
||||||
Example metrics:
|
|
||||||
```shell
|
|
||||||
# HELP flaresolverr_request_total Total requests with result
|
|
||||||
# TYPE flaresolverr_request_total counter
|
|
||||||
flaresolverr_request_total{domain="nowsecure.nl",result="solved"} 1.0
|
|
||||||
# HELP flaresolverr_request_created Total requests with result
|
|
||||||
# TYPE flaresolverr_request_created gauge
|
|
||||||
flaresolverr_request_created{domain="nowsecure.nl",result="solved"} 1.690141657157109e+09
|
|
||||||
# HELP flaresolverr_request_duration Request duration in seconds
|
|
||||||
# TYPE flaresolverr_request_duration histogram
|
|
||||||
flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="0.0"} 0.0
|
|
||||||
flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="10.0"} 1.0
|
|
||||||
flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="25.0"} 1.0
|
|
||||||
flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="50.0"} 1.0
|
|
||||||
flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="+Inf"} 1.0
|
|
||||||
flaresolverr_request_duration_count{domain="nowsecure.nl"} 1.0
|
|
||||||
flaresolverr_request_duration_sum{domain="nowsecure.nl"} 5.858
|
|
||||||
# HELP flaresolverr_request_duration_created Request duration in seconds
|
|
||||||
# TYPE flaresolverr_request_duration_created gauge
|
|
||||||
flaresolverr_request_duration_created{domain="nowsecure.nl"} 1.6901416571570296e+09
|
|
||||||
```
|
|
||||||
|
|
||||||
## Captcha Solvers
|
## Captcha Solvers
|
||||||
|
|
||||||
> **Warning**
|
:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
|
||||||
> At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
|
|
||||||
|
|
||||||
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
|
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
|
||||||
solve a captcha.
|
solve a captcha.
|
||||||
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
|
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
|
||||||
|
|
||||||
FlareSolverr can be customized to solve the CAPTCHA automatically by setting the environment variable `CAPTCHA_SOLVER`
|
FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
|
||||||
to the file name of one of the adapters inside the `/captcha` directory.
|
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
|
||||||
|
|
||||||
## Related projects
|
## Related projects
|
||||||
|
|
||||||
|
|||||||
@ -1,19 +0,0 @@
|
|||||||
[Unit]
|
|
||||||
Description=FlareSolverr
|
|
||||||
After=network.target
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
SyslogIdentifier=flaresolverr
|
|
||||||
Restart=always
|
|
||||||
RestartSec=5
|
|
||||||
Type=simple
|
|
||||||
User=flaresolverr
|
|
||||||
Group=flaresolverr
|
|
||||||
Environment="LOG_LEVEL=info"
|
|
||||||
Environment="CAPTCHA_SOLVER=none"
|
|
||||||
WorkingDirectory=/opt/flaresolverr
|
|
||||||
ExecStart=/opt/flaresolverr/flaresolverr
|
|
||||||
TimeoutStopSec=30
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "flaresolverr",
|
"name": "flaresolverr",
|
||||||
"version": "3.3.24",
|
"version": "3.0.1",
|
||||||
"description": "Proxy server to bypass Cloudflare protection",
|
"description": "Proxy server to bypass Cloudflare protection",
|
||||||
"author": "Diego Heras (ngosang / ngosang@hotmail.es)",
|
"author": "Diego Heras (ngosang / ngosang@hotmail.es)",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
|
|||||||
@ -1,13 +1,9 @@
|
|||||||
bottle==0.12.25
|
bottle==0.12.23
|
||||||
waitress==3.0.1
|
waitress==2.1.2
|
||||||
selenium==4.15.2
|
selenium==4.7.2
|
||||||
func-timeout==4.3.5
|
func-timeout==4.3.5
|
||||||
prometheus-client==0.17.1
|
|
||||||
# required by undetected_chromedriver
|
# required by undetected_chromedriver
|
||||||
requests==2.32.3
|
requests==2.28.1
|
||||||
certifi==2024.07.04
|
websockets==10.4
|
||||||
websockets==11.0.3
|
# only required for linux
|
||||||
# only required for linux and macos
|
xvfbwrapper==0.2.9
|
||||||
xvfbwrapper==0.2.9; platform_system != "Windows"
|
|
||||||
# only required for windows
|
|
||||||
pefile==2023.2.7; platform_system == "Windows"
|
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 8.8 KiB |
@ -1,66 +0,0 @@
|
|||||||
import logging
|
|
||||||
import os
|
|
||||||
import urllib.parse
|
|
||||||
|
|
||||||
from bottle import request
|
|
||||||
from dtos import V1RequestBase, V1ResponseBase
|
|
||||||
from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
|
|
||||||
|
|
||||||
PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
|
|
||||||
PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
|
|
||||||
|
|
||||||
|
|
||||||
def setup():
|
|
||||||
if PROMETHEUS_ENABLED:
|
|
||||||
start_metrics_http_server(PROMETHEUS_PORT)
|
|
||||||
|
|
||||||
|
|
||||||
def prometheus_plugin(callback):
|
|
||||||
"""
|
|
||||||
Bottle plugin to expose Prometheus metrics
|
|
||||||
http://bottlepy.org/docs/dev/plugindev.html
|
|
||||||
"""
|
|
||||||
def wrapper(*args, **kwargs):
|
|
||||||
actual_response = callback(*args, **kwargs)
|
|
||||||
|
|
||||||
if PROMETHEUS_ENABLED:
|
|
||||||
try:
|
|
||||||
export_metrics(actual_response)
|
|
||||||
except Exception as e:
|
|
||||||
logging.warning("Error exporting metrics: " + str(e))
|
|
||||||
|
|
||||||
return actual_response
|
|
||||||
|
|
||||||
def export_metrics(actual_response):
|
|
||||||
res = V1ResponseBase(actual_response)
|
|
||||||
|
|
||||||
if res.startTimestamp is None or res.endTimestamp is None:
|
|
||||||
# skip management and healthcheck endpoints
|
|
||||||
return
|
|
||||||
|
|
||||||
domain = "unknown"
|
|
||||||
if res.solution and res.solution.url:
|
|
||||||
domain = parse_domain_url(res.solution.url)
|
|
||||||
else:
|
|
||||||
# timeout error
|
|
||||||
req = V1RequestBase(request.json)
|
|
||||||
if req.url:
|
|
||||||
domain = parse_domain_url(req.url)
|
|
||||||
|
|
||||||
run_time = (res.endTimestamp - res.startTimestamp) / 1000
|
|
||||||
REQUEST_DURATION.labels(domain=domain).observe(run_time)
|
|
||||||
|
|
||||||
result = "unknown"
|
|
||||||
if res.message == "Challenge solved!":
|
|
||||||
result = "solved"
|
|
||||||
elif res.message == "Challenge not detected!":
|
|
||||||
result = "not_detected"
|
|
||||||
elif res.message.startswith("Error"):
|
|
||||||
result = "error"
|
|
||||||
REQUEST_COUNTER.labels(domain=domain, result=result).inc()
|
|
||||||
|
|
||||||
def parse_domain_url(url):
|
|
||||||
parsed_url = urllib.parse.urlparse(url)
|
|
||||||
return parsed_url.hostname
|
|
||||||
|
|
||||||
return wrapper
|
|
||||||
@ -1,110 +0,0 @@
|
|||||||
import os
|
|
||||||
import platform
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import zipfile
|
|
||||||
|
|
||||||
import requests
|
|
||||||
|
|
||||||
|
|
||||||
def clean_files():
|
|
||||||
try:
|
|
||||||
shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def download_chromium():
|
|
||||||
# https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
|
|
||||||
revision = "1453032" if os.name == 'nt' else '1453031'
|
|
||||||
arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64'
|
|
||||||
dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
|
|
||||||
dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
|
|
||||||
dl_path_folder = os.path.join(dl_path, dl_file)
|
|
||||||
dl_path_zip = dl_path_folder + '.zip'
|
|
||||||
|
|
||||||
# response = requests.get(
|
|
||||||
# f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
|
|
||||||
# timeout=30)
|
|
||||||
# revision = response.text.strip()
|
|
||||||
print("Downloading revision: " + revision)
|
|
||||||
|
|
||||||
os.mkdir(dl_path)
|
|
||||||
with requests.get(
|
|
||||||
f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
|
|
||||||
stream=True) as r:
|
|
||||||
r.raise_for_status()
|
|
||||||
with open(dl_path_zip, 'wb') as f:
|
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
|
||||||
f.write(chunk)
|
|
||||||
print("File downloaded: " + dl_path_zip)
|
|
||||||
with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
|
|
||||||
zip_ref.extractall(dl_path)
|
|
||||||
os.remove(dl_path_zip)
|
|
||||||
|
|
||||||
chrome_path = os.path.join(dl_path, "chrome")
|
|
||||||
shutil.move(dl_path_folder, chrome_path)
|
|
||||||
print("Extracted in: " + chrome_path)
|
|
||||||
|
|
||||||
if os.name != 'nt':
|
|
||||||
# Give executable permissions for *nix
|
|
||||||
# file * | grep executable | cut -d: -f1
|
|
||||||
print("Giving executable permissions...")
|
|
||||||
execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'xdg-mime', 'xdg-settings']
|
|
||||||
for exec_file in execs:
|
|
||||||
exec_path = os.path.join(chrome_path, exec_file)
|
|
||||||
os.chmod(exec_path, 0o755)
|
|
||||||
|
|
||||||
|
|
||||||
def run_pyinstaller():
|
|
||||||
sep = ';' if os.name == 'nt' else ':'
|
|
||||||
result = subprocess.run([sys.executable, "-m", "PyInstaller",
|
|
||||||
"--icon", "resources/flaresolverr_logo.ico",
|
|
||||||
"--add-data", f"package.json{sep}.",
|
|
||||||
"--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
|
|
||||||
os.path.join("src", "flaresolverr.py")],
|
|
||||||
cwd=os.pardir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
||||||
if result.returncode != 0:
|
|
||||||
print(result.stderr.decode('utf-8'))
|
|
||||||
raise Exception("Error running pyInstaller")
|
|
||||||
|
|
||||||
|
|
||||||
def compress_package():
|
|
||||||
dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
|
|
||||||
package_folder = os.path.join(dist_folder, 'package')
|
|
||||||
shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
|
|
||||||
print("Package folder: " + package_folder)
|
|
||||||
|
|
||||||
compr_format = 'zip' if os.name == 'nt' else 'gztar'
|
|
||||||
compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
|
|
||||||
compr_file_path = os.path.join(dist_folder, compr_file_name)
|
|
||||||
shutil.make_archive(compr_file_path, compr_format, package_folder)
|
|
||||||
print("Compressed file path: " + compr_file_path)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
print("Building package...")
|
|
||||||
print("Platform: " + platform.platform())
|
|
||||||
|
|
||||||
print("Cleaning previous build...")
|
|
||||||
clean_files()
|
|
||||||
|
|
||||||
print("Downloading Chromium...")
|
|
||||||
download_chromium()
|
|
||||||
|
|
||||||
print("Building pyinstaller executable... ")
|
|
||||||
run_pyinstaller()
|
|
||||||
|
|
||||||
print("Compressing package... ")
|
|
||||||
compress_package()
|
|
||||||
|
|
||||||
# NOTE: python -m pip install pyinstaller
|
|
||||||
@ -33,7 +33,6 @@ class V1RequestBase(object):
|
|||||||
maxTimeout: int = None
|
maxTimeout: int = None
|
||||||
proxy: dict = None
|
proxy: dict = None
|
||||||
session: str = None
|
session: str = None
|
||||||
session_ttl_minutes: int = None
|
|
||||||
headers: list = None # deprecated v2.0.0, not used
|
headers: list = None # deprecated v2.0.0, not used
|
||||||
userAgent: str = None # deprecated v2.0.0, not used
|
userAgent: str = None # deprecated v2.0.0, not used
|
||||||
|
|
||||||
@ -52,8 +51,6 @@ class V1ResponseBase(object):
|
|||||||
# V1ResponseBase
|
# V1ResponseBase
|
||||||
status: str = None
|
status: str = None
|
||||||
message: str = None
|
message: str = None
|
||||||
session: str = None
|
|
||||||
sessions: list[str] = None
|
|
||||||
startTimestamp: int = None
|
startTimestamp: int = None
|
||||||
endTimestamp: int = None
|
endTimestamp: int = None
|
||||||
version: str = None
|
version: str = None
|
||||||
|
|||||||
@ -3,13 +3,11 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import certifi
|
from bottle import run, response, Bottle, request
|
||||||
from bottle import run, response, Bottle, request, ServerAdapter
|
|
||||||
|
|
||||||
from bottle_plugins.error_plugin import error_plugin
|
from bottle_plugins.error_plugin import error_plugin
|
||||||
from bottle_plugins.logger_plugin import logger_plugin
|
from bottle_plugins.logger_plugin import logger_plugin
|
||||||
from bottle_plugins import prometheus_plugin
|
from dtos import IndexResponse, V1RequestBase
|
||||||
from dtos import V1RequestBase
|
|
||||||
import flaresolverr_service
|
import flaresolverr_service
|
||||||
import utils
|
import utils
|
||||||
|
|
||||||
@ -25,6 +23,10 @@ class JSONErrorBottle(Bottle):
|
|||||||
|
|
||||||
app = JSONErrorBottle()
|
app = JSONErrorBottle()
|
||||||
|
|
||||||
|
# plugin order is important
|
||||||
|
app.install(logger_plugin)
|
||||||
|
app.install(error_plugin)
|
||||||
|
|
||||||
|
|
||||||
@app.route('/')
|
@app.route('/')
|
||||||
def index():
|
def index():
|
||||||
@ -58,22 +60,6 @@ def controller_v1():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# check python version
|
|
||||||
if sys.version_info < (3, 9):
|
|
||||||
raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
|
|
||||||
|
|
||||||
# fix for HEADLESS=false in Windows binary
|
|
||||||
# https://stackoverflow.com/a/27694505
|
|
||||||
if os.name == 'nt':
|
|
||||||
import multiprocessing
|
|
||||||
multiprocessing.freeze_support()
|
|
||||||
|
|
||||||
# fix ssl certificates for compiled binaries
|
|
||||||
# https://github.com/pyinstaller/pyinstaller/issues/7229
|
|
||||||
# https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
|
|
||||||
os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
|
|
||||||
os.environ["SSL_CERT_FILE"] = certifi.where()
|
|
||||||
|
|
||||||
# validate configuration
|
# validate configuration
|
||||||
log_level = os.environ.get('LOG_LEVEL', 'info').upper()
|
log_level = os.environ.get('LOG_LEVEL', 'info').upper()
|
||||||
log_html = utils.get_config_log_html()
|
log_html = utils.get_config_log_html()
|
||||||
@ -101,25 +87,9 @@ if __name__ == "__main__":
|
|||||||
logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
|
logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
|
||||||
logging.debug('Debug log enabled')
|
logging.debug('Debug log enabled')
|
||||||
|
|
||||||
# Get current OS for global variable
|
|
||||||
utils.get_current_platform()
|
|
||||||
|
|
||||||
# test browser installation
|
# test browser installation
|
||||||
flaresolverr_service.test_browser_installation()
|
flaresolverr_service.test_browser_installation()
|
||||||
|
|
||||||
# start bootle plugins
|
|
||||||
# plugin order is important
|
|
||||||
app.install(logger_plugin)
|
|
||||||
app.install(error_plugin)
|
|
||||||
prometheus_plugin.setup()
|
|
||||||
app.install(prometheus_plugin.prometheus_plugin)
|
|
||||||
|
|
||||||
# start webserver
|
# start webserver
|
||||||
# default server 'wsgiref' does not support concurrent requests
|
# default server 'wsgiref' does not support concurrent requests
|
||||||
# https://github.com/FlareSolverr/FlareSolverr/issues/680
|
run(app, host=server_host, port=server_port, quiet=True, server='waitress')
|
||||||
# https://github.com/Pylons/waitress/issues/31
|
|
||||||
class WaitressServerPoll(ServerAdapter):
|
|
||||||
def run(self, handler):
|
|
||||||
from waitress import serve
|
|
||||||
serve(handler, host=self.host, port=self.port, asyncore_use_poll=True)
|
|
||||||
run(app, host=server_host, port=server_port, quiet=True, server=WaitressServerPoll)
|
|
||||||
|
|||||||
@ -1,79 +1,44 @@
|
|||||||
import logging
|
import logging
|
||||||
import platform
|
|
||||||
import sys
|
|
||||||
import time
|
import time
|
||||||
from datetime import timedelta
|
from urllib.parse import unquote
|
||||||
from html import escape
|
|
||||||
from urllib.parse import unquote, quote
|
|
||||||
|
|
||||||
from func_timeout import FunctionTimedOut, func_timeout
|
from func_timeout import func_timeout, FunctionTimedOut
|
||||||
from selenium.common import TimeoutException
|
from selenium.common import TimeoutException
|
||||||
from selenium.webdriver.chrome.webdriver import WebDriver
|
from selenium.webdriver.chrome.webdriver import WebDriver
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
from selenium.webdriver.common.keys import Keys
|
|
||||||
from selenium.webdriver.support.expected_conditions import (
|
|
||||||
presence_of_element_located, staleness_of, title_is)
|
|
||||||
from selenium.webdriver.common.action_chains import ActionChains
|
|
||||||
from selenium.webdriver.support.wait import WebDriverWait
|
from selenium.webdriver.support.wait import WebDriverWait
|
||||||
|
from selenium.webdriver.support.expected_conditions import presence_of_element_located, staleness_of, title_is
|
||||||
|
|
||||||
|
from dtos import V1RequestBase, V1ResponseBase, ChallengeResolutionT, ChallengeResolutionResultT, IndexResponse, \
|
||||||
|
HealthResponse, STATUS_OK, STATUS_ERROR
|
||||||
import utils
|
import utils
|
||||||
from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
|
|
||||||
ChallengeResolutionT, HealthResponse, IndexResponse,
|
|
||||||
V1RequestBase, V1ResponseBase)
|
|
||||||
from sessions import SessionsStorage
|
|
||||||
|
|
||||||
ACCESS_DENIED_TITLES = [
|
|
||||||
# Cloudflare
|
|
||||||
'Access denied',
|
|
||||||
# Cloudflare http://bitturk.net/ Firefox
|
|
||||||
'Attention Required! | Cloudflare'
|
|
||||||
]
|
|
||||||
ACCESS_DENIED_SELECTORS = [
|
ACCESS_DENIED_SELECTORS = [
|
||||||
# Cloudflare
|
# Cloudflare
|
||||||
'div.cf-error-title span.cf-code-label span',
|
'div.cf-error-title span.cf-code-label span'
|
||||||
# Cloudflare http://bitturk.net/ Firefox
|
# Cloudflare http://bitturk.net/ Firefox
|
||||||
'#cf-error-details div.cf-error-overview h1'
|
'#cf-error-details div.cf-error-overview h1'
|
||||||
]
|
]
|
||||||
CHALLENGE_TITLES = [
|
CHALLENGE_TITLE = [
|
||||||
# Cloudflare
|
# Cloudflare
|
||||||
'Just a moment...',
|
'Just a moment...',
|
||||||
# DDoS-GUARD
|
# DDoS-GUARD
|
||||||
'DDoS-Guard'
|
'DDOS-GUARD',
|
||||||
]
|
]
|
||||||
CHALLENGE_SELECTORS = [
|
CHALLENGE_SELECTORS = [
|
||||||
# Cloudflare
|
# Cloudflare
|
||||||
'#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring',
|
'#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js',
|
||||||
# Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
|
# Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
|
||||||
'td.info #js_info',
|
'td.info #js_info'
|
||||||
# Fairlane / pararius.com
|
|
||||||
'div.vc div.text-box h2'
|
|
||||||
]
|
]
|
||||||
SHORT_TIMEOUT = 1
|
SHORT_TIMEOUT = 10
|
||||||
SESSIONS_STORAGE = SessionsStorage()
|
|
||||||
|
|
||||||
|
|
||||||
def test_browser_installation():
|
def test_browser_installation():
|
||||||
logging.info("Testing web browser installation...")
|
logging.info("Testing web browser installation...")
|
||||||
logging.info("Platform: " + platform.platform())
|
|
||||||
|
|
||||||
chrome_exe_path = utils.get_chrome_exe_path()
|
|
||||||
if chrome_exe_path is None:
|
|
||||||
logging.error("Chrome / Chromium web browser not installed!")
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
logging.info("Chrome / Chromium path: " + chrome_exe_path)
|
|
||||||
|
|
||||||
chrome_major_version = utils.get_chrome_major_version()
|
|
||||||
if chrome_major_version == '':
|
|
||||||
logging.error("Chrome / Chromium version not detected!")
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
logging.info("Chrome / Chromium major version: " + chrome_major_version)
|
|
||||||
|
|
||||||
logging.info("Launching web browser...")
|
|
||||||
user_agent = utils.get_user_agent()
|
user_agent = utils.get_user_agent()
|
||||||
logging.info("FlareSolverr User-Agent: " + user_agent)
|
logging.info("FlareSolverr User-Agent: " + user_agent)
|
||||||
logging.info("Test successful!")
|
logging.info("Test successful")
|
||||||
|
|
||||||
|
|
||||||
def index_endpoint() -> IndexResponse:
|
def index_endpoint() -> IndexResponse:
|
||||||
@ -121,17 +86,17 @@ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
|
|||||||
logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
|
logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
|
||||||
|
|
||||||
# set default values
|
# set default values
|
||||||
if req.maxTimeout is None or int(req.maxTimeout) < 1:
|
if req.maxTimeout is None or req.maxTimeout < 1:
|
||||||
req.maxTimeout = 60000
|
req.maxTimeout = 60000
|
||||||
|
|
||||||
# execute the command
|
# execute the command
|
||||||
res: V1ResponseBase
|
res: V1ResponseBase
|
||||||
if req.cmd == 'sessions.create':
|
if req.cmd == 'sessions.create':
|
||||||
res = _cmd_sessions_create(req)
|
raise Exception("Not implemented yet.")
|
||||||
elif req.cmd == 'sessions.list':
|
elif req.cmd == 'sessions.list':
|
||||||
res = _cmd_sessions_list(req)
|
raise Exception("Not implemented yet.")
|
||||||
elif req.cmd == 'sessions.destroy':
|
elif req.cmd == 'sessions.destroy':
|
||||||
res = _cmd_sessions_destroy(req)
|
raise Exception("Not implemented yet.")
|
||||||
elif req.cmd == 'request.get':
|
elif req.cmd == 'request.get':
|
||||||
res = _cmd_request_get(req)
|
res = _cmd_request_get(req)
|
||||||
elif req.cmd == 'request.post':
|
elif req.cmd == 'request.post':
|
||||||
@ -178,108 +143,19 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
|
|||||||
return res
|
return res
|
||||||
|
|
||||||
|
|
||||||
def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
|
|
||||||
logging.debug("Creating new session...")
|
|
||||||
|
|
||||||
session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
|
|
||||||
session_id = session.session_id
|
|
||||||
|
|
||||||
if not fresh:
|
|
||||||
return V1ResponseBase({
|
|
||||||
"status": STATUS_OK,
|
|
||||||
"message": "Session already exists.",
|
|
||||||
"session": session_id
|
|
||||||
})
|
|
||||||
|
|
||||||
return V1ResponseBase({
|
|
||||||
"status": STATUS_OK,
|
|
||||||
"message": "Session created successfully.",
|
|
||||||
"session": session_id
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _cmd_sessions_list(req: V1RequestBase) -> V1ResponseBase:
|
|
||||||
session_ids = SESSIONS_STORAGE.session_ids()
|
|
||||||
|
|
||||||
return V1ResponseBase({
|
|
||||||
"status": STATUS_OK,
|
|
||||||
"message": "",
|
|
||||||
"sessions": session_ids
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
|
|
||||||
session_id = req.session
|
|
||||||
existed = SESSIONS_STORAGE.destroy(session_id)
|
|
||||||
|
|
||||||
if not existed:
|
|
||||||
raise Exception("The session doesn't exist.")
|
|
||||||
|
|
||||||
return V1ResponseBase({
|
|
||||||
"status": STATUS_OK,
|
|
||||||
"message": "The session has been removed."
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
|
def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
|
||||||
timeout = int(req.maxTimeout) / 1000
|
timeout = req.maxTimeout / 1000
|
||||||
driver = None
|
driver = None
|
||||||
try:
|
try:
|
||||||
if req.session:
|
driver = utils.get_webdriver()
|
||||||
session_id = req.session
|
|
||||||
ttl = timedelta(minutes=req.session_ttl_minutes) if req.session_ttl_minutes else None
|
|
||||||
session, fresh = SESSIONS_STORAGE.get(session_id, ttl)
|
|
||||||
|
|
||||||
if fresh:
|
|
||||||
logging.debug(f"new session created to perform the request (session_id={session_id})")
|
|
||||||
else:
|
|
||||||
logging.debug(f"existing session is used to perform the request (session_id={session_id}, "
|
|
||||||
f"lifetime={str(session.lifetime())}, ttl={str(ttl)})")
|
|
||||||
|
|
||||||
driver = session.driver
|
|
||||||
else:
|
|
||||||
driver = utils.get_webdriver(req.proxy)
|
|
||||||
logging.debug('New instance of webdriver has been created to perform the request')
|
|
||||||
return func_timeout(timeout, _evil_logic, (req, driver, method))
|
return func_timeout(timeout, _evil_logic, (req, driver, method))
|
||||||
except FunctionTimedOut:
|
except FunctionTimedOut:
|
||||||
raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
|
raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
|
raise Exception('Error solving the challenge. ' + str(e))
|
||||||
finally:
|
finally:
|
||||||
if not req.session and driver is not None:
|
if driver is not None:
|
||||||
if utils.PLATFORM_VERSION == "nt":
|
|
||||||
driver.close()
|
|
||||||
driver.quit()
|
driver.quit()
|
||||||
logging.debug('A used instance of webdriver has been destroyed')
|
|
||||||
|
|
||||||
|
|
||||||
def click_verify(driver: WebDriver):
|
|
||||||
try:
|
|
||||||
logging.debug("Try to find the Cloudflare verify checkbox...")
|
|
||||||
actions = ActionChains(driver)
|
|
||||||
actions.pause(5).send_keys(Keys.TAB).pause(1).send_keys(Keys.SPACE).perform()
|
|
||||||
logging.debug("Cloudflare verify checkbox found and clicked!")
|
|
||||||
except Exception:
|
|
||||||
logging.debug("Cloudflare verify checkbox not found on the page.")
|
|
||||||
finally:
|
|
||||||
driver.switch_to.default_content()
|
|
||||||
|
|
||||||
try:
|
|
||||||
logging.debug("Try to find the Cloudflare 'Verify you are human' button...")
|
|
||||||
button = driver.find_element(
|
|
||||||
by=By.XPATH,
|
|
||||||
value="//input[@type='button' and @value='Verify you are human']",
|
|
||||||
)
|
|
||||||
if button:
|
|
||||||
actions = ActionChains(driver)
|
|
||||||
actions.move_to_element_with_offset(button, 5, 7)
|
|
||||||
actions.click(button)
|
|
||||||
actions.perform()
|
|
||||||
logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
|
|
||||||
except Exception:
|
|
||||||
logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
|
|
||||||
|
|
||||||
time.sleep(2)
|
|
||||||
|
|
||||||
|
|
||||||
def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
|
def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
|
||||||
@ -287,37 +163,18 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
|||||||
res.status = STATUS_OK
|
res.status = STATUS_OK
|
||||||
res.message = ""
|
res.message = ""
|
||||||
|
|
||||||
|
|
||||||
# navigate to the page
|
# navigate to the page
|
||||||
logging.debug(f'Navigating to... {req.url}')
|
logging.debug(f'Navigating to... {req.url}')
|
||||||
if method == 'POST':
|
if method == 'POST':
|
||||||
_post_request(req, driver)
|
_post_request(req, driver)
|
||||||
else:
|
else:
|
||||||
driver.get(req.url)
|
driver.get(req.url)
|
||||||
|
|
||||||
# set cookies if required
|
|
||||||
if req.cookies is not None and len(req.cookies) > 0:
|
|
||||||
logging.debug(f'Setting cookies...')
|
|
||||||
for cookie in req.cookies:
|
|
||||||
driver.delete_cookie(cookie['name'])
|
|
||||||
driver.add_cookie(cookie)
|
|
||||||
# reload the page
|
|
||||||
if method == 'POST':
|
|
||||||
_post_request(req, driver)
|
|
||||||
else:
|
|
||||||
driver.get(req.url)
|
|
||||||
|
|
||||||
# wait for the page
|
|
||||||
if utils.get_config_log_html():
|
if utils.get_config_log_html():
|
||||||
logging.debug(f"Response HTML:\n{driver.page_source}")
|
logging.debug(f"Response HTML:\n{driver.page_source}")
|
||||||
html_element = driver.find_element(By.TAG_NAME, "html")
|
|
||||||
page_title = driver.title
|
|
||||||
|
|
||||||
# find access denied titles
|
# wait for the page
|
||||||
for title in ACCESS_DENIED_TITLES:
|
html_element = driver.find_element(By.TAG_NAME, "html")
|
||||||
if title == page_title:
|
|
||||||
raise Exception('Cloudflare has blocked this request. '
|
|
||||||
'Probably your IP is banned for this site, check in your web browser.')
|
|
||||||
# find access denied selectors
|
# find access denied selectors
|
||||||
for selector in ACCESS_DENIED_SELECTORS:
|
for selector in ACCESS_DENIED_SELECTORS:
|
||||||
found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
|
found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
|
||||||
@ -327,10 +184,11 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
|||||||
|
|
||||||
# find challenge by title
|
# find challenge by title
|
||||||
challenge_found = False
|
challenge_found = False
|
||||||
for title in CHALLENGE_TITLES:
|
page_title = driver.title
|
||||||
if title.lower() == page_title.lower():
|
for title in CHALLENGE_TITLE:
|
||||||
|
if title == page_title:
|
||||||
challenge_found = True
|
challenge_found = True
|
||||||
logging.info("Challenge detected. Title found: " + page_title)
|
logging.info("Challenge detected. Title found: " + title)
|
||||||
break
|
break
|
||||||
if not challenge_found:
|
if not challenge_found:
|
||||||
# find challenge by selectors
|
# find challenge by selectors
|
||||||
@ -341,19 +199,17 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
|||||||
logging.info("Challenge detected. Selector found: " + selector)
|
logging.info("Challenge detected. Selector found: " + selector)
|
||||||
break
|
break
|
||||||
|
|
||||||
attempt = 0
|
|
||||||
if challenge_found:
|
if challenge_found:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
attempt = attempt + 1
|
# wait until the title change
|
||||||
# wait until the title changes
|
for title in CHALLENGE_TITLE:
|
||||||
for title in CHALLENGE_TITLES:
|
logging.debug("Waiting for title: " + title)
|
||||||
logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title)
|
|
||||||
WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
|
WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
|
||||||
|
|
||||||
# then wait until all the selectors disappear
|
# then wait until all the selectors disappear
|
||||||
for selector in CHALLENGE_SELECTORS:
|
for selector in CHALLENGE_SELECTORS:
|
||||||
logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector)
|
logging.debug("Waiting for selector: " + selector)
|
||||||
WebDriverWait(driver, SHORT_TIMEOUT).until_not(
|
WebDriverWait(driver, SHORT_TIMEOUT).until_not(
|
||||||
presence_of_element_located((By.CSS_SELECTOR, selector)))
|
presence_of_element_located((By.CSS_SELECTOR, selector)))
|
||||||
|
|
||||||
@ -362,9 +218,6 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
|||||||
|
|
||||||
except TimeoutException:
|
except TimeoutException:
|
||||||
logging.debug("Timeout waiting for selector")
|
logging.debug("Timeout waiting for selector")
|
||||||
|
|
||||||
click_verify(driver)
|
|
||||||
|
|
||||||
# update the html (cloudflare reloads the page every 5 s)
|
# update the html (cloudflare reloads the page every 5 s)
|
||||||
html_element = driver.find_element(By.TAG_NAME, "html")
|
html_element = driver.find_element(By.TAG_NAME, "html")
|
||||||
|
|
||||||
@ -414,7 +267,7 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
|
|||||||
value = unquote(parts[1])
|
value = unquote(parts[1])
|
||||||
except Exception:
|
except Exception:
|
||||||
value = parts[1]
|
value = parts[1]
|
||||||
post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
|
post_form += f'<input type="text" name="{name}" value="{value}"><br>'
|
||||||
post_form += '</form>'
|
post_form += '</form>'
|
||||||
html_content = f"""
|
html_content = f"""
|
||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
@ -424,4 +277,4 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
|
|||||||
<script>document.getElementById('hackForm').submit();</script>
|
<script>document.getElementById('hackForm').submit();</script>
|
||||||
</body>
|
</body>
|
||||||
</html>"""
|
</html>"""
|
||||||
driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
|
driver.get("data:text/html;charset=utf-8," + html_content)
|
||||||
|
|||||||
@ -1,32 +0,0 @@
|
|||||||
import logging
|
|
||||||
|
|
||||||
from prometheus_client import Counter, Histogram, start_http_server
|
|
||||||
import time
|
|
||||||
|
|
||||||
REQUEST_COUNTER = Counter(
|
|
||||||
name='flaresolverr_request',
|
|
||||||
documentation='Total requests with result',
|
|
||||||
labelnames=['domain', 'result']
|
|
||||||
)
|
|
||||||
REQUEST_DURATION = Histogram(
|
|
||||||
name='flaresolverr_request_duration',
|
|
||||||
documentation='Request duration in seconds',
|
|
||||||
labelnames=['domain'],
|
|
||||||
buckets=[0, 10, 25, 50]
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def serve(port):
|
|
||||||
start_http_server(port=port)
|
|
||||||
while True:
|
|
||||||
time.sleep(600)
|
|
||||||
|
|
||||||
|
|
||||||
def start_metrics_http_server(prometheus_port: int):
|
|
||||||
logging.info(f"Serving Prometheus exporter on http://0.0.0.0:{prometheus_port}/metrics")
|
|
||||||
from threading import Thread
|
|
||||||
Thread(
|
|
||||||
target=serve,
|
|
||||||
kwargs=dict(port=prometheus_port),
|
|
||||||
daemon=True,
|
|
||||||
).start()
|
|
||||||
@ -1,84 +0,0 @@
|
|||||||
import logging
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
from typing import Optional, Tuple
|
|
||||||
from uuid import uuid1
|
|
||||||
|
|
||||||
from selenium.webdriver.chrome.webdriver import WebDriver
|
|
||||||
|
|
||||||
import utils
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class Session:
|
|
||||||
session_id: str
|
|
||||||
driver: WebDriver
|
|
||||||
created_at: datetime
|
|
||||||
|
|
||||||
def lifetime(self) -> timedelta:
|
|
||||||
return datetime.now() - self.created_at
|
|
||||||
|
|
||||||
|
|
||||||
class SessionsStorage:
|
|
||||||
"""SessionsStorage creates, stores and process all the sessions"""
|
|
||||||
|
|
||||||
def __init__(self):
|
|
||||||
self.sessions = {}
|
|
||||||
|
|
||||||
def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
|
|
||||||
force_new: Optional[bool] = False) -> Tuple[Session, bool]:
|
|
||||||
"""create creates new instance of WebDriver if necessary,
|
|
||||||
assign defined (or newly generated) session_id to the instance
|
|
||||||
and returns the session object. If a new session has been created
|
|
||||||
second argument is set to True.
|
|
||||||
|
|
||||||
Note: The function is idempotent, so in case if session_id
|
|
||||||
already exists in the storage a new instance of WebDriver won't be created
|
|
||||||
and existing session will be returned. Second argument defines if
|
|
||||||
new session has been created (True) or an existing one was used (False).
|
|
||||||
"""
|
|
||||||
session_id = session_id or str(uuid1())
|
|
||||||
|
|
||||||
if force_new:
|
|
||||||
self.destroy(session_id)
|
|
||||||
|
|
||||||
if self.exists(session_id):
|
|
||||||
return self.sessions[session_id], False
|
|
||||||
|
|
||||||
driver = utils.get_webdriver(proxy)
|
|
||||||
created_at = datetime.now()
|
|
||||||
session = Session(session_id, driver, created_at)
|
|
||||||
|
|
||||||
self.sessions[session_id] = session
|
|
||||||
|
|
||||||
return session, True
|
|
||||||
|
|
||||||
def exists(self, session_id: str) -> bool:
|
|
||||||
return session_id in self.sessions
|
|
||||||
|
|
||||||
def destroy(self, session_id: str) -> bool:
|
|
||||||
"""destroy closes the driver instance and removes session from the storage.
|
|
||||||
The function is noop if session_id doesn't exist.
|
|
||||||
The function returns True if session was found and destroyed,
|
|
||||||
and False if session_id wasn't found.
|
|
||||||
"""
|
|
||||||
if not self.exists(session_id):
|
|
||||||
return False
|
|
||||||
|
|
||||||
session = self.sessions.pop(session_id)
|
|
||||||
if utils.PLATFORM_VERSION == "nt":
|
|
||||||
session.driver.close()
|
|
||||||
session.driver.quit()
|
|
||||||
return True
|
|
||||||
|
|
||||||
def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:
|
|
||||||
session, fresh = self.create(session_id)
|
|
||||||
|
|
||||||
if ttl is not None and not fresh and session.lifetime() > ttl:
|
|
||||||
logging.debug(f'session\'s lifetime has expired, so the session is recreated (session_id={session_id})')
|
|
||||||
session, fresh = self.create(session_id, force_new=True)
|
|
||||||
|
|
||||||
return session, fresh
|
|
||||||
|
|
||||||
def session_ids(self) -> list[str]:
|
|
||||||
return list(self.sessions.keys())
|
|
||||||
293
src/tests.py
293
src/tests.py
@ -1,5 +1,4 @@
|
|||||||
import unittest
|
import unittest
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from webtest import TestApp
|
from webtest import TestApp
|
||||||
|
|
||||||
@ -8,7 +7,7 @@ import flaresolverr
|
|||||||
import utils
|
import utils
|
||||||
|
|
||||||
|
|
||||||
def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
|
def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
|
||||||
for obj in _list:
|
for obj in _list:
|
||||||
if obj[key] == value:
|
if obj[key] == value:
|
||||||
return obj
|
return obj
|
||||||
@ -24,13 +23,10 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
cloudflare_url = "https://nowsecure.nl"
|
cloudflare_url = "https://nowsecure.nl"
|
||||||
cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
|
cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
|
||||||
ddos_guard_url = "https://anidex.info/"
|
ddos_guard_url = "https://anidex.info/"
|
||||||
fairlane_url = "https://www.pararius.com/apartments/amsterdam"
|
|
||||||
custom_cloudflare_url = "https://www.muziekfabriek.org"
|
custom_cloudflare_url = "https://www.muziekfabriek.org"
|
||||||
cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
|
cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
|
||||||
|
|
||||||
app = TestApp(flaresolverr.app)
|
app = TestApp(flaresolverr.app)
|
||||||
# wait until the server is ready
|
|
||||||
app.get('/')
|
|
||||||
|
|
||||||
def test_wrong_endpoint(self):
|
def test_wrong_endpoint(self):
|
||||||
res = self.app.get('/wrong', status=404)
|
res = self.app.get('/wrong', status=404)
|
||||||
@ -170,32 +166,6 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
|
self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
|
||||||
self.assertGreater(len(cf_cookie["value"]), 10)
|
self.assertGreater(len(cf_cookie["value"]), 10)
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_fairlane_js(self):
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.fairlane_url
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Challenge solved!", body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
solution = body.solution
|
|
||||||
self.assertIn(self.fairlane_url, solution.url)
|
|
||||||
self.assertEqual(solution.status, 200)
|
|
||||||
self.assertIs(len(solution.headers), 0)
|
|
||||||
self.assertIn("<title>Rental Apartments Amsterdam</title>", solution.response)
|
|
||||||
self.assertGreater(len(solution.cookies), 0)
|
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
|
||||||
|
|
||||||
cf_cookie = _find_obj_by_key("name", "fl_pass_v2_b", solution.cookies)
|
|
||||||
self.assertIsNotNone(cf_cookie, "Fairlane cookie not found")
|
|
||||||
self.assertGreater(len(cf_cookie["value"]), 50)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_custom_cloudflare_js(self):
|
def test_v1_endpoint_request_get_custom_cloudflare_js(self):
|
||||||
res = self.app.post_json('/v1', {
|
res = self.app.post_json('/v1', {
|
||||||
"cmd": "request.get",
|
"cmd": "request.get",
|
||||||
@ -239,45 +209,7 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_cookies_param(self):
|
# todo: test Cmd 'request.get' should return OK with 'cookies' param
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.google_url,
|
|
||||||
"cookies": [
|
|
||||||
{
|
|
||||||
"name": "testcookie1",
|
|
||||||
"value": "testvalue1"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "testcookie2",
|
|
||||||
"value": "testvalue2"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Challenge not detected!", body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
solution = body.solution
|
|
||||||
self.assertIn(self.google_url, solution.url)
|
|
||||||
self.assertEqual(solution.status, 200)
|
|
||||||
self.assertIs(len(solution.headers), 0)
|
|
||||||
self.assertIn("<title>Google</title>", solution.response)
|
|
||||||
self.assertGreater(len(solution.cookies), 1)
|
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
|
||||||
|
|
||||||
user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
|
|
||||||
self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
|
|
||||||
self.assertEqual("testvalue1", user_cookie1["value"])
|
|
||||||
|
|
||||||
user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
|
|
||||||
self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
|
|
||||||
self.assertEqual("testvalue2", user_cookie2["value"])
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
|
def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
|
||||||
res = self.app.post_json('/v1', {
|
res = self.app.post_json('/v1', {
|
||||||
@ -302,124 +234,10 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertGreater(len(solution.cookies), 0)
|
self.assertGreater(len(solution.cookies), 0)
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
self.assertIn("Chrome/", solution.userAgent)
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_proxy_http_param(self):
|
# todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param
|
||||||
"""
|
# todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials
|
||||||
To configure TinyProxy in local:
|
# todo: test Cmd 'request.get' should return OK with SOCKSv5 'proxy' param
|
||||||
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
# todo: test Cmd 'request.get' should fail with wrong 'proxy' param
|
||||||
* edit => LogFile "/tmp/tinyproxy.log"
|
|
||||||
* edit => Syslog Off
|
|
||||||
* sudo tinyproxy -d
|
|
||||||
* sudo tail -f /tmp/tinyproxy.log
|
|
||||||
"""
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.google_url,
|
|
||||||
"proxy": {
|
|
||||||
"url": self.proxy_url
|
|
||||||
}
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Challenge not detected!", body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
solution = body.solution
|
|
||||||
self.assertIn(self.google_url, solution.url)
|
|
||||||
self.assertEqual(solution.status, 200)
|
|
||||||
self.assertIs(len(solution.headers), 0)
|
|
||||||
self.assertIn("<title>Google</title>", solution.response)
|
|
||||||
self.assertGreater(len(solution.cookies), 0)
|
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self):
|
|
||||||
"""
|
|
||||||
To configure TinyProxy in local:
|
|
||||||
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
|
||||||
* edit => LogFile "/tmp/tinyproxy.log"
|
|
||||||
* edit => Syslog Off
|
|
||||||
* add => BasicAuth testuser testpass
|
|
||||||
* sudo tinyproxy -d
|
|
||||||
* sudo tail -f /tmp/tinyproxy.log
|
|
||||||
"""
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.google_url,
|
|
||||||
"proxy": {
|
|
||||||
"url": self.proxy_url,
|
|
||||||
"username": "testuser",
|
|
||||||
"password": "testpass"
|
|
||||||
}
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Challenge not detected!", body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
solution = body.solution
|
|
||||||
self.assertIn(self.google_url, solution.url)
|
|
||||||
self.assertEqual(solution.status, 200)
|
|
||||||
self.assertIs(len(solution.headers), 0)
|
|
||||||
self.assertIn("<title>Google</title>", solution.response)
|
|
||||||
self.assertGreater(len(solution.cookies), 0)
|
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_proxy_socks_param(self):
|
|
||||||
"""
|
|
||||||
To configure Dante in local:
|
|
||||||
* https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
|
|
||||||
* sudo vim /etc/sockd.conf
|
|
||||||
* sudo systemctl restart sockd.service
|
|
||||||
* curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
|
|
||||||
"""
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.google_url,
|
|
||||||
"proxy": {
|
|
||||||
"url": self.proxy_socks_url
|
|
||||||
}
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Challenge not detected!", body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
solution = body.solution
|
|
||||||
self.assertIn(self.google_url, solution.url)
|
|
||||||
self.assertEqual(solution.status, 200)
|
|
||||||
self.assertIs(len(solution.headers), 0)
|
|
||||||
self.assertIn("<title>Google</title>", solution.response)
|
|
||||||
self.assertGreater(len(solution.cookies), 0)
|
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_proxy_wrong_param(self):
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"url": self.google_url,
|
|
||||||
"proxy": {
|
|
||||||
"url": "http://127.0.0.1:43210"
|
|
||||||
}
|
|
||||||
}, status=500)
|
|
||||||
self.assertEqual(res.status_code, 500)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_ERROR, body.status)
|
|
||||||
self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
|
|
||||||
body.message)
|
|
||||||
self.assertGreater(body.startTimestamp, 10000)
|
|
||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_fail_timeout(self):
|
def test_v1_endpoint_request_get_fail_timeout(self):
|
||||||
res = self.app.post_json('/v1', {
|
res = self.app.post_json('/v1', {
|
||||||
@ -533,99 +351,12 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertEqual(STATUS_OK, body.status)
|
self.assertEqual(STATUS_OK, body.status)
|
||||||
self.assertEqual("Challenge not detected!", body.message)
|
self.assertEqual("Challenge not detected!", body.message)
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_create_without_session(self):
|
# todo: test Cmd 'sessions.create' should return OK
|
||||||
res = self.app.post_json('/v1', {
|
# todo: test Cmd 'sessions.create' should return OK with session
|
||||||
"cmd": "sessions.create"
|
# todo: test Cmd 'sessions.list' should return OK
|
||||||
})
|
# todo: test Cmd 'sessions.destroy' should return OK
|
||||||
self.assertEqual(res.status_code, 200)
|
# todo: test Cmd 'sessions.destroy' should fail
|
||||||
|
# todo: test Cmd 'request.get' should use session
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Session created successfully.", body.message)
|
|
||||||
self.assertIsNotNone(body.session)
|
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_create_with_session(self):
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"session": "test_create_session"
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Session created successfully.", body.message)
|
|
||||||
self.assertEqual(body.session, "test_create_session")
|
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_create_with_proxy(self):
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"proxy": {
|
|
||||||
"url": self.proxy_url
|
|
||||||
}
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("Session created successfully.", body.message)
|
|
||||||
self.assertIsNotNone(body.session)
|
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_list(self):
|
|
||||||
self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"session": "test_list_sessions"
|
|
||||||
})
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.list"
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("", body.message)
|
|
||||||
self.assertGreaterEqual(len(body.sessions), 1)
|
|
||||||
self.assertIn("test_list_sessions", body.sessions)
|
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_destroy_existing_session(self):
|
|
||||||
self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"session": "test_destroy_sessions"
|
|
||||||
})
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.destroy",
|
|
||||||
"session": "test_destroy_sessions"
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
self.assertEqual("The session has been removed.", body.message)
|
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_destroy_non_existing_session(self):
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.destroy",
|
|
||||||
"session": "non_existing_session_name"
|
|
||||||
}, status=500)
|
|
||||||
self.assertEqual(res.status_code, 500)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_ERROR, body.status)
|
|
||||||
self.assertEqual("Error: The session doesn't exist.", body.message)
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_with_session(self):
|
|
||||||
self.app.post_json('/v1', {
|
|
||||||
"cmd": "sessions.create",
|
|
||||||
"session": "test_request_sessions"
|
|
||||||
})
|
|
||||||
res = self.app.post_json('/v1', {
|
|
||||||
"cmd": "request.get",
|
|
||||||
"session": "test_request_sessions",
|
|
||||||
"url": self.google_url
|
|
||||||
})
|
|
||||||
self.assertEqual(res.status_code, 200)
|
|
||||||
|
|
||||||
body = V1ResponseBase(res.json)
|
|
||||||
self.assertEqual(STATUS_OK, body.status)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|||||||
@ -39,8 +39,6 @@ def asset_cloudflare_solution(self, res, site_url, site_text):
|
|||||||
|
|
||||||
class TestFlareSolverr(unittest.TestCase):
|
class TestFlareSolverr(unittest.TestCase):
|
||||||
app = TestApp(flaresolverr.app)
|
app = TestApp(flaresolverr.app)
|
||||||
# wait until the server is ready
|
|
||||||
app.get('/')
|
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_cloudflare(self):
|
def test_v1_endpoint_request_get_cloudflare(self):
|
||||||
sites_get = [
|
sites_get = [
|
||||||
|
|||||||
@ -17,12 +17,11 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
|
||||||
__version__ = "3.5.5"
|
__version__ = "3.2.1"
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
@ -34,7 +33,7 @@ from weakref import finalize
|
|||||||
import selenium.webdriver.chrome.service
|
import selenium.webdriver.chrome.service
|
||||||
import selenium.webdriver.chrome.webdriver
|
import selenium.webdriver.chrome.webdriver
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
import selenium.webdriver.chromium.service
|
import selenium.webdriver.common.service
|
||||||
import selenium.webdriver.remote.command
|
import selenium.webdriver.remote.command
|
||||||
import selenium.webdriver.remote.webdriver
|
import selenium.webdriver.remote.webdriver
|
||||||
|
|
||||||
@ -110,11 +109,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
browser_executable_path=None,
|
browser_executable_path=None,
|
||||||
port=0,
|
port=0,
|
||||||
enable_cdp_events=False,
|
enable_cdp_events=False,
|
||||||
# service_args=None,
|
service_args=None,
|
||||||
# service_creationflags=None,
|
service_creationflags=None,
|
||||||
desired_capabilities=None,
|
desired_capabilities=None,
|
||||||
advanced_elements=False,
|
advanced_elements=False,
|
||||||
# service_log_path=None,
|
service_log_path=None,
|
||||||
keep_alive=True,
|
keep_alive=True,
|
||||||
log_level=0,
|
log_level=0,
|
||||||
headless=False,
|
headless=False,
|
||||||
@ -125,7 +124,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
debug=False,
|
debug=False,
|
||||||
no_sandbox=True,
|
no_sandbox=True,
|
||||||
windows_headless=False,
|
windows_headless=False,
|
||||||
user_multi_procs: bool = False,
|
|
||||||
**kw,
|
**kw,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
@ -237,28 +235,17 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
|
uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
|
||||||
this option has a default of True since many people seem to run this as root (....) , and chrome does not start
|
this option has a default of True since many people seem to run this as root (....) , and chrome does not start
|
||||||
when running as root without using --no-sandbox flag.
|
when running as root without using --no-sandbox flag.
|
||||||
|
|
||||||
user_multi_procs:
|
|
||||||
set to true when you are using multithreads/multiprocessing
|
|
||||||
ensures not all processes are trying to modify a binary which is in use by another.
|
|
||||||
for this to work. YOU MUST HAVE AT LEAST 1 UNDETECTED_CHROMEDRIVER BINARY IN YOUR ROAMING DATA FOLDER.
|
|
||||||
this requirement can be easily satisfied, by just running this program "normal" and close/kill it.
|
|
||||||
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
finalize(self, self._ensure_close, self)
|
finalize(self, self._ensure_close, self)
|
||||||
self.debug = debug
|
self.debug = debug
|
||||||
self.patcher = Patcher(
|
patcher = Patcher(
|
||||||
executable_path=driver_executable_path,
|
executable_path=driver_executable_path,
|
||||||
force=patcher_force_close,
|
force=patcher_force_close,
|
||||||
version_main=version_main,
|
version_main=version_main,
|
||||||
user_multi_procs=user_multi_procs,
|
|
||||||
)
|
)
|
||||||
# self.patcher.auto(user_multiprocess = user_multi_num_procs)
|
patcher.auto()
|
||||||
self.patcher.auto()
|
self.patcher = patcher
|
||||||
|
|
||||||
# self.patcher = patcher
|
|
||||||
if not options:
|
if not options:
|
||||||
options = ChromeOptions()
|
options = ChromeOptions()
|
||||||
|
|
||||||
@ -300,11 +287,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
|
|
||||||
# see if a custom user profile is specified in options
|
# see if a custom user profile is specified in options
|
||||||
for arg in options.arguments:
|
for arg in options.arguments:
|
||||||
|
|
||||||
if any([_ in arg for _ in ("--headless", "headless")]):
|
|
||||||
options.arguments.remove(arg)
|
|
||||||
options.headless = True
|
|
||||||
|
|
||||||
if "lang" in arg:
|
if "lang" in arg:
|
||||||
m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
|
m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
|
||||||
try:
|
try:
|
||||||
@ -374,18 +356,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
browser_executable_path or find_chrome_executable()
|
browser_executable_path or find_chrome_executable()
|
||||||
)
|
)
|
||||||
|
|
||||||
if not options.binary_location or not \
|
|
||||||
pathlib.Path(options.binary_location).exists():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
"\n---------------------\n"
|
|
||||||
"Could not determine browser executable."
|
|
||||||
"\n---------------------\n"
|
|
||||||
"Make sure your browser is installed in the default location (path).\n"
|
|
||||||
"If you are sure about the browser executable, you can specify it using\n"
|
|
||||||
"the `browser_executable_path='{}` parameter.\n\n"
|
|
||||||
.format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
|
|
||||||
)
|
|
||||||
|
|
||||||
self._delay = 3
|
self._delay = 3
|
||||||
|
|
||||||
self.user_data_dir = user_data_dir
|
self.user_data_dir = user_data_dir
|
||||||
@ -395,25 +365,13 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
|
options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
|
||||||
if no_sandbox:
|
if no_sandbox:
|
||||||
options.arguments.extend(["--no-sandbox", "--test-type"])
|
options.arguments.extend(["--no-sandbox", "--test-type"])
|
||||||
|
if headless or options.headless:
|
||||||
if headless or getattr(options, 'headless', None):
|
options.headless = True
|
||||||
#workaround until a better checking is found
|
options.add_argument("--window-size=1920,1080")
|
||||||
try:
|
options.add_argument("--start-maximized")
|
||||||
v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
|
options.add_argument("--no-sandbox")
|
||||||
if v_main < 108:
|
# fixes "could not connect to chrome" error when running
|
||||||
options.add_argument("--headless=chrome")
|
# on linux using privileged user like root (which i don't recommend)
|
||||||
elif v_main >= 108:
|
|
||||||
options.add_argument("--headless=new")
|
|
||||||
except:
|
|
||||||
logger.warning("could not detect version_main."
|
|
||||||
"therefore, we are assuming it is chrome 108 or higher")
|
|
||||||
options.add_argument("--headless=new")
|
|
||||||
|
|
||||||
options.add_argument("--window-size=1920,1080")
|
|
||||||
options.add_argument("--start-maximized")
|
|
||||||
options.add_argument("--no-sandbox")
|
|
||||||
# fixes "could not connect to chrome" error when running
|
|
||||||
# on linux using privileged user like root (which i don't recommend)
|
|
||||||
|
|
||||||
options.add_argument(
|
options.add_argument(
|
||||||
"--log-level=%d" % log_level
|
"--log-level=%d" % log_level
|
||||||
@ -451,10 +409,8 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
options.binary_location, *options.arguments
|
options.binary_location, *options.arguments
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
startupinfo = None
|
startupinfo = subprocess.STARTUPINFO()
|
||||||
if os.name == 'nt' and windows_headless:
|
if os.name == 'nt' and windows_headless:
|
||||||
# STARTUPINFO() is Windows only
|
|
||||||
startupinfo = subprocess.STARTUPINFO()
|
|
||||||
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
|
||||||
browser = subprocess.Popen(
|
browser = subprocess.Popen(
|
||||||
[options.binary_location, *options.arguments],
|
[options.binary_location, *options.arguments],
|
||||||
@ -466,15 +422,26 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
)
|
)
|
||||||
self.browser_pid = browser.pid
|
self.browser_pid = browser.pid
|
||||||
|
|
||||||
|
if service_creationflags:
|
||||||
service = selenium.webdriver.chromium.service.ChromiumService(
|
service = selenium.webdriver.common.service.Service(
|
||||||
self.patcher.executable_path
|
patcher.executable_path, port, service_args, service_log_path
|
||||||
)
|
)
|
||||||
|
for attr_name in ("creationflags", "creation_flags"):
|
||||||
|
if hasattr(service, attr_name):
|
||||||
|
setattr(service, attr_name, service_creationflags)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
service = None
|
||||||
|
|
||||||
super(Chrome, self).__init__(
|
super(Chrome, self).__init__(
|
||||||
service=service,
|
executable_path=patcher.executable_path,
|
||||||
|
port=port,
|
||||||
options=options,
|
options=options,
|
||||||
|
service_args=service_args,
|
||||||
|
desired_capabilities=desired_capabilities,
|
||||||
|
service_log_path=service_log_path,
|
||||||
keep_alive=keep_alive,
|
keep_alive=keep_alive,
|
||||||
|
service=service, # needed or the service will be re-created
|
||||||
)
|
)
|
||||||
|
|
||||||
self.reactor = None
|
self.reactor = None
|
||||||
@ -493,7 +460,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
else:
|
else:
|
||||||
self._web_element_cls = WebElement
|
self._web_element_cls = WebElement
|
||||||
|
|
||||||
if headless or getattr(options, 'headless', None):
|
if options.headless:
|
||||||
self._configure_headless()
|
self._configure_headless()
|
||||||
|
|
||||||
def _configure_headless(self):
|
def _configure_headless(self):
|
||||||
@ -507,17 +474,19 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
"Page.addScriptToEvaluateOnNewDocument",
|
"Page.addScriptToEvaluateOnNewDocument",
|
||||||
{
|
{
|
||||||
"source": """
|
"source": """
|
||||||
Object.defineProperty(window, "navigator", {
|
|
||||||
value: new Proxy(navigator, {
|
Object.defineProperty(window, 'navigator', {
|
||||||
has: (target, key) => (key === "webdriver" ? false : key in target),
|
value: new Proxy(navigator, {
|
||||||
get: (target, key) =>
|
has: (target, key) => (key === 'webdriver' ? false : key in target),
|
||||||
key === "webdriver"
|
get: (target, key) =>
|
||||||
? false
|
key === 'webdriver' ?
|
||||||
: typeof target[key] === "function"
|
false :
|
||||||
? target[key].bind(target)
|
typeof target[key] === 'function' ?
|
||||||
: target[key],
|
target[key].bind(target) :
|
||||||
}),
|
target[key]
|
||||||
});
|
})
|
||||||
|
});
|
||||||
|
|
||||||
"""
|
"""
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@ -636,38 +605,37 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
|
|
||||||
self.get = get_wrapped
|
self.get = get_wrapped
|
||||||
|
|
||||||
# def _get_cdc_props(self):
|
def _get_cdc_props(self):
|
||||||
# return self.execute_script(
|
return self.execute_script(
|
||||||
# """
|
"""
|
||||||
# let objectToInspect = window,
|
let objectToInspect = window,
|
||||||
# result = [];
|
result = [];
|
||||||
# while(objectToInspect !== null)
|
while(objectToInspect !== null)
|
||||||
# { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
|
{ result = result.concat(Object.getOwnPropertyNames(objectToInspect));
|
||||||
# objectToInspect = Object.getPrototypeOf(objectToInspect); }
|
objectToInspect = Object.getPrototypeOf(objectToInspect); }
|
||||||
#
|
return result.filter(i => i.match(/.+_.+_(Array|Promise|Symbol)/ig))
|
||||||
# return result.filter(i => i.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig))
|
"""
|
||||||
# """
|
)
|
||||||
# )
|
|
||||||
#
|
def _hook_remove_cdc_props(self):
|
||||||
# def _hook_remove_cdc_props(self):
|
self.execute_cdp_cmd(
|
||||||
# self.execute_cdp_cmd(
|
"Page.addScriptToEvaluateOnNewDocument",
|
||||||
# "Page.addScriptToEvaluateOnNewDocument",
|
{
|
||||||
# {
|
"source": """
|
||||||
# "source": """
|
let objectToInspect = window,
|
||||||
# let objectToInspect = window,
|
result = [];
|
||||||
# result = [];
|
while(objectToInspect !== null)
|
||||||
# while(objectToInspect !== null)
|
{ result = result.concat(Object.getOwnPropertyNames(objectToInspect));
|
||||||
# { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
|
objectToInspect = Object.getPrototypeOf(objectToInspect); }
|
||||||
# objectToInspect = Object.getPrototypeOf(objectToInspect); }
|
result.forEach(p => p.match(/.+_.+_(Array|Promise|Symbol)/ig)
|
||||||
# result.forEach(p => p.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig)
|
&&delete window[p]&&console.log('removed',p))
|
||||||
# &&delete window[p]&&console.log('removed',p))
|
"""
|
||||||
# """
|
},
|
||||||
# },
|
)
|
||||||
# )
|
|
||||||
|
|
||||||
def get(self, url):
|
def get(self, url):
|
||||||
# if self._get_cdc_props():
|
if self._get_cdc_props():
|
||||||
# self._hook_remove_cdc_props()
|
self._hook_remove_cdc_props()
|
||||||
return super().get(url)
|
return super().get(url)
|
||||||
|
|
||||||
def add_cdp_listener(self, event_name, callback):
|
def add_cdp_listener(self, event_name, callback):
|
||||||
@ -728,50 +696,13 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
if not capabilities:
|
if not capabilities:
|
||||||
capabilities = self.options.to_capabilities()
|
capabilities = self.options.to_capabilities()
|
||||||
super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
|
super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
|
||||||
capabilities
|
capabilities, browser_profile
|
||||||
)
|
)
|
||||||
# super(Chrome, self).start_session(capabilities, browser_profile)
|
# super(Chrome, self).start_session(capabilities, browser_profile)
|
||||||
|
|
||||||
def find_elements_recursive(self, by, value):
|
|
||||||
"""
|
|
||||||
find elements in all frames
|
|
||||||
this is a generator function, which is needed
|
|
||||||
since if it would return a list of elements, they
|
|
||||||
will be stale on arrival.
|
|
||||||
using generator, when the element is returned we are in the correct frame
|
|
||||||
to use it directly
|
|
||||||
Args:
|
|
||||||
by: By
|
|
||||||
value: str
|
|
||||||
Returns: Generator[webelement.WebElement]
|
|
||||||
"""
|
|
||||||
def search_frame(f=None):
|
|
||||||
if not f:
|
|
||||||
# ensure we are on main content frame
|
|
||||||
self.switch_to.default_content()
|
|
||||||
else:
|
|
||||||
self.switch_to.frame(f)
|
|
||||||
for elem in self.find_elements(by, value):
|
|
||||||
yield elem
|
|
||||||
# switch back to main content, otherwise we will get StaleElementReferenceException
|
|
||||||
self.switch_to.default_content()
|
|
||||||
|
|
||||||
# search root frame
|
|
||||||
for elem in search_frame():
|
|
||||||
yield elem
|
|
||||||
# get iframes
|
|
||||||
frames = self.find_elements('css selector', 'iframe')
|
|
||||||
|
|
||||||
# search per frame
|
|
||||||
for f in frames:
|
|
||||||
for elem in search_frame(f):
|
|
||||||
yield elem
|
|
||||||
|
|
||||||
def quit(self):
|
def quit(self):
|
||||||
try:
|
try:
|
||||||
self.service.stop()
|
|
||||||
self.service.process.kill()
|
self.service.process.kill()
|
||||||
self.command_executor.close()
|
|
||||||
self.service.process.wait(5)
|
self.service.process.wait(5)
|
||||||
logger.debug("webdriver process ended")
|
logger.debug("webdriver process ended")
|
||||||
except (AttributeError, RuntimeError, OSError):
|
except (AttributeError, RuntimeError, OSError):
|
||||||
@ -785,7 +716,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
os.kill(self.browser_pid, 15)
|
os.kill(self.browser_pid, 15)
|
||||||
logger.debug("gracefully closed browser")
|
logger.debug("gracefully closed browser")
|
||||||
except Exception as e: # noqa
|
except Exception as e: # noqa
|
||||||
pass
|
logger.debug(e, exc_info=True)
|
||||||
if (
|
if (
|
||||||
hasattr(self, "keep_user_data_dir")
|
hasattr(self, "keep_user_data_dir")
|
||||||
and hasattr(self, "user_data_dir")
|
and hasattr(self, "user_data_dir")
|
||||||
@ -804,11 +735,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
else:
|
else:
|
||||||
logger.debug("successfully removed %s" % self.user_data_dir)
|
logger.debug("successfully removed %s" % self.user_data_dir)
|
||||||
break
|
break
|
||||||
|
time.sleep(0.1)
|
||||||
try:
|
|
||||||
time.sleep(0.1)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# dereference patcher, so patcher can start cleaning up as well.
|
# dereference patcher, so patcher can start cleaning up as well.
|
||||||
# this must come last, otherwise it will throw 'in use' errors
|
# this must come last, otherwise it will throw 'in use' errors
|
||||||
@ -903,10 +830,10 @@ def find_chrome_executable():
|
|||||||
if item is not None:
|
if item is not None:
|
||||||
for subitem in (
|
for subitem in (
|
||||||
"Google/Chrome/Application",
|
"Google/Chrome/Application",
|
||||||
|
"Google/Chrome Beta/Application",
|
||||||
|
"Google/Chrome Canary/Application",
|
||||||
):
|
):
|
||||||
candidates.add(os.sep.join((item, subitem, "chrome.exe")))
|
candidates.add(os.sep.join((item, subitem, "chrome.exe")))
|
||||||
for candidate in candidates:
|
for candidate in candidates:
|
||||||
logger.debug('checking if %s exists and is executable' % candidate)
|
|
||||||
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
|
if os.path.exists(candidate) and os.access(candidate, os.X_OK):
|
||||||
logger.debug('found! using %s' % candidate)
|
|
||||||
return os.path.normpath(candidate)
|
return os.path.normpath(candidate)
|
||||||
|
|||||||
262
src/undetected_chromedriver/_compat.py
Normal file
262
src/undetected_chromedriver/_compat.py
Normal file
@ -0,0 +1,262 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
# this module is part of undetected_chromedriver
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
888 888 d8b
|
||||||
|
888 888 Y8P
|
||||||
|
888 888
|
||||||
|
.d8888b 88888b. 888d888 .d88b. 88888b.d88b. .d88b. .d88888 888d888 888 888 888 .d88b. 888d888
|
||||||
|
d88P" 888 "88b 888P" d88""88b 888 "888 "88b d8P Y8b d88" 888 888P" 888 888 888 d8P Y8b 888P"
|
||||||
|
888 888 888 888 888 888 888 888 888 88888888 888 888 888 888 Y88 88P 88888888 888
|
||||||
|
Y88b. 888 888 888 Y88..88P 888 888 888 Y8b. Y88b 888 888 888 Y8bd8P Y8b. 888
|
||||||
|
"Y8888P 888 888 888 "Y88P" 888 888 888 "Y8888 "Y88888 888 888 Y88P "Y8888 888 88888888
|
||||||
|
|
||||||
|
by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
from distutils.version import LooseVersion
|
||||||
|
import io
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import re
|
||||||
|
import string
|
||||||
|
import sys
|
||||||
|
from urllib.request import urlopen
|
||||||
|
from urllib.request import urlretrieve
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
from selenium.webdriver import Chrome as _Chrome
|
||||||
|
from selenium.webdriver import ChromeOptions as _ChromeOptions
|
||||||
|
|
||||||
|
|
||||||
|
TARGET_VERSION = 0
|
||||||
|
logger = logging.getLogger("uc")
|
||||||
|
|
||||||
|
|
||||||
|
class Chrome:
|
||||||
|
def __new__(cls, *args, emulate_touch=False, **kwargs):
|
||||||
|
|
||||||
|
if not ChromeDriverManager.installed:
|
||||||
|
ChromeDriverManager(*args, **kwargs).install()
|
||||||
|
if not ChromeDriverManager.selenium_patched:
|
||||||
|
ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()
|
||||||
|
if not kwargs.get("executable_path"):
|
||||||
|
kwargs["executable_path"] = "./{}".format(
|
||||||
|
ChromeDriverManager(*args, **kwargs).executable_path
|
||||||
|
)
|
||||||
|
if not kwargs.get("options"):
|
||||||
|
kwargs["options"] = ChromeOptions()
|
||||||
|
instance = object.__new__(_Chrome)
|
||||||
|
instance.__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
instance._orig_get = instance.get
|
||||||
|
|
||||||
|
def _get_wrapped(*args, **kwargs):
|
||||||
|
if instance.execute_script("return navigator.webdriver"):
|
||||||
|
instance.execute_cdp_cmd(
|
||||||
|
"Page.addScriptToEvaluateOnNewDocument",
|
||||||
|
{
|
||||||
|
"source": """
|
||||||
|
|
||||||
|
Object.defineProperty(window, 'navigator', {
|
||||||
|
value: new Proxy(navigator, {
|
||||||
|
has: (target, key) => (key === 'webdriver' ? false : key in target),
|
||||||
|
get: (target, key) =>
|
||||||
|
key === 'webdriver'
|
||||||
|
? undefined
|
||||||
|
: typeof target[key] === 'function'
|
||||||
|
? target[key].bind(target)
|
||||||
|
: target[key]
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
"""
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return instance._orig_get(*args, **kwargs)
|
||||||
|
|
||||||
|
instance.get = _get_wrapped
|
||||||
|
instance.get = _get_wrapped
|
||||||
|
instance.get = _get_wrapped
|
||||||
|
|
||||||
|
original_user_agent_string = instance.execute_script(
|
||||||
|
"return navigator.userAgent"
|
||||||
|
)
|
||||||
|
instance.execute_cdp_cmd(
|
||||||
|
"Network.setUserAgentOverride",
|
||||||
|
{
|
||||||
|
"userAgent": original_user_agent_string.replace("Headless", ""),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if emulate_touch:
|
||||||
|
instance.execute_cdp_cmd(
|
||||||
|
"Page.addScriptToEvaluateOnNewDocument",
|
||||||
|
{
|
||||||
|
"source": """
|
||||||
|
Object.defineProperty(navigator, 'maxTouchPoints', {
|
||||||
|
get: () => 1
|
||||||
|
})"""
|
||||||
|
},
|
||||||
|
)
|
||||||
|
logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})")
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class ChromeOptions:
|
||||||
|
def __new__(cls, *args, **kwargs):
|
||||||
|
if not ChromeDriverManager.installed:
|
||||||
|
ChromeDriverManager(*args, **kwargs).install()
|
||||||
|
if not ChromeDriverManager.selenium_patched:
|
||||||
|
ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()
|
||||||
|
|
||||||
|
instance = object.__new__(_ChromeOptions)
|
||||||
|
instance.__init__()
|
||||||
|
instance.add_argument("start-maximized")
|
||||||
|
instance.add_experimental_option("excludeSwitches", ["enable-automation"])
|
||||||
|
instance.add_argument("--disable-blink-features=AutomationControlled")
|
||||||
|
return instance
|
||||||
|
|
||||||
|
|
||||||
|
class ChromeDriverManager(object):
|
||||||
|
installed = False
|
||||||
|
selenium_patched = False
|
||||||
|
target_version = None
|
||||||
|
|
||||||
|
DL_BASE = "https://chromedriver.storage.googleapis.com/"
|
||||||
|
|
||||||
|
def __init__(self, executable_path=None, target_version=None, *args, **kwargs):
|
||||||
|
|
||||||
|
_platform = sys.platform
|
||||||
|
|
||||||
|
if TARGET_VERSION:
|
||||||
|
# use global if set
|
||||||
|
self.target_version = TARGET_VERSION
|
||||||
|
|
||||||
|
if target_version:
|
||||||
|
# use explicitly passed target
|
||||||
|
self.target_version = target_version # user override
|
||||||
|
|
||||||
|
if not self.target_version:
|
||||||
|
# none of the above (default) and just get current version
|
||||||
|
self.target_version = self.get_release_version_number().version[
|
||||||
|
0
|
||||||
|
] # only major version int
|
||||||
|
|
||||||
|
self._base = base_ = "chromedriver{}"
|
||||||
|
|
||||||
|
exe_name = self._base
|
||||||
|
if _platform in ("win32",):
|
||||||
|
exe_name = base_.format(".exe")
|
||||||
|
if _platform in ("linux",):
|
||||||
|
_platform += "64"
|
||||||
|
exe_name = exe_name.format("")
|
||||||
|
if _platform in ("darwin",):
|
||||||
|
_platform = "mac64"
|
||||||
|
exe_name = exe_name.format("")
|
||||||
|
self.platform = _platform
|
||||||
|
self.executable_path = executable_path or exe_name
|
||||||
|
self._exe_name = exe_name
|
||||||
|
|
||||||
|
def patch_selenium_webdriver(self_):
|
||||||
|
"""
|
||||||
|
Patches selenium package Chrome, ChromeOptions classes for current session
|
||||||
|
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
import selenium.webdriver.chrome.service
|
||||||
|
import selenium.webdriver
|
||||||
|
|
||||||
|
selenium.webdriver.Chrome = Chrome
|
||||||
|
selenium.webdriver.ChromeOptions = ChromeOptions
|
||||||
|
logger.info("Selenium patched. Safe to import Chrome / ChromeOptions")
|
||||||
|
self_.__class__.selenium_patched = True
|
||||||
|
|
||||||
|
def install(self, patch_selenium=True):
|
||||||
|
"""
|
||||||
|
Initialize the patch
|
||||||
|
|
||||||
|
This will:
|
||||||
|
download chromedriver if not present
|
||||||
|
patch the downloaded chromedriver
|
||||||
|
patch selenium package if <patch_selenium> is True (default)
|
||||||
|
|
||||||
|
:param patch_selenium: patch selenium webdriver classes for Chrome and ChromeDriver (for current python session)
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
if not os.path.exists(self.executable_path):
|
||||||
|
self.fetch_chromedriver()
|
||||||
|
if not self.__class__.installed:
|
||||||
|
if self.patch_binary():
|
||||||
|
self.__class__.installed = True
|
||||||
|
|
||||||
|
if patch_selenium:
|
||||||
|
self.patch_selenium_webdriver()
|
||||||
|
|
||||||
|
def get_release_version_number(self):
|
||||||
|
"""
|
||||||
|
Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
|
||||||
|
|
||||||
|
:return: version string
|
||||||
|
"""
|
||||||
|
path = (
|
||||||
|
"LATEST_RELEASE"
|
||||||
|
if not self.target_version
|
||||||
|
else f"LATEST_RELEASE_{self.target_version}"
|
||||||
|
)
|
||||||
|
return LooseVersion(urlopen(self.__class__.DL_BASE + path).read().decode())
|
||||||
|
|
||||||
|
def fetch_chromedriver(self):
|
||||||
|
"""
|
||||||
|
Downloads ChromeDriver from source and unpacks the executable
|
||||||
|
|
||||||
|
:return: on success, name of the unpacked executable
|
||||||
|
"""
|
||||||
|
base_ = self._base
|
||||||
|
zip_name = base_.format(".zip")
|
||||||
|
ver = self.get_release_version_number().vstring
|
||||||
|
if os.path.exists(self.executable_path):
|
||||||
|
return self.executable_path
|
||||||
|
urlretrieve(
|
||||||
|
f"{self.__class__.DL_BASE}{ver}/{base_.format(f'_{self.platform}')}.zip",
|
||||||
|
filename=zip_name,
|
||||||
|
)
|
||||||
|
with zipfile.ZipFile(zip_name) as zf:
|
||||||
|
zf.extract(self._exe_name)
|
||||||
|
os.remove(zip_name)
|
||||||
|
if sys.platform != "win32":
|
||||||
|
os.chmod(self._exe_name, 0o755)
|
||||||
|
return self._exe_name
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def random_cdc():
|
||||||
|
cdc = random.choices(string.ascii_lowercase, k=26)
|
||||||
|
cdc[-6:-4] = map(str.upper, cdc[-6:-4])
|
||||||
|
cdc[2] = cdc[0]
|
||||||
|
cdc[3] = "_"
|
||||||
|
return "".join(cdc).encode()
|
||||||
|
|
||||||
|
def patch_binary(self):
|
||||||
|
"""
|
||||||
|
Patches the ChromeDriver binary
|
||||||
|
|
||||||
|
:return: False on failure, binary name on success
|
||||||
|
"""
|
||||||
|
linect = 0
|
||||||
|
replacement = self.random_cdc()
|
||||||
|
with io.open(self.executable_path, "r+b") as fh:
|
||||||
|
for line in iter(lambda: fh.readline(), b""):
|
||||||
|
if b"cdc_" in line:
|
||||||
|
fh.seek(-len(line), 1)
|
||||||
|
newline = re.sub(b"cdc_.{22}", replacement, line)
|
||||||
|
fh.write(newline)
|
||||||
|
linect += 1
|
||||||
|
return linect
|
||||||
|
|
||||||
|
|
||||||
|
def install(executable_path=None, target_version=None, *args, **kwargs):
|
||||||
|
ChromeDriverManager(executable_path, target_version, *args, **kwargs).install()
|
||||||
@ -2,7 +2,6 @@ import asyncio
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
import os
|
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
@ -188,6 +187,4 @@ def test():
|
|||||||
|
|
||||||
time.sleep(10)
|
time.sleep(10)
|
||||||
|
|
||||||
if os.name == "nt":
|
|
||||||
driver.close()
|
|
||||||
driver.quit()
|
driver.quit()
|
||||||
|
|||||||
@ -41,12 +41,12 @@ def start_detached(executable, *args):
|
|||||||
# close pipes
|
# close pipes
|
||||||
writer.close()
|
writer.close()
|
||||||
reader.close()
|
reader.close()
|
||||||
process.close()
|
|
||||||
|
|
||||||
return pid
|
return pid
|
||||||
|
|
||||||
|
|
||||||
def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
|
def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
|
||||||
|
|
||||||
# configure launch
|
# configure launch
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
if platform.system() == "Windows":
|
if platform.system() == "Windows":
|
||||||
|
|||||||
@ -56,6 +56,7 @@ class ChromeOptions(_ChromiumOptions):
|
|||||||
def handle_prefs(self, user_data_dir):
|
def handle_prefs(self, user_data_dir):
|
||||||
prefs = self.experimental_options.get("prefs")
|
prefs = self.experimental_options.get("prefs")
|
||||||
if prefs:
|
if prefs:
|
||||||
|
|
||||||
user_data_dir = user_data_dir or self._user_data_dir
|
user_data_dir = user_data_dir or self._user_data_dir
|
||||||
default_path = os.path.join(user_data_dir, "Default")
|
default_path = os.path.join(user_data_dir, "Default")
|
||||||
os.makedirs(default_path, exist_ok=True)
|
os.makedirs(default_path, exist_ok=True)
|
||||||
|
|||||||
@ -3,37 +3,45 @@
|
|||||||
|
|
||||||
from distutils.version import LooseVersion
|
from distutils.version import LooseVersion
|
||||||
import io
|
import io
|
||||||
import json
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import pathlib
|
|
||||||
import platform
|
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
import shutil
|
import secrets
|
||||||
import string
|
import string
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from urllib.request import urlopen
|
from urllib.request import urlopen
|
||||||
from urllib.request import urlretrieve
|
from urllib.request import urlretrieve
|
||||||
import zipfile
|
import zipfile
|
||||||
from multiprocessing import Lock
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2", "freebsd"))
|
IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
|
||||||
|
|
||||||
|
|
||||||
class Patcher(object):
|
class Patcher(object):
|
||||||
lock = Lock()
|
url_repo = "https://chromedriver.storage.googleapis.com"
|
||||||
|
zip_name = "chromedriver_%s.zip"
|
||||||
exe_name = "chromedriver%s"
|
exe_name = "chromedriver%s"
|
||||||
|
|
||||||
platform = sys.platform
|
platform = sys.platform
|
||||||
|
if platform.endswith("win32"):
|
||||||
|
zip_name %= "win32"
|
||||||
|
exe_name %= ".exe"
|
||||||
|
if platform.endswith(("linux", "linux2")):
|
||||||
|
zip_name %= "linux64"
|
||||||
|
exe_name %= ""
|
||||||
|
if platform.endswith("darwin"):
|
||||||
|
zip_name %= "mac64"
|
||||||
|
exe_name %= ""
|
||||||
|
|
||||||
if platform.endswith("win32"):
|
if platform.endswith("win32"):
|
||||||
d = "~/appdata/roaming/undetected_chromedriver"
|
d = "~/appdata/roaming/undetected_chromedriver"
|
||||||
elif "LAMBDA_TASK_ROOT" in os.environ:
|
elif "LAMBDA_TASK_ROOT" in os.environ:
|
||||||
d = "/tmp/undetected_chromedriver"
|
d = "/tmp/undetected_chromedriver"
|
||||||
elif platform.startswith(("linux", "linux2")):
|
elif platform.startswith(("linux","linux2")):
|
||||||
d = "~/.local/share/undetected_chromedriver"
|
d = "~/.local/share/undetected_chromedriver"
|
||||||
elif platform.endswith("darwin"):
|
elif platform.endswith("darwin"):
|
||||||
d = "~/Library/Application Support/undetected_chromedriver"
|
d = "~/Library/Application Support/undetected_chromedriver"
|
||||||
@ -41,14 +49,9 @@ class Patcher(object):
|
|||||||
d = "~/.undetected_chromedriver"
|
d = "~/.undetected_chromedriver"
|
||||||
data_path = os.path.abspath(os.path.expanduser(d))
|
data_path = os.path.abspath(os.path.expanduser(d))
|
||||||
|
|
||||||
def __init__(
|
def __init__(self, executable_path=None, force=False, version_main: int = 0):
|
||||||
self,
|
|
||||||
executable_path=None,
|
|
||||||
force=False,
|
|
||||||
version_main: int = 0,
|
|
||||||
user_multi_procs=False,
|
|
||||||
):
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
executable_path: None = automatic
|
executable_path: None = automatic
|
||||||
a full file path to the chromedriver executable
|
a full file path to the chromedriver executable
|
||||||
@ -57,39 +60,18 @@ class Patcher(object):
|
|||||||
version_main: 0 = auto
|
version_main: 0 = auto
|
||||||
specify main chrome version (rounded, ex: 82)
|
specify main chrome version (rounded, ex: 82)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self.force = force
|
self.force = force
|
||||||
self._custom_exe_path = False
|
self.executable_path = None
|
||||||
prefix = "undetected"
|
prefix = secrets.token_hex(8)
|
||||||
self.user_multi_procs = user_multi_procs
|
|
||||||
|
|
||||||
try:
|
|
||||||
# Try to convert version_main into an integer
|
|
||||||
version_main_int = int(version_main)
|
|
||||||
# check if version_main_int is less than or equal to e.g 114
|
|
||||||
self.is_old_chromedriver = version_main and version_main_int <= 114
|
|
||||||
except (ValueError,TypeError):
|
|
||||||
# Check not running inside Docker
|
|
||||||
if not os.path.exists("/app/chromedriver"):
|
|
||||||
# If the conversion fails, log an error message
|
|
||||||
logging.info("version_main cannot be converted to an integer")
|
|
||||||
# Set self.is_old_chromedriver to False if the conversion fails
|
|
||||||
self.is_old_chromedriver = False
|
|
||||||
|
|
||||||
# Needs to be called before self.exe_name is accessed
|
|
||||||
self._set_platform_name()
|
|
||||||
|
|
||||||
if not os.path.exists(self.data_path):
|
if not os.path.exists(self.data_path):
|
||||||
os.makedirs(self.data_path, exist_ok=True)
|
os.makedirs(self.data_path, exist_ok=True)
|
||||||
|
|
||||||
if not executable_path:
|
if not executable_path:
|
||||||
if sys.platform.startswith("freebsd"):
|
self.executable_path = os.path.join(
|
||||||
self.executable_path = os.path.join(
|
self.data_path, "_".join([prefix, self.exe_name])
|
||||||
self.data_path, self.exe_name
|
)
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.executable_path = os.path.join(
|
|
||||||
self.data_path, "_".join([prefix, self.exe_name])
|
|
||||||
)
|
|
||||||
|
|
||||||
if not IS_POSIX:
|
if not IS_POSIX:
|
||||||
if executable_path:
|
if executable_path:
|
||||||
@ -99,67 +81,20 @@ class Patcher(object):
|
|||||||
self.zip_path = os.path.join(self.data_path, prefix)
|
self.zip_path = os.path.join(self.data_path, prefix)
|
||||||
|
|
||||||
if not executable_path:
|
if not executable_path:
|
||||||
if not self.user_multi_procs:
|
self.executable_path = os.path.abspath(
|
||||||
self.executable_path = os.path.abspath(
|
os.path.join(".", self.executable_path)
|
||||||
os.path.join(".", self.executable_path)
|
)
|
||||||
)
|
|
||||||
|
self._custom_exe_path = False
|
||||||
|
|
||||||
if executable_path:
|
if executable_path:
|
||||||
self._custom_exe_path = True
|
self._custom_exe_path = True
|
||||||
self.executable_path = executable_path
|
self.executable_path = executable_path
|
||||||
|
|
||||||
# Set the correct repository to download the Chromedriver from
|
|
||||||
if self.is_old_chromedriver:
|
|
||||||
self.url_repo = "https://chromedriver.storage.googleapis.com"
|
|
||||||
else:
|
|
||||||
self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
|
|
||||||
|
|
||||||
self.version_main = version_main
|
self.version_main = version_main
|
||||||
self.version_full = None
|
self.version_full = None
|
||||||
|
|
||||||
def _set_platform_name(self):
|
def auto(self, executable_path=None, force=False, version_main=None):
|
||||||
"""
|
""""""
|
||||||
Set the platform and exe name based on the platform undetected_chromedriver is running on
|
|
||||||
in order to download the correct chromedriver.
|
|
||||||
"""
|
|
||||||
if self.platform.endswith("win32"):
|
|
||||||
self.platform_name = "win32"
|
|
||||||
self.exe_name %= ".exe"
|
|
||||||
if self.platform.endswith(("linux", "linux2")):
|
|
||||||
self.platform_name = "linux64"
|
|
||||||
self.exe_name %= ""
|
|
||||||
if self.platform.endswith("darwin"):
|
|
||||||
if self.is_old_chromedriver:
|
|
||||||
self.platform_name = "mac64"
|
|
||||||
else:
|
|
||||||
self.platform_name = "mac-x64"
|
|
||||||
self.exe_name %= ""
|
|
||||||
if self.platform.startswith("freebsd"):
|
|
||||||
self.platform_name = "freebsd"
|
|
||||||
self.exe_name %= ""
|
|
||||||
|
|
||||||
def auto(self, executable_path=None, force=False, version_main=None, _=None):
|
|
||||||
"""
|
|
||||||
|
|
||||||
Args:
|
|
||||||
executable_path:
|
|
||||||
force:
|
|
||||||
version_main:
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
|
|
||||||
"""
|
|
||||||
p = pathlib.Path(self.data_path)
|
|
||||||
if self.user_multi_procs:
|
|
||||||
with Lock():
|
|
||||||
files = list(p.rglob("*chromedriver*"))
|
|
||||||
most_recent = max(files, key=lambda f: f.stat().st_mtime)
|
|
||||||
files.remove(most_recent)
|
|
||||||
list(map(lambda f: f.unlink(), files))
|
|
||||||
if self.is_binary_patched(most_recent):
|
|
||||||
self.executable_path = str(most_recent)
|
|
||||||
return True
|
|
||||||
|
|
||||||
if executable_path:
|
if executable_path:
|
||||||
self.executable_path = executable_path
|
self.executable_path = executable_path
|
||||||
self._custom_exe_path = True
|
self._custom_exe_path = True
|
||||||
@ -176,104 +111,27 @@ class Patcher(object):
|
|||||||
if force is True:
|
if force is True:
|
||||||
self.force = force
|
self.force = force
|
||||||
|
|
||||||
|
|
||||||
if self.platform_name == "freebsd":
|
|
||||||
chromedriver_path = shutil.which("chromedriver")
|
|
||||||
|
|
||||||
if not os.path.isfile(chromedriver_path) or not os.access(chromedriver_path, os.X_OK):
|
|
||||||
logging.error("Chromedriver not installed!")
|
|
||||||
return
|
|
||||||
|
|
||||||
version_path = os.path.join(os.path.dirname(self.executable_path), "version.txt")
|
|
||||||
|
|
||||||
process = os.popen(f'"{chromedriver_path}" --version')
|
|
||||||
chromedriver_version = process.read().split(' ')[1].split(' ')[0]
|
|
||||||
process.close()
|
|
||||||
|
|
||||||
current_version = None
|
|
||||||
if os.path.isfile(version_path) or os.access(version_path, os.X_OK):
|
|
||||||
with open(version_path, 'r') as f:
|
|
||||||
current_version = f.read()
|
|
||||||
|
|
||||||
if current_version != chromedriver_version:
|
|
||||||
logging.info("Copying chromedriver executable...")
|
|
||||||
shutil.copy(chromedriver_path, self.executable_path)
|
|
||||||
os.chmod(self.executable_path, 0o755)
|
|
||||||
|
|
||||||
with open(version_path, 'w') as f:
|
|
||||||
f.write(chromedriver_version)
|
|
||||||
|
|
||||||
logging.info("Chromedriver executable copied!")
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
os.unlink(self.executable_path)
|
|
||||||
except PermissionError:
|
|
||||||
if self.force:
|
|
||||||
self.force_kill_instances(self.executable_path)
|
|
||||||
return self.auto(force=not self.force)
|
|
||||||
try:
|
|
||||||
if self.is_binary_patched():
|
|
||||||
# assumes already running AND patched
|
|
||||||
return True
|
|
||||||
except PermissionError:
|
|
||||||
pass
|
|
||||||
# return False
|
|
||||||
except FileNotFoundError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
release = self.fetch_release_number()
|
|
||||||
self.version_main = release.version[0]
|
|
||||||
self.version_full = release
|
|
||||||
self.unzip_package(self.fetch_package())
|
|
||||||
|
|
||||||
return self.patch()
|
|
||||||
|
|
||||||
def driver_binary_in_use(self, path: str = None) -> bool:
|
|
||||||
"""
|
|
||||||
naive test to check if a found chromedriver binary is
|
|
||||||
currently in use
|
|
||||||
|
|
||||||
Args:
|
|
||||||
path: a string or PathLike object to the binary to check.
|
|
||||||
if not specified, we check use this object's executable_path
|
|
||||||
"""
|
|
||||||
if not path:
|
|
||||||
path = self.executable_path
|
|
||||||
p = pathlib.Path(path)
|
|
||||||
|
|
||||||
if not p.exists():
|
|
||||||
raise OSError("file does not exist: %s" % p)
|
|
||||||
try:
|
try:
|
||||||
with open(p, mode="a+b") as fs:
|
os.unlink(self.executable_path)
|
||||||
exc = []
|
except PermissionError:
|
||||||
try:
|
if self.force:
|
||||||
|
self.force_kill_instances(self.executable_path)
|
||||||
fs.seek(0, 0)
|
return self.auto(force=not self.force)
|
||||||
except PermissionError as e:
|
try:
|
||||||
exc.append(e) # since some systems apprently allow seeking
|
if self.is_binary_patched():
|
||||||
# we conduct another test
|
# assumes already running AND patched
|
||||||
try:
|
|
||||||
fs.readline()
|
|
||||||
except PermissionError as e:
|
|
||||||
exc.append(e)
|
|
||||||
|
|
||||||
if exc:
|
|
||||||
|
|
||||||
return True
|
return True
|
||||||
return False
|
except PermissionError:
|
||||||
# ok safe to assume this is in use
|
pass
|
||||||
except Exception as e:
|
# return False
|
||||||
# logger.exception("whoops ", e)
|
except FileNotFoundError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def cleanup_unused_files(self):
|
release = self.fetch_release_number()
|
||||||
p = pathlib.Path(self.data_path)
|
self.version_main = release.version[0]
|
||||||
items = list(p.glob("*undetected*"))
|
self.version_full = release
|
||||||
for item in items:
|
self.unzip_package(self.fetch_package())
|
||||||
try:
|
return self.patch()
|
||||||
item.unlink()
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
def patch(self):
|
def patch(self):
|
||||||
self.patch_exe()
|
self.patch_exe()
|
||||||
@ -285,32 +143,12 @@ class Patcher(object):
|
|||||||
:return: version string
|
:return: version string
|
||||||
:rtype: LooseVersion
|
:rtype: LooseVersion
|
||||||
"""
|
"""
|
||||||
# Endpoint for old versions of Chromedriver (114 and below)
|
path = "/latest_release"
|
||||||
if self.is_old_chromedriver:
|
if self.version_main:
|
||||||
path = f"/latest_release_{self.version_main}"
|
path += f"_{self.version_main}"
|
||||||
path = path.upper()
|
path = path.upper()
|
||||||
logger.debug("getting release number from %s" % path)
|
|
||||||
return LooseVersion(urlopen(self.url_repo + path).read().decode())
|
|
||||||
|
|
||||||
# Endpoint for new versions of Chromedriver (115+)
|
|
||||||
if not self.version_main:
|
|
||||||
# Fetch the latest version
|
|
||||||
path = "/last-known-good-versions-with-downloads.json"
|
|
||||||
logger.debug("getting release number from %s" % path)
|
|
||||||
with urlopen(self.url_repo + path) as conn:
|
|
||||||
response = conn.read().decode()
|
|
||||||
|
|
||||||
last_versions = json.loads(response)
|
|
||||||
return LooseVersion(last_versions["channels"]["Stable"]["version"])
|
|
||||||
|
|
||||||
# Fetch the latest minor version of the major version provided
|
|
||||||
path = "/latest-versions-per-milestone-with-downloads.json"
|
|
||||||
logger.debug("getting release number from %s" % path)
|
logger.debug("getting release number from %s" % path)
|
||||||
with urlopen(self.url_repo + path) as conn:
|
return LooseVersion(urlopen(self.url_repo + path).read().decode())
|
||||||
response = conn.read().decode()
|
|
||||||
|
|
||||||
major_versions = json.loads(response)
|
|
||||||
return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
|
|
||||||
|
|
||||||
def parse_exe_version(self):
|
def parse_exe_version(self):
|
||||||
with io.open(self.executable_path, "rb") as f:
|
with io.open(self.executable_path, "rb") as f:
|
||||||
@ -325,16 +163,10 @@ class Patcher(object):
|
|||||||
|
|
||||||
:return: path to downloaded file
|
:return: path to downloaded file
|
||||||
"""
|
"""
|
||||||
zip_name = f"chromedriver_{self.platform_name}.zip"
|
u = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, self.zip_name)
|
||||||
if self.is_old_chromedriver:
|
logger.debug("downloading from %s" % u)
|
||||||
download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
|
# return urlretrieve(u, filename=self.data_path)[0]
|
||||||
else:
|
return urlretrieve(u)[0]
|
||||||
zip_name = zip_name.replace("_", "-", 1)
|
|
||||||
download_url = "https://storage.googleapis.com/chrome-for-testing-public/%s/%s/%s"
|
|
||||||
download_url %= (self.version_full.vstring, self.platform_name, zip_name)
|
|
||||||
|
|
||||||
logger.debug("downloading from %s" % download_url)
|
|
||||||
return urlretrieve(download_url)[0]
|
|
||||||
|
|
||||||
def unzip_package(self, fp):
|
def unzip_package(self, fp):
|
||||||
"""
|
"""
|
||||||
@ -342,12 +174,6 @@ class Patcher(object):
|
|||||||
|
|
||||||
:return: path to unpacked executable
|
:return: path to unpacked executable
|
||||||
"""
|
"""
|
||||||
exe_path = self.exe_name
|
|
||||||
if not self.is_old_chromedriver:
|
|
||||||
# The new chromedriver unzips into its own folder
|
|
||||||
zip_name = f"chromedriver-{self.platform_name}"
|
|
||||||
exe_path = os.path.join(zip_name, self.exe_name)
|
|
||||||
|
|
||||||
logger.debug("unzipping %s" % fp)
|
logger.debug("unzipping %s" % fp)
|
||||||
try:
|
try:
|
||||||
os.unlink(self.zip_path)
|
os.unlink(self.zip_path)
|
||||||
@ -356,10 +182,10 @@ class Patcher(object):
|
|||||||
|
|
||||||
os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
|
os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
|
||||||
with zipfile.ZipFile(fp, mode="r") as zf:
|
with zipfile.ZipFile(fp, mode="r") as zf:
|
||||||
zf.extractall(self.zip_path)
|
zf.extract(self.exe_name, self.zip_path)
|
||||||
os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
|
os.rename(os.path.join(self.zip_path, self.exe_name), self.executable_path)
|
||||||
os.remove(fp)
|
os.remove(fp)
|
||||||
shutil.rmtree
|
os.rmdir(self.zip_path)
|
||||||
os.chmod(self.executable_path, 0o755)
|
os.chmod(self.executable_path, 0o755)
|
||||||
return self.executable_path
|
return self.executable_path
|
||||||
|
|
||||||
@ -380,46 +206,43 @@ class Patcher(object):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def gen_random_cdc():
|
def gen_random_cdc():
|
||||||
cdc = random.choices(string.ascii_letters, k=27)
|
cdc = random.choices(string.ascii_lowercase, k=26)
|
||||||
|
cdc[-6:-4] = map(str.upper, cdc[-6:-4])
|
||||||
|
cdc[2] = cdc[0]
|
||||||
|
cdc[3] = "_"
|
||||||
return "".join(cdc).encode()
|
return "".join(cdc).encode()
|
||||||
|
|
||||||
def is_binary_patched(self, executable_path=None):
|
def is_binary_patched(self, executable_path=None):
|
||||||
|
"""simple check if executable is patched.
|
||||||
|
|
||||||
|
:return: False if not patched, else True
|
||||||
|
"""
|
||||||
executable_path = executable_path or self.executable_path
|
executable_path = executable_path or self.executable_path
|
||||||
try:
|
with io.open(executable_path, "rb") as fh:
|
||||||
with io.open(executable_path, "rb") as fh:
|
for line in iter(lambda: fh.readline(), b""):
|
||||||
return fh.read().find(b"undetected chromedriver") != -1
|
if b"cdc_" in line:
|
||||||
except FileNotFoundError:
|
return False
|
||||||
return False
|
else:
|
||||||
|
return True
|
||||||
|
|
||||||
def patch_exe(self):
|
def patch_exe(self):
|
||||||
start = time.perf_counter()
|
"""
|
||||||
|
Patches the ChromeDriver binary
|
||||||
|
|
||||||
|
:return: False on failure, binary name on success
|
||||||
|
"""
|
||||||
logger.info("patching driver executable %s" % self.executable_path)
|
logger.info("patching driver executable %s" % self.executable_path)
|
||||||
|
|
||||||
|
linect = 0
|
||||||
|
replacement = self.gen_random_cdc()
|
||||||
with io.open(self.executable_path, "r+b") as fh:
|
with io.open(self.executable_path, "r+b") as fh:
|
||||||
content = fh.read()
|
for line in iter(lambda: fh.readline(), b""):
|
||||||
# match_injected_codeblock = re.search(rb"{window.*;}", content)
|
if b"cdc_" in line:
|
||||||
match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
|
fh.seek(-len(line), 1)
|
||||||
if match_injected_codeblock:
|
newline = re.sub(b"cdc_.{22}", replacement, line)
|
||||||
target_bytes = match_injected_codeblock[0]
|
fh.write(newline)
|
||||||
new_target_bytes = (
|
linect += 1
|
||||||
b'{console.log("undetected chromedriver 1337!")}'.ljust(
|
return linect
|
||||||
len(target_bytes), b" "
|
|
||||||
)
|
|
||||||
)
|
|
||||||
new_content = content.replace(target_bytes, new_target_bytes)
|
|
||||||
if new_content == content:
|
|
||||||
logger.warning(
|
|
||||||
"something went wrong patching the driver binary. could not find injection code block"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.debug(
|
|
||||||
"found block:\n%s\nreplacing with:\n%s"
|
|
||||||
% (target_bytes, new_target_bytes)
|
|
||||||
)
|
|
||||||
fh.seek(0)
|
|
||||||
fh.write(new_content)
|
|
||||||
logger.debug(
|
|
||||||
"patching took us {:.2f} seconds".format(time.perf_counter() - start)
|
|
||||||
)
|
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return "{0:s}({1:s})".format(
|
return "{0:s}({1:s})".format(
|
||||||
@ -428,6 +251,7 @@ class Patcher(object):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def __del__(self):
|
def __del__(self):
|
||||||
|
|
||||||
if self._custom_exe_path:
|
if self._custom_exe_path:
|
||||||
# if the driver binary is specified by user
|
# if the driver binary is specified by user
|
||||||
# we assume it is important enough to not delete it
|
# we assume it is important enough to not delete it
|
||||||
@ -435,17 +259,21 @@ class Patcher(object):
|
|||||||
else:
|
else:
|
||||||
timeout = 3 # stop trying after this many seconds
|
timeout = 3 # stop trying after this many seconds
|
||||||
t = time.monotonic()
|
t = time.monotonic()
|
||||||
now = lambda: time.monotonic()
|
while True:
|
||||||
while now() - t > timeout:
|
now = time.monotonic()
|
||||||
# we don't want to wait until the end of time
|
if now - t > timeout:
|
||||||
|
# we don't want to wait until the end of time
|
||||||
|
logger.debug(
|
||||||
|
"could not unlink %s in time (%d seconds)"
|
||||||
|
% (self.executable_path, timeout)
|
||||||
|
)
|
||||||
|
break
|
||||||
try:
|
try:
|
||||||
if self.user_multi_procs:
|
|
||||||
break
|
|
||||||
os.unlink(self.executable_path)
|
os.unlink(self.executable_path)
|
||||||
logger.debug("successfully unlinked %s" % self.executable_path)
|
logger.debug("successfully unlinked %s" % self.executable_path)
|
||||||
break
|
break
|
||||||
except (OSError, RuntimeError, PermissionError):
|
except (OSError, RuntimeError, PermissionError):
|
||||||
time.sleep(0.01)
|
time.sleep(0.1)
|
||||||
continue
|
continue
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
break
|
break
|
||||||
|
|||||||
@ -6,7 +6,6 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
@ -64,7 +63,9 @@ class Reactor(threading.Thread):
|
|||||||
break
|
break
|
||||||
|
|
||||||
async def listen(self):
|
async def listen(self):
|
||||||
|
|
||||||
while self.running:
|
while self.running:
|
||||||
|
|
||||||
await self._wait_service_started()
|
await self._wait_service_started()
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
@ -73,7 +74,9 @@ class Reactor(threading.Thread):
|
|||||||
log_entries = self.driver.get_log("performance")
|
log_entries = self.driver.get_log("performance")
|
||||||
|
|
||||||
for entry in log_entries:
|
for entry in log_entries:
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|
||||||
obj_serialized: str = entry.get("message")
|
obj_serialized: str = entry.get("message")
|
||||||
obj = json.loads(obj_serialized)
|
obj = json.loads(obj_serialized)
|
||||||
message = obj.get("message")
|
message = obj.get("message")
|
||||||
|
|||||||
4
src/undetected_chromedriver/v2.py
Normal file
4
src/undetected_chromedriver/v2.py
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
# for backward compatibility
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.modules[__name__] = sys.modules[__package__]
|
||||||
@ -1,7 +1,6 @@
|
|||||||
from typing import List
|
|
||||||
|
|
||||||
from selenium.webdriver.common.by import By
|
from selenium.webdriver.common.by import By
|
||||||
import selenium.webdriver.remote.webelement
|
import selenium.webdriver.remote.webelement
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
|
||||||
class WebElement(selenium.webdriver.remote.webelement.WebElement):
|
class WebElement(selenium.webdriver.remote.webelement.WebElement):
|
||||||
|
|||||||
221
src/utils.py
221
src/utils.py
@ -1,19 +1,13 @@
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import platform
|
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
import urllib.parse
|
|
||||||
|
|
||||||
from selenium.webdriver.chrome.webdriver import WebDriver
|
from selenium.webdriver.chrome.webdriver import WebDriver
|
||||||
import undetected_chromedriver as uc
|
import undetected_chromedriver as uc
|
||||||
|
|
||||||
FLARESOLVERR_VERSION = None
|
FLARESOLVERR_VERSION = None
|
||||||
PLATFORM_VERSION = None
|
|
||||||
CHROME_EXE_PATH = None
|
|
||||||
CHROME_MAJOR_VERSION = None
|
CHROME_MAJOR_VERSION = None
|
||||||
USER_AGENT = None
|
USER_AGENT = None
|
||||||
XVFB_DISPLAY = None
|
XVFB_DISPLAY = None
|
||||||
@ -34,139 +28,26 @@ def get_flaresolverr_version() -> str:
|
|||||||
return FLARESOLVERR_VERSION
|
return FLARESOLVERR_VERSION
|
||||||
|
|
||||||
package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
|
package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
|
||||||
if not os.path.isfile(package_path):
|
|
||||||
package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
|
|
||||||
with open(package_path) as f:
|
with open(package_path) as f:
|
||||||
FLARESOLVERR_VERSION = json.loads(f.read())['version']
|
FLARESOLVERR_VERSION = json.loads(f.read())['version']
|
||||||
return FLARESOLVERR_VERSION
|
return FLARESOLVERR_VERSION
|
||||||
|
|
||||||
def get_current_platform() -> str:
|
|
||||||
global PLATFORM_VERSION
|
|
||||||
if PLATFORM_VERSION is not None:
|
|
||||||
return PLATFORM_VERSION
|
|
||||||
PLATFORM_VERSION = os.name
|
|
||||||
return PLATFORM_VERSION
|
|
||||||
|
|
||||||
|
def get_webdriver() -> WebDriver:
|
||||||
def create_proxy_extension(proxy: dict) -> str:
|
global PATCHED_DRIVER_PATH
|
||||||
parsed_url = urllib.parse.urlparse(proxy['url'])
|
|
||||||
scheme = parsed_url.scheme
|
|
||||||
host = parsed_url.hostname
|
|
||||||
port = parsed_url.port
|
|
||||||
username = proxy['username']
|
|
||||||
password = proxy['password']
|
|
||||||
manifest_json = """
|
|
||||||
{
|
|
||||||
"version": "1.0.0",
|
|
||||||
"manifest_version": 2,
|
|
||||||
"name": "Chrome Proxy",
|
|
||||||
"permissions": [
|
|
||||||
"proxy",
|
|
||||||
"tabs",
|
|
||||||
"unlimitedStorage",
|
|
||||||
"storage",
|
|
||||||
"<all_urls>",
|
|
||||||
"webRequest",
|
|
||||||
"webRequestBlocking"
|
|
||||||
],
|
|
||||||
"background": {"scripts": ["background.js"]},
|
|
||||||
"minimum_chrome_version": "76.0.0"
|
|
||||||
}
|
|
||||||
"""
|
|
||||||
|
|
||||||
background_js = """
|
|
||||||
var config = {
|
|
||||||
mode: "fixed_servers",
|
|
||||||
rules: {
|
|
||||||
singleProxy: {
|
|
||||||
scheme: "%s",
|
|
||||||
host: "%s",
|
|
||||||
port: %d
|
|
||||||
},
|
|
||||||
bypassList: ["localhost"]
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
|
|
||||||
|
|
||||||
function callbackFn(details) {
|
|
||||||
return {
|
|
||||||
authCredentials: {
|
|
||||||
username: "%s",
|
|
||||||
password: "%s"
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
chrome.webRequest.onAuthRequired.addListener(
|
|
||||||
callbackFn,
|
|
||||||
{ urls: ["<all_urls>"] },
|
|
||||||
['blocking']
|
|
||||||
);
|
|
||||||
""" % (
|
|
||||||
scheme,
|
|
||||||
host,
|
|
||||||
port,
|
|
||||||
username,
|
|
||||||
password
|
|
||||||
)
|
|
||||||
|
|
||||||
proxy_extension_dir = tempfile.mkdtemp()
|
|
||||||
|
|
||||||
with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f:
|
|
||||||
f.write(manifest_json)
|
|
||||||
|
|
||||||
with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f:
|
|
||||||
f.write(background_js)
|
|
||||||
|
|
||||||
return proxy_extension_dir
|
|
||||||
|
|
||||||
|
|
||||||
def get_webdriver(proxy: dict = None) -> WebDriver:
|
|
||||||
global PATCHED_DRIVER_PATH, USER_AGENT
|
|
||||||
logging.debug('Launching web browser...')
|
logging.debug('Launching web browser...')
|
||||||
|
|
||||||
# undetected_chromedriver
|
# undetected_chromedriver
|
||||||
options = uc.ChromeOptions()
|
options = uc.ChromeOptions()
|
||||||
options.add_argument('--no-sandbox')
|
options.add_argument('--no-sandbox')
|
||||||
options.add_argument('--window-size=1920,1080')
|
options.add_argument('--window-size=1920,1080')
|
||||||
options.add_argument('--disable-search-engine-choice-screen')
|
|
||||||
# todo: this param shows a warning in chrome head-full
|
# todo: this param shows a warning in chrome head-full
|
||||||
options.add_argument('--disable-setuid-sandbox')
|
options.add_argument('--disable-setuid-sandbox')
|
||||||
options.add_argument('--disable-dev-shm-usage')
|
options.add_argument('--disable-dev-shm-usage')
|
||||||
# this option removes the zygote sandbox (it seems that the resolution is a bit faster)
|
# this option removes the zygote sandbox (it seems that the resolution is a bit faster)
|
||||||
options.add_argument('--no-zygote')
|
options.add_argument('--no-zygote')
|
||||||
# attempt to fix Docker ARM32 build
|
|
||||||
IS_ARMARCH = platform.machine().startswith(('arm', 'aarch'))
|
|
||||||
if IS_ARMARCH:
|
|
||||||
options.add_argument('--disable-gpu-sandbox')
|
|
||||||
options.add_argument('--disable-software-rasterizer')
|
|
||||||
options.add_argument('--ignore-certificate-errors')
|
|
||||||
options.add_argument('--ignore-ssl-errors')
|
|
||||||
# fix GL errors in ASUSTOR NAS
|
|
||||||
# https://github.com/FlareSolverr/FlareSolverr/issues/782
|
|
||||||
# https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
|
|
||||||
# https://peter.sh/experiments/chromium-command-line-switches/#use-gl
|
|
||||||
options.add_argument('--use-gl=swiftshader')
|
|
||||||
|
|
||||||
language = os.environ.get('LANG', None)
|
# note: headless mode is detected (options.headless = True)
|
||||||
if language is not None:
|
|
||||||
options.add_argument('--accept-lang=%s' % language)
|
|
||||||
|
|
||||||
# Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
|
|
||||||
if USER_AGENT is not None:
|
|
||||||
options.add_argument('--user-agent=%s' % USER_AGENT)
|
|
||||||
|
|
||||||
proxy_extension_dir = None
|
|
||||||
if proxy and all(key in proxy for key in ['url', 'username', 'password']):
|
|
||||||
proxy_extension_dir = create_proxy_extension(proxy)
|
|
||||||
options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
|
|
||||||
elif proxy and 'url' in proxy:
|
|
||||||
proxy_url = proxy['url']
|
|
||||||
logging.debug("Using webdriver proxy: %s", proxy_url)
|
|
||||||
options.add_argument('--proxy-server=%s' % proxy_url)
|
|
||||||
|
|
||||||
# note: headless mode is detected (headless = True)
|
|
||||||
# we launch the browser in head-full mode with the window hidden
|
# we launch the browser in head-full mode with the window hidden
|
||||||
windows_headless = False
|
windows_headless = False
|
||||||
if get_config_headless():
|
if get_config_headless():
|
||||||
@ -174,8 +55,6 @@ def get_webdriver(proxy: dict = None) -> WebDriver:
|
|||||||
windows_headless = True
|
windows_headless = True
|
||||||
else:
|
else:
|
||||||
start_xvfb_display()
|
start_xvfb_display()
|
||||||
# For normal headless mode:
|
|
||||||
# options.add_argument('--headless')
|
|
||||||
|
|
||||||
# if we are inside the Docker container, we avoid downloading the driver
|
# if we are inside the Docker container, we avoid downloading the driver
|
||||||
driver_exe_path = None
|
driver_exe_path = None
|
||||||
@ -188,27 +67,15 @@ def get_webdriver(proxy: dict = None) -> WebDriver:
|
|||||||
if PATCHED_DRIVER_PATH is not None:
|
if PATCHED_DRIVER_PATH is not None:
|
||||||
driver_exe_path = PATCHED_DRIVER_PATH
|
driver_exe_path = PATCHED_DRIVER_PATH
|
||||||
|
|
||||||
# detect chrome path
|
|
||||||
browser_executable_path = get_chrome_exe_path()
|
|
||||||
|
|
||||||
# downloads and patches the chromedriver
|
# downloads and patches the chromedriver
|
||||||
# if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
|
# if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
|
||||||
try:
|
driver = uc.Chrome(options=options, driver_executable_path=driver_exe_path, version_main=version_main,
|
||||||
driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
|
windows_headless=windows_headless)
|
||||||
driver_executable_path=driver_exe_path, version_main=version_main,
|
|
||||||
windows_headless=windows_headless, headless=get_config_headless())
|
|
||||||
except Exception as e:
|
|
||||||
logging.error("Error starting Chrome: %s" % e)
|
|
||||||
|
|
||||||
# save the patched driver to avoid re-downloads
|
# save the patched driver to avoid re-downloads
|
||||||
if driver_exe_path is None:
|
if driver_exe_path is None:
|
||||||
PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
|
PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
|
||||||
if PATCHED_DRIVER_PATH != driver.patcher.executable_path:
|
shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
|
||||||
shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
|
|
||||||
|
|
||||||
# clean up proxy extension directory
|
|
||||||
if proxy_extension_dir is not None:
|
|
||||||
shutil.rmtree(proxy_extension_dir)
|
|
||||||
|
|
||||||
# selenium vanilla
|
# selenium vanilla
|
||||||
# options = webdriver.ChromeOptions()
|
# options = webdriver.ChromeOptions()
|
||||||
@ -221,45 +88,23 @@ def get_webdriver(proxy: dict = None) -> WebDriver:
|
|||||||
return driver
|
return driver
|
||||||
|
|
||||||
|
|
||||||
def get_chrome_exe_path() -> str:
|
|
||||||
global CHROME_EXE_PATH
|
|
||||||
if CHROME_EXE_PATH is not None:
|
|
||||||
return CHROME_EXE_PATH
|
|
||||||
# linux pyinstaller bundle
|
|
||||||
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
|
|
||||||
if os.path.exists(chrome_path):
|
|
||||||
if not os.access(chrome_path, os.X_OK):
|
|
||||||
raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
|
|
||||||
f'Please, extract the archive with "tar xzf <file.tar.gz>".')
|
|
||||||
CHROME_EXE_PATH = chrome_path
|
|
||||||
return CHROME_EXE_PATH
|
|
||||||
# windows pyinstaller bundle
|
|
||||||
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
|
|
||||||
if os.path.exists(chrome_path):
|
|
||||||
CHROME_EXE_PATH = chrome_path
|
|
||||||
return CHROME_EXE_PATH
|
|
||||||
# system
|
|
||||||
CHROME_EXE_PATH = uc.find_chrome_executable()
|
|
||||||
return CHROME_EXE_PATH
|
|
||||||
|
|
||||||
|
|
||||||
def get_chrome_major_version() -> str:
|
def get_chrome_major_version() -> str:
|
||||||
global CHROME_MAJOR_VERSION
|
global CHROME_MAJOR_VERSION
|
||||||
if CHROME_MAJOR_VERSION is not None:
|
if CHROME_MAJOR_VERSION is not None:
|
||||||
return CHROME_MAJOR_VERSION
|
return CHROME_MAJOR_VERSION
|
||||||
|
|
||||||
if os.name == 'nt':
|
if os.name == 'nt':
|
||||||
# Example: '104.0.5112.79'
|
|
||||||
try:
|
try:
|
||||||
complete_version = extract_version_nt_executable(get_chrome_exe_path())
|
stream = os.popen(
|
||||||
|
'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
|
||||||
|
output = stream.read()
|
||||||
|
# Example: '104.0.5112.79'
|
||||||
|
complete_version = extract_version_registry(output)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
# Example: '104.0.5112.79'
|
||||||
complete_version = extract_version_nt_registry()
|
complete_version = extract_version_folder()
|
||||||
except Exception:
|
|
||||||
# Example: '104.0.5112.79'
|
|
||||||
complete_version = extract_version_nt_folder()
|
|
||||||
else:
|
else:
|
||||||
chrome_path = get_chrome_exe_path()
|
chrome_path = uc.find_chrome_executable()
|
||||||
process = os.popen(f'"{chrome_path}" --version')
|
process = os.popen(f'"{chrome_path}" --version')
|
||||||
# Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
|
# Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
|
||||||
# Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
|
# Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
|
||||||
@ -267,32 +112,24 @@ def get_chrome_major_version() -> str:
|
|||||||
process.close()
|
process.close()
|
||||||
|
|
||||||
CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
|
CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
|
||||||
|
logging.info(f"Chrome major version: {CHROME_MAJOR_VERSION}")
|
||||||
return CHROME_MAJOR_VERSION
|
return CHROME_MAJOR_VERSION
|
||||||
|
|
||||||
|
|
||||||
def extract_version_nt_executable(exe_path: str) -> str:
|
def extract_version_registry(output) -> str:
|
||||||
import pefile
|
try:
|
||||||
pe = pefile.PE(exe_path, fast_load=True)
|
google_version = ''
|
||||||
pe.parse_data_directories(
|
for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]:
|
||||||
directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
|
if letter != '\n':
|
||||||
)
|
google_version += letter
|
||||||
return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
|
else:
|
||||||
|
break
|
||||||
|
return google_version.strip()
|
||||||
|
except TypeError:
|
||||||
|
return ''
|
||||||
|
|
||||||
|
|
||||||
def extract_version_nt_registry() -> str:
|
def extract_version_folder() -> str:
|
||||||
stream = os.popen(
|
|
||||||
'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
|
|
||||||
output = stream.read()
|
|
||||||
google_version = ''
|
|
||||||
for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]:
|
|
||||||
if letter != '\n':
|
|
||||||
google_version += letter
|
|
||||||
else:
|
|
||||||
break
|
|
||||||
return google_version.strip()
|
|
||||||
|
|
||||||
|
|
||||||
def extract_version_nt_folder() -> str:
|
|
||||||
# Check if the Chrome folder exists in the x32 or x64 Program Files folders.
|
# Check if the Chrome folder exists in the x32 or x64 Program Files folders.
|
||||||
for i in range(2):
|
for i in range(2):
|
||||||
path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'
|
path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'
|
||||||
@ -317,15 +154,11 @@ def get_user_agent(driver=None) -> str:
|
|||||||
if driver is None:
|
if driver is None:
|
||||||
driver = get_webdriver()
|
driver = get_webdriver()
|
||||||
USER_AGENT = driver.execute_script("return navigator.userAgent")
|
USER_AGENT = driver.execute_script("return navigator.userAgent")
|
||||||
# Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
|
|
||||||
USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
|
|
||||||
return USER_AGENT
|
return USER_AGENT
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise Exception("Error getting browser User-Agent. " + str(e))
|
raise Exception("Error getting browser User-Agent. " + str(e))
|
||||||
finally:
|
finally:
|
||||||
if driver is not None:
|
if driver is not None:
|
||||||
if PLATFORM_VERSION == "nt":
|
|
||||||
driver.close()
|
|
||||||
driver.quit()
|
driver.quit()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user