diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index fd29059..68f2733 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -7,7 +7,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e0e1ec..24a4b4c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,26 +6,15 @@ on: - 'v*.*.*' jobs: - build: + create-release: name: Create release - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout code uses: actions/checkout@v3 with: fetch-depth: 0 # get all commits, branches and tags (required for the changelog) - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: '16' - - - name: Build artifacts - run: | - npm install - npm run build - npm run package - - name: Build changelog id: github_changelog run: | @@ -47,9 +36,60 @@ jobs: draft: false prerelease: false + build-linux: + name: Build Linux binary + needs: create-release + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # get all commits, branches and tags (required for the changelog) + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Build artifacts + run: | + python -m pip install -r requirements.txt + python -m pip install pyinstaller==5.9.0 + cd src + python build_package.py + - name: Upload release artifacts - uses: alexellis/upload-assets@0.2.2 + uses: alexellis/upload-assets@0.4.0 env: GITHUB_TOKEN: ${{ secrets.GH_PAT }} with: - asset_paths: '["./bin/*.zip"]' + asset_paths: '["./dist/flaresolverr_*"]' + + build-windows: + name: Build Windows binary + needs: create-release + runs-on: windows-2022 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 # get all commits, branches and tags (required for the changelog) + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Build artifacts + run: | + python -m pip install -r requirements.txt + python -m pip install pyinstaller==5.9.0 + cd src + python build_package.py + + - name: Upload release artifacts + uses: alexellis/upload-assets@0.4.0 + env: + GITHUB_TOKEN: ${{ secrets.GH_PAT }} + with: + asset_paths: '["./dist/flaresolverr_*"]' diff --git a/.gitignore b/.gitignore index 8439f8d..b84573e 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ __pycache__/ build/ develop-eggs/ dist/ +dist_chrome/ downloads/ eggs/ .eggs/ diff --git a/README.md b/README.md index da301d5..2e95950 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,13 @@ Remember to restart the Docker daemon and the container after the update. ### Precompiled binaries -Precompiled binaries are not currently available for v3. Please see https://github.com/FlareSolverr/FlareSolverr/issues/660 for updates, -or below for instructions of how to build FlareSolverr from source code. +This is the recommended way for Windows users. +* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64. +* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration. ### From source code -* Install [Python 3.10](https://www.python.org/downloads/). +* Install [Python 3.11](https://www.python.org/downloads/). * Install [Chrome](https://www.google.com/intl/en_us/chrome/) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) web browser. * (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package. * Clone this repository and open a shell in that path. diff --git a/requirements.txt b/requirements.txt index 0f4a8d4..f68ea4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,9 @@ selenium==4.8.2 func-timeout==4.3.5 # required by undetected_chromedriver requests==2.28.2 +certifi==2022.12.7 websockets==10.4 # only required for linux xvfbwrapper==0.2.9 +# only required for windows +pefile==2023.2.7 diff --git a/src/build_package.py b/src/build_package.py new file mode 100644 index 0000000..7a16e83 --- /dev/null +++ b/src/build_package.py @@ -0,0 +1,86 @@ +import os +import platform +import shutil +import subprocess +import sys +import zipfile + +import requests + + +def clean_files(): + try: + shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build')) + except Exception: + pass + try: + shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')) + except Exception: + pass + try: + shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')) + except Exception: + pass + + +def download_chromium(): + # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/ + revision = "1090006" if os.name == 'nt' else '1090007' + arch = 'Win' if os.name == 'nt' else 'Linux_x64' + dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux' + dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome') + dl_path_folder = os.path.join(dl_path, dl_file) + dl_path_zip = dl_path_folder + '.zip' + + # response = requests.get( + # f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE', + # timeout=30) + # revision = response.text.strip() + print("Downloading revision: " + revision) + + os.mkdir(dl_path) + with requests.get( + f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip', + stream=True) as r: + r.raise_for_status() + with open(dl_path_zip, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + print("File downloaded: " + dl_path_zip) + with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref: + zip_ref.extractall(dl_path) + os.remove(dl_path_zip) + shutil.move(dl_path_folder, os.path.join(dl_path, "chrome")) + + +def run_pyinstaller(): + sep = ';' if os.name == 'nt' else ':' + subprocess.check_call([sys.executable, "-m", "PyInstaller", + "--onefile", + "--add-data", f"package.json{sep}.", + "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome", + os.path.join("src", "flaresolverr.py")], + cwd=os.pardir) + exe_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist') + exe_name = 'flaresolverr.exe' if os.name == 'nt' else 'flaresolverr' + exe_new_name = 'flaresolverr_windows_x64.exe' if os.name == 'nt' else 'flaresolverr_linux_x64' + exe_path = os.path.join(exe_folder, exe_name) + exe_new_path = os.path.join(exe_folder, exe_new_name) + shutil.move(exe_path, exe_new_path) + print("Executable path: " + exe_new_path) + + +if __name__ == "__main__": + print("Building package...") + print("Platform: " + platform.platform()) + + print("Cleaning previous build...") + clean_files() + + print("Downloading Chromium...") + download_chromium() + + print("Building pyinstaller executable... ") + run_pyinstaller() + +# NOTE: python -m pip install pyinstaller diff --git a/src/flaresolverr.py b/src/flaresolverr.py index cf27177..a477316 100644 --- a/src/flaresolverr.py +++ b/src/flaresolverr.py @@ -3,6 +3,7 @@ import logging import os import sys +import certifi from bottle import run, response, Bottle, request, ServerAdapter from bottle_plugins.error_plugin import error_plugin @@ -60,6 +61,12 @@ def controller_v1(): if __name__ == "__main__": + # fix ssl certificates for compiled binaries + # https://github.com/pyinstaller/pyinstaller/issues/7229 + # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3 + os.environ["REQUESTS_CA_BUNDLE"] = certifi.where() + os.environ["SSL_CERT_FILE"] = certifi.where() + # validate configuration log_level = os.environ.get('LOG_LEVEL', 'info').upper() log_html = utils.get_config_log_html() diff --git a/src/utils.py b/src/utils.py index 2c8739e..eee8868 100644 --- a/src/utils.py +++ b/src/utils.py @@ -8,6 +8,7 @@ from selenium.webdriver.chrome.webdriver import WebDriver import undetected_chromedriver as uc FLARESOLVERR_VERSION = None +CHROME_EXE_PATH = None CHROME_MAJOR_VERSION = None USER_AGENT = None XVFB_DISPLAY = None @@ -28,6 +29,8 @@ def get_flaresolverr_version() -> str: return FLARESOLVERR_VERSION package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json') + if not os.path.isfile(package_path): + package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json') with open(package_path) as f: FLARESOLVERR_VERSION = json.loads(f.read())['version'] return FLARESOLVERR_VERSION @@ -72,9 +75,13 @@ def get_webdriver() -> WebDriver: if PATCHED_DRIVER_PATH is not None: driver_exe_path = PATCHED_DRIVER_PATH + # detect chrome path + browser_executable_path = get_chrome_exe_path() + # downloads and patches the chromedriver # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time - driver = uc.Chrome(options=options, driver_executable_path=driver_exe_path, version_main=version_main, + driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path, + driver_executable_path=driver_exe_path, version_main=version_main, windows_headless=windows_headless) # save the patched driver to avoid re-downloads @@ -94,7 +101,22 @@ def get_webdriver() -> WebDriver: def get_chrome_exe_path() -> str: - return uc.find_chrome_executable() + global CHROME_EXE_PATH + if CHROME_EXE_PATH is not None: + return CHROME_EXE_PATH + # linux pyinstaller bundle + chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome") + if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK): + CHROME_EXE_PATH = chrome_path + return CHROME_EXE_PATH + # windows pyinstaller bundle + chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe") + if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK): + CHROME_EXE_PATH = chrome_path + return CHROME_EXE_PATH + # system + CHROME_EXE_PATH = uc.find_chrome_executable() + return CHROME_EXE_PATH def get_chrome_major_version() -> str: @@ -103,17 +125,17 @@ def get_chrome_major_version() -> str: return CHROME_MAJOR_VERSION if os.name == 'nt': + # Example: '104.0.5112.79' try: - stream = os.popen( - 'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"') - output = stream.read() - # Example: '104.0.5112.79' - complete_version = extract_version_registry(output) + complete_version = extract_version_nt_executable(get_chrome_exe_path()) except Exception: - # Example: '104.0.5112.79' - complete_version = extract_version_folder() + try: + complete_version = extract_version_nt_registry() + except Exception: + # Example: '104.0.5112.79' + complete_version = extract_version_nt_folder() else: - chrome_path = uc.find_chrome_executable() + chrome_path = get_chrome_exe_path() process = os.popen(f'"{chrome_path}" --version') # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n' # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n' @@ -124,20 +146,29 @@ def get_chrome_major_version() -> str: return CHROME_MAJOR_VERSION -def extract_version_registry(output) -> str: - try: - google_version = '' - for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]: - if letter != '\n': - google_version += letter - else: - break - return google_version.strip() - except TypeError: - return '' +def extract_version_nt_executable(exe_path: str) -> str: + import pefile + pe = pefile.PE(exe_path, fast_load=True) + pe.parse_data_directories( + directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]] + ) + return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8') -def extract_version_folder() -> str: +def extract_version_nt_registry() -> str: + stream = os.popen( + 'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"') + output = stream.read() + google_version = '' + for letter in output[output.rindex('DisplayVersion REG_SZ') + 24:]: + if letter != '\n': + google_version += letter + else: + break + return google_version.strip() + + +def extract_version_nt_folder() -> str: # Check if the Chrome folder exists in the x32 or x64 Program Files folders. for i in range(2): path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'