From e2e5452fe227bfa4ab34ee07045983932c12905f Mon Sep 17 00:00:00 2001 From: Kennedy Oliveira Date: Sat, 27 Jan 2024 20:08:59 -0300 Subject: [PATCH 1/3] Add wait time after challenge --- README.md | 1 + src/dtos.py | 3 +++ src/flaresolverr_service.py | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/README.md b/README.md index b5208fc..7e73376 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ session. When you no longer need to use a session you should make sure to close | cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. | | returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. | | proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) | +| waitInSeconds | Optional, default none, how long to wait with the page open after solving the challenge and before returning the results, useful to allow it to load dynamic content. | > **Warning** > If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge. diff --git a/src/dtos.py b/src/dtos.py index 1e9aace..6ec5842 100644 --- a/src/dtos.py +++ b/src/dtos.py @@ -43,6 +43,9 @@ class V1RequestBase(object): returnOnlyCookies: bool = None download: bool = None # deprecated v2.0.0, not used returnRawHtml: bool = None # deprecated v2.0.0, not used + # Wait the specified amount in seconds after resolving the challenge and before returning the response + # this can be useful for loading dynamic content of the page + waitInSeconds: int = None def __init__(self, _dict): self.__dict__.update(_dict) diff --git a/src/flaresolverr_service.py b/src/flaresolverr_service.py index 6cef3e4..09c01f9 100644 --- a/src/flaresolverr_service.py +++ b/src/flaresolverr_service.py @@ -398,6 +398,11 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge if not req.returnOnlyCookies: challenge_res.headers = {} # todo: fix, selenium not provides this info + + if req.waitInSeconds and req.waitInSeconds > 0: + logging.info("Waiting " + str(req.waitInSeconds) + " seconds before returning the response...") + time.sleep(req.waitInSeconds) + challenge_res.response = driver.page_source res.result = challenge_res From febb9f0e052f875e12f584be33e97e7d50a079ed Mon Sep 17 00:00:00 2001 From: Kennedy Oliveira Date: Sat, 27 Jan 2024 20:47:29 -0300 Subject: [PATCH 2/3] Use GITHUB_TOKEN instead of GH_PAT --- .github/workflows/release-docker.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 357f906..e5dbd1e 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -40,7 +40,7 @@ jobs: with: registry: ghcr.io username: ${{ github.repository_owner }} - password: ${{ secrets.GH_PAT }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push uses: docker/build-push-action@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9aab370..f8589b9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,7 +61,7 @@ jobs: - name: Upload release artifacts uses: alexellis/upload-assets@0.4.0 env: - GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: asset_paths: '["./dist/flaresolverr_*"]' From b8554bd2b6d5c816dcc18892eebf81dc61293225 Mon Sep 17 00:00:00 2001 From: Kennedy Oliveira Date: Sat, 27 Jan 2024 20:54:40 -0300 Subject: [PATCH 3/3] Use GITHUB_TOKEN instead of GH_PAT --- .github/workflows/autotag.yml | 2 +- .github/workflows/release.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/autotag.yml b/.github/workflows/autotag.yml index 2363337..e9d74b2 100644 --- a/.github/workflows/autotag.yml +++ b/.github/workflows/autotag.yml @@ -16,5 +16,5 @@ jobs: name: Auto Tag uses: Klemensas/action-autotag@stable with: - GITHUB_TOKEN: "${{ secrets.GH_PAT }}" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" tag_prefix: "v" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f8589b9..b75788f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: id: create_release uses: actions/create-release@v1 env: - GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: tag_name: ${{ github.ref }} release_name: ${{ github.ref }} @@ -90,6 +90,6 @@ jobs: - name: Upload release artifacts uses: alexellis/upload-assets@0.4.0 env: - GITHUB_TOKEN: ${{ secrets.GH_PAT }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: asset_paths: '["./dist/flaresolverr_*"]'