Add support for Custom CloudFlare challenge

EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands

src/providers/cloudflare.ts

@@ -9,16 +9,27 @@ import log from "../services/log";
 const BAN_SELECTORS = ['.text-gray-600'];
 const CHALLENGE_SELECTORS = [
     '#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait', // CloudFlare
-    '#link-ddg' // DDoS-GUARD
+    '#link-ddg', // DDoS-GUARD
+    'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
 ];
 const CAPTCHA_SELECTORS = ['input[name="cf_captcha_kind"]'];
 
 export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
 
   // look for challenge and return fast if not detected
-  if (response.headers().server &&
+  let cfDetected = response.headers().server && response.headers().server.startsWith('cloudflare');
-      response.headers().server.startsWith('cloudflare') &&
+  if (cfDetected) {
-      (response.status() == 403 || response.status() == 503)) {
+    if (response.status() == 403 || response.status() == 503) {
+      cfDetected = true; // Defected CloudFlare and DDoS-GUARD
+    } else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
+        response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
+      cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    } else {
+      cfDetected = false;
+    }
+  }
+
+  if (cfDetected) {
     log.info('Cloudflare detected');
   } else {
     log.info('Cloudflare not detected');
@@ -29,76 +40,68 @@ export default async function resolveChallenge(url: string, page: Page, response
     throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
   }
 
+  // find Cloudflare selectors
   let selectorFound = false;
-  if (response.status() > 400) {
+  let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
+  if (selector) {
+    selectorFound = true;
+    log.debug(`Javascript challenge element '${selector}' detected.`)
+    log.debug('Waiting for Cloudflare challenge...')
 
-    // find Cloudflare selectors
+    while (true) {
-    let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
+      try {
-    if (selector) {
-      selectorFound = true;
-      log.debug(`Javascript challenge element '${selector}' detected.`)
-      log.debug('Waiting for Cloudflare challenge...')
 
-      while (true) {
+        selector = await findAnySelector(page, CHALLENGE_SELECTORS)
-        try {
+        if (!selector) {
+          // solved!
+          log.debug('Challenge element not found')
+          break
+        } else {
+          log.debug(`Javascript challenge element '${selector}' detected.`)
 
-          selector = await findAnySelector(page, CHALLENGE_SELECTORS)
+          // new Cloudflare Challenge #cf-please-wait
-          if (!selector) {
+          const displayStyle = await page.evaluate((selector) => {
-            // solved!
+            return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
-            log.debug('Challenge element not found')
+          }, selector);
+          if (displayStyle == "none") {
+            // spinner is hidden, could be a captcha or not
+            log.debug('Challenge element is hidden')
+            // wait until redirecting disappears
+            while (true) {
+              try {
+                await page.waitFor(1000)
+                const displayStyle2 = await page.evaluate(() => {
+                  return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                });
+                if (displayStyle2 == "none") {
+                  break // hCaptcha detected
+                }
+              } catch (error) {
+                break // redirection completed
+              }
+            }
             break
           } else {
-            log.debug(`Javascript challenge element '${selector}' detected.`)
+            log.debug('Challenge element is visible')
-
-            // new Cloudflare Challenge #cf-please-wait
-            const displayStyle = await page.evaluate((selector) => {
-              return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
-            }, selector);
-            if (displayStyle == "none") {
-              // spinner is hidden, could be a captcha or not
-              log.debug('Challenge element is hidden')
-              // wait until redirecting disappears
-              while (true) {
-                try {
-                  await page.waitFor(1000)
-                  const displayStyle2 = await page.evaluate(() => {
-                    return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
-                  });
-                  if (displayStyle2 == "none") {
-                    break // hCaptcha detected
-                  }
-                } catch (error) {
-                  break // redirection completed
-                }
-              }
-              break
-            } else {
-              log.debug('Challenge element is visible')
-            }
-          }
-          log.debug('Found challenge element again')
-
-        } catch (error)
-        {
-          log.debug("Unexpected error: " + error);
-          if (!error.toString().includes("Execution context was destroyed")) {
-            break
           }
         }
+        log.debug('Found challenge element again')
 
-        log.debug('Waiting for Cloudflare challenge...')
+      } catch (error)
-        await page.waitFor(1000)
+      {
+        log.debug("Unexpected error: " + error);
+        if (!error.toString().includes("Execution context was destroyed")) {
+          break
+        }
       }
 
-      log.debug('Validating HTML code...')
+      log.debug('Waiting for Cloudflare challenge...')
-    } else {
+      await page.waitFor(1000)
-      log.debug(`No challenge element detected.`)
     }
 
+    log.debug('Validating HTML code...')
   } else {
-    // some sites use cloudflare but there is no challenge
+    log.debug(`No challenge element detected.`)
-    log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
-    selectorFound = true;
   }
 
   // check for CAPTCHA challenge

src/tests/app.test.ts

@@ -16,6 +16,7 @@ const cfUrl = "https://pirateiro.com/torrents/?search=harry";
 const cfCaptchaUrl = "https://idope.se"
 const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
 const ddgUrl = "https://www.erai-raws.info/feed/?type=magnet";
+const ccfUrl = "https://www.muziekfabriek.org";
 
 beforeAll(async () => {
     // Init session
@@ -198,6 +199,35 @@ describe("Test '/v1' path", () => {
         expect(cfCookie.length).toBeGreaterThan(10)
     });
 
+    test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ccfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ccfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<html><head>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "ct_anti_ddos_key";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
     test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
         const payload = {
             "cmd": "request.get",