From 075b53ee2471b5a8429f946bf8f7083bf90e4e42 Mon Sep 17 00:00:00 2001 From: ngosang Date: Mon, 18 Oct 2021 21:45:21 +0200 Subject: [PATCH] Improve Firefox start time and code clean up --- package-lock.json | 379 ++---------------------------------- package.json | 1 - src/providers/cloudflare.ts | 12 +- src/services/sessions.ts | 76 ++++---- src/services/solver.ts | 1 + 5 files changed, 63 insertions(+), 406 deletions(-) diff --git a/package-lock.json b/package-lock.json index 23bb451..9147aba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,6 @@ "body-parser": "^1.19.0", "console-log-level": "^1.4.1", "express": "^4.17.1", - "got": "^11.8.2", "puppeteer": "^3.3.0", "uuid": "^8.3.2" }, @@ -1328,17 +1327,6 @@ "node": ">= 8" } }, - "node_modules/@sindresorhus/is": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.2.0.tgz", - "integrity": "sha512-VkE3KLBmJwcCaVARtQpfuKcKv8gcBmUubrfHGF84dXuuW6jgsRYxPtzcIhPyK9WAPpRt2/xY6zkD9MnRaJzSyw==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/is?sponsor=1" - } - }, "node_modules/@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -1359,17 +1347,6 @@ "@sinonjs/commons": "^1.7.0" } }, - "node_modules/@szmarczak/http-timer": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", - "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", - "dependencies": { - "defer-to-connect": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -1465,17 +1442,6 @@ "@types/node": "*" } }, - "node_modules/@types/cacheable-request": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", - "integrity": "sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==", - "dependencies": { - "@types/http-cache-semantics": "*", - "@types/keyv": "*", - "@types/node": "*", - "@types/responselike": "*" - } - }, "node_modules/@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -1524,11 +1490,6 @@ "@types/node": "*" } }, - "node_modules/@types/http-cache-semantics": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", - "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" - }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz", @@ -1563,14 +1524,6 @@ "pretty-format": "^27.0.0" } }, - "node_modules/@types/keyv": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.3.tgz", - "integrity": "sha512-FXCJgyyN3ivVgRoml4h94G/p3kY+u/B86La+QptcqJaWtBWtmc6TtkNfS40n9bIvyLteHh7zXOtgbobORKPbDg==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -1580,7 +1533,8 @@ "node_modules/@types/node": { "version": "14.17.27", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.27.tgz", - "integrity": "sha512-94+Ahf9IcaDuJTle/2b+wzvjmutxXAEXU6O81JHblYXUg2BDG+dnBy7VxIPHKAyEEDHzCMQydTJuWvrE+Aanzw==" + "integrity": "sha512-94+Ahf9IcaDuJTle/2b+wzvjmutxXAEXU6O81JHblYXUg2BDG+dnBy7VxIPHKAyEEDHzCMQydTJuWvrE+Aanzw==", + "devOptional": true }, "node_modules/@types/prettier": { "version": "2.4.1", @@ -1610,14 +1564,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, - "node_modules/@types/responselike": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", - "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/serve-static": { "version": "1.13.10", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", @@ -2499,31 +2445,6 @@ "node": ">= 0.8" } }, - "node_modules/cacheable-lookup": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", - "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==", - "engines": { - "node": ">=10.6.0" - } - }, - "node_modules/cacheable-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz", - "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==", - "dependencies": { - "clone-response": "^1.0.2", - "get-stream": "^5.1.0", - "http-cache-semantics": "^4.0.0", - "keyv": "^4.0.0", - "lowercase-keys": "^2.0.0", - "normalize-url": "^6.0.1", - "responselike": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -2727,6 +2648,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, "dependencies": { "mimic-response": "^1.0.0" } @@ -3063,31 +2985,6 @@ "dev": true, "peer": true }, - "node_modules/decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "dependencies": { - "mimic-response": "^3.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/decompress-response/node_modules/mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -3120,14 +3017,6 @@ "node": ">=0.10.0" } }, - "node_modules/defer-to-connect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==", - "engines": { - "node": ">=10" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -3983,30 +3872,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/got": { - "version": "11.8.2", - "resolved": "https://registry.npmjs.org/got/-/got-11.8.2.tgz", - "integrity": "sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==", - "dependencies": { - "@sindresorhus/is": "^4.0.0", - "@szmarczak/http-timer": "^4.0.5", - "@types/cacheable-request": "^6.0.1", - "@types/responselike": "^1.0.0", - "cacheable-lookup": "^5.0.3", - "cacheable-request": "^7.0.1", - "decompress-response": "^6.0.0", - "http2-wrapper": "^1.0.0-beta.5.2", - "lowercase-keys": "^2.0.0", - "p-cancelable": "^2.0.0", - "responselike": "^2.0.0" - }, - "engines": { - "node": ">=10.19.0" - }, - "funding": { - "url": "https://github.com/sindresorhus/got?sponsor=1" - } - }, "node_modules/graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", @@ -4084,7 +3949,8 @@ "node_modules/http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", - "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true }, "node_modules/http-errors": { "version": "1.7.2", @@ -4146,18 +4012,6 @@ "dev": true, "peer": true }, - "node_modules/http2-wrapper": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", - "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", - "dependencies": { - "quick-lru": "^5.1.1", - "resolve-alpn": "^1.0.0" - }, - "engines": { - "node": ">=10.19.0" - } - }, "node_modules/https-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", @@ -5492,11 +5346,6 @@ "node": ">=4" } }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" - }, "node_modules/json5": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", @@ -5524,14 +5373,6 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/keyv": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz", - "integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==", - "dependencies": { - "json-buffer": "3.0.1" - } - }, "node_modules/kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -5678,6 +5519,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true, "engines": { "node": ">=8" } @@ -5828,6 +5670,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true, "engines": { "node": ">=4" } @@ -6009,17 +5852,6 @@ "node": ">=0.10.0" } }, - "node_modules/normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6131,14 +5963,6 @@ "node": ">= 0.8.0" } }, - "node_modules/p-cancelable": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", - "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==", - "engines": { - "node": ">=8" - } - }, "node_modules/p-is-promise": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz", @@ -6808,17 +6632,6 @@ } ] }, - "node_modules/quick-lru": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -6942,11 +6755,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/resolve-alpn": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" - }, "node_modules/resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -6970,14 +6778,6 @@ "node": ">=8" } }, - "node_modules/responselike": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", - "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==", - "dependencies": { - "lowercase-keys": "^2.0.0" - } - }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -9542,11 +9342,6 @@ "fastq": "^1.6.0" } }, - "@sindresorhus/is": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.2.0.tgz", - "integrity": "sha512-VkE3KLBmJwcCaVARtQpfuKcKv8gcBmUubrfHGF84dXuuW6jgsRYxPtzcIhPyK9WAPpRt2/xY6zkD9MnRaJzSyw==" - }, "@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -9567,14 +9362,6 @@ "@sinonjs/commons": "^1.7.0" } }, - "@szmarczak/http-timer": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.6.tgz", - "integrity": "sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==", - "requires": { - "defer-to-connect": "^2.0.0" - } - }, "@tootallnate/once": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", @@ -9667,17 +9454,6 @@ "@types/node": "*" } }, - "@types/cacheable-request": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", - "integrity": "sha512-B3xVo+dlKM6nnKTcmm5ZtY/OL8bOAOd2Olee9M1zft65ox50OzjEHW91sDiU9j6cvW8Ejg1/Qkf4xd2kugApUA==", - "requires": { - "@types/http-cache-semantics": "*", - "@types/keyv": "*", - "@types/node": "*", - "@types/responselike": "*" - } - }, "@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -9726,11 +9502,6 @@ "@types/node": "*" } }, - "@types/http-cache-semantics": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", - "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" - }, "@types/istanbul-lib-coverage": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz", @@ -9765,14 +9536,6 @@ "pretty-format": "^27.0.0" } }, - "@types/keyv": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.3.tgz", - "integrity": "sha512-FXCJgyyN3ivVgRoml4h94G/p3kY+u/B86La+QptcqJaWtBWtmc6TtkNfS40n9bIvyLteHh7zXOtgbobORKPbDg==", - "requires": { - "@types/node": "*" - } - }, "@types/mime": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", @@ -9782,7 +9545,8 @@ "@types/node": { "version": "14.17.27", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.27.tgz", - "integrity": "sha512-94+Ahf9IcaDuJTle/2b+wzvjmutxXAEXU6O81JHblYXUg2BDG+dnBy7VxIPHKAyEEDHzCMQydTJuWvrE+Aanzw==" + "integrity": "sha512-94+Ahf9IcaDuJTle/2b+wzvjmutxXAEXU6O81JHblYXUg2BDG+dnBy7VxIPHKAyEEDHzCMQydTJuWvrE+Aanzw==", + "devOptional": true }, "@types/prettier": { "version": "2.4.1", @@ -9812,14 +9576,6 @@ "integrity": "sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==", "dev": true }, - "@types/responselike": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", - "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==", - "requires": { - "@types/node": "*" - } - }, "@types/serve-static": { "version": "1.13.10", "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.13.10.tgz", @@ -10526,25 +10282,6 @@ "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" }, - "cacheable-lookup": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz", - "integrity": "sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA==" - }, - "cacheable-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.2.tgz", - "integrity": "sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew==", - "requires": { - "clone-response": "^1.0.2", - "get-stream": "^5.1.0", - "http-cache-semantics": "^4.0.0", - "keyv": "^4.0.0", - "lowercase-keys": "^2.0.0", - "normalize-url": "^6.0.1", - "responselike": "^2.0.0" - } - }, "call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -10698,6 +10435,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, "requires": { "mimic-response": "^1.0.0" } @@ -10984,21 +10722,6 @@ "dev": true, "peer": true }, - "decompress-response": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", - "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", - "requires": { - "mimic-response": "^3.1.0" - }, - "dependencies": { - "mimic-response": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", - "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==" - } - } - }, "dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", @@ -11025,11 +10748,6 @@ "dev": true, "peer": true }, - "defer-to-connect": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", - "integrity": "sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==" - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -11697,24 +11415,6 @@ "slash": "^3.0.0" } }, - "got": { - "version": "11.8.2", - "resolved": "https://registry.npmjs.org/got/-/got-11.8.2.tgz", - "integrity": "sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ==", - "requires": { - "@sindresorhus/is": "^4.0.0", - "@szmarczak/http-timer": "^4.0.5", - "@types/cacheable-request": "^6.0.1", - "@types/responselike": "^1.0.0", - "cacheable-lookup": "^5.0.3", - "cacheable-request": "^7.0.1", - "decompress-response": "^6.0.0", - "http2-wrapper": "^1.0.0-beta.5.2", - "lowercase-keys": "^2.0.0", - "p-cancelable": "^2.0.0", - "responselike": "^2.0.0" - } - }, "graceful-fs": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", @@ -11774,7 +11474,8 @@ "http-cache-semantics": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", - "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==" + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true }, "http-errors": { "version": "1.7.2", @@ -11826,15 +11527,6 @@ } } }, - "http2-wrapper": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-1.0.3.tgz", - "integrity": "sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg==", - "requires": { - "quick-lru": "^5.1.1", - "resolve-alpn": "^1.0.0" - } - }, "https-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", @@ -12867,11 +12559,6 @@ "dev": true, "peer": true }, - "json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==" - }, "json5": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.0.tgz", @@ -12891,14 +12578,6 @@ "universalify": "^2.0.0" } }, - "keyv": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.3.tgz", - "integrity": "sha512-zdGa2TOpSZPq5mU6iowDARnMBZgtCqJ11dJROFi6tg6kTn4nuUdU09lFyLFSaHrWqpIJ+EBq4E8/Dc0Vx5vLdA==", - "requires": { - "json-buffer": "3.0.1" - } - }, "kleur": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", @@ -13028,7 +12707,8 @@ "lowercase-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", - "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==" + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true }, "lru-cache": { "version": "6.0.0", @@ -13138,7 +12818,8 @@ "mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true }, "minimatch": { "version": "3.0.4", @@ -13271,11 +12952,6 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, - "normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==" - }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -13363,11 +13039,6 @@ "word-wrap": "~1.2.3" } }, - "p-cancelable": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.1.1.tgz", - "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==" - }, "p-is-promise": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-3.0.0.tgz", @@ -13874,11 +13545,6 @@ "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", "dev": true }, - "quick-lru": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", - "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==" - }, "range-parser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", @@ -13975,11 +13641,6 @@ "path-parse": "^1.0.6" } }, - "resolve-alpn": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/resolve-alpn/-/resolve-alpn-1.2.1.tgz", - "integrity": "sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==" - }, "resolve-cwd": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", @@ -13997,14 +13658,6 @@ "dev": true, "peer": true }, - "responselike": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz", - "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==", - "requires": { - "lowercase-keys": "^2.0.0" - } - }, "reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", diff --git a/package.json b/package.json index 14ef033..85c9596 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "body-parser": "^1.19.0", "console-log-level": "^1.4.1", "express": "^4.17.1", - "got": "^11.8.2", "puppeteer": "^3.3.0", "uuid": "^8.3.2" }, diff --git a/src/providers/cloudflare.ts b/src/providers/cloudflare.ts index 80b76b1..ba5cea4 100644 --- a/src/providers/cloudflare.ts +++ b/src/providers/cloudflare.ts @@ -39,7 +39,7 @@ export default async function resolveChallenge(url: string, page: Page, response selector = await findAnySelector(page, CHALLENGE_SELECTORS) if (!selector) { // solved! - log.debug('Challenge element not found.') + log.debug('Challenge element not found') break } else { log.debug(`Javascript challenge element '${selector}' detected.`) @@ -50,7 +50,7 @@ export default async function resolveChallenge(url: string, page: Page, response }, selector); if (displayStyle == "none") { // spinner is hidden, could be a captcha or not - log.debug('Challenge element is hidden.') + log.debug('Challenge element is hidden') // wait until redirecting disappears while (true) { try { @@ -67,10 +67,10 @@ export default async function resolveChallenge(url: string, page: Page, response } break } else { - log.debug('Challenge element is visible.') + log.debug('Challenge element is visible') } } - log.debug('Found challenge element again.') + log.debug('Found challenge element again') } catch (error) { @@ -97,7 +97,7 @@ export default async function resolveChallenge(url: string, page: Page, response // check for CAPTCHA challenge if (await findAnySelector(page, CAPTCHA_SELECTORS)) { - log.info('CAPTCHA challenge detected.'); + log.info('CAPTCHA challenge detected'); throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.'); // const captchaSolver = getCaptchaSolver() @@ -120,7 +120,7 @@ export default async function resolveChallenge(url: string, page: Page, response { throw new Error('No challenge selectors found, unable to proceed.') } else { - log.info('Challenge solved.'); + log.info('Challenge solved'); } } diff --git a/src/services/sessions.ts b/src/services/sessions.ts index 8fe95d8..8729840 100644 --- a/src/services/sessions.ts +++ b/src/services/sessions.ts @@ -32,7 +32,7 @@ let webBrowserUserAgent: string; function userDataDirFromId(id: string): string { - return path.join(os.tmpdir(), `/puppeteer_profile_${id}`) + return path.join(os.tmpdir(), `/puppeteer_cprofile_${id}`) } function prepareBrowserProfile(id: string, proxy: Proxy): string { @@ -42,31 +42,49 @@ function prepareBrowserProfile(id: string, proxy: Proxy): string { fs.mkdirSync(userDataDir, { recursive: true }) } + // Some parameters to configure Firefox + // https://github.com/puppeteer/puppeteer/blob/943477cc1eb4b129870142873b3554737d5ef252/experimental/puppeteer-firefox/misc/puppeteer.cfg + let prefs = `// Any comment. You must start the file with a comment! + +// Disable newtabpage +user_pref("browser.newtabpage.enabled", false); +user_pref("browser.startup.homepage", "about:blank"); + +// Do not warn when closing all open tabs +user_pref("browser.tabs.warnOnClose", false); + +// Disable telemetry +user_pref("toolkit.telemetry.reportingpolicy.firstRun", false); + +// Disable first-run welcome page +user_pref("startup.homepage_welcome_url", "about:blank"); +user_pref("startup.homepage_welcome_url.additional", ""); + +// Disable images to speed up load +user_pref("permissions.default.image", 2); + +` + // proxy.url format => http://: if (proxy && proxy.url) { let [host, port] = proxy.url.replace(/https?:\/\//g, '').split(':'); + prefs += ` - let prefs = ` - user_pref("browser.newtabpage.enabled", false); - user_pref("browser.startup.homepage", "about:blank"); - user_pref("browser.tabs.warnOnClose", false); - user_pref("toolkit.telemetry.reportingpolicy.firstRun", false); - user_pref("trailhead.firstrun.branches", "nofirstrun-empty"); - user_pref("browser.aboutwelcome.enabled", false); - user_pref("network.proxy.ftp", "${host}"); - user_pref("network.proxy.ftp_port", ${port}); - user_pref("network.proxy.http", "${host}"); - user_pref("network.proxy.http_port", ${port}); - user_pref("network.proxy.share_proxy_settings", true); - user_pref("network.proxy.socks", "${host}"); - user_pref("network.proxy.socks_port", ${port}); - user_pref("network.proxy.ssl", "${host}"); - user_pref("network.proxy.ssl_port", ${port}); - user_pref("network.proxy.type", 1); - ` +// Proxy configuration +user_pref("network.proxy.ftp", "${host}"); +user_pref("network.proxy.ftp_port", ${port}); +user_pref("network.proxy.http", "${host}"); +user_pref("network.proxy.http_port", ${port}); +user_pref("network.proxy.share_proxy_settings", true); +user_pref("network.proxy.socks", "${host}"); +user_pref("network.proxy.socks_port", ${port}); +user_pref("network.proxy.ssl", "${host}"); +user_pref("network.proxy.ssl_port", ${port}); +user_pref("network.proxy.type", 1); - fs.writeFileSync(path.join(userDataDir, './prefs.js'), prefs); +` } + fs.writeFileSync(path.join(userDataDir, './prefs.js'), prefs); return userDataDir } @@ -118,23 +136,9 @@ export async function create(session: string, options: SessionCreateOptions): Pr } log.debug('Launching web browser...') - - // todo: the retries are required? - let launchTries = 3 - let browser: Browser; - while (0 <= launchTries--) { - try { - browser = await puppeteer.launch(puppeteerOptions) - break - } catch (e) { - if (e.message !== 'Failed to launch the browser process!') - throw e - log.warn('Failed to open browser, trying again...') - } - } - + let browser: Browser = await puppeteer.launch(puppeteerOptions) if (!browser) { - throw Error(`Failed to launch browser 3 times in a row.`) + throw Error(`Failed to launch web browser.`) } sessionCache[sessionId] = { diff --git a/src/services/solver.ts b/src/services/solver.ts index ada736a..0a52013 100644 --- a/src/services/solver.ts +++ b/src/services/solver.ts @@ -88,6 +88,7 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem): } // reload the page to be sure we get the real page + log.debug("Reloading the page") response = await gotoPage(params, page); const payload: ChallengeResolutionT = {