From f3dbdffd522e0d3a53b43b7e6d6cb92f9f65aadf Mon Sep 17 00:00:00 2001 From: Lovi <62809003+Lovi-0@users.noreply.github.com> Date: Sat, 8 Feb 2025 16:07:52 +0100 Subject: [PATCH] From fake-useragent to ua-generator --- .github/workflows/build.yml | 8 +- .../Lib/Downloader/HLS/downloader.py | 6 +- .../Lib/Downloader/HLS/segments.py | 8 +- StreamingCommunity/Util/headers.py | 175 ++++-------------- config.json | 1 + requirements.txt | 2 +- 6 files changed, 45 insertions(+), 155 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c971627..eee0d40 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,12 +77,12 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install pyinstaller - python -m pip install fake-useragent==1.1.3 + - name: Build executable with PyInstaller (Windows) if: matrix.os == 'windows-latest' shell: pwsh run: | - pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=fake_useragent ` + pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=ua_generator ` --hidden-import=qbittorrentapi --hidden-import=qbittorrent --hidden-import=googlesearch ` --hidden-import=bs4 --hidden-import=httpx --hidden-import=rich --hidden-import=tqdm ` --hidden-import=m3u8 --hidden-import=psutil --hidden-import=unidecode ` @@ -93,10 +93,11 @@ jobs: --hidden-import=pyTelegramBotAPI --additional-hooks-dir=pyinstaller/hooks ` --add-data "StreamingCommunity;StreamingCommunity" ` --name=StreamingCommunity --icon=".github/media/logo.ico" test_run.py + - name: Build executable with PyInstaller (Linux) if: matrix.os == 'ubuntu-latest' run: | - pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=fake_useragent \ + pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=ua_generator \ --hidden-import=qbittorrentapi --hidden-import=qbittorrent --hidden-import=googlesearch \ --hidden-import=bs4 --hidden-import=httpx --hidden-import=rich --hidden-import=tqdm \ --hidden-import=m3u8 --hidden-import=psutil --hidden-import=unidecode \ @@ -107,6 +108,7 @@ jobs: --hidden-import=pyTelegramBotAPI --additional-hooks-dir=pyinstaller/hooks \ --add-data "StreamingCommunity:StreamingCommunity" \ --name=StreamingCommunity test_run.py + - name: Upload executable (Windows) if: matrix.os == 'windows-latest' uses: actions/upload-artifact@v4 diff --git a/StreamingCommunity/Lib/Downloader/HLS/downloader.py b/StreamingCommunity/Lib/Downloader/HLS/downloader.py index dc8d436..3d6f738 100644 --- a/StreamingCommunity/Lib/Downloader/HLS/downloader.py +++ b/StreamingCommunity/Lib/Downloader/HLS/downloader.py @@ -159,11 +159,7 @@ class M3U8Manager: If it's a master playlist, only selects video stream. """ if not self.is_master: - if FILTER_CUSTOM_REOLUTION != -1: - self.video_url, self.video_res = self.parser._video.get_custom_uri(y_resolution=FILTER_CUSTOM_REOLUTION) - else: - self.video_url, self.video_res = self.parser._video.get_best_uri() - + self.video_url, self.video_res = self.m3u8_url, "0p" self.audio_streams = [] self.sub_streams = [] diff --git a/StreamingCommunity/Lib/Downloader/HLS/segments.py b/StreamingCommunity/Lib/Downloader/HLS/segments.py index 195ebdd..081a434 100644 --- a/StreamingCommunity/Lib/Downloader/HLS/segments.py +++ b/StreamingCommunity/Lib/Downloader/HLS/segments.py @@ -47,6 +47,7 @@ PROXY_START_MAX = config_manager.get_float('REQUESTS', 'proxy_start_max') DEFAULT_VIDEO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_video_workser') DEFAULT_AUDIO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_audio_workser') MAX_TIMEOOUT = config_manager.get_int("REQUESTS", "timeout") +SEGMENT_MAX_TIMEOUT = config_manager.get_int("M3U8_DOWNLOAD", "segment_timeout") @@ -169,8 +170,9 @@ class M3U8_Segments: def _get_http_client(self, index: int = None): client_params = { - 'headers': random_headers(self.key_base_url) if hasattr(self, 'key_base_url') else {'User-Agent': get_headers()}, - 'timeout': MAX_TIMEOOUT, + #'headers': random_headers(self.key_base_url) if hasattr(self, 'key_base_url') else {'User-Agent': get_headers()}, + 'headers': {'User-Agent': get_headers()}, + 'timeout': SEGMENT_MAX_TIMEOUT, 'follow_redirects': True, 'http2': False } @@ -391,7 +393,7 @@ class M3U8_Segments: f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{description}{Colors.WHITE}): " f"{Colors.RED}{{percentage:.2f}}% " f"{Colors.MAGENTA}{{bar}} " - f"{Colors.WHITE}[ {Colors.YELLOW}{{elapsed}}{Colors.WHITE} < {Colors.CYAN}{{remaining}}{Colors.WHITE}{{postfix}}{Colors.WHITE} ]" + f"{Colors.YELLOW}{{elapsed}}{Colors.WHITE} < {Colors.CYAN}{{remaining}}{Colors.WHITE}{{postfix}}{Colors.WHITE}" ) def _get_worker_count(self, stream_type: str) -> int: diff --git a/StreamingCommunity/Util/headers.py b/StreamingCommunity/Util/headers.py index 6879845..0ee6257 100644 --- a/StreamingCommunity/Util/headers.py +++ b/StreamingCommunity/Util/headers.py @@ -1,153 +1,12 @@ # 4.04.24 -import re -import sys import random -from importlib.metadata import version, PackageNotFoundError # External library -from fake_useragent import UserAgent +import ua_generator -# Variable -try: - ua_version = version('fake-useragent') -except PackageNotFoundError: - ua_version = None - -if not getattr(sys, 'frozen', False): - if ua_version == '1.1.3': - ua = UserAgent(use_external_data=True) - else: - ua = UserAgent() -else: - ua = UserAgent() - - -def extract_versions(user_agent): - """ - Extract browser versions from the user agent. - - Parameters: - user_agent (str): User agent of the browser. - - Returns: - list: List of browser versions. - """ - - # Patterns to extract versions from various user agents - patterns = { - 'chrome': re.compile(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)'), - 'firefox': re.compile(r'Firefox/(\d+)\.?(\d+)?\.?(\d+)?'), - 'safari': re.compile(r'Version/(\d+)\.(\d+)\.(\d+) Safari/(\d+)\.(\d+)\.(\d+)'), - 'edge': re.compile(r'Edg/(\d+)\.(\d+)\.(\d+)\.(\d+)'), - 'edgios': re.compile(r'EdgiOS/(\d+)\.(\d+)\.(\d+)\.(\d+)'), - 'crios': re.compile(r'CriOS/(\d+)\.(\d+)\.(\d+)\.(\d+)'), - } - - for key, pattern in patterns.items(): - match = pattern.search(user_agent) - if match: - return [match.group(i+1) for i in range(match.lastindex)] - - # Fallback values if specific versions are not found - return ['99', '0', '0', '0'] - -def get_platform(user_agent): - """ - Determine the device platform from the user agent. - - Parameters: - user_agent (str): User agent of the browser. - - Returns: - str: Device platform. - """ - if 'Windows' in user_agent: - return '"Windows"' - elif 'Mac OS X' in user_agent: - return '"macOS"' - elif 'Android' in user_agent: - return '"Android"' - elif 'iPhone' in user_agent or 'iPad' in user_agent: - return '"iOS"' - elif 'Linux' in user_agent: - return '"Linux"' - return '"Unknown"' - -def get_model(user_agent): - """ - Determine the device model from the user agent. - - Parameters: - user_agent (str): User agent of the browser. - - Returns: - str: Device model. - """ - if 'iPhone' in user_agent: - return '"iPhone"' - elif 'iPad' in user_agent: - return '"iPad"' - elif 'Android' in user_agent: - return '"Android"' - elif 'Windows' in user_agent: - return '"PC"' - elif 'Mac OS X' in user_agent: - return '"Mac"' - elif 'Linux' in user_agent: - return '"Linux"' - return '"Unknown"' - -def random_headers(referer: str = None): - """ - Generate random HTTP headers to simulate human-like behavior. - - Returns: - dict: Generated HTTP headers. - """ - user_agent = ua.random - versions = extract_versions(user_agent) - platform = get_platform(user_agent) - model = get_model(user_agent) - is_mobile = 'Mobi' in user_agent or 'Android' in user_agent - - # Generate sec-ch-ua string based on the browser - if 'Chrome' in user_agent or 'CriOS' in user_agent: - sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Chromium";v="{versions[0]}", "Google Chrome";v="{versions[0]}"' - elif 'Edg' in user_agent or 'EdgiOS' in user_agent: - sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Chromium";v="{versions[0]}", "Microsoft Edge";v="{versions[0]}"' - elif 'Firefox' in user_agent: - sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Firefox";v="{versions[0]}"' - elif 'Safari' in user_agent: - sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Safari";v="{versions[0]}"' - else: - sec_ch_ua = f'" Not;A Brand";v="{versions[0]}"' - - headers = { - 'User-Agent': user_agent, - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', - 'Accept-Language': random.choice(['en-US', 'en-GB', 'fr-FR', 'es-ES', 'de-DE']), - 'Accept-Encoding': 'gzip, deflate, br', - 'Connection': 'keep-alive', - 'Upgrade-Insecure-Requests': '1', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': 'none', - 'Sec-Fetch-User': '?1', - 'sec-ch-ua-mobile': '?1' if is_mobile else '?0', - 'sec-ch-ua-platform': platform, - 'sec-ch-ua': sec_ch_ua, - 'sec-ch-ua-model': model - } - - if referer: - headers['Origin'] = referer - headers['Referer'] = referer - - return headers - def get_headers() -> str: """ Generate a random user agent to use in HTTP requests. @@ -157,4 +16,34 @@ def get_headers() -> str: """ # Get a random user agent string from the user agent rotator - return str(ua.chrome) \ No newline at end of file + user_agent = ua_generator.generate().text + return user_agent + + +def random_headers(referer: str = None): + """ + Generate random HTTP headers to simulate human-like behavior. + + Returns: + dict: Generated HTTP headers. + """ + ua = ua_generator.generate() + + headers = { + 'User-Agent': ua.text, + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': random.choice(['en-US', 'en-GB', 'fr-FR', 'es-ES', 'de-DE']), + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Upgrade-Insecure-Requests': '1', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': 'none', + 'Sec-Fetch-User': '?1', + } + + if referer: + headers['Origin'] = referer + headers['Referer'] = referer + + return headers \ No newline at end of file diff --git a/config.json b/config.json index 0fc83e6..05664c0 100644 --- a/config.json +++ b/config.json @@ -31,6 +31,7 @@ "tqdm_delay": 0.01, "default_video_workser": 12, "default_audio_workser": 12, + "segment_timeout": 7, "download_audio": true, "merge_audio": true, "specific_list_audio": [ diff --git a/requirements.txt b/requirements.txt index 0294b55..428b1fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ jsbeautifier pathvalidate pycryptodomex googlesearch-python -fake-useragent<2.0.0 +ua-generator qbittorrent-api python-qbittorrent Pillow