From fake-useragent to ua-generator

.github/workflows/build.yml

@@ -77,12 +77,12 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -r requirements.txt
           python -m pip install pyinstaller
-          python -m pip install fake-useragent==1.1.3
+
       - name: Build executable with PyInstaller (Windows)
         if: matrix.os == 'windows-latest'
         shell: pwsh
         run: |
-          pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=fake_useragent `
+          pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=ua_generator `
             --hidden-import=qbittorrentapi --hidden-import=qbittorrent --hidden-import=googlesearch `
             --hidden-import=bs4 --hidden-import=httpx --hidden-import=rich --hidden-import=tqdm `
             --hidden-import=m3u8 --hidden-import=psutil --hidden-import=unidecode `
@@ -93,10 +93,11 @@ jobs:
             --hidden-import=pyTelegramBotAPI --additional-hooks-dir=pyinstaller/hooks `
             --add-data "StreamingCommunity;StreamingCommunity" `
             --name=StreamingCommunity --icon=".github/media/logo.ico" test_run.py
+
       - name: Build executable with PyInstaller (Linux)
         if: matrix.os == 'ubuntu-latest'
         run: |
-          pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=fake_useragent \
+          pyinstaller --onefile --hidden-import=pycryptodomex --hidden-import=ua_generator \
             --hidden-import=qbittorrentapi --hidden-import=qbittorrent --hidden-import=googlesearch \
             --hidden-import=bs4 --hidden-import=httpx --hidden-import=rich --hidden-import=tqdm \
             --hidden-import=m3u8 --hidden-import=psutil --hidden-import=unidecode \
@@ -107,6 +108,7 @@ jobs:
             --hidden-import=pyTelegramBotAPI --additional-hooks-dir=pyinstaller/hooks \
             --add-data "StreamingCommunity:StreamingCommunity" \
             --name=StreamingCommunity test_run.py
+            
       - name: Upload executable (Windows)
         if: matrix.os == 'windows-latest'
         uses: actions/upload-artifact@v4

StreamingCommunity/Lib/Downloader/HLS/downloader.py

@@ -159,11 +159,7 @@ class M3U8Manager:
         If it's a master playlist, only selects video stream.
         """
         if not self.is_master:
-            if FILTER_CUSTOM_REOLUTION != -1:
+            self.video_url, self.video_res = self.m3u8_url, "0p"
-                self.video_url, self.video_res = self.parser._video.get_custom_uri(y_resolution=FILTER_CUSTOM_REOLUTION)
-            else:
-                self.video_url, self.video_res = self.parser._video.get_best_uri()
-
             self.audio_streams = []
             self.sub_streams = []
             

StreamingCommunity/Lib/Downloader/HLS/segments.py

@@ -47,6 +47,7 @@ PROXY_START_MAX = config_manager.get_float('REQUESTS', 'proxy_start_max')
 DEFAULT_VIDEO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_video_workser')
 DEFAULT_AUDIO_WORKERS = config_manager.get_int('M3U8_DOWNLOAD', 'default_audio_workser')
 MAX_TIMEOOUT = config_manager.get_int("REQUESTS", "timeout")
+SEGMENT_MAX_TIMEOUT = config_manager.get_int("M3U8_DOWNLOAD", "segment_timeout")
 
 
 
@@ -169,8 +170,9 @@ class M3U8_Segments:
 
     def _get_http_client(self, index: int = None):
         client_params = {
-            'headers': random_headers(self.key_base_url) if hasattr(self, 'key_base_url') else {'User-Agent': get_headers()},
+            #'headers': random_headers(self.key_base_url) if hasattr(self, 'key_base_url') else {'User-Agent': get_headers()},
-            'timeout': MAX_TIMEOOUT,
+            'headers': {'User-Agent': get_headers()},
+            'timeout': SEGMENT_MAX_TIMEOUT,
             'follow_redirects': True,
             'http2': False
         }
@@ -391,7 +393,7 @@ class M3U8_Segments:
                 f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{description}{Colors.WHITE}): "
                 f"{Colors.RED}{{percentage:.2f}}% "
                 f"{Colors.MAGENTA}{{bar}} "
-                f"{Colors.WHITE}[ {Colors.YELLOW}{{elapsed}}{Colors.WHITE} < {Colors.CYAN}{{remaining}}{Colors.WHITE}{{postfix}}{Colors.WHITE} ]"
+                f"{Colors.YELLOW}{{elapsed}}{Colors.WHITE} < {Colors.CYAN}{{remaining}}{Colors.WHITE}{{postfix}}{Colors.WHITE}"
             )
     
     def _get_worker_count(self, stream_type: str) -> int:

StreamingCommunity/Util/headers.py

@@ -1,153 +1,12 @@
 # 4.04.24
 
-import re
-import sys
 import random
-from importlib.metadata import version, PackageNotFoundError
 
 
 # External library
-from fake_useragent import UserAgent
+import ua_generator
 
 
-# Variable
-try:
-    ua_version = version('fake-useragent')
-except PackageNotFoundError:
-    ua_version = None
-
-if not getattr(sys, 'frozen', False):
-    if ua_version == '1.1.3':
-        ua = UserAgent(use_external_data=True)
-    else:
-        ua = UserAgent()
-else:
-    ua = UserAgent()
-
-
-def extract_versions(user_agent):
-    """
-    Extract browser versions from the user agent.
-
-    Parameters:
-        user_agent (str): User agent of the browser.
-
-    Returns:
-        list: List of browser versions.
-    """
-
-    # Patterns to extract versions from various user agents
-    patterns = {
-        'chrome': re.compile(r'Chrome/(\d+)\.(\d+)\.(\d+)\.(\d+)'),
-        'firefox': re.compile(r'Firefox/(\d+)\.?(\d+)?\.?(\d+)?'),
-        'safari': re.compile(r'Version/(\d+)\.(\d+)\.(\d+) Safari/(\d+)\.(\d+)\.(\d+)'),
-        'edge': re.compile(r'Edg/(\d+)\.(\d+)\.(\d+)\.(\d+)'),
-        'edgios': re.compile(r'EdgiOS/(\d+)\.(\d+)\.(\d+)\.(\d+)'),
-        'crios': re.compile(r'CriOS/(\d+)\.(\d+)\.(\d+)\.(\d+)'),
-    }
-
-    for key, pattern in patterns.items():
-        match = pattern.search(user_agent)
-        if match:
-            return [match.group(i+1) for i in range(match.lastindex)]
-
-    # Fallback values if specific versions are not found
-    return ['99', '0', '0', '0']
-
-def get_platform(user_agent):
-    """
-    Determine the device platform from the user agent.
-
-    Parameters:
-        user_agent (str): User agent of the browser.
-
-    Returns:
-        str: Device platform.
-    """
-    if 'Windows' in user_agent:
-        return '"Windows"'
-    elif 'Mac OS X' in user_agent:
-        return '"macOS"'
-    elif 'Android' in user_agent:
-        return '"Android"'
-    elif 'iPhone' in user_agent or 'iPad' in user_agent:
-        return '"iOS"'
-    elif 'Linux' in user_agent:
-        return '"Linux"'
-    return '"Unknown"'
-
-def get_model(user_agent):
-    """
-    Determine the device model from the user agent.
-
-    Parameters:
-        user_agent (str): User agent of the browser.
-
-    Returns:
-        str: Device model.
-    """
-    if 'iPhone' in user_agent:
-        return '"iPhone"'
-    elif 'iPad' in user_agent:
-        return '"iPad"'
-    elif 'Android' in user_agent:
-        return '"Android"'
-    elif 'Windows' in user_agent:
-        return '"PC"'
-    elif 'Mac OS X' in user_agent:
-        return '"Mac"'
-    elif 'Linux' in user_agent:
-        return '"Linux"'
-    return '"Unknown"'
-
-def random_headers(referer: str = None):
-    """
-    Generate random HTTP headers to simulate human-like behavior.
-
-    Returns:
-        dict: Generated HTTP headers.
-    """
-    user_agent = ua.random
-    versions = extract_versions(user_agent)
-    platform = get_platform(user_agent)
-    model = get_model(user_agent)
-    is_mobile = 'Mobi' in user_agent or 'Android' in user_agent
-
-    # Generate sec-ch-ua string based on the browser
-    if 'Chrome' in user_agent or 'CriOS' in user_agent:
-        sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Chromium";v="{versions[0]}", "Google Chrome";v="{versions[0]}"'
-    elif 'Edg' in user_agent or 'EdgiOS' in user_agent:
-        sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Chromium";v="{versions[0]}", "Microsoft Edge";v="{versions[0]}"'
-    elif 'Firefox' in user_agent:
-        sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Firefox";v="{versions[0]}"'
-    elif 'Safari' in user_agent:
-        sec_ch_ua = f'" Not;A Brand";v="{versions[0]}", "Safari";v="{versions[0]}"'
-    else:
-        sec_ch_ua = f'" Not;A Brand";v="{versions[0]}"'
-
-    headers = {
-        'User-Agent': user_agent,
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-        'Accept-Language': random.choice(['en-US', 'en-GB', 'fr-FR', 'es-ES', 'de-DE']),
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1',
-        'Sec-Fetch-Dest': 'document',
-        'Sec-Fetch-Mode': 'navigate',
-        'Sec-Fetch-Site': 'none',
-        'Sec-Fetch-User': '?1',
-        'sec-ch-ua-mobile': '?1' if is_mobile else '?0',
-        'sec-ch-ua-platform': platform,
-        'sec-ch-ua': sec_ch_ua,
-        'sec-ch-ua-model': model
-    }
-
-    if referer:
-        headers['Origin'] = referer
-        headers['Referer'] = referer
-    
-    return headers
-
 def get_headers() -> str:
     """
     Generate a random user agent to use in HTTP requests.
@@ -157,4 +16,34 @@ def get_headers() -> str:
     """
     
     # Get a random user agent string from the user agent rotator
-    return str(ua.chrome)
+    user_agent =  ua_generator.generate().text
+    return user_agent
+
+
+def random_headers(referer: str = None):
+    """
+    Generate random HTTP headers to simulate human-like behavior.
+
+    Returns:
+        dict: Generated HTTP headers.
+    """
+    ua = ua_generator.generate()
+
+    headers = {
+        'User-Agent': ua.text,
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+        'Accept-Language': random.choice(['en-US', 'en-GB', 'fr-FR', 'es-ES', 'de-DE']),
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+    }
+
+    if referer:
+        headers['Origin'] = referer
+        headers['Referer'] = referer
+    
+    return headers

config.json

@@ -31,6 +31,7 @@
         "tqdm_delay": 0.01,
         "default_video_workser": 12,
         "default_audio_workser": 12,
+        "segment_timeout": 7,
         "download_audio": true,
         "merge_audio": true,
         "specific_list_audio": [

requirements.txt

@@ -9,7 +9,7 @@ jsbeautifier
 pathvalidate
 pycryptodomex
 googlesearch-python
-fake-useragent<2.0.0
+ua-generator
 qbittorrent-api
 python-qbittorrent
 Pillow