diff --git a/Src/Api/Streamingcommunity/costant.py b/Src/Api/Streamingcommunity/costant.py index de48a7f..4aa5e97 100644 --- a/Src/Api/Streamingcommunity/costant.py +++ b/Src/Api/Streamingcommunity/costant.py @@ -2,6 +2,4 @@ STREAMING_FOLDER = "streamingcommunity" MOVIE_FOLDER = "Movie" -SERIES_FOLDER = "Serie" - -STATIC_IP_SERVER = ['57.129.7.85', '57.129.7.188', '57.129.7.174', '57.129.4.77', '57.129.16.196', '57.129.16.156', '57.129.16.139', '57.129.16.135', '57.129.13.175', '57.129.13.157', '51.38.112.237', '51.195.107.7', '51.195.107.230', '162.19.255.78', '162.19.255.36', '162.19.255.224', '162.19.255.223', '162.19.254.244', '162.19.254.232', '162.19.254.230', '162.19.253.242', '162.19.249.48', '162.19.245.142', '162.19.231.20', '162.19.229.177', '162.19.228.128', '162.19.228.127', '162.19.228.105', '141.95.1.32', '141.95.1.196', '141.95.1.102', '141.95.0.50', '141.95.0.248', '135.125.237.84', '135.125.233.236'] \ No newline at end of file +SERIES_FOLDER = "Serie" \ No newline at end of file diff --git a/Src/Api/Streamingcommunity/film.py b/Src/Api/Streamingcommunity/film.py index e55fd8f..2b9d7bf 100644 --- a/Src/Api/Streamingcommunity/film.py +++ b/Src/Api/Streamingcommunity/film.py @@ -18,7 +18,7 @@ from .Core.Vix_player.player import VideoSource # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, MOVIE_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, MOVIE_FOLDER # Variable diff --git a/Src/Api/Streamingcommunity/series.py b/Src/Api/Streamingcommunity/series.py index 71396d8..9b2ecc0 100644 --- a/Src/Api/Streamingcommunity/series.py +++ b/Src/Api/Streamingcommunity/series.py @@ -20,7 +20,7 @@ from .Core.Util import manage_selection, map_episode_title # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, SERIES_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, SERIES_FOLDER # Variable diff --git a/Src/Lib/Hls/downloader.py b/Src/Lib/Hls/downloader.py index 26bd443..3feb9e8 100644 --- a/Src/Lib/Hls/downloader.py +++ b/Src/Lib/Hls/downloader.py @@ -232,15 +232,12 @@ class Downloader(): console.print(f"[cyan]Find codec [white]=> ([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") - def __donwload_video__(self, server_ip: list = None): + def __donwload_video__(self): """ Downloads and manages video segments. This method downloads video segments if necessary and updates the list of downloaded video segments. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Construct full path for the video segment directory @@ -256,7 +253,6 @@ class Downloader(): # Create an instance of M3U8_Segments to handle video segments video_m3u8 = M3U8_Segments(self.m3u8_index, full_path_video) - video_m3u8.add_server_ip(server_ip) # Get information about the video segments video_m3u8.get_info() @@ -270,15 +266,12 @@ class Downloader(): else: console.log("[cyan]Video [red]already exists.") - def __donwload_audio__(self, server_ip: list = None): + def __donwload_audio__(self): """ Downloads and manages audio segments. This method iterates over available audio tracks, downloads them if necessary, and updates the list of downloaded audio tracks. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Iterate over each available audio track @@ -305,7 +298,6 @@ class Downloader(): # If the audio segment directory doesn't exist, download audio segments audio_m3u8 = M3U8_Segments(obj_audio.get('uri'), full_path_audio) - audio_m3u8.add_server_ip(server_ip) # Get information about the audio segments audio_m3u8.get_info() @@ -494,12 +486,9 @@ class Downloader(): else: logging.info("Video file converted already exist.") - def start(self, server_ip: list = None) -> None: + def start(self) -> None: """ Start the process of fetching, downloading, joining, and cleaning up the video. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Check if file already exist @@ -532,9 +521,9 @@ class Downloader(): # Start all download ... if DOWNLOAD_VIDEO: - self.__donwload_video__(server_ip) + self.__donwload_video__() if DOWNLOAD_AUDIO: - self.__donwload_audio__(server_ip) + self.__donwload_audio__() if DOWNLOAD_SUBTITLE: self.__download_subtitle__() diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 33375e1..50a206a 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -13,6 +13,7 @@ from urllib.parse import urljoin, urlparse, urlunparse # External libraries import requests +from requests.exceptions import HTTPError, ConnectionError, Timeout, RequestException from tqdm import tqdm @@ -36,16 +37,19 @@ from ..M3U8 import ( import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # Config TQDM_MAX_WORKER = config_manager.get_int('M3U8_DOWNLOAD', 'tdqm_workers') TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar') -REQUEST_VERIFY_SSL = config_manager.get_bool('REQUESTS', 'verify_ssl') +REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout') +PROXY_LIST = config_manager.get_list('REQUESTS', 'proxy') # Variable headers_index = config_manager.get_dict('REQUESTS', 'index') headers_segments = config_manager.get_dict('REQUESTS', 'segments') - +session = requests.Session() +session.verify = config_manager.get_bool('REQUESTS', 'verify_ssl') class M3U8_Segments: @@ -58,7 +62,6 @@ class M3U8_Segments: - tmp_folder (str): The temporary folder to store downloaded segments. """ self.url = url - self.fake_proxy = False self.tmp_folder = tmp_folder self.tmp_file_path = os.path.join(self.tmp_folder, "0.ts") os.makedirs(self.tmp_folder, exist_ok=True) @@ -73,17 +76,6 @@ class M3U8_Segments: self.segment_queue = queue.PriorityQueue() # Priority queue to maintain the order of segments self.condition = threading.Condition() # Condition variable for thread synchronization - def add_server_ip(self, list_ip): - """ - Add server IP addresses - - Args: - list_ip (list): A list of IP addresses to be added. - """ - if list_ip is not None: - self.fake_proxy = True - self.fake_proxy_ip = list_ip - def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: """ Retrieves the encryption key from the M3U8 playlist. @@ -115,41 +107,6 @@ class M3U8_Segments: logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})") return byte_content - def __test_ip(self, url_to_test: str): - """ - Tests each proxy IP by sending a request to a corresponding segment URL. - """ - - failed_ips = [] - - for i in range(len(self.fake_proxy_ip)): - - try: - response = requests.get(url_to_test, verify=False, retries=0) - - if response == None: - logging.error(f"[Work] to make request using: {url_to_test}") - failed_ips.append(i) - - except: - - # Log the error and add the IP to the list of failed IPs - logging.error(f"[Fail] to make request using IP in this request: {url_to_test}") - failed_ips.append(i) - - # Remove the failed IPs from the fake_proxy_ip list - self.fake_proxy_ip = [ip for j, ip in enumerate(self.fake_proxy_ip) if j not in failed_ips] - - # Exit the program if 50% requests failed - if len(failed_ips) / 2 > len(self.fake_proxy_ip): - logging.error("All requests with ip failed.") - - # Set to not use proxy - self.fake_proxy_ip = None - self.fake_proxy = False - - return False - def parse_data(self, m3u8_content: str) -> None: """ Parses the M3U8 content to extract segment information. @@ -190,25 +147,6 @@ class M3U8_Segments: self.segments[i] = self.class_url_fixer.generate_full_url(segment_url) logging.info(f"Generated new URL: {self.segments[i]}, from: {segment_url}") - # Change IP address of server - if self.fake_proxy: - for i in range(len(self.segments)): - segment_url = self.segments[i] - - # Set to not use proxy if 50% failed - if not self.__test_ip(segment_url): - console.log("[red]Cant use proxy switch to normal url.") - self.fake_proxy = False - break - - self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url)) - - # Save new playlist of segment - path_m3u8_file = os.path.join(self.tmp_folder, "playlist_fix.m3u8") - with open(path_m3u8_file, "w") as file: - for item in self.segments: - file.write(f"{item}\n") - # Update segments for estimator self.class_ts_estimator.total_segments = len(self.segments) logging.info(f"Segmnets to donwload: [{len(self.segments)}]") @@ -231,29 +169,6 @@ class M3U8_Segments: # Parse the text from the M3U8 index file self.parse_data(response.text) - def __gen_proxy__(self, url: str, url_index: int) -> str: - """ - Change the IP address of the provided URL based on the given index. - - Args: - - url (str): The original URL that needs its IP address replaced. - - url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP. - - Returns: - str: The modified URL with the new IP address. - """ - if self.fake_proxy: - - new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)] - - # Parse the original URL and replace the hostname with the new IP address - parsed_url = urlparse(url)._replace(netloc=new_ip_address) - - return urlunparse(parsed_url) - - else: - return url - def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: """ Downloads a TS segment and adds it to the segment queue. @@ -269,9 +184,27 @@ class M3U8_Segments: try: - # Make request and calculate time duration start_time = time.time() - response = requests.get(ts_url, headers=headers_segments, verify=REQUEST_VERIFY_SSL, timeout=10) + + # Generate proxy + if len(PROXY_LIST) > 0: + + # Generate proxy + new_proxy = PROXY_LIST[index % len(PROXY_LIST)] + proxy_url = f"http://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" + logging.info(f"Generate proxy url: {proxy_url}") + + # Make request + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies={'http': proxy_url, 'https': proxy_url}) + response.raise_for_status() + + else: + + # Make request + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + # Calculate duration duration = time.time() - start_time logging.info(f"Make request to get segment: [{index} - {len(self.segments)}] in: {duration}, len data: {len(response.content)}") @@ -293,8 +226,10 @@ class M3U8_Segments: else: logging.error(f"Failed to download segment: {ts_url}") + except (HTTPError, ConnectionError, Timeout, RequestException) as e: + logging.error(f"Request-related exception while downloading segment: {e}") except Exception as e: - logging.error(f"Exception while downloading segment: {e}") + logging.error(f"An unexpected exception occurred while download segment: {e}") # Update bar progress_bar.update(1) diff --git a/config.json b/config.json index 4cfd9e6..11f12cd 100644 --- a/config.json +++ b/config.json @@ -14,7 +14,8 @@ "max_retry": 3, "verify_ssl": false, "index": {"user-agent": ""}, - "segments": { "user-agent": ""} + "segments": { "user-agent": ""}, + "proxy": [] }, "M3U8_DOWNLOAD": { "tdqm_workers": 4,