diff --git a/README.md b/README.md index a05a96e..6bb3962 100644 --- a/README.md +++ b/README.md @@ -96,12 +96,12 @@ You can change some behaviors by tweaking the configuration file. * **timeout**: The timeout value for requests. - **Default Value**: `10` - * **max_retry**: Maximum number of retries for requests. - - **Default Value**: `3` - * **verify_ssl**: Whether to verify SSL certificates. - **Default Value**: `false` + * **proxy**: The proxy to use for requests. (Note: This parameter works only with HTTP and HTTPS protocols.) + - **Example Value**: `[{'protocol': 'http', 'ip': '123.45.67.89', 'port': '8080', 'username': 'your_username', 'password': 'your_password'}, {'protocol': 'https', 'ip': '123.45.67.89', 'port': '8080', 'username': 'your_username', 'password': 'your_password'}]` +
diff --git a/Src/Api/Streamingcommunity/costant.py b/Src/Api/Streamingcommunity/costant.py index de48a7f..4aa5e97 100644 --- a/Src/Api/Streamingcommunity/costant.py +++ b/Src/Api/Streamingcommunity/costant.py @@ -2,6 +2,4 @@ STREAMING_FOLDER = "streamingcommunity" MOVIE_FOLDER = "Movie" -SERIES_FOLDER = "Serie" - -STATIC_IP_SERVER = ['57.129.7.85', '57.129.7.188', '57.129.7.174', '57.129.4.77', '57.129.16.196', '57.129.16.156', '57.129.16.139', '57.129.16.135', '57.129.13.175', '57.129.13.157', '51.38.112.237', '51.195.107.7', '51.195.107.230', '162.19.255.78', '162.19.255.36', '162.19.255.224', '162.19.255.223', '162.19.254.244', '162.19.254.232', '162.19.254.230', '162.19.253.242', '162.19.249.48', '162.19.245.142', '162.19.231.20', '162.19.229.177', '162.19.228.128', '162.19.228.127', '162.19.228.105', '141.95.1.32', '141.95.1.196', '141.95.1.102', '141.95.0.50', '141.95.0.248', '135.125.237.84', '135.125.233.236'] \ No newline at end of file +SERIES_FOLDER = "Serie" \ No newline at end of file diff --git a/Src/Api/Streamingcommunity/film.py b/Src/Api/Streamingcommunity/film.py index 765a5cf..2b9d7bf 100644 --- a/Src/Api/Streamingcommunity/film.py +++ b/Src/Api/Streamingcommunity/film.py @@ -18,7 +18,7 @@ from .Core.Vix_player.player import VideoSource # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, MOVIE_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, MOVIE_FOLDER # Variable @@ -59,4 +59,4 @@ def download_film(id_film: str, title_name: str, domain: str): Downloader( m3u8_playlist = master_playlist, output_filename = os.path.join(mp4_path, mp4_format) - ).start(STATIC_IP_SERVER) \ No newline at end of file + ).start() \ No newline at end of file diff --git a/Src/Api/Streamingcommunity/series.py b/Src/Api/Streamingcommunity/series.py index 1d0458b..9b2ecc0 100644 --- a/Src/Api/Streamingcommunity/series.py +++ b/Src/Api/Streamingcommunity/series.py @@ -20,7 +20,7 @@ from .Core.Util import manage_selection, map_episode_title # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, SERIES_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, SERIES_FOLDER # Variable @@ -96,7 +96,7 @@ def donwload_video(tv_name: str, index_season_selected: int, index_episode_selec Downloader( m3u8_playlist = master_playlist, output_filename = os.path.join(mp4_path, mp4_name) - ).start(STATIC_IP_SERVER) + ).start() def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: bool = False) -> None: diff --git a/Src/Lib/Hls/downloader.py b/Src/Lib/Hls/downloader.py index 26bd443..3feb9e8 100644 --- a/Src/Lib/Hls/downloader.py +++ b/Src/Lib/Hls/downloader.py @@ -232,15 +232,12 @@ class Downloader(): console.print(f"[cyan]Find codec [white]=> ([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") - def __donwload_video__(self, server_ip: list = None): + def __donwload_video__(self): """ Downloads and manages video segments. This method downloads video segments if necessary and updates the list of downloaded video segments. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Construct full path for the video segment directory @@ -256,7 +253,6 @@ class Downloader(): # Create an instance of M3U8_Segments to handle video segments video_m3u8 = M3U8_Segments(self.m3u8_index, full_path_video) - video_m3u8.add_server_ip(server_ip) # Get information about the video segments video_m3u8.get_info() @@ -270,15 +266,12 @@ class Downloader(): else: console.log("[cyan]Video [red]already exists.") - def __donwload_audio__(self, server_ip: list = None): + def __donwload_audio__(self): """ Downloads and manages audio segments. This method iterates over available audio tracks, downloads them if necessary, and updates the list of downloaded audio tracks. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Iterate over each available audio track @@ -305,7 +298,6 @@ class Downloader(): # If the audio segment directory doesn't exist, download audio segments audio_m3u8 = M3U8_Segments(obj_audio.get('uri'), full_path_audio) - audio_m3u8.add_server_ip(server_ip) # Get information about the audio segments audio_m3u8.get_info() @@ -494,12 +486,9 @@ class Downloader(): else: logging.info("Video file converted already exist.") - def start(self, server_ip: list = None) -> None: + def start(self) -> None: """ Start the process of fetching, downloading, joining, and cleaning up the video. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Check if file already exist @@ -532,9 +521,9 @@ class Downloader(): # Start all download ... if DOWNLOAD_VIDEO: - self.__donwload_video__(server_ip) + self.__donwload_video__() if DOWNLOAD_AUDIO: - self.__donwload_audio__(server_ip) + self.__donwload_audio__() if DOWNLOAD_SUBTITLE: self.__download_subtitle__() diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 050706e..03b2d7d 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -13,6 +13,7 @@ from urllib.parse import urljoin, urlparse, urlunparse # External libraries import requests +from requests.exceptions import HTTPError, ConnectionError, Timeout, RequestException from tqdm import tqdm @@ -36,16 +37,19 @@ from ..M3U8 import ( import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # Config TQDM_MAX_WORKER = config_manager.get_int('M3U8_DOWNLOAD', 'tdqm_workers') TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar') -REQUEST_VERIFY_SSL = config_manager.get_bool('REQUESTS', 'verify_ssl') +REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout') +PROXY_LIST = config_manager.get_list('REQUESTS', 'proxy') # Variable headers_index = config_manager.get_dict('REQUESTS', 'index') headers_segments = config_manager.get_dict('REQUESTS', 'segments') - +session = requests.Session() +session.verify = config_manager.get_bool('REQUESTS', 'verify_ssl') class M3U8_Segments: @@ -58,7 +62,6 @@ class M3U8_Segments: - tmp_folder (str): The temporary folder to store downloaded segments. """ self.url = url - self.fake_proxy = False self.tmp_folder = tmp_folder self.tmp_file_path = os.path.join(self.tmp_folder, "0.ts") os.makedirs(self.tmp_folder, exist_ok=True) @@ -73,17 +76,6 @@ class M3U8_Segments: self.segment_queue = queue.PriorityQueue() # Priority queue to maintain the order of segments self.condition = threading.Condition() # Condition variable for thread synchronization - def add_server_ip(self, list_ip): - """ - Add server IP addresses - - Args: - list_ip (list): A list of IP addresses to be added. - """ - if list_ip is not None: - self.fake_proxy = True - self.fake_proxy_ip = list_ip - def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: """ Retrieves the encryption key from the M3U8 playlist. @@ -115,41 +107,6 @@ class M3U8_Segments: logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})") return byte_content - def __test_ip(self, url_to_test: str): - """ - Tests each proxy IP by sending a request to a corresponding segment URL. - """ - - failed_ips = [] - - for i in range(len(self.fake_proxy_ip)): - - try: - response = requests.get(url_to_test, verify=False, retries=0) - - if response == None: - logging.error(f"[Work] to make request using: {url_to_test}") - failed_ips.append(i) - - except: - - # Log the error and add the IP to the list of failed IPs - logging.error(f"[Fail] to make request using IP in this request: {url_to_test}") - failed_ips.append(i) - - # Remove the failed IPs from the fake_proxy_ip list - self.fake_proxy_ip = [ip for j, ip in enumerate(self.fake_proxy_ip) if j not in failed_ips] - - # Exit the program if 50% requests failed - if len(failed_ips) / 2 > len(self.fake_proxy_ip): - logging.error("All requests with ip failed.") - - # Set to not use proxy - self.fake_proxy_ip = None - self.fake_proxy = False - - return False - def parse_data(self, m3u8_content: str) -> None: """ Parses the M3U8 content to extract segment information. @@ -190,25 +147,6 @@ class M3U8_Segments: self.segments[i] = self.class_url_fixer.generate_full_url(segment_url) logging.info(f"Generated new URL: {self.segments[i]}, from: {segment_url}") - # Change IP address of server - if self.fake_proxy: - for i in range(len(self.segments)): - segment_url = self.segments[i] - - # Set to not use proxy if 50% failed - if not self.__test_ip(segment_url): - console.log("[red]Cant use proxy switch to normal url.") - self.fake_proxy = False - break - - self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url)) - - # Save new playlist of segment - path_m3u8_file = os.path.join(self.tmp_folder, "playlist_fix.m3u8") - with open(path_m3u8_file, "w") as file: - for item in self.segments: - file.write(f"{item}\n") - # Update segments for estimator self.class_ts_estimator.total_segments = len(self.segments) logging.info(f"Segmnets to donwload: [{len(self.segments)}]") @@ -231,28 +169,37 @@ class M3U8_Segments: # Parse the text from the M3U8 index file self.parse_data(response.text) - def __gen_proxy__(self, url: str, url_index: int) -> str: + def get_proxy(self, index): """ - Change the IP address of the provided URL based on the given index. - - Args: - - url (str): The original URL that needs its IP address replaced. - - url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP. - - Returns: - str: The modified URL with the new IP address. - """ - if self.fake_proxy: - - new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)] - - # Parse the original URL and replace the hostname with the new IP address - parsed_url = urlparse(url)._replace(netloc=new_ip_address) - - return urlunparse(parsed_url) + Returns the proxy configuration for the given index. - else: - return url + Args: + - index (int): The index to select the proxy from the PROXY_LIST. + + Returns: + - dict: A dictionary containing the proxy scheme and proxy URL. + """ + try: + + # Select the proxy from the list using the index + new_proxy = PROXY_LIST[index % len(PROXY_LIST)] + proxy_scheme = new_proxy["protocol"] + + # Construct the proxy URL based on the presence of user and pass keys + if "user" in new_proxy and "pass" in new_proxy: + proxy_url = f"{proxy_scheme}://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" + elif "user" in new_proxy: + proxy_url = f"{proxy_scheme}://{new_proxy['user']}@{new_proxy['ip']}:{new_proxy['port']}" + else: + proxy_url = f"{proxy_scheme}://{new_proxy['ip']}:{new_proxy['port']}" + + logging.info(f"Proxy URL generated: {proxy_url}") + return {proxy_scheme: proxy_url} + + except KeyError as e: + logging.error(f"KeyError: Missing required key {e} in proxy configuration.") + except Exception as e: + logging.error(f"An unexpected error occurred while generating proxy URL: {e}") def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: """ @@ -269,9 +216,23 @@ class M3U8_Segments: try: - # Make request and calculate time duration start_time = time.time() - response = requests.get(ts_url, headers=headers_segments, verify=REQUEST_VERIFY_SSL, timeout=15) + + # Generate proxy + if len(PROXY_LIST) > 0: + + # Make request + proxy = self.get_proxy(index) + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=proxy) + response.raise_for_status() + + else: + + # Make request + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + # Calculate duration duration = time.time() - start_time logging.info(f"Make request to get segment: [{index} - {len(self.segments)}] in: {duration}, len data: {len(response.content)}") @@ -293,8 +254,10 @@ class M3U8_Segments: else: logging.error(f"Failed to download segment: {ts_url}") + except (HTTPError, ConnectionError, Timeout, RequestException) as e: + logging.error(f"Request-related exception while downloading segment: {e}") except Exception as e: - logging.error(f"Exception while downloading segment: {e}") + logging.error(f"An unexpected exception occurred while download segment: {e}") # Update bar progress_bar.update(1) diff --git a/config.json b/config.json index 4cfd9e6..e815bf8 100644 --- a/config.json +++ b/config.json @@ -10,11 +10,11 @@ "not_close": false }, "REQUESTS": { - "timeout": 10, - "max_retry": 3, + "timeout": 5, "verify_ssl": false, "index": {"user-agent": ""}, - "segments": { "user-agent": ""} + "segments": { "user-agent": ""}, + "proxy": [] }, "M3U8_DOWNLOAD": { "tdqm_workers": 4,