From 3ccb9b29468cbbce491e910bac90e68eb58e70d0 Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Thu, 6 Jun 2024 23:06:32 +0200 Subject: [PATCH 1/5] Remove ip --- Src/Api/Streamingcommunity/film.py | 2 +- Src/Api/Streamingcommunity/series.py | 2 +- Src/Lib/Hls/segments.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/Api/Streamingcommunity/film.py b/Src/Api/Streamingcommunity/film.py index 765a5cf..e55fd8f 100644 --- a/Src/Api/Streamingcommunity/film.py +++ b/Src/Api/Streamingcommunity/film.py @@ -59,4 +59,4 @@ def download_film(id_film: str, title_name: str, domain: str): Downloader( m3u8_playlist = master_playlist, output_filename = os.path.join(mp4_path, mp4_format) - ).start(STATIC_IP_SERVER) \ No newline at end of file + ).start() \ No newline at end of file diff --git a/Src/Api/Streamingcommunity/series.py b/Src/Api/Streamingcommunity/series.py index 1d0458b..71396d8 100644 --- a/Src/Api/Streamingcommunity/series.py +++ b/Src/Api/Streamingcommunity/series.py @@ -96,7 +96,7 @@ def donwload_video(tv_name: str, index_season_selected: int, index_episode_selec Downloader( m3u8_playlist = master_playlist, output_filename = os.path.join(mp4_path, mp4_name) - ).start(STATIC_IP_SERVER) + ).start() def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: bool = False) -> None: diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 050706e..33375e1 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -271,7 +271,7 @@ class M3U8_Segments: # Make request and calculate time duration start_time = time.time() - response = requests.get(ts_url, headers=headers_segments, verify=REQUEST_VERIFY_SSL, timeout=15) + response = requests.get(ts_url, headers=headers_segments, verify=REQUEST_VERIFY_SSL, timeout=10) duration = time.time() - start_time logging.info(f"Make request to get segment: [{index} - {len(self.segments)}] in: {duration}, len data: {len(response.content)}") From 22d69b174f5f9b5c07b6eb933f6b3cbe22988b0b Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:03:20 +0200 Subject: [PATCH 2/5] Remove static ip, introduce proxy. --- Src/Api/Streamingcommunity/costant.py | 4 +- Src/Api/Streamingcommunity/film.py | 2 +- Src/Api/Streamingcommunity/series.py | 2 +- Src/Lib/Hls/downloader.py | 21 ++--- Src/Lib/Hls/segments.py | 123 ++++++-------------------- config.json | 3 +- 6 files changed, 39 insertions(+), 116 deletions(-) diff --git a/Src/Api/Streamingcommunity/costant.py b/Src/Api/Streamingcommunity/costant.py index de48a7f..4aa5e97 100644 --- a/Src/Api/Streamingcommunity/costant.py +++ b/Src/Api/Streamingcommunity/costant.py @@ -2,6 +2,4 @@ STREAMING_FOLDER = "streamingcommunity" MOVIE_FOLDER = "Movie" -SERIES_FOLDER = "Serie" - -STATIC_IP_SERVER = ['57.129.7.85', '57.129.7.188', '57.129.7.174', '57.129.4.77', '57.129.16.196', '57.129.16.156', '57.129.16.139', '57.129.16.135', '57.129.13.175', '57.129.13.157', '51.38.112.237', '51.195.107.7', '51.195.107.230', '162.19.255.78', '162.19.255.36', '162.19.255.224', '162.19.255.223', '162.19.254.244', '162.19.254.232', '162.19.254.230', '162.19.253.242', '162.19.249.48', '162.19.245.142', '162.19.231.20', '162.19.229.177', '162.19.228.128', '162.19.228.127', '162.19.228.105', '141.95.1.32', '141.95.1.196', '141.95.1.102', '141.95.0.50', '141.95.0.248', '135.125.237.84', '135.125.233.236'] \ No newline at end of file +SERIES_FOLDER = "Serie" \ No newline at end of file diff --git a/Src/Api/Streamingcommunity/film.py b/Src/Api/Streamingcommunity/film.py index e55fd8f..2b9d7bf 100644 --- a/Src/Api/Streamingcommunity/film.py +++ b/Src/Api/Streamingcommunity/film.py @@ -18,7 +18,7 @@ from .Core.Vix_player.player import VideoSource # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, MOVIE_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, MOVIE_FOLDER # Variable diff --git a/Src/Api/Streamingcommunity/series.py b/Src/Api/Streamingcommunity/series.py index 71396d8..9b2ecc0 100644 --- a/Src/Api/Streamingcommunity/series.py +++ b/Src/Api/Streamingcommunity/series.py @@ -20,7 +20,7 @@ from .Core.Util import manage_selection, map_episode_title # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') -from .costant import STREAMING_FOLDER, SERIES_FOLDER, STATIC_IP_SERVER +from .costant import STREAMING_FOLDER, SERIES_FOLDER # Variable diff --git a/Src/Lib/Hls/downloader.py b/Src/Lib/Hls/downloader.py index 26bd443..3feb9e8 100644 --- a/Src/Lib/Hls/downloader.py +++ b/Src/Lib/Hls/downloader.py @@ -232,15 +232,12 @@ class Downloader(): console.print(f"[cyan]Find codec [white]=> ([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") - def __donwload_video__(self, server_ip: list = None): + def __donwload_video__(self): """ Downloads and manages video segments. This method downloads video segments if necessary and updates the list of downloaded video segments. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Construct full path for the video segment directory @@ -256,7 +253,6 @@ class Downloader(): # Create an instance of M3U8_Segments to handle video segments video_m3u8 = M3U8_Segments(self.m3u8_index, full_path_video) - video_m3u8.add_server_ip(server_ip) # Get information about the video segments video_m3u8.get_info() @@ -270,15 +266,12 @@ class Downloader(): else: console.log("[cyan]Video [red]already exists.") - def __donwload_audio__(self, server_ip: list = None): + def __donwload_audio__(self): """ Downloads and manages audio segments. This method iterates over available audio tracks, downloads them if necessary, and updates the list of downloaded audio tracks. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Iterate over each available audio track @@ -305,7 +298,6 @@ class Downloader(): # If the audio segment directory doesn't exist, download audio segments audio_m3u8 = M3U8_Segments(obj_audio.get('uri'), full_path_audio) - audio_m3u8.add_server_ip(server_ip) # Get information about the audio segments audio_m3u8.get_info() @@ -494,12 +486,9 @@ class Downloader(): else: logging.info("Video file converted already exist.") - def start(self, server_ip: list = None) -> None: + def start(self) -> None: """ Start the process of fetching, downloading, joining, and cleaning up the video. - - Args: - - server_ip (list): A list of IP addresses to use in requests. """ # Check if file already exist @@ -532,9 +521,9 @@ class Downloader(): # Start all download ... if DOWNLOAD_VIDEO: - self.__donwload_video__(server_ip) + self.__donwload_video__() if DOWNLOAD_AUDIO: - self.__donwload_audio__(server_ip) + self.__donwload_audio__() if DOWNLOAD_SUBTITLE: self.__download_subtitle__() diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 33375e1..50a206a 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -13,6 +13,7 @@ from urllib.parse import urljoin, urlparse, urlunparse # External libraries import requests +from requests.exceptions import HTTPError, ConnectionError, Timeout, RequestException from tqdm import tqdm @@ -36,16 +37,19 @@ from ..M3U8 import ( import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + # Config TQDM_MAX_WORKER = config_manager.get_int('M3U8_DOWNLOAD', 'tdqm_workers') TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar') -REQUEST_VERIFY_SSL = config_manager.get_bool('REQUESTS', 'verify_ssl') +REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout') +PROXY_LIST = config_manager.get_list('REQUESTS', 'proxy') # Variable headers_index = config_manager.get_dict('REQUESTS', 'index') headers_segments = config_manager.get_dict('REQUESTS', 'segments') - +session = requests.Session() +session.verify = config_manager.get_bool('REQUESTS', 'verify_ssl') class M3U8_Segments: @@ -58,7 +62,6 @@ class M3U8_Segments: - tmp_folder (str): The temporary folder to store downloaded segments. """ self.url = url - self.fake_proxy = False self.tmp_folder = tmp_folder self.tmp_file_path = os.path.join(self.tmp_folder, "0.ts") os.makedirs(self.tmp_folder, exist_ok=True) @@ -73,17 +76,6 @@ class M3U8_Segments: self.segment_queue = queue.PriorityQueue() # Priority queue to maintain the order of segments self.condition = threading.Condition() # Condition variable for thread synchronization - def add_server_ip(self, list_ip): - """ - Add server IP addresses - - Args: - list_ip (list): A list of IP addresses to be added. - """ - if list_ip is not None: - self.fake_proxy = True - self.fake_proxy_ip = list_ip - def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: """ Retrieves the encryption key from the M3U8 playlist. @@ -115,41 +107,6 @@ class M3U8_Segments: logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})") return byte_content - def __test_ip(self, url_to_test: str): - """ - Tests each proxy IP by sending a request to a corresponding segment URL. - """ - - failed_ips = [] - - for i in range(len(self.fake_proxy_ip)): - - try: - response = requests.get(url_to_test, verify=False, retries=0) - - if response == None: - logging.error(f"[Work] to make request using: {url_to_test}") - failed_ips.append(i) - - except: - - # Log the error and add the IP to the list of failed IPs - logging.error(f"[Fail] to make request using IP in this request: {url_to_test}") - failed_ips.append(i) - - # Remove the failed IPs from the fake_proxy_ip list - self.fake_proxy_ip = [ip for j, ip in enumerate(self.fake_proxy_ip) if j not in failed_ips] - - # Exit the program if 50% requests failed - if len(failed_ips) / 2 > len(self.fake_proxy_ip): - logging.error("All requests with ip failed.") - - # Set to not use proxy - self.fake_proxy_ip = None - self.fake_proxy = False - - return False - def parse_data(self, m3u8_content: str) -> None: """ Parses the M3U8 content to extract segment information. @@ -190,25 +147,6 @@ class M3U8_Segments: self.segments[i] = self.class_url_fixer.generate_full_url(segment_url) logging.info(f"Generated new URL: {self.segments[i]}, from: {segment_url}") - # Change IP address of server - if self.fake_proxy: - for i in range(len(self.segments)): - segment_url = self.segments[i] - - # Set to not use proxy if 50% failed - if not self.__test_ip(segment_url): - console.log("[red]Cant use proxy switch to normal url.") - self.fake_proxy = False - break - - self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url)) - - # Save new playlist of segment - path_m3u8_file = os.path.join(self.tmp_folder, "playlist_fix.m3u8") - with open(path_m3u8_file, "w") as file: - for item in self.segments: - file.write(f"{item}\n") - # Update segments for estimator self.class_ts_estimator.total_segments = len(self.segments) logging.info(f"Segmnets to donwload: [{len(self.segments)}]") @@ -231,29 +169,6 @@ class M3U8_Segments: # Parse the text from the M3U8 index file self.parse_data(response.text) - def __gen_proxy__(self, url: str, url_index: int) -> str: - """ - Change the IP address of the provided URL based on the given index. - - Args: - - url (str): The original URL that needs its IP address replaced. - - url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP. - - Returns: - str: The modified URL with the new IP address. - """ - if self.fake_proxy: - - new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)] - - # Parse the original URL and replace the hostname with the new IP address - parsed_url = urlparse(url)._replace(netloc=new_ip_address) - - return urlunparse(parsed_url) - - else: - return url - def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: """ Downloads a TS segment and adds it to the segment queue. @@ -269,9 +184,27 @@ class M3U8_Segments: try: - # Make request and calculate time duration start_time = time.time() - response = requests.get(ts_url, headers=headers_segments, verify=REQUEST_VERIFY_SSL, timeout=10) + + # Generate proxy + if len(PROXY_LIST) > 0: + + # Generate proxy + new_proxy = PROXY_LIST[index % len(PROXY_LIST)] + proxy_url = f"http://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" + logging.info(f"Generate proxy url: {proxy_url}") + + # Make request + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies={'http': proxy_url, 'https': proxy_url}) + response.raise_for_status() + + else: + + # Make request + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT) + response.raise_for_status() + + # Calculate duration duration = time.time() - start_time logging.info(f"Make request to get segment: [{index} - {len(self.segments)}] in: {duration}, len data: {len(response.content)}") @@ -293,8 +226,10 @@ class M3U8_Segments: else: logging.error(f"Failed to download segment: {ts_url}") + except (HTTPError, ConnectionError, Timeout, RequestException) as e: + logging.error(f"Request-related exception while downloading segment: {e}") except Exception as e: - logging.error(f"Exception while downloading segment: {e}") + logging.error(f"An unexpected exception occurred while download segment: {e}") # Update bar progress_bar.update(1) diff --git a/config.json b/config.json index 4cfd9e6..11f12cd 100644 --- a/config.json +++ b/config.json @@ -14,7 +14,8 @@ "max_retry": 3, "verify_ssl": false, "index": {"user-agent": ""}, - "segments": { "user-agent": ""} + "segments": { "user-agent": ""}, + "proxy": [] }, "M3U8_DOWNLOAD": { "tdqm_workers": 4, From eb213187e84fd86e71f38bd9c2924b4617e6bf88 Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Fri, 7 Jun 2024 22:32:12 +0200 Subject: [PATCH 3/5] Finish proxy addition. --- README.md | 6 +++--- Src/Lib/Hls/segments.py | 37 +++++++++++++++++++++++++++++++------ config.json | 3 +-- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a05a96e..6bb3962 100644 --- a/README.md +++ b/README.md @@ -96,12 +96,12 @@ You can change some behaviors by tweaking the configuration file. * **timeout**: The timeout value for requests. - **Default Value**: `10` - * **max_retry**: Maximum number of retries for requests. - - **Default Value**: `3` - * **verify_ssl**: Whether to verify SSL certificates. - **Default Value**: `false` + * **proxy**: The proxy to use for requests. (Note: This parameter works only with HTTP and HTTPS protocols.) + - **Example Value**: `[{'protocol': 'http', 'ip': '123.45.67.89', 'port': '8080', 'username': 'your_username', 'password': 'your_password'}, {'protocol': 'https', 'ip': '123.45.67.89', 'port': '8080', 'username': 'your_username', 'password': 'your_password'}]` +
diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 50a206a..4120881 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -169,6 +169,36 @@ class M3U8_Segments: # Parse the text from the M3U8 index file self.parse_data(response.text) + def get_proxy(self, index): + """ + Returns the proxy configuration for the given index. + + Args: + - index (int): The index to select the proxy from the PROXY_LIST. + + Returns: + - dict: A dictionary containing the proxy scheme and proxy URL. + """ + try: + + # Select the proxy from the list using the index + new_proxy = PROXY_LIST[index % len(PROXY_LIST)] + proxy_scheme = new_proxy["protocol"] + + # Construct the proxy URL based on the presence of user and pass keys + if "user" in new_proxy and "pass" in new_proxy: + proxy_url = f"{proxy_scheme}://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" + else: + proxy_url = f"{proxy_scheme}://{new_proxy['ip']}:{new_proxy['port']}" + + logging.info(f"Proxy URL generated: {proxy_url}") + return {proxy_scheme: proxy_url} + + except KeyError as e: + logging.error(f"KeyError: Missing required key {e} in proxy configuration.") + except Exception as e: + logging.error(f"An unexpected error occurred while generating proxy URL: {e}") + def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: """ Downloads a TS segment and adds it to the segment queue. @@ -189,13 +219,8 @@ class M3U8_Segments: # Generate proxy if len(PROXY_LIST) > 0: - # Generate proxy - new_proxy = PROXY_LIST[index % len(PROXY_LIST)] - proxy_url = f"http://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" - logging.info(f"Generate proxy url: {proxy_url}") - # Make request - response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies={'http': proxy_url, 'https': proxy_url}) + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=self.get_proxy(index)) response.raise_for_status() else: diff --git a/config.json b/config.json index 11f12cd..e815bf8 100644 --- a/config.json +++ b/config.json @@ -10,8 +10,7 @@ "not_close": false }, "REQUESTS": { - "timeout": 10, - "max_retry": 3, + "timeout": 5, "verify_ssl": false, "index": {"user-agent": ""}, "segments": { "user-agent": ""}, From 500fb965f43d4cdd9f2b017f0fc3be32bb31b185 Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Fri, 7 Jun 2024 23:04:55 +0200 Subject: [PATCH 4/5] Add other proxy option --- Src/Lib/Hls/segments.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 4120881..1eb5b80 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -188,6 +188,8 @@ class M3U8_Segments: # Construct the proxy URL based on the presence of user and pass keys if "user" in new_proxy and "pass" in new_proxy: proxy_url = f"{proxy_scheme}://{new_proxy['user']}:{new_proxy['pass']}@{new_proxy['ip']}:{new_proxy['port']}" + elif "user" in new_proxy: + proxy_url = f"{proxy_scheme}://{new_proxy['user']}@{new_proxy['ip']}:{new_proxy['port']}" else: proxy_url = f"{proxy_scheme}://{new_proxy['ip']}:{new_proxy['port']}" @@ -210,6 +212,7 @@ class M3U8_Segments: """ # Generate new user agent + proxy = self.get_proxy(index) headers_segments['user-agent'] = get_headers() try: @@ -220,7 +223,7 @@ class M3U8_Segments: if len(PROXY_LIST) > 0: # Make request - response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=self.get_proxy(index)) + response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=proxy) response.raise_for_status() else: From 86ecac7eddf8736837c06eb97503a269fd6ad80f Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Fri, 7 Jun 2024 23:30:07 +0200 Subject: [PATCH 5/5] Fix error proxy 1 --- Src/Lib/Hls/segments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 1eb5b80..03b2d7d 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -212,7 +212,6 @@ class M3U8_Segments: """ # Generate new user agent - proxy = self.get_proxy(index) headers_segments['user-agent'] = get_headers() try: @@ -223,6 +222,7 @@ class M3U8_Segments: if len(PROXY_LIST) > 0: # Make request + proxy = self.get_proxy(index) response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=proxy) response.raise_for_status()