diff --git a/Src/Api/Template/Util/recall_search.py b/Src/Api/Template/Util/recall_search.py new file mode 100644 index 0000000..e93372d --- /dev/null +++ b/Src/Api/Template/Util/recall_search.py @@ -0,0 +1,35 @@ +# 19.10.24 + +import os +import sys + +def execute_search(info): + """ + Dynamically imports and executes a specified function from a module defined in the info dictionary. + + Parameters: + info (dict): A dictionary containing the function name, folder, and module information. + """ + + # Step 1: Define the project path using the folder from the info dictionary + project_path = os.path.dirname(info['folder']) # Get the base path for the project + + # Step 2: Add the project path to sys.path + if project_path not in sys.path: + sys.path.append(project_path) + + # Attempt to import the specified function from the module + try: + # Construct the import statement dynamically + module_path = f"Src.Api.{info['folder_base']}" + exec(f"from {module_path} import {info['function']}") + + # Call the specified function + eval(info['function'])() # Calls the search function + + except ModuleNotFoundError as e: + print(f"ModuleNotFoundError: {e}") + except ImportError as e: + print(f"ImportError: {e}") + except Exception as e: + print(f"An error occurred: {e}") diff --git a/Src/Api/Template/__init__.py b/Src/Api/Template/__init__.py index 38a9119..ce9d599 100644 --- a/Src/Api/Template/__init__.py +++ b/Src/Api/Template/__init__.py @@ -1,5 +1,6 @@ # 19.06.24 from .site import get_select_title +from .Util.recall_search import execute_search from .Util.get_domain import search_domain from .Util.manage_ep import manage_selection, map_episode_title, validate_episode_selection, validate_selection \ No newline at end of file diff --git a/Src/Api/altadefinizione/film.py b/Src/Api/altadefinizione/film.py index db6d0d2..b32d2f8 100644 --- a/Src/Api/altadefinizione/film.py +++ b/Src/Api/altadefinizione/film.py @@ -8,7 +8,6 @@ import logging # Internal utilities from Src.Util.message import start_message from Src.Util.console import console -from Src.Util.os import create_folder, can_create_file, remove_special_characters from Src.Lib.Downloader import HLS_Downloader @@ -38,16 +37,8 @@ def download_film(select_title: MediaItem): video_source = VideoSource(select_title.url) # Define output path - mp4_name = remove_special_characters(select_title.name) + ".mp4" - mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.name)) - - # Ensure the folder path exists - create_folder(mp4_path) - - # Check if the MP4 file can be created - if not can_create_file(mp4_name): - logging.error("Invalid mp4 name.") - sys.exit(0) + mp4_name = select_title.name + ".mp4" + mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name) # Get m3u8 master playlist master_playlist = video_source.get_playlist() diff --git a/Src/Api/cb01/film.py b/Src/Api/cb01/film.py index 1d77c74..9a9f44f 100644 --- a/Src/Api/cb01/film.py +++ b/Src/Api/cb01/film.py @@ -8,7 +8,6 @@ import logging # Internal utilities from Src.Util.console import console from Src.Util.message import start_message -from Src.Util.os import create_folder, can_create_file, remove_special_characters from Src.Lib.Downloader import HLS_Downloader @@ -37,17 +36,8 @@ def download_film(select_title: MediaItem): video_source = VideoSource(select_title.url) # Define output path - title_name = remove_special_characters(select_title.name) - mp4_name = remove_special_characters(title_name) +".mp4" - mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, title_name) - - # Ensure the folder path exists - create_folder(mp4_path) - - # Check if the MP4 file can be created - if not can_create_file(mp4_name): - logging.error("Invalid mp4 name.") - sys.exit(0) + mp4_name = select_title.name +".mp4" + mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name) # Get m3u8 master playlist master_playlist = video_source.get_playlist() diff --git a/Src/Api/guardaserie/series.py b/Src/Api/guardaserie/series.py index c7f3752..043921c 100644 --- a/Src/Api/guardaserie/series.py +++ b/Src/Api/guardaserie/series.py @@ -2,16 +2,16 @@ import os import sys -import logging +import time # Internal utilities from Src.Util.console import console, msg -from Src.Util.os import create_folder, can_create_file from Src.Util.message import start_message +from Src.Util.call_stack import get_call_stack from Src.Util.table import TVShowManager from Src.Lib.Downloader import HLS_Downloader -from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection +from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search # Logic class @@ -48,24 +48,19 @@ def download_video(scape_info_serie: GetSerieInfo, index_season_selected: int, i mp4_name = f"{map_episode_title(scape_info_serie.tv_name, index_season_selected, index_episode_selected, obj_episode.get('name'))}.mp4" mp4_path = os.path.join(ROOT_PATH, SITE_NAME, SERIES_FOLDER, scape_info_serie.tv_name, f"S{index_season_selected}") - # Ensure the folder path exists - create_folder(mp4_path) - - # Check if the MP4 file can be created - if not can_create_file(mp4_name): - logging.error("Invalid mp4 name.") - sys.exit(0) - # Setup video source video_source.setup(obj_episode.get('url')) # Get m3u8 master playlist master_playlist = video_source.get_playlist() - HLS_Downloader( - m3u8_playlist = master_playlist, - output_filename = os.path.join(mp4_path, mp4_name) - ).start() + if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404: + time.sleep(2) + + # Re call search function + if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n": + frames = get_call_stack() + execute_search(frames[-4]) def download_episode(scape_info_serie: GetSerieInfo, index_season_selected: int, download_all: bool = False) -> None: diff --git a/Src/Api/streamingcommunity/film.py b/Src/Api/streamingcommunity/film.py index 2705091..159c2c8 100644 --- a/Src/Api/streamingcommunity/film.py +++ b/Src/Api/streamingcommunity/film.py @@ -8,7 +8,6 @@ import logging # Internal utilities from Src.Util.console import console from Src.Util.message import start_message -from Src.Util.os import create_folder, can_create_file, remove_special_characters from Src.Lib.Downloader import HLS_Downloader @@ -44,17 +43,8 @@ def download_film(select_title: MediaItem, domain: str, version: str): master_playlist = video_source.get_playlist() # Define the filename and path for the downloaded film - mp4_name = remove_special_characters(select_title.slug) - mp4_format = (mp4_name) + ".mp4" - mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.slug)) - - # Ensure the folder path exists - create_folder(mp4_path) - - # Check if the MP4 file can be created - if not can_create_file(mp4_name): - logging.error("Invalid mp4 name.") - sys.exit(0) + mp4_format = (select_title.slug) + ".mp4" + mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.slug) # Download the film using the m3u8 playlist, and output filename HLS_Downloader( diff --git a/Src/Api/streamingcommunity/series.py b/Src/Api/streamingcommunity/series.py index 36db672..e983329 100644 --- a/Src/Api/streamingcommunity/series.py +++ b/Src/Api/streamingcommunity/series.py @@ -2,15 +2,16 @@ import os import sys -import logging +import time # Internal utilities from Src.Util.console import console, msg from Src.Util.message import start_message +from Src.Util.call_stack import get_call_stack from Src.Util.table import TVShowManager from Src.Lib.Downloader import HLS_Downloader -from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection +from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search # Logic class @@ -52,11 +53,13 @@ def download_video(tv_name: str, index_season_selected: int, index_episode_selec master_playlist = video_source.get_playlist() # Download the episode - HLS_Downloader( - m3u8_playlist = master_playlist, - output_filename = os.path.join(mp4_path, mp4_name) - ).start() + if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404: + time.sleep(2) + # Re call search function + if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n": + frames = get_call_stack() + execute_search(frames[-4]) def download_episode(tv_name: str, index_season_selected: int, download_all: bool = False) -> None: """ diff --git a/Src/Lib/Downloader/HLS/downloader.py b/Src/Lib/Downloader/HLS/downloader.py index 9c6f790..9587065 100644 --- a/Src/Lib/Downloader/HLS/downloader.py +++ b/Src/Lib/Downloader/HLS/downloader.py @@ -111,8 +111,8 @@ class HttpClient: return response.text # Return the response text except Exception as e: - logging.error(f"Request to {url} failed: {e}") - return None + logging.error(f"Request to {url} failed: {response.status_code} when get text.") + return 404 def get_content(self, url, timeout=20): """ @@ -128,7 +128,7 @@ class HttpClient: return response.content # Return the raw response content except Exception as e: - logging.error(f"Request to {url} failed: {e}") + logging.error(f"Request to {url} failed: {response.status_code} when get content.") return None @@ -179,13 +179,14 @@ class ContentExtractor: result = list(set(available_languages) & set(set_language)) # Create a formatted table to display audio info - table = Table(show_header=False, box=None) - table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") - table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}") - table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") + if len(available_languages) > 0: + table = Table(show_header=False, box=None) + table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") + table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}") + table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") - console.rule("[bold green] AUDIO ", style="bold red") - console.print(table) + console.rule("[bold green] AUDIO ", style="bold red") + console.print(table) else: console.log("[red]Can't find a list of audios") @@ -207,13 +208,14 @@ class ContentExtractor: result = list(set(available_languages) & set(set_language)) # Create a formatted table to display subtitle info - table = Table(show_header=False, box=None) - table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") - table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}") - table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") + if len(available_languages) > 0: + table = Table(show_header=False, box=None) + table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") + table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}") + table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") - console.rule("[bold green] SUBTITLE ", style="bold red") - console.print(table) + console.rule("[bold green] SUBTITLE ", style="bold red") + console.print(table) else: console.log("[red]Can't find a list of subtitles") @@ -242,7 +244,10 @@ class ContentExtractor: table.add_row(f"[green]Downloadable:", f"[purple]{video_res[0]}x{video_res[1]}") if self.codec is not None: - table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") + if config_manager.get_bool("M3U8_CONVERSION", "use_codec"): + table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") + else: + table.add_row(f"[green]Codec:", "[purple]copy") console.rule("[bold green] VIDEO ", style="bold red") console.print(table) @@ -387,8 +392,7 @@ class ContentDownloader: video_m3u8.download_streams(f"{Colors.MAGENTA}video") # Print duration information of the downloaded video - print_duration_table(downloaded_video[0].get('path')) - print("") + #print_duration_table(downloaded_video[0].get('path')) else: console.log("[cyan]Video [red]already exists.") @@ -416,7 +420,7 @@ class ContentDownloader: audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}") # Print duration information of the downloaded audio - print_duration_table(obj_audio.get('path')) + #print_duration_table(obj_audio.get('path')) else: console.log(f"[cyan]Audio [white]([green]{obj_audio.get('language')}[white]) [red]already exists.") @@ -444,7 +448,7 @@ class ContentDownloader: ) # Print the status of the subtitle download - console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}") + #console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}") # Write the content to the specified file with open(obj_subtitle.get("path"), "wb") as f: @@ -461,7 +465,7 @@ class ContentJoiner: """ self.path_manager: PathManager = path_manager - def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle): + def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle, codec = None): """ Sets up the content joiner with downloaded media files. @@ -473,21 +477,26 @@ class ContentJoiner: self.downloaded_video = downloaded_video self.downloaded_audio = downloaded_audio self.downloaded_subtitle = downloaded_subtitle + self.codec = codec # Initialize flags to check if media is available self.converted_out_path = None - self.there_is_video = (len(downloaded_video) > 0) - self.there_is_audio = (len(downloaded_audio) > 0) - self.there_is_subtitle = (len(downloaded_subtitle) > 0) + self.there_is_video = len(downloaded_video) > 0 + self.there_is_audio = len(downloaded_audio) > 0 + self.there_is_subtitle = len(downloaded_subtitle) > 0 - # Display the status of available media - table = Table(show_header=False, box=None) - table.add_row(f"[green]Video - audio:", f"[yellow]{self.there_is_audio}") - table.add_row(f"[green]Video - Subtitle:", f"[yellow]{self.there_is_subtitle}") + if self.there_is_audio or self.there_is_subtitle: - console.rule("[bold green] JOIN ", style="bold red") - console.print(table) - print("") + # Display the status of available media + table = Table(show_header=False, box=None) + + table.add_row(f"[green]Video - audio", f"[yellow]{self.there_is_audio}") + table.add_row(f"[green]Video - Subtitle", f"[yellow]{self.there_is_subtitle}") + + print("") + console.rule("[bold green] JOIN ", style="bold red") + console.print(table) + print("") # Start the joining process self.conversione() @@ -575,8 +584,8 @@ class ContentJoiner: if not os.path.exists(path_join_video): # Set codec to None if not defined in class - if not hasattr(self, 'codec'): - self.codec = None + #if not hasattr(self, 'codec'): + # self.codec = None # Join the video segments into a single video file join_video( @@ -604,8 +613,8 @@ class ContentJoiner: if not os.path.exists(path_join_video_audio): # Set codec to None if not defined in class - if not hasattr(self, 'codec'): - self.codec = None + #if not hasattr(self, 'codec'): + # self.codec = None # Join the video with audio segments join_audios( @@ -661,7 +670,6 @@ class HLS_Downloader: self.output_filename = self._generate_output_filename(output_filename, m3u8_playlist, m3u8_index) self.path_manager = PathManager(self.output_filename) self.download_tracker = DownloadTracker(self.path_manager) - self.http_client = HttpClient(headers_index) self.content_extractor = ContentExtractor() self.content_downloader = ContentDownloader() self.content_joiner = ContentJoiner(self.path_manager) @@ -728,7 +736,10 @@ class HLS_Downloader: # Determine whether to process a playlist or index if self.m3u8_playlist: - self._process_playlist() + r_proc = self._process_playlist() + + if r_proc == 404: + return 404 elif self.m3u8_index: self._process_index() @@ -780,6 +791,7 @@ class HLS_Downloader: missing_ts = True # Prepare the report panel content + print("") panel_content = ( f"[bold green]Download completed![/bold green]\n" f"[cyan]File size: [bold red]{formatted_size}[/bold red]\n" @@ -815,8 +827,14 @@ class HLS_Downloader: # Retrieve the m3u8 playlist content if self.is_playlist_url: - m3u8_playlist_text = HttpClient(headers=headers_index).get(self.m3u8_playlist) - m3u8_url_fixer.set_playlist(self.m3u8_playlist) + response_text = HttpClient(headers=headers_index).get(self.m3u8_playlist) + + if response_text != 404: + m3u8_playlist_text = response_text + m3u8_url_fixer.set_playlist(self.m3u8_playlist) + + else: + return 404 else: m3u8_playlist_text = self.m3u8_playlist @@ -849,7 +867,7 @@ class HLS_Downloader: self.content_downloader.download_subtitle(self.download_tracker.downloaded_subtitle) # Join downloaded content - self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle) + self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle, self.content_extractor.codec) # Clean up temporary files and directories self._clean(self.content_joiner.converted_out_path) @@ -862,7 +880,8 @@ class HLS_Downloader: # Download video self.download_tracker.add_video(self.m3u8_index) - + self.content_downloader.download_video(self.download_tracker.downloaded_video) + # Join video self.content_joiner.setup(self.download_tracker.downloaded_video, [], []) diff --git a/Src/Lib/Downloader/HLS/segments.py b/Src/Lib/Downloader/HLS/segments.py index 8d5842a..c0121ee 100644 --- a/Src/Lib/Downloader/HLS/segments.py +++ b/Src/Lib/Downloader/HLS/segments.py @@ -8,7 +8,7 @@ import logging import binascii import threading from queue import PriorityQueue -from urllib.parse import urljoin, urlparse, urlunparse +from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor @@ -78,9 +78,6 @@ class M3U8_Segments: self.queue = PriorityQueue() self.stop_event = threading.Event() - # Server ip - self.fake_proxy = False - def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: """ Retrieves the encryption key from the M3U8 playlist. @@ -111,27 +108,8 @@ class M3U8_Segments: hex_content = binascii.hexlify(response.content).decode('utf-8') byte_content = bytes.fromhex(hex_content) - logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})") return byte_content - def __gen_proxy__(self, url: str, url_index: int) -> str: - """ - Change the IP address of the provided URL based on the given index. - - Args: - - url (str): The original URL that needs its IP address replaced. - - url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP. - - Returns: - str: The modified URL with the new IP address. - """ - new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)] - - # Parse the original URL and replace the hostname with the new IP address - parsed_url = urlparse(url)._replace(netloc=new_ip_address) - - return urlunparse(parsed_url) - def parse_data(self, m3u8_content: str) -> None: """ Parses the M3U8 content to extract segment information. @@ -185,12 +163,6 @@ class M3U8_Segments: if len(self.valid_proxy) == 0: sys.exit(0) - # Server ip - if self.fake_proxy: - for i in range(len(self.segments)): - segment_url = self.segments[i] - self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url)) - def get_info(self) -> None: """ Makes a request to the index M3U8 file to get information about segments. @@ -214,70 +186,78 @@ class M3U8_Segments: else: # Parser data of content of index pass in input to class - self.parse_data(self.url) + self.parse_data(self.url) - def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: + def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm, retries: int = 3, backoff_factor: float = 1.5) -> None: """ - Downloads a TS segment and adds it to the segment queue. + Downloads a TS segment and adds it to the segment queue with retry logic. Parameters: - ts_url (str): The URL of the TS segment. - index (int): The index of the segment. - progress_bar (tqdm): Progress counter for tracking download progress. + - retries (int): The number of times to retry on failure (default is 3). + - backoff_factor (float): The backoff factor for exponential backoff (default is 1.5 seconds). """ need_verify = REQUEST_VERIFY - # Set to false for only fake proxy that use real ip of server - if self.fake_proxy: - need_verify = False + for attempt in range(retries): + try: + start_time = time.time() - try: - start_time = time.time() + # Make request to get content + if THERE_IS_PROXY_LIST: - # Make request to get content - if THERE_IS_PROXY_LIST: + # Get proxy from list + proxy = self.valid_proxy[index % len(self.valid_proxy)] + logging.info(f"Use proxy: {proxy}") - # Get proxy from list - proxy = self.valid_proxy[index % len(self.valid_proxy)] - logging.info(f"Use proxy: {proxy}") + with httpx.Client(proxies=proxy, verify=need_verify) as client: + if 'key_base_url' in self.__dict__: + response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) + else: + response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True) - with httpx.Client(proxies=proxy, verify=need_verify) as client: - if 'key_base_url' in self.__dict__: - response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) - else: - response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True) + else: + with httpx.Client(verify=need_verify) as client_2: + if 'key_base_url' in self.__dict__: + response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) + else: + response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True) - else: - with httpx.Client(verify=need_verify) as client_2: - if 'key_base_url' in self.__dict__: - response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) - else: - response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True) + # Get response content + response.raise_for_status() # Raise exception for HTTP errors + duration = time.time() - start_time + segment_content = response.content - # Get response content - response.raise_for_status() - duration = time.time() - start_time - segment_content = response.content + # Update bar + response_size = int(response.headers.get('Content-Length', 0)) or len(segment_content) - # Update bar - response_size = int(response.headers.get('Content-Length', 0)) + # Update progress bar with custom Class + self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar) - if response_size == 0: - response_size = int(len(response.content)) + # Decrypt the segment content if decryption is needed + if self.decryption is not None: + segment_content = self.decryption.decrypt(segment_content) - # Update progress bar with custom Class - self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar) + # Add the segment to the queue + self.queue.put((index, segment_content)) + progress_bar.update(1) - # Decrypt the segment content if decryption is needed - if self.decryption is not None: - segment_content = self.decryption.decrypt(segment_content) + # Break out of the loop on success + return - # Add the segment to the queue - self.queue.put((index, segment_content)) - progress_bar.update(1) - - except Exception as e: - console.print(f"\nFailed to download: '{ts_url}' with error: {e}") + except (httpx.RequestError, httpx.HTTPStatusError) as e: + console.print(f"\nAttempt {attempt + 1} failed for '{ts_url}' with error: {e}") + + if attempt + 1 == retries: + console.print(f"\nFailed after {retries} attempts. Skipping '{ts_url}'") + break + + # Exponential backoff before retrying + sleep_time = backoff_factor * (2 ** attempt) + console.print(f"Retrying in {sleep_time} seconds...") + time.sleep(sleep_time) def write_segments_to_file(self): """ @@ -393,7 +373,6 @@ class M3U8_Segments: delay = TQDM_DELAY_WORKER # Start all workers - logging.info(f"Worker to use: {max_workers}") with ThreadPoolExecutor(max_workers=max_workers) as executor: for index, segment_url in enumerate(self.segments): time.sleep(delay) diff --git a/Src/Lib/Downloader/MP4/downloader.py b/Src/Lib/Downloader/MP4/downloader.py index 74b0100..0070abe 100644 --- a/Src/Lib/Downloader/MP4/downloader.py +++ b/Src/Lib/Downloader/MP4/downloader.py @@ -29,7 +29,7 @@ REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout') -def MP4_downloader(url: str, path: str, referer: str = None): +def MP4_downloader(url: str, path: str, referer: str = None, headers_: str = None): """ Downloads an MP4 video from a given URL using the specified referer header. @@ -40,55 +40,63 @@ def MP4_downloader(url: str, path: str, referer: str = None): - referer (str): The referer header value to include in the HTTP request headers. """ + headers = None + if "http" not in str(url).lower().strip() or "https" not in str(url).lower().strip(): logging.error(f"Invalid url: {url}") sys.exit(0) - # Make request to get content of video - logging.info(f"Make request to fetch mp4 from: {url}") - if referer != None: headers = {'Referer': referer, 'user-agent': get_headers()} - else: + if headers == None: headers = {'user-agent': get_headers()} + else: + headers = headers_ + # Make request to get content of video with httpx.Client(verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT) as client: with client.stream("GET", url, headers=headers, timeout=10) as response: total = int(response.headers.get('content-length', 0)) - # Create bar format - if TQDM_USE_LARGE_BAR: - bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): " - f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ " - f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] " - f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| " - f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]") + if total != 0: + + # Create bar format + if TQDM_USE_LARGE_BAR: + bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): " + f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ " + f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] " + f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| " + f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]") + else: + bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% " + f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]") + + # Create progress bar + progress_bar = tqdm( + total=total, + ascii='░▒█', + bar_format=bar_format, + unit_scale=True, + unit_divisor=1024, + mininterval=0.05 + ) + + # Download file + with open(path, 'wb') as file, progress_bar as bar: + for chunk in response.iter_bytes(chunk_size=1024): + if chunk: + size = file.write(chunk) + bar.update(size) + else: - bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% " - f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]") + console.print("[red]Cant find any stream.") - # Create progress bar - progress_bar = tqdm( - total=total, - ascii='░▒█', - bar_format=bar_format, - unit_scale=True, - unit_divisor=1024, - mininterval=0.05 - ) - - # Download file - with open(path, 'wb') as file, progress_bar as bar: - for chunk in response.iter_bytes(chunk_size=1024): - if chunk: - size = file.write(chunk) - bar.update(size) - - # Get summary - console.print(Panel( - f"[bold green]Download completed![/bold green]\n" - f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n" - f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]", - title=f"{os.path.basename(path.replace('.mp4', ''))}", - border_style="green" - )) \ No newline at end of file + # Get summary + if total != 0: + console.print(Panel( + f"[bold green]Download completed![/bold green]\n" + f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n" + f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]", + title=f"{os.path.basename(path.replace('.mp4', ''))}", + border_style="green" + )) diff --git a/Src/Lib/FFmpeg/command.py b/Src/Lib/FFmpeg/command.py index 6cb0cb6..aca810a 100644 --- a/Src/Lib/FFmpeg/command.py +++ b/Src/Lib/FFmpeg/command.py @@ -55,7 +55,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): # Enabled the use of gpu if USE_GPU: - ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda']) + ffmpeg_cmd.extend(['-hwaccel', 'cuda']) # Add mpegts to force to detect input file as ts file if need_to_force_to_ts(video_path): @@ -67,7 +67,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): ffmpeg_cmd.extend(['-i', video_path]) # Add output Parameters - if USE_CODEC: + if USE_CODEC and codec != None: if USE_VCODEC: if codec.video_codec_name: if not USE_GPU: @@ -76,6 +76,10 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) else: console.log("[red]Cant find vcodec for 'join_audios'") + else: + if USE_GPU: + ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) + if USE_ACODEC: if codec.audio_codec_name: @@ -98,7 +102,6 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): # Overwrite ffmpeg_cmd += [out_path, "-y"] - logging.info(f"FFmpeg command: {ffmpeg_cmd}") # Run join if DEBUG_MODE: @@ -143,7 +146,7 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s # Enabled the use of gpu if USE_GPU: - ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda']) + ffmpeg_cmd.extend(['-hwaccel', 'cuda']) # Insert input video path ffmpeg_cmd.extend(['-i', video_path]) @@ -173,6 +176,9 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) else: console.log("[red]Cant find vcodec for 'join_audios'") + else: + if USE_GPU: + ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) if USE_ACODEC: if codec.audio_codec_name: @@ -195,12 +201,11 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s # Use shortest input path for video and audios if not video_audio_same_duration: - console.log("[red]Use shortest input.") + logging.info("[red]Use shortest input.") ffmpeg_cmd.extend(['-shortest', '-strict', 'experimental']) # Overwrite ffmpeg_cmd += [out_path, "-y"] - logging.info(f"FFmpeg command: {ffmpeg_cmd}") # Run join if DEBUG_MODE: diff --git a/Src/Lib/M3U8/parser.py b/Src/Lib/M3U8/parser.py index aba31f7..791fa01 100644 --- a/Src/Lib/M3U8/parser.py +++ b/Src/Lib/M3U8/parser.py @@ -164,6 +164,13 @@ class M3U8_Codec: else: logging.warning("No bandwidth provided. Bitrates cannot be calculated.") + def __str__(self): + return (f"M3U8_Codec(bandwidth={self.bandwidth}, " + f"codecs='{self.codecs}', " + f"audio_codec='{self.audio_codec}', " + f"video_codec='{self.video_codec}', " + f"audio_codec_name='{self.audio_codec_name}', " + f"video_codec_name='{self.video_codec_name}')") class M3U8_Video: diff --git a/Src/Util/os.py b/Src/Util/os.py index 42ac26b..bcdf57f 100644 --- a/Src/Util/os.py +++ b/Src/Util/os.py @@ -164,7 +164,6 @@ def check_file_existence(file_path): return True else: - logging.warning(f"The file '{file_path}' does not exist.") return False except Exception as e: diff --git a/Test/t_down_mp4.py b/Test/t_down_mp4.py index 43e0a61..5e9652f 100644 --- a/Test/t_down_mp4.py +++ b/Test/t_down_mp4.py @@ -17,5 +17,5 @@ from Src.Lib.Downloader import MP4_downloader # Test MP4_downloader( "", - "EP_1.mp4", + "EP_1.mp4" ) diff --git a/config.json b/config.json index cdeb94a..e77a4fb 100644 --- a/config.json +++ b/config.json @@ -42,8 +42,7 @@ "eng", "spa" ], - "cleanup_tmp_folder": true, - "create_report": false + "cleanup_tmp_folder": true }, "M3U8_CONVERSION": { "use_codec": false, @@ -58,23 +57,23 @@ }, "SITE": { "streamingcommunity": { - "video_workers": 4, - "audio_workers": 4, + "video_workers": 6, + "audio_workers": 6, "domain": "computer" }, "altadefinizione": { - "video_workers": -1, - "audio_workers": -1, + "video_workers": 12, + "audio_workers": 12, "domain": "my" }, "guardaserie": { - "video_workers": -1, - "audio_workers": -1, + "video_workers": 12, + "audio_workers": 12, "domain": "dev" }, "mostraguarda": { - "video_workers": -1, - "audio_workers": -1, + "video_workers": 12, + "audio_workers": 12, "domain": "stream" }, "ddlstreamitaly": { @@ -88,12 +87,6 @@ "animeunity": { "domain": "to" }, - "watch_lonelil": { - "domain": "ru" - }, - "uhdmovies": { - "domain": "mov" - }, "bitsearch": { "domain": "to" },