diff --git a/Src/Lib/FFmpeg/capture.py b/Src/Lib/FFmpeg/capture.py index b347b94..fe4ddbd 100644 --- a/Src/Lib/FFmpeg/capture.py +++ b/Src/Lib/FFmpeg/capture.py @@ -60,8 +60,8 @@ def capture_output(process: subprocess.Popen, description: str) -> None: time_now = datetime.now().strftime('%H:%M:%S') # Construct the progress string with formatted output information - progress_string = (f"[blue][{time_now}][purple] FFmpeg [white][{description}]: " - f"[white]([green]'speed': [yellow]{data.get('speed', 'N/A')}[white], " + progress_string = (f"[blue][{time_now}][purple] FFmpeg [white][{description}[white]]: " + f"([green]'speed': [yellow]{data.get('speed', 'N/A')}[white], " f"[green]'size': [yellow]{format_size(byte_size)}[white])") max_length = max(max_length, len(progress_string)) diff --git a/Src/Lib/FFmpeg/command.py b/Src/Lib/FFmpeg/command.py index 5f11d36..af2f57b 100644 --- a/Src/Lib/FFmpeg/command.py +++ b/Src/Lib/FFmpeg/command.py @@ -5,7 +5,6 @@ import sys import time import logging import shutil -import threading import subprocess from typing import List, Dict @@ -18,13 +17,13 @@ except: pass # Internal utilities from Src.Util._jsonConfig import config_manager -from Src.Util.os import check_file_existence +from Src.Util.os import check_file_existence, suppress_output from Src.Util.console import console from .util import has_audio_stream, need_to_force_to_ts, check_ffmpeg_input from .capture import capture_ffmpeg_real_time -# Variable +# Config DEBUG_MODE = config_manager.get_bool("DEFAULT", "debug") DEBUG_FFMPEG = "debug" if DEBUG_MODE else "error" USE_CODECS = config_manager.get_bool("M3U8_CONVERSION", "use_codec") @@ -33,6 +32,10 @@ FFMPEG_DEFAULT_PRESET = config_manager.get("M3U8_CONVERSION", "default_preset") CHECK_OUTPUT_CONVERSION = config_manager.get_bool("M3U8_CONVERSION", "check_output_after_ffmpeg") +# Variable +TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar') + + # --> v 1.0 (deprecated) def __concatenate_and_save(file_list_path: str, output_filename: str, v_codec: str = None, a_codec: str = None, bandwidth: int = None, prefix: str = "segments", output_directory: str = None): @@ -314,15 +317,25 @@ def join_video(video_path: str, out_path: str, vcodec: str = None, acodec: str = ffmpeg_cmd += [out_path, "-y"] logging.info(f"FFmpeg command: {ffmpeg_cmd}") + # Run join if DEBUG_MODE: subprocess.run(ffmpeg_cmd, check=True) else: - capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video") - print() + + if TQDM_USE_LARGE_BAR: + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video") + print() + + else: + console.log(f"[purple]FFmpeg [white][[cyan]Join video[white]] ...") + with suppress_output(): + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video") + print() - # Check file + + # Check file output if CHECK_OUTPUT_CONVERSION: console.log("[red]Check output ffmpeg") time.sleep(0.5) @@ -380,15 +393,24 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s ffmpeg_cmd += [out_path, "-y"] logging.info(f"FFmpeg command: {ffmpeg_cmd}") + # Run join if DEBUG_MODE: subprocess.run(ffmpeg_cmd, check=True) else: - capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio") - print() + + if TQDM_USE_LARGE_BAR: + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio") + print() + + else: + console.log(f"[purple]FFmpeg [white][[cyan]Join audio[white]] ...") + with suppress_output(): + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio") + print() - # Check file + # Check file output if CHECK_OUTPUT_CONVERSION: console.log("[red]Check output ffmpeg") time.sleep(0.5) @@ -443,15 +465,24 @@ def join_subtitle(video_path: str, subtitles_list: List[Dict[str, str]], out_pat ffmpeg_cmd += [out_path, "-y"] logging.info(f"FFmpeg command: {ffmpeg_cmd}") + # Run join if DEBUG_MODE: subprocess.run(ffmpeg_cmd, check=True) else: - capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle") - print() - - # Check file + if TQDM_USE_LARGE_BAR: + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle") + print() + + else: + console.log(f"[purple]FFmpeg [white][[cyan]Join subtitle[white]] ...") + with suppress_output(): + capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle") + print() + + + # Check file output if CHECK_OUTPUT_CONVERSION: console.log("[red]Check output ffmpeg") time.sleep(0.5) diff --git a/Src/Lib/Hls/downloader.py b/Src/Lib/Hls/downloader.py index c1349b1..a3fea2c 100644 --- a/Src/Lib/Hls/downloader.py +++ b/Src/Lib/Hls/downloader.py @@ -50,7 +50,8 @@ DOWNLOAD_SPECIFIC_AUDIO = config_manager.get_list('M3U8_DOWNLOAD', 'specific_lis DOWNLOAD_SPECIFIC_SUBTITLE = config_manager.get_list('M3U8_DOWNLOAD', 'specific_list_subtitles') DOWNLOAD_VIDEO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_video') DOWNLOAD_AUDIO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_audio') -DOWNLOAD_SUB = config_manager.get_bool('M3U8_DOWNLOAD', 'download_sub') +MERGE_AUDIO = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_audio') +DOWNLOAD_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'download_sub') MERGE_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_subs') REMOVE_SEGMENTS_FOLDER = config_manager.get_bool('M3U8_DOWNLOAD', 'cleanup_tmp_folder') FILTER_CUSTOM_REOLUTION = config_manager.get_int('M3U8_PARSER', 'force_resolution') @@ -472,7 +473,7 @@ class Downloader(): os.rename(out_path, self.output_filename) # Print size of the file - console.print(Panel(f"[bold green]Download completed![/bold green]\nFile size: [bold]{format_size(os.path.getsize(self.output_filename))}[/bold]", title=f"{os.path.basename(self.output_filename.replace('.mp4', ''))}", border_style="green")) + console.print(Panel(f"[bold green]Download completed![/bold green]\nFile size: [bold red]{format_size(os.path.getsize(self.output_filename))}[/bold red]", title=f"{os.path.basename(self.output_filename.replace('.mp4', ''))}", border_style="green")) # Delete all files except the output file delete_files_except_one(self.base_path, os.path.basename(self.output_filename)) @@ -519,14 +520,16 @@ class Downloader(): # Collect information about the playlist self.__manage_playlist__(m3u8_playlist_text) + # Start all download ... if DOWNLOAD_VIDEO: self.__donwload_video__(server_ip) if DOWNLOAD_AUDIO: self.__donwload_audio__(server_ip) - if DOWNLOAD_SUB: + if DOWNLOAD_SUBTITLE: self.__download_subtitle__() + # Check file to convert converted_out_path = None there_is_video: bool = (len(self.downloaded_video) > 0) @@ -534,38 +537,72 @@ class Downloader(): there_is_subtitle: bool = (len(self.downloaded_subtitle) > 0) console.log(f"[cyan]Conversion [white]=> ([green]Audio: [yellow]{there_is_audio}[white], [green]Subtitle: [yellow]{there_is_subtitle}[white])") + # Join audio and video if there_is_audio: - converted_out_path = self.__join_video_audio__() + if MERGE_AUDIO: + converted_out_path = self.__join_video_audio__() + + else: + for obj_audio in self.downloaded_audio: + language = obj_audio.get('language') + path = obj_audio.get('path') + + # Set the new path for regular audio + new_path = self.output_filename.replace(".mp4", f"_{language}.mp4") + + try: + # Rename the audio file to the new path + os.rename(path, new_path) + logging.info(f"Audio moved to {new_path}") + + except Exception as e: + logging.error(f"Failed to move audio {path} to {new_path}: {e}") + + + # Convert video + if there_is_video: + converted_out_path = self.__join_video__() + # Join only video ( audio is present in the same ts files ) else: if there_is_video: converted_out_path = self.__join_video__() + # Join subtitle if there_is_subtitle: if MERGE_SUBTITLE: if converted_out_path is not None: converted_out_path = self.__join_video_subtitles__(converted_out_path) + else: for obj_sub in self.downloaded_subtitle: language = obj_sub.get('language') path = obj_sub.get('path') forced = 'forced' in language + # Check if the language includes "forced" + forced = 'forced' in language + + # Remove "forced-" from the language if present and set the new path with "forced" if forced: language = language.replace("forced-", "") new_path = self.output_filename.replace(".mp4", f".{language}.forced.vtt") else: + + # Set the new path for regular languages new_path = self.output_filename.replace(".mp4", f".{language}.vtt") try: + # Rename the subtitle file to the new path os.rename(path, new_path) logging.info(f"Subtitle moved to {new_path}") + except Exception as e: logging.error(f"Failed to move subtitle {path} to {new_path}: {e}") - + # Clean all tmp file self.__clean__(converted_out_path) diff --git a/Src/Lib/Hls/segments.py b/Src/Lib/Hls/segments.py index 7c049b2..beeb831 100644 --- a/Src/Lib/Hls/segments.py +++ b/Src/Lib/Hls/segments.py @@ -64,7 +64,7 @@ class M3U8_Segments: self.ctrl_c_detected = False # Global variable to track Ctrl+C detection os.makedirs(self.tmp_folder, exist_ok=True) # Create the temporary folder if it does not exist - self.class_ts_estimator = M3U8_Ts_Estimator(TQDM_MAX_WORKER, 0) + self.class_ts_estimator = M3U8_Ts_Estimator(0) self.class_url_fixer = M3U8_UrlFix(url) self.fake_proxy = False @@ -304,7 +304,7 @@ class M3U8_Segments: progress_bar = tqdm( total=len(self.segments), unit='s', - ascii=' #', + ascii='░▒█', bar_format=bar_format, dynamic_ncols=True, ncols=80, diff --git a/Src/Lib/M3U8/estimator.py b/Src/Lib/M3U8/estimator.py index d46427a..e0e6596 100644 --- a/Src/Lib/M3U8/estimator.py +++ b/Src/Lib/M3U8/estimator.py @@ -1,7 +1,7 @@ # 20.02.24 +import threading import logging - from collections import deque @@ -21,7 +21,7 @@ TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar class M3U8_Ts_Estimator: - def __init__(self, workers: int, total_segments: int): + def __init__(self, total_segments: int): """ Initialize the TSFileSizeCalculator object. @@ -31,11 +31,11 @@ class M3U8_Ts_Estimator: """ self.ts_file_sizes = [] self.now_downloaded_size = 0 - self.average_over = 5 + self.average_over = 6 self.list_speeds = deque(maxlen=self.average_over) self.smoothed_speeds = [] - self.tqdm_workers = workers self.total_segments = total_segments + self.lock = threading.Lock() def add_ts_file(self, size: int, size_download: int, duration: float): """ @@ -50,30 +50,26 @@ class M3U8_Ts_Estimator: logging.error("Invalid input values: size=%d, size_download=%d, duration=%f", size, size_download, duration) return - self.ts_file_sizes.append(size) - self.now_downloaded_size += size_download - - # Only for the start - if len(self.smoothed_speeds) <= 3: - size_download = size_download / self.tqdm_workers - + # Calculate speed outside of the lock try: - # Calculate mbps - speed_mbps = (size_download * 8) / (duration * 1_000_000) * self.tqdm_workers - + speed_mbps = (size_download * 16) / (duration * 1_000_000) except ZeroDivisionError as e: logging.error("Division by zero error while calculating speed: %s", e) return - self.list_speeds.append(speed_mbps) + # Only update shared data within the lock + with self.lock: + self.ts_file_sizes.append(size) + self.now_downloaded_size += size_download + self.list_speeds.append(speed_mbps) - # Calculate moving average - smoothed_speed = sum(self.list_speeds) / len(self.list_speeds) - self.smoothed_speeds.append(smoothed_speed) + # Calculate moving average + smoothed_speed = sum(self.list_speeds) / len(self.list_speeds) + self.smoothed_speeds.append(smoothed_speed) - # Update smooth speeds - if len(self.smoothed_speeds) > self.average_over: - self.smoothed_speeds.pop(0) + # Update smooth speeds + if len(self.smoothed_speeds) > self.average_over: + self.smoothed_speeds.pop(0) def calculate_total_size(self) -> str: """ @@ -107,7 +103,7 @@ class M3U8_Ts_Estimator: Returns: float: The average speed in megabytes per second (MB/s). """ - return (sum(self.smoothed_speeds) / len(self.smoothed_speeds)) / 10 # MB/s + return ((sum(self.smoothed_speeds) / len(self.smoothed_speeds)) / 8 ) * 10 # MB/s def get_downloaded_size(self) -> str: """ diff --git a/Src/Lib/M3U8/parser.py b/Src/Lib/M3U8/parser.py index 4c4d476..2e3b647 100644 --- a/Src/Lib/M3U8/parser.py +++ b/Src/Lib/M3U8/parser.py @@ -52,17 +52,15 @@ class M3U8_Codec: Represents codec information for an M3U8 playlist. """ - def __init__(self, bandwidth, resolution, codecs): + def __init__(self, bandwidth, codecs): """ Initializes the M3U8Codec object with the provided parameters. Args: - bandwidth (int): Bandwidth of the codec. - - resolution (str): Resolution of the codec. - codecs (str): Codecs information in the format "avc1.xxxxxx,mp4a.xx". """ self.bandwidth = bandwidth - self.resolution = resolution self.codecs = codecs self.audio_codec = None self.video_codec = None @@ -76,7 +74,10 @@ class M3U8_Codec: """ # Split the codecs string by comma - codecs_list = self.codecs.split(',') + try: + codecs_list = self.codecs.split(',') + except Exception as e: + logging.error(f"Cant split codec list: {self.codecs} with error {e}") # Separate audio and video codecs for codec in codecs_list: @@ -448,14 +449,14 @@ class M3U8_Parser: try: for playlist in m3u8_obj.playlists: - there_is_codec = not M3U8_Parser.extract_resolution(playlist.uri) == (0,0) + there_is_codec = not playlist.stream_info.codecs is None + logging.info(f"There is coded: {there_is_codec}") if there_is_codec: self.codec = M3U8_Codec( playlist.stream_info.bandwidth, - None, playlist.stream_info.codecs - ) + ) # Direct access resolutions in m3u8 obj if playlist.stream_info.resolution is not None: diff --git a/Src/Util/message.py b/Src/Util/message.py index 8576feb..4a5706c 100644 --- a/Src/Util/message.py +++ b/Src/Util/message.py @@ -20,7 +20,7 @@ def start_message(): Display a start message. """ - msg = ''' + msg = r''' _____ _ _ _____ _ _ / ____| | (_) / ____| (_) | diff --git a/Src/Util/os.py b/Src/Util/os.py index 6f7260f..f2ab1de 100644 --- a/Src/Util/os.py +++ b/Src/Util/os.py @@ -1,6 +1,7 @@ # 24.01.24 import re +import io import os import sys import ssl @@ -14,6 +15,7 @@ import zipfile import platform import importlib import subprocess +import contextlib import importlib.metadata from typing import List @@ -28,6 +30,7 @@ from .console import console + # --> OS FILE ASCII special_chars_to_remove = ['!','@','#','$','%','^','&','*','(',')','[',']','{','}','<','|','`','~',"'",'"',';',':',',','?',"\\","/","\t"] @@ -103,6 +106,14 @@ def remove_special_characters(input_string): +# --> OS MANAGE OUTPUT +@contextlib.contextmanager +def suppress_output(): + with contextlib.redirect_stdout(io.StringIO()): + yield + + + # --> OS MANAGE FOLDER def create_folder(folder_name: str) -> None: """ diff --git a/config.json b/config.json index 22ed5f6..2603713 100644 --- a/config.json +++ b/config.json @@ -22,10 +22,11 @@ "tqdm_use_large_bar": true, "download_video": true, "download_audio": true, + "merge_audio": true, + "specific_list_audio": ["ita"], "download_sub": true, "merge_subs": true, - "specific_list_audio": ["ita"], - "specific_list_subtitles": ["eng"], + "specific_list_subtitles": ["eng", "spa"], "cleanup_tmp_folder": true, "create_report": false },