From f5ad9b71875880cd92b29d87f1c1a43aa48ce869 Mon Sep 17 00:00:00 2001 From: Lovi <62809003+Lovi-0@users.noreply.github.com> Date: Sat, 7 Dec 2024 16:25:42 +0100 Subject: [PATCH] Improve ffmpeg check and semgents info. --- .../Lib/Downloader/HLS/downloader.py | 4 +- .../Lib/Downloader/HLS/segments.py | 38 +++++++++++++++---- StreamingCommunity/Lib/FFmpeg/command.py | 18 +++++---- StreamingCommunity/Lib/FFmpeg/util.py | 32 +++++++++------- StreamingCommunity/Util/os.py | 34 ++++++++++------- Test/Download/HLS.py | 2 +- Test/Download/MP4.py | 4 +- config.json | 2 +- 8 files changed, 87 insertions(+), 47 deletions(-) diff --git a/StreamingCommunity/Lib/Downloader/HLS/downloader.py b/StreamingCommunity/Lib/Downloader/HLS/downloader.py index 54c625b..b666057 100644 --- a/StreamingCommunity/Lib/Downloader/HLS/downloader.py +++ b/StreamingCommunity/Lib/Downloader/HLS/downloader.py @@ -397,7 +397,7 @@ class ContentDownloader: self.expected_real_time = video_m3u8.expected_real_time # Download the video streams and print status - video_m3u8.download_streams(f"{Colors.MAGENTA}video") + video_m3u8.download_streams(f"{Colors.MAGENTA}video", "video") # Print duration information of the downloaded video #print_duration_table(downloaded_video[0].get('path')) @@ -427,7 +427,7 @@ class ContentDownloader: audio_m3u8.get_info() # Download the audio segments and print status - audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}") + audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}", "audio") # Print duration information of the downloaded audio #print_duration_table(obj_audio.get('path')) diff --git a/StreamingCommunity/Lib/Downloader/HLS/segments.py b/StreamingCommunity/Lib/Downloader/HLS/segments.py index a44e9d3..9f31644 100644 --- a/StreamingCommunity/Lib/Downloader/HLS/segments.py +++ b/StreamingCommunity/Lib/Downloader/HLS/segments.py @@ -4,10 +4,11 @@ import os import sys import time import queue +import signal import logging import binascii import threading -import signal + from queue import PriorityQueue from urllib.parse import urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed @@ -34,6 +35,7 @@ from ...M3U8 import ( M3U8_Parser, M3U8_UrlFix ) +from ...FFmpeg.util import print_duration_table from .proxyes import main_test_proxy # Config @@ -93,6 +95,10 @@ class M3U8_Segments: self.interrupt_flag = threading.Event() self.download_interrupted = False + # OTHER INFO + self.info_maxRetry = 0 + self.info_nRetry = 0 + def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: """ Retrieves the encryption key from the M3U8 playlist. @@ -317,11 +323,16 @@ class M3U8_Segments: except Exception as e: logging.info(f"Attempt {attempt + 1} failed for segment {index} - '{ts_url}': {e}") + # Update stat variable class + if attempt > self.info_maxRetry: + self.info_maxRetry = ( attempt + 1 ) + self.info_nRetry += 1 + if attempt + 1 == REQUEST_MAX_RETRY: console.log(f"[red]Final retry failed for segment: {index}") self.queue.put((index, None)) # Marker for failed segment progress_bar.update(1) - break + #break sleep_time = backoff_factor * (2 ** attempt) logging.info(f"Retrying segment {index} in {sleep_time} seconds...") @@ -382,12 +393,13 @@ class M3U8_Segments: except Exception as e: logging.error(f"Error writing segment {index}: {str(e)}") - def download_streams(self, add_desc): + def download_streams(self, description: str, type: str): """ Downloads all TS segments in parallel and writes them to a file. Parameters: - - add_desc (str): Additional description for the progress bar. + - description: Description to insert on tqdm bar + - type (str): Type of download: 'video' or 'audio' """ self.setup_interrupt_handler() @@ -414,15 +426,18 @@ class M3U8_Segments: AUDIO_WORKERS = DEFAULT_AUDIO_WORKERS # Differnt workers for audio and video - if "video" in str(add_desc): + if "video" == str(type): TQDM_MAX_WORKER = VIDEO_WORKERS - if "audio" in str(add_desc): + + if "audio" == str(type): TQDM_MAX_WORKER = AUDIO_WORKERS + console.print(f"[cyan]Video workers[white]: [green]{VIDEO_WORKERS} [white]| [cyan]Audio workers[white]: [green]{AUDIO_WORKERS}") + # Custom bar for mobile and pc if TQDM_USE_LARGE_BAR: bar_format = ( - f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{add_desc}{Colors.WHITE}): " + f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{description}{Colors.WHITE}): " f"{Colors.RED}{{percentage:.2f}}% " f"{Colors.MAGENTA}{{bar}} " f"{Colors.WHITE}[ {Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] " @@ -535,4 +550,11 @@ class M3U8_Segments: if file_size == 0: raise Exception("Output file is empty") - logging.info(f"Download completed. File size: {file_size} bytes") + console.print(f"[cyan]Max retry per URL[white]: [green]{self.info_maxRetry}[green] [white]| [cyan]Total retry done[white]: [green]{self.info_nRetry}[green] [white]| [cyan]Duration: {print_duration_table(self.tmp_file_path, None, True)} \n") + + if self.info_nRetry >= len(self.segments) * (1/3.33): + console.print( + "[yellow]⚠ Warning:[/yellow] Too many retries detected! " + "Consider reducing the number of [cyan]workers[/cyan] in the [magenta]config.json[/magenta] file. " + "This will impact [bold]performance[/bold]." + ) \ No newline at end of file diff --git a/StreamingCommunity/Lib/FFmpeg/command.py b/StreamingCommunity/Lib/FFmpeg/command.py index bd426cd..916bb0d 100644 --- a/StreamingCommunity/Lib/FFmpeg/command.py +++ b/StreamingCommunity/Lib/FFmpeg/command.py @@ -9,8 +9,11 @@ from typing import List, Dict # Internal utilities from StreamingCommunity.Util._jsonConfig import config_manager -from StreamingCommunity.Util.os import os_manager, suppress_output +from StreamingCommunity.Util.os import os_manager, os_summary, suppress_output from StreamingCommunity.Util.console import console + + +# Logic class from .util import need_to_force_to_ts, check_duration_v_a from .capture import capture_ffmpeg_real_time from ..M3U8 import M3U8_Codec @@ -29,6 +32,7 @@ FFMPEG_DEFAULT_PRESET = config_manager.get("M3U8_CONVERSION", "default_preset") # Variable TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar') +FFMPEG_PATH = os_summary.ffmpeg_path def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): @@ -49,8 +53,8 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None): logging.error("Missing input video for ffmpeg conversion.") sys.exit(0) - # Start command - ffmpeg_cmd = ['ffmpeg'] + # Start command with locate ffmpeg + ffmpeg_cmd = [FFMPEG_PATH] # Enabled the use of gpu if USE_GPU: @@ -140,8 +144,8 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s video_audio_same_duration = check_duration_v_a(video_path, audio_tracks[0].get('path')) - # Start command - ffmpeg_cmd = ['ffmpeg'] + # Start command with locate ffmpeg + ffmpeg_cmd = [FFMPEG_PATH] # Enabled the use of gpu if USE_GPU: @@ -242,8 +246,8 @@ def join_subtitle(video_path: str, subtitles_list: List[Dict[str, str]], out_pat logging.error("Missing input video for ffmpeg conversion.") sys.exit(0) - # Start command - ffmpeg_cmd = ["ffmpeg", "-i", video_path] + # Start command with locate ffmpeg + ffmpeg_cmd = [FFMPEG_PATH, "-i", video_path] # Add subtitle input files first for subtitle in subtitles_list: diff --git a/StreamingCommunity/Lib/FFmpeg/util.py b/StreamingCommunity/Lib/FFmpeg/util.py index 52db553..354f02d 100644 --- a/StreamingCommunity/Lib/FFmpeg/util.py +++ b/StreamingCommunity/Lib/FFmpeg/util.py @@ -10,6 +10,12 @@ from typing import Tuple # Internal utilities from StreamingCommunity.Util.console import console +from StreamingCommunity.Util.os import os_summary + + +# Variable +FFPROB_PATH = os_summary.ffprobe_path + def has_audio_stream(video_path: str) -> bool: @@ -23,7 +29,7 @@ def has_audio_stream(video_path: str) -> bool: has_audio (bool): True if the input video has an audio stream, False otherwise. """ try: - ffprobe_cmd = ['ffprobe', '-v', 'error', '-print_format', 'json', '-select_streams', 'a', '-show_streams', video_path] + ffprobe_cmd = [FFPROB_PATH, '-v', 'error', '-print_format', 'json', '-select_streams', 'a', '-show_streams', video_path] logging.info(f"FFmpeg command: {ffprobe_cmd}") with subprocess.Popen(ffprobe_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc: @@ -47,12 +53,11 @@ def get_video_duration(file_path: str) -> float: - file_path (str): The path to the video file. Returns: - (float): The duration of the video in seconds if successful, - None if there's an error. + (float): The duration of the video in seconds if successful, None if there's an error. """ try: - ffprobe_cmd = ['ffprobe', '-v', 'error', '-show_format', '-print_format', 'json', file_path] + ffprobe_cmd = [FFPROB_PATH, '-v', 'error', '-show_format', '-print_format', 'json', file_path] logging.info(f"FFmpeg command: {ffprobe_cmd}") # Use a with statement to ensure the subprocess is cleaned up properly @@ -77,15 +82,16 @@ def get_video_duration(file_path: str) -> float: logging.error(f"Error get video duration: {e}") sys.exit(0) + def get_video_duration_s(filename): """ Get the duration of a video file using ffprobe. Parameters: - - filename (str): Path to the video file (e.g., 'sim.mp4') + - filename (str): Path to the video file (e.g., 'sim.mp4') Returns: - - duration (float): Duration of the video in seconds, or None if an error occurs. + - duration (float): Duration of the video in seconds, or None if an error occurs. """ ffprobe_cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename] @@ -160,14 +166,14 @@ def get_ffprobe_info(file_path): Get format and codec information for a media file using ffprobe. Parameters: - file_path (str): Path to the media file. + - file_path (str): Path to the media file. Returns: dict: A dictionary containing the format name and a list of codec names. """ try: result = subprocess.run( - ['ffprobe', '-v', 'error', '-show_format', '-show_streams', '-print_format', 'json', file_path], + [FFPROB_PATH, '-v', 'error', '-show_format', '-show_streams', '-print_format', 'json', file_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True ) output = result.stdout @@ -195,7 +201,7 @@ def is_png_format_or_codec(file_info): Check if the format is 'png_pipe' or if any codec is 'png'. Parameters: - file_info (dict): The dictionary containing file information. + - file_info (dict): The dictionary containing file information. Returns: bool: True if the format is 'png_pipe' or any codec is 'png', otherwise False. @@ -210,7 +216,7 @@ def need_to_force_to_ts(file_path): Get if a file to TS format if it is in PNG format or contains a PNG codec. Parameters: - file_path (str): Path to the input media file. + - file_path (str): Path to the input media file. """ logging.info(f"Processing file: {file_path}") file_info = get_ffprobe_info(file_path) @@ -225,11 +231,11 @@ def check_duration_v_a(video_path, audio_path): Check if the duration of the video and audio matches. Parameters: - - video_path (str): Path to the video file. - - audio_path (str): Path to the audio file. + - video_path (str): Path to the video file. + - audio_path (str): Path to the audio file. Returns: - - bool: True if the duration of the video and audio matches, False otherwise. + - bool: True if the duration of the video and audio matches, False otherwise. """ # Ottieni la durata del video diff --git a/StreamingCommunity/Util/os.py b/StreamingCommunity/Util/os.py index 311e6f3..0077ed8 100644 --- a/StreamingCommunity/Util/os.py +++ b/StreamingCommunity/Util/os.py @@ -306,7 +306,12 @@ class InternManager(): print() -class OsSummary(): +class OsSummary: + + def __init__(self): + ffmpeg_path, ffprobe_path = check_ffmpeg() + self.ffmpeg_path = ffmpeg_path + self.ffprobe_path = ffprobe_path def get_executable_version(self, command: list): """ @@ -441,27 +446,29 @@ class OsSummary(): console.print(f"[cyan]Python[white]: [bold red]{python_version} ({python_implementation} {arch}) - {os_info} ({glibc_version})[/bold red]") logging.info(f"Python: {python_version} ({python_implementation} {arch}) - {os_info} ({glibc_version})") - # ffmpeg and ffprobe versions - ffmpeg_path, ffprobe_path = check_ffmpeg() - + # Usa il comando 'where' su Windows if platform.system() == "Windows": - # Usa il comando 'where' su Windows command = 'where' + + # Usa il comando 'which' su Unix/Linux else: - # Usa il comando 'which' su Unix/Linux command = 'which' # Locate ffmpeg and ffprobe - if "binary" not in ffmpeg_path: - ffmpeg_path = self.check_ffmpeg_location([command, 'ffmpeg']) + if self.ffmpeg_path != None and "binary" not in self.ffmpeg_path: + self.ffmpeg_path = self.check_ffmpeg_location([command, 'ffmpeg']) - if "binary" not in ffprobe_path: - ffprobe_path = self.check_ffmpeg_location([command, 'ffprobe']) + if self.ffprobe_path != None and "binary" not in self.ffprobe_path: + self.ffprobe_path = self.check_ffmpeg_location([command, 'ffprobe']) - ffmpeg_version = self.get_executable_version([ffprobe_path, '-version']) - ffprobe_version = self.get_executable_version([ffprobe_path, '-version']) + if self.ffmpeg_path is None or self.ffprobe_path is None: + console.log("[red]Cant locate ffmpeg or ffprobe") + sys.exit(0) - console.print(f"[cyan]Path[white]: [red]ffmpeg [bold yellow]'{ffmpeg_path}'[/bold yellow][white], [red]ffprobe '[bold yellow]{ffprobe_path}'[/bold yellow]") + ffmpeg_version = self.get_executable_version([self.ffprobe_path, '-version']) + ffprobe_version = self.get_executable_version([self.ffprobe_path, '-version']) + + console.print(f"[cyan]Path[white]: [red]ffmpeg [bold yellow]'{self.ffmpeg_path}'[/bold yellow][white], [red]ffprobe '[bold yellow]{self.ffprobe_path}'[/bold yellow]") console.print(f"[cyan]Exe versions[white]: [bold red]ffmpeg {ffmpeg_version}, ffprobe {ffprobe_version}[/bold red]") # Check if requirements.txt exists, if not on pyinstaller @@ -500,6 +507,7 @@ os_manager = OsManager() internet_manager = InternManager() os_summary = OsSummary() + @contextlib.contextmanager def suppress_output(): with contextlib.redirect_stdout(io.StringIO()): diff --git a/Test/Download/HLS.py b/Test/Download/HLS.py index 5601d95..17dc60b 100644 --- a/Test/Download/HLS.py +++ b/Test/Download/HLS.py @@ -18,6 +18,6 @@ from StreamingCommunity.Lib.Downloader import HLS_Downloader start_message() logger = Logger() print("Return: ", HLS_Downloader( - output_filename="", + output_filename="test.mp4", m3u8_index="" ).start()) \ No newline at end of file diff --git a/Test/Download/MP4.py b/Test/Download/MP4.py index ecb125b..ac5bce4 100644 --- a/Test/Download/MP4.py +++ b/Test/Download/MP4.py @@ -18,6 +18,6 @@ from StreamingCommunity.Lib.Downloader import MP4_downloader start_message() logger = Logger() print("Return: ", MP4_downloader( - "", - ".\Video\undefined.mp4" + url="", + path=r".\Video\undefined.mp4" )) diff --git a/config.json b/config.json index e6f1c75..287471a 100644 --- a/config.json +++ b/config.json @@ -20,7 +20,7 @@ }, "REQUESTS": { "timeout": 20, - "max_retry": 5, + "max_retry": 8, "verify_ssl": true, "proxy_start_min": 0.1, "proxy_start_max": 0.5,