Fix small bar with join ffmpeg.

This commit is contained in:
Ghost 2024-05-30 17:00:01 +02:00
parent add0f2f01d
commit 7f6799d276
9 changed files with 131 additions and 54 deletions

View File

@ -60,8 +60,8 @@ def capture_output(process: subprocess.Popen, description: str) -> None:
time_now = datetime.now().strftime('%H:%M:%S') time_now = datetime.now().strftime('%H:%M:%S')
# Construct the progress string with formatted output information # Construct the progress string with formatted output information
progress_string = (f"[blue][{time_now}][purple] FFmpeg [white][{description}]: " progress_string = (f"[blue][{time_now}][purple] FFmpeg [white][{description}[white]]: "
f"[white]([green]'speed': [yellow]{data.get('speed', 'N/A')}[white], " f"([green]'speed': [yellow]{data.get('speed', 'N/A')}[white], "
f"[green]'size': [yellow]{format_size(byte_size)}[white])") f"[green]'size': [yellow]{format_size(byte_size)}[white])")
max_length = max(max_length, len(progress_string)) max_length = max(max_length, len(progress_string))

View File

@ -5,7 +5,6 @@ import sys
import time import time
import logging import logging
import shutil import shutil
import threading
import subprocess import subprocess
from typing import List, Dict from typing import List, Dict
@ -18,13 +17,13 @@ except: pass
# Internal utilities # Internal utilities
from Src.Util._jsonConfig import config_manager from Src.Util._jsonConfig import config_manager
from Src.Util.os import check_file_existence from Src.Util.os import check_file_existence, suppress_output
from Src.Util.console import console from Src.Util.console import console
from .util import has_audio_stream, need_to_force_to_ts, check_ffmpeg_input from .util import has_audio_stream, need_to_force_to_ts, check_ffmpeg_input
from .capture import capture_ffmpeg_real_time from .capture import capture_ffmpeg_real_time
# Variable # Config
DEBUG_MODE = config_manager.get_bool("DEFAULT", "debug") DEBUG_MODE = config_manager.get_bool("DEFAULT", "debug")
DEBUG_FFMPEG = "debug" if DEBUG_MODE else "error" DEBUG_FFMPEG = "debug" if DEBUG_MODE else "error"
USE_CODECS = config_manager.get_bool("M3U8_CONVERSION", "use_codec") USE_CODECS = config_manager.get_bool("M3U8_CONVERSION", "use_codec")
@ -33,6 +32,10 @@ FFMPEG_DEFAULT_PRESET = config_manager.get("M3U8_CONVERSION", "default_preset")
CHECK_OUTPUT_CONVERSION = config_manager.get_bool("M3U8_CONVERSION", "check_output_after_ffmpeg") CHECK_OUTPUT_CONVERSION = config_manager.get_bool("M3U8_CONVERSION", "check_output_after_ffmpeg")
# Variable
TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar')
# --> v 1.0 (deprecated) # --> v 1.0 (deprecated)
def __concatenate_and_save(file_list_path: str, output_filename: str, v_codec: str = None, a_codec: str = None, bandwidth: int = None, prefix: str = "segments", output_directory: str = None): def __concatenate_and_save(file_list_path: str, output_filename: str, v_codec: str = None, a_codec: str = None, bandwidth: int = None, prefix: str = "segments", output_directory: str = None):
@ -314,15 +317,25 @@ def join_video(video_path: str, out_path: str, vcodec: str = None, acodec: str =
ffmpeg_cmd += [out_path, "-y"] ffmpeg_cmd += [out_path, "-y"]
logging.info(f"FFmpeg command: {ffmpeg_cmd}") logging.info(f"FFmpeg command: {ffmpeg_cmd}")
# Run join # Run join
if DEBUG_MODE: if DEBUG_MODE:
subprocess.run(ffmpeg_cmd, check=True) subprocess.run(ffmpeg_cmd, check=True)
else: else:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video")
print() if TQDM_USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video")
print()
else:
console.log(f"[purple]FFmpeg [white][[cyan]Join video[white]] ...")
with suppress_output():
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video")
print()
# Check file
# Check file output
if CHECK_OUTPUT_CONVERSION: if CHECK_OUTPUT_CONVERSION:
console.log("[red]Check output ffmpeg") console.log("[red]Check output ffmpeg")
time.sleep(0.5) time.sleep(0.5)
@ -380,15 +393,24 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
ffmpeg_cmd += [out_path, "-y"] ffmpeg_cmd += [out_path, "-y"]
logging.info(f"FFmpeg command: {ffmpeg_cmd}") logging.info(f"FFmpeg command: {ffmpeg_cmd}")
# Run join # Run join
if DEBUG_MODE: if DEBUG_MODE:
subprocess.run(ffmpeg_cmd, check=True) subprocess.run(ffmpeg_cmd, check=True)
else: else:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio")
print() if TQDM_USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio")
print()
else:
console.log(f"[purple]FFmpeg [white][[cyan]Join audio[white]] ...")
with suppress_output():
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio")
print()
# Check file # Check file output
if CHECK_OUTPUT_CONVERSION: if CHECK_OUTPUT_CONVERSION:
console.log("[red]Check output ffmpeg") console.log("[red]Check output ffmpeg")
time.sleep(0.5) time.sleep(0.5)
@ -443,15 +465,24 @@ def join_subtitle(video_path: str, subtitles_list: List[Dict[str, str]], out_pat
ffmpeg_cmd += [out_path, "-y"] ffmpeg_cmd += [out_path, "-y"]
logging.info(f"FFmpeg command: {ffmpeg_cmd}") logging.info(f"FFmpeg command: {ffmpeg_cmd}")
# Run join # Run join
if DEBUG_MODE: if DEBUG_MODE:
subprocess.run(ffmpeg_cmd, check=True) subprocess.run(ffmpeg_cmd, check=True)
else: else:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle")
print() if TQDM_USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle")
print()
else:
console.log(f"[purple]FFmpeg [white][[cyan]Join subtitle[white]] ...")
with suppress_output():
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle")
print()
# Check file # Check file output
if CHECK_OUTPUT_CONVERSION: if CHECK_OUTPUT_CONVERSION:
console.log("[red]Check output ffmpeg") console.log("[red]Check output ffmpeg")
time.sleep(0.5) time.sleep(0.5)

View File

@ -50,7 +50,8 @@ DOWNLOAD_SPECIFIC_AUDIO = config_manager.get_list('M3U8_DOWNLOAD', 'specific_lis
DOWNLOAD_SPECIFIC_SUBTITLE = config_manager.get_list('M3U8_DOWNLOAD', 'specific_list_subtitles') DOWNLOAD_SPECIFIC_SUBTITLE = config_manager.get_list('M3U8_DOWNLOAD', 'specific_list_subtitles')
DOWNLOAD_VIDEO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_video') DOWNLOAD_VIDEO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_video')
DOWNLOAD_AUDIO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_audio') DOWNLOAD_AUDIO = config_manager.get_bool('M3U8_DOWNLOAD', 'download_audio')
DOWNLOAD_SUB = config_manager.get_bool('M3U8_DOWNLOAD', 'download_sub') MERGE_AUDIO = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_audio')
DOWNLOAD_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'download_sub')
MERGE_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_subs') MERGE_SUBTITLE = config_manager.get_bool('M3U8_DOWNLOAD', 'merge_subs')
REMOVE_SEGMENTS_FOLDER = config_manager.get_bool('M3U8_DOWNLOAD', 'cleanup_tmp_folder') REMOVE_SEGMENTS_FOLDER = config_manager.get_bool('M3U8_DOWNLOAD', 'cleanup_tmp_folder')
FILTER_CUSTOM_REOLUTION = config_manager.get_int('M3U8_PARSER', 'force_resolution') FILTER_CUSTOM_REOLUTION = config_manager.get_int('M3U8_PARSER', 'force_resolution')
@ -472,7 +473,7 @@ class Downloader():
os.rename(out_path, self.output_filename) os.rename(out_path, self.output_filename)
# Print size of the file # Print size of the file
console.print(Panel(f"[bold green]Download completed![/bold green]\nFile size: [bold]{format_size(os.path.getsize(self.output_filename))}[/bold]", title=f"{os.path.basename(self.output_filename.replace('.mp4', ''))}", border_style="green")) console.print(Panel(f"[bold green]Download completed![/bold green]\nFile size: [bold red]{format_size(os.path.getsize(self.output_filename))}[/bold red]", title=f"{os.path.basename(self.output_filename.replace('.mp4', ''))}", border_style="green"))
# Delete all files except the output file # Delete all files except the output file
delete_files_except_one(self.base_path, os.path.basename(self.output_filename)) delete_files_except_one(self.base_path, os.path.basename(self.output_filename))
@ -519,14 +520,16 @@ class Downloader():
# Collect information about the playlist # Collect information about the playlist
self.__manage_playlist__(m3u8_playlist_text) self.__manage_playlist__(m3u8_playlist_text)
# Start all download ... # Start all download ...
if DOWNLOAD_VIDEO: if DOWNLOAD_VIDEO:
self.__donwload_video__(server_ip) self.__donwload_video__(server_ip)
if DOWNLOAD_AUDIO: if DOWNLOAD_AUDIO:
self.__donwload_audio__(server_ip) self.__donwload_audio__(server_ip)
if DOWNLOAD_SUB: if DOWNLOAD_SUBTITLE:
self.__download_subtitle__() self.__download_subtitle__()
# Check file to convert # Check file to convert
converted_out_path = None converted_out_path = None
there_is_video: bool = (len(self.downloaded_video) > 0) there_is_video: bool = (len(self.downloaded_video) > 0)
@ -534,35 +537,69 @@ class Downloader():
there_is_subtitle: bool = (len(self.downloaded_subtitle) > 0) there_is_subtitle: bool = (len(self.downloaded_subtitle) > 0)
console.log(f"[cyan]Conversion [white]=> ([green]Audio: [yellow]{there_is_audio}[white], [green]Subtitle: [yellow]{there_is_subtitle}[white])") console.log(f"[cyan]Conversion [white]=> ([green]Audio: [yellow]{there_is_audio}[white], [green]Subtitle: [yellow]{there_is_subtitle}[white])")
# Join audio and video # Join audio and video
if there_is_audio: if there_is_audio:
converted_out_path = self.__join_video_audio__() if MERGE_AUDIO:
converted_out_path = self.__join_video_audio__()
else:
for obj_audio in self.downloaded_audio:
language = obj_audio.get('language')
path = obj_audio.get('path')
# Set the new path for regular audio
new_path = self.output_filename.replace(".mp4", f"_{language}.mp4")
try:
# Rename the audio file to the new path
os.rename(path, new_path)
logging.info(f"Audio moved to {new_path}")
except Exception as e:
logging.error(f"Failed to move audio {path} to {new_path}: {e}")
# Convert video
if there_is_video:
converted_out_path = self.__join_video__()
# Join only video ( audio is present in the same ts files ) # Join only video ( audio is present in the same ts files )
else: else:
if there_is_video: if there_is_video:
converted_out_path = self.__join_video__() converted_out_path = self.__join_video__()
# Join subtitle # Join subtitle
if there_is_subtitle: if there_is_subtitle:
if MERGE_SUBTITLE: if MERGE_SUBTITLE:
if converted_out_path is not None: if converted_out_path is not None:
converted_out_path = self.__join_video_subtitles__(converted_out_path) converted_out_path = self.__join_video_subtitles__(converted_out_path)
else: else:
for obj_sub in self.downloaded_subtitle: for obj_sub in self.downloaded_subtitle:
language = obj_sub.get('language') language = obj_sub.get('language')
path = obj_sub.get('path') path = obj_sub.get('path')
forced = 'forced' in language forced = 'forced' in language
# Check if the language includes "forced"
forced = 'forced' in language
# Remove "forced-" from the language if present and set the new path with "forced"
if forced: if forced:
language = language.replace("forced-", "") language = language.replace("forced-", "")
new_path = self.output_filename.replace(".mp4", f".{language}.forced.vtt") new_path = self.output_filename.replace(".mp4", f".{language}.forced.vtt")
else: else:
# Set the new path for regular languages
new_path = self.output_filename.replace(".mp4", f".{language}.vtt") new_path = self.output_filename.replace(".mp4", f".{language}.vtt")
try: try:
# Rename the subtitle file to the new path
os.rename(path, new_path) os.rename(path, new_path)
logging.info(f"Subtitle moved to {new_path}") logging.info(f"Subtitle moved to {new_path}")
except Exception as e: except Exception as e:
logging.error(f"Failed to move subtitle {path} to {new_path}: {e}") logging.error(f"Failed to move subtitle {path} to {new_path}: {e}")

View File

@ -64,7 +64,7 @@ class M3U8_Segments:
self.ctrl_c_detected = False # Global variable to track Ctrl+C detection self.ctrl_c_detected = False # Global variable to track Ctrl+C detection
os.makedirs(self.tmp_folder, exist_ok=True) # Create the temporary folder if it does not exist os.makedirs(self.tmp_folder, exist_ok=True) # Create the temporary folder if it does not exist
self.class_ts_estimator = M3U8_Ts_Estimator(TQDM_MAX_WORKER, 0) self.class_ts_estimator = M3U8_Ts_Estimator(0)
self.class_url_fixer = M3U8_UrlFix(url) self.class_url_fixer = M3U8_UrlFix(url)
self.fake_proxy = False self.fake_proxy = False
@ -304,7 +304,7 @@ class M3U8_Segments:
progress_bar = tqdm( progress_bar = tqdm(
total=len(self.segments), total=len(self.segments),
unit='s', unit='s',
ascii=' #', ascii='░▒█',
bar_format=bar_format, bar_format=bar_format,
dynamic_ncols=True, dynamic_ncols=True,
ncols=80, ncols=80,

View File

@ -1,7 +1,7 @@
# 20.02.24 # 20.02.24
import threading
import logging import logging
from collections import deque from collections import deque
@ -21,7 +21,7 @@ TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar
class M3U8_Ts_Estimator: class M3U8_Ts_Estimator:
def __init__(self, workers: int, total_segments: int): def __init__(self, total_segments: int):
""" """
Initialize the TSFileSizeCalculator object. Initialize the TSFileSizeCalculator object.
@ -31,11 +31,11 @@ class M3U8_Ts_Estimator:
""" """
self.ts_file_sizes = [] self.ts_file_sizes = []
self.now_downloaded_size = 0 self.now_downloaded_size = 0
self.average_over = 5 self.average_over = 6
self.list_speeds = deque(maxlen=self.average_over) self.list_speeds = deque(maxlen=self.average_over)
self.smoothed_speeds = [] self.smoothed_speeds = []
self.tqdm_workers = workers
self.total_segments = total_segments self.total_segments = total_segments
self.lock = threading.Lock()
def add_ts_file(self, size: int, size_download: int, duration: float): def add_ts_file(self, size: int, size_download: int, duration: float):
""" """
@ -50,30 +50,26 @@ class M3U8_Ts_Estimator:
logging.error("Invalid input values: size=%d, size_download=%d, duration=%f", size, size_download, duration) logging.error("Invalid input values: size=%d, size_download=%d, duration=%f", size, size_download, duration)
return return
self.ts_file_sizes.append(size) # Calculate speed outside of the lock
self.now_downloaded_size += size_download
# Only for the start
if len(self.smoothed_speeds) <= 3:
size_download = size_download / self.tqdm_workers
try: try:
# Calculate mbps speed_mbps = (size_download * 16) / (duration * 1_000_000)
speed_mbps = (size_download * 8) / (duration * 1_000_000) * self.tqdm_workers
except ZeroDivisionError as e: except ZeroDivisionError as e:
logging.error("Division by zero error while calculating speed: %s", e) logging.error("Division by zero error while calculating speed: %s", e)
return return
self.list_speeds.append(speed_mbps) # Only update shared data within the lock
with self.lock:
self.ts_file_sizes.append(size)
self.now_downloaded_size += size_download
self.list_speeds.append(speed_mbps)
# Calculate moving average # Calculate moving average
smoothed_speed = sum(self.list_speeds) / len(self.list_speeds) smoothed_speed = sum(self.list_speeds) / len(self.list_speeds)
self.smoothed_speeds.append(smoothed_speed) self.smoothed_speeds.append(smoothed_speed)
# Update smooth speeds # Update smooth speeds
if len(self.smoothed_speeds) > self.average_over: if len(self.smoothed_speeds) > self.average_over:
self.smoothed_speeds.pop(0) self.smoothed_speeds.pop(0)
def calculate_total_size(self) -> str: def calculate_total_size(self) -> str:
""" """
@ -107,7 +103,7 @@ class M3U8_Ts_Estimator:
Returns: Returns:
float: The average speed in megabytes per second (MB/s). float: The average speed in megabytes per second (MB/s).
""" """
return (sum(self.smoothed_speeds) / len(self.smoothed_speeds)) / 10 # MB/s return ((sum(self.smoothed_speeds) / len(self.smoothed_speeds)) / 8 ) * 10 # MB/s
def get_downloaded_size(self) -> str: def get_downloaded_size(self) -> str:
""" """

View File

@ -52,17 +52,15 @@ class M3U8_Codec:
Represents codec information for an M3U8 playlist. Represents codec information for an M3U8 playlist.
""" """
def __init__(self, bandwidth, resolution, codecs): def __init__(self, bandwidth, codecs):
""" """
Initializes the M3U8Codec object with the provided parameters. Initializes the M3U8Codec object with the provided parameters.
Args: Args:
- bandwidth (int): Bandwidth of the codec. - bandwidth (int): Bandwidth of the codec.
- resolution (str): Resolution of the codec.
- codecs (str): Codecs information in the format "avc1.xxxxxx,mp4a.xx". - codecs (str): Codecs information in the format "avc1.xxxxxx,mp4a.xx".
""" """
self.bandwidth = bandwidth self.bandwidth = bandwidth
self.resolution = resolution
self.codecs = codecs self.codecs = codecs
self.audio_codec = None self.audio_codec = None
self.video_codec = None self.video_codec = None
@ -76,7 +74,10 @@ class M3U8_Codec:
""" """
# Split the codecs string by comma # Split the codecs string by comma
codecs_list = self.codecs.split(',') try:
codecs_list = self.codecs.split(',')
except Exception as e:
logging.error(f"Cant split codec list: {self.codecs} with error {e}")
# Separate audio and video codecs # Separate audio and video codecs
for codec in codecs_list: for codec in codecs_list:
@ -448,14 +449,14 @@ class M3U8_Parser:
try: try:
for playlist in m3u8_obj.playlists: for playlist in m3u8_obj.playlists:
there_is_codec = not M3U8_Parser.extract_resolution(playlist.uri) == (0,0) there_is_codec = not playlist.stream_info.codecs is None
logging.info(f"There is coded: {there_is_codec}")
if there_is_codec: if there_is_codec:
self.codec = M3U8_Codec( self.codec = M3U8_Codec(
playlist.stream_info.bandwidth, playlist.stream_info.bandwidth,
None,
playlist.stream_info.codecs playlist.stream_info.codecs
) )
# Direct access resolutions in m3u8 obj # Direct access resolutions in m3u8 obj
if playlist.stream_info.resolution is not None: if playlist.stream_info.resolution is not None:

View File

@ -20,7 +20,7 @@ def start_message():
Display a start message. Display a start message.
""" """
msg = ''' msg = r'''
_____ _ _ _____ _ _ _____ _ _ _____ _ _
/ ____| | (_) / ____| (_) | / ____| | (_) / ____| (_) |

View File

@ -1,6 +1,7 @@
# 24.01.24 # 24.01.24
import re import re
import io
import os import os
import sys import sys
import ssl import ssl
@ -14,6 +15,7 @@ import zipfile
import platform import platform
import importlib import importlib
import subprocess import subprocess
import contextlib
import importlib.metadata import importlib.metadata
from typing import List from typing import List
@ -28,6 +30,7 @@ from .console import console
# --> OS FILE ASCII # --> OS FILE ASCII
special_chars_to_remove = ['!','@','#','$','%','^','&','*','(',')','[',']','{','}','<','|','`','~',"'",'"',';',':',',','?',"\\","/","\t"] special_chars_to_remove = ['!','@','#','$','%','^','&','*','(',')','[',']','{','}','<','|','`','~',"'",'"',';',':',',','?',"\\","/","\t"]
@ -103,6 +106,14 @@ def remove_special_characters(input_string):
# --> OS MANAGE OUTPUT
@contextlib.contextmanager
def suppress_output():
with contextlib.redirect_stdout(io.StringIO()):
yield
# --> OS MANAGE FOLDER # --> OS MANAGE FOLDER
def create_folder(folder_name: str) -> None: def create_folder(folder_name: str) -> None:
""" """

View File

@ -22,10 +22,11 @@
"tqdm_use_large_bar": true, "tqdm_use_large_bar": true,
"download_video": true, "download_video": true,
"download_audio": true, "download_audio": true,
"merge_audio": true,
"specific_list_audio": ["ita"],
"download_sub": true, "download_sub": true,
"merge_subs": true, "merge_subs": true,
"specific_list_audio": ["ita"], "specific_list_subtitles": ["eng", "spa"],
"specific_list_subtitles": ["eng"],
"cleanup_tmp_folder": true, "cleanup_tmp_folder": true,
"create_report": false "create_report": false
}, },