github/StreamingCommunity
1
1
Fork 0
mirror of https://github.com/Arrowar/StreamingCommunity.git synced 2025-06-05 02:55:25 +00:00
Code Issues 2 Packages Projects Releases Wiki Activity

Improve ffmpeg check and semgents info.

Browse Source
This commit is contained in:
Lovi 2024-12-07 16:25:42 +01:00
parent 475bd88d33
commit f5ad9b7187
8 changed files with 87 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
StreamingCommunity/Lib/Downloader/HLS/downloader.py
View File

@ -397,7 +397,7 @@ class ContentDownloader:
self.expected_real_time = video_m3u8.expected_real_time
# Download the video streams and print status
video_m3u8.download_streams(f"{Colors.MAGENTA}video")
video_m3u8.download_streams(f"{Colors.MAGENTA}video", "video")
# Print duration information of the downloaded video
#print_duration_table(downloaded_video[0].get('path'))
@ -427,7 +427,7 @@ class ContentDownloader:
audio_m3u8.get_info()
# Download the audio segments and print status
audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}")
audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}", "audio")
# Print duration information of the downloaded audio
#print_duration_table(obj_audio.get('path'))

38
StreamingCommunity/Lib/Downloader/HLS/segments.py
View File

@ -4,10 +4,11 @@ import os
import sys
import time
import queue
import signal
import logging
import binascii
import threading
import signal
from queue import PriorityQueue
from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor, as_completed
@ -34,6 +35,7 @@ from ...M3U8 import (
M3U8_Parser,
M3U8_UrlFix
)
from ...FFmpeg.util import print_duration_table
from .proxyes import main_test_proxy
# Config
@ -93,6 +95,10 @@ class M3U8_Segments:
self.interrupt_flag = threading.Event()
self.download_interrupted = False
# OTHER INFO
self.info_maxRetry = 0
self.info_nRetry = 0
def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes:
"""
Retrieves the encryption key from the M3U8 playlist.
@ -317,11 +323,16 @@ class M3U8_Segments:
except Exception as e:
logging.info(f"Attempt {attempt + 1} failed for segment {index} - '{ts_url}': {e}")
# Update stat variable class
if attempt > self.info_maxRetry:
self.info_maxRetry = ( attempt + 1 )
self.info_nRetry += 1
if attempt + 1 == REQUEST_MAX_RETRY:
console.log(f"[red]Final retry failed for segment: {index}")
self.queue.put((index, None)) # Marker for failed segment
progress_bar.update(1)
break
#break
sleep_time = backoff_factor * (2 ** attempt)
logging.info(f"Retrying segment {index} in {sleep_time} seconds...")
@ -382,12 +393,13 @@ class M3U8_Segments:
except Exception as e:
logging.error(f"Error writing segment {index}: {str(e)}")
def download_streams(self, add_desc):
def download_streams(self, description: str, type: str):
"""
Downloads all TS segments in parallel and writes them to a file.
Parameters:
- add_desc (str): Additional description for the progress bar.
- description: Description to insert on tqdm bar
- type (str): Type of download: 'video' or 'audio'
"""
self.setup_interrupt_handler()
@ -414,15 +426,18 @@ class M3U8_Segments:
AUDIO_WORKERS = DEFAULT_AUDIO_WORKERS
# Differnt workers for audio and video
if "video" in str(add_desc):
if "video" == str(type):
TQDM_MAX_WORKER = VIDEO_WORKERS
if "audio" in str(add_desc):
if "audio" == str(type):
TQDM_MAX_WORKER = AUDIO_WORKERS
console.print(f"[cyan]Video workers[white]: [green]{VIDEO_WORKERS} [white]| [cyan]Audio workers[white]: [green]{AUDIO_WORKERS}")
# Custom bar for mobile and pc
if TQDM_USE_LARGE_BAR:
bar_format = (
f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{add_desc}{Colors.WHITE}): "
f"{Colors.YELLOW}[HLS] {Colors.WHITE}({Colors.CYAN}{description}{Colors.WHITE}): "
f"{Colors.RED}{{percentage:.2f}}% "
f"{Colors.MAGENTA}{{bar}} "
f"{Colors.WHITE}[ {Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] "
@ -535,4 +550,11 @@ class M3U8_Segments:
if file_size == 0:
raise Exception("Output file is empty")
logging.info(f"Download completed. File size: {file_size} bytes")
console.print(f"[cyan]Max retry per URL[white]: [green]{self.info_maxRetry}[green] [white]| [cyan]Total retry done[white]: [green]{self.info_nRetry}[green] [white]| [cyan]Duration: {print_duration_table(self.tmp_file_path, None, True)} \n")
if self.info_nRetry >= len(self.segments) * (1/3.33):
console.print(
"[yellow]⚠ Warning:[/yellow] Too many retries detected! "
"Consider reducing the number of [cyan]workers[/cyan] in the [magenta]config.json[/magenta] file. "
"This will impact [bold]performance[/bold]."
)

18
StreamingCommunity/Lib/FFmpeg/command.py
View File

@ -9,8 +9,11 @@ from typing import List, Dict
# Internal utilities
from StreamingCommunity.Util._jsonConfig import config_manager
from StreamingCommunity.Util.os import os_manager, suppress_output
from StreamingCommunity.Util.os import os_manager, os_summary, suppress_output
from StreamingCommunity.Util.console import console
# Logic class
from .util import need_to_force_to_ts, check_duration_v_a
from .capture import capture_ffmpeg_real_time
from ..M3U8 import M3U8_Codec
@ -29,6 +32,7 @@ FFMPEG_DEFAULT_PRESET = config_manager.get("M3U8_CONVERSION", "default_preset")
# Variable
TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar')
FFMPEG_PATH = os_summary.ffmpeg_path
def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
@ -49,8 +53,8 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
logging.error("Missing input video for ffmpeg conversion.")
sys.exit(0)
# Start command
ffmpeg_cmd = ['ffmpeg']
# Start command with locate ffmpeg
ffmpeg_cmd = [FFMPEG_PATH]
# Enabled the use of gpu
if USE_GPU:
@ -140,8 +144,8 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
video_audio_same_duration = check_duration_v_a(video_path, audio_tracks[0].get('path'))
# Start command
ffmpeg_cmd = ['ffmpeg']
# Start command with locate ffmpeg
ffmpeg_cmd = [FFMPEG_PATH]
# Enabled the use of gpu
if USE_GPU:
@ -242,8 +246,8 @@ def join_subtitle(video_path: str, subtitles_list: List[Dict[str, str]], out_pat
logging.error("Missing input video for ffmpeg conversion.")
sys.exit(0)
# Start command
ffmpeg_cmd = ["ffmpeg", "-i", video_path]
# Start command with locate ffmpeg
ffmpeg_cmd = [FFMPEG_PATH, "-i", video_path]
# Add subtitle input files first
for subtitle in subtitles_list:

32
StreamingCommunity/Lib/FFmpeg/util.py
View File

@ -10,6 +10,12 @@ from typing import Tuple
# Internal utilities
from StreamingCommunity.Util.console import console
from StreamingCommunity.Util.os import os_summary
# Variable
FFPROB_PATH = os_summary.ffprobe_path
def has_audio_stream(video_path: str) -> bool:
@ -23,7 +29,7 @@ def has_audio_stream(video_path: str) -> bool:
has_audio (bool): True if the input video has an audio stream, False otherwise.
"""
try:
ffprobe_cmd = ['ffprobe', '-v', 'error', '-print_format', 'json', '-select_streams', 'a', '-show_streams', video_path]
ffprobe_cmd = [FFPROB_PATH, '-v', 'error', '-print_format', 'json', '-select_streams', 'a', '-show_streams', video_path]
logging.info(f"FFmpeg command: {ffprobe_cmd}")
with subprocess.Popen(ffprobe_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as proc:
@ -47,12 +53,11 @@ def get_video_duration(file_path: str) -> float:
- file_path (str): The path to the video file.
Returns:
(float): The duration of the video in seconds if successful,
None if there's an error.
(float): The duration of the video in seconds if successful, None if there's an error.
"""
try:
ffprobe_cmd = ['ffprobe', '-v', 'error', '-show_format', '-print_format', 'json', file_path]
ffprobe_cmd = [FFPROB_PATH, '-v', 'error', '-show_format', '-print_format', 'json', file_path]
logging.info(f"FFmpeg command: {ffprobe_cmd}")
# Use a with statement to ensure the subprocess is cleaned up properly
@ -77,15 +82,16 @@ def get_video_duration(file_path: str) -> float:
logging.error(f"Error get video duration: {e}")
sys.exit(0)
def get_video_duration_s(filename):
"""
Get the duration of a video file using ffprobe.
Parameters:
- filename (str): Path to the video file (e.g., 'sim.mp4')
- filename (str): Path to the video file (e.g., 'sim.mp4')
Returns:
- duration (float): Duration of the video in seconds, or None if an error occurs.
- duration (float): Duration of the video in seconds, or None if an error occurs.
"""
ffprobe_cmd = ['ffprobe', '-v', 'error', '-show_entries', 'format=duration', '-of', 'default=noprint_wrappers=1:nokey=1', filename]
@ -160,14 +166,14 @@ def get_ffprobe_info(file_path):
Get format and codec information for a media file using ffprobe.
Parameters:
file_path (str): Path to the media file.
- file_path (str): Path to the media file.
Returns:
dict: A dictionary containing the format name and a list of codec names.
"""
try:
result = subprocess.run(
['ffprobe', '-v', 'error', '-show_format', '-show_streams', '-print_format', 'json', file_path],
[FFPROB_PATH, '-v', 'error', '-show_format', '-show_streams', '-print_format', 'json', file_path],
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True
)
output = result.stdout
@ -195,7 +201,7 @@ def is_png_format_or_codec(file_info):
Check if the format is 'png_pipe' or if any codec is 'png'.
Parameters:
file_info (dict): The dictionary containing file information.
- file_info (dict): The dictionary containing file information.
Returns:
bool: True if the format is 'png_pipe' or any codec is 'png', otherwise False.
@ -210,7 +216,7 @@ def need_to_force_to_ts(file_path):
Get if a file to TS format if it is in PNG format or contains a PNG codec.
Parameters:
file_path (str): Path to the input media file.
- file_path (str): Path to the input media file.
"""
logging.info(f"Processing file: {file_path}")
file_info = get_ffprobe_info(file_path)
@ -225,11 +231,11 @@ def check_duration_v_a(video_path, audio_path):
Check if the duration of the video and audio matches.
Parameters:
- video_path (str): Path to the video file.
- audio_path (str): Path to the audio file.
- video_path (str): Path to the video file.
- audio_path (str): Path to the audio file.
Returns:
- bool: True if the duration of the video and audio matches, False otherwise.
- bool: True if the duration of the video and audio matches, False otherwise.
"""
# Ottieni la durata del video

34
StreamingCommunity/Util/os.py
View File

@ -306,7 +306,12 @@ class InternManager():
print()
class OsSummary():
class OsSummary:
def __init__(self):
ffmpeg_path, ffprobe_path = check_ffmpeg()
self.ffmpeg_path = ffmpeg_path
self.ffprobe_path = ffprobe_path
def get_executable_version(self, command: list):
"""
@ -441,27 +446,29 @@ class OsSummary():
console.print(f"[cyan]Python[white]: [bold red]{python_version} ({python_implementation} {arch}) - {os_info} ({glibc_version})[/bold red]")
logging.info(f"Python: {python_version} ({python_implementation} {arch}) - {os_info} ({glibc_version})")
# ffmpeg and ffprobe versions
ffmpeg_path, ffprobe_path = check_ffmpeg()
# Usa il comando 'where' su Windows
if platform.system() == "Windows":
# Usa il comando 'where' su Windows
command = 'where'
# Usa il comando 'which' su Unix/Linux
else:
# Usa il comando 'which' su Unix/Linux
command = 'which'
# Locate ffmpeg and ffprobe
if "binary" not in ffmpeg_path:
ffmpeg_path = self.check_ffmpeg_location([command, 'ffmpeg'])
if self.ffmpeg_path != None and "binary" not in self.ffmpeg_path:
self.ffmpeg_path = self.check_ffmpeg_location([command, 'ffmpeg'])
if "binary" not in ffprobe_path:
ffprobe_path = self.check_ffmpeg_location([command, 'ffprobe'])
if self.ffprobe_path != None and "binary" not in self.ffprobe_path:
self.ffprobe_path = self.check_ffmpeg_location([command, 'ffprobe'])
ffmpeg_version = self.get_executable_version([ffprobe_path, '-version'])
ffprobe_version = self.get_executable_version([ffprobe_path, '-version'])
if self.ffmpeg_path is None or self.ffprobe_path is None:
console.log("[red]Cant locate ffmpeg or ffprobe")
sys.exit(0)
console.print(f"[cyan]Path[white]: [red]ffmpeg [bold yellow]'{ffmpeg_path}'[/bold yellow][white], [red]ffprobe '[bold yellow]{ffprobe_path}'[/bold yellow]")
ffmpeg_version = self.get_executable_version([self.ffprobe_path, '-version'])
ffprobe_version = self.get_executable_version([self.ffprobe_path, '-version'])
console.print(f"[cyan]Path[white]: [red]ffmpeg [bold yellow]'{self.ffmpeg_path}'[/bold yellow][white], [red]ffprobe '[bold yellow]{self.ffprobe_path}'[/bold yellow]")
console.print(f"[cyan]Exe versions[white]: [bold red]ffmpeg {ffmpeg_version}, ffprobe {ffprobe_version}[/bold red]")
# Check if requirements.txt exists, if not on pyinstaller
@ -500,6 +507,7 @@ os_manager = OsManager()
internet_manager = InternManager()
os_summary = OsSummary()
@contextlib.contextmanager
def suppress_output():
with contextlib.redirect_stdout(io.StringIO()):

2
Test/Download/HLS.py
View File

@ -18,6 +18,6 @@ from StreamingCommunity.Lib.Downloader import HLS_Downloader
start_message()
logger = Logger()
print("Return: ", HLS_Downloader(
output_filename="",
output_filename="test.mp4",
m3u8_index=""
).start())

4
Test/Download/MP4.py
View File

@ -18,6 +18,6 @@ from StreamingCommunity.Lib.Downloader import MP4_downloader
start_message()
logger = Logger()
print("Return: ", MP4_downloader(
"",
".\Video\undefined.mp4"
url="",
path=r".\Video\undefined.mp4"
))

2
config.json
View File

@ -20,7 +20,7 @@
},
"REQUESTS": {
"timeout": 20,
"max_retry": 5,
"max_retry": 8,
"verify_ssl": true,
"proxy_start_min": 0.1,
"proxy_start_max": 0.5,