From 38a666e53d9ca0a24b5b7f52c6eccb83a202a59c Mon Sep 17 00:00:00 2001 From: Ghost <62809003+Ghost6446@users.noreply.github.com> Date: Sat, 27 Jan 2024 17:19:41 +0100 Subject: [PATCH] add debug --- .gitignore | 5 ++- Src/Util/FFmpeg/installer.py | 4 +-- Src/Util/FFmpeg/m3u8.py | 68 ++++++++++++++++++++++++++---------- Src/Util/FFmpeg/util.py | 10 ++++-- Src/Util/Helper/console.py | 12 +++++++ Src/Util/Helper/os.py | 18 ++++++++-- 6 files changed, 91 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index ce62ea0..0eedbff 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ + +# Folder bin lib lib64 @@ -6,4 +8,5 @@ pyvenv.cfg # Project specific videos/ -tmp/ \ No newline at end of file +tmp/ +debug.log \ No newline at end of file diff --git a/Src/Util/FFmpeg/installer.py b/Src/Util/FFmpeg/installer.py index fa1f01a..3f95030 100644 --- a/Src/Util/FFmpeg/installer.py +++ b/Src/Util/FFmpeg/installer.py @@ -38,9 +38,9 @@ def check_ffmpeg(): try: subprocess.run(["ffmpeg", "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) console.print("[blue]FFmpeg [white]=> [red]Add to system path") - except subprocess.CalledProcessError: - console.print("[cyan]FFmpeg is not in the PATH. Downloading and adding to the PATH...[/cyan]") + except: try: + console.print("[cyan]FFmpeg is not in the PATH. Downloading and adding to the PATH...[/cyan]") download_ffmpeg() sys.exit(0) # Exit except: diff --git a/Src/Util/FFmpeg/m3u8.py b/Src/Util/FFmpeg/m3u8.py index bb5600a..7905fa7 100644 --- a/Src/Util/FFmpeg/m3u8.py +++ b/Src/Util/FFmpeg/m3u8.py @@ -1,19 +1,20 @@ # 5.01.24 -> 7.01.24 + +# Class import +from Src.Util.Helper.console import console, config_logger +from Src.Util.Helper.headers import get_headers +from Src.Util.FFmpeg.util import there_is_audio, merge_ts_files + + # Import -import requests, re, os, ffmpeg, shutil, time, sys, warnings +import requests, re, os, ffmpeg, time, sys, warnings, logging, shutil from tqdm.rich import tqdm from concurrent.futures import ThreadPoolExecutor from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend -# Class import -from Src.Util.Helper.console import console -from Src.Util.Helper.headers import get_headers -from Src.Util.FFmpeg.util import there_is_audio, merge_ts_files - - # Disable warning from tqdm import TqdmExperimentalWarning warnings.filterwarnings("ignore", category=TqdmExperimentalWarning) @@ -22,6 +23,8 @@ warnings.filterwarnings("ignore", category=UserWarning, module="cryptography") # Variable os.makedirs("videos", exist_ok=True) +DOWNLOAD_WORKERS = 30 +USE_MULTI_THREAD = True # [ main class ] @@ -44,6 +47,11 @@ class M3U8Downloader: self.max_retry = 3 self.failed_segments = [] + # Debug + logging.debug(m3u8_url) + logging.debug(m3u8_audio) + logging.debug(self.key) + def decode_ext_x_key(self, key_str): key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:") v_list = re.findall(r"[^,=]+", key_str) @@ -79,7 +87,7 @@ class M3U8Downloader: if self.m3u8_audio != None: self.segments_audio.append(m3u8_audio_line[i+1]) - console.log(f"[cyan]Find: {len(self.segments)} ts file to download") + console.log(f"[cyan]Find: {len(self.segments)} ts video file to download") # Check video ts segment if len(self.segments) == 0: @@ -108,6 +116,7 @@ class M3U8Downloader: # Check there is audio in first ts file path_test_ts_file = os.path.join(self.temp_folder, "ts_test.ts") + if self.key and self.iv: open(path_test_ts_file, "wb").write(self.decrypt_ts(requests.get(self.segments[0]).content)) else: @@ -115,9 +124,7 @@ class M3U8Downloader: if not there_is_audio(path_test_ts_file): self.download_audio = True - #console.log("[yellow]=> Make req to get video and audio file") - #console.log("[yellow]=> Download audio") os.remove(path_test_ts_file) def decrypt_ts(self, encrypted_data): @@ -132,9 +139,10 @@ class M3U8Downloader: if retry == self.max_retry: console.log(f"[red]Failed download: {ts_url}") self.segments.remove(ts_url) + logging.error(f"Failed: {ts_url}") return None - req = requests.get(ts_url, headers={'user-agent': get_headers()}, timeout=10, allow_redirects=True) + req = requests.get(ts_url, headers={'user-agent': get_headers()}, timeout=5, allow_redirects=True) if req.status_code == 200: return req.content @@ -146,6 +154,7 @@ class M3U8Downloader: video_ts_url = self.segments[index] video_ts_filename = os.path.join(self.temp_folder, f"{index}_v.ts") + logging.debug(f"Download video ts file: {video_ts_url}") # Download video or audio ts file if not os.path.exists(video_ts_filename): # Only for media that not use audio @@ -160,10 +169,15 @@ class M3U8Downloader: else: with open(video_ts_filename, "wb") as ts_file: ts_file.write(ts_response) + + else: + logging.debug(f"Cant save video ts: {video_ts_url}") # Donwload only audio ts file and merge with video if self.download_audio: audio_ts_url = self.segments_audio[index] + logging.debug(f"Download audio ts file: {audio_ts_url}") + audio_ts_filename = os.path.join(self.temp_folder, f"{index}_a.ts") video_audio_ts_filename = os.path.join(self.temp_folder, f"{index}_v_a.ts") @@ -192,13 +206,29 @@ class M3U8Downloader: else: self.failed_segments.append(index) os.remove(audio_ts_filename) + + else: + logging.debug(f"Cant save audio ts: {audio_ts_url}") def download_and_save_ts(self): - with ThreadPoolExecutor(max_workers=30) as executor: - list(tqdm(executor.map(self.decrypt_and_save, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download")) - - if len(self.failed_segments) > 0: - console.log(f"[red]Segment ts: {self.failed_segments}, cant use audio") + + try: + + if USE_MULTI_THREAD: + with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor: + list(tqdm(executor.map(self.decrypt_and_save, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download")) + else: + for index in range(len(self.segments)): + console.log(f"[yellow]Download: [red]{index}") + self.decrypt_and_save(index) + + + if len(self.failed_segments) > 0: + console.log(f"[red]Segment ts: {self.failed_segments}, cant use audio") + + except KeyboardInterrupt: + console.log("[yellow]Interruption detected. Exiting program.") + sys.exit(0) def join_ts_files(self): @@ -222,12 +252,12 @@ class M3U8Downloader: ( ffmpeg.input(file_list_path, format='concat', safe=0).output(self.output_filename, c='copy', loglevel='quiet').run() ) - console.log(f"[cyan]Clean ...") except ffmpeg.Error as e: console.log(f"[red]Error saving MP4: {e.stdout}") sys.exit(0) - - time.sleep(2) + + time.sleep(1) + console.log(f"[cyan]Clean ...") os.remove(file_list_path) shutil.rmtree("tmp", ignore_errors=True) diff --git a/Src/Util/FFmpeg/util.py b/Src/Util/FFmpeg/util.py index a14eb79..b65ff40 100644 --- a/Src/Util/FFmpeg/util.py +++ b/Src/Util/FFmpeg/util.py @@ -1,7 +1,10 @@ # 4.01.2023 -# Import -import ffmpeg, subprocess +# Class import +from Src.Util.Helper.console import console, config_logger + +# General import +import ffmpeg, subprocess, logging def there_is_audio(ts_file_path): probe = ffmpeg.probe(ts_file_path) @@ -10,11 +13,14 @@ def there_is_audio(ts_file_path): def merge_ts_files(video_path, audio_path, output_path): input_video = ffmpeg.input(video_path) input_audio = ffmpeg.input(audio_path) + logging.debug(f"Merge video ts: {input_video}, with audio ts: {input_audio}, to: {output_path}") ffmpeg_command = ffmpeg.output(input_video, input_audio, output_path, format='mpegts', acodec='copy', vcodec='copy', loglevel='quiet').compile() try: subprocess.run(ffmpeg_command, check=True, stderr=subprocess.PIPE) + logging.debug(f"Saving: {output_path}") return True except subprocess.CalledProcessError as e: + logging.error(f"Can save: {output_path}") return False \ No newline at end of file diff --git a/Src/Util/Helper/console.py b/Src/Util/Helper/console.py index 3872bf3..907713c 100644 --- a/Src/Util/Helper/console.py +++ b/Src/Util/Helper/console.py @@ -3,8 +3,20 @@ # Import from rich.console import Console from rich.prompt import Prompt +import logging # Variable msg = Prompt() console = Console() +SAVE_DEBUG = False +class ConfigurazioneLogger: + def __init__(self, nome_file_log='debug.log'): + self.nome_file_log = nome_file_log + self.configura_logger() + + def configura_logger(self): + if SAVE_DEBUG: + logging.basicConfig(filename=self.nome_file_log, filemode='w', level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') + +config_logger = ConfigurazioneLogger() \ No newline at end of file diff --git a/Src/Util/Helper/os.py b/Src/Util/Helper/os.py index 18d083d..11774eb 100644 --- a/Src/Util/Helper/os.py +++ b/Src/Util/Helper/os.py @@ -1,11 +1,25 @@ # 24.01.24 # Import -import shutil, os +import shutil, os, time def remove_folder(folder_path): if os.path.exists(folder_path): try: shutil.rmtree(folder_path) except OSError as e: - print(f"Error removing folder '{folder_path}': {e}") \ No newline at end of file + print(f"Error removing folder '{folder_path}': {e}") + + +def remove_file(file_path): + if os.path.exists(file_path): + time.sleep(1) + + try: + os.remove(file_path) + except OSError as e: + print(f"Error removing file '{file_path}': {e}") + else: + print(f"File '{file_path}' does not exist.") + + time.sleep(1) \ No newline at end of file