From c2489c028d25c6add86f2ca0a8d51d39fad78f7e Mon Sep 17 00:00:00 2001 From: Helper0x <158275011+Helper0x@users.noreply.github.com> Date: Wed, 31 Jan 2024 20:04:40 +0100 Subject: [PATCH] fix audio sync --- Src/Api/film.py | 1 - Src/Api/page.py | 1 + Src/Api/tv.py | 1 - Src/Util/FFmpeg/m3u8.py | 336 ++++++++++++++++------------------------ Src/Util/FFmpeg/util.py | 49 +++--- run.py | 3 +- 6 files changed, 160 insertions(+), 231 deletions(-) diff --git a/Src/Api/film.py b/Src/Api/film.py index 3644e19..0f1649f 100644 --- a/Src/Api/film.py +++ b/Src/Api/film.py @@ -103,5 +103,4 @@ def main_dw_film(id_film, title_name, domain): if m3u8_url_audio != None: console.print("[blue]Use m3u8 audio => [red]True") - print("\n") dw_m3u8(m3u8_url, m3u8_url_audio, m3u8_key, mp4_path) diff --git a/Src/Api/page.py b/Src/Api/page.py index 258dca7..cd299a9 100644 --- a/Src/Api/page.py +++ b/Src/Api/page.py @@ -17,6 +17,7 @@ def domain_version(): site_req = requests.get(f"https://streamingcommunity.{req_repo.json()['domain']}/", headers={'user-agent': get_headers()}).text soup = BeautifulSoup(site_req, "lxml") version = json.loads(soup.find("div", {"id": "app"}).get("data-page"))['version'] + console.print(f"[blue]Rules [white]=> [red].{req_repo.json()['domain']}") return req_repo.json()['domain'], version diff --git a/Src/Api/tv.py b/Src/Api/tv.py index 50cef20..f764621 100644 --- a/Src/Api/tv.py +++ b/Src/Api/tv.py @@ -130,7 +130,6 @@ def dw_single_ep(tv_id, eps, index_ep_select, domain, token, tv_name, season_sel if m3u8_url_audio != None: console.print("[blue]Use m3u8 audio => [red]True") - print("\n") dw_m3u8(m3u8_url, m3u8_url_audio, m3u8_key, mp4_path) def main_dw_tv(tv_id, tv_name, version, domain): diff --git a/Src/Util/FFmpeg/m3u8.py b/Src/Util/FFmpeg/m3u8.py index 7905fa7..e9b7e86 100644 --- a/Src/Util/FFmpeg/m3u8.py +++ b/Src/Util/FFmpeg/m3u8.py @@ -1,245 +1,137 @@ # 5.01.24 -> 7.01.24 - # Class import from Src.Util.Helper.console import console, config_logger from Src.Util.Helper.headers import get_headers -from Src.Util.FFmpeg.util import there_is_audio, merge_ts_files - +from Src.Util.FFmpeg.util import print_duration_table # Import -import requests, re, os, ffmpeg, time, sys, warnings, logging, shutil +import requests, re, os, ffmpeg, time, sys, warnings, logging, shutil, subprocess from tqdm.rich import tqdm from concurrent.futures import ThreadPoolExecutor from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.backends import default_backend - # Disable warning from tqdm import TqdmExperimentalWarning warnings.filterwarnings("ignore", category=TqdmExperimentalWarning) warnings.filterwarnings("ignore", category=UserWarning, module="cryptography") - # Variable os.makedirs("videos", exist_ok=True) DOWNLOAD_WORKERS = 30 -USE_MULTI_THREAD = True # [ main class ] -class M3U8Downloader: - - def __init__(self, m3u8_url, m3u8_audio = None, key=None, output_filename="output.mp4"): - self.m3u8_url = m3u8_url - self.m3u8_audio = m3u8_audio - self.key = key - self.output_filename = output_filename - - self.segments = [] - self.segments_audio = [] +class Decryption(): + def __init__(self, key): self.iv = None - if key != None: self.key = bytes.fromhex(key) - - self.temp_folder = "tmp" - os.makedirs(self.temp_folder, exist_ok=True) - self.download_audio = False - self.max_retry = 3 - self.failed_segments = [] - - # Debug - logging.debug(m3u8_url) - logging.debug(m3u8_audio) - logging.debug(self.key) + self.key = key def decode_ext_x_key(self, key_str): + logging.debug(f"String to decode: {key_str}") key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:") v_list = re.findall(r"[^,=]+", key_str) key_map = {v_list[i]: v_list[i+1] for i in range(0, len(v_list), 2)} - - return key_map # URI | METHOD | IV - + logging.debug(f"Output string: {key_map}") + return key_map + def parse_key(self, raw_iv): self.iv = bytes.fromhex(raw_iv.replace("0x", "")) - def parse_m3u8(self, m3u8_content): - if self.m3u8_audio != None: - m3u8_audio_line = str(requests.get(self.m3u8_audio).content).split("\\n") - - m3u8_base_url = self.m3u8_url.rstrip(self.m3u8_url.split("/")[-1]) - lines = m3u8_content.split('\n') - - for i in range(len(lines)): - line = str(lines[i]) - - if line.startswith("#EXT-X-KEY:"): - x_key_dict = self.decode_ext_x_key(line) - self.parse_key(x_key_dict['IV']) - - if line.startswith("#EXTINF"): - ts_url = lines[i+1] - - if not ts_url.startswith("http"): - ts_url = m3u8_base_url + ts_url - - self.segments.append(ts_url) - - if self.m3u8_audio != None: - self.segments_audio.append(m3u8_audio_line[i+1]) - - console.log(f"[cyan]Find: {len(self.segments)} ts video file to download") - - # Check video ts segment - if len(self.segments) == 0: - console.log("[red]No ts files to download") - sys.exit(0) - - # Check audio ts segment - if self.m3u8_audio != None: - console.log(f"[cyan]Find: {len(self.segments_audio)} ts audio file to download") - - if len(self.segments_audio) == 0: - console.log("[red]No ts audio files to download") - sys.exit(0) - - def download_m3u8(self): - response = requests.get(self.m3u8_url, headers={'user-agent': get_headers()}) - - if response.ok: - m3u8_content = response.text - self.parse_m3u8(m3u8_content) - else: - console.log("[red]Wrong m3u8 url") - sys.exit(0) - - if self.m3u8_audio != None: - - # Check there is audio in first ts file - path_test_ts_file = os.path.join(self.temp_folder, "ts_test.ts") - - if self.key and self.iv: - open(path_test_ts_file, "wb").write(self.decrypt_ts(requests.get(self.segments[0]).content)) - else: - open(path_test_ts_file, "wb").write(requests.get(self.segments[0]).content) - - if not there_is_audio(path_test_ts_file): - self.download_audio = True - - os.remove(path_test_ts_file) - def decrypt_ts(self, encrypted_data): cipher = Cipher(algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend()) decryptor = cipher.decryptor() decrypted_data = decryptor.update(encrypted_data) + decryptor.finalize() return decrypted_data - - def make_req_single_ts_file(self, ts_url, retry=0): - if retry == self.max_retry: - console.log(f"[red]Failed download: {ts_url}") - self.segments.remove(ts_url) - logging.error(f"Failed: {ts_url}") - return None +class M3U8(): + def __init__(self, url, key=None): + self.url = url + self.key = bytes.fromhex(key) if key is not None else key + self.temp_folder = "tmp" + os.makedirs(self.temp_folder, exist_ok=True) - req = requests.get(ts_url, headers={'user-agent': get_headers()}, timeout=5, allow_redirects=True) + def parse_data(self, m3u8_content): + self.decription = Decryption(self.key) + self.segments = [] + base_url = self.url.rstrip(self.url.split("/")[-1]) + lines = m3u8_content.split('\n') - if req.status_code == 200: - return req.content + for i in range(len(lines)): + line = str(lines[i]) + + if line.startswith("#EXT-X-KEY:"): + x_key_dict = self.decription.decode_ext_x_key(line) + self.decription.parse_key(x_key_dict['IV']) + + if line.startswith("#EXTINF"): + ts_url = lines[i+1] + + if not ts_url.startswith("http"): + ts_url = base_url + ts_url + + logging.debug(f"Add to segment: {ts_url}") + self.segments.append(ts_url) + + def get_info(self): + self.max_retry = 3 + response = requests.get(self.url, headers={'user-agent': get_headers()}) + + if response.ok: + self.parse_data(response.text) + console.log(f"[red]Ts segments find [white]=> [yellow]{len(self.segments)}") else: - retry += 1 - return self.make_req_single_ts_file(ts_url, retry) - - def decrypt_and_save(self, index): - - video_ts_url = self.segments[index] - video_ts_filename = os.path.join(self.temp_folder, f"{index}_v.ts") - logging.debug(f"Download video ts file: {video_ts_url}") - - # Download video or audio ts file - if not os.path.exists(video_ts_filename): # Only for media that not use audio - ts_response = self.make_req_single_ts_file(video_ts_url) - - if ts_response != None: - if self.key and self.iv: - decrypted_data = self.decrypt_ts(ts_response) - with open(video_ts_filename, "wb") as ts_file: - ts_file.write(decrypted_data) - - else: - with open(video_ts_filename, "wb") as ts_file: - ts_file.write(ts_response) - - else: - logging.debug(f"Cant save video ts: {video_ts_url}") - - # Donwload only audio ts file and merge with video - if self.download_audio: - audio_ts_url = self.segments_audio[index] - logging.debug(f"Download audio ts file: {audio_ts_url}") - - audio_ts_filename = os.path.join(self.temp_folder, f"{index}_a.ts") - video_audio_ts_filename = os.path.join(self.temp_folder, f"{index}_v_a.ts") - - if not os.path.exists(video_audio_ts_filename): # Only for media use audio - ts_response = self.make_req_single_ts_file(audio_ts_url) - - if ts_response != None: - if self.key and self.iv: - decrypted_data = self.decrypt_ts(ts_response) - - with open(audio_ts_filename, "wb") as ts_file: - ts_file.write(decrypted_data) - - else: - with open(audio_ts_filename, "wb") as ts_file: - ts_file.write(ts_response) - - # Join ts video and audio - res_merge = merge_ts_files(video_ts_filename, audio_ts_filename, video_audio_ts_filename) - - if res_merge: - os.remove(video_ts_filename) - os.remove(audio_ts_filename) - - # If merge fail, so we have only video and audio, take only video - else: - self.failed_segments.append(index) - os.remove(audio_ts_filename) - - else: - logging.debug(f"Cant save audio ts: {audio_ts_url}") - - def download_and_save_ts(self): - - try: - - if USE_MULTI_THREAD: - with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor: - list(tqdm(executor.map(self.decrypt_and_save, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download")) - else: - for index in range(len(self.segments)): - console.log(f"[yellow]Download: [red]{index}") - self.decrypt_and_save(index) - - - if len(self.failed_segments) > 0: - console.log(f"[red]Segment ts: {self.failed_segments}, cant use audio") - - except KeyboardInterrupt: - console.log("[yellow]Interruption detected. Exiting program.") + console.log("[red]Wrong m3u8 url") sys.exit(0) - def join_ts_files(self): + def get_req_ts(self, ts_url): + + try: + response = requests.get(ts_url, headers={'user-agent': get_headers()}) + + if response.status_code == 200: + return response.content + else: + print(f"Failed: {ts_url}, with error: {response.status_code}") + self.segments.remove(ts_url) + logging.error(f"Failed download: {ts_url}") + return None + + except Exception as e: + print(f"Failed: {ts_url}, with error: {e}") + self.segments.remove(ts_url) + logging.error(f"Failed download: {ts_url}") + return None + + def save_ts(self, index): + ts_url = self.segments[index] + ts_filename = os.path.join(self.temp_folder, f"{index}.ts") + + if not os.path.exists(ts_filename): + ts_content = self.get_req_ts(ts_url) + + if ts_content is not None: + with open(ts_filename, "wb") as ts_file: + if self.key and self.decription.iv: + decrypted_data = self.decription.decrypt_ts(ts_content) + ts_file.write(decrypted_data) + else: + ts_file.write(ts_content) + + def download_ts(self): + with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor: + list(tqdm(executor.map(self.save_ts, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download")) + + def join(self, output_filename): current_dir = os.path.dirname(os.path.realpath(__file__)) file_list_path = os.path.join(current_dir, 'file_list.txt') - # Make sort by number ts_files = [f for f in os.listdir(self.temp_folder) if f.endswith(".ts")] def extract_number(file_name): return int(''.join(filter(str.isdigit, file_name))) - ts_files.sort(key=extract_number) with open(file_list_path, 'w') as f: @@ -249,24 +141,66 @@ class M3U8Downloader: console.log("[cyan]Start join all file") try: - ( - ffmpeg.input(file_list_path, format='concat', safe=0).output(self.output_filename, c='copy', loglevel='quiet').run() - ) + ffmpeg.input(file_list_path, format='concat', safe=0).output(output_filename, c='copy', loglevel='quiet').run() except ffmpeg.Error as e: console.log(f"[red]Error saving MP4: {e.stdout}") sys.exit(0) - time.sleep(1) console.log(f"[cyan]Clean ...") os.remove(file_list_path) shutil.rmtree("tmp", ignore_errors=True) +class M3U8Downloader: + def __init__(self, m3u8_url, m3u8_audio = None, key=None, output_filename="output.mp4"): + self.m3u8_url = m3u8_url + self.m3u8_audio = m3u8_audio + self.key = key + self.video_path = output_filename + self.audio_path = os.path.join("videos", "audio.mp4") + + def start(self): + video_m3u8 = M3U8(self.m3u8_url, self.key) + console.log("[green]Download video ts") + video_m3u8.get_info() + video_m3u8.download_ts() + video_m3u8.join(self.video_path) + print_duration_table(self.video_path) + print("\n") + + if self.m3u8_audio != None: + audio_m3u8 = M3U8(self.m3u8_audio, self.key) + console.log("[green]Download audio ts") + audio_m3u8.get_info() + audio_m3u8.download_ts() + audio_m3u8.join(self.audio_path) + print_duration_table(self.audio_path) + + self.join_audio() + + def join_audio(self): + command = [ + "ffmpeg", + "-y", + "-i", self.video_path, + "-i", self.audio_path, + "-c", "copy", + "-map", "0:v:0", + "-map", "1:a:0", + "-shortest", + "-strict", "experimental", + self.video_path + ".mp4" + ] + + try: + out = subprocess.run(command, check=True, stderr=subprocess.PIPE) + console.print("\n[green]Merge completed successfully.") + except subprocess.CalledProcessError as e: + print("ffmpeg output:", e.stderr.decode()) + + os.remove(self.video_path) + os.remove(self.audio_path) # [ main function ] def dw_m3u8(url, audio_url=None, key=None, output_filename="output.mp4"): - - downloader = M3U8Downloader(url, audio_url, key, output_filename) - - downloader.download_m3u8() - downloader.download_and_save_ts() - downloader.join_ts_files() + print("\n") + M3U8Downloader(url, audio_url, key, output_filename).start() diff --git a/Src/Util/FFmpeg/util.py b/Src/Util/FFmpeg/util.py index f8f4a7d..9829c15 100644 --- a/Src/Util/FFmpeg/util.py +++ b/Src/Util/FFmpeg/util.py @@ -1,31 +1,28 @@ -# 4.01.2023 +# 31.01.24 # Class import -from Src.Util.Helper.console import console, config_logger +from Src.Util.Helper.console import console -# General import -import ffmpeg, subprocess, logging - -def there_is_audio(ts_file_path): - probe = ffmpeg.probe(ts_file_path) - return any(stream['codec_type'] == 'audio' for stream in probe['streams']) - -def merge_ts_files(video_path, audio_path, output_path): - input_video = ffmpeg.input(video_path) - input_audio = ffmpeg.input(audio_path) - logging.debug(f"Merge video ts: {input_video}, with audio ts: {input_audio}, to: {output_path}") - - ffmpeg_command = ffmpeg.output(input_video, input_audio, output_path, - format='mpegts', - acodec='copy', - vcodec='copy', - loglevel='quiet', - ).compile() +# Import +import ffmpeg +def get_video_duration(file_path): try: - subprocess.run(ffmpeg_command, check=True, stderr=subprocess.PIPE) - logging.debug(f"Saving: {output_path}") - return True - except subprocess.CalledProcessError as e: - logging.error(f"Can save: {output_path}") - return False \ No newline at end of file + probe = ffmpeg.probe(file_path) + duration = float(probe['format']['duration']) + return duration + except ffmpeg.Error as e: + print(f"Error: {e.stderr}") + return None + +def format_duration(seconds): + hours, remainder = divmod(seconds, 3600) + minutes, seconds = divmod(remainder, 60) + return int(hours), int(minutes), int(seconds) + +def print_duration_table(file_path): + video_duration = get_video_duration(file_path) + + if video_duration is not None: + hours, minutes, seconds = format_duration(video_duration) + console.log(f"[cyan]Info [green]'{file_path}': [purple]{int(hours)}[red]h [purple]{int(minutes)}[red]m [purple]{int(seconds)}[red]s") \ No newline at end of file diff --git a/run.py b/run.py index 580f7ee..89eefb2 100644 --- a/run.py +++ b/run.py @@ -1,4 +1,4 @@ -# 10.12.23 +# 10.12.23 -> 31.01.24 # Class import import Src.Api.page as Page @@ -24,7 +24,6 @@ def initialize(): except Exception as e: console.print(f"[blue]Req github [white]=> [red]Failed: {e}") - console.print(f"[blue]Find system [white]=> [red]{sys.platform}") check_ffmpeg() print("\n")