Merge pull request #29 from Helper0x/fix_sync_audio

Fix audio sync

Src/Api/film.py

@@ -103,5 +103,4 @@ def main_dw_film(id_film, title_name, domain):
     if m3u8_url_audio != None:
         console.print("[blue]Use m3u8 audio => [red]True")
     
-    print("\n")
     dw_m3u8(m3u8_url, m3u8_url_audio, m3u8_key, mp4_path)

Src/Api/page.py

@@ -17,6 +17,7 @@ def domain_version():
         site_req = requests.get(f"https://streamingcommunity.{req_repo.json()['domain']}/", headers={'user-agent': get_headers()}).text
         soup = BeautifulSoup(site_req, "lxml")
         version = json.loads(soup.find("div", {"id": "app"}).get("data-page"))['version']
+        console.print(f"[blue]Rules [white]=> [red].{req_repo.json()['domain']}")
 
         return req_repo.json()['domain'], version
     

Src/Api/tv.py

@@ -130,7 +130,6 @@ def dw_single_ep(tv_id, eps, index_ep_select, domain, token, tv_name, season_sel
     if m3u8_url_audio != None:
         console.print("[blue]Use m3u8 audio => [red]True")
 
-    print("\n")
     dw_m3u8(m3u8_url, m3u8_url_audio, m3u8_key, mp4_path)
     
 def main_dw_tv(tv_id, tv_name, version, domain):

Src/Util/FFmpeg/m3u8.py

@@ -1,245 +1,137 @@
 # 5.01.24 -> 7.01.24
 
-
 # Class import
 from Src.Util.Helper.console import console, config_logger
 from Src.Util.Helper.headers import get_headers
-from Src.Util.FFmpeg.util import there_is_audio, merge_ts_files
+from Src.Util.FFmpeg.util import print_duration_table
-
 
 # Import
-import requests, re,  os, ffmpeg, time, sys, warnings, logging, shutil
+import requests, re,  os, ffmpeg, time, sys, warnings, logging, shutil, subprocess
 from tqdm.rich import tqdm
 from concurrent.futures import ThreadPoolExecutor
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.backends import default_backend
 
-
 # Disable warning
 from tqdm import TqdmExperimentalWarning
 warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
 warnings.filterwarnings("ignore", category=UserWarning, module="cryptography")
 
-
 # Variable
 os.makedirs("videos", exist_ok=True)
 DOWNLOAD_WORKERS = 30
-USE_MULTI_THREAD = True
 
 
 # [ main class ]
-class M3U8Downloader:
+class Decryption():
-
+    def __init__(self, key):
-    def __init__(self, m3u8_url, m3u8_audio = None, key=None, output_filename="output.mp4"):
-        self.m3u8_url = m3u8_url
-        self.m3u8_audio = m3u8_audio
-        self.key = key
-        self.output_filename = output_filename
-        
-        self.segments = []
-        self.segments_audio = []
         self.iv = None
-        if key != None: self.key = bytes.fromhex(key)
+        self.key = key
-
-        self.temp_folder = "tmp"
-        os.makedirs(self.temp_folder, exist_ok=True)
-        self.download_audio = False
-        self.max_retry = 3
-        self.failed_segments = []
-
-        # Debug
-        logging.debug(m3u8_url)
-        logging.debug(m3u8_audio)
-        logging.debug(self.key)
 
     def decode_ext_x_key(self, key_str):
+        logging.debug(f"String to decode: {key_str}")
         key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:")
         v_list = re.findall(r"[^,=]+", key_str)
         key_map = {v_list[i]: v_list[i+1] for i in range(0, len(v_list), 2)}
-
+        logging.debug(f"Output string: {key_map}")
-        return key_map # URI | METHOD | IV
+        return key_map
-
+    
     def parse_key(self, raw_iv):
         self.iv = bytes.fromhex(raw_iv.replace("0x", ""))
 
-    def parse_m3u8(self, m3u8_content):
-        if self.m3u8_audio != None: 
-            m3u8_audio_line = str(requests.get(self.m3u8_audio).content).split("\\n")
-
-        m3u8_base_url = self.m3u8_url.rstrip(self.m3u8_url.split("/")[-1])
-        lines = m3u8_content.split('\n')
-
-        for i in range(len(lines)):
-            line = str(lines[i])
-
-            if line.startswith("#EXT-X-KEY:"):
-                x_key_dict = self.decode_ext_x_key(line)
-                self.parse_key(x_key_dict['IV'])
-
-            if line.startswith("#EXTINF"):
-                ts_url = lines[i+1]
-
-                if not ts_url.startswith("http"):
-                    ts_url = m3u8_base_url + ts_url
-
-                self.segments.append(ts_url)
-
-                if self.m3u8_audio != None: 
-                    self.segments_audio.append(m3u8_audio_line[i+1])
-
-        console.log(f"[cyan]Find: {len(self.segments)} ts video file to download")
-
-        # Check video ts segment
-        if len(self.segments) == 0:
-            console.log("[red]No ts files to download")
-            sys.exit(0)
-
-        # Check audio ts segment
-        if self.m3u8_audio != None:
-            console.log(f"[cyan]Find: {len(self.segments_audio)} ts audio file to download")
-
-            if len(self.segments_audio) == 0:
-                console.log("[red]No ts audio files to download")
-                sys.exit(0)
-
-    def download_m3u8(self):
-        response = requests.get(self.m3u8_url, headers={'user-agent': get_headers()})
-
-        if response.ok:
-            m3u8_content = response.text
-            self.parse_m3u8(m3u8_content)
-        else:
-            console.log("[red]Wrong m3u8 url")
-            sys.exit(0)
-
-        if self.m3u8_audio != None:
-
-            # Check there is audio in first ts file
-            path_test_ts_file = os.path.join(self.temp_folder, "ts_test.ts")
-
-            if self.key and self.iv:
-                open(path_test_ts_file, "wb").write(self.decrypt_ts(requests.get(self.segments[0]).content))
-            else:
-                open(path_test_ts_file, "wb").write(requests.get(self.segments[0]).content)
-
-            if not there_is_audio(path_test_ts_file):
-                self.download_audio = True
-
-            os.remove(path_test_ts_file)
-
     def decrypt_ts(self, encrypted_data):
         cipher = Cipher(algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend())
         decryptor = cipher.decryptor()
         decrypted_data = decryptor.update(encrypted_data) + decryptor.finalize()
 
         return decrypted_data
-    
-    def make_req_single_ts_file(self, ts_url, retry=0):
 
-        if retry == self.max_retry:
+class M3U8():
-            console.log(f"[red]Failed download: {ts_url}")
+    def __init__(self, url, key=None):
-            self.segments.remove(ts_url)
+        self.url = url
-            logging.error(f"Failed: {ts_url}")
+        self.key = bytes.fromhex(key) if key is not None else key
-            return None
+        self.temp_folder = "tmp"
+        os.makedirs(self.temp_folder, exist_ok=True)
 
-        req = requests.get(ts_url, headers={'user-agent': get_headers()}, timeout=5, allow_redirects=True)
+    def parse_data(self, m3u8_content):
+        self.decription = Decryption(self.key)
+        self.segments = []
+        base_url = self.url.rstrip(self.url.split("/")[-1])
+        lines = m3u8_content.split('\n')
 
-        if req.status_code == 200:
+        for i in range(len(lines)):
-            return req.content
+            line = str(lines[i])
+
+            if line.startswith("#EXT-X-KEY:"):
+                x_key_dict = self.decription.decode_ext_x_key(line)
+                self.decription.parse_key(x_key_dict['IV'])
+
+            if line.startswith("#EXTINF"):
+                ts_url = lines[i+1]
+
+                if not ts_url.startswith("http"):
+                    ts_url = base_url + ts_url
+
+                logging.debug(f"Add to segment: {ts_url}")
+                self.segments.append(ts_url)
+
+    def get_info(self):
+        self.max_retry = 3
+        response = requests.get(self.url, headers={'user-agent': get_headers()})
+
+        if response.ok:
+            self.parse_data(response.text)
+            console.log(f"[red]Ts segments find [white]=> [yellow]{len(self.segments)}")
         else:
-            retry += 1
+            console.log("[red]Wrong m3u8 url")
-            return self.make_req_single_ts_file(ts_url, retry)
-
-    def decrypt_and_save(self, index):
-        
-        video_ts_url = self.segments[index]
-        video_ts_filename = os.path.join(self.temp_folder, f"{index}_v.ts")
-        logging.debug(f"Download video ts file: {video_ts_url}")
-
-        # Download video or audio ts file 
-        if not os.path.exists(video_ts_filename):   # Only for media that not use audio
-            ts_response = self.make_req_single_ts_file(video_ts_url)
-
-            if ts_response != None:
-                if self.key and self.iv:
-                    decrypted_data = self.decrypt_ts(ts_response)
-                    with open(video_ts_filename, "wb") as ts_file:
-                        ts_file.write(decrypted_data)
-
-                else:
-                    with open(video_ts_filename, "wb") as ts_file:
-                        ts_file.write(ts_response)
-            
-            else:
-                logging.debug(f"Cant save video ts: {video_ts_url}")
-
-        # Donwload only audio ts file and merge with video
-        if self.download_audio: 
-            audio_ts_url = self.segments_audio[index]
-            logging.debug(f"Download audio ts file: {audio_ts_url}")
-
-            audio_ts_filename = os.path.join(self.temp_folder, f"{index}_a.ts")
-            video_audio_ts_filename = os.path.join(self.temp_folder, f"{index}_v_a.ts")
-
-            if not os.path.exists(video_audio_ts_filename):  # Only for media use audio
-                ts_response = self.make_req_single_ts_file(audio_ts_url)
-
-                if ts_response != None:
-                    if self.key and self.iv:
-                        decrypted_data = self.decrypt_ts(ts_response)
-
-                        with open(audio_ts_filename, "wb") as ts_file:
-                            ts_file.write(decrypted_data)
-
-                    else:
-                        with open(audio_ts_filename, "wb") as ts_file:
-                            ts_file.write(ts_response)
-                    
-                    # Join ts video and audio
-                    res_merge = merge_ts_files(video_ts_filename, audio_ts_filename, video_audio_ts_filename)
-
-                    if res_merge:
-                        os.remove(video_ts_filename)
-                        os.remove(audio_ts_filename)
-
-                    # If merge fail, so we have only video and audio, take only video
-                    else:
-                        self.failed_segments.append(index)
-                        os.remove(audio_ts_filename)
-
-                else:
-                    logging.debug(f"Cant save audio ts: {audio_ts_url}")
-                         
-    def download_and_save_ts(self):
-
-        try:
-                
-            if USE_MULTI_THREAD:
-                with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor:
-                    list(tqdm(executor.map(self.decrypt_and_save, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download"))
-            else:
-                for index in range(len(self.segments)):
-                    console.log(f"[yellow]Download: [red]{index}")
-                    self.decrypt_and_save(index)
-                
-
-            if len(self.failed_segments) > 0:
-                console.log(f"[red]Segment ts: {self.failed_segments}, cant use audio")
-
-        except KeyboardInterrupt:
-            console.log("[yellow]Interruption detected. Exiting program.")
             sys.exit(0)
 
-    def join_ts_files(self):
+    def get_req_ts(self, ts_url):
+
+        try:
+            response = requests.get(ts_url, headers={'user-agent': get_headers()})
+
+            if response.status_code == 200:
+                return response.content
+            else:
+                print(f"Failed: {ts_url}, with error: {response.status_code}")
+                self.segments.remove(ts_url)
+                logging.error(f"Failed download: {ts_url}")
+                return None
+            
+        except Exception as e:
+            print(f"Failed: {ts_url}, with error: {e}")
+            self.segments.remove(ts_url)
+            logging.error(f"Failed download: {ts_url}")
+            return None
+        
+    def save_ts(self, index):
+        ts_url = self.segments[index]
+        ts_filename = os.path.join(self.temp_folder, f"{index}.ts")
+
+        if not os.path.exists(ts_filename):
+            ts_content = self.get_req_ts(ts_url)
+
+            if ts_content is not None:
+                with open(ts_filename, "wb") as ts_file:
+                    if self.key and self.decription.iv:
+                        decrypted_data = self.decription.decrypt_ts(ts_content)
+                        ts_file.write(decrypted_data)
+                    else:
+                        ts_file.write(ts_content)
+          
+    def download_ts(self):
+        with ThreadPoolExecutor(max_workers=DOWNLOAD_WORKERS) as executor:
+            list(tqdm(executor.map(self.save_ts, range(len(self.segments)) ), total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download"))
+
+    def join(self, output_filename):
 
         current_dir = os.path.dirname(os.path.realpath(__file__))
         file_list_path = os.path.join(current_dir, 'file_list.txt')
 
-        # Make sort by number
         ts_files = [f for f in os.listdir(self.temp_folder) if f.endswith(".ts")]
         def extract_number(file_name):
             return int(''.join(filter(str.isdigit, file_name)))
-        
         ts_files.sort(key=extract_number)
 
         with open(file_list_path, 'w') as f:
@@ -249,24 +141,66 @@ class M3U8Downloader:
 
         console.log("[cyan]Start join all file")
         try:
-            (
+            ffmpeg.input(file_list_path, format='concat', safe=0).output(output_filename, c='copy', loglevel='quiet').run()
-                ffmpeg.input(file_list_path, format='concat', safe=0).output(self.output_filename, c='copy', loglevel='quiet').run()
-            )
         except ffmpeg.Error as e:
             console.log(f"[red]Error saving MP4: {e.stdout}")
             sys.exit(0)
             
-        time.sleep(1)
         console.log(f"[cyan]Clean ...")
         os.remove(file_list_path)
         shutil.rmtree("tmp", ignore_errors=True)
 
+class M3U8Downloader:
+    def __init__(self, m3u8_url, m3u8_audio = None, key=None, output_filename="output.mp4"):
+        self.m3u8_url = m3u8_url
+        self.m3u8_audio = m3u8_audio
+        self.key = key
+        self.video_path = output_filename
+        self.audio_path = os.path.join("videos", "audio.mp4")
+
+    def start(self):
+        video_m3u8 = M3U8(self.m3u8_url, self.key)
+        console.log("[green]Download video ts")
+        video_m3u8.get_info()
+        video_m3u8.download_ts()
+        video_m3u8.join(self.video_path)
+        print_duration_table(self.video_path)
+        print("\n")
+
+        if self.m3u8_audio != None:
+            audio_m3u8 = M3U8(self.m3u8_audio, self.key)
+            console.log("[green]Download audio ts")
+            audio_m3u8.get_info()
+            audio_m3u8.download_ts()
+            audio_m3u8.join(self.audio_path)
+            print_duration_table(self.audio_path)
+
+            self.join_audio()
+
+    def join_audio(self):
+        command = [
+            "ffmpeg",
+            "-y",
+            "-i", self.video_path,
+            "-i", self.audio_path,
+            "-c", "copy",
+            "-map", "0:v:0",
+            "-map", "1:a:0",
+            "-shortest",
+            "-strict", "experimental",
+            self.video_path + ".mp4"
+        ]
+
+        try:
+            out = subprocess.run(command, check=True, stderr=subprocess.PIPE)
+            console.print("\n[green]Merge completed successfully.")
+        except subprocess.CalledProcessError as e:
+            print("ffmpeg output:", e.stderr.decode())
+
+        os.remove(self.video_path)
+        os.remove(self.audio_path)
 
 # [ main function ]
 def dw_m3u8(url, audio_url=None, key=None, output_filename="output.mp4"):
-
+    print("\n")
-    downloader = M3U8Downloader(url, audio_url, key, output_filename)
+    M3U8Downloader(url, audio_url, key, output_filename).start()
-
-    downloader.download_m3u8()
-    downloader.download_and_save_ts()
-    downloader.join_ts_files()

Src/Util/FFmpeg/util.py

@@ -1,31 +1,28 @@
-# 4.01.2023
+# 31.01.24
 
 # Class import
-from Src.Util.Helper.console import console, config_logger
+from Src.Util.Helper.console import console
 
-# General import
+# Import
-import ffmpeg, subprocess, logging
+import ffmpeg 
-
-def there_is_audio(ts_file_path):
-    probe = ffmpeg.probe(ts_file_path)
-    return any(stream['codec_type'] == 'audio' for stream in probe['streams'])
-
-def merge_ts_files(video_path, audio_path, output_path):
-    input_video = ffmpeg.input(video_path)
-    input_audio = ffmpeg.input(audio_path)
-    logging.debug(f"Merge video ts: {input_video}, with audio ts: {input_audio}, to: {output_path}")
-
-    ffmpeg_command = ffmpeg.output(input_video, input_audio, output_path, 
-        format='mpegts', 
-        acodec='copy', 
-        vcodec='copy', 
-        loglevel='quiet',
-        ).compile()
 
+def get_video_duration(file_path):
     try:
-        subprocess.run(ffmpeg_command, check=True, stderr=subprocess.PIPE)
+        probe = ffmpeg.probe(file_path)
-        logging.debug(f"Saving: {output_path}")
+        duration = float(probe['format']['duration'])
-        return True
+        return duration
-    except subprocess.CalledProcessError as e:
+    except ffmpeg.Error as e:
-        logging.error(f"Can save: {output_path}")
+        print(f"Error: {e.stderr}")
-        return False
+        return None
+
+def format_duration(seconds):
+    hours, remainder = divmod(seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return int(hours), int(minutes), int(seconds)
+
+def print_duration_table(file_path):
+    video_duration = get_video_duration(file_path)
+
+    if video_duration is not None:
+        hours, minutes, seconds = format_duration(video_duration)
+        console.log(f"[cyan]Info [green]'{file_path}': [purple]{int(hours)}[red]h [purple]{int(minutes)}[red]m [purple]{int(seconds)}[red]s")

run.py

@@ -1,4 +1,4 @@
-# 10.12.23
+# 10.12.23 -> 31.01.24
 
 # Class import
 import Src.Api.page as Page
@@ -24,7 +24,6 @@ def initialize():
     except Exception as e:
         console.print(f"[blue]Req github [white]=> [red]Failed: {e}")
 
-    console.print(f"[blue]Find system [white]=> [red]{sys.platform}")
     check_ffmpeg()
     print("\n")