change m3u8 class

README.md

@@ -13,7 +13,6 @@ Script to download film from streaming community without selenium, select movie
 ## Requirement
 * python [3.9](https://www.python.org/downloads/release/python-390/)
 * ffmpeg [win](https://www.gyan.dev/ffmpeg/builds/)
-* open ssl [win](https://slproweb.com/products/Win32OpenSSL.html)
 
 ## Installation library
 ```bash
@@ -27,7 +26,8 @@ python run.py
 
 ## Features
 - Search for movies.
-- Search and download TV series episodes.
+- Search and download TV series episode.
+- Add audio if missing.
 
 ## Authors
 

Stream/api/film.py

@@ -6,7 +6,7 @@ from Stream.util.m3u8 import dw_m3u8
 from Stream.util.util import convert_utf8_name
 
 # General import
-import requests, sys, re, json
+import requests, os, re, json
 from bs4 import BeautifulSoup
 
 # [func]
@@ -51,4 +51,5 @@ def main_dw_film(id_film, title_name, domain):
     m3u8_url = get_m3u8_url(json_win_video, json_win_param)
     m3u8_key = get_m3u8_key(json_win_video, json_win_param, title_name)
     
-    dw_m3u8(m3u8_url, requests.get(m3u8_url, headers={"User-agent": get_headers()}).text, "", m3u8_key, lower_title_name.replace("+", " ").replace(",", "") + ".mp4")
+    path_film = os.path.join("videos", lower_title_name.replace("+", " ").replace(",", "") + ".mp4")
+    dw_m3u8(m3u8_url, m3u8_key, path_film)

Stream/api/tv.py

@@ -101,14 +101,15 @@ def main_dw_tv(tv_id, tv_name, version, domain):
     base_path_mp4 = os.path.join("videos", mp4_format)
     base_audio_path = os.path.join("videos", mp4_format + "_audio.mp4")
 
-    dw_m3u8(m3u8_url, requests.get(m3u8_url, headers={"User-agent": get_headers()}).text, "", m3u8_key, mp4_format)
+    dw_m3u8(m3u8_url, m3u8_key, base_path_mp4)
 
     if not check_audio_presence(base_path_mp4):
-        console.log("[red]Audio is not present, start download (Use all CPU)")
-        m3u8_url_audio = get_m3u8_audio(json_win_video, json_win_param, tv_name, season_select, index_ep_select+1, eps[index_ep_select]['name'])
-        dw_m3u8(m3u8_url_audio, requests.get(m3u8_url_audio, headers={"User-agent": get_headers()}).text, "", m3u8_key, "audio.mp4")
         
-        temp_audio_path = os.path.join("videos", "audio.mp4")
+        console.print("[red]Audio is not present, start download (Use all CPU)")
-        join_audio_to_video(temp_audio_path, base_path_mp4, base_audio_path)
+        m3u8_url_audio = get_m3u8_audio(json_win_video, json_win_param, tv_name, season_select, index_ep_select+1, eps[index_ep_select]['name'])
-        os.remove(temp_audio_path)
+
+        dw_m3u8(m3u8_url_audio, m3u8_key, base_audio_path)
+
+        join_audio_to_video(base_audio_path, base_path_mp4, base_audio_path)
+        os.remove(base_audio_path)
         os.remove(base_path_mp4)

Stream/upload/__version__.py

@@ -1,5 +1,5 @@
 __title__ = 'Streaming_community'
-__version__ = 'v0.6.0'
+__version__ = 'v0.6.1'
 __author__ = 'Ghost6446'
 __description__ = 'A command-line program to download film'
 __license__ = 'MIT License'

Stream/util/m3u8.py

@@ -1,190 +1,136 @@
-# 4.08.2023 -> 14.09.2023 -> 17.09.2023 -> 3.12.2023
+# 5.01.24
 
 # Import
-import re, os, sys, glob, time, requests, shutil, ffmpeg, subprocess
+import requests, re,  os, ffmpeg, shutil
-from functools import partial
-from multiprocessing.dummy import Pool
 from tqdm.rich import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+from cryptography.hazmat.backends import default_backend
 import moviepy.editor as mp
 
 # Class import
 from Stream.util.console import console
-
+from Stream.util.headers import get_headers
-# Disable warning
-import warnings
-from tqdm import TqdmExperimentalWarning
-warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
-
 
 # Variable
-main_out_folder = "videos"
+os.makedirs("videos", exist_ok=True)
-os.makedirs(main_out_folder, exist_ok=True)
-path_out_without_key = "temp_ts"
-path_out_with_key = "out_ts"
 
 
-# [ decoder ]
+# [ main class ]
-def decode_aes_128(path_video_frame, decript_key, x_key):
+class M3U8Downloader:
-    frame_name = path_video_frame.split("\\")[-1].split("-")[0] + ".ts"
-    iv = x_key["IV"].lstrip("0x") if "IV" in x_key.keys() else ""
 
-    out = subprocess.run([
+    def __init__(self, m3u8_url, key=None, output_filename="output.mp4"):
-        "openssl",
+        self.m3u8_url = m3u8_url
-        "aes-128-cbc", "-d",
+        self.key = key
-        "-in", path_video_frame,
+        self.output_filename = output_filename
-        "-out", os.path.join(path_out_with_key, frame_name),
-        "-nosalt","-iv", iv,
-        "-K", decript_key 
-    ], capture_output=True)
         
-def decode_ext_x_key(key_str: str):
+        self.segments = []
-    key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:")
+        self.iv = None
-    v_list = re.findall(r"[^,=]+", key_str)
+        self.key = bytes.fromhex(key)
-    key_map = {v_list[i]: v_list[i+1] for i in range(0, len(v_list), 2)}
+
-    return key_map
+        self.temp_folder = "tmp"
+        os.makedirs(self.temp_folder, exist_ok=True)
+
+    def decode_ext_x_key(self, key_str):
+        key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:")
+        v_list = re.findall(r"[^,=]+", key_str)
+        key_map = {v_list[i]: v_list[i+1] for i in range(0, len(v_list), 2)}
+
+        return key_map # URI | METHOD | IV
+
+    def parse_key(self, raw_iv):
+        self.iv = bytes.fromhex(raw_iv.replace("0x", ""))
+
+    def parse_m3u8(self, m3u8_content):
+        m3u8_base_url = self.m3u8_url.rstrip(self.m3u8_url.split("/")[-1])
+        lines = m3u8_content.split('\n')
+
+        for i in range(len(lines)):
+            line = str(lines[i])
+
+            if line.startswith("#EXT-X-KEY:"):
+                x_key_dict = self.decode_ext_x_key(line)
+                self.parse_key(x_key_dict['IV'])
+
+            if line.startswith("#EXTINF"):
+                ts_url = lines[i+1]
+
+                if not ts_url.startswith("http"):
+                    ts_url = m3u8_base_url + ts_url
+                self.segments.append(ts_url)
+
+        console.print(f"[cyan]Find: {len(self.segments)} ts file to download")
+
+    def download_m3u8(self):
+        response = requests.get(self.m3u8_url, headers={'user-agent': get_headers()})
+
+        if response.ok:
+            m3u8_content = response.text
+            self.parse_m3u8(m3u8_content)
+
+    def decrypt_ts(self, encrypted_data):
+        cipher = Cipher(algorithms.AES(self.key), modes.CBC(self.iv), backend=default_backend())
+        decryptor = cipher.decryptor()
+        decrypted_data = decryptor.update(encrypted_data) + decryptor.finalize()
+
+        return decrypted_data
+    
+    def decrypt_and_save(self, args):
+        ts_url, index = args
+        ts_filename = os.path.join(self.temp_folder, f"{index}.ts")
+
+        if not os.path.exists(ts_filename):
+            ts_response = requests.get(ts_url, headers={'user-agent': get_headers()}).content
+
+            if self.key and self.iv:
+                decrypted_data = self.decrypt_ts(ts_response)
+                with open(ts_filename, "wb") as ts_file:
+                    ts_file.write(decrypted_data)
+
+            else:
+                with open(ts_filename, "wb") as ts_file:
+                    ts_file.write(ts_response)
+
+    def download_and_save_ts(self):
+        with ThreadPoolExecutor(max_workers=30) as executor:
+
+            list(tqdm(executor.map(self.decrypt_and_save, zip(self.segments, range(len(self.segments)))),
+                total=len(self.segments), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download"))
+            
+    def join_ts_files(self):
+
+        current_dir = os.path.dirname(os.path.realpath(__file__))
+        file_list_path = os.path.join(current_dir, 'file_list.txt')
+
+        ts_files = [f for f in os.listdir(self.temp_folder) if f.endswith(".ts")]
+        ts_files.sort()
+
+        with open(file_list_path, 'w') as f:
+            for ts_file in ts_files:
+                relative_path = os.path.relpath(os.path.join(self.temp_folder, ts_file), current_dir)
+                f.write(f"file '{relative_path}'\n")
+
+        console.print("[cyan]Start join all file")
+        try:
+            (
+                ffmpeg.input(file_list_path, format='concat', safe=0).output(self.output_filename, c="copy", loglevel="quiet").run()
+            )
+            console.print(f"[cyan]Clean ...")
+        except ffmpeg.Error as e:
+            print(f"Errore durante il salvataggio del file MP4: {e}")
+        finally:
+            os.remove(file_list_path)
+            shutil.rmtree("tmp", ignore_errors=True)
 
 
-# [ util ] 
+# [ function ]
-def save_in_part(folder_ts, merged_mp4, file_extension = ".ts"):
+def dw_m3u8(url, key=None, output_filename="output.mp4"):
 
-    # Get list of ts file in order
+    downloader = M3U8Downloader(url, key, output_filename)
-    os.chdir(folder_ts)
 
-    # Order all ts file
+    downloader.download_m3u8()
-    try: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"), key=lambda x:float(re.findall("(\d+)", x.split("_")[1])[0]))
+    downloader.download_and_save_ts()
-    except: 
+    downloader.join_ts_files()
-        try: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"), key=lambda x:float(re.findall("(\d+)", x.split("-")[1])[0]))
-        except: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"))
-
-    open("concat.txt", "wb")
-    open("part_list.txt", "wb")
-
-    # Variable for download
-    list_mp4_part = []
-    part = 0
-    start = 0
-    end = 200
-
-    # Create mp4 from start ts to end
-    def save_part_ts(start, end, part):
-        list_mp4_part.append(f"{part}.mp4")
-
-        with open(f"{part}_concat.txt", "w") as f:
-            for i in range(start, end):
-                f.write(f"file {ordered_ts_names[i]} \n")
-
-        ffmpeg.input(f"{part}_concat.txt", format='concat', safe=0).output(f"{part}.mp4", c='copy', loglevel="quiet").run()
-
-
-    # Save first part
-    save_part_ts(start, end, part)
-
-    # Save all other part
-    for _ in range(start, end):
-
-        # Increment progress ts file
-        start+= 200
-        end += 200
-        part+=1
-
-        # Check if end or not
-        if(end < len(ordered_ts_names)): 
-            save_part_ts(start, end, part)
-        else:
-            save_part_ts(start, len(ordered_ts_names), part)
-            break
-
-    # Merge all part
-    console.log(f"[purple]Merge all: {file_extension} file")
-    with open("part_list.txt", 'w') as f:
-        for mp4_fname in list_mp4_part:
-            f.write(f"file {mp4_fname}\n")
-            
-    ffmpeg.input("part_list.txt", format='concat', safe=0).output(merged_mp4, c='copy', loglevel="quiet").run()
-    
-def download_ts_file(ts_url: str, store_dir: str, headers):
-
-    # Get ts name and folder
-    ts_name = ts_url.split('/')[-1].split("?")[0]
-    ts_dir = os.path.join(store_dir, ts_name)
-
-    if(not os.path.isfile(ts_dir)):
-        ts_res = requests.get(ts_url, headers=headers)
-
-        if(ts_res.status_code == 200):
-            with open(ts_dir, 'wb+') as f:
-                f.write(ts_res.content)
-        else:
-            print(f"Failed to download streaming file: {ts_name}.") 
-
-        time.sleep(0.5)
-
-
-# [ donwload ]
-def dw_m3u8(m3u8_link, m3u8_content, m3u8_headers="", decrypt_key="", merged_mp4="test.mp4"):
-
-    # Reading the m3u8 file
-    m3u8_base_url = m3u8_link.rstrip(m3u8_link.split("/")[-1])
-    m3u8 = m3u8_content.split('\n')
-
-    ts_url_list = []
-    ts_names = []
-    x_key_dict = dict()
-
-    is_encryped = False
-    os.makedirs(path_out_without_key, exist_ok=True)
-    os.makedirs(path_out_with_key, exist_ok=True)
-
-    # Parsing the content in m3u8 with creation of url_list with url of ts file
-    for i in range(len(m3u8)):
-        if "AES-128" in str(m3u8[i]):
-            is_encryped = True
-
-        if m3u8[i].startswith("#EXT-X-KEY:"):
-            x_key_dict = decode_ext_x_key(m3u8[i])
-
-        if m3u8[i].startswith("#EXTINF"):
-            ts_url = m3u8[i+1]
-            ts_names.append(ts_url.split('/')[-1])
-
-            if not ts_url.startswith("http"):
-                ts_url = m3u8_base_url + ts_url
-
-            ts_url_list.append(ts_url)
-    console.log(f"[blue]Find [white]=> [red]{len(ts_url_list)}[blue] ts file to download")
-
-    if is_encryped and decrypt_key == "": 
-        console.log(f"[red]M3U8 Is encryped")
-        sys.exit(0)
-
-
-    #  Using multithreading to download all ts file
-    pool = Pool(15)
-    gen = pool.imap(partial(download_ts_file, store_dir=path_out_without_key, headers=m3u8_headers), ts_url_list)
-    for _ in tqdm(gen, total=len(ts_url_list), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download m3u8"):
-        pass
-    pool.close()
-    pool.join()
-
-    # Merge all ts 
-    if is_encryped:
-        for ts_fname in tqdm(glob.glob(f"{path_out_without_key}/*.ts"), desc="[yellow]Decoding m3u8"):
-            decode_aes_128(ts_fname, decrypt_key, x_key_dict)
-        save_in_part(path_out_with_key, merged_mp4)
-    else:
-        save_in_part(path_out_without_key, merged_mp4)
-
-    # Clean temp file
-    os.chdir("..")
-    console.log("[green]Clean")
-
-    # Move mp4 file to main folder
-    if is_encryped: shutil.move(path_out_with_key+"/"+merged_mp4 , main_out_folder+"/")
-    else: shutil.move(path_out_without_key+"/"+merged_mp4 , main_out_folder+"/")
-
-    # Remove folder out_ts and temp_ts
-    shutil.rmtree(path_out_with_key, ignore_errors=True)
-    shutil.rmtree(path_out_without_key, ignore_errors=True)
 
 def join_audio_to_video(audio_path, video_path, out_path):
 
@@ -197,4 +143,3 @@ def join_audio_to_video(audio_path, video_path, out_path):
 
     # Join all
     final.write_videofile(out_path)
-

requirements.txt

@@ -6,3 +6,4 @@ rich
 random-user-agent
 ffmpeg-python
 moviepy
+cryptography

run.py

@@ -21,7 +21,7 @@ def main():
     main_update()
     console.print(f"[blue]Find system [white]=> [red]{sys.platform} \n")
     
-    film_search = msg.ask("[blue]Insert word to search in all site: ").strip()
+    film_search = msg.ask("\n[blue]Insert word to search in all site: ").strip()
     db_title = Page.search(film_search, domain)
 
     for i in range(len(db_title)):
@@ -29,12 +29,14 @@ def main():
     index_select = int(msg.ask("\n[blue]Index to download: "))
 
     if db_title[index_select]['type'] == "movie":
-        console.log(f"[green]Movie select: {db_title[index_select]['name']}")
+        console.print(f"[green]\nMovie select: {db_title[index_select]['name']}")
         download_film(db_title[index_select]['id'], db_title[index_select]['name'].replace(" ", "+"), domain)
 
     else:
-        console.log(f"[green]Tv select: {db_title[index_select]['name']}")
+        console.print(f"[green]\nTv select: {db_title[index_select]['name']}")
         download_tv(db_title[index_select]['id'], db_title[index_select]['name'].replace(" ", "+"), site_version, domain)
 
+    console.print("\n[red]Done")
+
 if __name__ == '__main__':
     main()