...

README.md

@@ -1,2 +1,33 @@
-# StreamingCommunity_api
-Script to download film from streaming community without selenium
+<p align="center">
+	<img src="Stream/assets/min_logo.png" style="max-width: 55%;" alt="video working" />
+</p>
+
+## Streaming community downloader
+
+Script to download film from streaming community without selenium
+
+<p align="center">
+	<img src="Stream/assets/run.gif" style="max-width: 55%;" alt="video working" />
+</p>
+
+## Requirement
+
+* python [3.9](https://www.python.org/downloads/release/python-390/)
+
+## Installation
+
+* requirement for library of python
+
+```bash
+	pip install -r requirements.txt
+```
+
+## Run
+
+```bash
+	python run.py
+```
+
+## Authors
+
+- [@Ghost6446](https://www.github.com/Ghost6446)

Stream/api/film.py

@@ -0,0 +1,51 @@
+# 3.12.23 -> 10.12.23
+
+# Class import
+from Stream.util.headers import get_headers
+from Stream.util.console import console
+from Stream.util.m3u8 import dw_m3u8
+
+# General import
+import requests, sys, re, json
+from bs4 import BeautifulSoup
+
+# [func]
+def get_iframe(id_title, domain):
+    req_iframe = requests.get(url = f"https://streamingcommunity.{domain}/iframe/{id_title}", headers = {
+        "User-agent": get_headers()
+    })
+
+    url_embed = BeautifulSoup(req_iframe.text, "lxml").find("iframe").get("src")
+    req_embed = requests.get(url_embed, headers = {"User-agent": get_headers()}).text
+    return BeautifulSoup(req_embed, "lxml").find("body").find("script").text
+    
+def parse_content(embed_content):
+
+    # Parse parameter from req embed content
+    win_video = re.search(r"window.video = {.*}", str(embed_content)).group()
+    win_param = re.search(r"params: {[\s\S]*}", str(embed_content)).group()
+
+    # Parse parameter to make read for json
+    json_win_video = "{"+win_video.split("{")[1].split("}")[0]+"}"
+    json_win_param = "{"+win_param.split("{")[1].split("}")[0].replace("\n", "").replace(" ", "") + "}"
+    json_win_param = json_win_param.replace(",}", "}").replace("'", '"')
+    return json.loads(json_win_video), json.loads(json_win_param)
+
+def get_m3u8_url(json_win_video, json_win_param):
+    return f"https://vixcloud.co/playlist/{json_win_video['id']}?type=video&rendition=720p&token={json_win_param['token720p']}&expires={json_win_param['expires']}"
+
+def get_m3u8_key(json_win_video, json_win_param, title_name):
+    req_key = requests.get('https://vixcloud.co/storage/enc.key', headers={
+        'referer': f'https://vixcloud.co/embed/{json_win_video["id"]}?token={json_win_param["token720p"]}&title={title_name.replace(" ", "+")}&referer=1&expires={json_win_param["expires"]}',
+    }).content
+
+    return "".join(["{:02x}".format(c) for c in req_key])
+
+def main_dw_film(id_film, title_name, domain):
+
+    embed_content = get_iframe(id_film, domain)
+    json_win_video, json_win_param = parse_content(embed_content)
+    m3u8_url = get_m3u8_url(json_win_video, json_win_param)
+    m3u8_key = get_m3u8_key(json_win_video, json_win_param, title_name)
+    
+    dw_m3u8(m3u8_url, requests.get(m3u8_url, headers={"User-agent": get_headers()}).text, "", m3u8_key, str(title_name).replace("+", " ").replace(",", "") + ".mp4")

Stream/api/page.py

@@ -0,0 +1,37 @@
+# 10.12.23
+
+# Class import
+from Stream.util.headers import get_headers
+from Stream.util.console import console
+
+# General import
+import requests, json, sys
+from bs4 import BeautifulSoup
+
+def get_version(domain):
+
+    try:
+        r = requests.get(f'https://streamingcommunity.{domain}/', headers={
+            'Authority': f'streamingcommunity.{domain}',
+            'User-Agent': get_headers(),
+        })
+        soup = BeautifulSoup(r.text, "lxml")
+        info_data_page = soup.find("div", {'id': 'app'}).attrs["data-page"]
+
+        return json.loads(info_data_page)['version']
+    
+    except:
+
+        console.log("[red]UPDATE DOMANIN")
+        sys.exit(0)
+    
+
+def search(title_search, domain):
+
+    title_search = str(title_search).replace(" ", "+")
+    r = requests.get(
+        url = f"https://streamingcommunity.{domain}/api/search?q={title_search}",
+        headers = {"User-agent": get_headers()}
+    )
+
+    return [{'name': title['name'], 'type': title['type'], 'id': title['id']} for title in r.json()['data']]

Stream/api/tv.py

@@ -0,0 +1,86 @@
+# 3.12.23 -> 10.12.23
+
+# Class import
+from Stream.util.headers import get_headers
+from Stream.util.console import console, msg, console_print
+from Stream.util.m3u8 import dw_m3u8
+
+# General import
+import requests, sys, re, json
+from bs4 import BeautifulSoup
+
+# [func]
+def get_token(id_tv, domain):
+    session = requests.Session()
+    session.get(f"https://streamingcommunity.{domain}/watch/{id_tv}")
+    return session.cookies['XSRF-TOKEN']
+
+def get_info_tv(id_film, title_name, site_version, domain):
+    r = requests.get(f"https://streamingcommunity.{domain}/titles/{id_film}-{title_name}", headers={
+        'X-Inertia': 'true', 
+        'X-Inertia-Version': site_version,
+        'User-Agent': get_headers()
+    })
+
+    return r.json()['props']['title']['seasons_count']
+
+def get_info_season(tv_id, tv_name, domain, version, token, n_stagione):
+    r = requests.get(f'https://streamingcommunity.broker/titles/{tv_id}-{tv_name}/stagione-{n_stagione}', headers={
+        'authority': f'streamingcommunity.{domain}', 'referer': f'https://streamingcommunity.broker/titles/{tv_id}-{tv_name}',
+        'user-agent': get_headers(), 'x-inertia': 'true', 'x-inertia-version': version, 'x-xsrf-token': token,
+    })
+
+    return [{'id': ep['id'], 'n': ep['number'], 'name': ep['name']} for ep in r.json()['props']['loadedSeason']['episodes']]
+
+def get_iframe(tv_id, ep_id, domain, token):
+    r = requests.get(f'https://streamingcommunity.broker/iframe/{tv_id}', params={'episode_id': ep_id, 'next_episode': '1'}, cookies={'XSRF-TOKEN': token}, headers={
+        'referer': f'https://streamingcommunity.{domain}/watch/{tv_id}?e={ep_id}',
+        'user-agent': get_headers()
+    })
+
+    url_embed = BeautifulSoup(r.text, "lxml").find("iframe").get("src")
+    req_embed = requests.get(url_embed, headers = {"User-agent": get_headers()}).text
+    return BeautifulSoup(req_embed, "lxml").find("body").find("script").text
+
+def parse_content(embed_content):
+
+    # Parse parameter from req embed content
+    win_video = re.search(r"window.video = {.*}", str(embed_content)).group()
+    win_param = re.search(r"params: {[\s\S]*}", str(embed_content)).group()
+
+    # Parse parameter to make read for json
+    json_win_video = "{"+win_video.split("{")[1].split("}")[0]+"}"
+    json_win_param = "{"+win_param.split("{")[1].split("}")[0].replace("\n", "").replace(" ", "") + "}"
+    json_win_param = json_win_param.replace(",}", "}").replace("'", '"')
+    return json.loads(json_win_video), json.loads(json_win_param)
+
+def get_m3u8_url(json_win_video, json_win_param):
+    return f"https://vixcloud.co/playlist/{json_win_video['id']}?type=video&rendition=720p&token={json_win_param['token720p']}&expires={json_win_param['expires']}"
+
+def get_m3u8_key_ep(json_win_video, json_win_param, tv_name, n_stagione, n_ep, ep_title):
+    req_key = requests.get('https://vixcloud.co/storage/enc.key', headers={
+        'referer': f'https://vixcloud.co/embed/{json_win_video["id"]}?token={json_win_param["token720p"]}&title={tv_name.replace("-", "+")}&referer=1&expires={json_win_param["expires"]}&description=S{n_stagione}%3AE{n_ep}+{ep_title.replace(" ", "+")}&nextEpisode=1',
+    }).content
+
+    return "".join(["{:02x}".format(c) for c in req_key])
+
+def main_dw_tv(tv_id, tv_name, version, domain):
+
+    token = get_token(tv_id, domain)
+
+    tv_name = str(tv_name.replace('+', '-')).lower()
+    console.log(f"[blue]Season find: [red]{get_info_tv(tv_id, tv_name, version, domain)}")
+    season_select = msg.ask("[green]Insert season number: ")
+
+    eps = get_info_season(tv_id, tv_name, domain, version, token, season_select)
+    for ep in eps:
+        console_print(f"[green]Ep: [blue]{ep['n']} [green]=> [purple]{ep['name']}")
+
+    index_ep_select = int(msg.ask("[green]Insert ep number: ")) - 1
+    embed_content = get_iframe(tv_id, eps[index_ep_select]['id'], domain, token)
+    json_win_video, json_win_param = parse_content(embed_content)
+    m3u8_url = get_m3u8_url(json_win_video, json_win_param)
+    m3u8_key = get_m3u8_key_ep(json_win_video, json_win_param, tv_name, season_select, index_ep_select+1, eps[index_ep_select]['name'])
+    
+    dw_m3u8(m3u8_url, requests.get(m3u8_url, headers={"User-agent": get_headers()}).text, "", m3u8_key, tv_name.replace("+", "_") + "_"+str(season_select)+"__"+str(index_ep_select+1) + ".mp4")
+

Stream/util/console.py

@@ -0,0 +1,11 @@
+# 17.09.2023 -> 3.12.23
+
+# Import
+from rich.console import Console
+from rich.prompt import Prompt
+from rich import print as console_print
+
+# Variable
+msg = Prompt()
+console = Console()
+

Stream/util/headers.py

@@ -0,0 +1,12 @@
+# 3.12.23 -> 10.12.23
+
+# Import
+from random_user_agent.user_agent import UserAgent
+from random_user_agent.params import SoftwareName, OperatingSystem
+
+# [func]
+def get_headers():
+    software_names = [SoftwareName.CHROME.value]
+    operating_systems = [OperatingSystem.WINDOWS.value, OperatingSystem.LINUX.value]   
+    user_agent_rotator = UserAgent(software_names=software_names, operating_systems=operating_systems, limit=10)
+    return user_agent_rotator.get_random_user_agent()

Stream/util/m3u8.py

@@ -0,0 +1,291 @@
+# 4.08.2023 -> 14.09.2023 -> 17.09.2023 -> 3.12.2023
+
+# Import
+import re, os, sys, glob, time, requests, shutil, ffmpeg, subprocess
+from functools import partial
+from multiprocessing.dummy import Pool
+from tqdm.rich import tqdm
+
+# Class import
+#from Stream.util.console import console
+from Stream.util.console import console
+
+# Disable warning
+import warnings
+from tqdm import TqdmExperimentalWarning
+warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
+
+# Variable
+main_out_folder = "videos"
+os.makedirs("videos", exist_ok=True)
+
+# [ decoder ] -> costant = ou_ts
+class Video_Decoder(object):
+
+    iv = ""
+    uri = ""
+    method = ""
+
+    def __init__(self, x_key, uri):
+        self.method = x_key["METHOD"] if "METHOD" in x_key.keys() else ""
+        self.uri = uri
+        self.iv = x_key["IV"].lstrip("0x") if "IV" in x_key.keys() else ""
+
+    def decode_aes_128(self, video_fname: str):
+        frame_name = video_fname.split("\\")[-1].split("-")[0] + ".ts"
+        res_cmd = subprocess.run(["openssl","aes-128-cbc","-d","-in", video_fname,"-out", "ou_ts/"+frame_name,"-nosalt","-iv", self.iv,"-K", self.uri ], capture_output=True)
+
+        res_cmd_str = res_cmd.stderr.decode("utf-8")
+        res_cmd_fix = res_cmd_str.replace("b", "").replace("\n", "").replace("\r", "")
+
+        if "lengthad" in res_cmd_fix:
+            console.log("[red]Wrong m3u8 key or remove key from input !!!")
+            sys.exit(0)
+
+def decode_ext_x_key(key_str: str):
+    key_str = key_str.replace('"', '').lstrip("#EXT-X-KEY:")
+    v_list = re.findall(r"[^,=]+", key_str)
+    key_map = {v_list[i]: v_list[i+1] for i in range(0, len(v_list), 2)}
+    return key_map
+
+
+# [ util ] 
+def save_in_part(folder_ts, merged_mp4, file_extension = ".ts"):
+
+    # Get list of ts file in order
+    os.chdir(folder_ts)
+
+    # Order all ts file
+    try: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"), key=lambda x:float(re.findall("(\d+)", x.split("_")[1])[0]))
+    except: 
+        try: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"), key=lambda x:float(re.findall("(\d+)", x.split("-")[1])[0]))
+        except: ordered_ts_names = sorted(glob.glob(f"*{file_extension}"))
+
+    open("concat.txt", "wb")
+    open("part_list.txt", "wb")
+
+    # Variable for download
+    list_mp4_part = []
+    part = 0
+    start = 0
+    end = 200
+
+    # Create mp4 from start ts to end
+    def save_part_ts(start, end, part):
+        #console.log(f"[blue]Process part [green][[red]{part}[green]]")
+        list_mp4_part.append(f"{part}.mp4")
+
+        with open(f"{part}_concat.txt", "w") as f:
+            for i in range(start, end):
+                f.write(f"file {ordered_ts_names[i]} \n")
+                
+        ffmpeg.input(f"{part}_concat.txt", format='concat', safe=0).output(f"{part}.mp4", c='copy', loglevel="quiet").run()
+        
+    # Save first part
+    save_part_ts(start, end, part)
+
+    # Save all other part
+    for _ in range(start, end):
+
+        # Increment progress ts file
+        start+= 200
+        end += 200
+        part+=1
+
+        # Check if end or not
+        if(end < len(ordered_ts_names)): 
+            save_part_ts(start, end, part)
+        else:
+            save_part_ts(start, len(ordered_ts_names), part)
+            break
+
+    # Merge all part
+    console.log(f"[purple]Merge all: {file_extension} file")
+    with open("part_list.txt", 'w') as f:
+        for mp4_fname in list_mp4_part:
+            f.write(f"file {mp4_fname}\n")
+            
+    ffmpeg.input("part_list.txt", format='concat', safe=0).output(merged_mp4, c='copy', loglevel="quiet").run()
+    
+def download_ts_file(ts_url: str, store_dir: str, headers):
+
+    # Get ts name and folder
+    ts_name = ts_url.split('/')[-1].split("?")[0]
+    ts_dir = store_dir + "/" + ts_name
+
+    # Check if exist
+    if(not os.path.isfile(ts_dir)):
+
+        # Download
+        ts_res = requests.get(ts_url, headers=headers)
+
+        if(ts_res.status_code == 200):
+            with open(ts_dir, 'wb+') as f:
+                f.write(ts_res.content)
+        else:
+            print(f"Failed to download streaming file: {ts_name}.") 
+
+        time.sleep(0.5)
+
+def download_vvt_sub(content, language, folder_id):
+
+    # Get content of vvt
+    url_main_sub = ""
+    vvt = content.split("\n")
+
+    # Find main url or vvt
+    for i in range(len(vvt)):
+        line = str(vvt[i])
+
+        if line.startswith("#EXTINF"):
+            url_main_sub = vvt[i+1]
+            
+    # Save subtitle to main folder out
+    path = os.path.join(main_out_folder, str(folder_id))
+    os.makedirs(path, exist_ok=True)
+    open(os.path.join(path, "sub_"+str(language)+".vtt"), "wb").write(requests.get(url_main_sub).content)
+
+# [ donwload ]
+def dw_m3u8(m3u8_link, m3u8_content, m3u8_headers="", decrypt_key="", merged_mp4="test.mp4"):
+
+    # Reading the m3u8 file
+    m3u8_http_base = m3u8_link.rstrip(m3u8_link.split("/")[-1])
+    m3u8 = m3u8_content.split('\n')
+    ts_url_list = []
+    ts_names = []
+    x_key_dict = dict()
+    is_encryped = False
+
+    # Parsing the content in m3u8 with creation of url_list with url of ts file
+    for i_str in range(len(m3u8)):
+        line_str = m3u8[i_str]
+
+        if "AES-128" in str(line_str):
+            is_encryped = True
+
+        if line_str.startswith("#EXT-X-KEY:"):
+            x_key_dict = decode_ext_x_key(line_str)
+
+        if line_str.startswith("#EXTINF"):
+            ts_url = m3u8[i_str+1]
+            ts_names.append(ts_url.split('/')[-1])
+
+            if not ts_url.startswith("http"):
+                ts_url = m3u8_http_base + ts_url
+
+            ts_url_list.append(ts_url)
+    #console.log(f"[blue]Find [white]=> [red]{len(ts_url_list)}[blue] ts file to download")
+    #console.log(f"[green]Is m3u8 encryped => [red]{is_encryped}")
+
+    if is_encryped and decrypt_key == "": 
+        console.log(f"[red]M3U8 Is encryped")
+        sys.exit(0)
+
+    if is_encryped:
+        #console.log(f"[blue]Use decrypting")
+
+        video_decoder = Video_Decoder(x_key=x_key_dict, uri=decrypt_key)
+        os.makedirs("ou_ts", exist_ok=True)
+
+    #  Using multithreading to download all ts file
+    os.makedirs("temp_ts", exist_ok=True)
+    pool = Pool(15)
+    gen = pool.imap(partial(download_ts_file, store_dir="temp_ts", headers=m3u8_headers), ts_url_list)
+    for _ in tqdm(gen, total=len(ts_url_list), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download"):
+        pass
+    pool.close()
+    pool.join()
+
+    if is_encryped:
+        for ts_fname in tqdm(glob.glob("temp_ts\*.ts"), desc="[yellow]Decoding"):
+            video_decoder.decode_aes_128(ts_fname)
+
+        # Start to merge all *.ts files
+        save_in_part("ou_ts", merged_mp4)
+    else:
+        save_in_part("temp_ts", merged_mp4)
+
+
+    # Clean temp file
+    os.chdir("..")
+    console.log("[green]Clean")
+
+    if is_encryped: 
+        shutil.move("ou_ts\\"+merged_mp4 , main_out_folder+"\\")
+    else: 
+        shutil.move("temp_ts\\"+merged_mp4 , main_out_folder+"\\")
+
+    shutil.rmtree("ou_ts", ignore_errors=True)
+    shutil.rmtree("temp_ts", ignore_errors=True)
+
+def dw_aac(m3u8_link, m3u8_content, m3u8_headers, merged_mp3):
+
+    # Reading the m3u8 file
+    url_base = m3u8_link.rstrip(m3u8_link.split("/")[-1])
+    m3u8 = m3u8_content.split('\n')
+    ts_url_list = []
+    ts_names = []
+
+    # Parsing the content in m3u8 with creation of url_list with url of ts file
+    for i in range(len(m3u8)):
+        line = m3u8[i]
+
+        if line.startswith("#EXTINF"):
+            ts_url = m3u8[i+1]
+            ts_names.append(ts_url.split('/')[-1])
+
+            if not ts_url.startswith("http"):
+                ts_url = url_base + ts_url
+
+            ts_url_list.append(ts_url)
+    console.log(f"[blue]Find [white]=> [red]{len(ts_url_list)}[blue] ts file to download")
+
+    #  Using multithreading to download all ts file
+    os.makedirs("temp_ts", exist_ok=True)
+    pool = Pool(15)
+    gen = pool.imap(partial(download_ts_file, store_dir="temp_ts", headers=m3u8_headers), ts_url_list)
+    for _ in tqdm(gen, total=len(ts_url_list), unit="bytes", unit_scale=True, unit_divisor=1024, desc="[yellow]Download"):
+        pass
+    pool.close()
+    pool.join()
+
+    save_in_part("temp_ts", merged_mp3, file_extension=".aac")
+
+    # Clean temp file
+    os.chdir("..")
+    console.log("[green]Clean")
+    shutil.move("temp_ts\\"+merged_mp3 , ".")
+
+    shutil.rmtree("ou_ts", ignore_errors=True)
+    shutil.rmtree("temp_ts", ignore_errors=True)
+
+def dw_vvt_sub(url, headers, folder_id) -> (None):
+
+    print(url, headers, folder_id)
+
+    # Get content of m3u8 vvt
+    req = requests.get(url, headers=headers)
+    vvts = req.text.split('\n')
+    vvt_data = []
+
+    # Parsing the content in m3u8 of vvt with creation of url_list with url and name of language
+    for line in vvts:
+        line = line.split(",")
+        if line[0] == "#EXT-X-MEDIA:TYPE=SUBTITLES":
+
+            vvt_data.append({
+                'language': line[2].split("=")[1].replace('"', ""),
+                'url': line[-1].split("URI=")[1].replace('"', "")
+            })
+
+    
+    # Check array is not empty
+    if len(vvt_data) > 0:
+
+        # Download all subtitle
+        for i in range(len(vvts)):
+            console.log(f"[blue]Download [red]sub => [green]{vvt_data[i]['language']}")
+            download_vvt_sub(requests.get(vvt_data[i]['url']).text, vvt_data[i]['language'], folder_id)
+
+    else:
+        console.log("[red]Cant find info of subtitle [SKIP]")

Stream/util/message.py

@@ -0,0 +1,20 @@
+# 3.12.23
+
+# Import
+from Stream.util.console import console
+
+# [Function]
+def msg_start():
+
+    msg = """
+        _____ _                            _                                               _ _         
+        / ____| |                          (_)                                             (_) |        
+        | (___ | |_ _ __ ___  __ _ _ __ ___  _ _ __   __ _    ___ ___  _ __ ___  _   _ _ __  _| |_ _   _ 
+        \___ \| __| '__/ _ \/ _` | '_ ` _ \| | '_ \ / _` |  / __/ _ \| '_ ` _ \| | | | '_ \| | __| | | |
+        ____) | |_| | |  __/ (_| | | | | | | | | | | (_| | | (_| (_) | | | | | | |_| | | | | | |_| |_| |
+        |_____/ \__|_|  \___|\__,_|_| |_| |_|_|_| |_|\__, |  \___\___/|_| |_| |_|\__,_|_| |_|_|\__|\__, |
+                                                    __/ |                                         __/ |
+                                                    |___/                                         |___/ 
+            """
+
+    console.log(f"[purple]{msg}")

requirements.txt

@@ -0,0 +1,7 @@
+requests
+bs4 
+lxml
+tqdm
+rich
+random-user-agent
+ffmpeg-python

run.py

@@ -0,0 +1,32 @@
+# 10.12.23
+
+# Class import
+import Stream.api.page as Page
+from Stream.util.message import msg_start
+from Stream.util.console import console, msg, console_print
+from Stream.api.film import main_dw_film as download_film
+from Stream.api.tv import main_dw_tv as download_tv
+
+domain = "cz"
+site_version = Page.get_version(domain)
+
+def main():
+    msg_start()
+
+    film_search = msg.ask("[blue]Insert film to search: ").strip()
+    db_title = Page.search(film_search, domain)
+
+    for i in range(len(db_title)):
+        console_print(f"[yellow]{i} [white]-> [green]{db_title[i]['name']} [white]- [cyan]{db_title[i]['type']}")
+    index_select = int(msg.ask("[blue]Index to download: "))
+
+    if db_title[index_select]['type'] == "movie":
+        console.log(f"[green]Movie select: {db_title[index_select]['name']}")
+        download_film(db_title[index_select]['id'], db_title[index_select]['name'].replace(" ", "+"), domain)
+
+    else:
+        console.log(f"[green]Tv select: {db_title[index_select]['name']}")
+        download_tv(db_title[index_select]['id'], db_title[index_select]['name'].replace(" ", "+"), site_version, domain)
+
+if __name__ == '__main__':
+    main()