Fix mp4 downloader

This commit is contained in:
Lovi 2025-02-07 10:16:36 +01:00
parent 438adb2f4c
commit 26ff364e19
9 changed files with 97 additions and 162 deletions

View File

@ -352,7 +352,6 @@ The configuration file is divided into several main sections:
```json
{
"tqdm_delay": 0.01,
"tqdm_use_large_bar": true,
"default_video_workser": 12,
"default_audio_workser": 12,
"cleanup_tmp_folder": true
@ -360,7 +359,6 @@ The configuration file is divided into several main sections:
```
- `tqdm_delay`: Delay between progress bar updates
- `tqdm_use_large_bar`: Use detailed progress bar (recommended for desktop) set to false for mobile
- `default_video_workser`: Number of threads for video download
* Can be changed from terminal with `--default_video_worker <number>`
<br/><br/>
@ -371,9 +369,6 @@ The configuration file is divided into several main sections:
- `cleanup_tmp_folder`: Remove temporary .ts files after download
> [!IMPORTANT]
> Set `tqdm_use_large_bar` to `false` when using Termux or terminals with limited width to prevent network monitoring issues
### Language Settings
@ -532,17 +527,17 @@ python3 telegram_bot.py
# Website Status
| Website | Status |
|:-------------------|:------:|
| [1337xx](https://1337xx.to/) | ✅ |
| [AltadefinizioneGratis](https://altadefinizionegratis.pro/) | ✅ |
| [AnimeUnity](https://animeunity.so/) | ✅ |
| [Ilcorsaronero](https://ilcorsaronero.link/) | ✅ |
| [CB01New](https://cb01new.media/) | ✅ |
| [DDLStreamItaly](https://ddlstreamitaly.co/) | ✅ |
| [GuardaSerie](https://guardaserie.meme/) | ✅ |
| [MostraGuarda](https://mostraguarda.stream/) | ✅ |
| [StreamingCommunity](https://streamingcommunity.paris/) | ✅ |
| Website | Status | Command |
|:-------------------|:------:|:--------:|
| [1337xx](https://1337xx.to/) | ✅ | -133 |
| [AltadefinizioneGratis](https://altadefinizionegratis.pro/) | ✅ | -ALT |
| [AnimeUnity](https://animeunity.so/) | ✅ | -ANI |
| [Ilcorsaronero](https://ilcorsaronero.link/) | ✅ | `-ILC` |
| [CB01New](https://cb01new.media/) | ✅ | -CB0 |
| [DDLStreamItaly](https://ddlstreamitaly.co/) | ✅ | -DDL |
| [GuardaSerie](https://guardaserie.meme/) | ✅ | -GUA |
| [MostraGuarda](https://mostraguarda.stream/) | ✅ | -MOS |
| [StreamingCommunity](https://streamingcommunity.paris/) | ✅ | -STR |
# Tutorials
@ -554,7 +549,8 @@ python3 telegram_bot.py
# To Do
- Finish [website API](https://github.com/Arrowar/StreamingCommunity/tree/test_gui_1)
- To Finish [website API](https://github.com/Arrowar/StreamingCommunity/tree/test_gui_1)
- To finish [website API 2](https://github.com/hydrosh/StreamingCommunity/tree/test_gui_1)
# Contributing

View File

@ -1,7 +1,6 @@
# 11.03.24
import os
import sys
import logging
@ -78,16 +77,12 @@ def download_episode(index_select: int, scrape_serie: ScrapeSerieAnime, video_so
os_manager.create_path(mp4_path)
# Start downloading
r_proc = MP4_downloader(
path, kill_handler = MP4_downloader(
url=str(video_source.src_mp4).strip(),
path=os.path.join(mp4_path, title_name)
)
if r_proc != None:
console.print("[green]Result: ")
console.print(r_proc)
return os.path.join(mp4_path, title_name)
return path, kill_handler
else:
logging.error(f"Skip index: {index_select} cant find info with api.")
@ -101,6 +96,8 @@ def download_series(select_title: MediaItem):
- tv_id (int): The ID of the TV series.
- tv_name (str): The name of the TV series.
"""
start_message()
if TELEGRAM_BOT:
bot = get_bot_instance()
@ -134,15 +131,16 @@ def download_series(select_title: MediaItem):
# Download selected episodes
if len(list_episode_select) == 1 and last_command != "*":
download_episode(list_episode_select[0]-1, scrape_serie, video_source)[0]
path, _ = download_episode(list_episode_select[0]-1, scrape_serie, video_source)[0]
return path
# Download all other episodes selecter
else:
kill_handler=bool(False)
kill_handler = False
for i_episode in list_episode_select:
if kill_handler:
break
kill_handler= download_episode(i_episode-1, scrape_serie, video_source)[1]
break
_, kill_handler = download_episode(i_episode-1, scrape_serie, video_source)
if TELEGRAM_BOT:
bot.send_message(f"Finito di scaricare tutte le serie e episodi", None)

View File

@ -38,7 +38,7 @@ from .proxyes import main_test_proxy
# Config
TQDM_DELAY_WORKER = config_manager.get_float('M3U8_DOWNLOAD', 'tqdm_delay')
TQDM_USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
REQUEST_MAX_RETRY = config_manager.get_int('REQUESTS', 'max_retry')
REQUEST_VERIFY = False
THERE_IS_PROXY_LIST = os_manager.check_file("list_proxy.txt")
@ -378,7 +378,7 @@ class M3U8_Segments:
"""
Generate platform-appropriate progress bar format.
"""
if not TQDM_USE_LARGE_BAR:
if not USE_LARGE_BAR:
return (
f"{Colors.YELLOW}Proc{Colors.WHITE}: "
f"{Colors.RED}{{percentage:.2f}}% "

View File

@ -5,6 +5,7 @@ import re
import sys
import signal
import logging
from functools import partial
# External libraries
@ -32,15 +33,17 @@ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
# Config
GET_ONLY_LINK = config_manager.get_bool('M3U8_PARSER', 'get_only_link')
TQDM_USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout')
TELEGRAM_BOT = config_manager.get_bool('DEFAULT', 'telegram_bot')
#Ending constant
KILL_HANDLER = bool(False)
def signal_handler(signum, frame, kill_handler):
"""Signal handler for graceful interruption"""
kill_handler[0] = True
print("\nReceived interrupt signal. Completing current download...")
def MP4_downloader(url: str, path: str, referer: str = None, headers_: dict = None):
"""
@ -88,32 +91,32 @@ def MP4_downloader(url: str, path: str, referer: str = None, headers_: dict = No
logging.error(f"Error preparing headers: {header_err}")
console.print(f"[bold red]Error preparing headers: {header_err}[/bold red]")
return None
temp_path = f"{path}.temp"
kill_handler = [False] # Using list for mutable state
original_handler = signal.signal(signal.SIGINT, partial(signal_handler, kill_handler=kill_handler))
try:
# Create a custom transport that bypasses SSL verification
transport = httpx.HTTPTransport(
verify=False,
verify=False,
http2=True
)
# Download with streaming and progress tracking
with httpx.Client(transport=transport, timeout=httpx.Timeout(60.0)) as client:
with httpx.Client(transport=transport, timeout=httpx.Timeout(60)) as client:
with client.stream("GET", url, headers=headers, timeout=REQUEST_TIMEOUT) as response:
response.raise_for_status()
# Get total file size
total = int(response.headers.get('content-length', 0))
# Handle empty streams
if total == 0:
console.print("[bold red]No video stream found.[/bold red]")
return None
# Create progress bar
progress_bar = tqdm(
total=total,
ascii='░▒█',
bar_format=f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): "
ascii='░▒█',
bar_format=f"{Colors.YELLOW}[MP4]{Colors.WHITE}: "
f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ "
f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] "
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| "
@ -124,42 +127,30 @@ def MP4_downloader(url: str, path: str, referer: str = None, headers_: dict = No
mininterval=0.05
)
# Ensure directory exists
os.makedirs(os.path.dirname(path), exist_ok=True)
downloaded = 0
with open(temp_path, 'wb') as file, progress_bar as bar:
try:
for chunk in response.iter_bytes(chunk_size=1024):
if kill_handler[0]:
console.print("\n[bold yellow]Interrupting download...[/bold yellow]")
return None, True
if chunk:
size = file.write(chunk)
downloaded += size
bar.update(size)
def signal_handler(*args):
"""
Signal handler for SIGINT
except KeyboardInterrupt:
console.print("\n[bold red]Download interrupted by user.[/bold red]")
if os.path.exists(temp_path):
os.remove(temp_path)
return None, True
Parameters:
- args (tuple): The signal arguments (to prevent errors).
"""
if(downloaded<total/2):
raise KeyboardInterrupt
else:
console.print("[bold green]Download almost completed, will exit next[/bold green]")
print("KILL_HANDLER: ", KILL_HANDLER)
# Rename temp file to final file
if os.path.exists(temp_path):
os.rename(temp_path, path)
# Download file
with open(path, 'wb') as file, progress_bar as bar:
downloaded = 0
#Test check stop download
#atexit.register(quit_gracefully)
for chunk in response.iter_bytes(chunk_size=1024):
signal.signal(signal.SIGINT,signal_handler)
if chunk:
size = file.write(chunk)
downloaded += size
bar.update(size)
# Optional: Add a check to stop download if needed
# if downloaded > MAX_DOWNLOAD_SIZE:
# break
# Post-download processing
if os.path.exists(path) and os.path.getsize(path) > 0:
if os.path.exists(path):
console.print(Panel(
f"[bold green]Download completed![/bold green]\n"
f"[cyan]File size: [bold red]{internet_manager.format_file_size(os.path.getsize(path))}[/bold red]\n"
@ -172,18 +163,19 @@ def MP4_downloader(url: str, path: str, referer: str = None, headers_: dict = No
message = f"Download completato\nDimensione: {internet_manager.format_file_size(os.path.getsize(path))}\nDurata: {print_duration_table(path, description=False, return_string=True)}\nTitolo: {os.path.basename(path.replace('.mp4', ''))}"
clean_message = re.sub(r'\[[a-zA-Z]+\]', '', message)
bot.send_message(clean_message, None)
return path
return path, kill_handler[0]
else:
console.print("[bold red]Download failed or file is empty.[/bold red]")
return None
return None, kill_handler[0]
except Exception as e:
logging.error(f"Unexpected error during download: {e}")
logging.error(f"Unexpected error: {e}")
console.print(f"[bold red]Unexpected Error: {e}[/bold red]")
return None
if os.path.exists(temp_path):
os.remove(temp_path)
return None, kill_handler[0]
except KeyboardInterrupt:
console.print("[bold red]Download stopped by user.[/bold red]")
return None
finally:
# Restore original signal handler
signal.signal(signal.SIGINT, original_handler)

View File

@ -29,7 +29,7 @@ PASSWORD = str(config_manager.get_dict('DEFAULT', 'config_qbit_tor')['pass'])
# Config
TQDM_USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout')
@ -163,12 +163,13 @@ class TOR_downloader:
try:
# Custom bar for mobile and pc
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
bar_format = (
f"{Colors.YELLOW}[TOR] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): "
f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ "
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]"
)
else:
bar_format = (
f"{Colors.YELLOW}Proc{Colors.WHITE}: "
@ -216,7 +217,7 @@ class TOR_downloader:
average_internet_unit = average_internet_str.split(' ')[1]
# Update the progress bar's postfix
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
pbar.set_postfix_str(
f"{Colors.WHITE}[ {Colors.GREEN}{downloaded_size} {Colors.WHITE}< {Colors.GREEN}{total_size} {Colors.RED}{total_size_unit} "
f"{Colors.WHITE}| {Colors.CYAN}{average_internet} {Colors.RED}{average_internet_unit}"

View File

@ -30,7 +30,7 @@ FFMPEG_DEFAULT_PRESET = config_manager.get("M3U8_CONVERSION", "default_preset")
# Variable
TQDM_USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
FFMPEG_PATH = os_summary.ffmpeg_path
@ -100,7 +100,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
subprocess.run(ffmpeg_cmd, check=True)
else:
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join video")
print()
@ -196,7 +196,7 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
subprocess.run(ffmpeg_cmd, check=True)
else:
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join audio")
print()
@ -251,7 +251,7 @@ def join_subtitle(video_path: str, subtitles_list: List[Dict[str, str]], out_pat
subprocess.run(ffmpeg_cmd, check=True)
else:
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
capture_ffmpeg_real_time(ffmpeg_cmd, "[cyan]Join subtitle")
print()

View File

@ -18,7 +18,7 @@ from StreamingCommunity.Util.os import internet_manager
# Variable
TQDM_USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
USE_LARGE_BAR = not ("android" in sys.platform or "ios" in sys.platform)
class M3U8_Ts_Estimator:
@ -36,14 +36,14 @@ class M3U8_Ts_Estimator:
self.lock = threading.Lock()
self.speed = {"upload": "N/A", "download": "N/A"}
if TQDM_USE_LARGE_BAR:
logging.debug("TQDM_USE_LARGE_BAR is True, starting speed capture thread")
if USE_LARGE_BAR:
logging.debug("USE_LARGE_BAR is True, starting speed capture thread")
self.speed_thread = threading.Thread(target=self.capture_speed)
self.speed_thread.daemon = True
self.speed_thread.start()
else:
logging.debug("TQDM_USE_LARGE_BAR is False, speed capture thread not started")
logging.debug("USE_LARGE_BAR is False, speed capture thread not started")
def add_ts_file(self, size: int, size_download: int, duration: float):
"""Add a file size to the list of file sizes."""
@ -114,7 +114,7 @@ class M3U8_Ts_Estimator:
units_file_downloaded = downloaded_file_size_str.split(' ')[1]
units_file_total_size = file_total_size.split(' ')[1]
if TQDM_USE_LARGE_BAR:
if USE_LARGE_BAR:
speed_data = self.speed['download'].split(" ")
if len(speed_data) >= 2:

View File

@ -3,7 +3,6 @@
import os
import sys
import json
import httpx
import logging
from typing import Any, List

View File

@ -9,9 +9,7 @@ sys.path.append(src_path)
# Other
import time
import glob
import logging
import importlib
import json
from rich.console import Console
@ -25,61 +23,10 @@ console = Console()
README_PATH = "README.md"
def load_site_names():
modules = []
site_names = {}
# Traverse the Api directory
api_dir = os.path.join(os.path.dirname(__file__), '..', 'StreamingCommunity', 'Api', 'Site')
init_files = glob.glob(os.path.join(api_dir, '*', '__init__.py'))
# Retrieve modules and their indices
for init_file in init_files:
# Get folder name as module name
module_name = os.path.basename(os.path.dirname(init_file))
logging.info(f"Load module name: {module_name}")
try:
# Dynamically import the module
mod = importlib.import_module(f'StreamingCommunity.Api.Site.{module_name}')
# Get 'indice' from the module
indice = getattr(mod, 'indice', 0)
is_deprecate = bool(getattr(mod, '_deprecate', True))
use_for = getattr(mod, '_useFor', 'other')
if not is_deprecate:
modules.append((module_name, indice, use_for))
except Exception as e:
console.print(f"[red]Failed to import module {module_name}: {str(e)}")
# Sort modules by 'indice'
modules.sort(key=lambda x: x[1])
# Load SITE_NAME from each module in the sorted order
for module_name, _, use_for in modules:
# Construct a unique alias for the module
module_alias = f'{module_name}'
logging.info(f"Module alias: {module_alias}")
try:
# Dynamically import the module
mod = importlib.import_module(f'StreamingCommunity.Api.Site.{module_name}')
# Get the SITE_NAME variable from the module
site_name = getattr(mod, 'SITE_NAME', None)
if site_name:
# Add the SITE_NAME to the dictionary
site_names[module_alias] = (site_name, use_for)
except Exception as e:
console.print(f"[red]Failed to load SITE_NAME from module {module_name}: {str(e)}")
return site_names
def get_config():
with open("config.json", "r", encoding="utf-8") as file:
return json.load(file)
def update_readme(site_names, domain_to_use):
if not os.path.exists(README_PATH):
@ -97,10 +44,12 @@ def update_readme(site_names, domain_to_use):
alias = f"{site_name.lower()}"
if alias in site_names:
command = f"-{site_name[:3].upper()}"
if site_name == "animeunity":
updated_line = f"| [{site_name}](https://www.{alias}.{domain_to_use}/) | ✅ |\n"
updated_line = f"| [{site_name}](https://www.{alias}.{domain_to_use}/) | ✅ | {command} |\n"
else:
updated_line = f"| [{site_name}](https://{alias}.{domain_to_use}/) | ✅ |\n"
updated_line = f"| [{site_name}](https://{alias}.{domain_to_use}/) | ✅ | {command} |\n"
updated_lines.append(updated_line)
continue
@ -110,17 +59,17 @@ def update_readme(site_names, domain_to_use):
with open(README_PATH, "w", encoding="utf-8") as file:
file.writelines(updated_lines)
if __name__ == "__main__":
site_names = load_site_names()
for alias, (site_name, use_for) in site_names.items():
original_domain = config_manager.get_list("SITE", alias)['domain']
for site_name, data in get_config()['SITE'].items():
original_domain = config_manager.get_dict("SITE", site_name)['domain']
if site_name != "ilcorsaronero":
if site_name == "animeunity":
domain_to_use, _ = search_domain(site_name=site_name, base_url=f"https://www.{site_name}.{original_domain}", get_first=True)
domain_to_use, _ = search_domain(site_name, f"https://www.{site_name}.{original_domain}", True)
else:
domain_to_use, _ = search_domain(site_name=site_name, base_url=f"https://{site_name}.{original_domain}", get_first=True)
domain_to_use, _ = search_domain(site_name, f"https://{site_name}.{original_domain}", True)
update_readme(alias, domain_to_use)
update_readme(site_name, domain_to_use)
print("\n------------------------------------")
time.sleep(1)