Fix 401 Unauthorized, add retry, fix parser gpu

This commit is contained in:
Lovi 2024-10-20 18:29:37 +02:00
parent 6566dd6b3d
commit ce0dc7ad78
15 changed files with 248 additions and 233 deletions

View File

@ -0,0 +1,35 @@
# 19.10.24
import os
import sys
def execute_search(info):
"""
Dynamically imports and executes a specified function from a module defined in the info dictionary.
Parameters:
info (dict): A dictionary containing the function name, folder, and module information.
"""
# Step 1: Define the project path using the folder from the info dictionary
project_path = os.path.dirname(info['folder']) # Get the base path for the project
# Step 2: Add the project path to sys.path
if project_path not in sys.path:
sys.path.append(project_path)
# Attempt to import the specified function from the module
try:
# Construct the import statement dynamically
module_path = f"Src.Api.{info['folder_base']}"
exec(f"from {module_path} import {info['function']}")
# Call the specified function
eval(info['function'])() # Calls the search function
except ModuleNotFoundError as e:
print(f"ModuleNotFoundError: {e}")
except ImportError as e:
print(f"ImportError: {e}")
except Exception as e:
print(f"An error occurred: {e}")

View File

@ -1,5 +1,6 @@
# 19.06.24 # 19.06.24
from .site import get_select_title from .site import get_select_title
from .Util.recall_search import execute_search
from .Util.get_domain import search_domain from .Util.get_domain import search_domain
from .Util.manage_ep import manage_selection, map_episode_title, validate_episode_selection, validate_selection from .Util.manage_ep import manage_selection, map_episode_title, validate_episode_selection, validate_selection

View File

@ -8,7 +8,6 @@ import logging
# Internal utilities # Internal utilities
from Src.Util.message import start_message from Src.Util.message import start_message
from Src.Util.console import console from Src.Util.console import console
from Src.Util.os import create_folder, can_create_file, remove_special_characters
from Src.Lib.Downloader import HLS_Downloader from Src.Lib.Downloader import HLS_Downloader
@ -38,16 +37,8 @@ def download_film(select_title: MediaItem):
video_source = VideoSource(select_title.url) video_source = VideoSource(select_title.url)
# Define output path # Define output path
mp4_name = remove_special_characters(select_title.name) + ".mp4" mp4_name = select_title.name + ".mp4"
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.name)) mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name)
# Ensure the folder path exists
create_folder(mp4_path)
# Check if the MP4 file can be created
if not can_create_file(mp4_name):
logging.error("Invalid mp4 name.")
sys.exit(0)
# Get m3u8 master playlist # Get m3u8 master playlist
master_playlist = video_source.get_playlist() master_playlist = video_source.get_playlist()

View File

@ -8,7 +8,6 @@ import logging
# Internal utilities # Internal utilities
from Src.Util.console import console from Src.Util.console import console
from Src.Util.message import start_message from Src.Util.message import start_message
from Src.Util.os import create_folder, can_create_file, remove_special_characters
from Src.Lib.Downloader import HLS_Downloader from Src.Lib.Downloader import HLS_Downloader
@ -37,17 +36,8 @@ def download_film(select_title: MediaItem):
video_source = VideoSource(select_title.url) video_source = VideoSource(select_title.url)
# Define output path # Define output path
title_name = remove_special_characters(select_title.name) mp4_name = select_title.name +".mp4"
mp4_name = remove_special_characters(title_name) +".mp4" mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name)
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, title_name)
# Ensure the folder path exists
create_folder(mp4_path)
# Check if the MP4 file can be created
if not can_create_file(mp4_name):
logging.error("Invalid mp4 name.")
sys.exit(0)
# Get m3u8 master playlist # Get m3u8 master playlist
master_playlist = video_source.get_playlist() master_playlist = video_source.get_playlist()

View File

@ -2,16 +2,16 @@
import os import os
import sys import sys
import logging import time
# Internal utilities # Internal utilities
from Src.Util.console import console, msg from Src.Util.console import console, msg
from Src.Util.os import create_folder, can_create_file
from Src.Util.message import start_message from Src.Util.message import start_message
from Src.Util.call_stack import get_call_stack
from Src.Util.table import TVShowManager from Src.Util.table import TVShowManager
from Src.Lib.Downloader import HLS_Downloader from Src.Lib.Downloader import HLS_Downloader
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search
# Logic class # Logic class
@ -48,24 +48,19 @@ def download_video(scape_info_serie: GetSerieInfo, index_season_selected: int, i
mp4_name = f"{map_episode_title(scape_info_serie.tv_name, index_season_selected, index_episode_selected, obj_episode.get('name'))}.mp4" mp4_name = f"{map_episode_title(scape_info_serie.tv_name, index_season_selected, index_episode_selected, obj_episode.get('name'))}.mp4"
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, SERIES_FOLDER, scape_info_serie.tv_name, f"S{index_season_selected}") mp4_path = os.path.join(ROOT_PATH, SITE_NAME, SERIES_FOLDER, scape_info_serie.tv_name, f"S{index_season_selected}")
# Ensure the folder path exists
create_folder(mp4_path)
# Check if the MP4 file can be created
if not can_create_file(mp4_name):
logging.error("Invalid mp4 name.")
sys.exit(0)
# Setup video source # Setup video source
video_source.setup(obj_episode.get('url')) video_source.setup(obj_episode.get('url'))
# Get m3u8 master playlist # Get m3u8 master playlist
master_playlist = video_source.get_playlist() master_playlist = video_source.get_playlist()
HLS_Downloader( if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404:
m3u8_playlist = master_playlist, time.sleep(2)
output_filename = os.path.join(mp4_path, mp4_name)
).start() # Re call search function
if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n":
frames = get_call_stack()
execute_search(frames[-4])
def download_episode(scape_info_serie: GetSerieInfo, index_season_selected: int, download_all: bool = False) -> None: def download_episode(scape_info_serie: GetSerieInfo, index_season_selected: int, download_all: bool = False) -> None:

View File

@ -8,7 +8,6 @@ import logging
# Internal utilities # Internal utilities
from Src.Util.console import console from Src.Util.console import console
from Src.Util.message import start_message from Src.Util.message import start_message
from Src.Util.os import create_folder, can_create_file, remove_special_characters
from Src.Lib.Downloader import HLS_Downloader from Src.Lib.Downloader import HLS_Downloader
@ -44,17 +43,8 @@ def download_film(select_title: MediaItem, domain: str, version: str):
master_playlist = video_source.get_playlist() master_playlist = video_source.get_playlist()
# Define the filename and path for the downloaded film # Define the filename and path for the downloaded film
mp4_name = remove_special_characters(select_title.slug) mp4_format = (select_title.slug) + ".mp4"
mp4_format = (mp4_name) + ".mp4" mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.slug)
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.slug))
# Ensure the folder path exists
create_folder(mp4_path)
# Check if the MP4 file can be created
if not can_create_file(mp4_name):
logging.error("Invalid mp4 name.")
sys.exit(0)
# Download the film using the m3u8 playlist, and output filename # Download the film using the m3u8 playlist, and output filename
HLS_Downloader( HLS_Downloader(

View File

@ -2,15 +2,16 @@
import os import os
import sys import sys
import logging import time
# Internal utilities # Internal utilities
from Src.Util.console import console, msg from Src.Util.console import console, msg
from Src.Util.message import start_message from Src.Util.message import start_message
from Src.Util.call_stack import get_call_stack
from Src.Util.table import TVShowManager from Src.Util.table import TVShowManager
from Src.Lib.Downloader import HLS_Downloader from Src.Lib.Downloader import HLS_Downloader
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search
# Logic class # Logic class
@ -52,11 +53,13 @@ def download_video(tv_name: str, index_season_selected: int, index_episode_selec
master_playlist = video_source.get_playlist() master_playlist = video_source.get_playlist()
# Download the episode # Download the episode
HLS_Downloader( if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404:
m3u8_playlist = master_playlist, time.sleep(2)
output_filename = os.path.join(mp4_path, mp4_name)
).start()
# Re call search function
if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n":
frames = get_call_stack()
execute_search(frames[-4])
def download_episode(tv_name: str, index_season_selected: int, download_all: bool = False) -> None: def download_episode(tv_name: str, index_season_selected: int, download_all: bool = False) -> None:
""" """

View File

@ -111,8 +111,8 @@ class HttpClient:
return response.text # Return the response text return response.text # Return the response text
except Exception as e: except Exception as e:
logging.error(f"Request to {url} failed: {e}") logging.error(f"Request to {url} failed: {response.status_code} when get text.")
return None return 404
def get_content(self, url, timeout=20): def get_content(self, url, timeout=20):
""" """
@ -128,7 +128,7 @@ class HttpClient:
return response.content # Return the raw response content return response.content # Return the raw response content
except Exception as e: except Exception as e:
logging.error(f"Request to {url} failed: {e}") logging.error(f"Request to {url} failed: {response.status_code} when get content.")
return None return None
@ -179,13 +179,14 @@ class ContentExtractor:
result = list(set(available_languages) & set(set_language)) result = list(set(available_languages) & set(set_language))
# Create a formatted table to display audio info # Create a formatted table to display audio info
table = Table(show_header=False, box=None) if len(available_languages) > 0:
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") table = Table(show_header=False, box=None)
table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}") table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}")
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
console.rule("[bold green] AUDIO ", style="bold red") console.rule("[bold green] AUDIO ", style="bold red")
console.print(table) console.print(table)
else: else:
console.log("[red]Can't find a list of audios") console.log("[red]Can't find a list of audios")
@ -207,13 +208,14 @@ class ContentExtractor:
result = list(set(available_languages) & set(set_language)) result = list(set(available_languages) & set(set_language))
# Create a formatted table to display subtitle info # Create a formatted table to display subtitle info
table = Table(show_header=False, box=None) if len(available_languages) > 0:
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}") table = Table(show_header=False, box=None)
table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}") table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}") table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}")
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
console.rule("[bold green] SUBTITLE ", style="bold red") console.rule("[bold green] SUBTITLE ", style="bold red")
console.print(table) console.print(table)
else: else:
console.log("[red]Can't find a list of subtitles") console.log("[red]Can't find a list of subtitles")
@ -242,7 +244,10 @@ class ContentExtractor:
table.add_row(f"[green]Downloadable:", f"[purple]{video_res[0]}x{video_res[1]}") table.add_row(f"[green]Downloadable:", f"[purple]{video_res[0]}x{video_res[1]}")
if self.codec is not None: if self.codec is not None:
table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))") if config_manager.get_bool("M3U8_CONVERSION", "use_codec"):
table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))")
else:
table.add_row(f"[green]Codec:", "[purple]copy")
console.rule("[bold green] VIDEO ", style="bold red") console.rule("[bold green] VIDEO ", style="bold red")
console.print(table) console.print(table)
@ -387,8 +392,7 @@ class ContentDownloader:
video_m3u8.download_streams(f"{Colors.MAGENTA}video") video_m3u8.download_streams(f"{Colors.MAGENTA}video")
# Print duration information of the downloaded video # Print duration information of the downloaded video
print_duration_table(downloaded_video[0].get('path')) #print_duration_table(downloaded_video[0].get('path'))
print("")
else: else:
console.log("[cyan]Video [red]already exists.") console.log("[cyan]Video [red]already exists.")
@ -416,7 +420,7 @@ class ContentDownloader:
audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}") audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}")
# Print duration information of the downloaded audio # Print duration information of the downloaded audio
print_duration_table(obj_audio.get('path')) #print_duration_table(obj_audio.get('path'))
else: else:
console.log(f"[cyan]Audio [white]([green]{obj_audio.get('language')}[white]) [red]already exists.") console.log(f"[cyan]Audio [white]([green]{obj_audio.get('language')}[white]) [red]already exists.")
@ -444,7 +448,7 @@ class ContentDownloader:
) )
# Print the status of the subtitle download # Print the status of the subtitle download
console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}") #console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}")
# Write the content to the specified file # Write the content to the specified file
with open(obj_subtitle.get("path"), "wb") as f: with open(obj_subtitle.get("path"), "wb") as f:
@ -461,7 +465,7 @@ class ContentJoiner:
""" """
self.path_manager: PathManager = path_manager self.path_manager: PathManager = path_manager
def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle): def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle, codec = None):
""" """
Sets up the content joiner with downloaded media files. Sets up the content joiner with downloaded media files.
@ -473,21 +477,26 @@ class ContentJoiner:
self.downloaded_video = downloaded_video self.downloaded_video = downloaded_video
self.downloaded_audio = downloaded_audio self.downloaded_audio = downloaded_audio
self.downloaded_subtitle = downloaded_subtitle self.downloaded_subtitle = downloaded_subtitle
self.codec = codec
# Initialize flags to check if media is available # Initialize flags to check if media is available
self.converted_out_path = None self.converted_out_path = None
self.there_is_video = (len(downloaded_video) > 0) self.there_is_video = len(downloaded_video) > 0
self.there_is_audio = (len(downloaded_audio) > 0) self.there_is_audio = len(downloaded_audio) > 0
self.there_is_subtitle = (len(downloaded_subtitle) > 0) self.there_is_subtitle = len(downloaded_subtitle) > 0
# Display the status of available media if self.there_is_audio or self.there_is_subtitle:
table = Table(show_header=False, box=None)
table.add_row(f"[green]Video - audio:", f"[yellow]{self.there_is_audio}")
table.add_row(f"[green]Video - Subtitle:", f"[yellow]{self.there_is_subtitle}")
console.rule("[bold green] JOIN ", style="bold red") # Display the status of available media
console.print(table) table = Table(show_header=False, box=None)
print("")
table.add_row(f"[green]Video - audio", f"[yellow]{self.there_is_audio}")
table.add_row(f"[green]Video - Subtitle", f"[yellow]{self.there_is_subtitle}")
print("")
console.rule("[bold green] JOIN ", style="bold red")
console.print(table)
print("")
# Start the joining process # Start the joining process
self.conversione() self.conversione()
@ -575,8 +584,8 @@ class ContentJoiner:
if not os.path.exists(path_join_video): if not os.path.exists(path_join_video):
# Set codec to None if not defined in class # Set codec to None if not defined in class
if not hasattr(self, 'codec'): #if not hasattr(self, 'codec'):
self.codec = None # self.codec = None
# Join the video segments into a single video file # Join the video segments into a single video file
join_video( join_video(
@ -604,8 +613,8 @@ class ContentJoiner:
if not os.path.exists(path_join_video_audio): if not os.path.exists(path_join_video_audio):
# Set codec to None if not defined in class # Set codec to None if not defined in class
if not hasattr(self, 'codec'): #if not hasattr(self, 'codec'):
self.codec = None # self.codec = None
# Join the video with audio segments # Join the video with audio segments
join_audios( join_audios(
@ -661,7 +670,6 @@ class HLS_Downloader:
self.output_filename = self._generate_output_filename(output_filename, m3u8_playlist, m3u8_index) self.output_filename = self._generate_output_filename(output_filename, m3u8_playlist, m3u8_index)
self.path_manager = PathManager(self.output_filename) self.path_manager = PathManager(self.output_filename)
self.download_tracker = DownloadTracker(self.path_manager) self.download_tracker = DownloadTracker(self.path_manager)
self.http_client = HttpClient(headers_index)
self.content_extractor = ContentExtractor() self.content_extractor = ContentExtractor()
self.content_downloader = ContentDownloader() self.content_downloader = ContentDownloader()
self.content_joiner = ContentJoiner(self.path_manager) self.content_joiner = ContentJoiner(self.path_manager)
@ -728,7 +736,10 @@ class HLS_Downloader:
# Determine whether to process a playlist or index # Determine whether to process a playlist or index
if self.m3u8_playlist: if self.m3u8_playlist:
self._process_playlist() r_proc = self._process_playlist()
if r_proc == 404:
return 404
elif self.m3u8_index: elif self.m3u8_index:
self._process_index() self._process_index()
@ -780,6 +791,7 @@ class HLS_Downloader:
missing_ts = True missing_ts = True
# Prepare the report panel content # Prepare the report panel content
print("")
panel_content = ( panel_content = (
f"[bold green]Download completed![/bold green]\n" f"[bold green]Download completed![/bold green]\n"
f"[cyan]File size: [bold red]{formatted_size}[/bold red]\n" f"[cyan]File size: [bold red]{formatted_size}[/bold red]\n"
@ -815,8 +827,14 @@ class HLS_Downloader:
# Retrieve the m3u8 playlist content # Retrieve the m3u8 playlist content
if self.is_playlist_url: if self.is_playlist_url:
m3u8_playlist_text = HttpClient(headers=headers_index).get(self.m3u8_playlist) response_text = HttpClient(headers=headers_index).get(self.m3u8_playlist)
m3u8_url_fixer.set_playlist(self.m3u8_playlist)
if response_text != 404:
m3u8_playlist_text = response_text
m3u8_url_fixer.set_playlist(self.m3u8_playlist)
else:
return 404
else: else:
m3u8_playlist_text = self.m3u8_playlist m3u8_playlist_text = self.m3u8_playlist
@ -849,7 +867,7 @@ class HLS_Downloader:
self.content_downloader.download_subtitle(self.download_tracker.downloaded_subtitle) self.content_downloader.download_subtitle(self.download_tracker.downloaded_subtitle)
# Join downloaded content # Join downloaded content
self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle) self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle, self.content_extractor.codec)
# Clean up temporary files and directories # Clean up temporary files and directories
self._clean(self.content_joiner.converted_out_path) self._clean(self.content_joiner.converted_out_path)
@ -862,6 +880,7 @@ class HLS_Downloader:
# Download video # Download video
self.download_tracker.add_video(self.m3u8_index) self.download_tracker.add_video(self.m3u8_index)
self.content_downloader.download_video(self.download_tracker.downloaded_video)
# Join video # Join video
self.content_joiner.setup(self.download_tracker.downloaded_video, [], []) self.content_joiner.setup(self.download_tracker.downloaded_video, [], [])

View File

@ -8,7 +8,7 @@ import logging
import binascii import binascii
import threading import threading
from queue import PriorityQueue from queue import PriorityQueue
from urllib.parse import urljoin, urlparse, urlunparse from urllib.parse import urljoin, urlparse
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
@ -78,9 +78,6 @@ class M3U8_Segments:
self.queue = PriorityQueue() self.queue = PriorityQueue()
self.stop_event = threading.Event() self.stop_event = threading.Event()
# Server ip
self.fake_proxy = False
def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes: def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes:
""" """
Retrieves the encryption key from the M3U8 playlist. Retrieves the encryption key from the M3U8 playlist.
@ -111,27 +108,8 @@ class M3U8_Segments:
hex_content = binascii.hexlify(response.content).decode('utf-8') hex_content = binascii.hexlify(response.content).decode('utf-8')
byte_content = bytes.fromhex(hex_content) byte_content = bytes.fromhex(hex_content)
logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})")
return byte_content return byte_content
def __gen_proxy__(self, url: str, url_index: int) -> str:
"""
Change the IP address of the provided URL based on the given index.
Args:
- url (str): The original URL that needs its IP address replaced.
- url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP.
Returns:
str: The modified URL with the new IP address.
"""
new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)]
# Parse the original URL and replace the hostname with the new IP address
parsed_url = urlparse(url)._replace(netloc=new_ip_address)
return urlunparse(parsed_url)
def parse_data(self, m3u8_content: str) -> None: def parse_data(self, m3u8_content: str) -> None:
""" """
Parses the M3U8 content to extract segment information. Parses the M3U8 content to extract segment information.
@ -185,12 +163,6 @@ class M3U8_Segments:
if len(self.valid_proxy) == 0: if len(self.valid_proxy) == 0:
sys.exit(0) sys.exit(0)
# Server ip
if self.fake_proxy:
for i in range(len(self.segments)):
segment_url = self.segments[i]
self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url))
def get_info(self) -> None: def get_info(self) -> None:
""" """
Makes a request to the index M3U8 file to get information about segments. Makes a request to the index M3U8 file to get information about segments.
@ -216,68 +188,76 @@ class M3U8_Segments:
# Parser data of content of index pass in input to class # Parser data of content of index pass in input to class
self.parse_data(self.url) self.parse_data(self.url)
def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None: def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm, retries: int = 3, backoff_factor: float = 1.5) -> None:
""" """
Downloads a TS segment and adds it to the segment queue. Downloads a TS segment and adds it to the segment queue with retry logic.
Parameters: Parameters:
- ts_url (str): The URL of the TS segment. - ts_url (str): The URL of the TS segment.
- index (int): The index of the segment. - index (int): The index of the segment.
- progress_bar (tqdm): Progress counter for tracking download progress. - progress_bar (tqdm): Progress counter for tracking download progress.
- retries (int): The number of times to retry on failure (default is 3).
- backoff_factor (float): The backoff factor for exponential backoff (default is 1.5 seconds).
""" """
need_verify = REQUEST_VERIFY need_verify = REQUEST_VERIFY
# Set to false for only fake proxy that use real ip of server for attempt in range(retries):
if self.fake_proxy: try:
need_verify = False start_time = time.time()
try: # Make request to get content
start_time = time.time() if THERE_IS_PROXY_LIST:
# Make request to get content # Get proxy from list
if THERE_IS_PROXY_LIST: proxy = self.valid_proxy[index % len(self.valid_proxy)]
logging.info(f"Use proxy: {proxy}")
# Get proxy from list with httpx.Client(proxies=proxy, verify=need_verify) as client:
proxy = self.valid_proxy[index % len(self.valid_proxy)] if 'key_base_url' in self.__dict__:
logging.info(f"Use proxy: {proxy}") response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
else:
response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
with httpx.Client(proxies=proxy, verify=need_verify) as client: else:
if 'key_base_url' in self.__dict__: with httpx.Client(verify=need_verify) as client_2:
response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) if 'key_base_url' in self.__dict__:
else: response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True) else:
response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
else: # Get response content
with httpx.Client(verify=need_verify) as client_2: response.raise_for_status() # Raise exception for HTTP errors
if 'key_base_url' in self.__dict__: duration = time.time() - start_time
response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True) segment_content = response.content
else:
response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
# Get response content # Update bar
response.raise_for_status() response_size = int(response.headers.get('Content-Length', 0)) or len(segment_content)
duration = time.time() - start_time
segment_content = response.content
# Update bar # Update progress bar with custom Class
response_size = int(response.headers.get('Content-Length', 0)) self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar)
if response_size == 0: # Decrypt the segment content if decryption is needed
response_size = int(len(response.content)) if self.decryption is not None:
segment_content = self.decryption.decrypt(segment_content)
# Update progress bar with custom Class # Add the segment to the queue
self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar) self.queue.put((index, segment_content))
progress_bar.update(1)
# Decrypt the segment content if decryption is needed # Break out of the loop on success
if self.decryption is not None: return
segment_content = self.decryption.decrypt(segment_content)
# Add the segment to the queue except (httpx.RequestError, httpx.HTTPStatusError) as e:
self.queue.put((index, segment_content)) console.print(f"\nAttempt {attempt + 1} failed for '{ts_url}' with error: {e}")
progress_bar.update(1)
except Exception as e: if attempt + 1 == retries:
console.print(f"\nFailed to download: '{ts_url}' with error: {e}") console.print(f"\nFailed after {retries} attempts. Skipping '{ts_url}'")
break
# Exponential backoff before retrying
sleep_time = backoff_factor * (2 ** attempt)
console.print(f"Retrying in {sleep_time} seconds...")
time.sleep(sleep_time)
def write_segments_to_file(self): def write_segments_to_file(self):
""" """
@ -393,7 +373,6 @@ class M3U8_Segments:
delay = TQDM_DELAY_WORKER delay = TQDM_DELAY_WORKER
# Start all workers # Start all workers
logging.info(f"Worker to use: {max_workers}")
with ThreadPoolExecutor(max_workers=max_workers) as executor: with ThreadPoolExecutor(max_workers=max_workers) as executor:
for index, segment_url in enumerate(self.segments): for index, segment_url in enumerate(self.segments):
time.sleep(delay) time.sleep(delay)

View File

@ -29,7 +29,7 @@ REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout')
def MP4_downloader(url: str, path: str, referer: str = None): def MP4_downloader(url: str, path: str, referer: str = None, headers_: str = None):
""" """
Downloads an MP4 video from a given URL using the specified referer header. Downloads an MP4 video from a given URL using the specified referer header.
@ -40,55 +40,63 @@ def MP4_downloader(url: str, path: str, referer: str = None):
- referer (str): The referer header value to include in the HTTP request headers. - referer (str): The referer header value to include in the HTTP request headers.
""" """
headers = None
if "http" not in str(url).lower().strip() or "https" not in str(url).lower().strip(): if "http" not in str(url).lower().strip() or "https" not in str(url).lower().strip():
logging.error(f"Invalid url: {url}") logging.error(f"Invalid url: {url}")
sys.exit(0) sys.exit(0)
# Make request to get content of video
logging.info(f"Make request to fetch mp4 from: {url}")
if referer != None: if referer != None:
headers = {'Referer': referer, 'user-agent': get_headers()} headers = {'Referer': referer, 'user-agent': get_headers()}
else: if headers == None:
headers = {'user-agent': get_headers()} headers = {'user-agent': get_headers()}
else:
headers = headers_
# Make request to get content of video
with httpx.Client(verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT) as client: with httpx.Client(verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT) as client:
with client.stream("GET", url, headers=headers, timeout=10) as response: with client.stream("GET", url, headers=headers, timeout=10) as response:
total = int(response.headers.get('content-length', 0)) total = int(response.headers.get('content-length', 0))
# Create bar format if total != 0:
if TQDM_USE_LARGE_BAR:
bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): " # Create bar format
f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ " if TQDM_USE_LARGE_BAR:
f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] " bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): "
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| " f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ "
f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]") f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] "
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| "
f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]")
else:
bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% "
f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]")
# Create progress bar
progress_bar = tqdm(
total=total,
ascii='░▒█',
bar_format=bar_format,
unit_scale=True,
unit_divisor=1024,
mininterval=0.05
)
# Download file
with open(path, 'wb') as file, progress_bar as bar:
for chunk in response.iter_bytes(chunk_size=1024):
if chunk:
size = file.write(chunk)
bar.update(size)
else: else:
bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% " console.print("[red]Cant find any stream.")
f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]")
# Create progress bar # Get summary
progress_bar = tqdm( if total != 0:
total=total, console.print(Panel(
ascii='░▒█', f"[bold green]Download completed![/bold green]\n"
bar_format=bar_format, f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n"
unit_scale=True, f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]",
unit_divisor=1024, title=f"{os.path.basename(path.replace('.mp4', ''))}",
mininterval=0.05 border_style="green"
) ))
# Download file
with open(path, 'wb') as file, progress_bar as bar:
for chunk in response.iter_bytes(chunk_size=1024):
if chunk:
size = file.write(chunk)
bar.update(size)
# Get summary
console.print(Panel(
f"[bold green]Download completed![/bold green]\n"
f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n"
f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]",
title=f"{os.path.basename(path.replace('.mp4', ''))}",
border_style="green"
))

View File

@ -55,7 +55,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
# Enabled the use of gpu # Enabled the use of gpu
if USE_GPU: if USE_GPU:
ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda']) ffmpeg_cmd.extend(['-hwaccel', 'cuda'])
# Add mpegts to force to detect input file as ts file # Add mpegts to force to detect input file as ts file
if need_to_force_to_ts(video_path): if need_to_force_to_ts(video_path):
@ -67,7 +67,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
ffmpeg_cmd.extend(['-i', video_path]) ffmpeg_cmd.extend(['-i', video_path])
# Add output Parameters # Add output Parameters
if USE_CODEC: if USE_CODEC and codec != None:
if USE_VCODEC: if USE_VCODEC:
if codec.video_codec_name: if codec.video_codec_name:
if not USE_GPU: if not USE_GPU:
@ -76,6 +76,10 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
else: else:
console.log("[red]Cant find vcodec for 'join_audios'") console.log("[red]Cant find vcodec for 'join_audios'")
else:
if USE_GPU:
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
if USE_ACODEC: if USE_ACODEC:
if codec.audio_codec_name: if codec.audio_codec_name:
@ -98,7 +102,6 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
# Overwrite # Overwrite
ffmpeg_cmd += [out_path, "-y"] ffmpeg_cmd += [out_path, "-y"]
logging.info(f"FFmpeg command: {ffmpeg_cmd}")
# Run join # Run join
if DEBUG_MODE: if DEBUG_MODE:
@ -143,7 +146,7 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
# Enabled the use of gpu # Enabled the use of gpu
if USE_GPU: if USE_GPU:
ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda']) ffmpeg_cmd.extend(['-hwaccel', 'cuda'])
# Insert input video path # Insert input video path
ffmpeg_cmd.extend(['-i', video_path]) ffmpeg_cmd.extend(['-i', video_path])
@ -173,6 +176,9 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc']) ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
else: else:
console.log("[red]Cant find vcodec for 'join_audios'") console.log("[red]Cant find vcodec for 'join_audios'")
else:
if USE_GPU:
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
if USE_ACODEC: if USE_ACODEC:
if codec.audio_codec_name: if codec.audio_codec_name:
@ -195,12 +201,11 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
# Use shortest input path for video and audios # Use shortest input path for video and audios
if not video_audio_same_duration: if not video_audio_same_duration:
console.log("[red]Use shortest input.") logging.info("[red]Use shortest input.")
ffmpeg_cmd.extend(['-shortest', '-strict', 'experimental']) ffmpeg_cmd.extend(['-shortest', '-strict', 'experimental'])
# Overwrite # Overwrite
ffmpeg_cmd += [out_path, "-y"] ffmpeg_cmd += [out_path, "-y"]
logging.info(f"FFmpeg command: {ffmpeg_cmd}")
# Run join # Run join
if DEBUG_MODE: if DEBUG_MODE:

View File

@ -164,6 +164,13 @@ class M3U8_Codec:
else: else:
logging.warning("No bandwidth provided. Bitrates cannot be calculated.") logging.warning("No bandwidth provided. Bitrates cannot be calculated.")
def __str__(self):
return (f"M3U8_Codec(bandwidth={self.bandwidth}, "
f"codecs='{self.codecs}', "
f"audio_codec='{self.audio_codec}', "
f"video_codec='{self.video_codec}', "
f"audio_codec_name='{self.audio_codec_name}', "
f"video_codec_name='{self.video_codec_name}')")
class M3U8_Video: class M3U8_Video:

View File

@ -164,7 +164,6 @@ def check_file_existence(file_path):
return True return True
else: else:
logging.warning(f"The file '{file_path}' does not exist.")
return False return False
except Exception as e: except Exception as e:

View File

@ -17,5 +17,5 @@ from Src.Lib.Downloader import MP4_downloader
# Test # Test
MP4_downloader( MP4_downloader(
"", "",
"EP_1.mp4", "EP_1.mp4"
) )

View File

@ -42,8 +42,7 @@
"eng", "eng",
"spa" "spa"
], ],
"cleanup_tmp_folder": true, "cleanup_tmp_folder": true
"create_report": false
}, },
"M3U8_CONVERSION": { "M3U8_CONVERSION": {
"use_codec": false, "use_codec": false,
@ -58,23 +57,23 @@
}, },
"SITE": { "SITE": {
"streamingcommunity": { "streamingcommunity": {
"video_workers": 4, "video_workers": 6,
"audio_workers": 4, "audio_workers": 6,
"domain": "computer" "domain": "computer"
}, },
"altadefinizione": { "altadefinizione": {
"video_workers": -1, "video_workers": 12,
"audio_workers": -1, "audio_workers": 12,
"domain": "my" "domain": "my"
}, },
"guardaserie": { "guardaserie": {
"video_workers": -1, "video_workers": 12,
"audio_workers": -1, "audio_workers": 12,
"domain": "dev" "domain": "dev"
}, },
"mostraguarda": { "mostraguarda": {
"video_workers": -1, "video_workers": 12,
"audio_workers": -1, "audio_workers": 12,
"domain": "stream" "domain": "stream"
}, },
"ddlstreamitaly": { "ddlstreamitaly": {
@ -88,12 +87,6 @@
"animeunity": { "animeunity": {
"domain": "to" "domain": "to"
}, },
"watch_lonelil": {
"domain": "ru"
},
"uhdmovies": {
"domain": "mov"
},
"bitsearch": { "bitsearch": {
"domain": "to" "domain": "to"
}, },