mirror of
https://github.com/Arrowar/StreamingCommunity.git
synced 2025-06-05 02:55:25 +00:00
Fix 401 Unauthorized, add retry, fix parser gpu
This commit is contained in:
parent
6566dd6b3d
commit
ce0dc7ad78
35
Src/Api/Template/Util/recall_search.py
Normal file
35
Src/Api/Template/Util/recall_search.py
Normal file
@ -0,0 +1,35 @@
|
||||
# 19.10.24
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
def execute_search(info):
|
||||
"""
|
||||
Dynamically imports and executes a specified function from a module defined in the info dictionary.
|
||||
|
||||
Parameters:
|
||||
info (dict): A dictionary containing the function name, folder, and module information.
|
||||
"""
|
||||
|
||||
# Step 1: Define the project path using the folder from the info dictionary
|
||||
project_path = os.path.dirname(info['folder']) # Get the base path for the project
|
||||
|
||||
# Step 2: Add the project path to sys.path
|
||||
if project_path not in sys.path:
|
||||
sys.path.append(project_path)
|
||||
|
||||
# Attempt to import the specified function from the module
|
||||
try:
|
||||
# Construct the import statement dynamically
|
||||
module_path = f"Src.Api.{info['folder_base']}"
|
||||
exec(f"from {module_path} import {info['function']}")
|
||||
|
||||
# Call the specified function
|
||||
eval(info['function'])() # Calls the search function
|
||||
|
||||
except ModuleNotFoundError as e:
|
||||
print(f"ModuleNotFoundError: {e}")
|
||||
except ImportError as e:
|
||||
print(f"ImportError: {e}")
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
@ -1,5 +1,6 @@
|
||||
# 19.06.24
|
||||
|
||||
from .site import get_select_title
|
||||
from .Util.recall_search import execute_search
|
||||
from .Util.get_domain import search_domain
|
||||
from .Util.manage_ep import manage_selection, map_episode_title, validate_episode_selection, validate_selection
|
@ -8,7 +8,6 @@ import logging
|
||||
# Internal utilities
|
||||
from Src.Util.message import start_message
|
||||
from Src.Util.console import console
|
||||
from Src.Util.os import create_folder, can_create_file, remove_special_characters
|
||||
from Src.Lib.Downloader import HLS_Downloader
|
||||
|
||||
|
||||
@ -38,16 +37,8 @@ def download_film(select_title: MediaItem):
|
||||
video_source = VideoSource(select_title.url)
|
||||
|
||||
# Define output path
|
||||
mp4_name = remove_special_characters(select_title.name) + ".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.name))
|
||||
|
||||
# Ensure the folder path exists
|
||||
create_folder(mp4_path)
|
||||
|
||||
# Check if the MP4 file can be created
|
||||
if not can_create_file(mp4_name):
|
||||
logging.error("Invalid mp4 name.")
|
||||
sys.exit(0)
|
||||
mp4_name = select_title.name + ".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name)
|
||||
|
||||
# Get m3u8 master playlist
|
||||
master_playlist = video_source.get_playlist()
|
||||
|
@ -8,7 +8,6 @@ import logging
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
from Src.Util.message import start_message
|
||||
from Src.Util.os import create_folder, can_create_file, remove_special_characters
|
||||
from Src.Lib.Downloader import HLS_Downloader
|
||||
|
||||
|
||||
@ -37,17 +36,8 @@ def download_film(select_title: MediaItem):
|
||||
video_source = VideoSource(select_title.url)
|
||||
|
||||
# Define output path
|
||||
title_name = remove_special_characters(select_title.name)
|
||||
mp4_name = remove_special_characters(title_name) +".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, title_name)
|
||||
|
||||
# Ensure the folder path exists
|
||||
create_folder(mp4_path)
|
||||
|
||||
# Check if the MP4 file can be created
|
||||
if not can_create_file(mp4_name):
|
||||
logging.error("Invalid mp4 name.")
|
||||
sys.exit(0)
|
||||
mp4_name = select_title.name +".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.name)
|
||||
|
||||
# Get m3u8 master playlist
|
||||
master_playlist = video_source.get_playlist()
|
||||
|
@ -2,16 +2,16 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import time
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console, msg
|
||||
from Src.Util.os import create_folder, can_create_file
|
||||
from Src.Util.message import start_message
|
||||
from Src.Util.call_stack import get_call_stack
|
||||
from Src.Util.table import TVShowManager
|
||||
from Src.Lib.Downloader import HLS_Downloader
|
||||
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection
|
||||
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search
|
||||
|
||||
|
||||
# Logic class
|
||||
@ -48,24 +48,19 @@ def download_video(scape_info_serie: GetSerieInfo, index_season_selected: int, i
|
||||
mp4_name = f"{map_episode_title(scape_info_serie.tv_name, index_season_selected, index_episode_selected, obj_episode.get('name'))}.mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, SERIES_FOLDER, scape_info_serie.tv_name, f"S{index_season_selected}")
|
||||
|
||||
# Ensure the folder path exists
|
||||
create_folder(mp4_path)
|
||||
|
||||
# Check if the MP4 file can be created
|
||||
if not can_create_file(mp4_name):
|
||||
logging.error("Invalid mp4 name.")
|
||||
sys.exit(0)
|
||||
|
||||
# Setup video source
|
||||
video_source.setup(obj_episode.get('url'))
|
||||
|
||||
# Get m3u8 master playlist
|
||||
master_playlist = video_source.get_playlist()
|
||||
|
||||
HLS_Downloader(
|
||||
m3u8_playlist = master_playlist,
|
||||
output_filename = os.path.join(mp4_path, mp4_name)
|
||||
).start()
|
||||
if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404:
|
||||
time.sleep(2)
|
||||
|
||||
# Re call search function
|
||||
if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n":
|
||||
frames = get_call_stack()
|
||||
execute_search(frames[-4])
|
||||
|
||||
|
||||
def download_episode(scape_info_serie: GetSerieInfo, index_season_selected: int, download_all: bool = False) -> None:
|
||||
|
@ -8,7 +8,6 @@ import logging
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
from Src.Util.message import start_message
|
||||
from Src.Util.os import create_folder, can_create_file, remove_special_characters
|
||||
from Src.Lib.Downloader import HLS_Downloader
|
||||
|
||||
|
||||
@ -44,17 +43,8 @@ def download_film(select_title: MediaItem, domain: str, version: str):
|
||||
master_playlist = video_source.get_playlist()
|
||||
|
||||
# Define the filename and path for the downloaded film
|
||||
mp4_name = remove_special_characters(select_title.slug)
|
||||
mp4_format = (mp4_name) + ".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, remove_special_characters(select_title.slug))
|
||||
|
||||
# Ensure the folder path exists
|
||||
create_folder(mp4_path)
|
||||
|
||||
# Check if the MP4 file can be created
|
||||
if not can_create_file(mp4_name):
|
||||
logging.error("Invalid mp4 name.")
|
||||
sys.exit(0)
|
||||
mp4_format = (select_title.slug) + ".mp4"
|
||||
mp4_path = os.path.join(ROOT_PATH, SITE_NAME, MOVIE_FOLDER, select_title.slug)
|
||||
|
||||
# Download the film using the m3u8 playlist, and output filename
|
||||
HLS_Downloader(
|
||||
|
@ -2,15 +2,16 @@
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
import time
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console, msg
|
||||
from Src.Util.message import start_message
|
||||
from Src.Util.call_stack import get_call_stack
|
||||
from Src.Util.table import TVShowManager
|
||||
from Src.Lib.Downloader import HLS_Downloader
|
||||
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection
|
||||
from ..Template import manage_selection, map_episode_title, validate_selection, validate_episode_selection, execute_search
|
||||
|
||||
|
||||
# Logic class
|
||||
@ -52,11 +53,13 @@ def download_video(tv_name: str, index_season_selected: int, index_episode_selec
|
||||
master_playlist = video_source.get_playlist()
|
||||
|
||||
# Download the episode
|
||||
HLS_Downloader(
|
||||
m3u8_playlist = master_playlist,
|
||||
output_filename = os.path.join(mp4_path, mp4_name)
|
||||
).start()
|
||||
if HLS_Downloader(os.path.join(mp4_path, mp4_name), master_playlist).start() == 404:
|
||||
time.sleep(2)
|
||||
|
||||
# Re call search function
|
||||
if msg.ask("[green]Do you want to continue [white]([red]y[white])[green] or return at home[white]([red]n[white]) ", choices=['y', 'n'], default='y', show_choices=True) == "n":
|
||||
frames = get_call_stack()
|
||||
execute_search(frames[-4])
|
||||
|
||||
def download_episode(tv_name: str, index_season_selected: int, download_all: bool = False) -> None:
|
||||
"""
|
||||
|
@ -111,8 +111,8 @@ class HttpClient:
|
||||
return response.text # Return the response text
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Request to {url} failed: {e}")
|
||||
return None
|
||||
logging.error(f"Request to {url} failed: {response.status_code} when get text.")
|
||||
return 404
|
||||
|
||||
def get_content(self, url, timeout=20):
|
||||
"""
|
||||
@ -128,7 +128,7 @@ class HttpClient:
|
||||
return response.content # Return the raw response content
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Request to {url} failed: {e}")
|
||||
logging.error(f"Request to {url} failed: {response.status_code} when get content.")
|
||||
return None
|
||||
|
||||
|
||||
@ -179,13 +179,14 @@ class ContentExtractor:
|
||||
result = list(set(available_languages) & set(set_language))
|
||||
|
||||
# Create a formatted table to display audio info
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
|
||||
table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}")
|
||||
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
|
||||
if len(available_languages) > 0:
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
|
||||
table.add_row(f"[red]Set audios:", f"[purple]{', '.join(set_language)}")
|
||||
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
|
||||
|
||||
console.rule("[bold green] AUDIO ", style="bold red")
|
||||
console.print(table)
|
||||
console.rule("[bold green] AUDIO ", style="bold red")
|
||||
console.print(table)
|
||||
|
||||
else:
|
||||
console.log("[red]Can't find a list of audios")
|
||||
@ -207,13 +208,14 @@ class ContentExtractor:
|
||||
result = list(set(available_languages) & set(set_language))
|
||||
|
||||
# Create a formatted table to display subtitle info
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
|
||||
table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}")
|
||||
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
|
||||
if len(available_languages) > 0:
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_row(f"[cyan]Available languages:", f"[purple]{', '.join(available_languages)}")
|
||||
table.add_row(f"[red]Set subtitles:", f"[purple]{', '.join(set_language)}")
|
||||
table.add_row(f"[green]Downloadable:", f"[purple]{', '.join(result)}")
|
||||
|
||||
console.rule("[bold green] SUBTITLE ", style="bold red")
|
||||
console.print(table)
|
||||
console.rule("[bold green] SUBTITLE ", style="bold red")
|
||||
console.print(table)
|
||||
|
||||
else:
|
||||
console.log("[red]Can't find a list of subtitles")
|
||||
@ -242,7 +244,10 @@ class ContentExtractor:
|
||||
table.add_row(f"[green]Downloadable:", f"[purple]{video_res[0]}x{video_res[1]}")
|
||||
|
||||
if self.codec is not None:
|
||||
table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))")
|
||||
if config_manager.get_bool("M3U8_CONVERSION", "use_codec"):
|
||||
table.add_row(f"[green]Codec:", f"([green]'v'[white]: [yellow]{self.codec.video_codec_name}[white] ([green]b[white]: [yellow]{self.codec.video_bitrate // 1000}k[white]), [green]'a'[white]: [yellow]{self.codec.audio_codec_name}[white] ([green]b[white]: [yellow]{self.codec.audio_bitrate // 1000}k[white]))")
|
||||
else:
|
||||
table.add_row(f"[green]Codec:", "[purple]copy")
|
||||
|
||||
console.rule("[bold green] VIDEO ", style="bold red")
|
||||
console.print(table)
|
||||
@ -387,8 +392,7 @@ class ContentDownloader:
|
||||
video_m3u8.download_streams(f"{Colors.MAGENTA}video")
|
||||
|
||||
# Print duration information of the downloaded video
|
||||
print_duration_table(downloaded_video[0].get('path'))
|
||||
print("")
|
||||
#print_duration_table(downloaded_video[0].get('path'))
|
||||
|
||||
else:
|
||||
console.log("[cyan]Video [red]already exists.")
|
||||
@ -416,7 +420,7 @@ class ContentDownloader:
|
||||
audio_m3u8.download_streams(f"{Colors.MAGENTA}audio {Colors.RED}{obj_audio.get('language')}")
|
||||
|
||||
# Print duration information of the downloaded audio
|
||||
print_duration_table(obj_audio.get('path'))
|
||||
#print_duration_table(obj_audio.get('path'))
|
||||
|
||||
else:
|
||||
console.log(f"[cyan]Audio [white]([green]{obj_audio.get('language')}[white]) [red]already exists.")
|
||||
@ -444,7 +448,7 @@ class ContentDownloader:
|
||||
)
|
||||
|
||||
# Print the status of the subtitle download
|
||||
console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}")
|
||||
#console.print(f"[cyan]Downloading subtitle: [red]{sub_language.lower()}")
|
||||
|
||||
# Write the content to the specified file
|
||||
with open(obj_subtitle.get("path"), "wb") as f:
|
||||
@ -461,7 +465,7 @@ class ContentJoiner:
|
||||
"""
|
||||
self.path_manager: PathManager = path_manager
|
||||
|
||||
def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle):
|
||||
def setup(self, downloaded_video, downloaded_audio, downloaded_subtitle, codec = None):
|
||||
"""
|
||||
Sets up the content joiner with downloaded media files.
|
||||
|
||||
@ -473,21 +477,26 @@ class ContentJoiner:
|
||||
self.downloaded_video = downloaded_video
|
||||
self.downloaded_audio = downloaded_audio
|
||||
self.downloaded_subtitle = downloaded_subtitle
|
||||
self.codec = codec
|
||||
|
||||
# Initialize flags to check if media is available
|
||||
self.converted_out_path = None
|
||||
self.there_is_video = (len(downloaded_video) > 0)
|
||||
self.there_is_audio = (len(downloaded_audio) > 0)
|
||||
self.there_is_subtitle = (len(downloaded_subtitle) > 0)
|
||||
self.there_is_video = len(downloaded_video) > 0
|
||||
self.there_is_audio = len(downloaded_audio) > 0
|
||||
self.there_is_subtitle = len(downloaded_subtitle) > 0
|
||||
|
||||
# Display the status of available media
|
||||
table = Table(show_header=False, box=None)
|
||||
table.add_row(f"[green]Video - audio:", f"[yellow]{self.there_is_audio}")
|
||||
table.add_row(f"[green]Video - Subtitle:", f"[yellow]{self.there_is_subtitle}")
|
||||
if self.there_is_audio or self.there_is_subtitle:
|
||||
|
||||
console.rule("[bold green] JOIN ", style="bold red")
|
||||
console.print(table)
|
||||
print("")
|
||||
# Display the status of available media
|
||||
table = Table(show_header=False, box=None)
|
||||
|
||||
table.add_row(f"[green]Video - audio", f"[yellow]{self.there_is_audio}")
|
||||
table.add_row(f"[green]Video - Subtitle", f"[yellow]{self.there_is_subtitle}")
|
||||
|
||||
print("")
|
||||
console.rule("[bold green] JOIN ", style="bold red")
|
||||
console.print(table)
|
||||
print("")
|
||||
|
||||
# Start the joining process
|
||||
self.conversione()
|
||||
@ -575,8 +584,8 @@ class ContentJoiner:
|
||||
if not os.path.exists(path_join_video):
|
||||
|
||||
# Set codec to None if not defined in class
|
||||
if not hasattr(self, 'codec'):
|
||||
self.codec = None
|
||||
#if not hasattr(self, 'codec'):
|
||||
# self.codec = None
|
||||
|
||||
# Join the video segments into a single video file
|
||||
join_video(
|
||||
@ -604,8 +613,8 @@ class ContentJoiner:
|
||||
if not os.path.exists(path_join_video_audio):
|
||||
|
||||
# Set codec to None if not defined in class
|
||||
if not hasattr(self, 'codec'):
|
||||
self.codec = None
|
||||
#if not hasattr(self, 'codec'):
|
||||
# self.codec = None
|
||||
|
||||
# Join the video with audio segments
|
||||
join_audios(
|
||||
@ -661,7 +670,6 @@ class HLS_Downloader:
|
||||
self.output_filename = self._generate_output_filename(output_filename, m3u8_playlist, m3u8_index)
|
||||
self.path_manager = PathManager(self.output_filename)
|
||||
self.download_tracker = DownloadTracker(self.path_manager)
|
||||
self.http_client = HttpClient(headers_index)
|
||||
self.content_extractor = ContentExtractor()
|
||||
self.content_downloader = ContentDownloader()
|
||||
self.content_joiner = ContentJoiner(self.path_manager)
|
||||
@ -728,7 +736,10 @@ class HLS_Downloader:
|
||||
|
||||
# Determine whether to process a playlist or index
|
||||
if self.m3u8_playlist:
|
||||
self._process_playlist()
|
||||
r_proc = self._process_playlist()
|
||||
|
||||
if r_proc == 404:
|
||||
return 404
|
||||
|
||||
elif self.m3u8_index:
|
||||
self._process_index()
|
||||
@ -780,6 +791,7 @@ class HLS_Downloader:
|
||||
missing_ts = True
|
||||
|
||||
# Prepare the report panel content
|
||||
print("")
|
||||
panel_content = (
|
||||
f"[bold green]Download completed![/bold green]\n"
|
||||
f"[cyan]File size: [bold red]{formatted_size}[/bold red]\n"
|
||||
@ -815,8 +827,14 @@ class HLS_Downloader:
|
||||
|
||||
# Retrieve the m3u8 playlist content
|
||||
if self.is_playlist_url:
|
||||
m3u8_playlist_text = HttpClient(headers=headers_index).get(self.m3u8_playlist)
|
||||
m3u8_url_fixer.set_playlist(self.m3u8_playlist)
|
||||
response_text = HttpClient(headers=headers_index).get(self.m3u8_playlist)
|
||||
|
||||
if response_text != 404:
|
||||
m3u8_playlist_text = response_text
|
||||
m3u8_url_fixer.set_playlist(self.m3u8_playlist)
|
||||
|
||||
else:
|
||||
return 404
|
||||
|
||||
else:
|
||||
m3u8_playlist_text = self.m3u8_playlist
|
||||
@ -849,7 +867,7 @@ class HLS_Downloader:
|
||||
self.content_downloader.download_subtitle(self.download_tracker.downloaded_subtitle)
|
||||
|
||||
# Join downloaded content
|
||||
self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle)
|
||||
self.content_joiner.setup(self.download_tracker.downloaded_video, self.download_tracker.downloaded_audio, self.download_tracker.downloaded_subtitle, self.content_extractor.codec)
|
||||
|
||||
# Clean up temporary files and directories
|
||||
self._clean(self.content_joiner.converted_out_path)
|
||||
@ -862,7 +880,8 @@ class HLS_Downloader:
|
||||
|
||||
# Download video
|
||||
self.download_tracker.add_video(self.m3u8_index)
|
||||
|
||||
self.content_downloader.download_video(self.download_tracker.downloaded_video)
|
||||
|
||||
# Join video
|
||||
self.content_joiner.setup(self.download_tracker.downloaded_video, [], [])
|
||||
|
||||
|
@ -8,7 +8,7 @@ import logging
|
||||
import binascii
|
||||
import threading
|
||||
from queue import PriorityQueue
|
||||
from urllib.parse import urljoin, urlparse, urlunparse
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
@ -78,9 +78,6 @@ class M3U8_Segments:
|
||||
self.queue = PriorityQueue()
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
# Server ip
|
||||
self.fake_proxy = False
|
||||
|
||||
def __get_key__(self, m3u8_parser: M3U8_Parser) -> bytes:
|
||||
"""
|
||||
Retrieves the encryption key from the M3U8 playlist.
|
||||
@ -111,27 +108,8 @@ class M3U8_Segments:
|
||||
hex_content = binascii.hexlify(response.content).decode('utf-8')
|
||||
byte_content = bytes.fromhex(hex_content)
|
||||
|
||||
logging.info(f"Key: ('hex': {hex_content}, 'byte': {byte_content})")
|
||||
return byte_content
|
||||
|
||||
def __gen_proxy__(self, url: str, url_index: int) -> str:
|
||||
"""
|
||||
Change the IP address of the provided URL based on the given index.
|
||||
|
||||
Args:
|
||||
- url (str): The original URL that needs its IP address replaced.
|
||||
- url_index (int): The index used to select a new IP address from the list of FAKE_PROXY_IP.
|
||||
|
||||
Returns:
|
||||
str: The modified URL with the new IP address.
|
||||
"""
|
||||
new_ip_address = self.fake_proxy_ip[url_index % len(self.fake_proxy_ip)]
|
||||
|
||||
# Parse the original URL and replace the hostname with the new IP address
|
||||
parsed_url = urlparse(url)._replace(netloc=new_ip_address)
|
||||
|
||||
return urlunparse(parsed_url)
|
||||
|
||||
def parse_data(self, m3u8_content: str) -> None:
|
||||
"""
|
||||
Parses the M3U8 content to extract segment information.
|
||||
@ -185,12 +163,6 @@ class M3U8_Segments:
|
||||
if len(self.valid_proxy) == 0:
|
||||
sys.exit(0)
|
||||
|
||||
# Server ip
|
||||
if self.fake_proxy:
|
||||
for i in range(len(self.segments)):
|
||||
segment_url = self.segments[i]
|
||||
self.segments[i] = self.__gen_proxy__(segment_url, self.segments.index(segment_url))
|
||||
|
||||
def get_info(self) -> None:
|
||||
"""
|
||||
Makes a request to the index M3U8 file to get information about segments.
|
||||
@ -214,70 +186,78 @@ class M3U8_Segments:
|
||||
else:
|
||||
|
||||
# Parser data of content of index pass in input to class
|
||||
self.parse_data(self.url)
|
||||
self.parse_data(self.url)
|
||||
|
||||
def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm) -> None:
|
||||
def make_requests_stream(self, ts_url: str, index: int, progress_bar: tqdm, retries: int = 3, backoff_factor: float = 1.5) -> None:
|
||||
"""
|
||||
Downloads a TS segment and adds it to the segment queue.
|
||||
Downloads a TS segment and adds it to the segment queue with retry logic.
|
||||
|
||||
Parameters:
|
||||
- ts_url (str): The URL of the TS segment.
|
||||
- index (int): The index of the segment.
|
||||
- progress_bar (tqdm): Progress counter for tracking download progress.
|
||||
- retries (int): The number of times to retry on failure (default is 3).
|
||||
- backoff_factor (float): The backoff factor for exponential backoff (default is 1.5 seconds).
|
||||
"""
|
||||
need_verify = REQUEST_VERIFY
|
||||
|
||||
# Set to false for only fake proxy that use real ip of server
|
||||
if self.fake_proxy:
|
||||
need_verify = False
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
# Make request to get content
|
||||
if THERE_IS_PROXY_LIST:
|
||||
|
||||
# Make request to get content
|
||||
if THERE_IS_PROXY_LIST:
|
||||
# Get proxy from list
|
||||
proxy = self.valid_proxy[index % len(self.valid_proxy)]
|
||||
logging.info(f"Use proxy: {proxy}")
|
||||
|
||||
# Get proxy from list
|
||||
proxy = self.valid_proxy[index % len(self.valid_proxy)]
|
||||
logging.info(f"Use proxy: {proxy}")
|
||||
with httpx.Client(proxies=proxy, verify=need_verify) as client:
|
||||
if 'key_base_url' in self.__dict__:
|
||||
response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
else:
|
||||
response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
|
||||
with httpx.Client(proxies=proxy, verify=need_verify) as client:
|
||||
if 'key_base_url' in self.__dict__:
|
||||
response = client.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
else:
|
||||
response = client.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
else:
|
||||
with httpx.Client(verify=need_verify) as client_2:
|
||||
if 'key_base_url' in self.__dict__:
|
||||
response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
else:
|
||||
response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
|
||||
else:
|
||||
with httpx.Client(verify=need_verify) as client_2:
|
||||
if 'key_base_url' in self.__dict__:
|
||||
response = client_2.get(ts_url, headers=random_headers(self.key_base_url), timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
else:
|
||||
response = client_2.get(ts_url, headers={'user-agent': get_headers()}, timeout=REQUEST_TIMEOUT, follow_redirects=True)
|
||||
# Get response content
|
||||
response.raise_for_status() # Raise exception for HTTP errors
|
||||
duration = time.time() - start_time
|
||||
segment_content = response.content
|
||||
|
||||
# Get response content
|
||||
response.raise_for_status()
|
||||
duration = time.time() - start_time
|
||||
segment_content = response.content
|
||||
# Update bar
|
||||
response_size = int(response.headers.get('Content-Length', 0)) or len(segment_content)
|
||||
|
||||
# Update bar
|
||||
response_size = int(response.headers.get('Content-Length', 0))
|
||||
# Update progress bar with custom Class
|
||||
self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar)
|
||||
|
||||
if response_size == 0:
|
||||
response_size = int(len(response.content))
|
||||
# Decrypt the segment content if decryption is needed
|
||||
if self.decryption is not None:
|
||||
segment_content = self.decryption.decrypt(segment_content)
|
||||
|
||||
# Update progress bar with custom Class
|
||||
self.class_ts_estimator.update_progress_bar(response_size, duration, progress_bar)
|
||||
# Add the segment to the queue
|
||||
self.queue.put((index, segment_content))
|
||||
progress_bar.update(1)
|
||||
|
||||
# Decrypt the segment content if decryption is needed
|
||||
if self.decryption is not None:
|
||||
segment_content = self.decryption.decrypt(segment_content)
|
||||
# Break out of the loop on success
|
||||
return
|
||||
|
||||
# Add the segment to the queue
|
||||
self.queue.put((index, segment_content))
|
||||
progress_bar.update(1)
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"\nFailed to download: '{ts_url}' with error: {e}")
|
||||
except (httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
console.print(f"\nAttempt {attempt + 1} failed for '{ts_url}' with error: {e}")
|
||||
|
||||
if attempt + 1 == retries:
|
||||
console.print(f"\nFailed after {retries} attempts. Skipping '{ts_url}'")
|
||||
break
|
||||
|
||||
# Exponential backoff before retrying
|
||||
sleep_time = backoff_factor * (2 ** attempt)
|
||||
console.print(f"Retrying in {sleep_time} seconds...")
|
||||
time.sleep(sleep_time)
|
||||
|
||||
def write_segments_to_file(self):
|
||||
"""
|
||||
@ -393,7 +373,6 @@ class M3U8_Segments:
|
||||
delay = TQDM_DELAY_WORKER
|
||||
|
||||
# Start all workers
|
||||
logging.info(f"Worker to use: {max_workers}")
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
for index, segment_url in enumerate(self.segments):
|
||||
time.sleep(delay)
|
||||
|
@ -29,7 +29,7 @@ REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout')
|
||||
|
||||
|
||||
|
||||
def MP4_downloader(url: str, path: str, referer: str = None):
|
||||
def MP4_downloader(url: str, path: str, referer: str = None, headers_: str = None):
|
||||
|
||||
"""
|
||||
Downloads an MP4 video from a given URL using the specified referer header.
|
||||
@ -40,55 +40,63 @@ def MP4_downloader(url: str, path: str, referer: str = None):
|
||||
- referer (str): The referer header value to include in the HTTP request headers.
|
||||
"""
|
||||
|
||||
headers = None
|
||||
|
||||
if "http" not in str(url).lower().strip() or "https" not in str(url).lower().strip():
|
||||
logging.error(f"Invalid url: {url}")
|
||||
sys.exit(0)
|
||||
|
||||
# Make request to get content of video
|
||||
logging.info(f"Make request to fetch mp4 from: {url}")
|
||||
|
||||
if referer != None:
|
||||
headers = {'Referer': referer, 'user-agent': get_headers()}
|
||||
else:
|
||||
if headers == None:
|
||||
headers = {'user-agent': get_headers()}
|
||||
else:
|
||||
headers = headers_
|
||||
|
||||
# Make request to get content of video
|
||||
with httpx.Client(verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT) as client:
|
||||
with client.stream("GET", url, headers=headers, timeout=10) as response:
|
||||
total = int(response.headers.get('content-length', 0))
|
||||
|
||||
# Create bar format
|
||||
if TQDM_USE_LARGE_BAR:
|
||||
bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): "
|
||||
f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ "
|
||||
f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] "
|
||||
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| "
|
||||
f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]")
|
||||
if total != 0:
|
||||
|
||||
# Create bar format
|
||||
if TQDM_USE_LARGE_BAR:
|
||||
bar_format = (f"{Colors.YELLOW}[MP4] {Colors.WHITE}({Colors.CYAN}video{Colors.WHITE}): "
|
||||
f"{Colors.RED}{{percentage:.2f}}% {Colors.MAGENTA}{{bar}} {Colors.WHITE}[ "
|
||||
f"{Colors.YELLOW}{{n_fmt}}{Colors.WHITE} / {Colors.RED}{{total_fmt}} {Colors.WHITE}] "
|
||||
f"{Colors.YELLOW}{{elapsed}} {Colors.WHITE}< {Colors.CYAN}{{remaining}} {Colors.WHITE}| "
|
||||
f"{Colors.YELLOW}{{rate_fmt}}{{postfix}} {Colors.WHITE}]")
|
||||
else:
|
||||
bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% "
|
||||
f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]")
|
||||
|
||||
# Create progress bar
|
||||
progress_bar = tqdm(
|
||||
total=total,
|
||||
ascii='░▒█',
|
||||
bar_format=bar_format,
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
mininterval=0.05
|
||||
)
|
||||
|
||||
# Download file
|
||||
with open(path, 'wb') as file, progress_bar as bar:
|
||||
for chunk in response.iter_bytes(chunk_size=1024):
|
||||
if chunk:
|
||||
size = file.write(chunk)
|
||||
bar.update(size)
|
||||
|
||||
else:
|
||||
bar_format = (f"{Colors.YELLOW}Proc{Colors.WHITE}: {Colors.RED}{{percentage:.2f}}% "
|
||||
f"{Colors.WHITE}| {Colors.CYAN}{{remaining}}{{postfix}} {Colors.WHITE}]")
|
||||
console.print("[red]Cant find any stream.")
|
||||
|
||||
# Create progress bar
|
||||
progress_bar = tqdm(
|
||||
total=total,
|
||||
ascii='░▒█',
|
||||
bar_format=bar_format,
|
||||
unit_scale=True,
|
||||
unit_divisor=1024,
|
||||
mininterval=0.05
|
||||
)
|
||||
|
||||
# Download file
|
||||
with open(path, 'wb') as file, progress_bar as bar:
|
||||
for chunk in response.iter_bytes(chunk_size=1024):
|
||||
if chunk:
|
||||
size = file.write(chunk)
|
||||
bar.update(size)
|
||||
|
||||
# Get summary
|
||||
console.print(Panel(
|
||||
f"[bold green]Download completed![/bold green]\n"
|
||||
f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n"
|
||||
f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]",
|
||||
title=f"{os.path.basename(path.replace('.mp4', ''))}",
|
||||
border_style="green"
|
||||
))
|
||||
# Get summary
|
||||
if total != 0:
|
||||
console.print(Panel(
|
||||
f"[bold green]Download completed![/bold green]\n"
|
||||
f"[cyan]File size: [bold red]{format_file_size(os.path.getsize(path))}[/bold red]\n"
|
||||
f"[cyan]Duration: [bold]{print_duration_table(path, description=False, return_string=True)}[/bold]",
|
||||
title=f"{os.path.basename(path.replace('.mp4', ''))}",
|
||||
border_style="green"
|
||||
))
|
||||
|
@ -55,7 +55,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
|
||||
|
||||
# Enabled the use of gpu
|
||||
if USE_GPU:
|
||||
ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda'])
|
||||
ffmpeg_cmd.extend(['-hwaccel', 'cuda'])
|
||||
|
||||
# Add mpegts to force to detect input file as ts file
|
||||
if need_to_force_to_ts(video_path):
|
||||
@ -67,7 +67,7 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
|
||||
ffmpeg_cmd.extend(['-i', video_path])
|
||||
|
||||
# Add output Parameters
|
||||
if USE_CODEC:
|
||||
if USE_CODEC and codec != None:
|
||||
if USE_VCODEC:
|
||||
if codec.video_codec_name:
|
||||
if not USE_GPU:
|
||||
@ -76,6 +76,10 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
|
||||
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
|
||||
else:
|
||||
console.log("[red]Cant find vcodec for 'join_audios'")
|
||||
else:
|
||||
if USE_GPU:
|
||||
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
|
||||
|
||||
|
||||
if USE_ACODEC:
|
||||
if codec.audio_codec_name:
|
||||
@ -98,7 +102,6 @@ def join_video(video_path: str, out_path: str, codec: M3U8_Codec = None):
|
||||
|
||||
# Overwrite
|
||||
ffmpeg_cmd += [out_path, "-y"]
|
||||
logging.info(f"FFmpeg command: {ffmpeg_cmd}")
|
||||
|
||||
# Run join
|
||||
if DEBUG_MODE:
|
||||
@ -143,7 +146,7 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
|
||||
|
||||
# Enabled the use of gpu
|
||||
if USE_GPU:
|
||||
ffmpeg_cmd.extend(['-hwaccel', 'cuda', '-hwaccel_output_format', 'cuda'])
|
||||
ffmpeg_cmd.extend(['-hwaccel', 'cuda'])
|
||||
|
||||
# Insert input video path
|
||||
ffmpeg_cmd.extend(['-i', video_path])
|
||||
@ -173,6 +176,9 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
|
||||
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
|
||||
else:
|
||||
console.log("[red]Cant find vcodec for 'join_audios'")
|
||||
else:
|
||||
if USE_GPU:
|
||||
ffmpeg_cmd.extend(['-c:v', 'h264_nvenc'])
|
||||
|
||||
if USE_ACODEC:
|
||||
if codec.audio_codec_name:
|
||||
@ -195,12 +201,11 @@ def join_audios(video_path: str, audio_tracks: List[Dict[str, str]], out_path: s
|
||||
|
||||
# Use shortest input path for video and audios
|
||||
if not video_audio_same_duration:
|
||||
console.log("[red]Use shortest input.")
|
||||
logging.info("[red]Use shortest input.")
|
||||
ffmpeg_cmd.extend(['-shortest', '-strict', 'experimental'])
|
||||
|
||||
# Overwrite
|
||||
ffmpeg_cmd += [out_path, "-y"]
|
||||
logging.info(f"FFmpeg command: {ffmpeg_cmd}")
|
||||
|
||||
# Run join
|
||||
if DEBUG_MODE:
|
||||
|
@ -164,6 +164,13 @@ class M3U8_Codec:
|
||||
else:
|
||||
logging.warning("No bandwidth provided. Bitrates cannot be calculated.")
|
||||
|
||||
def __str__(self):
|
||||
return (f"M3U8_Codec(bandwidth={self.bandwidth}, "
|
||||
f"codecs='{self.codecs}', "
|
||||
f"audio_codec='{self.audio_codec}', "
|
||||
f"video_codec='{self.video_codec}', "
|
||||
f"audio_codec_name='{self.audio_codec_name}', "
|
||||
f"video_codec_name='{self.video_codec_name}')")
|
||||
|
||||
|
||||
class M3U8_Video:
|
||||
|
@ -164,7 +164,6 @@ def check_file_existence(file_path):
|
||||
return True
|
||||
|
||||
else:
|
||||
logging.warning(f"The file '{file_path}' does not exist.")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
|
@ -17,5 +17,5 @@ from Src.Lib.Downloader import MP4_downloader
|
||||
# Test
|
||||
MP4_downloader(
|
||||
"",
|
||||
"EP_1.mp4",
|
||||
"EP_1.mp4"
|
||||
)
|
||||
|
25
config.json
25
config.json
@ -42,8 +42,7 @@
|
||||
"eng",
|
||||
"spa"
|
||||
],
|
||||
"cleanup_tmp_folder": true,
|
||||
"create_report": false
|
||||
"cleanup_tmp_folder": true
|
||||
},
|
||||
"M3U8_CONVERSION": {
|
||||
"use_codec": false,
|
||||
@ -58,23 +57,23 @@
|
||||
},
|
||||
"SITE": {
|
||||
"streamingcommunity": {
|
||||
"video_workers": 4,
|
||||
"audio_workers": 4,
|
||||
"video_workers": 6,
|
||||
"audio_workers": 6,
|
||||
"domain": "computer"
|
||||
},
|
||||
"altadefinizione": {
|
||||
"video_workers": -1,
|
||||
"audio_workers": -1,
|
||||
"video_workers": 12,
|
||||
"audio_workers": 12,
|
||||
"domain": "my"
|
||||
},
|
||||
"guardaserie": {
|
||||
"video_workers": -1,
|
||||
"audio_workers": -1,
|
||||
"video_workers": 12,
|
||||
"audio_workers": 12,
|
||||
"domain": "dev"
|
||||
},
|
||||
"mostraguarda": {
|
||||
"video_workers": -1,
|
||||
"audio_workers": -1,
|
||||
"video_workers": 12,
|
||||
"audio_workers": 12,
|
||||
"domain": "stream"
|
||||
},
|
||||
"ddlstreamitaly": {
|
||||
@ -88,12 +87,6 @@
|
||||
"animeunity": {
|
||||
"domain": "to"
|
||||
},
|
||||
"watch_lonelil": {
|
||||
"domain": "ru"
|
||||
},
|
||||
"uhdmovies": {
|
||||
"domain": "mov"
|
||||
},
|
||||
"bitsearch": {
|
||||
"domain": "to"
|
||||
},
|
||||
|
Loading…
x
Reference in New Issue
Block a user