diff --git a/README.md b/README.md index 2b1bdfc..b09e638 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This repository provide a simple script designed to facilitate the downloading of films and series from a popular streaming community platform. The script allows users to download individual films, entire series, or specific episodes, providing a seamless experience for content consumers. ## Join us -You can chat, help improve this repo, or just hang around for some fun in the **Git_StreamingCommunity** Discord [Server](https://discord.com/invite/qP3nsCXV5z) +You can chat, help improve this repo, or just hang around for some fun in the **Git_StreamingCommunity** Discord [Server](https://discord.gg/we8n4tfxFs) # Table of Contents * [INSTALLATION](#installation) diff --git a/Src/Api/Class/EpisodeType.py b/Src/Api/Class/EpisodeType.py index fcd20c2..c805c2f 100644 --- a/Src/Api/Class/EpisodeType.py +++ b/Src/Api/Class/EpisodeType.py @@ -21,6 +21,7 @@ class Episode: self.created_at: str = data.get('created_at', '') self.updated_at: str = data.get('updated_at', '') + class EpisodeManager: def __init__(self): """ diff --git a/Src/Api/Class/SearchType.py b/Src/Api/Class/SearchType.py index 0d369fc..f0d7fe9 100644 --- a/Src/Api/Class/SearchType.py +++ b/Src/Api/Class/SearchType.py @@ -1,6 +1,5 @@ # 03.03.24 -# Import from typing import List class Image: diff --git a/Src/Api/Class/SeriesType.py b/Src/Api/Class/SeriesType.py index efee669..9bb62e0 100644 --- a/Src/Api/Class/SeriesType.py +++ b/Src/Api/Class/SeriesType.py @@ -2,6 +2,7 @@ from typing import List, Dict, Union + class Title: def __init__(self, title_data: Dict[str, Union[int, str, None]]): """ @@ -20,6 +21,7 @@ class Title: self.updated_at: str = title_data.get('updated_at') self.episodes_count: int = title_data.get('episodes_count') + class TitleManager: def __init__(self): """ diff --git a/Src/Api/Class/Video.py b/Src/Api/Class/Video.py index 4bce998..06f0076 100644 --- a/Src/Api/Class/Video.py +++ b/Src/Api/Class/Video.py @@ -1,22 +1,26 @@ # 01.03.24 -# Class import -from Src.Util.headers import get_headers -from .SeriesType import TitleManager -from .EpisodeType import EpisodeManager -from .WindowType import WindowVideo, WindowParameter - -# Import import requests import re import json import binascii import logging import sys -from bs4 import BeautifulSoup from urllib.parse import urljoin, urlencode, quote +# External libraries +import requests +from bs4 import BeautifulSoup + + +# Internal utilities +from Src.Util.headers import get_headers +from .SeriesType import TitleManager +from .EpisodeType import EpisodeManager +from .WindowType import WindowVideo, WindowParameter + + class VideoSource: def __init__(self): """ diff --git a/Src/Api/Class/WindowType.py b/Src/Api/Class/WindowType.py index 61cd496..05047f4 100644 --- a/Src/Api/Class/WindowType.py +++ b/Src/Api/Class/WindowType.py @@ -2,6 +2,7 @@ from typing import Dict, Any + class WindowVideo: def __init__(self, data: Dict[str, Any]): """ @@ -25,6 +26,7 @@ class WindowVideo: self.folder_id: int = data.get('folder_id', '') self.created_at_diff: str = data.get('created_at_diff', '') + class WindowParameter: def __init__(self, data: Dict[str, Any]): """ diff --git a/Src/Api/anime.py b/Src/Api/anime.py index 3ab2611..6ce20e5 100644 --- a/Src/Api/anime.py +++ b/Src/Api/anime.py @@ -1,16 +1,20 @@ # 11.03.24 -# Class import +import os +import logging + + +# External libraries +import requests + + +# Internal utilities from Src.Util.console import console, msg from Src.Util.config import config_manager from Src.Lib.FFmpeg.my_m3u8 import Downloader from Src.Util.message import start_message from .Class import VideoSource -# General import -import os -import logging -import requests # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') @@ -20,6 +24,7 @@ SERIES_FOLDER = config_manager.get('DEFAULT', 'series_folder_name') URL_SITE_NAME = config_manager.get('SITE', 'anime_site_name') SITE_DOMAIN = config_manager.get('SITE', 'anime_domain') + # Variable video_source = VideoSource() diff --git a/Src/Api/film.py b/Src/Api/film.py index 2361ff7..41420ba 100644 --- a/Src/Api/film.py +++ b/Src/Api/film.py @@ -1,21 +1,23 @@ # 3.12.23 -> 10.12.23 -# Class import +import os +import logging + + +# Internal utilities from Src.Util.console import console from Src.Util.config import config_manager from Src.Lib.FFmpeg.my_m3u8 import Downloader from Src.Util.message import start_message from .Class import VideoSource -# General import -import os -import logging # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') MOVIE_FOLDER = config_manager.get('DEFAULT', 'movies_folder_name') STREAM_SITE_NAME = config_manager.get('SITE', 'streaming_site_name') + # Variable video_source = VideoSource() video_source.set_url_base_name(STREAM_SITE_NAME) diff --git a/Src/Api/series.py b/Src/Api/series.py index 6b9042c..6260d1f 100644 --- a/Src/Api/series.py +++ b/Src/Api/series.py @@ -1,6 +1,11 @@ # 3.12.23 -> 10.12.23 -# Class import +import os +import sys +import logging + + +# Internal utilities from Src.Util.console import console, msg from Src.Util.config import config_manager from Src.Util.table import TVShowManager @@ -9,16 +14,13 @@ from Src.Lib.Unidecode import transliterate from Src.Lib.FFmpeg.my_m3u8 import Downloader from .Class import VideoSource -# General import -import os -import logging -import sys # Config ROOT_PATH = config_manager.get('DEFAULT', 'root_path') SERIES_FOLDER = config_manager.get('DEFAULT', 'series_folder_name') STREAM_SITE_NAME = config_manager.get('SITE', 'streaming_site_name') + # Variable video_source = VideoSource() video_source.set_url_base_name(STREAM_SITE_NAME) @@ -56,6 +58,7 @@ def manage_selection(cmd_insert: str, max_count: int) -> list[int]: # Return list of selected seasons) return list_season_select + def display_episodes_list() -> str: """ Display episodes list and handle user input. @@ -133,6 +136,7 @@ def donwload_video(tv_name: str, index_season_selected: int, index_episode_selec logging.error(f"(donwload_video) Error downloading film: {e}") pass + def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: bool = False) -> None: """ Download all episodes of a season. @@ -174,6 +178,7 @@ def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: boo for i_episode in list_episode_select: donwload_video(tv_name, index_season_selected, i_episode) + def download_series(tv_id: str, tv_name: str, version: str, domain: str) -> None: """ Download all episodes of a TV series. diff --git a/Src/Api/site.py b/Src/Api/site.py index 201d6e2..195a561 100644 --- a/Src/Api/site.py +++ b/Src/Api/site.py @@ -1,26 +1,32 @@ # 10.12.23 -# Class import +import sys +import json +import logging + + +# External libraries +import requests +from bs4 import BeautifulSoup + + +# Internal utilities from Src.Util.table import TVShowManager from Src.Util.headers import get_headers from Src.Util.console import console from Src.Util.config import config_manager from .Class import MediaManager, MediaItem -# General import -import sys -import json -import logging -import requests -from bs4 import BeautifulSoup # Config GET_TITLES_OF_MOMENT = config_manager.get_bool('DEFAULT', 'get_moment_title') + # Variable media_search_manager = MediaManager() table_show_manager = TVShowManager() + def get_token(site_name: str, domain: str) -> dict: """ Function to retrieve session tokens from a specified website. @@ -61,6 +67,7 @@ def get_token(site_name: str, domain: str) -> dict: 'csrf_token': find_csrf_token } + def get_moment_titles(domain: str, version: str, prefix: str): """ Retrieves the title name from a specified domain using the provided version and prefix. @@ -100,6 +107,7 @@ def get_moment_titles(domain: str, version: str, prefix: str): logging.error("Error occurred: %s", str(e)) return None + def get_domain() -> str: """ Fetches the domain from a Telegra.ph API response. @@ -123,6 +131,7 @@ def get_domain() -> str: logging.error(f"Error fetching domain: {e}") sys.exit(0) + def test_site(domain: str) -> str: """ Tests the availability of a website. @@ -151,6 +160,7 @@ def test_site(domain: str) -> str: logging.error(f"Error testing site: {e}") return None + def get_version(text: str) -> str: """ Extracts the version from the HTML text of a webpage. @@ -174,6 +184,7 @@ def get_version(text: str) -> str: logging.error(f"Error extracting version: {e}") sys.exit(0) + def get_version_and_domain() -> tuple[str, str]: """ Retrieves the version and domain of a website. @@ -215,6 +226,7 @@ def get_version_and_domain() -> tuple[str, str]: logging.error(f"Error getting version and domain: {e}") sys.exit(0) + def search(title_search: str, domain: str) -> int: """ Search for titles based on a search query. @@ -237,6 +249,7 @@ def search(title_search: str, domain: str) -> int: # Return the number of titles found return media_search_manager.get_length() + def update_domain_anime(): """ Update the domain for anime streaming site. @@ -268,6 +281,7 @@ def update_domain_anime(): # Extract the domain from the URL and update the config config_manager.set_key('SITE', 'anime_domain', new_site_url.split(".")[-1]) + def anime_search(title_search: str) -> int: """ Function to perform an anime search using a provided title. @@ -322,6 +336,7 @@ def anime_search(title_search: str) -> int: # Return the length of media search manager return media_search_manager.get_length() + def get_select_title() -> MediaItem: """ Display a selection of titles and prompt the user to choose one. diff --git a/Src/Lib/FFmpeg/my_m3u8.py b/Src/Lib/FFmpeg/my_m3u8.py index beeaa58..60447dd 100644 --- a/Src/Lib/FFmpeg/my_m3u8.py +++ b/Src/Lib/FFmpeg/my_m3u8.py @@ -1,6 +1,5 @@ # 5.01.24 -> 7.01.24 -> 20.02.24 -> 29.03.24 -# Importing modules import os import sys import time @@ -8,16 +7,19 @@ import threading import logging import warnings + # Disable specific warnings from tqdm import TqdmExperimentalWarning warnings.filterwarnings("ignore", category=TqdmExperimentalWarning) warnings.filterwarnings("ignore", category=UserWarning, module="cryptography") + # External libraries import requests from tqdm.rich import tqdm from concurrent.futures import ThreadPoolExecutor, as_completed + # Internal utilities from Src.Util.console import console from Src.Util.headers import get_headers @@ -30,6 +32,7 @@ from Src.Util.os import ( convert_to_hex ) + # Logic class from .util import ( print_duration_table, @@ -44,6 +47,7 @@ from .util import ( M3U8_UrlFix ) + # Config Download_audio = config_manager.get_bool('M3U8_OPTIONS', 'download_audio') Donwload_subtitles = config_manager.get_bool('M3U8_OPTIONS', 'download_subtitles') @@ -59,12 +63,12 @@ TQDM_SHOW_PROGRESS = config_manager.get_bool('M3U8', 'tqdm_show_progress') MIN_TS_FILES_IN_FOLDER = config_manager.get_int('M3U8', 'minimum_ts_files_in_folder') REMOVE_SEGMENTS_FOLDER = config_manager.get_bool('M3U8', 'cleanup_tmp_folder') + # Variable config_headers = config_manager.get_dict('M3U8_OPTIONS', 'request') failed_segments = [] class_urlFix = M3U8_UrlFix() -# [ main class ] class M3U8_Segments: def __init__(self, url, folder, key=None): diff --git a/Src/Lib/FFmpeg/util/decryption.py b/Src/Lib/FFmpeg/util/decryption.py index 0bfe7f6..d30ae09 100644 --- a/Src/Lib/FFmpeg/util/decryption.py +++ b/Src/Lib/FFmpeg/util/decryption.py @@ -1,14 +1,15 @@ # 03.04.24 -# Import import subprocess import logging import os + # External library from Crypto.Cipher import AES from Crypto.Util.Padding import unpad + class AES_ECB: def __init__(self, key: bytes) -> None: """ @@ -49,6 +50,7 @@ class AES_ECB: decrypted_data = cipher.decrypt(ciphertext) return unpad(decrypted_data, AES.block_size) + class AES_CBC: def __init__(self, key: bytes, iv: bytes) -> None: """ @@ -91,6 +93,7 @@ class AES_CBC: decrypted_data = cipher.decrypt(ciphertext) return unpad(decrypted_data, AES.block_size) + class AES_CTR: def __init__(self, key: bytes, nonce: bytes) -> None: """ @@ -132,6 +135,7 @@ class AES_CTR: cipher = AES.new(self.key, AES.MODE_CTR, nonce=self.nonce) return cipher.decrypt(ciphertext) + class M3U8_Decryption: def __init__(self, key: bytes, iv: bytes = None) -> None: """ diff --git a/Src/Lib/FFmpeg/util/helper.py b/Src/Lib/FFmpeg/util/helper.py index 7d82c56..a86ac07 100644 --- a/Src/Lib/FFmpeg/util/helper.py +++ b/Src/Lib/FFmpeg/util/helper.py @@ -1,16 +1,20 @@ # 31.01.24 -# Class -from Src.Util.console import console - -# Import -import ffmpeg import subprocess import os import json import logging import shutil + +# External libraries +import ffmpeg + + +# Internal utilities +from Src.Util.console import console + + def has_audio_stream(video_path: str) -> bool: """ Check if the input video has an audio stream. @@ -37,6 +41,7 @@ def has_audio_stream(video_path: str) -> bool: logging.error(f"Error: {e.stderr}") return None + def get_video_duration(file_path: str) -> (float): """ Get the duration of a video file. @@ -64,6 +69,7 @@ def get_video_duration(file_path: str) -> (float): logging.error(f"Error: {e.stderr}") return None + def format_duration(seconds: float) -> list[int, int, int]: """ Format duration in seconds into hours, minutes, and seconds. @@ -80,6 +86,7 @@ def format_duration(seconds: float) -> list[int, int, int]: return int(hours), int(minutes), int(seconds) + def print_duration_table(file_path: str) -> None: """ Print duration of a video file in hours, minutes, and seconds. @@ -98,7 +105,7 @@ def print_duration_table(file_path: str) -> None: # Print the formatted duration console.log(f"[cyan]Info [green]'{file_path}': [purple]{int(hours)}[red]h [purple]{int(minutes)}[red]m [purple]{int(seconds)}[red]s") -# SINGLE SUBTITLE + def add_subtitle(input_video_path: str, input_subtitle_path: str, output_video_path: str, subtitle_language: str = 'ita', prefix: str = "single_sub") -> str: """ Convert a video with a single subtitle. @@ -153,7 +160,7 @@ def add_subtitle(input_video_path: str, input_subtitle_path: str, output_video_p # Return return output_video_path -# SEGMENTS + def concatenate_and_save(file_list_path: str, output_filename: str, video_decoding: str = None, audio_decoding: str = None, prefix: str = "segments", output_directory: str = None) -> str: """ Concatenate input files and save the output with specified decoding parameters. @@ -222,7 +229,7 @@ def concatenate_and_save(file_list_path: str, output_filename: str, video_decodi # Return return output_file_path -# AUDIOS + def join_audios(video_path: str, audio_tracks: list[dict[str, str]], prefix: str = "merged") -> str: """ Join video with multiple audio tracks and sync them if there are matching segments. @@ -300,7 +307,7 @@ def join_audios(video_path: str, audio_tracks: list[dict[str, str]], prefix: str logging.error("[M3U8_Downloader] Ffmpeg error: %s", ffmpeg_error) return "" -# SUBTITLES + def transcode_with_subtitles(video: str, subtitles_list: list[dict[str, str]], output_file: str, prefix: str = "transcoded") -> str: """ diff --git a/Src/Lib/FFmpeg/util/installer.py b/Src/Lib/FFmpeg/util/installer.py index 07a8f03..cf154c3 100644 --- a/Src/Lib/FFmpeg/util/installer.py +++ b/Src/Lib/FFmpeg/util/installer.py @@ -1,9 +1,5 @@ # 24.01.2023 -# Class -from Src.Util.console import console - -# Import import subprocess import os import requests @@ -12,7 +8,10 @@ import sys import ctypes -# [ func ] +# Internal utilities +from Src.Util.console import console + + def isAdmin() -> (bool): """ Check if the current user has administrative privileges. @@ -29,6 +28,7 @@ def isAdmin() -> (bool): return is_admin + def get_version(): """ Get the version of FFmpeg installed on the system. @@ -55,6 +55,7 @@ def get_version(): print("Error executing FFmpeg command:", e.output.strip()) raise e + def download_ffmpeg(): """ Download FFmpeg binary for Windows and add it to the system PATH. @@ -102,6 +103,7 @@ def download_ffmpeg(): print(f"Failed to extract FFmpeg zip file: {e}") raise e + def check_ffmpeg(): """ Check if FFmpeg is installed and available on the system PATH. diff --git a/Src/Lib/FFmpeg/util/math_calc.py b/Src/Lib/FFmpeg/util/math_calc.py index bdb4056..89d919d 100644 --- a/Src/Lib/FFmpeg/util/math_calc.py +++ b/Src/Lib/FFmpeg/util/math_calc.py @@ -1,7 +1,9 @@ # 29.02.24 +# Internal utilities from Src.Util.os import format_size + class M3U8_Ts_Files: def __init__(self): """ diff --git a/Src/Lib/FFmpeg/util/parser.py b/Src/Lib/FFmpeg/util/parser.py index a69c04d..df3759e 100644 --- a/Src/Lib/FFmpeg/util/parser.py +++ b/Src/Lib/FFmpeg/util/parser.py @@ -1,12 +1,16 @@ # 29.04.25 -# Class import +import logging + + +# Internal utilities from Src.Util.headers import get_headers -# Import -from m3u8 import M3U8 -import logging + +# External libraries import requests +from m3u8 import M3U8 + class M3U8_Parser: def __init__(self, DOWNLOAD_SPECIFIC_SUBTITLE = None): diff --git a/Src/Lib/FFmpeg/util/url_fix.py b/Src/Lib/FFmpeg/util/url_fix.py index d5febc7..f679c80 100644 --- a/Src/Lib/FFmpeg/util/url_fix.py +++ b/Src/Lib/FFmpeg/util/url_fix.py @@ -1,10 +1,10 @@ # 29.03.24 -# Import -import logging import sys +import logging from urllib.parse import urlparse, urljoin + class M3U8_UrlFix: def __init__(self) -> None: diff --git a/Src/Lib/Request/my_requests.py b/Src/Lib/Request/my_requests.py new file mode 100644 index 0000000..6610c87 --- /dev/null +++ b/Src/Lib/Request/my_requests.py @@ -0,0 +1,360 @@ +# 04.4.24 + +import base64 +import json +import logging +import ssl +import time +import re +import urllib.parse +import urllib.request +import urllib.error +from typing import Dict, Optional, Union, Unpack + + +# Constants +HTTP_TIMEOUT = 4 +HTTP_RETRIES = 2 +HTTP_DELAY = 1 + + +class RequestError(Exception): + """Custom exception class for request errors.""" + + def __init__(self, message: str, original_exception: Optional[Exception] = None) -> None: + """ + Initialize a RequestError instance. + + Args: + message (str): The error message. + original_exception (Optional[Exception], optional): The original exception that occurred. Defaults to None. + """ + super().__init__(message) + self.original_exception = original_exception + + def __str__(self) -> str: + """Return a string representation of the exception.""" + if self.original_exception: + return f"{super().__str__()} Original Exception: {type(self.original_exception).__name__}: {str(self.original_exception)}" + else: + return super().__str__() + + +class Response: + """Class representing an HTTP response.""" + def __init__( + self, + status: int, + text: str, + is_json: bool = False, + content: bytes = b"", + headers: Optional[Dict[str, str]] = None, + cookies: Optional[Dict[str, str]] = None, + redirect_url: Optional[str] = None, + response_time: Optional[float] = None, + timeout: Optional[float] = None, + ): + """ + Initialize a Response object. + + Args: + status (int): The HTTP status code of the response. + text (str): The response content as text. + is_json (bool, optional): Indicates if the response content is JSON. Defaults to False. + content (bytes, optional): The response content as bytes. Defaults to b"". + headers (Optional[Dict[str, str]], optional): The response headers. Defaults to None. + cookies (Optional[Dict[str, str]], optional): The cookies set in the response. Defaults to None. + redirect_url (Optional[str], optional): The URL if a redirection occurred. Defaults to None. + response_time (Optional[float], optional): The time taken to receive the response. Defaults to None. + timeout (Optional[float], optional): The request timeout. Defaults to None. + """ + self.status_code = status + self.text = text + self.is_json = is_json + self.content = content + self.headers = headers or {} + self.cookies = cookies or {} + self.redirect_url = redirect_url + self.response_time = response_time + self.timeout = timeout + self.ok = 200 <= status < 300 + + def raise_for_status(self): + """Raise an error if the response status code is not in the 2xx range.""" + if not self.ok: + raise RequestError(f"Request failed with status code {self.status_code}") + + def json(self): + """Return the response content as JSON if it is JSON.""" + if self.is_json: + return json.loads(self.text) + else: + return None + + +class ManageRequests: + """Class for managing HTTP requests.""" + def __init__( + self, + url: str, + method: str = 'GET', + headers: Optional[Dict[str, str]] = None, + timeout: float = HTTP_TIMEOUT, + retries: int = HTTP_RETRIES, + params: Optional[Dict[str, str]] = None, + verify_ssl: bool = True, + auth: Optional[tuple] = None, + proxy: Optional[str] = None, + cookies: Optional[Dict[str, str]] = None, + redirection_handling: bool = True, + ): + """ + Initialize a ManageRequests object. + + Args: + url (str): The URL to which the request will be sent. + method (str, optional): The HTTP method to be used for the request. Defaults to 'GET'. + headers (Optional[Dict[str, str]], optional): The request headers. Defaults to None. + timeout (float, optional): The request timeout. Defaults to HTTP_TIMEOUT. + retries (int, optional): The number of retries in case of request failure. Defaults to HTTP_RETRIES. + params (Optional[Dict[str, str]], optional): The query parameters for the request. Defaults to None. + verify_ssl (bool, optional): Indicates whether SSL certificate verification should be performed. Defaults to True. + auth (Optional[tuple], optional): Tuple containing the username and password for basic authentication. Defaults to None. + proxy (Optional[str], optional): The proxy URL. Defaults to None. + cookies (Optional[Dict[str, str]], optional): The cookies to be included in the request. Defaults to None. + redirection_handling (bool, optional): Indicates whether redirections should be followed. Defaults to True. + """ + self.url = url + self.method = method + self.headers = headers or {'User-Agent': 'Mozilla/5.0'} + self.timeout = timeout + self.retries = retries + self.params = params + self.verify_ssl = verify_ssl + self.auth = auth + self.proxy = proxy + self.cookies = cookies + self.redirection_handling = redirection_handling + + def add_header(self, key: str, value: str) -> None: + """Add a header to the request.""" + self.headers[key] = value + + def send(self) -> Response: + """Send the HTTP request.""" + start_time = time.time() + self.attempt = 0 + redirect_url = None + + while self.attempt < self.retries: + try: + req = self._build_request() + response = self._perform_request(req) + return self._process_response(response, start_time, redirect_url) + except (urllib.error.URLError, urllib.error.HTTPError) as e: + self._handle_error(e) + attempt += 1 + + def _build_request(self) -> urllib.request.Request: + """Build the urllib Request object.""" + headers = self.headers.copy() + if self.params: + url = self.url + '?' + urllib.parse.urlencode(self.params) + else: + url = self.url + req = urllib.request.Request(url, headers=headers, method=self.method) + if self.auth: + req.add_header('Authorization', 'Basic ' + base64.b64encode(f"{self.auth[0]}:{self.auth[1]}".encode()).decode()) + if self.cookies: + cookie_str = '; '.join([f"{name}={value}" for name, value in self.cookies.items()]) + req.add_header('Cookie', cookie_str) + return req + + def _perform_request(self, req: urllib.request.Request) -> urllib.response.addinfourl: + """Perform the HTTP request.""" + if self.proxy: + proxy_handler = urllib.request.ProxyHandler({'http': self.proxy, 'https': self.proxy}) + opener = urllib.request.build_opener(proxy_handler) + urllib.request.install_opener(opener) + if not self.verify_ssl: + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + response = urllib.request.urlopen(req, timeout=self.timeout, context=ssl_context) + else: + response = urllib.request.urlopen(req, timeout=self.timeout) + return response + + def _process_response(self, response: urllib.response.addinfourl, start_time: float, redirect_url: Optional[str]) -> Response: + """Process the HTTP response.""" + response_data = response.read() + content_type = response.headers.get('Content-Type', '').lower() + is_response_api = "json" in content_type + if self.redirection_handling and response.status in (301, 302, 303, 307, 308): + location = response.headers.get('Location') + logging.info(f"Redirecting to: {location}") + redirect_url = location + self.url = location + return self.send() + return self._build_response(response, response_data, start_time, redirect_url, content_type) + + def _build_response(self, response: urllib.response.addinfourl, response_data: bytes, start_time: float, redirect_url: Optional[str], content_type: str) -> Response: + """Build the Response object.""" + response_time = time.time() - start_time + response_headers = dict(response.headers) + response_cookies = {} + + for cookie in response.headers.get_all('Set-Cookie', []): + cookie_parts = cookie.split(';') + cookie_name, cookie_value = cookie_parts[0].split('=') + response_cookies[cookie_name.strip()] = cookie_value.strip() + + return Response( + status=response.status, + text=response_data.decode('latin-1'), + is_json=("json" in content_type), + content=response_data, + headers=response_headers, + cookies=response_cookies, + redirect_url=redirect_url, + response_time=response_time, + timeout=self.timeout, + ) + + def _handle_error(self, e: Union[urllib.error.URLError, urllib.error.HTTPError]) -> None: + """Handle request error.""" + logging.error(f"Request failed for URL '{self.url}': {str(e)}") + if self.attempt < self.retries: + logging.info(f"Retrying request for URL '{self.url}' (attempt {self.attempt}/{self.retries})") + time.sleep(HTTP_DELAY) + else: + logging.error(f"Maximum retries reached for URL '{self.url}'") + raise RequestError(str(e)) + + +class ValidateRequest: + """Class for validating request inputs.""" + @staticmethod + def validate_url(url: str) -> bool: + """Validate URL format.""" + url_regex = re.compile( + r'^(?:http|ftp)s?://' # http:// or https:// + r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|' # domain... + r'localhost|' # localhost... + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or IP + r'(?::\d+)?' # optional port + r'(?:/?|[/?]\S+)$', re.IGNORECASE) + return re.match(url_regex, url) is not None + + @staticmethod + def validate_headers(headers: Dict[str, str]) -> bool: + """Validate header values.""" + for key, value in headers.items(): + if not isinstance(key, str) or not isinstance(value, str): + return False + return True + + +class ValidateResponse: + """Class for validating response data.""" + @staticmethod + def is_valid_json(data: str) -> bool: + """Check if response data is a valid JSON.""" + try: + json.loads(data) + return True + except ValueError: + return False + + +class SSLHandler: + """Class for handling SSL certificates.""" + @staticmethod + def load_certificate(custom_cert_path: str) -> None: + """Load custom SSL certificate.""" + ssl_context = ssl.create_default_context(cafile=custom_cert_path) + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + + +class KwargsRequest(): + """Class representing keyword arguments for a request.""" + url: str + headers: Optional[Dict[str, str]] = None + timeout: float = HTTP_TIMEOUT + retries: int = HTTP_RETRIES + params: Optional[Dict[str, str]] = None + cookies: Optional[Dict[str, str]] = None + + +class Request: + """Class for making HTTP requests.""" + def __init__(self) -> None: + pass + + def get(self, url: str, **kwargs: Unpack[KwargsRequest]): + """ + Send a GET request. + + Args: + url (str): The URL to which the request will be sent. + **kwargs: Additional keyword arguments for the request. + + Returns: + Response: The response object. + """ + return self._send_request(url, 'GET', **kwargs) + + def post(self, url: str, **kwargs: Unpack[KwargsRequest]): + """ + Send a POST request. + + Args: + url (str): The URL to which the request will be sent. + **kwargs: Additional keyword arguments for the request. + + Returns: + Response: The response object. + """ + return self._send_request(url, 'POST', **kwargs) + + def put(self, url: str, **kwargs: Unpack[KwargsRequest]): + """ + Send a PUT request. + + Args: + url (str): The URL to which the request will be sent. + **kwargs: Additional keyword arguments for the request. + + Returns: + Response: The response object. + """ + return self._send_request(url, 'PUT', **kwargs) + + def delete(self, url: str, **kwargs: Unpack[KwargsRequest]): + """ + Send a DELETE request. + + Args: + url (str): The URL to which the request will be sent. + **kwargs: Additional keyword arguments for the request. + + Returns: + Response: The response object. + """ + return self._send_request(url, 'DELETE', **kwargs) + + def _send_request(self, url: str, method: str, **kwargs: Unpack[KwargsRequest]): + """Send an HTTP request.""" + # Add validation checks for URL and headers + if not ValidateRequest.validate_url(url): + raise ValueError("Invalid URL format") + + if 'headers' in kwargs and not ValidateRequest.validate_headers(kwargs['headers']): + raise ValueError("Invalid header values") + + return ManageRequests(url, method, **kwargs).send() + + +# Out +request = Request() \ No newline at end of file diff --git a/Src/Lib/Request/user_agent.py b/Src/Lib/Request/user_agent.py new file mode 100644 index 0000000..42ebf0c --- /dev/null +++ b/Src/Lib/Request/user_agent.py @@ -0,0 +1,105 @@ +# 04.4.24 + +import logging +import re +import os +import random +import threading +import json +from typing import Dict, List + +# Internal utilities +from .my_requests import request + + +def get_browser_user_agents_online(browser: str) -> List[str]: + """ + Retrieve browser user agent strings from a website. + + Args: + browser (str): The name of the browser (e.g., 'chrome', 'firefox', 'safari'). + + Returns: + List[str]: List of user agent strings for the specified browser. + """ + url = f"https://useragentstring.com/pages/{browser}/" + + try: + + # Make request and find all user agents + html = request.get(url).text + browser_user_agents = re.findall(r"(.+?)", html, re.UNICODE) + return [ua for ua in browser_user_agents if "more" not in ua.lower()] + + except Exception as e: + logging.error(f"Failed to fetch user agents for '{browser}': {str(e)}") + return [] + + +def update_user_agents(browser_name: str, browser_user_agents: Dict[str, List[str]]) -> None: + """ + Update browser user agents dictionary with new requests. + + Args: + browser_name (str): Name of the browser. + browser_user_agents (Dict[str, List[str]]): Dictionary to store browser user agents. + """ + browser_user_agents[browser_name] = get_browser_user_agents_online(browser_name) + + +def create_or_update_user_agent_file() -> None: + """ + Create or update the user agent file with browser user agents. + """ + user_agent_file = os.path.join(os.environ.get('TEMP'), 'fake_user_agent.json') + + if not os.path.exists(user_agent_file): + browser_user_agents: Dict[str, List[str]] = {} + threads = [] + + for browser_name in ['chrome', 'firefox', 'safari']: + t = threading.Thread(target=update_user_agents, args=(browser_name, browser_user_agents)) + threads.append(t) + t.start() + + for t in threads: + t.join() + + with open(user_agent_file, 'w') as f: + json.dump(browser_user_agents, f, indent=4) + logging.info(f"User agent file created at: {user_agent_file}") + + else: + logging.info("User agent file already exists.") + + +class UserAgentManager: + """ + Manager class to access browser user agents from a file. + """ + def __init__(self): + + # Get path to temp file where save all user agents + self.user_agent_file = os.path.join(os.environ.get('TEMP'), 'fake_user_agent.json') + + # If file dont exist, creaet it + if not os.path.exists(self.user_agent_file): + create_or_update_user_agent_file() + + def get_random_user_agent(self, browser: str) -> str: + """ + Get a random user agent for the specified browser. + + Args: + browser (str): The name of the browser ('chrome', 'firefox', 'safari'). + + Returns: + Optional[str]: Random user agent string for the specified browser. + """ + with open(self.user_agent_file, 'r') as f: + browser_user_agents = json.load(f) + return random.choice(browser_user_agents.get(browser.lower(), [])) + + +# Output +ua = UserAgentManager() \ No newline at end of file diff --git a/Src/Lib/Unidecode/__init__.py b/Src/Lib/Unidecode/__init__.py index ebce398..afa9e57 100644 --- a/Src/Lib/Unidecode/__init__.py +++ b/Src/Lib/Unidecode/__init__.py @@ -1,17 +1,18 @@ # 04.04.24 - -# Import import os import logging import importlib.util + # Variable Cache = {} + class UnidecodeError(ValueError): pass + def transliterate_nonascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str: """Transliterates non-ASCII characters in a string to their ASCII counterparts. @@ -25,6 +26,7 @@ def transliterate_nonascii(string: str, errors: str = 'ignore', replace_str: str """ return _transliterate(string, errors, replace_str) + def _get_ascii_representation(char: str) -> str: """Obtains the ASCII representation of a Unicode character. @@ -79,6 +81,7 @@ def _get_ascii_representation(char: str) -> str: else: return None + def _transliterate(string: str, errors: str, replace_str: str) -> str: """Main transliteration function. @@ -116,6 +119,7 @@ def _transliterate(string: str, errors: str, replace_str: str) -> str: return ''.join(retval) + def transliterate_expect_ascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str: """Transliterates non-ASCII characters in a string, expecting ASCII input. @@ -140,4 +144,6 @@ def transliterate_expect_ascii(string: str, errors: str = 'ignore', replace_str: # Otherwise, transliterate non-ASCII characters return _transliterate(string, errors, replace_str) + +# Out transliterate = transliterate_expect_ascii diff --git a/Src/Lib/Unidecode/x000.py b/Src/Lib/Unidecode/x000.py index 2c288a6..22bdb89 100644 --- a/Src/Lib/Unidecode/x000.py +++ b/Src/Lib/Unidecode/x000.py @@ -10,72 +10,72 @@ data = ( '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', -'', # 0x80 -'', # 0x81 -'', # 0x82 -'', # 0x83 -'', # 0x84 -'', # 0x85 -'', # 0x86 -'', # 0x87 -'', # 0x88 -'', # 0x89 -'', # 0x8a -'', # 0x8b -'', # 0x8c -'', # 0x8d -'', # 0x8e -'', # 0x8f -'', # 0x90 -'', # 0x91 -'', # 0x92 -'', # 0x93 -'', # 0x94 -'', # 0x95 -'', # 0x96 -'', # 0x97 -'', # 0x98 -'', # 0x99 -'', # 0x9a -'', # 0x9b -'', # 0x9c -'', # 0x9d -'', # 0x9e -'', # 0x9f +'', # 0x80 +'', # 0x81 +'', # 0x82 +'', # 0x83 +'', # 0x84 +'', # 0x85 +'', # 0x86 +'', # 0x87 +'', # 0x88 +'', # 0x89 +'', # 0x8a +'', # 0x8b +'', # 0x8c +'', # 0x8d +'', # 0x8e +'', # 0x8f +'', # 0x90 +'', # 0x91 +'', # 0x92 +'', # 0x93 +'', # 0x94 +'', # 0x95 +'', # 0x96 +'', # 0x97 +'', # 0x98 +'', # 0x99 +'', # 0x9a +'', # 0x9b +'', # 0x9c +'', # 0x9d +'', # 0x9e +'', # 0x9f ' ', # 0xa0 '!', # 0xa1 -'C/', # 0xa2 +'C/', # 0xa2 # Not "GBP" - Pound Sign is used for more than just British Pounds. 'PS', # 0xa3 '$?', # 0xa4 'Y=', # 0xa5 -'|', # 0xa6 +'|', # 0xa6 'SS', # 0xa7 -'"', # 0xa8 -'(c)', # 0xa9 -'a', # 0xaa +'"', # 0xa8 +'(c)', # 0xa9 +'a', # 0xaa '<<', # 0xab -'!', # 0xac -'', # 0xad -'(r)', # 0xae -'-', # 0xaf -'deg', # 0xb0 +'!', # 0xac +'', # 0xad +'(r)', # 0xae +'-', # 0xaf +'deg', # 0xb0 '+-', # 0xb1 # These might be combined with other superscript digits (u+2070 - u+2079) '2', # 0xb2 '3', # 0xb3 -'\'', # 0xb4 +'\'', # 0xb4 'u', # 0xb5 'P', # 0xb6 '*', # 0xb7 ',', # 0xb8 '1', # 0xb9 'o', # 0xba -'>>', # 0xbb +'>>', # 0xbb ' 1/4', # 0xbc ' 1/2', # 0xbd ' 3/4', # 0xbe @@ -89,7 +89,7 @@ data = ( 'A', # 0xc4 'A', # 0xc5 -'AE', # 0xc6 +'AE', # 0xc6 'C', # 0xc7 'E', # 0xc8 'E', # 0xc9 @@ -119,8 +119,8 @@ data = ( 'U', # 0xdc 'Y', # 0xdd -'Th', # 0xde -'ss', # 0xdf +'Th', # 0xde +'ss', # 0xdf 'a', # 0xe0 'a', # 0xe1 'a', # 0xe2 @@ -130,7 +130,7 @@ data = ( 'a', # 0xe4 'a', # 0xe5 -'ae', # 0xe6 +'ae', # 0xe6 'c', # 0xe7 'e', # 0xe8 'e', # 0xe9 @@ -160,6 +160,6 @@ data = ( 'u', # 0xfc 'y', # 0xfd -'th', # 0xfe +'th', # 0xfe 'y', # 0xff ) diff --git a/Src/Upload/update.py b/Src/Upload/update.py index d6c7cac..5725047 100644 --- a/Src/Upload/update.py +++ b/Src/Upload/update.py @@ -1,14 +1,15 @@ # 01.03.2023 -# Class import -from .version import __version__ -from Src.Util.console import console - -# General import import os import requests import time + +# Internal utilities +from .version import __version__ +from Src.Util.console import console + + # Variable repo_name = "StreamingCommunity_api" repo_user = "ghost6446" diff --git a/Src/Util/config.py b/Src/Util/config.py index 78ab8f5..68565e7 100644 --- a/Src/Util/config.py +++ b/Src/Util/config.py @@ -4,6 +4,7 @@ import json import os from typing import Any, List + class ConfigManager: def __init__(self, file_path: str = 'config.json') -> None: """Initialize the ConfigManager. diff --git a/Src/Util/console.py b/Src/Util/console.py index eb5f018..e23457d 100644 --- a/Src/Util/console.py +++ b/Src/Util/console.py @@ -1,9 +1,9 @@ # 24.02.24 -# Import from rich.console import Console from rich.prompt import Prompt + # Variable msg = Prompt() console = Console() \ No newline at end of file diff --git a/Src/Util/headers.py b/Src/Util/headers.py index f5d18a3..4bc4d04 100644 --- a/Src/Util/headers.py +++ b/Src/Util/headers.py @@ -1,10 +1,11 @@ -# 3.12.23 -> 10.12.23 -> 20.03.24 +# 4.04.24 -# Import -import fake_useragent +import logging + + +# Internal utilities +from Src.Lib.Request.user_agent import ua -# Variable -useragent = fake_useragent.UserAgent(use_external_data=True) def get_headers() -> str: """ @@ -15,4 +16,7 @@ def get_headers() -> str: """ # Get a random user agent string from the user agent rotator - return useragent.firefox \ No newline at end of file + random_headers = ua.get_random_user_agent("firefox") + + logging.info(f"Use headers: {random_headers}") + return random_headers \ No newline at end of file diff --git a/Src/Util/logger.py b/Src/Util/logger.py index 1788bcf..78360bb 100644 --- a/Src/Util/logger.py +++ b/Src/Util/logger.py @@ -1,12 +1,13 @@ # 26.03.24 -# Class import -from Src.Util.config import config_manager - -# Import import logging from logging.handlers import RotatingFileHandler + +# Internal utilities +from Src.Util.config import config_manager + + class Logger: def __init__(self): """ diff --git a/Src/Util/message.py b/Src/Util/message.py index 1ea0754..be62d00 100644 --- a/Src/Util/message.py +++ b/Src/Util/message.py @@ -1,17 +1,22 @@ # 3.12.23 -> 19.07.24 -# Class import -from .config import config_manager -from Src.Util.console import console - -# Import import os import platform + +# External libraries +from Src.Util.console import console + + +# Internal utilities +from .config import config_manager + + # Variable CLEAN = config_manager.get_bool('DEFAULT', 'clean_console') SHOW = config_manager.get_bool('DEFAULT', 'show_message') + def get_os_system(): """ This function returns the name of the operating system. @@ -19,6 +24,7 @@ def get_os_system(): os_system = platform.system() return os_system + def start_message(): """ Display a start message. diff --git a/Src/Util/os.py b/Src/Util/os.py index d72a4b9..ddac10f 100644 --- a/Src/Util/os.py +++ b/Src/Util/os.py @@ -1,6 +1,5 @@ # 24.01.24 -# Import import shutil import os import time @@ -9,6 +8,7 @@ import hashlib import logging import re + def remove_folder(folder_path: str) -> None: """ Remove a folder if it exists. @@ -23,6 +23,7 @@ def remove_folder(folder_path: str) -> None: except OSError as e: print(f"Error removing folder '{folder_path}': {e}") + def remove_file(file_path: str) -> None: """ Remove a file if it exists @@ -38,9 +39,8 @@ def remove_file(file_path: str) -> None: os.remove(file_path) except OSError as e: print(f"Error removing file '{file_path}': {e}") - #else: - # print(f"File '{file_path}' does not exist.") - + + def remove_special_characters(filename) -> str: """ Removes special characters from a filename to make it suitable for creating a filename in Windows. @@ -60,6 +60,7 @@ def remove_special_characters(filename) -> str: return cleaned_filename + def move_file_one_folder_up(file_path) -> None: """ Move a file one folder up from its current location. @@ -83,6 +84,7 @@ def move_file_one_folder_up(file_path) -> None: # Move the file os.rename(file_path, new_path) + def read_json(path: str): """Reads JSON file and returns its content. @@ -98,6 +100,7 @@ def read_json(path: str): return config + def save_json(json_obj, path: str) -> None: """Saves JSON object to the specified file path. @@ -109,6 +112,7 @@ def save_json(json_obj, path: str) -> None: with open(path, 'w') as file: json.dump(json_obj, file, indent=4) # Adjust the indentation as needed + def clean_json(path: str) -> None: """Reads JSON data from the file, cleans it, and saves it back. @@ -132,6 +136,7 @@ def clean_json(path: str) -> None: # Save the modified JSON data back to the file save_json(modified_data, path) + def format_size(size_bytes: float) -> str: """ Format the size in bytes into a human-readable format. @@ -154,6 +159,7 @@ def format_size(size_bytes: float) -> str: # Round the size to two decimal places and return with the appropriate unit return f"{size_bytes:.2f} {units[unit_index]}" + def compute_sha1_hash(input_string: str) -> str: """ Computes the SHA-1 hash of the input string. @@ -170,6 +176,7 @@ def compute_sha1_hash(input_string: str) -> str: # Return the hashed string return hashed_string + def decode_bytes(bytes_data: bytes, encodings_to_try: list[str] = None) -> str: """ Decode a byte sequence using a list of encodings and return the decoded string. @@ -200,6 +207,7 @@ def decode_bytes(bytes_data: bytes, encodings_to_try: list[str] = None) -> str: logging.info("Raw byte data: %s", bytes_data) return None + def convert_to_hex(bytes_data: bytes) -> str: """ Convert a byte sequence to its hexadecimal representation. diff --git a/Src/Util/table.py b/Src/Util/table.py index 52995d0..a34a2db 100644 --- a/Src/Util/table.py +++ b/Src/Util/table.py @@ -1,9 +1,5 @@ # 03.03.24 -# Class import -from .message import start_message - -# Import from rich.console import Console from rich.table import Table from rich.text import Text @@ -11,6 +7,11 @@ from rich.prompt import Prompt from rich.style import Style from typing import Dict, List, Any + +# Internal utilities +from .message import start_message + + class TVShowManager: def __init__(self): """ diff --git a/requirements.txt b/requirements.txt index 89806e6..de6588c 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/run.py b/run.py index bd33eae..07074b1 100644 --- a/run.py +++ b/run.py @@ -1,6 +1,11 @@ # 10.12.23 -> 31.01.24 -# Class +import sys +import logging +import platform + + +# Internal utilities from Src.Api import ( get_version_and_domain, download_series, @@ -19,10 +24,6 @@ from Src.Upload.update import update as git_update from Src.Lib.FFmpeg import check_ffmpeg from Src.Util.logger import Logger -# Import -import sys -import logging -import platform # Variable DEBUG_MODE = config_manager.get_bool("DEFAULT", "debug") @@ -31,7 +32,6 @@ SWITCH_TO = config_manager.get_bool('DEFAULT', 'swith_anime') CLOSE_CONSOLE = config_manager.get_bool('DEFAULT', 'not_close') -# [ main ] def initialize(): """ Initialize the application. diff --git a/update.py b/update.py index 4a64cf6..df14cfc 100644 --- a/update.py +++ b/update.py @@ -1,16 +1,25 @@ # 10.12.24 -# General imports -import requests import os import shutil -from zipfile import ZipFile from io import BytesIO +from zipfile import ZipFile + + +# Internal utilities +from Src.Util.config import config_manager + + +# External libraries +import requests from rich.console import Console + # Variable console = Console() local_path = os.path.join(".") +ROOT_PATH = config_manager.get('DEFAULT', 'root_path') + def move_content(source: str, destination: str) : """ @@ -36,6 +45,7 @@ def move_content(source: str, destination: str) : else: shutil.move(source_path, destination_path) + def keep_specific_items(directory: str, keep_folder: str, keep_file: str): """ Delete all items in the directory except for the specified folder and file. @@ -66,6 +76,7 @@ def keep_specific_items(directory: str, keep_folder: str, keep_file: str): except Exception as e: print(f"Error: {e}") + def download_and_extract_latest_commit(author: str, repo_name: str): """ Download and extract the latest commit from a GitHub repository. @@ -115,6 +126,7 @@ def download_and_extract_latest_commit(author: str, repo_name: str): else: console.log(f"[red]Failed to fetch commit information. Status code: {response.status_code}") + def main_upload(): """ Main function to upload the latest commit of a GitHub repository. @@ -128,10 +140,11 @@ def main_upload(): if cmd_insert == "yes": # Remove all old file - keep_specific_items(".", "videos", "upload.py") + keep_specific_items(".", ROOT_PATH, "upload.py") download_and_extract_latest_commit(repository_owner, repository_name) + main_upload() # win