mirror of
https://github.com/Arrowar/StreamingCommunity.git
synced 2025-06-06 19:45:24 +00:00
Improve organization
This commit is contained in:
Ghost
parent
92c35b489a
commit
bf7153214f
@ -7,7 +7,7 @@
|
||||
This repository provide a simple script designed to facilitate the downloading of films and series from a popular streaming community platform. The script allows users to download individual films, entire series, or specific episodes, providing a seamless experience for content consumers.
|
||||
|
||||
## Join us
|
||||
You can chat, help improve this repo, or just hang around for some fun in the **Git_StreamingCommunity** Discord [Server](https://discord.com/invite/qP3nsCXV5z)
|
||||
You can chat, help improve this repo, or just hang around for some fun in the **Git_StreamingCommunity** Discord [Server](https://discord.gg/we8n4tfxFs)
|
||||
# Table of Contents
|
||||
|
||||
* [INSTALLATION](#installation)
|
||||
|
||||
@ -21,6 +21,7 @@ class Episode:
|
||||
self.created_at: str = data.get('created_at', '')
|
||||
self.updated_at: str = data.get('updated_at', '')
|
||||
|
||||
|
||||
class EpisodeManager:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
# 03.03.24
|
||||
|
||||
# Import
|
||||
from typing import List
|
||||
|
||||
class Image:
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
from typing import List, Dict, Union
|
||||
|
||||
|
||||
class Title:
|
||||
def __init__(self, title_data: Dict[str, Union[int, str, None]]):
|
||||
"""
|
||||
@ -20,6 +21,7 @@ class Title:
|
||||
self.updated_at: str = title_data.get('updated_at')
|
||||
self.episodes_count: int = title_data.get('episodes_count')
|
||||
|
||||
|
||||
class TitleManager:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
@ -1,22 +1,26 @@
|
||||
# 01.03.24
|
||||
|
||||
# Class import
|
||||
from Src.Util.headers import get_headers
|
||||
from .SeriesType import TitleManager
|
||||
from .EpisodeType import EpisodeManager
|
||||
from .WindowType import WindowVideo, WindowParameter
|
||||
|
||||
# Import
|
||||
import requests
|
||||
import re
|
||||
import json
|
||||
import binascii
|
||||
import logging
|
||||
import sys
|
||||
from bs4 import BeautifulSoup
|
||||
from urllib.parse import urljoin, urlencode, quote
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.headers import get_headers
|
||||
from .SeriesType import TitleManager
|
||||
from .EpisodeType import EpisodeManager
|
||||
from .WindowType import WindowVideo, WindowParameter
|
||||
|
||||
|
||||
class VideoSource:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
class WindowVideo:
|
||||
def __init__(self, data: Dict[str, Any]):
|
||||
"""
|
||||
@ -25,6 +26,7 @@ class WindowVideo:
|
||||
self.folder_id: int = data.get('folder_id', '')
|
||||
self.created_at_diff: str = data.get('created_at_diff', '')
|
||||
|
||||
|
||||
class WindowParameter:
|
||||
def __init__(self, data: Dict[str, Any]):
|
||||
"""
|
||||
|
||||
@ -1,16 +1,20 @@
|
||||
# 11.03.24
|
||||
|
||||
# Class import
|
||||
import os
|
||||
import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console, msg
|
||||
from Src.Util.config import config_manager
|
||||
from Src.Lib.FFmpeg.my_m3u8 import Downloader
|
||||
from Src.Util.message import start_message
|
||||
from .Class import VideoSource
|
||||
|
||||
# General import
|
||||
import os
|
||||
import logging
|
||||
import requests
|
||||
|
||||
# Config
|
||||
ROOT_PATH = config_manager.get('DEFAULT', 'root_path')
|
||||
@ -20,6 +24,7 @@ SERIES_FOLDER = config_manager.get('DEFAULT', 'series_folder_name')
|
||||
URL_SITE_NAME = config_manager.get('SITE', 'anime_site_name')
|
||||
SITE_DOMAIN = config_manager.get('SITE', 'anime_domain')
|
||||
|
||||
|
||||
# Variable
|
||||
video_source = VideoSource()
|
||||
|
||||
|
||||
@ -1,21 +1,23 @@
|
||||
# 3.12.23 -> 10.12.23
|
||||
|
||||
# Class import
|
||||
import os
|
||||
import logging
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
from Src.Util.config import config_manager
|
||||
from Src.Lib.FFmpeg.my_m3u8 import Downloader
|
||||
from Src.Util.message import start_message
|
||||
from .Class import VideoSource
|
||||
|
||||
# General import
|
||||
import os
|
||||
import logging
|
||||
|
||||
# Config
|
||||
ROOT_PATH = config_manager.get('DEFAULT', 'root_path')
|
||||
MOVIE_FOLDER = config_manager.get('DEFAULT', 'movies_folder_name')
|
||||
STREAM_SITE_NAME = config_manager.get('SITE', 'streaming_site_name')
|
||||
|
||||
|
||||
# Variable
|
||||
video_source = VideoSource()
|
||||
video_source.set_url_base_name(STREAM_SITE_NAME)
|
||||
|
||||
@ -1,6 +1,11 @@
|
||||
# 3.12.23 -> 10.12.23
|
||||
|
||||
# Class import
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console, msg
|
||||
from Src.Util.config import config_manager
|
||||
from Src.Util.table import TVShowManager
|
||||
@ -9,16 +14,13 @@ from Src.Lib.Unidecode import transliterate
|
||||
from Src.Lib.FFmpeg.my_m3u8 import Downloader
|
||||
from .Class import VideoSource
|
||||
|
||||
# General import
|
||||
import os
|
||||
import logging
|
||||
import sys
|
||||
|
||||
# Config
|
||||
ROOT_PATH = config_manager.get('DEFAULT', 'root_path')
|
||||
SERIES_FOLDER = config_manager.get('DEFAULT', 'series_folder_name')
|
||||
STREAM_SITE_NAME = config_manager.get('SITE', 'streaming_site_name')
|
||||
|
||||
|
||||
# Variable
|
||||
video_source = VideoSource()
|
||||
video_source.set_url_base_name(STREAM_SITE_NAME)
|
||||
@ -56,6 +58,7 @@ def manage_selection(cmd_insert: str, max_count: int) -> list[int]:
|
||||
# Return list of selected seasons)
|
||||
return list_season_select
|
||||
|
||||
|
||||
def display_episodes_list() -> str:
|
||||
"""
|
||||
Display episodes list and handle user input.
|
||||
@ -133,6 +136,7 @@ def donwload_video(tv_name: str, index_season_selected: int, index_episode_selec
|
||||
logging.error(f"(donwload_video) Error downloading film: {e}")
|
||||
pass
|
||||
|
||||
|
||||
def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: bool = False) -> None:
|
||||
"""
|
||||
Download all episodes of a season.
|
||||
@ -174,6 +178,7 @@ def donwload_episode(tv_name: str, index_season_selected: int, donwload_all: boo
|
||||
for i_episode in list_episode_select:
|
||||
donwload_video(tv_name, index_season_selected, i_episode)
|
||||
|
||||
|
||||
def download_series(tv_id: str, tv_name: str, version: str, domain: str) -> None:
|
||||
"""
|
||||
Download all episodes of a TV series.
|
||||
|
||||
@ -1,26 +1,32 @@
|
||||
# 10.12.23
|
||||
|
||||
# Class import
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.table import TVShowManager
|
||||
from Src.Util.headers import get_headers
|
||||
from Src.Util.console import console
|
||||
from Src.Util.config import config_manager
|
||||
from .Class import MediaManager, MediaItem
|
||||
|
||||
# General import
|
||||
import sys
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Config
|
||||
GET_TITLES_OF_MOMENT = config_manager.get_bool('DEFAULT', 'get_moment_title')
|
||||
|
||||
|
||||
# Variable
|
||||
media_search_manager = MediaManager()
|
||||
table_show_manager = TVShowManager()
|
||||
|
||||
|
||||
def get_token(site_name: str, domain: str) -> dict:
|
||||
"""
|
||||
Function to retrieve session tokens from a specified website.
|
||||
@ -61,6 +67,7 @@ def get_token(site_name: str, domain: str) -> dict:
|
||||
'csrf_token': find_csrf_token
|
||||
}
|
||||
|
||||
|
||||
def get_moment_titles(domain: str, version: str, prefix: str):
|
||||
"""
|
||||
Retrieves the title name from a specified domain using the provided version and prefix.
|
||||
@ -100,6 +107,7 @@ def get_moment_titles(domain: str, version: str, prefix: str):
|
||||
logging.error("Error occurred: %s", str(e))
|
||||
return None
|
||||
|
||||
|
||||
def get_domain() -> str:
|
||||
"""
|
||||
Fetches the domain from a Telegra.ph API response.
|
||||
@ -123,6 +131,7 @@ def get_domain() -> str:
|
||||
logging.error(f"Error fetching domain: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def test_site(domain: str) -> str:
|
||||
"""
|
||||
Tests the availability of a website.
|
||||
@ -151,6 +160,7 @@ def test_site(domain: str) -> str:
|
||||
logging.error(f"Error testing site: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def get_version(text: str) -> str:
|
||||
"""
|
||||
Extracts the version from the HTML text of a webpage.
|
||||
@ -174,6 +184,7 @@ def get_version(text: str) -> str:
|
||||
logging.error(f"Error extracting version: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def get_version_and_domain() -> tuple[str, str]:
|
||||
"""
|
||||
Retrieves the version and domain of a website.
|
||||
@ -215,6 +226,7 @@ def get_version_and_domain() -> tuple[str, str]:
|
||||
logging.error(f"Error getting version and domain: {e}")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def search(title_search: str, domain: str) -> int:
|
||||
"""
|
||||
Search for titles based on a search query.
|
||||
@ -237,6 +249,7 @@ def search(title_search: str, domain: str) -> int:
|
||||
# Return the number of titles found
|
||||
return media_search_manager.get_length()
|
||||
|
||||
|
||||
def update_domain_anime():
|
||||
"""
|
||||
Update the domain for anime streaming site.
|
||||
@ -268,6 +281,7 @@ def update_domain_anime():
|
||||
# Extract the domain from the URL and update the config
|
||||
config_manager.set_key('SITE', 'anime_domain', new_site_url.split(".")[-1])
|
||||
|
||||
|
||||
def anime_search(title_search: str) -> int:
|
||||
"""
|
||||
Function to perform an anime search using a provided title.
|
||||
@ -322,6 +336,7 @@ def anime_search(title_search: str) -> int:
|
||||
# Return the length of media search manager
|
||||
return media_search_manager.get_length()
|
||||
|
||||
|
||||
def get_select_title() -> MediaItem:
|
||||
"""
|
||||
Display a selection of titles and prompt the user to choose one.
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
# 5.01.24 -> 7.01.24 -> 20.02.24 -> 29.03.24
|
||||
|
||||
# Importing modules
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
@ -8,16 +7,19 @@ import threading
|
||||
import logging
|
||||
import warnings
|
||||
|
||||
|
||||
# Disable specific warnings
|
||||
from tqdm import TqdmExperimentalWarning
|
||||
warnings.filterwarnings("ignore", category=TqdmExperimentalWarning)
|
||||
warnings.filterwarnings("ignore", category=UserWarning, module="cryptography")
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from tqdm.rich import tqdm
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
from Src.Util.headers import get_headers
|
||||
@ -30,6 +32,7 @@ from Src.Util.os import (
|
||||
convert_to_hex
|
||||
)
|
||||
|
||||
|
||||
# Logic class
|
||||
from .util import (
|
||||
print_duration_table,
|
||||
@ -44,6 +47,7 @@ from .util import (
|
||||
M3U8_UrlFix
|
||||
)
|
||||
|
||||
|
||||
# Config
|
||||
Download_audio = config_manager.get_bool('M3U8_OPTIONS', 'download_audio')
|
||||
Donwload_subtitles = config_manager.get_bool('M3U8_OPTIONS', 'download_subtitles')
|
||||
@ -59,12 +63,12 @@ TQDM_SHOW_PROGRESS = config_manager.get_bool('M3U8', 'tqdm_show_progress')
|
||||
MIN_TS_FILES_IN_FOLDER = config_manager.get_int('M3U8', 'minimum_ts_files_in_folder')
|
||||
REMOVE_SEGMENTS_FOLDER = config_manager.get_bool('M3U8', 'cleanup_tmp_folder')
|
||||
|
||||
|
||||
# Variable
|
||||
config_headers = config_manager.get_dict('M3U8_OPTIONS', 'request')
|
||||
failed_segments = []
|
||||
class_urlFix = M3U8_UrlFix()
|
||||
|
||||
# [ main class ]
|
||||
|
||||
class M3U8_Segments:
|
||||
def __init__(self, url, folder, key=None):
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
# 03.04.24
|
||||
|
||||
# Import
|
||||
import subprocess
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
# External library
|
||||
from Crypto.Cipher import AES
|
||||
from Crypto.Util.Padding import unpad
|
||||
|
||||
|
||||
class AES_ECB:
|
||||
def __init__(self, key: bytes) -> None:
|
||||
"""
|
||||
@ -49,6 +50,7 @@ class AES_ECB:
|
||||
decrypted_data = cipher.decrypt(ciphertext)
|
||||
return unpad(decrypted_data, AES.block_size)
|
||||
|
||||
|
||||
class AES_CBC:
|
||||
def __init__(self, key: bytes, iv: bytes) -> None:
|
||||
"""
|
||||
@ -91,6 +93,7 @@ class AES_CBC:
|
||||
decrypted_data = cipher.decrypt(ciphertext)
|
||||
return unpad(decrypted_data, AES.block_size)
|
||||
|
||||
|
||||
class AES_CTR:
|
||||
def __init__(self, key: bytes, nonce: bytes) -> None:
|
||||
"""
|
||||
@ -132,6 +135,7 @@ class AES_CTR:
|
||||
cipher = AES.new(self.key, AES.MODE_CTR, nonce=self.nonce)
|
||||
return cipher.decrypt(ciphertext)
|
||||
|
||||
|
||||
class M3U8_Decryption:
|
||||
def __init__(self, key: bytes, iv: bytes = None) -> None:
|
||||
"""
|
||||
|
||||
@ -1,16 +1,20 @@
|
||||
# 31.01.24
|
||||
|
||||
# Class
|
||||
from Src.Util.console import console
|
||||
|
||||
# Import
|
||||
import ffmpeg
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
|
||||
|
||||
# External libraries
|
||||
import ffmpeg
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
|
||||
|
||||
def has_audio_stream(video_path: str) -> bool:
|
||||
"""
|
||||
Check if the input video has an audio stream.
|
||||
@ -37,6 +41,7 @@ def has_audio_stream(video_path: str) -> bool:
|
||||
logging.error(f"Error: {e.stderr}")
|
||||
return None
|
||||
|
||||
|
||||
def get_video_duration(file_path: str) -> (float):
|
||||
"""
|
||||
Get the duration of a video file.
|
||||
@ -64,6 +69,7 @@ def get_video_duration(file_path: str) -> (float):
|
||||
logging.error(f"Error: {e.stderr}")
|
||||
return None
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> list[int, int, int]:
|
||||
"""
|
||||
Format duration in seconds into hours, minutes, and seconds.
|
||||
@ -80,6 +86,7 @@ def format_duration(seconds: float) -> list[int, int, int]:
|
||||
|
||||
return int(hours), int(minutes), int(seconds)
|
||||
|
||||
|
||||
def print_duration_table(file_path: str) -> None:
|
||||
"""
|
||||
Print duration of a video file in hours, minutes, and seconds.
|
||||
@ -98,7 +105,7 @@ def print_duration_table(file_path: str) -> None:
|
||||
# Print the formatted duration
|
||||
console.log(f"[cyan]Info [green]'{file_path}': [purple]{int(hours)}[red]h [purple]{int(minutes)}[red]m [purple]{int(seconds)}[red]s")
|
||||
|
||||
# SINGLE SUBTITLE
|
||||
|
||||
def add_subtitle(input_video_path: str, input_subtitle_path: str, output_video_path: str, subtitle_language: str = 'ita', prefix: str = "single_sub") -> str:
|
||||
"""
|
||||
Convert a video with a single subtitle.
|
||||
@ -153,7 +160,7 @@ def add_subtitle(input_video_path: str, input_subtitle_path: str, output_video_p
|
||||
# Return
|
||||
return output_video_path
|
||||
|
||||
# SEGMENTS
|
||||
|
||||
def concatenate_and_save(file_list_path: str, output_filename: str, video_decoding: str = None, audio_decoding: str = None, prefix: str = "segments", output_directory: str = None) -> str:
|
||||
"""
|
||||
Concatenate input files and save the output with specified decoding parameters.
|
||||
@ -222,7 +229,7 @@ def concatenate_and_save(file_list_path: str, output_filename: str, video_decodi
|
||||
# Return
|
||||
return output_file_path
|
||||
|
||||
# AUDIOS
|
||||
|
||||
def join_audios(video_path: str, audio_tracks: list[dict[str, str]], prefix: str = "merged") -> str:
|
||||
"""
|
||||
Join video with multiple audio tracks and sync them if there are matching segments.
|
||||
@ -300,7 +307,7 @@ def join_audios(video_path: str, audio_tracks: list[dict[str, str]], prefix: str
|
||||
logging.error("[M3U8_Downloader] Ffmpeg error: %s", ffmpeg_error)
|
||||
return ""
|
||||
|
||||
# SUBTITLES
|
||||
|
||||
def transcode_with_subtitles(video: str, subtitles_list: list[dict[str, str]], output_file: str, prefix: str = "transcoded") -> str:
|
||||
|
||||
"""
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
# 24.01.2023
|
||||
|
||||
# Class
|
||||
from Src.Util.console import console
|
||||
|
||||
# Import
|
||||
import subprocess
|
||||
import os
|
||||
import requests
|
||||
@ -12,7 +8,10 @@ import sys
|
||||
import ctypes
|
||||
|
||||
|
||||
# [ func ]
|
||||
# Internal utilities
|
||||
from Src.Util.console import console
|
||||
|
||||
|
||||
def isAdmin() -> (bool):
|
||||
"""
|
||||
Check if the current user has administrative privileges.
|
||||
@ -29,6 +28,7 @@ def isAdmin() -> (bool):
|
||||
|
||||
return is_admin
|
||||
|
||||
|
||||
def get_version():
|
||||
"""
|
||||
Get the version of FFmpeg installed on the system.
|
||||
@ -55,6 +55,7 @@ def get_version():
|
||||
print("Error executing FFmpeg command:", e.output.strip())
|
||||
raise e
|
||||
|
||||
|
||||
def download_ffmpeg():
|
||||
"""
|
||||
Download FFmpeg binary for Windows and add it to the system PATH.
|
||||
@ -102,6 +103,7 @@ def download_ffmpeg():
|
||||
print(f"Failed to extract FFmpeg zip file: {e}")
|
||||
raise e
|
||||
|
||||
|
||||
def check_ffmpeg():
|
||||
"""
|
||||
Check if FFmpeg is installed and available on the system PATH.
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
# 29.02.24
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.os import format_size
|
||||
|
||||
|
||||
class M3U8_Ts_Files:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
@ -1,12 +1,16 @@
|
||||
# 29.04.25
|
||||
|
||||
# Class import
|
||||
import logging
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.headers import get_headers
|
||||
|
||||
# Import
|
||||
from m3u8 import M3U8
|
||||
import logging
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from m3u8 import M3U8
|
||||
|
||||
|
||||
class M3U8_Parser:
|
||||
def __init__(self, DOWNLOAD_SPECIFIC_SUBTITLE = None):
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
# 29.03.24
|
||||
|
||||
# Import
|
||||
import logging
|
||||
import sys
|
||||
import logging
|
||||
from urllib.parse import urlparse, urljoin
|
||||
|
||||
|
||||
class M3U8_UrlFix:
|
||||
def __init__(self) -> None:
|
||||
|
||||
|
||||
360
Src/Lib/Request/my_requests.py
Normal file
360
Src/Lib/Request/my_requests.py
Normal file
@ -0,0 +1,360 @@
|
||||
# 04.4.24
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import time
|
||||
import re
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from typing import Dict, Optional, Union, Unpack
|
||||
|
||||
|
||||
# Constants
|
||||
HTTP_TIMEOUT = 4
|
||||
HTTP_RETRIES = 2
|
||||
HTTP_DELAY = 1
|
||||
|
||||
|
||||
class RequestError(Exception):
|
||||
"""Custom exception class for request errors."""
|
||||
|
||||
def __init__(self, message: str, original_exception: Optional[Exception] = None) -> None:
|
||||
"""
|
||||
Initialize a RequestError instance.
|
||||
|
||||
Args:
|
||||
message (str): The error message.
|
||||
original_exception (Optional[Exception], optional): The original exception that occurred. Defaults to None.
|
||||
"""
|
||||
super().__init__(message)
|
||||
self.original_exception = original_exception
|
||||
|
||||
def __str__(self) -> str:
|
||||
"""Return a string representation of the exception."""
|
||||
if self.original_exception:
|
||||
return f"{super().__str__()} Original Exception: {type(self.original_exception).__name__}: {str(self.original_exception)}"
|
||||
else:
|
||||
return super().__str__()
|
||||
|
||||
|
||||
class Response:
|
||||
"""Class representing an HTTP response."""
|
||||
def __init__(
|
||||
self,
|
||||
status: int,
|
||||
text: str,
|
||||
is_json: bool = False,
|
||||
content: bytes = b"",
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
cookies: Optional[Dict[str, str]] = None,
|
||||
redirect_url: Optional[str] = None,
|
||||
response_time: Optional[float] = None,
|
||||
timeout: Optional[float] = None,
|
||||
):
|
||||
"""
|
||||
Initialize a Response object.
|
||||
|
||||
Args:
|
||||
status (int): The HTTP status code of the response.
|
||||
text (str): The response content as text.
|
||||
is_json (bool, optional): Indicates if the response content is JSON. Defaults to False.
|
||||
content (bytes, optional): The response content as bytes. Defaults to b"".
|
||||
headers (Optional[Dict[str, str]], optional): The response headers. Defaults to None.
|
||||
cookies (Optional[Dict[str, str]], optional): The cookies set in the response. Defaults to None.
|
||||
redirect_url (Optional[str], optional): The URL if a redirection occurred. Defaults to None.
|
||||
response_time (Optional[float], optional): The time taken to receive the response. Defaults to None.
|
||||
timeout (Optional[float], optional): The request timeout. Defaults to None.
|
||||
"""
|
||||
self.status_code = status
|
||||
self.text = text
|
||||
self.is_json = is_json
|
||||
self.content = content
|
||||
self.headers = headers or {}
|
||||
self.cookies = cookies or {}
|
||||
self.redirect_url = redirect_url
|
||||
self.response_time = response_time
|
||||
self.timeout = timeout
|
||||
self.ok = 200 <= status < 300
|
||||
|
||||
def raise_for_status(self):
|
||||
"""Raise an error if the response status code is not in the 2xx range."""
|
||||
if not self.ok:
|
||||
raise RequestError(f"Request failed with status code {self.status_code}")
|
||||
|
||||
def json(self):
|
||||
"""Return the response content as JSON if it is JSON."""
|
||||
if self.is_json:
|
||||
return json.loads(self.text)
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class ManageRequests:
|
||||
"""Class for managing HTTP requests."""
|
||||
def __init__(
|
||||
self,
|
||||
url: str,
|
||||
method: str = 'GET',
|
||||
headers: Optional[Dict[str, str]] = None,
|
||||
timeout: float = HTTP_TIMEOUT,
|
||||
retries: int = HTTP_RETRIES,
|
||||
params: Optional[Dict[str, str]] = None,
|
||||
verify_ssl: bool = True,
|
||||
auth: Optional[tuple] = None,
|
||||
proxy: Optional[str] = None,
|
||||
cookies: Optional[Dict[str, str]] = None,
|
||||
redirection_handling: bool = True,
|
||||
):
|
||||
"""
|
||||
Initialize a ManageRequests object.
|
||||
|
||||
Args:
|
||||
url (str): The URL to which the request will be sent.
|
||||
method (str, optional): The HTTP method to be used for the request. Defaults to 'GET'.
|
||||
headers (Optional[Dict[str, str]], optional): The request headers. Defaults to None.
|
||||
timeout (float, optional): The request timeout. Defaults to HTTP_TIMEOUT.
|
||||
retries (int, optional): The number of retries in case of request failure. Defaults to HTTP_RETRIES.
|
||||
params (Optional[Dict[str, str]], optional): The query parameters for the request. Defaults to None.
|
||||
verify_ssl (bool, optional): Indicates whether SSL certificate verification should be performed. Defaults to True.
|
||||
auth (Optional[tuple], optional): Tuple containing the username and password for basic authentication. Defaults to None.
|
||||
proxy (Optional[str], optional): The proxy URL. Defaults to None.
|
||||
cookies (Optional[Dict[str, str]], optional): The cookies to be included in the request. Defaults to None.
|
||||
redirection_handling (bool, optional): Indicates whether redirections should be followed. Defaults to True.
|
||||
"""
|
||||
self.url = url
|
||||
self.method = method
|
||||
self.headers = headers or {'User-Agent': 'Mozilla/5.0'}
|
||||
self.timeout = timeout
|
||||
self.retries = retries
|
||||
self.params = params
|
||||
self.verify_ssl = verify_ssl
|
||||
self.auth = auth
|
||||
self.proxy = proxy
|
||||
self.cookies = cookies
|
||||
self.redirection_handling = redirection_handling
|
||||
|
||||
def add_header(self, key: str, value: str) -> None:
|
||||
"""Add a header to the request."""
|
||||
self.headers[key] = value
|
||||
|
||||
def send(self) -> Response:
|
||||
"""Send the HTTP request."""
|
||||
start_time = time.time()
|
||||
self.attempt = 0
|
||||
redirect_url = None
|
||||
|
||||
while self.attempt < self.retries:
|
||||
try:
|
||||
req = self._build_request()
|
||||
response = self._perform_request(req)
|
||||
return self._process_response(response, start_time, redirect_url)
|
||||
except (urllib.error.URLError, urllib.error.HTTPError) as e:
|
||||
self._handle_error(e)
|
||||
attempt += 1
|
||||
|
||||
def _build_request(self) -> urllib.request.Request:
|
||||
"""Build the urllib Request object."""
|
||||
headers = self.headers.copy()
|
||||
if self.params:
|
||||
url = self.url + '?' + urllib.parse.urlencode(self.params)
|
||||
else:
|
||||
url = self.url
|
||||
req = urllib.request.Request(url, headers=headers, method=self.method)
|
||||
if self.auth:
|
||||
req.add_header('Authorization', 'Basic ' + base64.b64encode(f"{self.auth[0]}:{self.auth[1]}".encode()).decode())
|
||||
if self.cookies:
|
||||
cookie_str = '; '.join([f"{name}={value}" for name, value in self.cookies.items()])
|
||||
req.add_header('Cookie', cookie_str)
|
||||
return req
|
||||
|
||||
def _perform_request(self, req: urllib.request.Request) -> urllib.response.addinfourl:
|
||||
"""Perform the HTTP request."""
|
||||
if self.proxy:
|
||||
proxy_handler = urllib.request.ProxyHandler({'http': self.proxy, 'https': self.proxy})
|
||||
opener = urllib.request.build_opener(proxy_handler)
|
||||
urllib.request.install_opener(opener)
|
||||
if not self.verify_ssl:
|
||||
ssl_context = ssl.create_default_context()
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
response = urllib.request.urlopen(req, timeout=self.timeout, context=ssl_context)
|
||||
else:
|
||||
response = urllib.request.urlopen(req, timeout=self.timeout)
|
||||
return response
|
||||
|
||||
def _process_response(self, response: urllib.response.addinfourl, start_time: float, redirect_url: Optional[str]) -> Response:
|
||||
"""Process the HTTP response."""
|
||||
response_data = response.read()
|
||||
content_type = response.headers.get('Content-Type', '').lower()
|
||||
is_response_api = "json" in content_type
|
||||
if self.redirection_handling and response.status in (301, 302, 303, 307, 308):
|
||||
location = response.headers.get('Location')
|
||||
logging.info(f"Redirecting to: {location}")
|
||||
redirect_url = location
|
||||
self.url = location
|
||||
return self.send()
|
||||
return self._build_response(response, response_data, start_time, redirect_url, content_type)
|
||||
|
||||
def _build_response(self, response: urllib.response.addinfourl, response_data: bytes, start_time: float, redirect_url: Optional[str], content_type: str) -> Response:
|
||||
"""Build the Response object."""
|
||||
response_time = time.time() - start_time
|
||||
response_headers = dict(response.headers)
|
||||
response_cookies = {}
|
||||
|
||||
for cookie in response.headers.get_all('Set-Cookie', []):
|
||||
cookie_parts = cookie.split(';')
|
||||
cookie_name, cookie_value = cookie_parts[0].split('=')
|
||||
response_cookies[cookie_name.strip()] = cookie_value.strip()
|
||||
|
||||
return Response(
|
||||
status=response.status,
|
||||
text=response_data.decode('latin-1'),
|
||||
is_json=("json" in content_type),
|
||||
content=response_data,
|
||||
headers=response_headers,
|
||||
cookies=response_cookies,
|
||||
redirect_url=redirect_url,
|
||||
response_time=response_time,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
def _handle_error(self, e: Union[urllib.error.URLError, urllib.error.HTTPError]) -> None:
|
||||
"""Handle request error."""
|
||||
logging.error(f"Request failed for URL '{self.url}': {str(e)}")
|
||||
if self.attempt < self.retries:
|
||||
logging.info(f"Retrying request for URL '{self.url}' (attempt {self.attempt}/{self.retries})")
|
||||
time.sleep(HTTP_DELAY)
|
||||
else:
|
||||
logging.error(f"Maximum retries reached for URL '{self.url}'")
|
||||
raise RequestError(str(e))
|
||||
|
||||
|
||||
class ValidateRequest:
|
||||
"""Class for validating request inputs."""
|
||||
@staticmethod
|
||||
def validate_url(url: str) -> bool:
|
||||
"""Validate URL format."""
|
||||
url_regex = re.compile(
|
||||
r'^(?:http|ftp)s?://' # http:// or https://
|
||||
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+[A-Z]{2,6}\.?|' # domain...
|
||||
r'localhost|' # localhost...
|
||||
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or IP
|
||||
r'(?::\d+)?' # optional port
|
||||
r'(?:/?|[/?]\S+)$', re.IGNORECASE)
|
||||
return re.match(url_regex, url) is not None
|
||||
|
||||
@staticmethod
|
||||
def validate_headers(headers: Dict[str, str]) -> bool:
|
||||
"""Validate header values."""
|
||||
for key, value in headers.items():
|
||||
if not isinstance(key, str) or not isinstance(value, str):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
class ValidateResponse:
|
||||
"""Class for validating response data."""
|
||||
@staticmethod
|
||||
def is_valid_json(data: str) -> bool:
|
||||
"""Check if response data is a valid JSON."""
|
||||
try:
|
||||
json.loads(data)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
class SSLHandler:
|
||||
"""Class for handling SSL certificates."""
|
||||
@staticmethod
|
||||
def load_certificate(custom_cert_path: str) -> None:
|
||||
"""Load custom SSL certificate."""
|
||||
ssl_context = ssl.create_default_context(cafile=custom_cert_path)
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
|
||||
|
||||
class KwargsRequest():
|
||||
"""Class representing keyword arguments for a request."""
|
||||
url: str
|
||||
headers: Optional[Dict[str, str]] = None
|
||||
timeout: float = HTTP_TIMEOUT
|
||||
retries: int = HTTP_RETRIES
|
||||
params: Optional[Dict[str, str]] = None
|
||||
cookies: Optional[Dict[str, str]] = None
|
||||
|
||||
|
||||
class Request:
|
||||
"""Class for making HTTP requests."""
|
||||
def __init__(self) -> None:
|
||||
pass
|
||||
|
||||
def get(self, url: str, **kwargs: Unpack[KwargsRequest]):
|
||||
"""
|
||||
Send a GET request.
|
||||
|
||||
Args:
|
||||
url (str): The URL to which the request will be sent.
|
||||
**kwargs: Additional keyword arguments for the request.
|
||||
|
||||
Returns:
|
||||
Response: The response object.
|
||||
"""
|
||||
return self._send_request(url, 'GET', **kwargs)
|
||||
|
||||
def post(self, url: str, **kwargs: Unpack[KwargsRequest]):
|
||||
"""
|
||||
Send a POST request.
|
||||
|
||||
Args:
|
||||
url (str): The URL to which the request will be sent.
|
||||
**kwargs: Additional keyword arguments for the request.
|
||||
|
||||
Returns:
|
||||
Response: The response object.
|
||||
"""
|
||||
return self._send_request(url, 'POST', **kwargs)
|
||||
|
||||
def put(self, url: str, **kwargs: Unpack[KwargsRequest]):
|
||||
"""
|
||||
Send a PUT request.
|
||||
|
||||
Args:
|
||||
url (str): The URL to which the request will be sent.
|
||||
**kwargs: Additional keyword arguments for the request.
|
||||
|
||||
Returns:
|
||||
Response: The response object.
|
||||
"""
|
||||
return self._send_request(url, 'PUT', **kwargs)
|
||||
|
||||
def delete(self, url: str, **kwargs: Unpack[KwargsRequest]):
|
||||
"""
|
||||
Send a DELETE request.
|
||||
|
||||
Args:
|
||||
url (str): The URL to which the request will be sent.
|
||||
**kwargs: Additional keyword arguments for the request.
|
||||
|
||||
Returns:
|
||||
Response: The response object.
|
||||
"""
|
||||
return self._send_request(url, 'DELETE', **kwargs)
|
||||
|
||||
def _send_request(self, url: str, method: str, **kwargs: Unpack[KwargsRequest]):
|
||||
"""Send an HTTP request."""
|
||||
# Add validation checks for URL and headers
|
||||
if not ValidateRequest.validate_url(url):
|
||||
raise ValueError("Invalid URL format")
|
||||
|
||||
if 'headers' in kwargs and not ValidateRequest.validate_headers(kwargs['headers']):
|
||||
raise ValueError("Invalid header values")
|
||||
|
||||
return ManageRequests(url, method, **kwargs).send()
|
||||
|
||||
|
||||
# Out
|
||||
request = Request()
|
||||
105
Src/Lib/Request/user_agent.py
Normal file
105
Src/Lib/Request/user_agent.py
Normal file
@ -0,0 +1,105 @@
|
||||
# 04.4.24
|
||||
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import json
|
||||
from typing import Dict, List
|
||||
|
||||
# Internal utilities
|
||||
from .my_requests import request
|
||||
|
||||
|
||||
def get_browser_user_agents_online(browser: str) -> List[str]:
|
||||
"""
|
||||
Retrieve browser user agent strings from a website.
|
||||
|
||||
Args:
|
||||
browser (str): The name of the browser (e.g., 'chrome', 'firefox', 'safari').
|
||||
|
||||
Returns:
|
||||
List[str]: List of user agent strings for the specified browser.
|
||||
"""
|
||||
url = f"https://useragentstring.com/pages/{browser}/"
|
||||
|
||||
try:
|
||||
|
||||
# Make request and find all user agents
|
||||
html = request.get(url).text
|
||||
browser_user_agents = re.findall(r"<a href=\'/.*?>(.+?)</a>", html, re.UNICODE)
|
||||
return [ua for ua in browser_user_agents if "more" not in ua.lower()]
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Failed to fetch user agents for '{browser}': {str(e)}")
|
||||
return []
|
||||
|
||||
|
||||
def update_user_agents(browser_name: str, browser_user_agents: Dict[str, List[str]]) -> None:
|
||||
"""
|
||||
Update browser user agents dictionary with new requests.
|
||||
|
||||
Args:
|
||||
browser_name (str): Name of the browser.
|
||||
browser_user_agents (Dict[str, List[str]]): Dictionary to store browser user agents.
|
||||
"""
|
||||
browser_user_agents[browser_name] = get_browser_user_agents_online(browser_name)
|
||||
|
||||
|
||||
def create_or_update_user_agent_file() -> None:
|
||||
"""
|
||||
Create or update the user agent file with browser user agents.
|
||||
"""
|
||||
user_agent_file = os.path.join(os.environ.get('TEMP'), 'fake_user_agent.json')
|
||||
|
||||
if not os.path.exists(user_agent_file):
|
||||
browser_user_agents: Dict[str, List[str]] = {}
|
||||
threads = []
|
||||
|
||||
for browser_name in ['chrome', 'firefox', 'safari']:
|
||||
t = threading.Thread(target=update_user_agents, args=(browser_name, browser_user_agents))
|
||||
threads.append(t)
|
||||
t.start()
|
||||
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
with open(user_agent_file, 'w') as f:
|
||||
json.dump(browser_user_agents, f, indent=4)
|
||||
logging.info(f"User agent file created at: {user_agent_file}")
|
||||
|
||||
else:
|
||||
logging.info("User agent file already exists.")
|
||||
|
||||
|
||||
class UserAgentManager:
|
||||
"""
|
||||
Manager class to access browser user agents from a file.
|
||||
"""
|
||||
def __init__(self):
|
||||
|
||||
# Get path to temp file where save all user agents
|
||||
self.user_agent_file = os.path.join(os.environ.get('TEMP'), 'fake_user_agent.json')
|
||||
|
||||
# If file dont exist, creaet it
|
||||
if not os.path.exists(self.user_agent_file):
|
||||
create_or_update_user_agent_file()
|
||||
|
||||
def get_random_user_agent(self, browser: str) -> str:
|
||||
"""
|
||||
Get a random user agent for the specified browser.
|
||||
|
||||
Args:
|
||||
browser (str): The name of the browser ('chrome', 'firefox', 'safari').
|
||||
|
||||
Returns:
|
||||
Optional[str]: Random user agent string for the specified browser.
|
||||
"""
|
||||
with open(self.user_agent_file, 'r') as f:
|
||||
browser_user_agents = json.load(f)
|
||||
return random.choice(browser_user_agents.get(browser.lower(), []))
|
||||
|
||||
|
||||
# Output
|
||||
ua = UserAgentManager()
|
||||
@ -1,17 +1,18 @@
|
||||
# 04.04.24
|
||||
|
||||
|
||||
# Import
|
||||
import os
|
||||
import logging
|
||||
import importlib.util
|
||||
|
||||
|
||||
# Variable
|
||||
Cache = {}
|
||||
|
||||
|
||||
class UnidecodeError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def transliterate_nonascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str:
|
||||
"""Transliterates non-ASCII characters in a string to their ASCII counterparts.
|
||||
|
||||
@ -25,6 +26,7 @@ def transliterate_nonascii(string: str, errors: str = 'ignore', replace_str: str
|
||||
"""
|
||||
return _transliterate(string, errors, replace_str)
|
||||
|
||||
|
||||
def _get_ascii_representation(char: str) -> str:
|
||||
"""Obtains the ASCII representation of a Unicode character.
|
||||
|
||||
@ -79,6 +81,7 @@ def _get_ascii_representation(char: str) -> str:
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def _transliterate(string: str, errors: str, replace_str: str) -> str:
|
||||
"""Main transliteration function.
|
||||
|
||||
@ -116,6 +119,7 @@ def _transliterate(string: str, errors: str, replace_str: str) -> str:
|
||||
|
||||
return ''.join(retval)
|
||||
|
||||
|
||||
def transliterate_expect_ascii(string: str, errors: str = 'ignore', replace_str: str = '?') -> str:
|
||||
"""Transliterates non-ASCII characters in a string, expecting ASCII input.
|
||||
|
||||
@ -140,4 +144,6 @@ def transliterate_expect_ascii(string: str, errors: str = 'ignore', replace_str:
|
||||
# Otherwise, transliterate non-ASCII characters
|
||||
return _transliterate(string, errors, replace_str)
|
||||
|
||||
|
||||
# Out
|
||||
transliterate = transliterate_expect_ascii
|
||||
|
||||
@ -10,72 +10,72 @@ data = (
|
||||
'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
|
||||
'', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
|
||||
|
||||
'', # 0x80
|
||||
'', # 0x81
|
||||
'', # 0x82
|
||||
'', # 0x83
|
||||
'', # 0x84
|
||||
'', # 0x85
|
||||
'', # 0x86
|
||||
'', # 0x87
|
||||
'', # 0x88
|
||||
'', # 0x89
|
||||
'', # 0x8a
|
||||
'', # 0x8b
|
||||
'', # 0x8c
|
||||
'', # 0x8d
|
||||
'', # 0x8e
|
||||
'', # 0x8f
|
||||
'', # 0x90
|
||||
'', # 0x91
|
||||
'', # 0x92
|
||||
'', # 0x93
|
||||
'', # 0x94
|
||||
'', # 0x95
|
||||
'', # 0x96
|
||||
'', # 0x97
|
||||
'', # 0x98
|
||||
'', # 0x99
|
||||
'', # 0x9a
|
||||
'', # 0x9b
|
||||
'', # 0x9c
|
||||
'', # 0x9d
|
||||
'', # 0x9e
|
||||
'', # 0x9f
|
||||
'', # 0x80
|
||||
'', # 0x81
|
||||
'', # 0x82
|
||||
'', # 0x83
|
||||
'', # 0x84
|
||||
'', # 0x85
|
||||
'', # 0x86
|
||||
'', # 0x87
|
||||
'', # 0x88
|
||||
'', # 0x89
|
||||
'', # 0x8a
|
||||
'', # 0x8b
|
||||
'', # 0x8c
|
||||
'', # 0x8d
|
||||
'', # 0x8e
|
||||
'', # 0x8f
|
||||
'', # 0x90
|
||||
'', # 0x91
|
||||
'', # 0x92
|
||||
'', # 0x93
|
||||
'', # 0x94
|
||||
'', # 0x95
|
||||
'', # 0x96
|
||||
'', # 0x97
|
||||
'', # 0x98
|
||||
'', # 0x99
|
||||
'', # 0x9a
|
||||
'', # 0x9b
|
||||
'', # 0x9c
|
||||
'', # 0x9d
|
||||
'', # 0x9e
|
||||
'', # 0x9f
|
||||
' ', # 0xa0
|
||||
'!', # 0xa1
|
||||
'C/', # 0xa2
|
||||
'C/', # 0xa2
|
||||
|
||||
# Not "GBP" - Pound Sign is used for more than just British Pounds.
|
||||
'PS', # 0xa3
|
||||
|
||||
'$?', # 0xa4
|
||||
'Y=', # 0xa5
|
||||
'|', # 0xa6
|
||||
'|', # 0xa6
|
||||
'SS', # 0xa7
|
||||
'"', # 0xa8
|
||||
'(c)', # 0xa9
|
||||
'a', # 0xaa
|
||||
'"', # 0xa8
|
||||
'(c)', # 0xa9
|
||||
'a', # 0xaa
|
||||
'<<', # 0xab
|
||||
'!', # 0xac
|
||||
'', # 0xad
|
||||
'(r)', # 0xae
|
||||
'-', # 0xaf
|
||||
'deg', # 0xb0
|
||||
'!', # 0xac
|
||||
'', # 0xad
|
||||
'(r)', # 0xae
|
||||
'-', # 0xaf
|
||||
'deg', # 0xb0
|
||||
'+-', # 0xb1
|
||||
|
||||
# These might be combined with other superscript digits (u+2070 - u+2079)
|
||||
'2', # 0xb2
|
||||
'3', # 0xb3
|
||||
|
||||
'\'', # 0xb4
|
||||
'\'', # 0xb4
|
||||
'u', # 0xb5
|
||||
'P', # 0xb6
|
||||
'*', # 0xb7
|
||||
',', # 0xb8
|
||||
'1', # 0xb9
|
||||
'o', # 0xba
|
||||
'>>', # 0xbb
|
||||
'>>', # 0xbb
|
||||
' 1/4', # 0xbc
|
||||
' 1/2', # 0xbd
|
||||
' 3/4', # 0xbe
|
||||
@ -89,7 +89,7 @@ data = (
|
||||
'A', # 0xc4
|
||||
|
||||
'A', # 0xc5
|
||||
'AE', # 0xc6
|
||||
'AE', # 0xc6
|
||||
'C', # 0xc7
|
||||
'E', # 0xc8
|
||||
'E', # 0xc9
|
||||
@ -119,8 +119,8 @@ data = (
|
||||
'U', # 0xdc
|
||||
|
||||
'Y', # 0xdd
|
||||
'Th', # 0xde
|
||||
'ss', # 0xdf
|
||||
'Th', # 0xde
|
||||
'ss', # 0xdf
|
||||
'a', # 0xe0
|
||||
'a', # 0xe1
|
||||
'a', # 0xe2
|
||||
@ -130,7 +130,7 @@ data = (
|
||||
'a', # 0xe4
|
||||
|
||||
'a', # 0xe5
|
||||
'ae', # 0xe6
|
||||
'ae', # 0xe6
|
||||
'c', # 0xe7
|
||||
'e', # 0xe8
|
||||
'e', # 0xe9
|
||||
@ -160,6 +160,6 @@ data = (
|
||||
'u', # 0xfc
|
||||
|
||||
'y', # 0xfd
|
||||
'th', # 0xfe
|
||||
'th', # 0xfe
|
||||
'y', # 0xff
|
||||
)
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
# 01.03.2023
|
||||
|
||||
# Class import
|
||||
from .version import __version__
|
||||
from Src.Util.console import console
|
||||
|
||||
# General import
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from .version import __version__
|
||||
from Src.Util.console import console
|
||||
|
||||
|
||||
# Variable
|
||||
repo_name = "StreamingCommunity_api"
|
||||
repo_user = "ghost6446"
|
||||
|
||||
@ -4,6 +4,7 @@ import json
|
||||
import os
|
||||
from typing import Any, List
|
||||
|
||||
|
||||
class ConfigManager:
|
||||
def __init__(self, file_path: str = 'config.json') -> None:
|
||||
"""Initialize the ConfigManager.
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
# 24.02.24
|
||||
|
||||
# Import
|
||||
from rich.console import Console
|
||||
from rich.prompt import Prompt
|
||||
|
||||
|
||||
# Variable
|
||||
msg = Prompt()
|
||||
console = Console()
|
||||
@ -1,10 +1,11 @@
|
||||
# 3.12.23 -> 10.12.23 -> 20.03.24
|
||||
# 4.04.24
|
||||
|
||||
# Import
|
||||
import fake_useragent
|
||||
import logging
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Lib.Request.user_agent import ua
|
||||
|
||||
# Variable
|
||||
useragent = fake_useragent.UserAgent(use_external_data=True)
|
||||
|
||||
def get_headers() -> str:
|
||||
"""
|
||||
@ -15,4 +16,7 @@ def get_headers() -> str:
|
||||
"""
|
||||
|
||||
# Get a random user agent string from the user agent rotator
|
||||
return useragent.firefox
|
||||
random_headers = ua.get_random_user_agent("firefox")
|
||||
|
||||
logging.info(f"Use headers: {random_headers}")
|
||||
return random_headers
|
||||
@ -1,12 +1,13 @@
|
||||
# 26.03.24
|
||||
|
||||
# Class import
|
||||
from Src.Util.config import config_manager
|
||||
|
||||
# Import
|
||||
import logging
|
||||
from logging.handlers import RotatingFileHandler
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.config import config_manager
|
||||
|
||||
|
||||
class Logger:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
@ -1,17 +1,22 @@
|
||||
# 3.12.23 -> 19.07.24
|
||||
|
||||
# Class import
|
||||
from .config import config_manager
|
||||
from Src.Util.console import console
|
||||
|
||||
# Import
|
||||
import os
|
||||
import platform
|
||||
|
||||
|
||||
# External libraries
|
||||
from Src.Util.console import console
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from .config import config_manager
|
||||
|
||||
|
||||
# Variable
|
||||
CLEAN = config_manager.get_bool('DEFAULT', 'clean_console')
|
||||
SHOW = config_manager.get_bool('DEFAULT', 'show_message')
|
||||
|
||||
|
||||
def get_os_system():
|
||||
"""
|
||||
This function returns the name of the operating system.
|
||||
@ -19,6 +24,7 @@ def get_os_system():
|
||||
os_system = platform.system()
|
||||
return os_system
|
||||
|
||||
|
||||
def start_message():
|
||||
"""
|
||||
Display a start message.
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
# 24.01.24
|
||||
|
||||
# Import
|
||||
import shutil
|
||||
import os
|
||||
import time
|
||||
@ -9,6 +8,7 @@ import hashlib
|
||||
import logging
|
||||
import re
|
||||
|
||||
|
||||
def remove_folder(folder_path: str) -> None:
|
||||
"""
|
||||
Remove a folder if it exists.
|
||||
@ -23,6 +23,7 @@ def remove_folder(folder_path: str) -> None:
|
||||
except OSError as e:
|
||||
print(f"Error removing folder '{folder_path}': {e}")
|
||||
|
||||
|
||||
def remove_file(file_path: str) -> None:
|
||||
"""
|
||||
Remove a file if it exists
|
||||
@ -38,9 +39,8 @@ def remove_file(file_path: str) -> None:
|
||||
os.remove(file_path)
|
||||
except OSError as e:
|
||||
print(f"Error removing file '{file_path}': {e}")
|
||||
#else:
|
||||
# print(f"File '{file_path}' does not exist.")
|
||||
|
||||
|
||||
|
||||
def remove_special_characters(filename) -> str:
|
||||
"""
|
||||
Removes special characters from a filename to make it suitable for creating a filename in Windows.
|
||||
@ -60,6 +60,7 @@ def remove_special_characters(filename) -> str:
|
||||
|
||||
return cleaned_filename
|
||||
|
||||
|
||||
def move_file_one_folder_up(file_path) -> None:
|
||||
"""
|
||||
Move a file one folder up from its current location.
|
||||
@ -83,6 +84,7 @@ def move_file_one_folder_up(file_path) -> None:
|
||||
# Move the file
|
||||
os.rename(file_path, new_path)
|
||||
|
||||
|
||||
def read_json(path: str):
|
||||
"""Reads JSON file and returns its content.
|
||||
|
||||
@ -98,6 +100,7 @@ def read_json(path: str):
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def save_json(json_obj, path: str) -> None:
|
||||
"""Saves JSON object to the specified file path.
|
||||
|
||||
@ -109,6 +112,7 @@ def save_json(json_obj, path: str) -> None:
|
||||
with open(path, 'w') as file:
|
||||
json.dump(json_obj, file, indent=4) # Adjust the indentation as needed
|
||||
|
||||
|
||||
def clean_json(path: str) -> None:
|
||||
"""Reads JSON data from the file, cleans it, and saves it back.
|
||||
|
||||
@ -132,6 +136,7 @@ def clean_json(path: str) -> None:
|
||||
# Save the modified JSON data back to the file
|
||||
save_json(modified_data, path)
|
||||
|
||||
|
||||
def format_size(size_bytes: float) -> str:
|
||||
"""
|
||||
Format the size in bytes into a human-readable format.
|
||||
@ -154,6 +159,7 @@ def format_size(size_bytes: float) -> str:
|
||||
# Round the size to two decimal places and return with the appropriate unit
|
||||
return f"{size_bytes:.2f} {units[unit_index]}"
|
||||
|
||||
|
||||
def compute_sha1_hash(input_string: str) -> str:
|
||||
"""
|
||||
Computes the SHA-1 hash of the input string.
|
||||
@ -170,6 +176,7 @@ def compute_sha1_hash(input_string: str) -> str:
|
||||
# Return the hashed string
|
||||
return hashed_string
|
||||
|
||||
|
||||
def decode_bytes(bytes_data: bytes, encodings_to_try: list[str] = None) -> str:
|
||||
"""
|
||||
Decode a byte sequence using a list of encodings and return the decoded string.
|
||||
@ -200,6 +207,7 @@ def decode_bytes(bytes_data: bytes, encodings_to_try: list[str] = None) -> str:
|
||||
logging.info("Raw byte data: %s", bytes_data)
|
||||
return None
|
||||
|
||||
|
||||
def convert_to_hex(bytes_data: bytes) -> str:
|
||||
"""
|
||||
Convert a byte sequence to its hexadecimal representation.
|
||||
|
||||
@ -1,9 +1,5 @@
|
||||
# 03.03.24
|
||||
|
||||
# Class import
|
||||
from .message import start_message
|
||||
|
||||
# Import
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.text import Text
|
||||
@ -11,6 +7,11 @@ from rich.prompt import Prompt
|
||||
from rich.style import Style
|
||||
from typing import Dict, List, Any
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from .message import start_message
|
||||
|
||||
|
||||
class TVShowManager:
|
||||
def __init__(self):
|
||||
"""
|
||||
|
||||
BIN
requirements.txt
BIN
requirements.txt
Binary file not shown.
12
run.py
12
run.py
@ -1,6 +1,11 @@
|
||||
# 10.12.23 -> 31.01.24
|
||||
|
||||
# Class
|
||||
import sys
|
||||
import logging
|
||||
import platform
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Api import (
|
||||
get_version_and_domain,
|
||||
download_series,
|
||||
@ -19,10 +24,6 @@ from Src.Upload.update import update as git_update
|
||||
from Src.Lib.FFmpeg import check_ffmpeg
|
||||
from Src.Util.logger import Logger
|
||||
|
||||
# Import
|
||||
import sys
|
||||
import logging
|
||||
import platform
|
||||
|
||||
# Variable
|
||||
DEBUG_MODE = config_manager.get_bool("DEFAULT", "debug")
|
||||
@ -31,7 +32,6 @@ SWITCH_TO = config_manager.get_bool('DEFAULT', 'swith_anime')
|
||||
CLOSE_CONSOLE = config_manager.get_bool('DEFAULT', 'not_close')
|
||||
|
||||
|
||||
# [ main ]
|
||||
def initialize():
|
||||
"""
|
||||
Initialize the application.
|
||||
|
||||
21
update.py
21
update.py
@ -1,16 +1,25 @@
|
||||
# 10.12.24
|
||||
|
||||
# General imports
|
||||
import requests
|
||||
import os
|
||||
import shutil
|
||||
from zipfile import ZipFile
|
||||
from io import BytesIO
|
||||
from zipfile import ZipFile
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util.config import config_manager
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from rich.console import Console
|
||||
|
||||
|
||||
# Variable
|
||||
console = Console()
|
||||
local_path = os.path.join(".")
|
||||
ROOT_PATH = config_manager.get('DEFAULT', 'root_path')
|
||||
|
||||
|
||||
def move_content(source: str, destination: str) :
|
||||
"""
|
||||
@ -36,6 +45,7 @@ def move_content(source: str, destination: str) :
|
||||
else:
|
||||
shutil.move(source_path, destination_path)
|
||||
|
||||
|
||||
def keep_specific_items(directory: str, keep_folder: str, keep_file: str):
|
||||
"""
|
||||
Delete all items in the directory except for the specified folder and file.
|
||||
@ -66,6 +76,7 @@ def keep_specific_items(directory: str, keep_folder: str, keep_file: str):
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
|
||||
def download_and_extract_latest_commit(author: str, repo_name: str):
|
||||
"""
|
||||
Download and extract the latest commit from a GitHub repository.
|
||||
@ -115,6 +126,7 @@ def download_and_extract_latest_commit(author: str, repo_name: str):
|
||||
else:
|
||||
console.log(f"[red]Failed to fetch commit information. Status code: {response.status_code}")
|
||||
|
||||
|
||||
def main_upload():
|
||||
"""
|
||||
Main function to upload the latest commit of a GitHub repository.
|
||||
@ -128,10 +140,11 @@ def main_upload():
|
||||
if cmd_insert == "yes":
|
||||
|
||||
# Remove all old file
|
||||
keep_specific_items(".", "videos", "upload.py")
|
||||
keep_specific_items(".", ROOT_PATH, "upload.py")
|
||||
|
||||
download_and_extract_latest_commit(repository_owner, repository_name)
|
||||
|
||||
|
||||
main_upload()
|
||||
|
||||
# win
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user