mirror of
https://github.com/Arrowar/StreamingCommunity.git
synced 2025-06-07 12:05:35 +00:00
Migrate to httpx
This commit is contained in:
Lovi
parent
661cb949e0
commit
fdd2823865
@ -99,8 +99,7 @@ You can change some behaviors by tweaking the configuration file.
|
||||
* **verify_ssl**: Whether to verify SSL certificates.
|
||||
- **Default Value**: `false`
|
||||
|
||||
* **proxy**: The proxy to use for requests. (Note: This parameter works only with HTTP and HTTPS protocols.)
|
||||
- **Example Value**: `["http://user:pass@38.154.227.167:5868"]`
|
||||
* **proxy**: To use proxy create a file with name list_proxy.txt and copy ip and port like "122.114.232.137:8080". They need to be http
|
||||
|
||||
</details>
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@ -47,12 +47,12 @@ class VideoSource:
|
||||
"""
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=self.headers)
|
||||
response = httpx.get(url, headers=self.headers, follow_redirects=True)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Request failed: {e}")
|
||||
logging.error(f"Request failed [supervideo]: {e}")
|
||||
return None
|
||||
|
||||
def parse_html(self, html_content: str) -> BeautifulSoup:
|
||||
|
||||
@ -5,7 +5,7 @@ import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from unidecode import unidecode
|
||||
|
||||
@ -44,7 +44,7 @@ def title_search(title_search: str) -> int:
|
||||
"""
|
||||
|
||||
# Send request to search for titles
|
||||
response = requests.get(f"https://{SITE_NAME}.{DOMAIN_NOW}/page/1/?story={unidecode(title_search.replace(' ', '+'))}&do=search&subaction=search&titleonly=3", headers={'user-agent': get_headers()})
|
||||
response = httpx.get(f"https://{SITE_NAME}.{DOMAIN_NOW}/page/1/?story={unidecode(title_search.replace(' ', '+'))}&do=search&subaction=search&titleonly=3", headers={'user-agent': get_headers()})
|
||||
response.raise_for_status()
|
||||
|
||||
# Create soup and find table
|
||||
|
||||
@ -6,7 +6,7 @@ from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@ -60,7 +60,7 @@ class VideoSource:
|
||||
"""
|
||||
try:
|
||||
|
||||
response = requests.get(f"https://www.{self.base_name}.{self.domain}/info_api/{self.media_id}/")
|
||||
response = httpx.get(f"https://www.{self.base_name}.{self.domain}/info_api/{self.media_id}/")
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse JSON response and return episode count
|
||||
@ -87,7 +87,7 @@ class VideoSource:
|
||||
"end_range": index_ep + 1
|
||||
}
|
||||
|
||||
response = requests.get(f"https://www.{self.base_name}.{self.domain}/info_api/{self.media_id}/{index_ep}", params = params)
|
||||
response = httpx.get(f"https://www.{self.base_name}.{self.domain}/info_api/{self.media_id}/{index_ep}", params = params)
|
||||
response.raise_for_status()
|
||||
|
||||
# Return information about the episode
|
||||
@ -110,7 +110,7 @@ class VideoSource:
|
||||
"""
|
||||
try:
|
||||
|
||||
response = requests.get(f"https://www.{self.base_name}.{self.domain}/embed-url/{episode_id}")
|
||||
response = httpx.get(f"https://www.{self.base_name}.{self.domain}/embed-url/{episode_id}")
|
||||
response.raise_for_status()
|
||||
|
||||
# Extract and clean embed URL
|
||||
@ -118,7 +118,7 @@ class VideoSource:
|
||||
self.iframe_src = embed_url
|
||||
|
||||
# Fetch video content using embed URL
|
||||
video_response = requests.get(embed_url)
|
||||
video_response = httpx.get(embed_url)
|
||||
video_response.raise_for_status()
|
||||
|
||||
|
||||
|
||||
@ -6,7 +6,7 @@ import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Internal utilities
|
||||
@ -28,7 +28,7 @@ def check_url_for_content(url: str, content: str) -> bool:
|
||||
try:
|
||||
|
||||
logging.info(f"Test site to extract domain: {url}")
|
||||
response = requests.get(url, timeout = 1)
|
||||
response = httpx.get(url, timeout = 1)
|
||||
response.raise_for_status()
|
||||
|
||||
if content in response.text:
|
||||
|
||||
@ -5,7 +5,7 @@ import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from unidecode import unidecode
|
||||
|
||||
@ -45,7 +45,7 @@ def get_token(site_name: str, domain: str) -> dict:
|
||||
"""
|
||||
|
||||
# Send a GET request to the specified URL composed of the site name and domain
|
||||
response = requests.get(f"https://www.{site_name}.{domain}")
|
||||
response = httpx.get(f"https://www.{site_name}.{domain}")
|
||||
response.raise_for_status()
|
||||
|
||||
# Initialize variables to store CSRF token
|
||||
@ -83,11 +83,11 @@ def update_domain():
|
||||
try:
|
||||
|
||||
console.log(f"[cyan]Test site: [red]https://{SITE_NAME}.{DOMAIN_NOW}")
|
||||
response = requests.get(f"https://www.{SITE_NAME}.{DOMAIN_NOW}")
|
||||
response = httpx.get(f"https://www.{SITE_NAME}.{DOMAIN_NOW}")
|
||||
response.status_code
|
||||
|
||||
# If the current site is inaccessible, try to obtain a new domain
|
||||
except:
|
||||
except Exception as e:
|
||||
|
||||
# Get new domain
|
||||
console.print("[red]\nExtract new DOMAIN from TLD list.")
|
||||
@ -166,7 +166,7 @@ def title_search(title: str) -> int:
|
||||
}
|
||||
|
||||
# Send a POST request to the API endpoint for live search
|
||||
response = requests.post(f'https://www.{SITE_NAME}.{url_domain}/livesearch', cookies=cookies, headers=headers, json_data=json_data)
|
||||
response = httpx.post(f'https://www.{SITE_NAME}.{url_domain}/livesearch', cookies=cookies, headers=headers, json=json_data)
|
||||
response.raise_for_status()
|
||||
|
||||
# Process each record returned in the response
|
||||
|
||||
@ -7,9 +7,8 @@ from urllib.parse import urlparse
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from unidecode import unidecode
|
||||
|
||||
|
||||
# Internal utilities
|
||||
@ -41,9 +40,10 @@ def title_search() -> int:
|
||||
|
||||
# Send request to search for titles
|
||||
try:
|
||||
response = requests.get(url_search, headers={'user-agent': get_headers()}, cookies=cookie_index)
|
||||
response = httpx.get(url_search, headers={'user-agent': get_headers()}, cookies=cookie_index)
|
||||
response.raise_for_status()
|
||||
except:
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Insert: {'ips4_IPSSessionFront': 'your_code', 'ips4_member_id': 'your_code'} in config file \ REQUESTS \ index, instead of user-agent. Use browser debug and cookie request with a valid account.")
|
||||
sys.exit(0)
|
||||
|
||||
@ -54,7 +54,8 @@ def title_search() -> int:
|
||||
# Get url and filename
|
||||
try:
|
||||
mp4_link = souce.get("src")
|
||||
except:
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Insert: {'ips4_IPSSessionFront': 'your_code', 'ips4_member_id': 'your_code'} in config file \ REQUESTS \ index, instead of user-agent. Use browser debug and cookie request with a valid account.")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
@ -1,12 +1,11 @@
|
||||
# 26.05.24
|
||||
|
||||
import re
|
||||
import sys
|
||||
import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@ -47,7 +46,7 @@ class VideoSource:
|
||||
"""
|
||||
|
||||
try:
|
||||
response = requests.get(url, headers=self.headers)
|
||||
response = httpx.get(url, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
with open('index.html', 'w', encoding='utf-8') as file:
|
||||
|
||||
@ -7,7 +7,7 @@ from urllib.parse import urlparse
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Internal utilities
|
||||
@ -36,7 +36,7 @@ def title_search() -> int:
|
||||
url_search = msg.ask(f"[cyan]Insert url title")
|
||||
|
||||
# Send request to search for titles
|
||||
response = requests.get(url_search, headers={'user-agent': get_headers()})
|
||||
response = httpx.get(url_search, headers={'user-agent': get_headers()})
|
||||
response.raise_for_status()
|
||||
|
||||
# Get playlist
|
||||
|
||||
@ -6,7 +6,7 @@ from urllib.parse import urljoin, urlparse, parse_qs, urlencode, urlunparse
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@ -66,7 +66,7 @@ class VideoSource:
|
||||
|
||||
try:
|
||||
|
||||
response = requests.get(f"https://{self.base_name}.{self.domain}/titles/{self.media_id}-{self.series_name}", headers=self.headers)
|
||||
response = httpx.get(f"https://{self.base_name}.{self.domain}/titles/{self.media_id}-{self.series_name}", headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
# Extract JSON response if available
|
||||
@ -90,7 +90,7 @@ class VideoSource:
|
||||
try:
|
||||
|
||||
# Make a request to collect information about a specific season
|
||||
response = requests.get(f'https://{self.base_name}.{self.domain}/titles/{self.media_id}-{self.series_name}/stagione-{number_season}', headers=self.headers)
|
||||
response = httpx.get(f'https://{self.base_name}.{self.domain}/titles/{self.media_id}-{self.series_name}/stagione-{number_season}', headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
# Extract JSON response if available
|
||||
@ -122,7 +122,7 @@ class VideoSource:
|
||||
try:
|
||||
|
||||
# Make a request to get iframe source
|
||||
response = requests.get(f"https://{self.base_name}.{self.domain}/iframe/{self.media_id}", params=params)
|
||||
response = httpx.get(f"https://{self.base_name}.{self.domain}/iframe/{self.media_id}", params=params)
|
||||
response.raise_for_status()
|
||||
|
||||
# Parse response with BeautifulSoup to get iframe source
|
||||
@ -164,15 +164,15 @@ class VideoSource:
|
||||
|
||||
# Make a request to get content
|
||||
try:
|
||||
response = requests.get(self.iframe_src, headers=self.headers)
|
||||
response = httpx.get(self.iframe_src, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
|
||||
except:
|
||||
except Exception as e:
|
||||
print("\n")
|
||||
console.print(Panel("[red bold]Coming soon", title="Notification", title_align="left", border_style="yellow"))
|
||||
sys.exit(0)
|
||||
|
||||
if response.ok:
|
||||
if response.status_code == 200:
|
||||
|
||||
# Parse response with BeautifulSoup to get content
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
|
||||
@ -6,7 +6,7 @@ import logging
|
||||
|
||||
|
||||
# External library
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Internal utilities
|
||||
@ -28,7 +28,7 @@ def check_url_for_content(url: str, content: str) -> bool:
|
||||
try:
|
||||
|
||||
logging.info(f"Test site to extract domain: {url}")
|
||||
response = requests.get(url, timeout = 1)
|
||||
response = httpx.get(url, timeout = 1)
|
||||
response.raise_for_status()
|
||||
|
||||
if content in response.text:
|
||||
|
||||
@ -8,7 +8,7 @@ from typing import Tuple
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
from unidecode import unidecode
|
||||
|
||||
@ -102,7 +102,7 @@ def get_version_and_domain(new_domain = None) -> Tuple[str, str]:
|
||||
|
||||
# Make requests to site to get text
|
||||
console.print(f"[cyan]Test site[white]: [red]https://{SITE_NAME}.{config_domain}")
|
||||
response = requests.get(f"https://{SITE_NAME}.{config_domain}")
|
||||
response = httpx.get(f"https://{SITE_NAME}.{config_domain}")
|
||||
console.print(f"[cyan]Test respost site[white]: [red]{response.status_code} \n")
|
||||
|
||||
# Extract version from the response
|
||||
@ -137,7 +137,7 @@ def title_search(title_search: str, domain: str) -> int:
|
||||
"""
|
||||
|
||||
# Send request to search for titles ( replace à to a and space to "+" )
|
||||
response = requests.get(f"https://{SITE_NAME}.{domain}/api/search?q={unidecode(title_search.replace(' ', '+'))}", headers={'user-agent': get_headers()})
|
||||
response = httpx.get(f"https://{SITE_NAME}.{domain}/api/search?q={unidecode(title_search.replace(' ', '+'))}", headers={'user-agent': get_headers()})
|
||||
response.raise_for_status()
|
||||
|
||||
# Add found titles to media search manager
|
||||
|
||||
@ -8,7 +8,7 @@ from typing import Generator, Optional
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
|
||||
@ -80,7 +80,7 @@ def search(query: str, num: int = 10, stop: Optional[int] = None, pause: float =
|
||||
time.sleep(pause)
|
||||
|
||||
# Fetch the HTML content of the search page
|
||||
html = requests.get(url).text
|
||||
html = httpx.get(url).text
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
try:
|
||||
|
||||
@ -6,7 +6,7 @@ import logging
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
@ -38,7 +38,7 @@ def MP4_downloader(url: str, path: str, referer: str, add_desc: str):
|
||||
|
||||
# Make request to get content of video
|
||||
logging.info(f"Make request to fetch mp4 from: {url}")
|
||||
response = requests.get(url, stream=True, headers={'Referer': referer, 'user-agent': get_headers()}, verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT)
|
||||
response = httpx.get(url, stream=True, headers={'Referer': referer, 'user-agent': get_headers()}, verify=REQUEST_VERIFY, timeout=REQUEST_TIMEOUT)
|
||||
total = int(response.headers.get('content-length', 0))
|
||||
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
from unidecode import unidecode
|
||||
|
||||
|
||||
@ -71,7 +71,7 @@ class Downloader():
|
||||
|
||||
Args:
|
||||
- output_filename (str): Output filename for the downloaded content.
|
||||
- m3u8_playlist (str, optional): URL to the main M3U8 playlist or text.
|
||||
- m3u8_playlist (str, optional): URL to the main M3U8 playlist.
|
||||
- m3u8_playlist (str, optional): URL to the main M3U8 index. ( NOT TEXT )
|
||||
"""
|
||||
|
||||
@ -139,9 +139,10 @@ class Downloader():
|
||||
# Send a GET request to the provided URL
|
||||
logging.info(f"Test url: {url}")
|
||||
headers_index['user-agent'] = get_headers()
|
||||
response = requests.get(url, headers=headers_index)
|
||||
response = httpx.get(url, headers=headers_index)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.ok:
|
||||
if response.status_code == 200:
|
||||
return response.text
|
||||
|
||||
else:
|
||||
@ -321,9 +322,10 @@ class Downloader():
|
||||
"""
|
||||
|
||||
# Send a GET request to download the subtitle content
|
||||
response = requests.get(uri)
|
||||
response = httpx.get(uri)
|
||||
response.raise_for_status()
|
||||
|
||||
if response.ok:
|
||||
if response.status_code == 200:
|
||||
|
||||
# Write the content to the specified file
|
||||
with open(path, "wb") as f:
|
||||
@ -368,7 +370,7 @@ class Downloader():
|
||||
m3u8_sub_parser = M3U8_Parser()
|
||||
m3u8_sub_parser.parse_data(
|
||||
uri = obj_subtitle.get('uri'),
|
||||
raw_content = requests.get(obj_subtitle.get('uri')).text
|
||||
raw_content = httpx.get(obj_subtitle.get('uri')).text
|
||||
)
|
||||
|
||||
# Initiate the download of the subtitle content
|
||||
@ -500,16 +502,15 @@ class Downloader():
|
||||
if self.m3u8_playlist:
|
||||
logging.info("Download from PLAYLIST")
|
||||
|
||||
# Fetch the M3U8 playlist content
|
||||
if not len(str(self.m3u8_playlist).split("\n")) > 2: # Is a single link
|
||||
m3u8_playlist_text = self.__df_make_req__(self.m3u8_playlist)
|
||||
|
||||
# Add full URL of the M3U8 playlist to fix next .ts without https if necessary
|
||||
self.m3u8_url_fixer.set_playlist(self.m3u8_playlist) # !!!!!!!!!!!!!!!!!! to fix for playlist with text
|
||||
m3u8_playlist_text = self.__df_make_req__(self.m3u8_playlist)
|
||||
|
||||
else:
|
||||
logging.warning("M3U8 master url not set.") # TO DO
|
||||
m3u8_playlist_text = self.m3u8_playlist
|
||||
# Add full URL of the M3U8 playlist to fix next .ts without https if necessary
|
||||
self.m3u8_url_fixer.set_playlist(self.m3u8_playlist)
|
||||
|
||||
if m3u8_playlist_text is None:
|
||||
console.log("[red]Playlist m3u8 to download is empty.")
|
||||
sys.exit(0)
|
||||
|
||||
# Save text playlist
|
||||
open(os.path.join(self.base_path, "tmp", "playlist.m3u8"), "w+").write(m3u8_playlist_text)
|
||||
|
||||
@ -1,16 +1,18 @@
|
||||
# 09.06.24
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from Src.Util._jsonConfig import config_manager
|
||||
from Src.Util.os import check_file_existence
|
||||
|
||||
|
||||
class ProxyManager:
|
||||
@ -39,15 +41,16 @@ class ProxyManager:
|
||||
- Proxy string if working, None otherwise
|
||||
"""
|
||||
protocol = proxy.split(":")[0].lower()
|
||||
protocol = f'{protocol}://'
|
||||
|
||||
try:
|
||||
response = requests.get(self.url, proxies={protocol: proxy}, timeout=self.timeout)
|
||||
response = httpx.get(self.url, proxies={protocol: proxy}, timeout=self.timeout)
|
||||
|
||||
if response.status_code == 200:
|
||||
logging.info(f"Proxy {proxy} is working.")
|
||||
return proxy
|
||||
|
||||
except requests.RequestException as e:
|
||||
except Exception as e:
|
||||
logging.error(f"Proxy {proxy} failed: {e}")
|
||||
self.failed_proxies[proxy] = time.time()
|
||||
return None
|
||||
@ -57,8 +60,9 @@ class ProxyManager:
|
||||
Verify all proxies in the list and store the working ones.
|
||||
"""
|
||||
logging.info("Starting proxy verification...")
|
||||
with ThreadPoolExecutor(max_workers=10) as executor:
|
||||
with ThreadPoolExecutor(max_workers=os.cpu_count()) as executor:
|
||||
self.verified_proxies = list(executor.map(self._check_proxy, self.proxy_list))
|
||||
|
||||
self.verified_proxies = [proxy for proxy in self.verified_proxies if proxy]
|
||||
logging.info(f"Verification complete. {len(self.verified_proxies)} proxies are working.")
|
||||
|
||||
@ -70,6 +74,7 @@ class ProxyManager:
|
||||
|
||||
for proxy in self.verified_proxies:
|
||||
protocol = proxy.split(":")[0].lower()
|
||||
protocol = f'{protocol}://' # For httpx
|
||||
validate_proxy.append({protocol: proxy})
|
||||
|
||||
return validate_proxy
|
||||
@ -77,12 +82,32 @@ class ProxyManager:
|
||||
|
||||
def main_test_proxy(url_test):
|
||||
|
||||
path_file_proxt_list = "list_proxy.txt"
|
||||
|
||||
if check_file_existence(path_file_proxt_list):
|
||||
|
||||
# Write test to pass THERE IS PROXY on config.json for segments
|
||||
config_manager.set_key("REQUESTS", "proxy", ["192.168.1.1"])
|
||||
|
||||
# Read file
|
||||
with open(path_file_proxt_list, 'r') as file:
|
||||
ip_addresses = file.readlines()
|
||||
|
||||
# Formatt ip
|
||||
ip_addresses = [ip.strip() for ip in ip_addresses]
|
||||
formatted_ips = [f"http://{ip}" for ip in ip_addresses]
|
||||
|
||||
# Get list of proxy from config.json
|
||||
proxy_list = config_manager.get_list('REQUESTS', 'proxy')
|
||||
proxy_list = formatted_ips
|
||||
|
||||
# Verify proxy
|
||||
manager = ProxyManager(proxy_list, url_test)
|
||||
manager.verify_proxies()
|
||||
|
||||
# Write valid ip in txt file
|
||||
with open(path_file_proxt_list, 'w') as file:
|
||||
for ip in ip_addresses:
|
||||
file.write(f"{ip}\n")
|
||||
|
||||
# Return valid proxy
|
||||
return manager.get_verified_proxies()
|
||||
|
||||
@ -13,8 +13,7 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
from requests.exceptions import HTTPError, ConnectionError, Timeout, RequestException
|
||||
import httpx
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
@ -23,6 +22,7 @@ from Src.Util.console import console
|
||||
from Src.Util.headers import get_headers
|
||||
from Src.Util.color import Colors
|
||||
from Src.Util._jsonConfig import config_manager
|
||||
from Src.Util.os import check_file_existence
|
||||
|
||||
|
||||
# Logic class
|
||||
@ -45,14 +45,12 @@ TQDM_MAX_WORKER = config_manager.get_int('M3U8_DOWNLOAD', 'tdqm_workers')
|
||||
TQDM_DELAY_WORKER = config_manager.get_float('M3U8_DOWNLOAD', 'tqdm_delay')
|
||||
TQDM_USE_LARGE_BAR = config_manager.get_int('M3U8_DOWNLOAD', 'tqdm_use_large_bar')
|
||||
REQUEST_TIMEOUT = config_manager.get_float('REQUESTS', 'timeout')
|
||||
THERE_IS_PROXY_LIST = len(config_manager.get_list('REQUESTS', 'proxy')) > 0
|
||||
THERE_IS_PROXY_LIST = check_file_existence("list_proxy.txt")
|
||||
|
||||
|
||||
# Variable
|
||||
headers_index = config_manager.get_dict('REQUESTS', 'index')
|
||||
headers_segments = config_manager.get_dict('REQUESTS', 'segments')
|
||||
session = requests.Session()
|
||||
session.verify = config_manager.get_bool('REQUESTS', 'verify_ssl')
|
||||
|
||||
|
||||
class M3U8_Segments:
|
||||
@ -96,7 +94,7 @@ class M3U8_Segments:
|
||||
logging.info(f"Uri key: {key_uri}")
|
||||
|
||||
try:
|
||||
response = requests.get(key_uri, headers=headers_index)
|
||||
response = httpx.get(key_uri, headers=headers_index)
|
||||
response.raise_for_status()
|
||||
|
||||
except Exception as e:
|
||||
@ -169,11 +167,11 @@ class M3U8_Segments:
|
||||
headers_index['user-agent'] = get_headers()
|
||||
|
||||
# Send a GET request to retrieve the index M3U8 file
|
||||
response = requests.get(self.url, headers=headers_index)
|
||||
response = httpx.get(self.url, headers=headers_index)
|
||||
response.raise_for_status()
|
||||
|
||||
# Save the M3U8 file to the temporary folder
|
||||
if response.ok:
|
||||
if response.status_code == 200:
|
||||
path_m3u8_file = os.path.join(self.tmp_folder, "playlist.m3u8")
|
||||
open(path_m3u8_file, "w+").write(response.text)
|
||||
|
||||
@ -199,9 +197,9 @@ class M3U8_Segments:
|
||||
if THERE_IS_PROXY_LIST:
|
||||
proxy = self.valid_proxy[index % len(self.valid_proxy)]
|
||||
logging.info(f"Use proxy: {proxy}")
|
||||
response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=proxy)
|
||||
response = httpx.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, proxies=proxy, verify=False)
|
||||
else:
|
||||
response = session.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT)
|
||||
response = httpx.get(ts_url, headers=headers_segments, timeout=REQUEST_TIMEOUT, verify=False)
|
||||
|
||||
# Get response content
|
||||
response.raise_for_status()
|
||||
@ -220,10 +218,6 @@ class M3U8_Segments:
|
||||
self.queue.put((index, segment_content))
|
||||
progress_bar.update(1)
|
||||
|
||||
except (HTTPError, ConnectionError, Timeout, RequestException) as e:
|
||||
progress_bar.update(1)
|
||||
logging.error(f"Request-related exception while downloading segment: {e}")
|
||||
|
||||
except Exception as e:
|
||||
progress_bar.update(1)
|
||||
logging.error(f"An unexpected exception occurred while download segment: {e}")
|
||||
@ -280,10 +274,30 @@ class M3U8_Segments:
|
||||
writer_thread = threading.Thread(target=self.write_segments_to_file)
|
||||
writer_thread.start()
|
||||
|
||||
# Ff proxy avaiable set max_workers to number of proxy
|
||||
# else set max_workers to TQDM_MAX_WORKER
|
||||
max_workers = len(self.valid_proxy) if THERE_IS_PROXY_LIST else TQDM_MAX_WORKER
|
||||
|
||||
# if proxy avaiable set timeout to variable time
|
||||
# else set timeout to TDQM_DELAY_WORKER
|
||||
if THERE_IS_PROXY_LIST:
|
||||
num_proxies = len(self.valid_proxy)
|
||||
self.working_proxy_list = self.valid_proxy
|
||||
|
||||
if num_proxies > 0:
|
||||
# calculate delay based on number of proxies
|
||||
# dalay should be between 0.5 and 1
|
||||
delay = max(0.5, min(1, 1 / (num_proxies + 1)))
|
||||
else:
|
||||
delay = TQDM_DELAY_WORKER
|
||||
|
||||
else:
|
||||
delay = TQDM_DELAY_WORKER
|
||||
|
||||
# Start all workers
|
||||
with ThreadPoolExecutor(max_workers=TQDM_MAX_WORKER) as executor:
|
||||
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
||||
for index, segment_url in enumerate(self.segments):
|
||||
time.sleep(TQDM_DELAY_WORKER)
|
||||
time.sleep(delay)
|
||||
executor.submit(self.make_requests_stream, segment_url, index, progress_bar)
|
||||
|
||||
# Wait for all tasks to complete
|
||||
|
||||
@ -1,9 +1,6 @@
|
||||
# 09.06.24
|
||||
# 02.04.24
|
||||
|
||||
from .helper import (
|
||||
M3U8_Decryption,
|
||||
M3U8_Ts_Estimator,
|
||||
M3U8_Parser,
|
||||
M3U8_Codec,
|
||||
M3U8_UrlFix
|
||||
)
|
||||
from .decryptor import M3U8_Decryption
|
||||
from .estimator import M3U8_Ts_Estimator
|
||||
from .parser import M3U8_Parser, M3U8_Codec
|
||||
from .url_fixer import M3U8_UrlFix
|
||||
@ -1,6 +0,0 @@
|
||||
# 02.04.24
|
||||
|
||||
from .decryptor import M3U8_Decryption
|
||||
from .estimator import M3U8_Ts_Estimator
|
||||
from .parser import M3U8_Parser, M3U8_Codec
|
||||
from .url_fixer import M3U8_UrlFix
|
||||
@ -1,14 +1,15 @@
|
||||
# 20.04.25
|
||||
|
||||
import sys
|
||||
import logging
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from ..parser import load
|
||||
from m3u8 import loads
|
||||
|
||||
|
||||
# External libraries
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Costant
|
||||
@ -372,7 +373,7 @@ class M3U8_Subtitle:
|
||||
|
||||
# Send a request to retrieve the subtitle content
|
||||
logging.info(f"Download subtitle: {obj_subtitle.get('name')}")
|
||||
response_subitle = requests.get(obj_subtitle.get('uri'))
|
||||
response_subitle = httpx.get(obj_subtitle.get('uri'))
|
||||
|
||||
try:
|
||||
# Try to extract the VTT URL from the subtitle content
|
||||
@ -418,10 +419,9 @@ class M3U8_Parser:
|
||||
|
||||
|
||||
# Get obj of the m3u8 text content download, dictionary with video, audio, segments, subtitles
|
||||
m3u8_obj = load(raw_content, uri)
|
||||
m3u8_obj = loads(raw_content, uri)
|
||||
|
||||
self.__parse_video_info__(m3u8_obj)
|
||||
self.__parse_encryption_keys__(m3u8_obj)
|
||||
self.__parse_subtitles_and_audio__(m3u8_obj)
|
||||
self.__parse_segments__(m3u8_obj)
|
||||
|
||||
@ -516,6 +516,7 @@ class M3U8_Parser:
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error parsing encryption keys: {e}")
|
||||
sys.exit(0)
|
||||
pass
|
||||
|
||||
def __parse_subtitles_and_audio__(self, m3u8_obj) -> None:
|
||||
@ -557,7 +558,11 @@ class M3U8_Parser:
|
||||
"""
|
||||
|
||||
try:
|
||||
|
||||
for segment in m3u8_obj.segments:
|
||||
|
||||
# Parse key
|
||||
self.__parse_encryption_keys__(segment)
|
||||
|
||||
# Collect all index duration
|
||||
self.duration += segment.duration
|
||||
@ -1,38 +0,0 @@
|
||||
# 15.04.24
|
||||
|
||||
import os
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from .model import M3U8
|
||||
|
||||
|
||||
def load(raw_content, uri):
|
||||
"""
|
||||
Parses the content of an M3U8 playlist and returns an M3U8 object.
|
||||
|
||||
Args:
|
||||
raw_content (str): The content of the M3U8 playlist as a string.
|
||||
uri (str): The URI of the M3U8 playlist file or stream.
|
||||
|
||||
Returns:
|
||||
M3U8: An object representing the parsed M3U8 playlist.
|
||||
|
||||
Raises:
|
||||
IOError: If the raw_content is empty or if the URI cannot be accessed.
|
||||
ValueError: If the raw_content is not a valid M3U8 playlist format.
|
||||
|
||||
Example:
|
||||
>>> m3u8_content = "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:10\n#EXT-X-MEDIA-SEQUENCE:0\n#EXTINF:10.0,\nhttp://example.com/segment0.ts\n#EXTINF:10.0,\nhttp://example.com/segment1.ts\n"
|
||||
>>> uri = "http://example.com/playlist.m3u8"
|
||||
>>> playlist = load(m3u8_content, uri)
|
||||
"""
|
||||
|
||||
if not raw_content:
|
||||
raise IOError("Empty content provided.")
|
||||
|
||||
if not uri:
|
||||
raise IOError("Empty URI provided.")
|
||||
|
||||
base_uri = os.path.dirname(uri)
|
||||
return M3U8(raw_content, base_uri=base_uri)
|
||||
@ -1,28 +0,0 @@
|
||||
# 19.04.24
|
||||
|
||||
import itertools
|
||||
|
||||
|
||||
def remove_quotes_parser(*attrs):
|
||||
"""
|
||||
Returns a dictionary mapping attribute names to a function that removes quotes from their values.
|
||||
"""
|
||||
return dict(zip(attrs, itertools.repeat(remove_quotes)))
|
||||
|
||||
|
||||
def remove_quotes(string):
|
||||
"""
|
||||
Removes quotes from a string.
|
||||
"""
|
||||
quotes = ('"', "'")
|
||||
if string and string[0] in quotes and string[-1] in quotes:
|
||||
return string[1:-1]
|
||||
return string
|
||||
|
||||
|
||||
def normalize_attribute(attribute):
|
||||
"""
|
||||
Normalizes an attribute name by converting hyphens to underscores and converting to lowercase.
|
||||
"""
|
||||
return attribute.replace('-', '_').lower().strip()
|
||||
|
||||
@ -1,358 +0,0 @@
|
||||
# 15.04.24
|
||||
|
||||
import os
|
||||
from collections import namedtuple
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from ..parser import parser
|
||||
|
||||
|
||||
# Variable
|
||||
StreamInfo = namedtuple('StreamInfo', ['bandwidth', 'program_id', 'resolution', 'codecs'])
|
||||
Media = namedtuple('Media', ['uri', 'type', 'group_id', 'language', 'name','default', 'autoselect', 'forced', 'characteristics'])
|
||||
|
||||
|
||||
class M3U8:
|
||||
"""
|
||||
Represents a single M3U8 playlist. Should be instantiated with the content as string.
|
||||
|
||||
Args:
|
||||
- content: the m3u8 content as string
|
||||
- base_path: all urls (key and segments url) will be updated with this base_path,
|
||||
ex: base_path = "http://videoserver.com/hls"
|
||||
- base_uri: uri the playlist comes from. it is propagated to SegmentList and Key
|
||||
ex: http://example.com/path/to
|
||||
|
||||
Attribute:
|
||||
- key: it's a `Key` object, the EXT-X-KEY from m3u8. Or None
|
||||
- segments: a `SegmentList` object, represents the list of `Segment`s from this playlist
|
||||
- is_variant: Returns true if this M3U8 is a variant playlist, with links to other M3U8s with different bitrates.
|
||||
If true, `playlists` is a list of the playlists available, and `iframe_playlists` is a list of the i-frame playlists available.
|
||||
- is_endlist: Returns true if EXT-X-ENDLIST tag present in M3U8.
|
||||
Info: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.8
|
||||
- playlists: If this is a variant playlist (`is_variant` is True), returns a list of Playlist objects
|
||||
- iframe_playlists: If this is a variant playlist (`is_variant` is True), returns a list of IFramePlaylist objects
|
||||
- playlist_type: A lower-case string representing the type of the playlist, which can be one of VOD (video on demand) or EVENT.
|
||||
- media: If this is a variant playlist (`is_variant` is True), returns a list of Media objects
|
||||
- target_duration: Returns the EXT-X-TARGETDURATION as an integer
|
||||
Info: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.2
|
||||
- media_sequence: Returns the EXT-X-MEDIA-SEQUENCE as an integer
|
||||
Info: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.3
|
||||
- program_date_time: Returns the EXT-X-PROGRAM-DATE-TIME as a string
|
||||
Info: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.5
|
||||
- version: Return the EXT-X-VERSION as is
|
||||
- allow_cache: Return the EXT-X-ALLOW-CACHE as is
|
||||
- files: Returns an iterable with all files from playlist, in order. This includes segments and key uri, if present.
|
||||
- base_uri: It is a property (getter and setter) used by SegmentList and Key to have absolute URIs.
|
||||
- is_i_frames_only: Returns true if EXT-X-I-FRAMES-ONLY tag present in M3U8.
|
||||
Guide: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.12
|
||||
|
||||
"""
|
||||
|
||||
# Mapping of simple attributes (obj attribute, parser attribute)
|
||||
SIMPLE_ATTRIBUTES = (
|
||||
('is_variant', 'is_variant'),
|
||||
('is_endlist', 'is_endlist'),
|
||||
('is_i_frames_only', 'is_i_frames_only'),
|
||||
('target_duration', 'targetduration'),
|
||||
('media_sequence', 'media_sequence'),
|
||||
('program_date_time', 'program_date_time'),
|
||||
('version', 'version'),
|
||||
('allow_cache', 'allow_cache'),
|
||||
('playlist_type', 'playlist_type')
|
||||
)
|
||||
|
||||
def __init__(self, content=None, base_path=None, base_uri=None):
|
||||
"""
|
||||
Initialize the M3U8 object.
|
||||
|
||||
Parameters:
|
||||
- content: M3U8 content (string).
|
||||
- base_path: Base path for relative URIs (string).
|
||||
- base_uri: Base URI for absolute URIs (string).
|
||||
"""
|
||||
if content is not None:
|
||||
self.data = parser.parse(content)
|
||||
else:
|
||||
self.data = {}
|
||||
self._base_uri = base_uri
|
||||
self.base_path = base_path
|
||||
self._initialize_attributes()
|
||||
|
||||
def _initialize_attributes(self):
|
||||
"""
|
||||
Initialize attributes based on parsed data.
|
||||
"""
|
||||
# Initialize key and segments
|
||||
self.key = Key(base_uri=self.base_uri, **self.data.get('key', {})) if 'key' in self.data else None
|
||||
self.segments = SegmentList([Segment(base_uri=self.base_uri, **params) for params in self.data.get('segments', [])])
|
||||
|
||||
# Initialize simple attributes
|
||||
for attr, param in self.SIMPLE_ATTRIBUTES:
|
||||
setattr(self, attr, self.data.get(param))
|
||||
|
||||
# Initialize files, media, playlists, and iframe_playlists
|
||||
self.files = []
|
||||
if self.key:
|
||||
self.files.append(self.key.uri)
|
||||
self.files.extend(self.segments.uri)
|
||||
|
||||
self.media = [Media(
|
||||
uri = media.get('uri'),
|
||||
type = media.get('type'),
|
||||
group_id = media.get('group_id'),
|
||||
language = media.get('language'),
|
||||
name = media.get('name'),
|
||||
default = media.get('default'),
|
||||
autoselect = media.get('autoselect'),
|
||||
forced = media.get('forced'),
|
||||
characteristics = media.get('characteristics'))
|
||||
for media in self.data.get('media', [])
|
||||
]
|
||||
self.playlists = PlaylistList([Playlist(
|
||||
base_uri = self.base_uri,
|
||||
media = self.media,
|
||||
**playlist
|
||||
)for playlist in self.data.get('playlists', [])
|
||||
])
|
||||
self.iframe_playlists = PlaylistList()
|
||||
for ifr_pl in self.data.get('iframe_playlists', []):
|
||||
self.iframe_playlists.append(
|
||||
IFramePlaylist(
|
||||
base_uri = self.base_uri,
|
||||
uri = ifr_pl['uri'],
|
||||
iframe_stream_info=ifr_pl['iframe_stream_info'])
|
||||
)
|
||||
|
||||
@property
|
||||
def base_uri(self):
|
||||
"""
|
||||
Get the base URI.
|
||||
"""
|
||||
return self._base_uri
|
||||
|
||||
@base_uri.setter
|
||||
def base_uri(self, new_base_uri):
|
||||
"""
|
||||
Set the base URI.
|
||||
"""
|
||||
self._base_uri = new_base_uri
|
||||
self.segments.base_uri = new_base_uri
|
||||
|
||||
|
||||
class BasePathMixin:
|
||||
"""
|
||||
Mixin class for managing base paths.
|
||||
"""
|
||||
@property
|
||||
def base_path(self):
|
||||
"""
|
||||
Get the base path.
|
||||
"""
|
||||
return os.path.dirname(self.uri)
|
||||
|
||||
@base_path.setter
|
||||
def base_path(self, newbase_path):
|
||||
"""
|
||||
Set the base path.
|
||||
"""
|
||||
if not self.base_path:
|
||||
self.uri = "%s/%s" % (newbase_path, self.uri)
|
||||
self.uri = self.uri.replace(self.base_path, newbase_path)
|
||||
|
||||
|
||||
class GroupedBasePathMixin:
|
||||
"""
|
||||
Mixin class for managing base paths across a group of items.
|
||||
"""
|
||||
|
||||
def _set_base_uri(self, new_base_uri):
|
||||
"""
|
||||
Set the base URI for each item in the group.
|
||||
"""
|
||||
for item in self:
|
||||
item.base_uri = new_base_uri
|
||||
|
||||
base_uri = property(None, _set_base_uri)
|
||||
|
||||
def _set_base_path(self, new_base_path):
|
||||
"""
|
||||
Set the base path for each item in the group.
|
||||
"""
|
||||
for item in self:
|
||||
item.base_path = new_base_path
|
||||
|
||||
base_path = property(None, _set_base_path)
|
||||
|
||||
|
||||
class Segment(BasePathMixin):
|
||||
"""
|
||||
Class representing a segment in an M3U8 playlist.
|
||||
Inherits from BasePathMixin for managing base paths.
|
||||
"""
|
||||
|
||||
def __init__(self, uri, base_uri, program_date_time=None, duration=None,
|
||||
title=None, byterange=None, discontinuity=False, key=None):
|
||||
"""
|
||||
Initialize a Segment object.
|
||||
|
||||
Args:
|
||||
- uri: URI of the segment.
|
||||
- base_uri: Base URI for the segment.
|
||||
- program_date_time: Returns the EXT-X-PROGRAM-DATE-TIME as a datetime
|
||||
Guide: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.5
|
||||
- duration: Duration of the segment (optional).
|
||||
- title: Title attribute from EXTINF parameter
|
||||
- byterange: Byterange information of the segment (optional).
|
||||
- discontinuity: Returns a boolean indicating if a EXT-X-DISCONTINUITY tag exists
|
||||
Guide: http://tools.ietf.org/html/draft-pantos-http-live-streaming-13#section-3.4.11
|
||||
- key: Key for encryption (optional).
|
||||
"""
|
||||
self.uri = uri
|
||||
self.duration = duration
|
||||
self.title = title
|
||||
self.base_uri = base_uri
|
||||
self.byterange = byterange
|
||||
self.program_date_time = program_date_time
|
||||
self.discontinuity = discontinuity
|
||||
#self.key = key
|
||||
|
||||
|
||||
class SegmentList(list, GroupedBasePathMixin):
|
||||
"""
|
||||
Class representing a list of segments in an M3U8 playlist.
|
||||
Inherits from list and GroupedBasePathMixin for managing base paths across a group of items.
|
||||
"""
|
||||
|
||||
@property
|
||||
def uri(self):
|
||||
"""
|
||||
Get the URI of each segment in the SegmentList.
|
||||
|
||||
Returns:
|
||||
- List of URIs of segments in the SegmentList.
|
||||
"""
|
||||
return [seg.uri for seg in self]
|
||||
|
||||
|
||||
class Key(BasePathMixin):
|
||||
"""
|
||||
Class representing a key used for encryption in an M3U8 playlist.
|
||||
Inherits from BasePathMixin for managing base paths.
|
||||
"""
|
||||
|
||||
def __init__(self, method, uri, base_uri, iv=None):
|
||||
"""
|
||||
Initialize a Key object.
|
||||
|
||||
Args:
|
||||
- method: Encryption method.
|
||||
ex: "AES-128"
|
||||
- uri: URI of the key.
|
||||
ex: "https://priv.example.com/key.php?r=52"
|
||||
- base_uri: Base URI for the key.
|
||||
ex: http://example.com/path/to
|
||||
- iv: Initialization vector (optional).
|
||||
ex: 0X12A
|
||||
"""
|
||||
self.method = method
|
||||
self.uri = uri
|
||||
self.iv = iv
|
||||
self.base_uri = base_uri
|
||||
|
||||
|
||||
class Playlist(BasePathMixin):
|
||||
"""
|
||||
Playlist object representing a link to a variant M3U8 with a specific bitrate.
|
||||
|
||||
More info: http://tools.ietf.org/html/draft-pantos-http-live-streaming-07#section-3.3.10
|
||||
"""
|
||||
|
||||
def __init__(self, uri, stream_info, media, base_uri):
|
||||
"""
|
||||
Initialize a Playlist object.
|
||||
|
||||
Args:
|
||||
- uri: URI of the playlist.
|
||||
- stream_info: is a named tuple containing the attributes: `program_id`,
|
||||
- media: List of Media objects associated with the playlist.
|
||||
- base_uri: Base URI for the playlist.
|
||||
"""
|
||||
self.uri = uri
|
||||
self.base_uri = base_uri
|
||||
|
||||
# Extract resolution information from stream_info
|
||||
resolution = stream_info.get('resolution')
|
||||
if resolution is not None:
|
||||
values = resolution.split('x')
|
||||
resolution_pair = (int(values[0]), int(values[1]))
|
||||
else:
|
||||
resolution_pair = None
|
||||
|
||||
# Create StreamInfo object
|
||||
self.stream_info = StreamInfo(
|
||||
bandwidth = stream_info['bandwidth'],
|
||||
program_id = stream_info.get('program_id'),
|
||||
resolution = resolution_pair,
|
||||
codecs = stream_info.get('codecs')
|
||||
)
|
||||
|
||||
# Filter media based on group ID and media type
|
||||
self.media = []
|
||||
for media_type in ('audio', 'video', 'subtitles'):
|
||||
group_id = stream_info.get(media_type)
|
||||
if group_id:
|
||||
self.media += filter(lambda m: m.group_id == group_id, media)
|
||||
|
||||
|
||||
class IFramePlaylist(BasePathMixin):
|
||||
"""
|
||||
Class representing an I-Frame playlist in an M3U8 playlist.
|
||||
Inherits from BasePathMixin for managing base paths.
|
||||
"""
|
||||
|
||||
def __init__(self, base_uri, uri, iframe_stream_info):
|
||||
"""
|
||||
Initialize an IFramePlaylist object.
|
||||
|
||||
Args:
|
||||
- base_uri: Base URI for the I-Frame playlist.
|
||||
- uri: URI of the I-Frame playlist.
|
||||
- iframe_stream_info, is a named tuple containing the attributes:
|
||||
`program_id`, `bandwidth`, `codecs` and `resolution` which is a tuple (w, h) of integers
|
||||
"""
|
||||
self.uri = uri
|
||||
self.base_uri = base_uri
|
||||
|
||||
# Extract resolution information from iframe_stream_info
|
||||
resolution = iframe_stream_info.get('resolution')
|
||||
if resolution is not None:
|
||||
values = resolution.split('x')
|
||||
resolution_pair = (int(values[0]), int(values[1]))
|
||||
else:
|
||||
resolution_pair = None
|
||||
|
||||
# Create StreamInfo object for I-Frame playlist
|
||||
self.iframe_stream_info = StreamInfo(
|
||||
bandwidth = iframe_stream_info.get('bandwidth'),
|
||||
program_id = iframe_stream_info.get('program_id'),
|
||||
resolution = resolution_pair,
|
||||
codecs = iframe_stream_info.get('codecs')
|
||||
)
|
||||
|
||||
class PlaylistList(list, GroupedBasePathMixin):
|
||||
"""
|
||||
Class representing a list of playlists in an M3U8 playlist.
|
||||
Inherits from list and GroupedBasePathMixin for managing base paths across a group of items.
|
||||
"""
|
||||
|
||||
def __str__(self):
|
||||
"""
|
||||
Return a string representation of the PlaylistList.
|
||||
|
||||
Returns:
|
||||
- String representation of the PlaylistList.
|
||||
"""
|
||||
output = [str(playlist) for playlist in self]
|
||||
return '\n'.join(output)
|
||||
@ -1,338 +0,0 @@
|
||||
# 15.04.24
|
||||
|
||||
import re
|
||||
import logging
|
||||
import datetime
|
||||
|
||||
|
||||
# Internal utilities
|
||||
from ..parser import protocol
|
||||
from ._util import (
|
||||
remove_quotes,
|
||||
remove_quotes_parser,
|
||||
normalize_attribute
|
||||
)
|
||||
|
||||
|
||||
# External utilities
|
||||
from Src.Util._jsonConfig import config_manager
|
||||
|
||||
|
||||
# Variable
|
||||
REMOVE_EMPTY_ROW = config_manager.get_bool('M3U8_PARSER', 'skip_empty_row_playlist')
|
||||
ATTRIBUTELISTPATTERN = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''')
|
||||
|
||||
|
||||
def parse(content):
|
||||
"""
|
||||
Given an M3U8 playlist content, parses the content and extracts metadata.
|
||||
|
||||
Args:
|
||||
content (str): The M3U8 playlist content.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the parsed metadata.
|
||||
"""
|
||||
|
||||
# Initialize data dictionary with default values
|
||||
data = {
|
||||
'is_variant': False,
|
||||
'is_endlist': False,
|
||||
'is_i_frames_only': False,
|
||||
'playlist_type': None,
|
||||
'playlists': [],
|
||||
'iframe_playlists': [],
|
||||
'segments': [],
|
||||
'media': [],
|
||||
}
|
||||
|
||||
# Initialize state dictionary for tracking parsing state
|
||||
state = {
|
||||
'expect_segment': False,
|
||||
'expect_playlist': False,
|
||||
}
|
||||
|
||||
# Iterate over lines in the content
|
||||
content = content.split("\n")
|
||||
content_length = len(content)
|
||||
i = 0
|
||||
|
||||
while i < content_length:
|
||||
line = content[i]
|
||||
line_stripped = line.strip()
|
||||
is_end = i + 1 == content_length - 2
|
||||
|
||||
if REMOVE_EMPTY_ROW:
|
||||
if i < content_length - 2:
|
||||
actual_row = extract_params(line_stripped)
|
||||
next_row = extract_params(content[i + 2].strip())
|
||||
|
||||
if actual_row is not None and next_row is None and not is_end:
|
||||
logging.info(f"Skip row: {line_stripped}")
|
||||
i += 1
|
||||
continue
|
||||
|
||||
i += 1
|
||||
|
||||
if line.startswith(protocol.ext_x_byterange):
|
||||
_parse_byterange(line, state)
|
||||
state['expect_segment'] = True
|
||||
|
||||
elif state['expect_segment']:
|
||||
_parse_ts_chunk(line, data, state)
|
||||
state['expect_segment'] = False
|
||||
|
||||
elif state['expect_playlist']:
|
||||
_parse_variant_playlist(line, data, state)
|
||||
state['expect_playlist'] = False
|
||||
|
||||
elif line.startswith(protocol.ext_x_targetduration):
|
||||
_parse_simple_parameter(line, data, float)
|
||||
elif line.startswith(protocol.ext_x_media_sequence):
|
||||
_parse_simple_parameter(line, data, int)
|
||||
elif line.startswith(protocol.ext_x_discontinuity):
|
||||
state['discontinuity'] = True
|
||||
elif line.startswith(protocol.ext_x_version):
|
||||
_parse_simple_parameter(line, data)
|
||||
elif line.startswith(protocol.ext_x_allow_cache):
|
||||
_parse_simple_parameter(line, data)
|
||||
|
||||
elif line.startswith(protocol.ext_x_key):
|
||||
state['current_key'] = _parse_key(line)
|
||||
data['key'] = data.get('key', state['current_key'])
|
||||
|
||||
elif line.startswith(protocol.extinf):
|
||||
_parse_extinf(line, data, state)
|
||||
state['expect_segment'] = True
|
||||
|
||||
elif line.startswith(protocol.ext_x_stream_inf):
|
||||
state['expect_playlist'] = True
|
||||
_parse_stream_inf(line, data, state)
|
||||
|
||||
elif line.startswith(protocol.ext_x_i_frame_stream_inf):
|
||||
_parse_i_frame_stream_inf(line, data)
|
||||
|
||||
elif line.startswith(protocol.ext_x_media):
|
||||
_parse_media(line, data, state)
|
||||
|
||||
elif line.startswith(protocol.ext_x_playlist_type):
|
||||
_parse_simple_parameter(line, data)
|
||||
|
||||
elif line.startswith(protocol.ext_i_frames_only):
|
||||
data['is_i_frames_only'] = True
|
||||
|
||||
elif line.startswith(protocol.ext_x_endlist):
|
||||
data['is_endlist'] = True
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def extract_params(line):
|
||||
"""
|
||||
Extracts parameters from a formatted input string.
|
||||
|
||||
Args:
|
||||
- line (str): The string containing the parameters to extract.
|
||||
|
||||
Returns:
|
||||
dict or None: A dictionary containing the extracted parameters with their respective values.
|
||||
"""
|
||||
params = {}
|
||||
matches = re.findall(r'([A-Z\-]+)=("[^"]*"|[^",\s]*)', line)
|
||||
if not matches:
|
||||
return None
|
||||
for match in matches:
|
||||
param, value = match
|
||||
params[param] = value.strip('"')
|
||||
return params
|
||||
|
||||
def _parse_key(line):
|
||||
"""
|
||||
Parses the #EXT-X-KEY line and extracts key attributes.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXT-X-KEY line from the playlist.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the key attributes.
|
||||
"""
|
||||
params = ATTRIBUTELISTPATTERN.split(line.replace(protocol.ext_x_key + ':', ''))[1::2]
|
||||
key = {}
|
||||
for param in params:
|
||||
name, value = param.split('=', 1)
|
||||
key[normalize_attribute(name)] = remove_quotes(value)
|
||||
return key
|
||||
|
||||
def _parse_extinf(line, data, state):
|
||||
"""
|
||||
Parses the #EXTINF line and extracts segment duration and title.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXTINF line from the playlist.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
- state (dict): The parsing state.
|
||||
"""
|
||||
duration, title = line.replace(protocol.extinf + ':', '').split(',')
|
||||
state['segment'] = {'duration': float(duration), 'title': remove_quotes(title)}
|
||||
|
||||
def _parse_ts_chunk(line, data, state):
|
||||
"""
|
||||
Parses a segment URI line and adds it to the segment list.
|
||||
|
||||
Args:
|
||||
line (str): The segment URI line from the playlist.
|
||||
data (dict): The dictionary to store the parsed data.
|
||||
state (dict): The parsing state.
|
||||
"""
|
||||
segment = state.pop('segment')
|
||||
if state.get('current_program_date_time'):
|
||||
segment['program_date_time'] = state['current_program_date_time']
|
||||
state['current_program_date_time'] += datetime.timedelta(seconds=segment['duration'])
|
||||
segment['uri'] = line
|
||||
segment['discontinuity'] = state.pop('discontinuity', False)
|
||||
if state.get('current_key'):
|
||||
segment['key'] = state['current_key']
|
||||
data['segments'].append(segment)
|
||||
|
||||
def _parse_attribute_list(prefix, line, atribute_parser):
|
||||
"""
|
||||
Parses a line containing a list of attributes and their values.
|
||||
|
||||
Args:
|
||||
- prefix (str): The prefix to identify the line.
|
||||
- line (str): The line containing the attributes.
|
||||
- atribute_parser (dict): A dictionary mapping attribute names to parsing functions.
|
||||
|
||||
Returns:
|
||||
dict: A dictionary containing the parsed attributes.
|
||||
"""
|
||||
params = ATTRIBUTELISTPATTERN.split(line.replace(prefix + ':', ''))[1::2]
|
||||
|
||||
attributes = {}
|
||||
for param in params:
|
||||
name, value = param.split('=', 1)
|
||||
name = normalize_attribute(name)
|
||||
|
||||
if name in atribute_parser:
|
||||
value = atribute_parser[name](value)
|
||||
|
||||
attributes[name] = value
|
||||
|
||||
return attributes
|
||||
|
||||
def _parse_stream_inf(line, data, state):
|
||||
"""
|
||||
Parses the #EXT-X-STREAM-INF line and extracts stream information.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXT-X-STREAM-INF line from the playlist.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
- state (dict): The parsing state.
|
||||
"""
|
||||
data['is_variant'] = True
|
||||
atribute_parser = remove_quotes_parser('codecs', 'audio', 'video', 'subtitles')
|
||||
atribute_parser["program_id"] = int
|
||||
atribute_parser["bandwidth"] = int
|
||||
state['stream_info'] = _parse_attribute_list(protocol.ext_x_stream_inf, line, atribute_parser)
|
||||
|
||||
def _parse_i_frame_stream_inf(line, data):
|
||||
"""
|
||||
Parses the #EXT-X-I-FRAME-STREAM-INF line and extracts I-frame stream information.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXT-X-I-FRAME-STREAM-INF line from the playlist.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
"""
|
||||
atribute_parser = remove_quotes_parser('codecs', 'uri')
|
||||
atribute_parser["program_id"] = int
|
||||
atribute_parser["bandwidth"] = int
|
||||
iframe_stream_info = _parse_attribute_list(protocol.ext_x_i_frame_stream_inf, line, atribute_parser)
|
||||
iframe_playlist = {'uri': iframe_stream_info.pop('uri'),
|
||||
'iframe_stream_info': iframe_stream_info}
|
||||
|
||||
data['iframe_playlists'].append(iframe_playlist)
|
||||
|
||||
def _parse_media(line, data, state):
|
||||
"""
|
||||
Parses the #EXT-X-MEDIA line and extracts media attributes.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXT-X-MEDIA line from the playlist.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
- state (dict): The parsing state.
|
||||
"""
|
||||
quoted = remove_quotes_parser('uri', 'group_id', 'language', 'name', 'characteristics')
|
||||
media = _parse_attribute_list(protocol.ext_x_media, line, quoted)
|
||||
data['media'].append(media)
|
||||
|
||||
def _parse_variant_playlist(line, data, state):
|
||||
"""
|
||||
Parses a variant playlist line and extracts playlist information.
|
||||
|
||||
Args:
|
||||
- line (str): The variant playlist line from the playlist.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
- state (dict): The parsing state.
|
||||
"""
|
||||
playlist = {'uri': line, 'stream_info': state.pop('stream_info')}
|
||||
|
||||
data['playlists'].append(playlist)
|
||||
|
||||
def _parse_byterange(line, state):
|
||||
"""
|
||||
Parses the #EXT-X-BYTERANGE line and extracts byte range information.
|
||||
|
||||
Args:
|
||||
- line (str): The #EXT-X-BYTERANGE line from the playlist.
|
||||
- state (dict): The parsing state.
|
||||
"""
|
||||
state['segment']['byterange'] = line.replace(protocol.ext_x_byterange + ':', '')
|
||||
|
||||
def _parse_simple_parameter_raw_value(line, cast_to=str, normalize=False):
|
||||
"""
|
||||
Parses a line containing a simple parameter and its value.
|
||||
|
||||
Args:
|
||||
- line (str): The line containing the parameter and its value.
|
||||
- cast_to (type): The type to which the value should be cast.
|
||||
- normalize (bool): Whether to normalize the parameter name.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing the parameter name and its value.
|
||||
"""
|
||||
param, value = line.split(':', 1)
|
||||
param = normalize_attribute(param.replace('#EXT-X-', ''))
|
||||
if normalize:
|
||||
value = normalize_attribute(value)
|
||||
return param, cast_to(value)
|
||||
|
||||
def _parse_and_set_simple_parameter_raw_value(line, data, cast_to=str, normalize=False):
|
||||
"""
|
||||
Parses a line containing a simple parameter and its value, and sets it in the data dictionary.
|
||||
|
||||
Args:
|
||||
- line (str): The line containing the parameter and its value.
|
||||
- data (dict): The dictionary to store the parsed data.
|
||||
- cast_to (type): The type to which the value should be cast.
|
||||
- normalize (bool): Whether to normalize the parameter name.
|
||||
|
||||
Returns:
|
||||
The parsed value.
|
||||
"""
|
||||
param, value = _parse_simple_parameter_raw_value(line, cast_to, normalize)
|
||||
data[param] = value
|
||||
return data[param]
|
||||
|
||||
def _parse_simple_parameter(line, data, cast_to=str):
|
||||
"""
|
||||
Parses a line containing a simple parameter and its value, and sets it in the data dictionary.
|
||||
|
||||
Args:
|
||||
line (str): The line containing the parameter and its value.
|
||||
data (dict): The dictionary to store the parsed data.
|
||||
cast_to (type): The type to which the value should be cast.
|
||||
|
||||
Returns:
|
||||
The parsed value.
|
||||
"""
|
||||
return _parse_and_set_simple_parameter_raw_value(line, data, cast_to, True)
|
||||
@ -1,17 +0,0 @@
|
||||
# 15.04.24
|
||||
|
||||
ext_x_targetduration = '#EXT-X-TARGETDURATION'
|
||||
ext_x_media_sequence = '#EXT-X-MEDIA-SEQUENCE'
|
||||
ext_x_program_date_time = '#EXT-X-PROGRAM-DATE-TIME'
|
||||
ext_x_media = '#EXT-X-MEDIA'
|
||||
ext_x_playlist_type = '#EXT-X-PLAYLIST-TYPE'
|
||||
ext_x_key = '#EXT-X-KEY'
|
||||
ext_x_stream_inf = '#EXT-X-STREAM-INF'
|
||||
ext_x_version = '#EXT-X-VERSION'
|
||||
ext_x_allow_cache = '#EXT-X-ALLOW-CACHE'
|
||||
ext_x_endlist = '#EXT-X-ENDLIST'
|
||||
extinf = '#EXTINF'
|
||||
ext_i_frames_only = '#EXT-X-I-FRAMES-ONLY'
|
||||
ext_x_byterange = '#EXT-X-BYTERANGE'
|
||||
ext_x_i_frame_stream_inf = '#EXT-X-I-FRAME-STREAM-INF'
|
||||
ext_x_discontinuity = '#EXT-X-DISCONTINUITY'
|
||||
@ -10,7 +10,7 @@ from Src.Util.console import console
|
||||
|
||||
|
||||
# External library
|
||||
import requests
|
||||
import httpx
|
||||
|
||||
|
||||
# Variable
|
||||
@ -28,9 +28,10 @@ def update():
|
||||
|
||||
# Make the GitHub API requests and handle potential errors
|
||||
try:
|
||||
response_reposity = requests.get(f"https://api.github.com/repos/{repo_user}/{repo_name}").json()
|
||||
response_releases = requests.get(f"https://api.github.com/repos/{repo_user}/{repo_name}/releases").json()
|
||||
except requests.RequestException as e:
|
||||
response_reposity = httpx.get(f"https://api.github.com/repos/{repo_user}/{repo_name}").json()
|
||||
response_releases = httpx.get(f"https://api.github.com/repos/{repo_user}/{repo_name}/releases").json()
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error accessing GitHub API: {e}")
|
||||
return
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# 29.04.24
|
||||
|
||||
import requests
|
||||
import httpx
|
||||
import json
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
@ -22,10 +22,10 @@ preference_registry = ['Verisign', 'KSregistry', 'KNET']
|
||||
def scrape_new_gtld_applications(url):
|
||||
|
||||
# Send a GET request to the URL
|
||||
response = requests.get(url)
|
||||
response = httpx.get(url)
|
||||
|
||||
# Check if the response is successful
|
||||
if response.ok:
|
||||
if response.status_code == 200:
|
||||
|
||||
# Parse the HTML content of the page
|
||||
soup = BeautifulSoup(response.content, 'html.parser')
|
||||
|
||||
13
config.json
13
config.json
@ -13,8 +13,7 @@
|
||||
"timeout": 5,
|
||||
"verify_ssl": false,
|
||||
"index": {"user-agent": ""},
|
||||
"segments": { "user-agent": ""},
|
||||
"proxy": []
|
||||
"segments": {"user-agent": ""}
|
||||
},
|
||||
"M3U8_DOWNLOAD": {
|
||||
"tdqm_workers": 2,
|
||||
@ -32,10 +31,10 @@
|
||||
},
|
||||
"M3U8_CONVERSION": {
|
||||
"use_codec": false,
|
||||
"use_vcodec": true,
|
||||
"use_acodec": true,
|
||||
"use_bitrate": true,
|
||||
"use_gpu": true,
|
||||
"use_vcodec": true,
|
||||
"use_acodec": true,
|
||||
"use_bitrate": true,
|
||||
"use_gpu": false,
|
||||
"default_preset": "ultrafast",
|
||||
"check_output_after_ffmpeg": false
|
||||
},
|
||||
@ -48,4 +47,4 @@
|
||||
"animeunity": "to",
|
||||
"altadefinizione": "vodka"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
requests
|
||||
httpx
|
||||
bs4
|
||||
rich
|
||||
tqdm
|
||||
m3u8
|
||||
unidecode
|
||||
fake-useragent
|
||||
Loading…
x
Reference in New Issue
Block a user