diff --git a/.github/.domain/domain_update.py b/.github/.domain/domain_update.py new file mode 100644 index 0000000..937661f --- /dev/null +++ b/.github/.domain/domain_update.py @@ -0,0 +1,263 @@ +# 20.04.2024 + +import re +import os +import json +from datetime import datetime +from urllib.parse import urlparse, urlunparse + +import httpx +import ua_generator + +JSON_FILE_PATH = os.path.join(".github", ".domain", "domains.json") + + +def load_domains(file_path): + if not os.path.exists(file_path): + print(f"Error: The file {file_path} was not found.") + return None + + try: + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + + except Exception as e: + print(f"Error reading the file {file_path}: {e}") + return None + +def save_domains(file_path, data): + try: + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, indent=2, ensure_ascii=False) + print(f"Data successfully saved to {file_path}") + + except Exception as e: + print(f"Error saving the file {file_path}: {e}") + +def get_new_tld(full_url): + try: + parsed_url = urlparse(full_url) + hostname = parsed_url.hostname + if hostname: + parts = hostname.split('.') + return parts[-1] + + except Exception: + pass + + return None + +def extract_domain_from_response(response, original_url): + if 'location' in response.headers: + return response.headers['location'] + + if str(response.url) != original_url: + return str(response.url) + + try: + content_type = response.headers.get('content-type', '').lower() + if 'text/html' in content_type or 'text/plain' in content_type: + response_text = response.text + + js_redirect_patterns = [ + r'window\.location\.href\s*=\s*["\']([^"\']+)["\']', + r'window\.location\s*=\s*["\']([^"\']+)["\']', + r'location\.href\s*=\s*["\']([^"\']+)["\']', + r'document\.location\s*=\s*["\']([^"\']+)["\']' + ] + + for pattern in js_redirect_patterns: + js_match = re.search(pattern, response_text, re.IGNORECASE) + if js_match: + return js_match.group(1) + + meta_patterns = [ + r']*http-equiv=["\']?refresh["\']?[^>]*content=["\'][^"\']*url=([^"\'>\s]+)', + r']*content=["\'][^"\']*url=([^"\'>\s]+)[^>]*http-equiv=["\']?refresh["\']?' + ] + + for pattern in meta_patterns: + meta_match = re.search(pattern, response_text, re.IGNORECASE) + if meta_match: + return meta_match.group(1) + + canonical_match = re.search(r']*rel=["\']?canonical["\']?[^>]*href=["\']([^"\']+)["\']', response_text, re.IGNORECASE) + if canonical_match: + return canonical_match.group(1) + + base_match = re.search(r']*href=["\']([^"\']+)["\']', response_text, re.IGNORECASE) + if base_match: + return base_match.group(1) + + error_redirect_patterns = [ + r'[Rr]edirect(?:ed)?\s+to:?\s*([^\s<>"\']+)', + r'[Nn]ew\s+[Uu][Rr][Ll]:?\s*([^\s<>"\']+)', + r'[Mm]oved\s+to:?\s*([^\s<>"\']+)', + r'[Ff]ound\s+at:?\s*([^\s<>"\']+)' + ] + + for pattern in error_redirect_patterns: + error_match = re.search(pattern, response_text) + if error_match: + potential_url = error_match.group(1) + if potential_url.startswith(('http://', 'https://', '//')): + return potential_url + + except Exception as e: + print(f" [!] Error extracting from response content: {e}") + + return None + +def try_url(url_to_try, headers, timeout=15): + try: + with httpx.Client(headers=headers, timeout=timeout, follow_redirects=False) as client: + response = client.get(url_to_try) + + if response.status_code in [301, 302, 303, 307, 308]: + location = response.headers.get('location') + if location: + print(f" [+] Found redirect ({response.status_code}) to: {location}") + try: + final_response = client.get(location) + if 200 <= final_response.status_code < 400: + return final_response + else: + return httpx.Response( + status_code=200, + headers={"location": location}, + content=b"", + request=response.request + ) + except Exception: + return httpx.Response( + status_code=200, + headers={"location": location}, + content=b"", + request=response.request + ) + + elif response.status_code in [403, 409, 429, 503]: + print(f" [!] HTTP {response.status_code} - attempting to extract redirect info") + + location = response.headers.get('location') + if location: + print(f" [+] Found location header in error response: {location}") + return httpx.Response( + status_code=200, + headers={"location": location}, + content=b"", + request=response.request + ) + + new_url = extract_domain_from_response(response, url_to_try) + if new_url and new_url != url_to_try: + print(f" [+] Found redirect URL in error response content: {new_url}") + return httpx.Response( + status_code=200, + headers={"location": new_url}, + content=b"", + request=response.request + ) + + if 200 <= response.status_code < 400: + return response + + print(f" [!] HTTP {response.status_code} for {url_to_try}") + + except httpx.HTTPStatusError as http_err: + new_url = extract_domain_from_response(http_err.response, url_to_try) + if new_url: + print(f" [+] Found new URL from HTTPStatusError response: {new_url}") + return httpx.Response( + status_code=200, + headers={"location": new_url}, + content=b"", + request=http_err.request + ) + except Exception as e: + print(f" [!] Error for {url_to_try}: {type(e).__name__}") + + return None + +def update_domain_entries(data): + if not data: + return False + + updated_count = 0 + + for key, entry in data.items(): + print(f"\n--- [DOMAIN] {key} ---") + original_full_url = entry.get("full_url") + original_domain_in_entry = entry.get("domain") + + if not original_full_url: + print(f" [!] 'full_url' missing. Skipped.") + continue + + ua = ua_generator.generate(device=('desktop', 'mobile'), browser=('chrome', 'edge', 'firefox', 'safari')) + current_headers = ua.headers.get() + + print(f" [] Stored URL: {original_full_url}") + if original_domain_in_entry: + print(f" [] Stored Domain (TLD): {original_domain_in_entry}") + + print(f" [] Testing URL: {original_full_url}") + response = try_url(original_full_url, current_headers) + + if response: + final_url_from_request = str(response.url) + print(f" [+] Redirect/Response to: {final_url_from_request}") + + parsed_final_url = urlparse(final_url_from_request) + normalized_full_url = urlunparse(parsed_final_url._replace(path='/', params='', query='', fragment='')) + if parsed_final_url.path == '' and not normalized_full_url.endswith('/'): + normalized_full_url += '/' + + if normalized_full_url != final_url_from_request: + print(f" [+] Normalized URL: {normalized_full_url}") + + if normalized_full_url != original_full_url: + new_tld_val = get_new_tld(final_url_from_request) + + if new_tld_val: + entry["full_url"] = normalized_full_url + + if new_tld_val != original_domain_in_entry: + print(f" [-] Domain TLD Changed: '{original_domain_in_entry}' -> '{new_tld_val}'") + entry["old_domain"] = original_domain_in_entry if original_domain_in_entry else entry.get("old_domain", "") + entry["domain"] = new_tld_val + entry["time_change"] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + print(f" [-] Domain & URL Updated: New TLD '{new_tld_val}', New URL '{normalized_full_url}'") + else: + entry["domain"] = new_tld_val + print(f" [-] URL Updated (TLD Unchanged '{new_tld_val}'): New URL '{normalized_full_url}'") + + updated_count += 1 + + else: + print(f" [!] Could not extract TLD from {final_url_from_request}. URL not updated despite potential change.") + else: + print(f" [] Same Domain: {final_url_from_request}") + + else: + print(f" [-] No response for {key}") + + return updated_count > 0 + +def main(): + print("Starting domain update script...") + domain_data = load_domains(JSON_FILE_PATH) + + if domain_data: + if update_domain_entries(domain_data): + save_domains(JSON_FILE_PATH, domain_data) + print("\nUpdate complete. Some entries were modified.") + else: + print("\nUpdate complete. No domains were modified.") + else: + print("\nCannot proceed without domain data.") + + print("Script finished.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.github/.domain/domains.json b/.github/.domain/domains.json new file mode 100644 index 0000000..7cd57d0 --- /dev/null +++ b/.github/.domain/domains.json @@ -0,0 +1,62 @@ +{ + "1337xx": { + "domain": "to", + "full_url": "https://www.1337xx.to/", + "old_domain": "to", + "time_change": "2025-03-19 12:20:19" + }, + "cb01new": { + "domain": "life", + "full_url": "https://cb01net.life/", + "old_domain": "download", + "time_change": "2025-06-01 01:02:16" + }, + "animeunity": { + "domain": "so", + "full_url": "https://www.animeunity.so/", + "old_domain": "so", + "time_change": "2025-03-19 12:20:23" + }, + "animeworld": { + "domain": "ac", + "full_url": "https://www.animeworld.ac/", + "old_domain": "ac", + "time_change": "2025-03-21 12:20:27" + }, + "guardaserie": { + "domain": "meme", + "full_url": "https://guardaserie.meme/", + "old_domain": "meme", + "time_change": "2025-03-19 12:20:24" + }, + "ddlstreamitaly": { + "domain": "co", + "full_url": "https://ddlstreamitaly.co/", + "old_domain": "co", + "time_change": "2025-03-19 12:20:26" + }, + "streamingwatch": { + "domain": "org", + "full_url": "https://www.streamingwatch.org/", + "old_domain": "org", + "time_change": "2025-04-29 12:30:30" + }, + "altadefinizione": { + "domain": "spa", + "full_url": "https://altadefinizione.spa/", + "old_domain": "locker", + "time_change": "2025-05-26 23:22:45" + }, + "streamingcommunity": { + "domain": "bio", + "full_url": "https://streamingunity.bio/", + "old_domain": "blog", + "time_change": "2025-05-31 12:17:33" + }, + "altadefinizionegratis": { + "domain": "icu", + "full_url": "https://altadefinizionegratis.icu/", + "old_domain": "taipei", + "time_change": "2025-05-18 11:21:05" + } +} \ No newline at end of file diff --git a/.github/.site/js/script.js b/.github/.site/js/script.js index e89eb9f..727e297 100644 --- a/.github/.site/js/script.js +++ b/.github/.site/js/script.js @@ -113,43 +113,27 @@ async function checkSiteStatus(url, siteName) { } } -const supabaseUrl = 'https://zvfngpoxwrgswnzytadh.supabase.co'; -const supabaseKey = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inp2Zm5ncG94d3Jnc3duenl0YWRoIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDAxNTIxNjMsImV4cCI6MjA1NTcyODE2M30.FNTCCMwi0QaKjOu8gtZsT5yQttUW8QiDDGXmzkn89QE'; +const domainsJsonUrl = 'https://raw.githubusercontent.com/Arrowar/StreamingCommunity/refs/heads/main/.github/.domain/domains.json'; async function loadSiteData() { try { - console.log('Starting to load site data...'); + console.log('Starting to load site data from GitHub...'); createStatusIndicator(); - updateStatusIndicator('Loading...', 'Fetching site data from database...', 0); + updateStatusIndicator('Loading...', 'Fetching site data from GitHub repository...', 0); const siteList = document.getElementById('site-list'); - const headers = { - 'accept': '*/*', - 'accept-language': 'it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7', - 'apikey': supabaseKey, - 'authorization': `Bearer ${supabaseKey}`, - 'content-type': 'application/json', - 'cache-control': 'no-cache', - 'pragma': 'no-cache', - 'range': '0-9' - }; - - console.log('Fetching from Supabase with headers:', headers); - const response = await fetch(`${supabaseUrl}/rest/v1/public?select=*`, { - method: 'GET', - headers: headers - }); + console.log(`Fetching from GitHub: ${domainsJsonUrl}`); + const response = await fetch(domainsJsonUrl); if (!response.ok) throw new Error(`HTTP error! Status: ${response.status}`); - const data = await response.json(); + const configSite = await response.json(); // Directly get the site data object siteList.innerHTML = ''; - if (data && data.length > 0) { - const configSite = data[0].data; + if (configSite && Object.keys(configSite).length > 0) { // Check if configSite is a non-empty object totalSites = Object.keys(configSite).length; completedSites = 0; let latestUpdate = new Date(0); @@ -239,7 +223,7 @@ async function loadSiteData() { document.getElementById('last-update-time').textContent = formattedDate; } else { siteList.innerHTML = '
No sites available
'; - updateStatusIndicator('Ready', 'No sites found in database', 100); + updateStatusIndicator('Ready', 'No sites found in the JSON file.', 100); } } catch (error) { console.error('Errore:', error); diff --git a/.github/workflows/update_domain.yml b/.github/workflows/update_domain.yml new file mode 100644 index 0000000..231c795 --- /dev/null +++ b/.github/workflows/update_domain.yml @@ -0,0 +1,49 @@ +name: Update domains + +on: + schedule: + - cron: "0 */2 * * *" + workflow_dispatch: + +jobs: + update-domains: + runs-on: ubuntu-latest + permissions: + contents: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + pip install httpx ua-generator requests + pip install --upgrade pip setuptools wheel + + - name: Configure DNS + run: | + sudo sh -c 'echo "nameserver 9.9.9.9" > /etc/resolv.conf' + cat /etc/resolv.conf + + - name: Execute domain update script + run: python .github/.domain/domain_update.py + + - name: Commit and push changes (if any) + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + # Check if domains.json was modified + if ! git diff --quiet .github/.domain/domains.json; then + git add .github/.domain/domains.json + git commit -m "Automatic domain update [skip ci]" + echo "Changes committed. Attempting to push..." + git push + else + echo "No changes to .github/.domain/domains.json to commit." + fi \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5cf4a3f..9322c75 100644 --- a/.gitignore +++ b/.gitignore @@ -52,5 +52,4 @@ cmd.txt bot_config.json scripts.json active_requests.json -domains.json working_proxies.json \ No newline at end of file diff --git a/StreamingCommunity/Util/config_json.py b/StreamingCommunity/Util/config_json.py index 08070cd..62f68a4 100644 --- a/StreamingCommunity/Util/config_json.py +++ b/StreamingCommunity/Util/config_json.py @@ -268,33 +268,32 @@ class ConfigManager: self._load_site_data_from_file() def _load_site_data_from_api(self) -> None: - """Load site data from API.""" + """Load site data from GitHub.""" + domains_github_url = "https://raw.githubusercontent.com/Arrowar/StreamingCommunity/refs/heads/main/.github/.domain/domains.json" headers = { - "apikey": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inp2Zm5ncG94d3Jnc3duenl0YWRoIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDAxNTIxNjMsImV4cCI6MjA1NTcyODE2M30.FNTCCMwi0QaKjOu8gtZsT5yQttUW8QiDDGXmzkn89QE", - "Authorization": f"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6Inp2Zm5ncG94d3Jnc3duenl0YWRoIiwicm9sZSI6ImFub24iLCJpYXQiOjE3NDAxNTIxNjMsImV4cCI6MjA1NTcyODE2M30.FNTCCMwi0QaKjOu8gtZsT5yQttUW8QiDDGXmzkn89QE", - "Content-Type": "application/json", - "User-Agent": get_userAgent() + "User-Agent": get_userAgent() } try: - console.print("[bold cyan]Retrieving site data from API...[/bold cyan]") - response = requests.get("https://zvfngpoxwrgswnzytadh.supabase.co/rest/v1/public", timeout=8, headers=headers) + console.print(f"[bold cyan]Retrieving site data from GitHub:[/bold cyan] [green]{domains_github_url}[/green]") + response = requests.get(domains_github_url, timeout=8, headers=headers) if response.ok: - data = response.json() - if data and len(data) > 0: - self.configSite = data[0]['data'] - - site_count = len(self.configSite) if isinstance(self.configSite, dict) else 0 - - else: - console.print("[bold yellow]API returned an empty data set[/bold yellow]") + self.configSite = response.json() + + site_count = len(self.configSite) if isinstance(self.configSite, dict) else 0 + console.print(f"[bold green]Site data loaded from GitHub:[/bold green] {site_count} streaming services found.") + else: - console.print(f"[bold red]API request failed:[/bold red] HTTP {response.status_code}, {response.text[:100]}") + console.print(f"[bold red]GitHub request failed:[/bold red] HTTP {response.status_code}, {response.text[:100]}") self._handle_site_data_fallback() + except json.JSONDecodeError as e: + console.print(f"[bold red]Error parsing JSON from GitHub:[/bold red] {str(e)}") + self._handle_site_data_fallback() + except Exception as e: - console.print(f"[bold red]API connection error:[/bold red] {str(e)}") + console.print(f"[bold red]GitHub connection error:[/bold red] {str(e)}") self._handle_site_data_fallback() def _load_site_data_from_file(self) -> None: diff --git a/StreamingCommunity/Util/os.py b/StreamingCommunity/Util/os.py index 5e35f03..2d8f7d1 100644 --- a/StreamingCommunity/Util/os.py +++ b/StreamingCommunity/Util/os.py @@ -12,7 +12,7 @@ import inspect import subprocess import contextlib import importlib.metadata - +import socket # External library from unidecode import unidecode @@ -283,37 +283,61 @@ class InternManager(): else: return f"{bytes / (1024 * 1024):.2f} MB/s" - def check_dns_provider(self): + # def check_dns_provider(self): + # """ + # Check if the system's current DNS server matches any known DNS providers. + + # Returns: + # bool: True if the current DNS server matches a known provider, + # False if no match is found or in case of errors + # """ + # dns_providers = { + # "Cloudflare": ["1.1.1.1", "1.0.0.1"], + # "Google": ["8.8.8.8", "8.8.4.4"], + # "OpenDNS": ["208.67.222.222", "208.67.220.220"], + # "Quad9": ["9.9.9.9", "149.112.112.112"], + # "AdGuard": ["94.140.14.14", "94.140.15.15"], + # "Comodo": ["8.26.56.26", "8.20.247.20"], + # "Level3": ["209.244.0.3", "209.244.0.4"], + # "Norton": ["199.85.126.10", "199.85.127.10"], + # "CleanBrowsing": ["185.228.168.9", "185.228.169.9"], + # "Yandex": ["77.88.8.8", "77.88.8.1"] + # } + + # try: + # resolver = dns.resolver.Resolver() + # nameservers = resolver.nameservers + + # if not nameservers: + # return False + + # for server in nameservers: + # for provider, ips in dns_providers.items(): + # if server in ips: + # return True + # return False + + # except Exception: + # return False + + def check_dns_resolve(self): """ - Check if the system's current DNS server matches any known DNS providers. + Check if the system's current DNS server can resolve a domain name. + Works on both Windows and Unix-like systems. Returns: - bool: True if the current DNS server matches a known provider, - False if no match is found or in case of errors + bool: True if the current DNS server can resolve a domain name, + False if can't resolve or in case of errors """ - dns_providers = { - "Cloudflare": ["1.1.1.1", "1.0.0.1"], - "Google": ["8.8.8.8", "8.8.4.4"], - "OpenDNS": ["208.67.222.222", "208.67.220.220"], - "Quad9": ["9.9.9.9", "149.112.112.112"], - } + test_domains = ["github.com", "google.com", "microsoft.com", "amazon.com"] try: - resolver = dns.resolver.Resolver() - nameservers = resolver.nameservers - - if not nameservers: - return False - - for server in nameservers: - for provider, ips in dns_providers.items(): - if server in ips: - return True + for domain in test_domains: + # socket.gethostbyname() works consistently across all platforms + socket.gethostbyname(domain) + return True + except (socket.gaierror, socket.error): return False - - except Exception: - return False - class OsSummary: def __init__(self): diff --git a/StreamingCommunity/run.py b/StreamingCommunity/run.py index e37bd74..2db8a86 100644 --- a/StreamingCommunity/run.py +++ b/StreamingCommunity/run.py @@ -210,7 +210,19 @@ def main(script_id = 0): log_not = Logger() initialize() - if not internet_manager.check_dns_provider(): + # if not internet_manager.check_dns_provider(): + # print() + # console.print("[red]❌ ERROR: DNS configuration is required!") + # console.print("[red]The program cannot function correctly without proper DNS settings.") + # console.print("[yellow]Please configure one of these DNS servers:") + # console.print("[blue]• Cloudflare (1.1.1.1) 'https://developers.cloudflare.com/1.1.1.1/setup/windows/'") + # console.print("[blue]• Quad9 (9.9.9.9) 'https://docs.quad9.net/Setup_Guides/Windows/Windows_10/'") + # console.print("\n[yellow]⚠️ The program will not work until you configure your DNS settings.") + + # time.sleep(2) + # msg.ask("[yellow]Press Enter to continue ...") + + if not internet_manager.check_dns_resolve(): print() console.print("[red]❌ ERROR: DNS configuration is required!") console.print("[red]The program cannot function correctly without proper DNS settings.") @@ -219,8 +231,7 @@ def main(script_id = 0): console.print("[blue]• Quad9 (9.9.9.9) 'https://docs.quad9.net/Setup_Guides/Windows/Windows_10/'") console.print("\n[yellow]⚠️ The program will not work until you configure your DNS settings.") - time.sleep(2) - msg.ask("[yellow]Press Enter to continue ...") + os._exit(0) # Load search functions search_functions = load_search_functions()