diff --git a/app/api/settings.py b/app/api/settings.py index e0a4fd7..9051a0a 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -223,3 +223,18 @@ def set_setting(): value = ",".join(value) return {"result": value} + + +@background +def run_populate(): + populate.Populate(instance_key=get_random_str()) + + +@api.route("/settings/trigger-scan", methods=["GET"]) +def trigger_scan(): + """ + Triggers a scan. + """ + run_populate() + + return {"msg": "Scan triggered!"} diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py index e0884be..2bb7cb2 100644 --- a/app/lib/artistlib.py +++ b/app/lib/artistlib.py @@ -1,12 +1,11 @@ import urllib -from concurrent.futures import ProcessPoolExecutor as Pool +from concurrent.futures import ThreadPoolExecutor from io import BytesIO -from multiprocessing import Pool, cpu_count from pathlib import Path import requests from PIL import Image, UnidentifiedImageError -from requests.exceptions import ConnectionError as ReqConnError +from requests.exceptions import ConnectionError as RequestConnectionError from requests.exceptions import ReadTimeout from tqdm import tqdm @@ -16,6 +15,9 @@ from app.store import artists as artist_store from app.utils.hashing import create_hash +CHECK_ARTIST_IMAGES_KEY = "" + + def get_artist_image_link(artist: str): """ Returns an artist image url. @@ -36,7 +38,7 @@ def get_artist_image_link(artist: str): return res["picture_big"] return None - except (ReqConnError, ReadTimeout, IndexError, KeyError): + except (RequestConnectionError, ReadTimeout, IndexError, KeyError): return None @@ -73,13 +75,18 @@ class DownloadImage: class CheckArtistImages: - def __init__(self): - with Pool(cpu_count()) as pool: + def __init__(self, instance_key: str): + global CHECK_ARTIST_IMAGES_KEY + CHECK_ARTIST_IMAGES_KEY = instance_key + + key_artist_map = ( + (instance_key, artist) for artist in artist_store.ArtistStore.artists + ) + + with ThreadPoolExecutor(max_workers=4) as executor: res = list( tqdm( - pool.imap_unordered( - self.download_image, artist_store.ArtistStore.artists - ), + executor.map(self.download_image, key_artist_map), total=len(artist_store.ArtistStore.artists), desc="Downloading missing artist images", ) @@ -88,12 +95,17 @@ class CheckArtistImages: list(res) @staticmethod - def download_image(artist: Artist): + def download_image(_map: tuple[str, Artist]): """ Checks if an artist image exists and downloads it if not. :param artist: The artist name """ + instance_key, artist = _map + + if CHECK_ARTIST_IMAGES_KEY != instance_key: + return + img_path = ( Path(settings.Paths.get_artist_img_sm_path()) / f"{artist.artisthash}.webp" ) diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py index c6e87fd..4039080 100644 --- a/app/lib/colorlib.py +++ b/app/lib/colorlib.py @@ -15,6 +15,11 @@ from app.db.sqlite.utils import SQLiteManager from app.store.artists import ArtistStore from app.store.albums import AlbumStore +from app.logger import log +from app.lib.errors import PopulateCancelledError + +PROCESS_ALBUM_COLORS_KEY = "" +PROCESS_ARTIST_COLORS_KEY = "" def get_image_colors(image: str, count=1) -> list[str]: @@ -52,7 +57,10 @@ class ProcessAlbumColors: Extracts the most dominant color from the album art and saves it to the database. """ - def __init__(self) -> None: + def __init__(self, instance_key: str) -> None: + global PROCESS_ALBUM_COLORS_KEY + PROCESS_ALBUM_COLORS_KEY = instance_key + albums = [ a for a in AlbumStore.albums @@ -62,6 +70,15 @@ class ProcessAlbumColors: with SQLiteManager() as cur: try: for album in tqdm(albums, desc="Processing missing album colors"): + if PROCESS_ALBUM_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessAlbumColors' instance is running. Stopping this one." + ) + + # TODO: Stop hitting the database for every album. + # Instead, fetch all the data from the database and + # check from memory. + exists = aldb.exists(album.albumhash, cur=cur) if exists: continue @@ -83,14 +100,22 @@ class ProcessArtistColors: Extracts the most dominant color from the artist art and saves it to the database. """ - def __init__(self) -> None: + def __init__(self, instance_key: str) -> None: all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0] + global PROCESS_ARTIST_COLORS_KEY + PROCESS_ARTIST_COLORS_KEY = instance_key + with SQLiteManager() as cur: try: for artist in tqdm( all_artists, desc="Processing missing artist colors" ): + if PROCESS_ARTIST_COLORS_KEY != instance_key: + raise PopulateCancelledError( + "A newer 'ProcessArtistColors' instance is running. Stopping this one." + ) + exists = adb.exists(artist.artisthash, cur=cur) if exists: diff --git a/app/lib/errors.py b/app/lib/errors.py new file mode 100644 index 0000000..fcbb4a4 --- /dev/null +++ b/app/lib/errors.py @@ -0,0 +1,7 @@ +class PopulateCancelledError(Exception): + """ + Raised when the instance key of a looping function called + inside Populate is changed. + """ + + pass diff --git a/app/lib/populate.py b/app/lib/populate.py index 7504df7..097264a 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,5 +1,6 @@ import os from collections import deque +from concurrent.futures import ThreadPoolExecutor from typing import Generator from requests import ConnectionError as RequestConnectionError @@ -14,6 +15,7 @@ from app.db.sqlite.tracks import SQLiteTrackMethods from app.lib.albumslib import validate_albums from app.lib.artistlib import CheckArtistImages from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors +from app.lib.errors import PopulateCancelledError from app.lib.taglib import extract_thumb, get_tags from app.lib.trackslib import validate_tracks from app.logger import log @@ -33,10 +35,6 @@ remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths POPULATE_KEY = "" -class PopulateCancelledError(Exception): - pass - - class Populate: """ Populates the database with all songs in the music directory @@ -84,31 +82,39 @@ class Populate: self.extract_thumb_with_overwrite(modified_tracks) - ProcessTrackThumbnails() - ProcessAlbumColors() - ProcessArtistColors() + try: + ProcessTrackThumbnails(instance_key) + ProcessAlbumColors(instance_key) + ProcessArtistColors(instance_key) + except PopulateCancelledError as e: + log.warn(e) + return tried_to_download_new_images = False if Ping()(): tried_to_download_new_images = True try: - CheckArtistImages() + CheckArtistImages(instance_key) except (RequestConnectionError, ReadTimeout) as e: log.error( "Internet connection lost. Downloading artist images stopped." ) else: log.warning( - f"No internet connection. Downloading artist images halted for {settings.get_scan_sleep_time()} seconds." + f"No internet connection. Downloading artist images stopped!" ) # Re-process the new artist images. if tried_to_download_new_images: - ProcessArtistColors() + ProcessArtistColors(instance_key=instance_key) if Ping()(): - FetchSimilarArtistsLastFM() + try: + FetchSimilarArtistsLastFM(instance_key) + except PopulateCancelledError as e: + log.warn(e) + return ArtistStore.load_artists(instance_key) @@ -151,7 +157,8 @@ class Populate: for file in tqdm(untagged, desc="Reading files"): if POPULATE_KEY != key: - raise PopulateCancelledError("Populate key changed") + log.warning("'Populate.tag_untagged': Populate key changed") + return tags = get_tags(file) @@ -197,7 +204,7 @@ class Populate: continue -def get_image(album: Album): +def get_image(_map: tuple[str, Album]): """ The function retrieves an image from an album by iterating through its tracks and extracting the thumbnail from the first track that has one. @@ -206,6 +213,11 @@ def get_image(album: Album): :return: None """ + instance_key, album = _map + + if POPULATE_KEY != instance_key: + raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed") + matching_tracks = filter( lambda t: t.albumhash == album.albumhash, TrackStore.tracks ) @@ -226,7 +238,7 @@ def get_image(album: Album): pass -from multiprocessing import Pool, cpu_count +CPU_COUNT = os.cpu_count() // 2 class ProcessTrackThumbnails: @@ -234,11 +246,13 @@ class ProcessTrackThumbnails: Extracts the album art from all albums in album store. """ - def __init__(self) -> None: - with Pool(processes=cpu_count()) as pool: + def __init__(self, instance_key: str) -> None: + key_album_map = ((instance_key, album) for album in AlbumStore.albums) + + with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: results = list( tqdm( - pool.imap_unordered(get_image, AlbumStore.albums), + executor.map(get_image, key_album_map), total=len(AlbumStore.albums), desc="Extracting track images", ) @@ -247,11 +261,18 @@ class ProcessTrackThumbnails: list(results) -def save_similar_artists(artist: Artist): +def save_similar_artists(_map: tuple[str, Artist]): """ Downloads and saves similar artists to the database. """ + instance_key, artist = _map + + if POPULATE_KEY != instance_key: + raise PopulateCancelledError( + "'FetchSimilarArtistsLastFM': Populate key changed" + ) + if lastfmdb.exists(artist.artisthash): return @@ -266,17 +287,18 @@ def save_similar_artists(artist: Artist): class FetchSimilarArtistsLastFM: """ - Fetches similar artists from LastFM using a process pool. + Fetches similar artists from LastFM using a thread pool. """ - def __init__(self) -> None: + def __init__(self, instance_key: str) -> None: artists = ArtistStore.artists + key_artist_map = ((instance_key, artist) for artist in artists) - with Pool(processes=cpu_count()) as pool: + with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor: try: results = list( tqdm( - pool.imap_unordered(save_similar_artists, artists), + executor.map(save_similar_artists, key_artist_map), total=len(artists), desc="Fetching similar artists", ) @@ -284,6 +306,10 @@ class FetchSimilarArtistsLastFM: list(results) + except PopulateCancelledError as e: + raise e + # any exception that can be raised by the pool - except: + except Exception as e: + log.warn(e) pass diff --git a/app/periodic_scan.py b/app/periodic_scan.py index 787c3a7..bf7d8c7 100644 --- a/app/periodic_scan.py +++ b/app/periodic_scan.py @@ -7,7 +7,7 @@ from app.lib.populate import Populate, PopulateCancelledError from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time from app.utils.generators import get_random_str from app.utils.threading import background - +from app.logger import log @background def run_periodic_scans(): @@ -25,6 +25,7 @@ def run_periodic_scans(): try: Populate(instance_key=get_random_str()) except PopulateCancelledError: + log.error("'run_periodic_scans': Periodic scan cancelled.") pass sleep_time = get_scan_sleep_time() diff --git a/app/settings.py b/app/settings.py index d4d601c..40c2ccc 100644 --- a/app/settings.py +++ b/app/settings.py @@ -168,7 +168,7 @@ class SessionVars: CLEAN_ALBUM_TITLE = True REMOVE_REMASTER_FROM_TRACK = True - DO_PERIODIC_SCANS = True + DO_PERIODIC_SCANS = False PERIODIC_SCAN_INTERVAL = 600 # 10 minutes """ The interval between periodic scans in seconds.