add route to trigger Populate

+ use instance keys to stop multiple instances of populate
+ move Populate error to a new file
+ misc

app/api/settings.py

@@ -223,3 +223,18 @@ def set_setting():
         value = ",".join(value)
 
     return {"result": value}
+
+
+@background
+def run_populate():
+    populate.Populate(instance_key=get_random_str())
+
+
+@api.route("/settings/trigger-scan", methods=["GET"])
+def trigger_scan():
+    """
+    Triggers a scan.
+    """
+    run_populate()
+
+    return {"msg": "Scan triggered!"}

app/lib/artistlib.py

@@ -1,12 +1,11 @@
 import urllib
-from concurrent.futures import ProcessPoolExecutor as Pool
+from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO
-from multiprocessing import Pool, cpu_count
 from pathlib import Path
 
 import requests
 from PIL import Image, UnidentifiedImageError
-from requests.exceptions import ConnectionError as ReqConnError
+from requests.exceptions import ConnectionError as RequestConnectionError
 from requests.exceptions import ReadTimeout
 from tqdm import tqdm
 
@@ -16,6 +15,9 @@ from app.store import artists as artist_store
 from app.utils.hashing import create_hash
 
 
+CHECK_ARTIST_IMAGES_KEY = ""
+
+
 def get_artist_image_link(artist: str):
     """
     Returns an artist image url.
@@ -36,7 +38,7 @@ def get_artist_image_link(artist: str):
                 return res["picture_big"]
 
         return None
-    except (ReqConnError, ReadTimeout, IndexError, KeyError):
+    except (RequestConnectionError, ReadTimeout, IndexError, KeyError):
         return None
 
 
@@ -73,13 +75,18 @@ class DownloadImage:
 
 
 class CheckArtistImages:
-    def __init__(self):
+    def __init__(self, instance_key: str):
-        with Pool(cpu_count()) as pool:
+        global CHECK_ARTIST_IMAGES_KEY
+        CHECK_ARTIST_IMAGES_KEY = instance_key
+
+        key_artist_map = (
+            (instance_key, artist) for artist in artist_store.ArtistStore.artists
+        )
+
+        with ThreadPoolExecutor(max_workers=4) as executor:
             res = list(
                 tqdm(
-                    pool.imap_unordered(
+                    executor.map(self.download_image, key_artist_map),
-                        self.download_image, artist_store.ArtistStore.artists
-                    ),
                     total=len(artist_store.ArtistStore.artists),
                     desc="Downloading missing artist images",
                 )
@@ -88,12 +95,17 @@ class CheckArtistImages:
             list(res)
 
     @staticmethod
-    def download_image(artist: Artist):
+    def download_image(_map: tuple[str, Artist]):
         """
         Checks if an artist image exists and downloads it if not.
 
         :param artist: The artist name
         """
+        instance_key, artist = _map
+
+        if CHECK_ARTIST_IMAGES_KEY != instance_key:
+            return
+
         img_path = (
             Path(settings.Paths.get_artist_img_sm_path()) / f"{artist.artisthash}.webp"
         )

app/lib/colorlib.py

@@ -15,6 +15,11 @@ from app.db.sqlite.utils import SQLiteManager
 
 from app.store.artists import ArtistStore
 from app.store.albums import AlbumStore
+from app.logger import log
+from app.lib.errors import PopulateCancelledError
+
+PROCESS_ALBUM_COLORS_KEY = ""
+PROCESS_ARTIST_COLORS_KEY = ""
 
 
 def get_image_colors(image: str, count=1) -> list[str]:
@@ -52,7 +57,10 @@ class ProcessAlbumColors:
     Extracts the most dominant color from the album art and saves it to the database.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, instance_key: str) -> None:
+        global PROCESS_ALBUM_COLORS_KEY
+        PROCESS_ALBUM_COLORS_KEY = instance_key
+
         albums = [
             a
             for a in AlbumStore.albums
@@ -62,6 +70,15 @@ class ProcessAlbumColors:
         with SQLiteManager() as cur:
             try:
                 for album in tqdm(albums, desc="Processing missing album colors"):
+                    if PROCESS_ALBUM_COLORS_KEY != instance_key:
+                        raise PopulateCancelledError(
+                            "A newer 'ProcessAlbumColors' instance is running. Stopping this one."
+                        )
+
+                    # TODO: Stop hitting the database for every album.
+                    # Instead, fetch all the data from the database and
+                    # check from memory.
+
                     exists = aldb.exists(album.albumhash, cur=cur)
                     if exists:
                         continue
@@ -83,14 +100,22 @@ class ProcessArtistColors:
     Extracts the most dominant color from the artist art and saves it to the database.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, instance_key: str) -> None:
         all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0]
 
+        global PROCESS_ARTIST_COLORS_KEY
+        PROCESS_ARTIST_COLORS_KEY = instance_key
+
         with SQLiteManager() as cur:
             try:
                 for artist in tqdm(
                     all_artists, desc="Processing missing artist colors"
                 ):
+                    if PROCESS_ARTIST_COLORS_KEY != instance_key:
+                        raise PopulateCancelledError(
+                            "A newer 'ProcessArtistColors' instance is running. Stopping this one."
+                        )
+
                     exists = adb.exists(artist.artisthash, cur=cur)
 
                     if exists:

app/lib/errors.py

@@ -0,0 +1,7 @@
+class PopulateCancelledError(Exception):
+    """
+    Raised when the instance key of a looping function called
+    inside Populate is changed.
+    """
+
+    pass

app/lib/populate.py

@@ -1,5 +1,6 @@
 import os
 from collections import deque
+from concurrent.futures import ThreadPoolExecutor
 from typing import Generator
 
 from requests import ConnectionError as RequestConnectionError
@@ -14,6 +15,7 @@ from app.db.sqlite.tracks import SQLiteTrackMethods
 from app.lib.albumslib import validate_albums
 from app.lib.artistlib import CheckArtistImages
 from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors
+from app.lib.errors import PopulateCancelledError
 from app.lib.taglib import extract_thumb, get_tags
 from app.lib.trackslib import validate_tracks
 from app.logger import log
@@ -33,10 +35,6 @@ remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths
 POPULATE_KEY = ""
 
 
-class PopulateCancelledError(Exception):
-    pass
-
-
 class Populate:
     """
     Populates the database with all songs in the music directory
@@ -84,31 +82,39 @@ class Populate:
 
         self.extract_thumb_with_overwrite(modified_tracks)
 
-        ProcessTrackThumbnails()
+        try:
-        ProcessAlbumColors()
+            ProcessTrackThumbnails(instance_key)
-        ProcessArtistColors()
+            ProcessAlbumColors(instance_key)
+            ProcessArtistColors(instance_key)
+        except PopulateCancelledError as e:
+            log.warn(e)
+            return
 
         tried_to_download_new_images = False
 
         if Ping()():
             tried_to_download_new_images = True
             try:
-                CheckArtistImages()
+                CheckArtistImages(instance_key)
             except (RequestConnectionError, ReadTimeout) as e:
                 log.error(
                     "Internet connection lost. Downloading artist images stopped."
                 )
         else:
             log.warning(
-                f"No internet connection. Downloading artist images halted for {settings.get_scan_sleep_time()} seconds."
+                f"No internet connection. Downloading artist images stopped!"
             )
 
         # Re-process the new artist images.
         if tried_to_download_new_images:
-            ProcessArtistColors()
+            ProcessArtistColors(instance_key=instance_key)
 
         if Ping()():
-            FetchSimilarArtistsLastFM()
+            try:
+                FetchSimilarArtistsLastFM(instance_key)
+            except PopulateCancelledError as e:
+                log.warn(e)
+                return
 
         ArtistStore.load_artists(instance_key)
 
@@ -151,7 +157,8 @@ class Populate:
 
         for file in tqdm(untagged, desc="Reading files"):
             if POPULATE_KEY != key:
-                raise PopulateCancelledError("Populate key changed")
+                log.warning("'Populate.tag_untagged': Populate key changed")
+                return
 
             tags = get_tags(file)
 
@@ -197,7 +204,7 @@ class Populate:
                 continue
 
 
-def get_image(album: Album):
+def get_image(_map: tuple[str, Album]):
     """
     The function retrieves an image from an album by iterating through its tracks and extracting the thumbnail from the first track that has one.
 
@@ -206,6 +213,11 @@ def get_image(album: Album):
     :return: None
     """
 
+    instance_key, album = _map
+
+    if POPULATE_KEY != instance_key:
+        raise PopulateCancelledError("'ProcessTrackThumbnails': Populate key changed")
+
     matching_tracks = filter(
         lambda t: t.albumhash == album.albumhash, TrackStore.tracks
     )
@@ -226,7 +238,7 @@ def get_image(album: Album):
         pass
 
 
-from multiprocessing import Pool, cpu_count
+CPU_COUNT = os.cpu_count() // 2
 
 
 class ProcessTrackThumbnails:
@@ -234,11 +246,13 @@ class ProcessTrackThumbnails:
     Extracts the album art from all albums in album store.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, instance_key: str) -> None:
-        with Pool(processes=cpu_count()) as pool:
+        key_album_map = ((instance_key, album) for album in AlbumStore.albums)
+
+        with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
             results = list(
                 tqdm(
-                    pool.imap_unordered(get_image, AlbumStore.albums),
+                    executor.map(get_image, key_album_map),
                     total=len(AlbumStore.albums),
                     desc="Extracting track images",
                 )
@@ -247,11 +261,18 @@ class ProcessTrackThumbnails:
             list(results)
 
 
-def save_similar_artists(artist: Artist):
+def save_similar_artists(_map: tuple[str, Artist]):
     """
     Downloads and saves similar artists to the database.
     """
 
+    instance_key, artist = _map
+
+    if POPULATE_KEY != instance_key:
+        raise PopulateCancelledError(
+            "'FetchSimilarArtistsLastFM': Populate key changed"
+        )
+
     if lastfmdb.exists(artist.artisthash):
         return
 
@@ -266,17 +287,18 @@ def save_similar_artists(artist: Artist):
 
 class FetchSimilarArtistsLastFM:
     """
-    Fetches similar artists from LastFM using a process pool.
+    Fetches similar artists from LastFM using a thread pool.
     """
 
-    def __init__(self) -> None:
+    def __init__(self, instance_key: str) -> None:
         artists = ArtistStore.artists
+        key_artist_map = ((instance_key, artist) for artist in artists)
 
-        with Pool(processes=cpu_count()) as pool:
+        with ThreadPoolExecutor(max_workers=CPU_COUNT) as executor:
             try:
                 results = list(
                     tqdm(
-                        pool.imap_unordered(save_similar_artists, artists),
+                        executor.map(save_similar_artists, key_artist_map),
                         total=len(artists),
                         desc="Fetching similar artists",
                     )
@@ -284,6 +306,10 @@ class FetchSimilarArtistsLastFM:
 
                 list(results)
 
+            except PopulateCancelledError as e:
+                raise e
+
             # any exception that can be raised by the pool
-            except:
+            except Exception as e:
+                log.warn(e)
                 pass

app/periodic_scan.py

@@ -7,7 +7,7 @@ from app.lib.populate import Populate, PopulateCancelledError
 from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time
 from app.utils.generators import get_random_str
 from app.utils.threading import background
-
+from app.logger import log
 
 @background
 def run_periodic_scans():
@@ -25,6 +25,7 @@ def run_periodic_scans():
         try:
             Populate(instance_key=get_random_str())
         except PopulateCancelledError:
+            log.error("'run_periodic_scans': Periodic scan cancelled.")
             pass
 
         sleep_time = get_scan_sleep_time()

app/settings.py

@@ -168,7 +168,7 @@ class SessionVars:
     CLEAN_ALBUM_TITLE = True
     REMOVE_REMASTER_FROM_TRACK = True
 
-    DO_PERIODIC_SCANS = True
+    DO_PERIODIC_SCANS = False
     PERIODIC_SCAN_INTERVAL = 600  # 10 minutes
     """
     The interval between periodic scans in seconds.