fix duplicate artist and album color entry in db

+ Remove folder store
+ Reduce fuzzy search score cutoff from 90% to 75%
+ use inheritance to init Artist class
+ misc

app/api/colors.py

@@ -8,6 +8,11 @@ api = Blueprint("colors", __name__, url_prefix="/colors")
 def get_album_color(albumhash: str):
     album = Store.get_album_by_hash(albumhash)
 
+    if len(album.colors) > 0:
+        return {
+            "color": album.colors[0]
+        }
+
     return {
-        "color": album.colors[0]
+        "color": ""
     }

app/api/folder.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from flask import Blueprint, request
 
 from app import settings
-from app.lib.folderslib import GetFilesAndDirs, create_folder
+from app.lib.folderslib import GetFilesAndDirs, get_folders
 from app.db.sqlite.settings import SettingsSQLMethods as db
 from app.utils.wintools import win_replace_slash, is_windows
 
@@ -30,6 +30,7 @@ def get_folder_tree():
             req_dir = "$home"
 
     root_dirs = db.get_root_dirs()
+    root_dirs.sort()
 
     try:
         if req_dir == "$home" and root_dirs[0] == "$home":
@@ -38,19 +39,17 @@ def get_folder_tree():
         pass
 
     if req_dir == "$home":
-        folders = [Path(f) for f in root_dirs]
+        folders = get_folders(root_dirs)
 
         return {
-            "folders": [
+            "folders": folders,
-                create_folder(str(f)) for f in folders
-            ],
             "tracks": [],
         }
 
     if is_windows():
         # Trailing slash needed when drive letters are passed,
         # Remember, the trailing slash is removed in the client.
-        req_dir = req_dir + "/"
+        req_dir += "/"
     else:
         req_dir = "/" + req_dir + "/" if not req_dir.startswith("/") else req_dir + "/"
 
@@ -66,7 +65,7 @@ def get_all_drives(is_win: bool = False):
     """
     Returns a list of all the drives on a Windows machine.
     """
-    drives = psutil.disk_partitions(all=False)
+    drives = psutil.disk_partitions()
     drives = [d.mountpoint for d in drives]
 
     if is_win:
@@ -99,7 +98,7 @@ def list_folders():
         }
 
     if is_win:
-        req_dir = req_dir + "/"
+        req_dir += "/"
     else:
         req_dir = "/" + req_dir + "/"
         req_dir = str(Path(req_dir).resolve())

app/api/playlist.py

@@ -34,9 +34,9 @@ delete_playlist = PL.delete_playlist
 # get_tracks_by_trackhashes = SQLiteTrackMethods.get_tracks_by_trackhashes
 def duplicate_images(images: list):
     if len(images) == 1:
-        images = images * 4
+        images *= 4
     elif len(images) == 2:
-        images = images + list(reversed(images))
+        images += list(reversed(images))
     elif len(images) == 3:
         images = images + images[:1]
 

app/api/settings.py

@@ -8,7 +8,6 @@ from app.db.sqlite.settings import SettingsSQLMethods as sdb
 from app.utils.generators import get_random_str
 from app.utils.threading import background
 
-from app.store.folder import FolderStore
 from app.store.albums import AlbumStore
 from app.store.tracks import TrackStore
 from app.store.artists import ArtistStore
@@ -27,7 +26,6 @@ def reload_everything():
     Reloads all stores using the current database items
     """
     TrackStore.load_all_tracks()
-    FolderStore.process_folders()
     AlbumStore.load_albums()
     ArtistStore.load_artists()
 

app/db/sqlite/albums.py

@@ -1,6 +1,6 @@
 from sqlite3 import Cursor
 
-from .utils import SQLiteManager, tuple_to_album, tuples_to_albums
+from .utils import SQLiteManager, tuples_to_albums
 
 
 class SQLiteAlbumMethods:
@@ -10,30 +10,15 @@ class SQLiteAlbumMethods:
         Inserts one album into the database
         """
 
-        sql = """INSERT INTO albums(
+        sql = """INSERT OR REPLACE INTO albums(
             albumhash,
             colors
             ) VALUES(?,?)
             """
 
         cur.execute(sql, (albumhash, colors))
-
         return cur.lastrowid
 
-    # @classmethod
-    # def insert_many_albums(cls, albums: list[dict]):
-    #     """
-    #     Takes a generator of albums, and inserts them into the database
-
-    #     Parameters
-    #     ----------
-    #     albums : Generator
-    #         Generator
-    #     """
-    #     with SQLiteManager() as cur:
-    #         for album in albums:
-    #             cls.insert_one_album(cur, album["albumhash"], album["colors"])
-
     @classmethod
     def get_all_albums(cls):
         with SQLiteManager() as cur:
@@ -45,58 +30,6 @@ class SQLiteAlbumMethods:
 
         return []
 
-    # @staticmethod
-    # def get_album_by_id(album_id: int):
-    #     conn = get_sqlite_conn()
-    #     cur = conn.cursor()
-
-    #     cur.execute("SELECT * FROM albums WHERE id=?", (album_id,))
-    #     album = cur.fetchone()
-
-    #     conn.close()
-
-    #     if album is None:
-    #         return None
-
-    #     return tuple_to_album(album)
-
-    @staticmethod
-    def get_album_by_hash(album_hash: str):
-        with SQLiteManager() as cur:
-            cur.execute("SELECT * FROM albums WHERE albumhash=?", (album_hash,))
-            album = cur.fetchone()
-
-            if album is not None:
-                return tuple_to_album(album)
-
-        return None
-
-    @classmethod
-    def get_albums_by_hashes(cls, album_hashes: list):
-        """
-        Gets all the albums with the specified hashes. Returns a generator of albums or an empty list.
-        """
-        with SQLiteManager() as cur:
-            hashes = ",".join("?" * len(album_hashes))
-            cur.execute(
-                f"SELECT * FROM albums WHERE albumhash IN ({hashes})", album_hashes
-            )
-            albums = cur.fetchall()
-
-            if albums is not None:
-                return tuples_to_albums(albums)
-
-        return []
-
-    # @staticmethod
-    # def update_album_colors(album_hash: str, colors: list[str]):
-    #     sql = "UPDATE albums SET colors=? WHERE albumhash=?"
-
-    #     colors_str = json.dumps(colors)
-
-    #     with SQLiteManager() as cur:
-    #         cur.execute(sql, (colors_str, album_hash))
-
     @staticmethod
     def get_albums_by_albumartist(albumartist: str):
         with SQLiteManager() as cur:
@@ -107,17 +40,3 @@ class SQLiteAlbumMethods:
                 return tuples_to_albums(albums)
 
         return []
-
-    @staticmethod
-    def get_all_albums_raw():
-        """
-        Returns all the albums in the database, as a list of tuples.
-        """
-        with SQLiteManager() as cur:
-            cur.execute("SELECT * FROM albums")
-            albums = cur.fetchall()
-
-            if albums is not None:
-                return albums
-
-        return []

app/db/sqlite/artists.py

@@ -3,27 +3,27 @@ Contains methods for reading and writing to the sqlite artists database.
 """
 
 import json
+from sqlite3 import Cursor
+
 from .utils import SQLiteManager
 
 
 class SQLiteArtistMethods:
-    @classmethod
+    @staticmethod
-    def insert_one_artist(cls, artisthash: str, colors: str | list[str]):
+    def insert_one_artist(cur: Cursor, artisthash: str, colors: str | list[str]):
         """
         Inserts a single artist into the database.
         """
-        sql = """INSERT INTO artists(
+        sql = """INSERT OR REPLACE INTO artists(
             artisthash,
             colors
             ) VALUES(?,?)
             """
         colors = json.dumps(colors)
+        cur.execute(sql, (artisthash, colors))
 
-        with SQLiteManager() as cur:
+    @staticmethod
-            cur.execute(sql, (artisthash, colors))
+    def get_all_artists():
-
-    @classmethod
-    def get_all_artists(cls):
         """
         Get all artists from the database and return a generator of Artist objects
         """

app/db/sqlite/queries.py

@@ -45,13 +45,15 @@ CREATE TABLE IF NOT EXISTS tracks (
     genre text,
     title text NOT NULL,
     track integer NOT NULL,
-    trackhash text NOT NULL
+    trackhash text NOT NULL,
+    UNIQUE (filepath)
 );
 
 CREATE TABLE IF NOT EXISTS albums (
     id integer PRIMARY KEY,
     albumhash text NOT NULL,
-    colors text NOT NULL
+    colors text NOT NULL,
+    UNIQUE (albumhash)
 );
 
 
@@ -60,7 +62,8 @@ CREATE TABLE IF NOT EXISTS artists (
     id integer PRIMARY KEY,
     artisthash text NOT NULL,
     colors text,
-    bio text
+    bio text,
+    UNIQUE (artisthash)
 );
 
 CREATE TABLE IF NOT EXISTS folders (

app/db/sqlite/settings.py

@@ -52,6 +52,7 @@ class SettingsSQLMethods:
             for _dir in dirs:
                 cur.execute(sql, (_dir,))
 
+    # Not currently used anywhere, to be used later
     @staticmethod
     def add_excluded_dirs(dirs: list[str]):
         """

app/db/sqlite/tracks.py

@@ -82,26 +82,6 @@ class SQLiteTrackMethods:
 
             return None
 
-    @staticmethod
-    def get_tracks_by_trackhashes(hashes: list[str]):
-        """
-        Gets all tracks in a list of trackhashes.
-        Returns a generator of Track objects or an empty list.
-        """
-
-        sql = "SELECT * FROM tracks WHERE trackhash IN ({})".format(
-            ",".join("?" * len(hashes))
-        )
-
-        with SQLiteManager() as cur:
-            cur.execute(sql, hashes)
-            rows = cur.fetchall()
-
-            if rows is not None:
-                return tuples_to_tracks(rows)
-
-            return []
-
     @staticmethod
     def remove_track_by_filepath(filepath: str):
         """

app/lib/artistlib.py

@@ -1,7 +1,8 @@
 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 from io import BytesIO
-from PIL import Image
+
+from PIL import Image, UnidentifiedImageError
 import requests
 import urllib
 
@@ -55,7 +56,10 @@ class DownloadImage:
         """
         Downloads the image from the url.
         """
-        return Image.open(BytesIO(requests.get(url, timeout=10).content))
+        try:
+            return Image.open(BytesIO(requests.get(url, timeout=10).content))
+        except UnidentifiedImageError:
+            return None
 
     @staticmethod
     def save_img(img: Image.Image, sm_path: Path, lg_path: Path):

app/lib/colorlib.py

@@ -12,16 +12,15 @@ from app import settings
 from app.db.sqlite.albums import SQLiteAlbumMethods as db
 from app.db.sqlite.artists import SQLiteArtistMethods as adb
 from app.db.sqlite.utils import SQLiteManager
-from app.models import Album, Artist
 
 from app.store.artists import ArtistStore
 from app.store.albums import AlbumStore
 
 
-def get_image_colors(image: str) -> list[str]:
+def get_image_colors(image: str, count=1) -> list[str]:
-    """Extracts 2 of the most dominant colors from an image."""
+    """Extracts n number of the most dominant colors from an image."""
     try:
-        colors = sorted(colorgram.extract(image, 1), key=lambda c: c.hsl.h)
+        colors = sorted(colorgram.extract(image, count), key=lambda c: c.hsl.h)
     except OSError:
         return []
 
@@ -34,6 +33,16 @@ def get_image_colors(image: str) -> list[str]:
     return formatted_colors
 
 
+def process_color(item_hash: str, is_album=True):
+    path = settings.Paths.SM_THUMB_PATH if is_album else settings.Paths.ARTIST_IMG_SM_PATH
+    path = Path(path) / (item_hash + ".webp")
+
+    if not path.exists():
+        return
+
+    return get_image_colors(str(path))
+
+
 class ProcessAlbumColors:
     """
     Extracts the most dominant color from the album art and saves it to the database.
@@ -44,26 +53,22 @@ class ProcessAlbumColors:
 
         with SQLiteManager() as cur:
             for album in tqdm(albums, desc="Processing missing album colors"):
-                colors = self.process_color(album)
+                sql = "SELECT COUNT(1) FROM albums WHERE albumhash = ?"
+                cur.execute(sql, (album.albumhash,))
+                count = cur.fetchone()[0]
+
+                if count != 0:
+                    continue
+
+                colors = process_color(album.albumhash)
 
                 if colors is None:
                     continue
 
                 album.set_colors(colors)
-
                 color_str = json.dumps(colors)
                 db.insert_one_album(cur, album.albumhash, color_str)
 
-    @staticmethod
-    def process_color(album: Album):
-        path = Path(settings.Paths.SM_THUMB_PATH) / album.image
-
-        if not path.exists():
-            return
-
-        colors = get_image_colors(str(path))
-        return colors
-
 
 class ProcessArtistColors:
     """
@@ -73,27 +78,20 @@ class ProcessArtistColors:
     def __init__(self) -> None:
         all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0]
 
-        for artist in tqdm(all_artists, desc="Processing missing artist colors"):
+        with SQLiteManager() as cur:
-            self.process_color(artist)
+            for artist in tqdm(all_artists, desc="Processing missing artist colors"):
+                sql = "SELECT COUNT(1) FROM artists WHERE artisthash = ?"
 
-    @staticmethod
+                cur.execute(sql, (artist.artisthash,))
-    def process_color(artist: Artist):
+                count = cur.fetchone()[0]
-        path = Path(settings.Paths.ARTIST_IMG_SM_PATH) / artist.image
 
-        if not path.exists():
+                if count != 0:
-            return
+                    continue
 
-        colors = get_image_colors(str(path))
+                colors = process_color(artist.artisthash, is_album=False)
 
-        if len(colors) > 0:
+                if colors is None:
-            adb.insert_one_artist(artisthash=artist.artisthash, colors=colors)
+                    continue
-            ArtistStore.map_artist_color((0, artist.artisthash, json.dumps(colors)))
 
-# TODO: If item color is in db, get it, assign it to the item and continue.
+                artist.set_colors(colors)
-#   - Format all colors in the format: rgb(123, 123, 123)
+                adb.insert_one_artist(cur, artist.artisthash, colors)
-#   - Each digit should be 3 digits long.
-#   - Format all db colors into a master string of the format "-itemhash:colorhash-"
-#   - Find the item hash using index() and get the color using the index + number, where number
-#       is the length of the rgb string + 1
-#   - Assign the color to the item and continue.
-#   - If the color is not in the db, extract it and add it to the db.

app/lib/populate.py

@@ -12,7 +12,6 @@ from app.logger import log
 from app.models import Album, Artist, Track
 from app.utils.filesystem import run_fast_scandir
 
-from app.store.folder import FolderStore
 from app.store.albums import AlbumStore
 from app.store.tracks import TrackStore
 from app.store.artists import ArtistStore
@@ -102,7 +101,6 @@ class Populate:
                 track.is_favorite = track.trackhash in fav_tracks
 
                 TrackStore.add_track(track)
-                FolderStore.add_folder(track.folder)
 
                 if not AlbumStore.album_exists(track.albumhash):
                     AlbumStore.add_album(AlbumStore.create_album(track))

app/lib/searchlib.py

@@ -23,10 +23,10 @@ class Cutoff:
     Holds all the default cutoff values.
     """
 
-    tracks: int = 90
+    tracks: int = 75
-    albums: int = 90
+    albums: int = 75
-    artists: int = 90
+    artists: int = 75
-    playlists: int = 90
+    playlists: int = 75
 
 
 class Limit:
@@ -54,7 +54,6 @@ class SearchTracks:
         results = process.extract(
             self.query,
             track_titles,
-            scorer=fuzz.WRatio,
             score_cutoff=Cutoff.tracks,
             limit=Limit.tracks,
         )
@@ -77,7 +76,6 @@ class SearchArtists:
         results = process.extract(
             self.query,
             artists,
-            scorer=fuzz.WRatio,
             score_cutoff=Cutoff.artists,
             limit=Limit.artists,
         )
@@ -100,7 +98,6 @@ class SearchAlbums:
         results = process.extract(
             self.query,
             albums,
-            scorer=fuzz.WRatio,
             score_cutoff=Cutoff.albums,
             limit=Limit.albums,
         )
@@ -125,7 +122,6 @@ class SearchPlaylists:
         results = process.extract(
             self.query,
             playlists,
-            scorer=fuzz.WRatio,
             score_cutoff=Cutoff.playlists,
             limit=Limit.playlists,
         )
@@ -176,7 +172,6 @@ class SearchAll:
         results = process.extract(
             query=query,
             choices=items,
-            scorer=fuzz.WRatio,
             score_cutoff=Cutoff.tracks,
             limit=20
         )

app/lib/watchdogg.py

@@ -17,7 +17,6 @@ from app.db.sqlite.tracks import SQLiteManager
 from app.db.sqlite.tracks import SQLiteTrackMethods as db
 from app.db.sqlite.settings import SettingsSQLMethods as sdb
 
-from app.store.folder import FolderStore
 from app.store.tracks import TrackStore
 from app.store.albums import AlbumStore
 from app.store.artists import ArtistStore
@@ -144,8 +143,6 @@ def add_track(filepath: str) -> None:
     track = Track(**tags)
     TrackStore.add_track(track)
 
-    FolderStore.add_folder(track.folder)
-
     if not AlbumStore.album_exists(track.albumhash):
         album = AlbumStore.create_album(track)
         AlbumStore.add_album(album)
@@ -182,11 +179,6 @@ def remove_track(filepath: str) -> None:
         if empty_artist:
             ArtistStore.remove_artist_by_hash(artist.artisthash)
 
-    empty_folder = FolderStore.is_empty_folder(track.folder)
-
-    if empty_folder:
-        FolderStore.remove_folder(track.folder)
-
 
 class Handler(PatternMatchingEventHandler):
     files_to_process = []
@@ -204,7 +196,6 @@ class Handler(PatternMatchingEventHandler):
             self,
             patterns=patterns,
             ignore_directories=True,
-            case_sensitive=False,
         )
 
     def get_abs_path(self, path: str):

app/models/artist.py

@@ -5,27 +5,6 @@ from dataclasses import dataclass
 from app.utils.hashing import create_hash
 
 
-@dataclass(slots=True)
-class Artist:
-    """
-    Artist class
-    """
-
-    name: str
-    artisthash: str = ""
-    image: str = ""
-    trackcount: int = 0
-    albumcount: int = 0
-    duration: int = 0
-    colors: list[str] = dataclasses.field(default_factory=list)
-    is_favorite: bool = False
-
-    def __post_init__(self):
-        self.artisthash = create_hash(self.name, decode=True)
-        self.image = self.artisthash + ".webp"
-        self.colors = json.loads(str(self.colors))
-
-
 @dataclass(slots=True)
 class ArtistMinimal:
     """
@@ -36,6 +15,29 @@ class ArtistMinimal:
     artisthash: str = ""
     image: str = ""
 
-    def __post_init__(self):
+    def __init__(self, name: str):
+        self.name = name
         self.artisthash = create_hash(self.name, decode=True)
         self.image = self.artisthash + ".webp"
+
+
+@dataclass(slots=True)
+class Artist(ArtistMinimal):
+    """
+    Artist class
+    """
+
+    trackcount: int = 0
+    albumcount: int = 0
+    duration: int = 0
+    colors: list[str] = dataclasses.field(default_factory=list)
+    is_favorite: bool = False
+
+    def __post_init__(self):
+        super(Artist, self).__init__(self.name)
+        self.colors = json.loads(str(self.colors))
+
+    def set_colors(self, colors: list[str]):
+        self.colors = colors
+
+# TODO: Use inheritance to create the classes in this file.

app/setup/__init__.py

@@ -1,7 +1,6 @@
 """
 Prepares the server for use.
 """
-from app.store.folder import FolderStore
 from app.setup.files import create_config_dir
 from app.setup.sqlite import setup_sqlite, run_migrations
 
@@ -16,6 +15,5 @@ def run_setup():
     run_migrations()
 
     TrackStore.load_all_tracks()
-    FolderStore.process_folders()
     AlbumStore.load_albums()
     ArtistStore.load_artists()

app/store/folder.py

@@ -1,122 +0,0 @@
-"""
-In memory store.
-"""
-from pathlib import Path
-from tqdm import tqdm
-
-from app.models import Folder
-from app.utils.bisection import UseBisection
-from app.utils.hashing import create_folder_hash
-
-from app.lib import folderslib
-from .tracks import TrackStore
-
-
-class FolderStore:
-    """
-    This class holds all tracks in memory and provides methods for
-    interacting with them.
-    """
-
-    folders: list[Folder] = []
-
-    @classmethod
-    def check_has_tracks(cls, path: str):  # type: ignore
-        """
-        Checks if a folder has tracks.
-        """
-        path_hashes = "".join(f.path_hash for f in cls.folders)
-        path_hash = create_folder_hash(*Path(path).parts[1:])
-
-        return path_hash in path_hashes
-
-    @classmethod
-    def is_empty_folder(cls, path: str):
-        """
-        Checks if a folder has tracks using tracks in the store.
-        """
-
-        all_folders = set(track.folder for track in TrackStore.tracks)
-        folder_hashes = "".join(
-            create_folder_hash(*Path(f).parts[1:]) for f in all_folders
-        )
-
-        path_hash = create_folder_hash(*Path(path).parts[1:])
-        return path_hash in folder_hashes
-
-    @classmethod
-    def add_folder(cls, path: str):
-        """
-        Adds a folder to the store.
-        """
-
-        if cls.check_has_tracks(path):
-            return
-
-        folder = folderslib.create_folder(path)
-        cls.folders.append(folder)
-
-    @classmethod
-    def remove_folder(cls, path: str):
-        """
-        Removes a folder from the store.
-        """
-
-        for folder in cls.folders:
-            if folder.path == path:
-                cls.folders.remove(folder)
-                break
-
-    @classmethod
-    def process_folders(cls):
-        """
-        Creates a list of folders from the tracks in the store.
-        """
-        cls.folders.clear()
-
-        all_folders = [track.folder for track in TrackStore.tracks]
-        all_folders = set(all_folders)
-
-        all_folders = [
-            folder for folder in all_folders if not cls.check_has_tracks(folder)
-        ]
-
-        all_folders = [Path(f) for f in all_folders]
-        # all_folders = [f for f in all_folders if f.exists()]
-
-        valid_folders = []
-
-        for folder in all_folders:
-            try:
-                if folder.exists():
-                    valid_folders.append(folder)
-            except PermissionError:
-                pass
-
-        for path in tqdm(valid_folders, desc="Processing folders"):
-            folder = folderslib.create_folder(str(path))
-
-            cls.folders.append(folder)
-
-    @classmethod
-    def get_folder(cls, path: str):  # type: ignore
-        """
-        Returns a folder object by its path.
-        """
-        # TODO: Modify this method to accept a list of paths, sorting is computationally expensive.
-        folders = sorted(cls.folders, key=lambda x: x.path)
-        folder = UseBisection(folders, "path", [path])()[0]
-
-        if folder is not None:
-            return folder
-
-        has_tracks = cls.check_has_tracks(path)
-
-        if not has_tracks:
-            return None
-
-        folder = folderslib.create_folder(path)
-        cls.folders.append(folder)
-        return folder
-
-# TODO: Remove this file. it's no longer needed.

pyproject.toml

@@ -9,11 +9,11 @@ python = ">=3.10,<3.12"
 Flask = "^2.0.2"
 Flask-Cors = "^3.0.10"
 requests = "^2.27.1"
-watchdog = "^2.2.1"
+watchdog = "^3.0.0"
 gunicorn = "^20.1.0"
 Pillow = "^9.0.1"
 "colorgram.py" = "^1.2.0"
-tqdm = "^4.64.0"
+tqdm = "^4.65.0"
 rapidfuzz = "^2.13.7"
 tinytag = "^1.8.1"
 Unidecode = "^1.3.6"
@@ -23,7 +23,7 @@ psutil = "^5.9.4"
 pylint = "^2.15.5"
 pytest = "^7.1.3"
 hypothesis = "^6.56.3"
-pyinstaller = "^5.7.0"
+pyinstaller = "^5.9.0"
 
 [tool.poetry.dev-dependencies.black]
 version = "^22.6.0"