tcsenpai/swingmusic
1
0
Fork 0
mirror of https://github.com/tcsenpai/swingmusic.git synced 2025-06-06 03:05:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix duplicate artist and album color entry in db

Browse Source 
+ Remove folder store
+ Reduce fuzzy search score cutoff from 90% to 75%
+ use inheritance to init Artist class
+ misc
This commit is contained in:
geoffrey45 2023-03-26 18:01:26 +03:00
parent 357afeb700
commit 5487dad27b
18 changed files with 102 additions and 333 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/api/colors.py
View File

@ -8,6 +8,11 @@ api = Blueprint("colors", __name__, url_prefix="/colors")
def get_album_color(albumhash: str):
album = Store.get_album_by_hash(albumhash)
if len(album.colors) > 0:
return {
"color": album.colors[0]
}
return {
"color": album.colors[0]
"color": ""
}

15
app/api/folder.py
View File

@ -8,7 +8,7 @@ from pathlib import Path
from flask import Blueprint, request
from app import settings
from app.lib.folderslib import GetFilesAndDirs, create_folder
from app.lib.folderslib import GetFilesAndDirs, get_folders
from app.db.sqlite.settings import SettingsSQLMethods as db
from app.utils.wintools import win_replace_slash, is_windows
@ -30,6 +30,7 @@ def get_folder_tree():
req_dir = "$home"
root_dirs = db.get_root_dirs()
root_dirs.sort()
try:
if req_dir == "$home" and root_dirs[0] == "$home":
@ -38,19 +39,17 @@ def get_folder_tree():
pass
if req_dir == "$home":
folders = [Path(f) for f in root_dirs]
folders = get_folders(root_dirs)
return {
"folders": [
create_folder(str(f)) for f in folders
],
"folders": folders,
"tracks": [],
}
if is_windows():
# Trailing slash needed when drive letters are passed,
# Remember, the trailing slash is removed in the client.
req_dir = req_dir + "/"
req_dir += "/"
else:
req_dir = "/" + req_dir + "/" if not req_dir.startswith("/") else req_dir + "/"
@ -66,7 +65,7 @@ def get_all_drives(is_win: bool = False):
"""
Returns a list of all the drives on a Windows machine.
"""
drives = psutil.disk_partitions(all=False)
drives = psutil.disk_partitions()
drives = [d.mountpoint for d in drives]
if is_win:
@ -99,7 +98,7 @@ def list_folders():
}
if is_win:
req_dir = req_dir + "/"
req_dir += "/"
else:
req_dir = "/" + req_dir + "/"
req_dir = str(Path(req_dir).resolve())

4
app/api/playlist.py
View File

@ -34,9 +34,9 @@ delete_playlist = PL.delete_playlist
# get_tracks_by_trackhashes = SQLiteTrackMethods.get_tracks_by_trackhashes
def duplicate_images(images: list):
if len(images) == 1:
images = images * 4
images *= 4
elif len(images) == 2:
images = images + list(reversed(images))
images += list(reversed(images))
elif len(images) == 3:
images = images + images[:1]

2
app/api/settings.py
View File

@ -8,7 +8,6 @@ from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.utils.generators import get_random_str
from app.utils.threading import background
from app.store.folder import FolderStore
from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
from app.store.artists import ArtistStore
@ -27,7 +26,6 @@ def reload_everything():
Reloads all stores using the current database items
"""
TrackStore.load_all_tracks()
FolderStore.process_folders()
AlbumStore.load_albums()
ArtistStore.load_artists()

85
app/db/sqlite/albums.py
View File

@ -1,6 +1,6 @@
from sqlite3 import Cursor
from .utils import SQLiteManager, tuple_to_album, tuples_to_albums
from .utils import SQLiteManager, tuples_to_albums
class SQLiteAlbumMethods:
@ -10,30 +10,15 @@ class SQLiteAlbumMethods:
Inserts one album into the database
"""
sql = """INSERT INTO albums(
sql = """INSERT OR REPLACE INTO albums(
albumhash,
colors
) VALUES(?,?)
"""
cur.execute(sql, (albumhash, colors))
return cur.lastrowid
# @classmethod
# def insert_many_albums(cls, albums: list[dict]):
# """
# Takes a generator of albums, and inserts them into the database
# Parameters
# ----------
# albums : Generator
# Generator
# """
# with SQLiteManager() as cur:
# for album in albums:
# cls.insert_one_album(cur, album["albumhash"], album["colors"])
@classmethod
def get_all_albums(cls):
with SQLiteManager() as cur:
@ -45,58 +30,6 @@ class SQLiteAlbumMethods:
return []
# @staticmethod
# def get_album_by_id(album_id: int):
# conn = get_sqlite_conn()
# cur = conn.cursor()
# cur.execute("SELECT * FROM albums WHERE id=?", (album_id,))
# album = cur.fetchone()
# conn.close()
# if album is None:
# return None
# return tuple_to_album(album)
@staticmethod
def get_album_by_hash(album_hash: str):
with SQLiteManager() as cur:
cur.execute("SELECT * FROM albums WHERE albumhash=?", (album_hash,))
album = cur.fetchone()
if album is not None:
return tuple_to_album(album)
return None
@classmethod
def get_albums_by_hashes(cls, album_hashes: list):
"""
Gets all the albums with the specified hashes. Returns a generator of albums or an empty list.
"""
with SQLiteManager() as cur:
hashes = ",".join("?" * len(album_hashes))
cur.execute(
f"SELECT * FROM albums WHERE albumhash IN ({hashes})", album_hashes
)
albums = cur.fetchall()
if albums is not None:
return tuples_to_albums(albums)
return []
# @staticmethod
# def update_album_colors(album_hash: str, colors: list[str]):
# sql = "UPDATE albums SET colors=? WHERE albumhash=?"
# colors_str = json.dumps(colors)
# with SQLiteManager() as cur:
# cur.execute(sql, (colors_str, album_hash))
@staticmethod
def get_albums_by_albumartist(albumartist: str):
with SQLiteManager() as cur:
@ -107,17 +40,3 @@ class SQLiteAlbumMethods:
return tuples_to_albums(albums)
return []
@staticmethod
def get_all_albums_raw():
"""
Returns all the albums in the database, as a list of tuples.
"""
with SQLiteManager() as cur:
cur.execute("SELECT * FROM albums")
albums = cur.fetchall()
if albums is not None:
return albums
return []

16
app/db/sqlite/artists.py
View File

@ -3,27 +3,27 @@ Contains methods for reading and writing to the sqlite artists database.
"""
import json
from sqlite3 import Cursor
from .utils import SQLiteManager
class SQLiteArtistMethods:
@classmethod
def insert_one_artist(cls, artisthash: str, colors: str | list[str]):
@staticmethod
def insert_one_artist(cur: Cursor, artisthash: str, colors: str | list[str]):
"""
Inserts a single artist into the database.
"""
sql = """INSERT INTO artists(
sql = """INSERT OR REPLACE INTO artists(
artisthash,
colors
) VALUES(?,?)
"""
colors = json.dumps(colors)
cur.execute(sql, (artisthash, colors))
with SQLiteManager() as cur:
cur.execute(sql, (artisthash, colors))
@classmethod
def get_all_artists(cls):
@staticmethod
def get_all_artists():
"""
Get all artists from the database and return a generator of Artist objects
"""

9
app/db/sqlite/queries.py
View File

@ -45,13 +45,15 @@ CREATE TABLE IF NOT EXISTS tracks (
genre text,
title text NOT NULL,
track integer NOT NULL,
trackhash text NOT NULL
trackhash text NOT NULL,
UNIQUE (filepath)
);
CREATE TABLE IF NOT EXISTS albums (
id integer PRIMARY KEY,
albumhash text NOT NULL,
colors text NOT NULL
colors text NOT NULL,
UNIQUE (albumhash)
);
@ -60,7 +62,8 @@ CREATE TABLE IF NOT EXISTS artists (
id integer PRIMARY KEY,
artisthash text NOT NULL,
colors text,
bio text
bio text,
UNIQUE (artisthash)
);
CREATE TABLE IF NOT EXISTS folders (

1
app/db/sqlite/settings.py
View File

@ -52,6 +52,7 @@ class SettingsSQLMethods:
for _dir in dirs:
cur.execute(sql, (_dir,))
# Not currently used anywhere, to be used later
@staticmethod
def add_excluded_dirs(dirs: list[str]):
"""

20
app/db/sqlite/tracks.py
View File

@ -82,26 +82,6 @@ class SQLiteTrackMethods:
return None
@staticmethod
def get_tracks_by_trackhashes(hashes: list[str]):
"""
Gets all tracks in a list of trackhashes.
Returns a generator of Track objects or an empty list.
"""
sql = "SELECT * FROM tracks WHERE trackhash IN ({})".format(
",".join("?" * len(hashes))
)
with SQLiteManager() as cur:
cur.execute(sql, hashes)
rows = cur.fetchall()
if rows is not None:
return tuples_to_tracks(rows)
return []
@staticmethod
def remove_track_by_filepath(filepath: str):
"""

8
app/lib/artistlib.py
View File

@ -1,7 +1,8 @@
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from io import BytesIO
from PIL import Image
from PIL import Image, UnidentifiedImageError
import requests
import urllib
@ -55,7 +56,10 @@ class DownloadImage:
"""
Downloads the image from the url.
"""
return Image.open(BytesIO(requests.get(url, timeout=10).content))
try:
return Image.open(BytesIO(requests.get(url, timeout=10).content))
except UnidentifiedImageError:
return None
@staticmethod
def save_img(img: Image.Image, sm_path: Path, lg_path: Path):

68
app/lib/colorlib.py
View File

@ -12,16 +12,15 @@ from app import settings
from app.db.sqlite.albums import SQLiteAlbumMethods as db
from app.db.sqlite.artists import SQLiteArtistMethods as adb
from app.db.sqlite.utils import SQLiteManager
from app.models import Album, Artist
from app.store.artists import ArtistStore
from app.store.albums import AlbumStore
def get_image_colors(image: str) -> list[str]:
"""Extracts 2 of the most dominant colors from an image."""
def get_image_colors(image: str, count=1) -> list[str]:
"""Extracts n number of the most dominant colors from an image."""
try:
colors = sorted(colorgram.extract(image, 1), key=lambda c: c.hsl.h)
colors = sorted(colorgram.extract(image, count), key=lambda c: c.hsl.h)
except OSError:
return []
@ -34,6 +33,16 @@ def get_image_colors(image: str) -> list[str]:
return formatted_colors
def process_color(item_hash: str, is_album=True):
path = settings.Paths.SM_THUMB_PATH if is_album else settings.Paths.ARTIST_IMG_SM_PATH
path = Path(path) / (item_hash + ".webp")
if not path.exists():
return
return get_image_colors(str(path))
class ProcessAlbumColors:
"""
Extracts the most dominant color from the album art and saves it to the database.
@ -44,26 +53,22 @@ class ProcessAlbumColors:
with SQLiteManager() as cur:
for album in tqdm(albums, desc="Processing missing album colors"):
colors = self.process_color(album)
sql = "SELECT COUNT(1) FROM albums WHERE albumhash = ?"
cur.execute(sql, (album.albumhash,))
count = cur.fetchone()[0]
if count != 0:
continue
colors = process_color(album.albumhash)
if colors is None:
continue
album.set_colors(colors)
color_str = json.dumps(colors)
db.insert_one_album(cur, album.albumhash, color_str)
@staticmethod
def process_color(album: Album):
path = Path(settings.Paths.SM_THUMB_PATH) / album.image
if not path.exists():
return
colors = get_image_colors(str(path))
return colors
class ProcessArtistColors:
"""
@ -73,27 +78,20 @@ class ProcessArtistColors:
def __init__(self) -> None:
all_artists = [a for a in ArtistStore.artists if len(a.colors) == 0]
for artist in tqdm(all_artists, desc="Processing missing artist colors"):
self.process_color(artist)
with SQLiteManager() as cur:
for artist in tqdm(all_artists, desc="Processing missing artist colors"):
sql = "SELECT COUNT(1) FROM artists WHERE artisthash = ?"
@staticmethod
def process_color(artist: Artist):
path = Path(settings.Paths.ARTIST_IMG_SM_PATH) / artist.image
cur.execute(sql, (artist.artisthash,))
count = cur.fetchone()[0]
if not path.exists():
return
if count != 0:
continue
colors = get_image_colors(str(path))
colors = process_color(artist.artisthash, is_album=False)
if len(colors) > 0:
adb.insert_one_artist(artisthash=artist.artisthash, colors=colors)
ArtistStore.map_artist_color((0, artist.artisthash, json.dumps(colors)))
if colors is None:
continue
# TODO: If item color is in db, get it, assign it to the item and continue.
# - Format all colors in the format: rgb(123, 123, 123)
# - Each digit should be 3 digits long.
# - Format all db colors into a master string of the format "-itemhash:colorhash-"
# - Find the item hash using index() and get the color using the index + number, where number
# is the length of the rgb string + 1
# - Assign the color to the item and continue.
# - If the color is not in the db, extract it and add it to the db.
artist.set_colors(colors)
adb.insert_one_artist(cur, artist.artisthash, colors)

2
app/lib/populate.py
View File

@ -12,7 +12,6 @@ from app.logger import log
from app.models import Album, Artist, Track
from app.utils.filesystem import run_fast_scandir
from app.store.folder import FolderStore
from app.store.albums import AlbumStore
from app.store.tracks import TrackStore
from app.store.artists import ArtistStore
@ -102,7 +101,6 @@ class Populate:
track.is_favorite = track.trackhash in fav_tracks
TrackStore.add_track(track)
FolderStore.add_folder(track.folder)
if not AlbumStore.album_exists(track.albumhash):
AlbumStore.add_album(AlbumStore.create_album(track))

13
app/lib/searchlib.py
View File

@ -23,10 +23,10 @@ class Cutoff:
Holds all the default cutoff values.
"""
tracks: int = 90
albums: int = 90
artists: int = 90
playlists: int = 90
tracks: int = 75
albums: int = 75
artists: int = 75
playlists: int = 75
class Limit:
@ -54,7 +54,6 @@ class SearchTracks:
results = process.extract(
self.query,
track_titles,
scorer=fuzz.WRatio,
score_cutoff=Cutoff.tracks,
limit=Limit.tracks,
)
@ -77,7 +76,6 @@ class SearchArtists:
results = process.extract(
self.query,
artists,
scorer=fuzz.WRatio,
score_cutoff=Cutoff.artists,
limit=Limit.artists,
)
@ -100,7 +98,6 @@ class SearchAlbums:
results = process.extract(
self.query,
albums,
scorer=fuzz.WRatio,
score_cutoff=Cutoff.albums,
limit=Limit.albums,
)
@ -125,7 +122,6 @@ class SearchPlaylists:
results = process.extract(
self.query,
playlists,
scorer=fuzz.WRatio,
score_cutoff=Cutoff.playlists,
limit=Limit.playlists,
)
@ -176,7 +172,6 @@ class SearchAll:
results = process.extract(
query=query,
choices=items,
scorer=fuzz.WRatio,
score_cutoff=Cutoff.tracks,
limit=20
)

9
app/lib/watchdogg.py
View File

@ -17,7 +17,6 @@ from app.db.sqlite.tracks import SQLiteManager
from app.db.sqlite.tracks import SQLiteTrackMethods as db
from app.db.sqlite.settings import SettingsSQLMethods as sdb
from app.store.folder import FolderStore
from app.store.tracks import TrackStore
from app.store.albums import AlbumStore
from app.store.artists import ArtistStore
@ -144,8 +143,6 @@ def add_track(filepath: str) -> None:
track = Track(**tags)
TrackStore.add_track(track)
FolderStore.add_folder(track.folder)
if not AlbumStore.album_exists(track.albumhash):
album = AlbumStore.create_album(track)
AlbumStore.add_album(album)
@ -182,11 +179,6 @@ def remove_track(filepath: str) -> None:
if empty_artist:
ArtistStore.remove_artist_by_hash(artist.artisthash)
empty_folder = FolderStore.is_empty_folder(track.folder)
if empty_folder:
FolderStore.remove_folder(track.folder)
class Handler(PatternMatchingEventHandler):
files_to_process = []
@ -204,7 +196,6 @@ class Handler(PatternMatchingEventHandler):
self,
patterns=patterns,
ignore_directories=True,
case_sensitive=False,
)
def get_abs_path(self, path: str):

46
app/models/artist.py
View File

@ -5,27 +5,6 @@ from dataclasses import dataclass
from app.utils.hashing import create_hash
@dataclass(slots=True)
class Artist:
"""
Artist class
"""
name: str
artisthash: str = ""
image: str = ""
trackcount: int = 0
albumcount: int = 0
duration: int = 0
colors: list[str] = dataclasses.field(default_factory=list)
is_favorite: bool = False
def __post_init__(self):
self.artisthash = create_hash(self.name, decode=True)
self.image = self.artisthash + ".webp"
self.colors = json.loads(str(self.colors))
@dataclass(slots=True)
class ArtistMinimal:
"""
@ -36,6 +15,29 @@ class ArtistMinimal:
artisthash: str = ""
image: str = ""
def __post_init__(self):
def __init__(self, name: str):
self.name = name
self.artisthash = create_hash(self.name, decode=True)
self.image = self.artisthash + ".webp"
@dataclass(slots=True)
class Artist(ArtistMinimal):
"""
Artist class
"""
trackcount: int = 0
albumcount: int = 0
duration: int = 0
colors: list[str] = dataclasses.field(default_factory=list)
is_favorite: bool = False
def __post_init__(self):
super(Artist, self).__init__(self.name)
self.colors = json.loads(str(self.colors))
def set_colors(self, colors: list[str]):
self.colors = colors
# TODO: Use inheritance to create the classes in this file.

2
app/setup/__init__.py
View File

@ -1,7 +1,6 @@
"""
Prepares the server for use.
"""
from app.store.folder import FolderStore
from app.setup.files import create_config_dir
from app.setup.sqlite import setup_sqlite, run_migrations
@ -16,6 +15,5 @@ def run_setup():
run_migrations()
TrackStore.load_all_tracks()
FolderStore.process_folders()
AlbumStore.load_albums()
ArtistStore.load_artists()

122
app/store/folder.py
View File

@ -1,122 +0,0 @@
"""
In memory store.
"""
from pathlib import Path
from tqdm import tqdm
from app.models import Folder
from app.utils.bisection import UseBisection
from app.utils.hashing import create_folder_hash
from app.lib import folderslib
from .tracks import TrackStore
class FolderStore:
"""
This class holds all tracks in memory and provides methods for
interacting with them.
"""
folders: list[Folder] = []
@classmethod
def check_has_tracks(cls, path: str): # type: ignore
"""
Checks if a folder has tracks.
"""
path_hashes = "".join(f.path_hash for f in cls.folders)
path_hash = create_folder_hash(*Path(path).parts[1:])
return path_hash in path_hashes
@classmethod
def is_empty_folder(cls, path: str):
"""
Checks if a folder has tracks using tracks in the store.
"""
all_folders = set(track.folder for track in TrackStore.tracks)
folder_hashes = "".join(
create_folder_hash(*Path(f).parts[1:]) for f in all_folders
)
path_hash = create_folder_hash(*Path(path).parts[1:])
return path_hash in folder_hashes
@classmethod
def add_folder(cls, path: str):
"""
Adds a folder to the store.
"""
if cls.check_has_tracks(path):
return
folder = folderslib.create_folder(path)
cls.folders.append(folder)
@classmethod
def remove_folder(cls, path: str):
"""
Removes a folder from the store.
"""
for folder in cls.folders:
if folder.path == path:
cls.folders.remove(folder)
break
@classmethod
def process_folders(cls):
"""
Creates a list of folders from the tracks in the store.
"""
cls.folders.clear()
all_folders = [track.folder for track in TrackStore.tracks]
all_folders = set(all_folders)
all_folders = [
folder for folder in all_folders if not cls.check_has_tracks(folder)
]
all_folders = [Path(f) for f in all_folders]
# all_folders = [f for f in all_folders if f.exists()]
valid_folders = []
for folder in all_folders:
try:
if folder.exists():
valid_folders.append(folder)
except PermissionError:
pass
for path in tqdm(valid_folders, desc="Processing folders"):
folder = folderslib.create_folder(str(path))
cls.folders.append(folder)
@classmethod
def get_folder(cls, path: str): # type: ignore
"""
Returns a folder object by its path.
"""
# TODO: Modify this method to accept a list of paths, sorting is computationally expensive.
folders = sorted(cls.folders, key=lambda x: x.path)
folder = UseBisection(folders, "path", [path])()[0]
if folder is not None:
return folder
has_tracks = cls.check_has_tracks(path)
if not has_tracks:
return None
folder = folderslib.create_folder(path)
cls.folders.append(folder)
return folder
# TODO: Remove this file. it's no longer needed.

6
pyproject.toml
View File

@ -9,11 +9,11 @@ python = ">=3.10,<3.12"
Flask = "^2.0.2"
Flask-Cors = "^3.0.10"
requests = "^2.27.1"
watchdog = "^2.2.1"
watchdog = "^3.0.0"
gunicorn = "^20.1.0"
Pillow = "^9.0.1"
"colorgram.py" = "^1.2.0"
tqdm = "^4.64.0"
tqdm = "^4.65.0"
rapidfuzz = "^2.13.7"
tinytag = "^1.8.1"
Unidecode = "^1.3.6"
@ -23,7 +23,7 @@ psutil = "^5.9.4"
pylint = "^2.15.5"
pytest = "^7.1.3"
hypothesis = "^6.56.3"
pyinstaller = "^5.7.0"
pyinstaller = "^5.9.0"
[tool.poetry.dev-dependencies.black]
version = "^22.6.0"