From 88be80b25d0a8beb513d4ee50342c79b9cc3547c Mon Sep 17 00:00:00 2001 From: mungai-njoroge Date: Wed, 21 Jun 2023 13:20:09 +0300 Subject: [PATCH] fix db insert duplicate unique fields error + save album date as timestamp in db + Rewrite artist image downloader with multiprocessing.Pool --- app/api/album.py | 21 +++++++++++---------- app/db/sqlite/tracks.py | 2 +- app/lib/artistlib.py | 36 +++++++++++++++++++----------------- app/lib/taglib.py | 10 ++++++---- app/lib/watchdogg.py | 1 - 5 files changed, 37 insertions(+), 33 deletions(-) diff --git a/app/api/album.py b/app/api/album.py index 052f01c..a530cb8 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -115,7 +115,9 @@ def get_artist_albums(): albums = [ { "artisthash": a, - "albums": AlbumStore.get_albums_by_albumartist(a, limit, exclude=base_title), + "albums": AlbumStore.get_albums_by_albumartist( + a, limit, exclude=base_title + ), } for a in albumartists ] @@ -136,18 +138,17 @@ def get_album_versions(): if data is None: return {"msg": "No albumartist provided"} - og_album_title: str = data['og_album_title'] - base_title: str = data['base_title'] - artisthash: str = data['artisthash'] + og_album_title: str = data["og_album_title"] + base_title: str = data["base_title"] + artisthash: str = data["artisthash"] albums = AlbumStore.get_albums_by_artisthash(artisthash) albums = [ - a for a in albums - if - create_hash(a.base_title) == create_hash(base_title) and create_hash(og_album_title) != create_hash(a.og_title) + a + for a in albums + if create_hash(a.base_title) == create_hash(base_title) + and create_hash(og_album_title) != create_hash(a.og_title) ] - return { - "data": albums - } + return {"data": albums} diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py index 4ee679e..db39eb9 100644 --- a/app/db/sqlite/tracks.py +++ b/app/db/sqlite/tracks.py @@ -21,7 +21,7 @@ class SQLiteTrackMethods: """ Inserts a single track into the database. """ - sql = """INSERT INTO tracks( + sql = """INSERT OR REPLACE INTO tracks( album, albumartist, albumhash, diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py index f9105ef..c7c0d5e 100644 --- a/app/lib/artistlib.py +++ b/app/lib/artistlib.py @@ -1,19 +1,19 @@ -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from io import BytesIO - -from PIL import Image, UnidentifiedImageError -import requests import urllib +from concurrent.futures import ProcessPoolExecutor as Pool +from io import BytesIO +from multiprocessing import Pool, cpu_count +from pathlib import Path +import requests +from PIL import Image, UnidentifiedImageError +from requests.exceptions import ConnectionError as ReqConnError +from requests.exceptions import ReadTimeout from tqdm import tqdm -from requests.exceptions import ConnectionError as ReqConnError, ReadTimeout from app import settings -from app.models import Artist, Track, Album -from app.utils.hashing import create_hash - +from app.models import Album, Artist, Track from app.store import artists as artist_store +from app.utils.hashing import create_hash def get_artist_image_link(artist: str): @@ -74,15 +74,17 @@ class DownloadImage: class CheckArtistImages: def __init__(self): - with ThreadPoolExecutor() as pool: - list( + with Pool(cpu_count()) as pool: + res = list( tqdm( - pool.map(self.download_image, artist_store.ArtistStore.artists), + pool.imap_unordered(self.download_image, artist_store.ArtistStore.artists), total=len(artist_store.ArtistStore.artists), desc="Downloading missing artist images", ) ) + list(res) + @staticmethod def download_image(artist: Artist): """ @@ -90,7 +92,9 @@ class CheckArtistImages: :param artist: The artist name """ - img_path = Path(settings.Paths.get_artist_img_sm_path()) / f"{artist.artisthash}.webp" + img_path = ( + Path(settings.Paths.get_artist_img_sm_path()) / f"{artist.artisthash}.webp" + ) if img_path.exists(): return @@ -155,9 +159,7 @@ def get_albumartists(albums: list[Album]) -> set[str]: return artists -def get_all_artists( - tracks: list[Track], albums: list[Album] -) -> list[Artist]: +def get_all_artists(tracks: list[Track], albums: list[Album]) -> list[Artist]: artists_from_tracks = get_artists_from_tracks(tracks=tracks) artist_from_albums = get_albumartists(albums=albums) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 056af84..a27da35 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -1,6 +1,7 @@ import datetime import os from io import BytesIO +import pendulum from PIL import Image, UnidentifiedImageError from tinytag import TinyTag @@ -64,11 +65,12 @@ def extract_thumb(filepath: str, webp_path: str) -> bool: return False -def extract_date(date_str: str | None, timestamp: float) -> int: +def extract_date(date_str: str | None) -> int | None: try: - return int(date_str.split("-")[0]) + date = pendulum.parse(date_str, strict=False) + return int(date.timestamp()) except Exception as e: - return datetime.datetime.fromtimestamp(timestamp).year + return None def get_tags(filepath: str): @@ -141,7 +143,7 @@ def get_tags(filepath: str): tags.image = f"{tags.albumhash}.webp" tags.folder = win_replace_slash(os.path.dirname(filepath)) - tags.date = extract_date(tags.year, last_mod) + tags.date = extract_date(tags.year) or int(last_mod) tags.filepath = win_replace_slash(filepath) tags.filetype = filetype tags.last_mod = last_mod diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index ee56967..a19e615 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -131,7 +131,6 @@ def add_track(filepath: str) -> None: """ # remove the track if it already exists TrackStore.remove_track_by_filepath(filepath) - db.remove_tracks_by_filepaths(filepath) # add the track to the database and store. tags = get_tags(filepath)