From 9d4f7af581a74e6722489e5a15538687630ff53d Mon Sep 17 00:00:00 2001 From: mungai-njoroge Date: Wed, 21 Jun 2023 09:20:56 +0300 Subject: [PATCH] rewrite populate.get_image() to extract a track thumbnail from the first track in an album that has one. + rewrite Populate.remove_modified with sets + clean the SqliteManager utility class + Rewrite ProcessTrackThumbnails to use a process pool instead of a thread pool + rewrite track store's remove_tracks_by_filepaths to utilize sets --- app/db/sqlite/tracks.py | 4 +-- app/db/sqlite/utils.py | 28 +++++++-------- app/lib/populate.py | 79 ++++++++++++++++++++++++++++++----------- app/lib/taglib.py | 3 +- app/lib/watchdogg.py | 2 +- app/store/tracks.py | 5 ++- app/utils/parsers.py | 18 ---------- 7 files changed, 78 insertions(+), 61 deletions(-) diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py index c530e01..4ee679e 100644 --- a/app/db/sqlite/tracks.py +++ b/app/db/sqlite/tracks.py @@ -84,12 +84,12 @@ class SQLiteTrackMethods: return None @staticmethod - def remove_tracks_by_filepaths(filepaths: str | list[str]): + def remove_tracks_by_filepaths(filepaths: str | set[str]): """ Removes a track or tracks from the database using their filepaths. """ if isinstance(filepaths, str): - filepaths = [filepaths] + filepaths = {filepaths} with SQLiteManager() as cur: for filepath in filepaths: diff --git a/app/db/sqlite/utils.py b/app/db/sqlite/utils.py index 53a4dae..3b35e1a 100644 --- a/app/db/sqlite/utils.py +++ b/app/db/sqlite/utils.py @@ -5,6 +5,7 @@ Helper functions for use with the SQLite database. import sqlite3 from sqlite3 import Connection, Cursor import time +from typing import Optional from app.models import Album, Playlist, Track from app.settings import Db @@ -61,12 +62,12 @@ class SQLiteManager: for you. It also commits and closes the connection when you're done. """ - def __init__(self, conn: Connection | None = None, userdata_db=False) -> None: + def __init__(self, conn: Optional[Connection] = None, userdata_db=False) -> None: """ When a connection is passed in, don't close the connection, because it's a connection to the search database [in memory db]. """ - self.conn: Connection | None = conn + self.conn = conn self.CLOSE_CONN = True self.userdata_db = userdata_db @@ -90,19 +91,18 @@ class SQLiteManager: return self.conn.cursor() def __exit__(self, exc_type, exc_value, exc_traceback): - if self.conn: - trial_count = 0 + trial_count = 0 - while trial_count < 10: - try: - self.conn.commit() + while trial_count < 10: + try: + self.conn.commit() - if self.CLOSE_CONN: - self.conn.close() + if self.CLOSE_CONN: + self.conn.close() - return - except sqlite3.OperationalError: - trial_count += 1 - time.sleep(3) + return + except sqlite3.OperationalError: + trial_count += 1 + time.sleep(3) - self.conn.close() + self.conn.close() diff --git a/app/lib/populate.py b/app/lib/populate.py index 0b5ca7a..8ffb8b1 100644 --- a/app/lib/populate.py +++ b/app/lib/populate.py @@ -1,6 +1,7 @@ +from collections import deque import os +from typing import Generator from tqdm import tqdm -from concurrent.futures import ThreadPoolExecutor from requests import ConnectionError as RequestConnectionError from requests import ReadTimeout @@ -47,7 +48,6 @@ class Populate: validate_tracks() tracks = get_all_tracks() - tracks = list(tracks) dirs_to_scan = sdb.get_root_dirs() @@ -66,13 +66,13 @@ class Populate: except IndexError: pass - files = [] + files = set() for _dir in dirs_to_scan: - files.extend(run_fast_scandir(_dir, full=True)[1]) + files = files.union(run_fast_scandir(_dir, full=True)[1]) - self.remove_modified(tracks) - untagged = self.filter_untagged(tracks, files) + unmodified = self.remove_modified(tracks) + untagged = files - unmodified if len(untagged) != 0: self.tag_untagged(untagged, key) @@ -101,23 +101,31 @@ class Populate: ProcessArtistColors() @staticmethod - def remove_modified(tracks: list[Track]): - modified = [ - t.filepath for t in tracks if t.last_mod != os.path.getmtime(t.filepath) - ] + def remove_modified(tracks: Generator[Track, None, None]): + """ + Removes tracks from the database that have been modified + since they were added to the database. + """ + + unmodified = set() + modified = set() + + for track in tracks: + if track.last_mod == os.path.getmtime(track.filepath): + unmodified.add(track.filepath) + continue + + modified.add(track.filepath) TrackStore.remove_tracks_by_filepaths(modified) remove_tracks_by_filepaths(modified) - @staticmethod - def filter_untagged(tracks: list[Track], files: list[str]): - tagged_files = [t.filepath for t in tracks] - return set(files) - set(tagged_files) + return unmodified @staticmethod def tag_untagged(untagged: set[str], key: str): log.info("Found %s new tracks", len(untagged)) - tagged_tracks: list[dict] = [] + tagged_tracks: deque[dict] = deque() tagged_count = 0 fav_tracks = favdb.get_fav_tracks() @@ -159,18 +167,47 @@ class Populate: def get_image(album: Album): - for track in TrackStore.tracks: - if track.albumhash == album.albumhash: - extract_thumb(track.filepath, track.image) - break + """ + The function retrieves an image from an album by iterating through its tracks and extracting the thumbnail from the first track that has one. + + :param album: An instance of the `Album` class representing the album to retrieve the image from. + :type album: Album + :return: None + """ + + matching_tracks = filter( + lambda t: t.albumhash == album.albumhash, TrackStore.tracks + ) + + try: + track = next(matching_tracks) + extracted = extract_thumb(track.filepath, track.image) + + while not extracted: + try: + track = next(matching_tracks) + extracted = extract_thumb(track.filepath, track.image) + except StopIteration: + break + + return + except StopIteration: + pass + + +from multiprocessing import Pool, cpu_count class ProcessTrackThumbnails: + """ + Extracts the album art from all albums in album store. + """ + def __init__(self) -> None: - with ThreadPoolExecutor(max_workers=4) as pool: + with Pool(processes=cpu_count()) as pool: results = list( tqdm( - pool.map(get_image, AlbumStore.albums), + pool.imap_unordered(get_image, AlbumStore.albums), total=len(AlbumStore.albums), desc="Extracting track images", ) diff --git a/app/lib/taglib.py b/app/lib/taglib.py index 492f244..056af84 100644 --- a/app/lib/taglib.py +++ b/app/lib/taglib.py @@ -67,8 +67,7 @@ def extract_thumb(filepath: str, webp_path: str) -> bool: def extract_date(date_str: str | None, timestamp: float) -> int: try: return int(date_str.split("-")[0]) - except: # pylint: disable=bare-except - print(datetime.datetime.fromtimestamp(timestamp).year) + except Exception as e: return datetime.datetime.fromtimestamp(timestamp).year diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py index bb73dc7..ee56967 100644 --- a/app/lib/watchdogg.py +++ b/app/lib/watchdogg.py @@ -92,7 +92,7 @@ class Watcher: ) return except OSError as e: - log.error('Failed to start watchdog. %s', e) + log.error("Failed to start watchdog. %s", e) return try: diff --git a/app/store/tracks.py b/app/store/tracks.py index 2fe4db1..d093f4e 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -53,15 +53,14 @@ class TrackStore: break @classmethod - def remove_tracks_by_filepaths(cls, filepaths: list[str]): + def remove_tracks_by_filepaths(cls, filepaths: set[str]): """ Removes multiple tracks from the store by their filepaths. """ - paths_str = "~".join(filepaths) for track in cls.tracks: - if track.filepath in paths_str: + if track.filepath in filepaths: cls.tracks.remove(track) @classmethod diff --git a/app/utils/parsers.py b/app/utils/parsers.py index 1d91d95..2af5c5f 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -245,21 +245,3 @@ def clean_title(title: str) -> str: # # if "-" in title: # return remove_hyphen_remasters(title) - - -# Traceback (most recent call last): -# File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner -# self.run() -# File "/usr/lib/python3.10/threading.py", line 953, in run -# self._target(*self._args, **self._kwargs) -# File "/home/cwilvx/code/swingmusic/app/periodic_scan.py", line 29, in run_periodic_scans -# Populate(key=get_random_str()) -# File "/home/cwilvx/code/swingmusic/app/lib/populate.py", line 74, in __init__ -# self.tag_untagged(untagged, key) -# File "/home/cwilvx/code/swingmusic/app/lib/populate.py", line 123, in tag_untagged -# insert_many_tracks(tagged_tracks) -# File "/home/cwilvx/code/swingmusic/app/db/sqlite/tracks.py", line 54, in insert_many_tracks -# cls.insert_one_track(track, cur) -# File "/home/cwilvx/code/swingmusic/app/db/sqlite/tracks.py", line 45, in insert_one_track -# cur.execute(sql, track) -# sqlite3.IntegrityError: UNIQUE constraint failed: tracks.filepath