From 1eac009fde8fb37f85697fe532b67cb3b688a3bb Mon Sep 17 00:00:00 2001
From: mungai-njoroge <geoffreymungai45@gmail.com>
Date: Tue, 20 Jun 2023 16:34:56 +0300
Subject: [PATCH] prevent running migrations if is_fresh_install

+ fix: sqlite3.ProgrammingError: Cannot operate on a closed cursor on ProcessAlbumColors()
+ move processing artist images from periodic_scans to Populate
+ bump hash string limit from 7 to 10
+ add last_mod property to database
+ fix: TypeError: '<' not supported between instances of 'int' and 'str' on album page
---
 app/db/sqlite/albums.py              |  1 -
 app/db/sqlite/artists.py             |  1 -
 app/db/sqlite/queries.py             |  3 +-
 app/db/sqlite/tracks.py              | 13 +++++---
 app/lib/artistlib.py                 |  2 +-
 app/lib/colorlib.py                  | 10 +++++-
 app/lib/populate.py                  | 49 +++++++++++++++++++++++-----
 app/lib/taglib.py                    | 10 +++---
 app/lib/trackslib.py                 |  2 +-
 app/lib/watchdogg.py                 | 17 ++++------
 app/migrations/__init__.py           |  4 +++
 app/migrations/__preinit/__init__.py |  4 +++
 app/models/track.py                  |  3 +-
 app/periodic_scan.py                 | 11 -------
 app/store/tracks.py                  | 16 ++++++++-
 app/utils/hashing.py                 |  4 +--
 16 files changed, 103 insertions(+), 47 deletions(-)

diff --git a/app/db/sqlite/albums.py b/app/db/sqlite/albums.py
index fdf08e6..e3bdd41 100644
--- a/app/db/sqlite/albums.py
+++ b/app/db/sqlite/albums.py
@@ -18,7 +18,6 @@ class SQLiteAlbumMethods:
 
         cur.execute(sql, (albumhash, colors))
         lastrowid = cur.lastrowid
-        cur.close()
 
         return lastrowid
 
diff --git a/app/db/sqlite/artists.py b/app/db/sqlite/artists.py
index ed7f9cb..5849706 100644
--- a/app/db/sqlite/artists.py
+++ b/app/db/sqlite/artists.py
@@ -21,7 +21,6 @@ class SQLiteArtistMethods:
             """
         colors = json.dumps(colors)
         cur.execute(sql, (artisthash, colors))
-        cur.close()
 
     @staticmethod
     def get_all_artists():
diff --git a/app/db/sqlite/queries.py b/app/db/sqlite/queries.py
index a886e37..032cd15 100644
--- a/app/db/sqlite/queries.py
+++ b/app/db/sqlite/queries.py
@@ -37,12 +37,13 @@ CREATE TABLE IF NOT EXISTS tracks (
     artist text NOT NULL,
     bitrate integer NOT NULL,
     copyright text,
-    date text NOT NULL,
+    date integer NOT NULL,
     disc integer NOT NULL,
     duration integer NOT NULL,
     filepath text NOT NULL,
     folder text NOT NULL,
     genre text,
+    last_mod float NOT NULL,
     title text NOT NULL,
     track integer NOT NULL,
     trackhash text NOT NULL,
diff --git a/app/db/sqlite/tracks.py b/app/db/sqlite/tracks.py
index a624b3f..c530e01 100644
--- a/app/db/sqlite/tracks.py
+++ b/app/db/sqlite/tracks.py
@@ -34,11 +34,12 @@ class SQLiteTrackMethods:
             filepath,
             folder,
             genre,
+            last_mod,
             title,
             track,
             trackhash
             ) VALUES(:album, :albumartist, :albumhash, :artist, :bitrate, :copyright, 
-            :date, :disc, :duration, :filepath, :folder, :genre, :title, :track, :trackhash)
+            :date, :disc, :duration, :filepath, :folder, :genre, :last_mod, :title, :track, :trackhash)
             """
 
         track = OrderedDict(sorted(track.items()))
@@ -83,12 +84,16 @@ class SQLiteTrackMethods:
             return None
 
     @staticmethod
-    def remove_track_by_filepath(filepath: str):
+    def remove_tracks_by_filepaths(filepaths: str | list[str]):
         """
-        Removes a track from the database using its filepath.
+        Removes a track or tracks from the database using their filepaths.
         """
+        if isinstance(filepaths, str):
+            filepaths = [filepaths]
+
         with SQLiteManager() as cur:
-            cur.execute("DELETE FROM tracks WHERE filepath=?", (filepath,))
+            for filepath in filepaths:
+                cur.execute("DELETE FROM tracks WHERE filepath=?", (filepath,))
 
     @staticmethod
     def remove_tracks_by_folders(folders: set[str]):
diff --git a/app/lib/artistlib.py b/app/lib/artistlib.py
index f519fc8..f9105ef 100644
--- a/app/lib/artistlib.py
+++ b/app/lib/artistlib.py
@@ -79,7 +79,7 @@ class CheckArtistImages:
                 tqdm(
                     pool.map(self.download_image, artist_store.ArtistStore.artists),
                     total=len(artist_store.ArtistStore.artists),
-                    desc="Downloading artist images",
+                    desc="Downloading missing artist images",
                 )
             )
 
diff --git a/app/lib/colorlib.py b/app/lib/colorlib.py
index 245dfe7..20ad558 100644
--- a/app/lib/colorlib.py
+++ b/app/lib/colorlib.py
@@ -34,7 +34,11 @@ def get_image_colors(image: str, count=1) -> list[str]:
 
 
 def process_color(item_hash: str, is_album=True):
-    path = settings.Paths.get_sm_thumb_path() if is_album else settings.Paths.get_artist_img_sm_path()
+    path = (
+        settings.Paths.get_sm_thumb_path()
+        if is_album
+        else settings.Paths.get_artist_img_sm_path()
+    )
     path = Path(path) / (item_hash + ".webp")
 
     if not path.exists():
@@ -69,6 +73,8 @@ class ProcessAlbumColors:
                 color_str = json.dumps(colors)
                 db.insert_one_album(cur, album.albumhash, color_str)
 
+            cur.close()
+
 
 class ProcessArtistColors:
     """
@@ -95,3 +101,5 @@ class ProcessArtistColors:
 
                 artist.set_colors(colors)
                 adb.insert_one_artist(cur, artist.artisthash, colors)
+
+            cur.close()
diff --git a/app/lib/populate.py b/app/lib/populate.py
index cc7f436..0b5ca7a 100644
--- a/app/lib/populate.py
+++ b/app/lib/populate.py
@@ -1,10 +1,14 @@
-from concurrent.futures import ThreadPoolExecutor
+import os
 from tqdm import tqdm
+from concurrent.futures import ThreadPoolExecutor
+from requests import ConnectionError as RequestConnectionError
+from requests import ReadTimeout
 
 from app import settings
 from app.db.sqlite.tracks import SQLiteTrackMethods
 from app.db.sqlite.settings import SettingsSQLMethods as sdb
 from app.db.sqlite.favorite import SQLiteFavoriteMethods as favdb
+from app.lib.artistlib import CheckArtistImages
 from app.lib.colorlib import ProcessAlbumColors, ProcessArtistColors
 
 from app.lib.taglib import extract_thumb, get_tags
@@ -16,9 +20,11 @@ from app.utils.filesystem import run_fast_scandir
 from app.store.albums import AlbumStore
 from app.store.tracks import TrackStore
 from app.store.artists import ArtistStore
+from app.utils.network import Ping
 
 get_all_tracks = SQLiteTrackMethods.get_all_tracks
 insert_many_tracks = SQLiteTrackMethods.insert_many_tracks
+remove_tracks_by_filepaths = SQLiteTrackMethods.remove_tracks_by_filepaths
 
 POPULATE_KEY = ""
 
@@ -48,8 +54,8 @@ class Populate:
         if len(dirs_to_scan) == 0:
             log.warning(
                 (
-                        "The root directory is not configured. "
-                        + "Open the app in your webbrowser to configure."
+                    "The root directory is not configured. "
+                    + "Open the app in your webbrowser to configure."
                 )
             )
             return
@@ -65,18 +71,44 @@ class Populate:
         for _dir in dirs_to_scan:
             files.extend(run_fast_scandir(_dir, full=True)[1])
 
+        self.remove_modified(tracks)
         untagged = self.filter_untagged(tracks, files)
 
-        if len(untagged) == 0:
-            log.info("All clear, no unread files.")
-            return
-
-        self.tag_untagged(untagged, key)
+        if len(untagged) != 0:
+            self.tag_untagged(untagged, key)
 
         ProcessTrackThumbnails()
         ProcessAlbumColors()
         ProcessArtistColors()
 
+        tried_to_download_new_images = False
+
+        if Ping()():
+            tried_to_download_new_images = True
+            try:
+                CheckArtistImages()
+            except (RequestConnectionError, ReadTimeout):
+                log.error(
+                    "Internet connection lost. Downloading artist images stopped."
+                )
+        else:
+            log.warning(
+                f"No internet connection. Downloading artist images halted for {settings.get_scan_sleep_time()} seconds."
+            )
+
+        # Re-process the new artist images.
+        if tried_to_download_new_images:
+            ProcessArtistColors()
+
+    @staticmethod
+    def remove_modified(tracks: list[Track]):
+        modified = [
+            t.filepath for t in tracks if t.last_mod != os.path.getmtime(t.filepath)
+        ]
+
+        TrackStore.remove_tracks_by_filepaths(modified)
+        remove_tracks_by_filepaths(modified)
+
     @staticmethod
     def filter_untagged(tracks: list[Track], files: list[str]):
         tagged_files = [t.filepath for t in tracks]
@@ -120,6 +152,7 @@ class Populate:
                 log.warning("Could not read file: %s", file)
 
         if len(tagged_tracks) > 0:
+            log.info("Adding %s tracks to database", len(tagged_tracks))
             insert_many_tracks(tagged_tracks)
 
         log.info("Added %s/%s tracks", tagged_count, len(untagged))
diff --git a/app/lib/taglib.py b/app/lib/taglib.py
index d0f5cab..492f244 100644
--- a/app/lib/taglib.py
+++ b/app/lib/taglib.py
@@ -64,17 +64,18 @@ def extract_thumb(filepath: str, webp_path: str) -> bool:
     return False
 
 
-def extract_date(date_str: str | None, filepath: str) -> int:
+def extract_date(date_str: str | None, timestamp: float) -> int:
     try:
         return int(date_str.split("-")[0])
     except:  # pylint: disable=bare-except
-        # TODO: USE FILEPATH LAST-MOD DATE instead of current date
-        return datetime.date.today().today().year
+        print(datetime.datetime.fromtimestamp(timestamp).year)
+        return datetime.datetime.fromtimestamp(timestamp).year
 
 
 def get_tags(filepath: str):
     filetype = filepath.split(".")[-1]
     filename = (filepath.split("/")[-1]).replace(f".{filetype}", "")
+    last_mod = os.path.getmtime(filepath)
 
     try:
         tags = TinyTag.get(filepath)
@@ -141,9 +142,10 @@ def get_tags(filepath: str):
     tags.image = f"{tags.albumhash}.webp"
     tags.folder = win_replace_slash(os.path.dirname(filepath))
 
-    tags.date = extract_date(tags.year, filepath)
+    tags.date = extract_date(tags.year, last_mod)
     tags.filepath = win_replace_slash(filepath)
     tags.filetype = filetype
+    tags.last_mod = last_mod
 
     tags = tags.__dict__
 
diff --git a/app/lib/trackslib.py b/app/lib/trackslib.py
index caa00ca..a423185 100644
--- a/app/lib/trackslib.py
+++ b/app/lib/trackslib.py
@@ -16,4 +16,4 @@ def validate_tracks() -> None:
     for track in tqdm(TrackStore.tracks, desc="Checking for deleted tracks"):
         if not os.path.exists(track.filepath):
             TrackStore.tracks.remove(track)
-            tdb.remove_track_by_filepath(track.filepath)
+            tdb.remove_tracks_by_filepaths(track.filepath)
diff --git a/app/lib/watchdogg.py b/app/lib/watchdogg.py
index 93625be..bb73dc7 100644
--- a/app/lib/watchdogg.py
+++ b/app/lib/watchdogg.py
@@ -8,18 +8,16 @@ import time
 from watchdog.events import PatternMatchingEventHandler
 from watchdog.observers import Observer
 
-from app.logger import log
-from app.lib.taglib import get_tags
-from app.models import Artist, Track
 from app import settings
-
+from app.db.sqlite.settings import SettingsSQLMethods as sdb
 from app.db.sqlite.tracks import SQLiteManager
 from app.db.sqlite.tracks import SQLiteTrackMethods as db
-from app.db.sqlite.settings import SettingsSQLMethods as sdb
-
-from app.store.tracks import TrackStore
+from app.lib.taglib import get_tags
+from app.logger import log
+from app.models import Artist, Track
 from app.store.albums import AlbumStore
 from app.store.artists import ArtistStore
+from app.store.tracks import TrackStore
 
 
 class Watcher:
@@ -133,7 +131,7 @@ def add_track(filepath: str) -> None:
     """
     # remove the track if it already exists
     TrackStore.remove_track_by_filepath(filepath)
-    db.remove_track_by_filepath(filepath)
+    db.remove_tracks_by_filepaths(filepath)
 
     # add the track to the database and store.
     tags = get_tags(filepath)
@@ -142,7 +140,6 @@ def add_track(filepath: str) -> None:
         return
 
     with SQLiteManager() as cur:
-        db.remove_track_by_filepath(tags["filepath"])
         db.insert_one_track(tags, cur)
 
     track = Track(**tags)
@@ -168,7 +165,7 @@ def remove_track(filepath: str) -> None:
     except IndexError:
         return
 
-    db.remove_track_by_filepath(filepath)
+    db.remove_tracks_by_filepaths(filepath)
     TrackStore.remove_track_by_filepath(filepath)
 
     empty_album = TrackStore.count_tracks_by_hash(track.albumhash) > 0
diff --git a/app/migrations/__init__.py b/app/migrations/__init__.py
index d3a1ab3..ac7ae21 100644
--- a/app/migrations/__init__.py
+++ b/app/migrations/__init__.py
@@ -24,6 +24,10 @@ def apply_migrations():
     userdb_version = MigrationManager.get_userdatadb_postinit_version()
     maindb_version = MigrationManager.get_maindb_postinit_version()
 
+    # No migrations to run
+    if userdb_version == 0 and maindb_version == 0:
+        return
+
     for migration in main_db_migrations:
         if migration.version > maindb_version:
             log.info("Running new MAIN-DB post-init migration: %s", migration.name)
diff --git a/app/migrations/__preinit/__init__.py b/app/migrations/__preinit/__init__.py
index e748779..0e5c5f3 100644
--- a/app/migrations/__preinit/__init__.py
+++ b/app/migrations/__preinit/__init__.py
@@ -28,6 +28,10 @@ def run_preinit_migrations():
     except OperationalError:
         userdb_version = 0
 
+    # No migrations to run
+    if userdb_version == 0:
+        return
+
     for migration in all_preinits:
         if migration.version > userdb_version:
             log.warn("Running new pre-init migration: %s", migration.name)
diff --git a/app/models/track.py b/app/models/track.py
index ff4f519..6c411c9 100644
--- a/app/models/track.py
+++ b/app/models/track.py
@@ -19,12 +19,13 @@ class Track:
     artist: str | list[ArtistMinimal]
     bitrate: int
     copyright: str
-    date: str
+    date: int
     disc: int
     duration: int
     filepath: str
     folder: str
     genre: str | list[str]
+    last_mod: float
     title: str
     track: int
     trackhash: str
diff --git a/app/periodic_scan.py b/app/periodic_scan.py
index f884b39..41c744e 100644
--- a/app/periodic_scan.py
+++ b/app/periodic_scan.py
@@ -2,11 +2,8 @@
 This module contains functions for the server
 """
 import time
-from requests import ReadTimeout
-from requests import ConnectionError as RequestConnectionError
 
 from app.logger import log
-from app.lib.artistlib import CheckArtistImages
 from app.lib.populate import Populate, PopulateCancelledError
 
 from app.utils.generators import get_random_str
@@ -30,13 +27,5 @@ def run_periodic_scans():
         except PopulateCancelledError:
             pass
 
-        if Ping()():
-            try:
-                CheckArtistImages()
-            except (RequestConnectionError, ReadTimeout):
-                log.error(
-                    "Internet connection lost. Downloading artist images stopped."
-                )
-
         sleep_time = get_scan_sleep_time()
         time.sleep(sleep_time)
diff --git a/app/store/tracks.py b/app/store/tracks.py
index 60925c6..2fe4db1 100644
--- a/app/store/tracks.py
+++ b/app/store/tracks.py
@@ -52,6 +52,18 @@ class TrackStore:
                 cls.tracks.remove(track)
                 break
 
+    @classmethod
+    def remove_tracks_by_filepaths(cls, filepaths: list[str]):
+        """
+        Removes multiple tracks from the store by their filepaths.
+        """
+
+        paths_str = "~".join(filepaths)
+
+        for track in cls.tracks:
+            if track.filepath in paths_str:
+                cls.tracks.remove(track)
+
     @classmethod
     def remove_tracks_by_dir_except(cls, dirs: list[str]):
         """Removes all tracks not in the root directories."""
@@ -98,7 +110,9 @@ class TrackStore:
                 track.is_favorite = False
 
     @classmethod
-    def append_track_artists(cls, albumhash: str, artists: list[str], new_album_title: str):
+    def append_track_artists(
+        cls, albumhash: str, artists: list[str], new_album_title: str
+    ):
         tracks = cls.get_tracks_by_albumhash(albumhash)
 
         for track in tracks:
diff --git a/app/utils/hashing.py b/app/utils/hashing.py
index b976e22..6492784 100644
--- a/app/utils/hashing.py
+++ b/app/utils/hashing.py
@@ -3,7 +3,7 @@ import hashlib
 from unidecode import unidecode
 
 
-def create_hash(*args: str, decode=False, limit=7) -> str:
+def create_hash(*args: str, decode=False, limit=10) -> str:
     """
     Creates a simple hash for an album
     """
@@ -19,7 +19,7 @@ def create_hash(*args: str, decode=False, limit=7) -> str:
     return str_[-limit:]
 
 
-def create_folder_hash(*args: str, limit=7) -> str:
+def create_folder_hash(*args: str, limit=10) -> str:
     """
     Creates a simple hash for an album
     """