From 13475b0630e9749a500e4c3c7130864fc23aee2d Mon Sep 17 00:00:00 2001 From: mungai-njoroge Date: Tue, 29 Aug 2023 20:04:30 +0300 Subject: [PATCH] rewrite remove duplicates to support removing duplicates in albums tracks efficiently + remove flags added to client settings page + misc --- README.md | 13 +++++++++++++ app/api/album.py | 8 ++------ app/api/settings.py | 14 +++++++------- app/arg_handler.py | 35 ++++------------------------------ app/db/sqlite/settings.py | 15 ++++++++------- app/lib/albumslib.py | 6 ++++++ app/models/album.py | 18 ++++++++--------- app/models/track.py | 16 ++++++++++------ app/periodic_scan.py | 4 ++-- app/print_help.py | 11 +++-------- app/serializers/track.py | 2 +- app/settings.py | 23 ++++++++++++++-------- app/start_info_logger.py | 7 ++++--- app/store/tracks.py | 2 +- app/utils/parsers.py | 4 ++-- app/utils/remove_duplicates.py | 35 ++++++++++++++++++++++++++++++---- 16 files changed, 118 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 38abd90..0929b01 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,19 @@ Usage: swingmusic [options] | | | | --build | Build the application (in development) | + +| Option | Short | Description | +|--------------------------|-------|-------------------------------------------------------| +| `--help` | `-h` | Show help message | +| `--version` | `-v` | Show the app version | +| `--host` | | Set the host | +| `--port` | | Set the port | +| `--config` | | Set the config path | +| `--no-periodic-scan`| `-nps` | Disable periodic scan | +| `--scan-interval` | `-psi` | Set the periodic scan interval in seconds. Default is 300 seconds (5 minutes) | +| `--build` | | Build the application (in development) | + + To stream your music across your local network, use the `--host` flag to run the app in all ports. Like this: ```sh diff --git a/app/api/album.py b/app/api/album.py index 1feb022..b0bf059 100644 --- a/app/api/album.py +++ b/app/api/album.py @@ -3,8 +3,6 @@ Contains all the album routes. """ import random -from dataclasses import asdict -from typing import Any from flask import Blueprint, request @@ -18,7 +16,6 @@ from app.serializers.track import serialize_track from app.store.albums import AlbumStore from app.store.tracks import TrackStore from app.utils.hashing import create_hash -from app.utils.remove_duplicates import remove_duplicates get_albums_by_albumartist = adb.get_albums_by_albumartist check_is_fav = favdb.check_is_favorite @@ -67,13 +64,12 @@ def get_album_tracks_and_info(): return list(genres) album.genres = get_album_genres(tracks) - tracks = remove_duplicates(tracks) - album.count = len(tracks) + album.get_date_from_tracks(tracks) try: - album.duration = sum((t.duration for t in tracks)) + album.duration = sum(t.duration for t in tracks) except AttributeError: album.duration = 0 diff --git a/app/api/settings.py b/app/api/settings.py index f6190d7..e0a4fd7 100644 --- a/app/api/settings.py +++ b/app/api/settings.py @@ -4,7 +4,7 @@ from app.db.sqlite.settings import SettingsSQLMethods as sdb from app.lib import populate from app.lib.watchdogg import Watcher as WatchDog from app.logger import log -from app.settings import ParserFlags, Paths, set_flag +from app.settings import Paths, SessionVarKeys, set_flag from app.store.albums import AlbumStore from app.store.artists import ArtistStore from app.store.tracks import TrackStore @@ -143,12 +143,12 @@ def get_root_dirs(): # maps settings to their parser flags mapp = { - "artist_separators": ParserFlags.ARTIST_SEPARATORS, - "extract_feat": ParserFlags.EXTRACT_FEAT, - "remove_prod": ParserFlags.REMOVE_PROD, - "clean_album_title": ParserFlags.CLEAN_ALBUM_TITLE, - "remove_remaster": ParserFlags.REMOVE_REMASTER_FROM_TRACK, - "merge_albums": ParserFlags.MERGE_ALBUM_VERSIONS, + "artist_separators": SessionVarKeys.ARTIST_SEPARATORS, + "extract_feat": SessionVarKeys.EXTRACT_FEAT, + "remove_prod": SessionVarKeys.REMOVE_PROD, + "clean_album_title": SessionVarKeys.CLEAN_ALBUM_TITLE, + "remove_remaster": SessionVarKeys.REMOVE_REMASTER_FROM_TRACK, + "merge_albums": SessionVarKeys.MERGE_ALBUM_VERSIONS, } diff --git a/app/arg_handler.py b/app/arg_handler.py index 8a07cfc..0d95177 100644 --- a/app/arg_handler.py +++ b/app/arg_handler.py @@ -3,14 +3,14 @@ Handles arguments passed to the program. """ import os.path import sys - from configparser import ConfigParser + import PyInstaller.__main__ as bundler from app import settings +from app.logger import log from app.print_help import HELP_MESSAGE from app.utils.wintools import is_windows -from app.logger import log from app.utils.xdg_utils import get_xdg_config_dir # from app.api.imgserver import set_app_dir @@ -29,10 +29,6 @@ class HandleArgs: self.handle_port() self.handle_config_path() - self.handle_no_feat() - self.handle_remove_prod() - self.handle_cleaning_albums() - self.handle_cleaning_tracks() self.handle_periodic_scan() self.handle_periodic_scan_interval() @@ -122,31 +118,10 @@ class HandleArgs: settings.Paths.set_config_dir(get_xdg_config_dir()) - @staticmethod - def handle_no_feat(): - # if ArgsEnum.no_feat in ARGS: - if any((a in ARGS for a in ALLARGS.show_feat)): - settings.FromFlags.EXTRACT_FEAT = False - - @staticmethod - def handle_remove_prod(): - if any((a in ARGS for a in ALLARGS.show_prod)): - settings.FromFlags.REMOVE_PROD = False - - @staticmethod - def handle_cleaning_albums(): - if any((a in ARGS for a in ALLARGS.dont_clean_albums)): - settings.FromFlags.CLEAN_ALBUM_TITLE = False - - @staticmethod - def handle_cleaning_tracks(): - if any((a in ARGS for a in ALLARGS.dont_clean_tracks)): - settings.FromFlags.REMOVE_REMASTER_FROM_TRACK = False - @staticmethod def handle_periodic_scan(): if any((a in ARGS for a in ALLARGS.no_periodic_scan)): - settings.FromFlags.DO_PERIODIC_SCANS = False + settings.SessionVars.DO_PERIODIC_SCANS = False @staticmethod def handle_periodic_scan_interval(): @@ -161,8 +136,6 @@ class HandleArgs: print("ERROR: Interval not specified") sys.exit(0) - # psi = 0 - try: psi = int(interval) except ValueError: @@ -173,7 +146,7 @@ class HandleArgs: print("WADAFUCK ARE YOU TRYING?") sys.exit(0) - settings.FromFlags.PERIODIC_SCAN_INTERVAL = psi + settings.SessionVars.PERIODIC_SCAN_INTERVAL = psi @staticmethod def handle_help(): diff --git a/app/db/sqlite/settings.py b/app/db/sqlite/settings.py index c27b7cc..9c1a490 100644 --- a/app/db/sqlite/settings.py +++ b/app/db/sqlite/settings.py @@ -1,8 +1,9 @@ from pprint import pprint from typing import Any + from app.db.sqlite.utils import SQLiteManager +from app.settings import SessionVars from app.utils.wintools import win_replace_slash -from app.settings import FromFlags class SettingsSQLMethods: @@ -138,11 +139,11 @@ def load_settings(): separators = db_separators.split(",") separators = set(separators) - FromFlags.ARTIST_SEPARATORS = separators + SessionVars.ARTIST_SEPARATORS = separators # boolean settings - FromFlags.EXTRACT_FEAT = bool(s[1]) - FromFlags.REMOVE_PROD = bool(s[2]) - FromFlags.CLEAN_ALBUM_TITLE = bool(s[3]) - FromFlags.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) - FromFlags.MERGE_ALBUM_VERSIONS = bool(s[5]) + SessionVars.EXTRACT_FEAT = bool(s[1]) + SessionVars.REMOVE_PROD = bool(s[2]) + SessionVars.CLEAN_ALBUM_TITLE = bool(s[3]) + SessionVars.REMOVE_REMASTER_FROM_TRACK = bool(s[4]) + SessionVars.MERGE_ALBUM_VERSIONS = bool(s[5]) diff --git a/app/lib/albumslib.py b/app/lib/albumslib.py index da2b3ae..a0817fc 100644 --- a/app/lib/albumslib.py +++ b/app/lib/albumslib.py @@ -30,6 +30,12 @@ def validate_albums(): AlbumStore.remove_album(album) bar() +def remove_duplicate_on_merge_versions(tracks: list[Track]) -> list[Track]: + """ + Removes duplicate tracks when merging versions of the same album. + """ + + pass def sort_by_track_no(tracks: list[Track]) -> list[dict[str, Any]]: tracks = [asdict(t) for t in tracks] diff --git a/app/models/album.py b/app/models/album.py index 97a8204..91606fc 100644 --- a/app/models/album.py +++ b/app/models/album.py @@ -1,13 +1,13 @@ import dataclasses -from dataclasses import dataclass import datetime +from dataclasses import dataclass + +from app.settings import SessionVarKeys, get_flag -from .track import Track -from .artist import Artist from ..utils.hashing import create_hash -from ..utils.parsers import parse_feat_from_title, get_base_title_and_versions - -from app.settings import get_flag, ParserFlags +from ..utils.parsers import get_base_title_and_versions, parse_feat_from_title +from .artist import Artist +from .track import Track @dataclass(slots=True) @@ -44,7 +44,7 @@ class Album: self.image = self.albumhash + ".webp" # Fetch album artists from title - if get_flag(ParserFlags.EXTRACT_FEAT): + if get_flag(SessionVarKeys.EXTRACT_FEAT): featured, self.title = parse_feat_from_title(self.title) if len(featured) > 0: @@ -58,8 +58,8 @@ class Album: TrackStore.append_track_artists(self.albumhash, featured, self.title) # Handle album version data - if get_flag(ParserFlags.CLEAN_ALBUM_TITLE): - get_versions = not get_flag(ParserFlags.MERGE_ALBUM_VERSIONS) + if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): + get_versions = not get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS) self.title, self.versions = get_base_title_and_versions( self.title, get_versions=get_versions diff --git a/app/models/track.py b/app/models/track.py index 42b2a98..f3104d7 100644 --- a/app/models/track.py +++ b/app/models/track.py @@ -1,6 +1,6 @@ from dataclasses import dataclass -from app.settings import ParserFlags, get_flag +from app.settings import SessionVarKeys, get_flag from app.utils.hashing import create_hash from app.utils.parsers import ( clean_title, @@ -41,6 +41,10 @@ class Track: artist_hashes: str = "" is_favorite: bool = False + # temporary attributes + _pos: int = 0 # for sorting tracks by disc and track number + _ati: str = "" # (album track identifier) for removing duplicates when merging album versions + og_title: str = "" og_album: str = "" @@ -53,31 +57,31 @@ class Track: artists = split_artists(self.artists) new_title = self.title - if get_flag(ParserFlags.EXTRACT_FEAT): + if get_flag(SessionVarKeys.EXTRACT_FEAT): featured, new_title = parse_feat_from_title(self.title) original_lower = "-".join([create_hash(a) for a in artists]) artists.extend( [a for a in featured if create_hash(a) not in original_lower] ) - if get_flag(ParserFlags.REMOVE_PROD): + if get_flag(SessionVarKeys.REMOVE_PROD): new_title = remove_prod(new_title) # if track is a single if self.og_title == self.album: self.rename_album(new_title) - if get_flag(ParserFlags.REMOVE_REMASTER_FROM_TRACK): + if get_flag(SessionVarKeys.REMOVE_REMASTER_FROM_TRACK): new_title = clean_title(new_title) self.title = new_title - if get_flag(ParserFlags.CLEAN_ALBUM_TITLE): + if get_flag(SessionVarKeys.CLEAN_ALBUM_TITLE): self.album, _ = get_base_title_and_versions( self.album, get_versions=False ) - if get_flag(ParserFlags.MERGE_ALBUM_VERSIONS): + if get_flag(SessionVarKeys.MERGE_ALBUM_VERSIONS): self.recreate_albumhash() self.artist_hashes = "-".join(create_hash(a, decode=True) for a in artists) diff --git a/app/periodic_scan.py b/app/periodic_scan.py index b0d338b..787c3a7 100644 --- a/app/periodic_scan.py +++ b/app/periodic_scan.py @@ -4,7 +4,7 @@ This module contains functions for the server import time from app.lib.populate import Populate, PopulateCancelledError -from app.settings import ParserFlags, get_flag, get_scan_sleep_time +from app.settings import SessionVarKeys, get_flag, get_scan_sleep_time from app.utils.generators import get_random_str from app.utils.threading import background @@ -20,7 +20,7 @@ def run_periodic_scans(): run_periodic_scan = True while run_periodic_scan: - run_periodic_scan = get_flag(ParserFlags.DO_PERIODIC_SCANS) + run_periodic_scan = get_flag(SessionVarKeys.DO_PERIODIC_SCANS) try: Populate(instance_key=get_random_str()) diff --git a/app/print_help.py b/app/print_help.py index a9dd79a..cda3581 100644 --- a/app/print_help.py +++ b/app/print_help.py @@ -10,19 +10,14 @@ Usage: swingmusic [options] Options: {', '.join(args.help)}: Show this help message {', '.join(args.version)}: Show the app version - + {args.host}: Set the host {args.port}: Set the port {args.config}: Set the config path - - {', '.join(args.show_feat)}: Do not extract featured artists from the song title - {', '.join(args.show_prod)}: Do not hide producers in the song title - {', '.join(args.dont_clean_albums)}: Don't clean album titles. Cleaning is done by removing information in - parentheses and showing it separately - {', '.join(args.dont_clean_tracks)}: Don't remove remaster information from track titles + {', '.join(args.no_periodic_scan)}: Disable periodic scan {', '.join(args.periodic_scan_interval)}: Set the periodic scan interval in seconds. Default is 300 seconds (5 minutes) - + {args.build}: Build the application (in development) """ diff --git a/app/serializers/track.py b/app/serializers/track.py index 933e37d..bb195f7 100644 --- a/app/serializers/track.py +++ b/app/serializers/track.py @@ -21,7 +21,7 @@ def serialize_track(track: Track, to_remove: set = {}, remove_disc=True) -> dict props.remove("disc") props.remove("track") - props.update(key for key in album_dict.keys() if key.startswith("is_")) + props.update(key for key in album_dict.keys() if key.startswith(("is_", "_"))) props.remove("is_favorite") for key in props: diff --git a/app/settings.py b/app/settings.py index 8bb5532..d4d601c 100644 --- a/app/settings.py +++ b/app/settings.py @@ -150,7 +150,11 @@ class ALLARGS: version = ("--version", "-v") -class FromFlags: +class SessionVars: + """ + Variables that can be altered per session. + """ + EXTRACT_FEAT = True """ Whether to extract the featured artists from the song title. @@ -165,14 +169,17 @@ class FromFlags: REMOVE_REMASTER_FROM_TRACK = True DO_PERIODIC_SCANS = True - PERIODIC_SCAN_INTERVAL = 300 # seconds + PERIODIC_SCAN_INTERVAL = 600 # 10 minutes + """ + The interval between periodic scans in seconds. + """ MERGE_ALBUM_VERSIONS = False ARTIST_SEPARATORS = set() # TODO: Find a way to eliminate this class without breaking typings -class ParserFlags: +class SessionVarKeys: EXTRACT_FEAT = "EXTRACT_FEAT" REMOVE_PROD = "REMOVE_PROD" CLEAN_ALBUM_TITLE = "CLEAN_ALBUM_TITLE" @@ -183,16 +190,16 @@ class ParserFlags: ARTIST_SEPARATORS = "ARTIST_SEPARATORS" -def get_flag(flag: ParserFlags) -> bool: - return getattr(FromFlags, flag) +def get_flag(key: SessionVarKeys) -> bool: + return getattr(SessionVars, key) -def set_flag(flag: ParserFlags, value: Any): - setattr(FromFlags, flag, value) +def set_flag(key: SessionVarKeys, value: Any): + setattr(SessionVars, key, value) def get_scan_sleep_time() -> int: - return FromFlags.PERIODIC_SCAN_INTERVAL + return SessionVars.PERIODIC_SCAN_INTERVAL class TCOLOR: diff --git a/app/start_info_logger.py b/app/start_info_logger.py index 574976a..3260f63 100644 --- a/app/start_info_logger.py +++ b/app/start_info_logger.py @@ -1,6 +1,7 @@ import os -from app.settings import TCOLOR, Release, FLASKVARS, Paths, get_flag, ParserFlags +from app.settings import (FLASKVARS, TCOLOR, Paths, Release, SessionVarKeys, + get_flag) from app.utils.network import get_ip @@ -30,11 +31,11 @@ def log_startup_info(): to_print = [ [ "Extract featured artists from titles", - get_flag(ParserFlags.EXTRACT_FEAT) + get_flag(SessionVarKeys.EXTRACT_FEAT) ], [ "Remove prod. from titles", - get_flag(ParserFlags.REMOVE_PROD) + get_flag(SessionVarKeys.REMOVE_PROD) ] ] diff --git a/app/store/tracks.py b/app/store/tracks.py index 15d04fd..8828828 100644 --- a/app/store/tracks.py +++ b/app/store/tracks.py @@ -157,7 +157,7 @@ class TrackStore: Returns all tracks matching the given album hash. """ tracks = [t for t in cls.tracks if t.albumhash == album_hash] - return remove_duplicates(tracks) + return remove_duplicates(tracks, is_album_tracks=True) @classmethod def get_tracks_by_artisthash(cls, artisthash: str): diff --git a/app/utils/parsers.py b/app/utils/parsers.py index 7e77657..ba8b128 100644 --- a/app/utils/parsers.py +++ b/app/utils/parsers.py @@ -1,14 +1,14 @@ import re from app.enums.album_versions import AlbumVersionEnum -from app.settings import get_flag, ParserFlags +from app.settings import SessionVarKeys, get_flag def split_artists(src: str): """ Splits a string of artists into a list of artists. """ - separators: set = get_flag(ParserFlags.ARTIST_SEPARATORS) + separators: set = get_flag(SessionVarKeys.ARTIST_SEPARATORS) separators = separators.union({","}) for sep in separators: diff --git a/app/utils/remove_duplicates.py b/app/utils/remove_duplicates.py index 461dd21..f32afc0 100644 --- a/app/utils/remove_duplicates.py +++ b/app/utils/remove_duplicates.py @@ -2,21 +2,48 @@ from collections import defaultdict from operator import attrgetter from app.models import Track +from app.utils.hashing import create_hash -def remove_duplicates(tracks: list[Track]) -> list[Track]: +def remove_duplicates(tracks: list[Track], is_album_tracks=False) -> list[Track]: """ Remove duplicates from a list of Track objects based on the trackhash attribute. + Retain objects with the highest bitrate. """ - hash_to_tracks = defaultdict(list) + tracks_dict = defaultdict(list) + # if is_album_tracks, sort by disc and track number + if is_album_tracks: + for t in tracks: + # _pos is used for sorting tracks by disc and track number + t._pos = int(f"{t.disc}{str(t.track).zfill(3)}") + + # _ati is used to remove duplicates when merging album versions + t._ati = f"{t._pos}{create_hash(t.title)}" + + # create groups of tracks with the same _ati + for track in tracks: + tracks_dict[track._ati].append(track) + + tracks = [] + + # pick the track with max bitrate for each group + for track_group in tracks_dict.values(): + max_bitrate_track = max(track_group, key=attrgetter("bitrate")) + tracks.append(max_bitrate_track) + + return sorted(tracks, key=lambda t: t._pos) + + # else, sort by trackhash for track in tracks: - hash_to_tracks[track.trackhash].append(track) + # create groups of tracks with the same trackhash + tracks_dict[track.trackhash].append(track) tracks = [] - for track_group in hash_to_tracks.values(): + # pick the track with max bitrate for each trackhash group + for track_group in tracks_dict.values(): max_bitrate_track = max(track_group, key=attrgetter("bitrate")) tracks.append(max_bitrate_track)