tcsenpai/swingmusic
1
0
Fork 0
mirror of https://github.com/tcsenpai/swingmusic.git synced 2025-06-06 03:05:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

rewrite migrations

Browse Source 
+ delete older migrations ... oops
+ change migratrions from "migrations" to "dbmigrations"
+ restructure migrations, order them based on release version
+ add a utils/decorators.py file with a coroutine decorator
This commit is contained in:
mungai-njoroge 2023-07-29 06:46:28 +03:00
parent a0c51d5f82
commit 93de3d2f0c
19 changed files with 327 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
app/api/playlist.py
View File

@ -26,7 +26,6 @@ count_playlist_by_name = PL.count_playlist_by_name
get_all_playlists = PL.get_all_playlists
get_playlist_by_id = PL.get_playlist_by_id
tracks_to_playlist = PL.add_tracks_to_playlist
add_artist_to_playlist = PL.add_artists_to_playlist
update_playlist = PL.update_playlist
delete_playlist = PL.delete_playlist
remove_image = PL.remove_banner
@ -101,7 +100,6 @@ def send_all_playlists():
def insert_playlist(name: str):
playlist = {
"artisthashes": json.dumps([]),
"image": None,
"last_updated": create_new_date(),
"name": name,
@ -154,7 +152,6 @@ def add_track_to_playlist(playlist_id: str):
if insert_count == 0:
return {"error": "Track already exists in playlist"}, 409
add_artist_to_playlist(int(playlist_id), [trackhash])
PL.update_last_updated(int(playlist_id))
return {"msg": "Done"}, 200
@ -218,7 +215,6 @@ def update_playlist_info(playlistid: str):
playlist = {
"id": int(playlistid),
"artisthashes": json.dumps([]),
"image": db_playlist.image,
"last_updated": create_new_date(),
"name": str(data.get("name")).strip(),
@ -360,19 +356,12 @@ def save_folder_as_folder():
if len(trackhashes) == 0:
return {"error": "No tracks found in folder"}, 404
artisthashes = set()
for t in tracks:
for a in t.artist:
artisthashes.add(a.artisthash)
playlist = insert_playlist(name)
if playlist is None:
return {"error": "Playlist could not be created"}, 500
tracks_to_playlist(playlist.id, trackhashes)
PL.add_artists_to_playlist(playlist.id, artisthashes=artisthashes)
PL.update_last_updated(playlist.id)
return {"playlist_id": playlist.id}, 201

64
app/db/sqlite/migrations.py
View File

@ -6,72 +6,26 @@ from app.db.sqlite.utils import SQLiteManager
class MigrationManager:
all_get_sql = "SELECT * FROM migrations"
_base = "UPDATE migrations SET"
_end = "= ? WHERE id = 1"
pre_init_set_sql = f"{_base} pre_init_version {_end}"
post_init_set_sql = f"{_base} post_init_version {_end}"
@classmethod
def get_preinit_version(cls) -> int:
@staticmethod
def get_version() -> int:
"""
Returns the latest userdata pre-init database version.
Returns the latest userdata database version.
"""
sql = "SELECT * FROM dbmigrations"
with SQLiteManager() as cur:
cur.execute(cls.all_get_sql)
cur.execute(sql)
ver = int(cur.fetchone()[1])
cur.close()
return ver
@classmethod
def get_maindb_postinit_version(cls) -> int:
"""
Returns the latest maindb post-init database version.
"""
with SQLiteManager() as cur:
cur.execute(cls.all_get_sql)
ver = int(cur.fetchone()[2])
cur.close()
return ver
@classmethod
def get_userdatadb_postinit_version(cls) -> int:
"""
Returns the latest userdata post-init database version.
"""
with SQLiteManager(userdata_db=True) as cur:
cur.execute(cls.all_get_sql)
ver = cur.fetchone()[2]
cur.close()
return ver
# 👇 Setters 👇
@classmethod
def set_preinit_version(cls, version: int):
@staticmethod
def set_version(version: int):
"""
Sets the userdata pre-init database version.
"""
sql = "UPDATE dbmigrations SET version = ? WHERE id = 1"
with SQLiteManager() as cur:
cur.execute(cls.pre_init_set_sql, (version,))
cur.execute(sql, (version,))
cur.close()
@classmethod
def set_maindb_postinit_version(cls, version: int):
"""
Sets the maindb post-init database version.
"""
with SQLiteManager() as cur:
cur.execute(cls.post_init_set_sql, (version,))
@classmethod
def set_userdatadb_postinit_version(cls, version: int):
"""
Sets the userdata post-init database version.
"""
with SQLiteManager(userdata_db=True) as cur:
cur.execute(cls.post_init_set_sql, (version,))

25
app/db/sqlite/playlists.py
View File

@ -18,13 +18,12 @@ class SQLitePlaylistMethods:
# banner_pos,
# has_gif,
sql = """INSERT INTO playlists(
artisthashes,
image,
last_updated,
name,
settings,
trackhashes
) VALUES(:artisthashes, :image, :last_updated, :name, :settings, :trackhashes)
) VALUES(:image, :last_updated, :name, :settings, :trackhashes)
"""
playlist = OrderedDict(sorted(playlist.items()))
@ -124,26 +123,6 @@ class SQLitePlaylistMethods:
def add_tracks_to_playlist(cls, playlist_id: int, trackhashes: list[str]):
return cls.add_item_to_json_list(playlist_id, "trackhashes", trackhashes)
@classmethod
@background
def add_artists_to_playlist(
cls,
playlist_id: int,
trackhashes: list[str] = [],
artisthashes: set[str] = None,
):
if not artisthashes:
track = [SQLiteTrackMethods.get_track_by_trackhash(t) for t in trackhashes]
tracks = [t for t in track if t is not None]
artisthashes: set[str] = set() # type: ignore
for track in tracks:
for a in track.artist:
artisthashes.add(a.artisthash)
cls.add_item_to_json_list(playlist_id, "artisthashes", artisthashes)
@staticmethod
def update_playlist(playlist_id: int, playlist: dict):
sql = """UPDATE playlists SET
@ -156,10 +135,8 @@ class SQLitePlaylistMethods:
del playlist["id"]
del playlist["trackhashes"]
del playlist["artisthashes"]
playlist["settings"] = json.dumps(playlist["settings"])
playlist = OrderedDict(sorted(playlist.items()))
params = (*playlist.values(), playlist_id)

18
app/db/sqlite/queries.py
View File

@ -5,10 +5,10 @@ This file contains the SQL queries to create the database tables.
# banner_pos integer NOT NULL,
# has_gif integer,
CREATE_USERDATA_TABLES = """
CREATE TABLE IF NOT EXISTS playlists (
id integer PRIMARY KEY,
artisthashes text,
image text,
last_updated text not null,
name text not null,
@ -52,10 +52,10 @@ CREATE TABLE IF NOT EXISTS tracks (
filepath text NOT NULL,
folder text NOT NULL,
genre text,
last_mod float NOT NULL,
title text NOT NULL,
track integer NOT NULL,
trackhash text NOT NULL,
last_mod float NOT NULL,
UNIQUE (filepath)
);
@ -81,14 +81,16 @@ CREATE TABLE IF NOT EXISTS folders (
);
"""
# changed from migrations to dbmigrations in v1.3.0
# to avoid conflicts with the previous migrations.
CREATE_MIGRATIONS_TABLE = """
CREATE TABLE IF NOT EXISTS migrations (
CREATE TABLE IF NOT EXISTS dbmigrations (
id integer PRIMARY KEY,
pre_init_version integer NOT NULL DEFAULT 0,
post_init_version integer NOT NULL DEFAULT 0
version integer NOT NULL DEFAULT 0
);
INSERT INTO migrations (pre_init_version, post_init_version)
SELECT 0, 0
WHERE NOT EXISTS (SELECT 1 FROM migrations);
INSERT INTO dbmigrations (version)
SELECT 0
WHERE NOT EXISTS (SELECT 1 FROM dbmigrations);
"""

2
app/db/sqlite/utils.py
View File

@ -2,6 +2,7 @@
Helper functions for use with the SQLite database.
"""
from pprint import pprint
import sqlite3
from sqlite3 import Connection, Cursor
import time
@ -45,6 +46,7 @@ def tuple_to_playlist(playlist: tuple):
"""
Takes a tuple and returns a Playlist object
"""
pprint(playlist)
return Playlist(*playlist)

47
app/migrations/__init__.py
View File

@ -6,14 +6,25 @@ Reads and applies the latest database migrations.
PLEASE NOTE: OLDER MIGRATIONS CAN NEVER BE DELETED.
ONLY MODIFY OLD MIGRATIONS FOR BUG FIXES OR ENHANCEMENTS ONLY
[TRY NOT TO MODIFY BEHAVIOR, UNLESS YOU KNOW WHAT YOU'RE DOING].
PS: Fuck that! Do what you want.
"""
from app.db.sqlite.migrations import MigrationManager
from app.logger import log
from app.migrations import v1_3_0
from .main import main_db_migrations
from .userdata import userdata_db_migrations
migrations = [
[
# v1.3.0
v1_3_0.RemovePlaylistArtistHashes,
v1_3_0.AddSettingsToPlaylistTable,
v1_3_0.AddLastUpdatedToTrackTable,
v1_3_0.MovePlaylistsAndFavoritesTo10BitHashes,
v1_3_0.RemoveAllTracks,
]
]
def apply_migrations():
@ -21,28 +32,14 @@ def apply_migrations():
Applies the latest database migrations.
"""
userdb_version = MigrationManager.get_userdatadb_postinit_version()
maindb_version = MigrationManager.get_maindb_postinit_version()
version = MigrationManager.get_version()
# No migrations to run
if userdb_version == 0 and maindb_version == 0:
return
if version != len(migrations):
# run migrations after the previous migration version
for migration in migrations[(version - 1) :]:
for m in migration:
log.info("Running new migration: %s", m.name)
m.migrate()
for migration in main_db_migrations:
if migration.version > maindb_version:
log.info("Running new MAIN-DB post-init migration: %s", migration.name)
migration.migrate()
for migration in userdata_db_migrations:
if migration.version > userdb_version:
log.info("Running new USERDATA-DB post-init migration: %s", migration.name)
migration.migrate()
def set_postinit_migration_versions():
"""
Sets the post-init migration versions.
"""
# TODO: Don't forget to remove the zeros below when you add a valid migration 👇.
MigrationManager.set_maindb_postinit_version(0)
MigrationManager.set_userdatadb_postinit_version(0)
# update migration version
MigrationManager.set_version(len(migrations))

45
app/migrations/__preinit/__init__.py
View File

@ -1,45 +0,0 @@
"""
Pre-init migrations are executed before the database is created.
Useful when you need to move files or folders before the database is created.
`Example use cases: Moving files around, dropping tables, etc.`
PLEASE NOTE: OLDER MIGRATIONS CAN NEVER BE DELETED.
ONLY MODIFY OLD MIGRATIONS FOR BUG FIXES OR ENHANCEMENTS ONLY.
[TRY NOT TO MODIFY BEHAVIOR, UNLESS YOU KNOW WHAT YOU'RE DOING].
"""
from sqlite3 import OperationalError
from app.db.sqlite.migrations import MigrationManager
from app.logger import log
from .drop_artist_and_album_color_tables import DropArtistAndAlbumColorTables
from .move_to_xdg_folder import MoveToXdgFolder
all_preinits = [MoveToXdgFolder, DropArtistAndAlbumColorTables]
def run_preinit_migrations():
"""
Runs all pre-init migrations.
"""
try:
userdb_version = MigrationManager.get_preinit_version()
except OperationalError:
userdb_version = 0
# No migrations to run
if userdb_version == 0:
return
for migration in all_preinits:
if migration.version > userdb_version:
log.warn("Running new pre-init migration: %s", migration.name)
migration.migrate()
def set_preinit_migration_versions():
"""
Sets the migration versions.
"""
MigrationManager.set_preinit_version(all_preinits[-1].version)

24
app/migrations/__preinit/drop_artist_and_album_color_tables.py
View File

@ -1,24 +0,0 @@
"""
Another shot at attempting to fix duplicate album and artist color entries.
This release should finally fix the issue. The migration script will now remove
the album and artist color tables and recreate them.
"""
from app.db.sqlite.utils import SQLiteManager
from app.logger import log
class DropArtistAndAlbumColorTables:
version = 2
name = "DropArtistAndAlbumColorTables"
@staticmethod
def migrate():
with SQLiteManager() as cur:
tables = ["artists", "albums"]
for table in tables:
cur.execute(f"DROP TABLE IF EXISTS {table}")
cur.execute("VACUUM")
log.info("Deleted artist and album color data to fix a few bugs. ✅")

49
app/migrations/__preinit/move_to_xdg_folder.py
View File

@ -1,49 +0,0 @@
"""
This migration handles moving the config folder to the XDG standard location.
It also handles moving the userdata and the downloaded artist images to the new location.
"""
import os
import shutil
from app.settings import Paths
from app.logger import log
class MoveToXdgFolder:
version = 1
name = "MoveToXdgFolder"
@staticmethod
def migrate():
old_config_dir = os.path.join(Paths.USER_HOME_DIR, ".swing")
new_config_dir = Paths.get_app_dir()
if not os.path.exists(old_config_dir):
log.info("No old config folder found. Skipping migration.")
return
log.info("Found old config folder: %s", old_config_dir)
old_imgs_dir = os.path.join(old_config_dir, "images")
# move images to new location
if os.path.exists(old_imgs_dir):
shutil.copytree(
old_imgs_dir,
os.path.join(new_config_dir, "images"),
copy_function=shutil.copy2,
dirs_exist_ok=True,
)
log.warn("Moved artist images to: %s", new_config_dir)
# move userdata.db to new location
userdata_db = os.path.join(old_config_dir, "userdata.db")
if os.path.exists(userdata_db):
shutil.copy2(userdata_db, new_config_dir)
log.warn("Moved userdata.db to: %s", new_config_dir)
log.warn("Migration complete. ✅")
# swing.db is not moved because the new code fixes bugs which require
# the whole database to be recreated anyway. (ie. the bug which caused duplicate album and artist color entries)

10
app/migrations/main/__init__.py
View File

@ -1,10 +0,0 @@
"""
Migrations for the main database.
PLEASE NOTE: OLDER MIGRATIONS CAN NEVER BE DELETED.
ONLY MODIFY OLD MIGRATIONS FOR BUG FIXES OR ENHANCEMENTS ONLY
[TRY NOT TO MODIFY BEHAVIOR, UNLESS YOU KNOW WHAT YOU'RE DOING].
"""
main_db_migrations = []

10
app/migrations/userdata/__init__.py
View File

@ -1,10 +0,0 @@
"""
Migrations for the userdata database.
PLEASE NOTE: OLDER MIGRATIONS CAN NEVER BE DELETED.
ONLY MODIFY OLD MIGRATIONS FOR BUG FIXES OR ENHANCEMENTS ONLY
[TRY NOT TO MODIFY BEHAVIOR, UNLESS YOU KNOW WHAT YOU'RE DOING].
"""
userdata_db_migrations = []

257
app/migrations/v1_3_0/__init__.py Normal file
View File

@ -0,0 +1,257 @@
import json
import time
from collections import OrderedDict
from typing import Generator
from app.db.sqlite.utils import SQLiteManager
from app.utils.decorators import coroutine
from app.utils.hashing import create_hash
# playlists table
# ---------------
# 0: id
# 1: banner_pos
# 2: has_gif
# 3: image
# 4: last_updated
# 5: name
# 6: trackhashes
class RemovePlaylistArtistHashes:
"""
This migration removes the artisthashes column from the playlists table.
"""
name = "RemovePlaylistArtistHashes"
@staticmethod
def migrate():
# remove artisthashes column
sql = "ALTER TABLE playlists DROP COLUMN artisthashes"
with SQLiteManager(userdata_db=True) as cur:
cur.execute(sql)
cur.close()
class AddSettingsToPlaylistTable:
"""
This migration adds the settings column and removes the banner_pos and has_gif columns
to the playlists table.
"""
name = "AddSettingsToPlaylistTable"
@staticmethod
def migrate():
# existing_playlists = []
select_playlists_sql = "SELECT * FROM playlists"
with SQLiteManager(userdata_db=True) as cur:
create_playlist_table_sql = """CREATE TABLE IF NOT EXISTS playlists (
id integer PRIMARY KEY,
image text,
last_updated text not null,
name text not null,
settings text,
trackhashes text
);"""
insert_playlist_sql = """INSERT INTO playlists(
image,
last_updated,
name,
settings,
trackhashes
) VALUES(:image, :last_updated, :name, :settings, :trackhashes)
"""
cur.execute(select_playlists_sql)
# load all playlists
playlists = cur.fetchall()
# drop old playlists table
cur.execute("DROP TABLE playlists")
# create new playlists table
cur.execute(create_playlist_table_sql)
def transform_playlists(pipeline: Generator, playlists: tuple):
for playlist in playlists:
# create dict that matches the new schema
p = {
"id": playlist[0],
"name": playlist[5],
"image": playlist[3],
"trackhashes": playlist[6],
"last_updated": playlist[4],
"settings": json.dumps(
{
"has_gif": False,
"banner_pos": playlist[1],
"square_img": False,
}
),
}
pipeline.send(p)
@coroutine
def insert_playlist():
while True:
playlist = yield
p = OrderedDict(sorted(playlist.items()))
cur.execute(insert_playlist_sql, p)
# insert playlists using a coroutine
# (my first coroutine)
pipeline = insert_playlist()
transform_playlists(pipeline, playlists)
cur.close()
class AddLastUpdatedToTrackTable:
"""
This migration adds the last modified column to the tracks table.
"""
name = "AddLastUpdatedToTrackTable"
@staticmethod
def migrate():
# add last_mod column and default to current timestamp
timestamp = time.time()
sql = f"ALTER TABLE tracks ADD COLUMN last_mod text not null DEFAULT '{timestamp}'"
with SQLiteManager() as cur:
cur.execute(sql)
cur.close()
class MovePlaylistsAndFavoritesTo10BitHashes:
"""
This migration moves the playlists and favorites to 10 bit hashes.
"""
name = "MovePlaylistsAndFavoritesTo10BitHashes"
@staticmethod
def migrate():
def get_track_data_by_hash(trackhash: str, tracks: list[tuple]) -> tuple:
for track in tracks:
# trackhash is the 15th bit hash
if track[15] == trackhash:
# return artist, album, title
return track[4], track[1], track[13]
def get_track_by_albumhash(albumhash: str, tracks: list[tuple]) -> tuple:
for track in tracks:
# albumhash is the 3rd bit hash
if track[3] == albumhash:
# return album, albumartist
return track[1], track[2]
_base = "SELECT * FROM"
fetch_playlists_sql = f"{_base} playlists"
fetch_tracks_sql = f"{_base} tracks"
update_playlist_hashes_sql = (
"UPDATE playlists SET trackhashes = :trackhashes WHERE id = :id"
)
fetch_favorites_sql = f"{_base} favorites"
update_fav_sql = "UPDATE favorites SET hash = :hash WHERE id = :id"
remove_fav_sql = "DELETE FROM favorites WHERE id = :id"
db_tracks = []
# read tracks from db
with SQLiteManager() as cur:
cur.execute(fetch_tracks_sql)
db_tracks.extend(cur.fetchall())
cur.close()
# update playlists
with SQLiteManager(userdata_db=True) as cur:
cur.execute(fetch_playlists_sql)
playlists = cur.fetchall()
# for each playlist
for p in playlists:
pid = p[0]
# load trackhashes
trackhashes: list[str] = json.loads(p[5])
for index, t in enumerate(trackhashes):
(artist, album, title) = get_track_data_by_hash(t, db_tracks)
# create new hash
new_hash = create_hash(artist, album, title, decode=True, limit=10)
trackhashes[index] = new_hash
# convert to string
trackhashes = json.dumps(trackhashes)
# save to db
cur.execute(
update_playlist_hashes_sql, {"trackhashes": trackhashes, "id": pid}
)
cur.close()
# update favorites
with SQLiteManager(userdata_db=True) as cur:
cur.execute(fetch_favorites_sql)
favorites = cur.fetchall()
# for each favorite
for f in favorites:
fid = f[0]
fhash: str = f[1]
ftype: str = f[2] # "track" || "album"
if ftype == "album":
(album, albumartist) = get_track_by_albumhash(fhash, db_tracks)
# create new hash
new_hash = create_hash(album, albumartist, decode=True, limit=10)
# save to db
cur.execute(update_fav_sql, {"hash": new_hash, "id": fid})
continue
if ftype == "track":
(artist, album, title) = get_track_data_by_hash(fhash, db_tracks)
# create new hash
new_hash = create_hash(artist, album, title, decode=True, limit=10)
# save to db
cur.execute(update_fav_sql, {"hash": new_hash, "id": fid})
continue
# remove favorites that are not track or album. ie. artists
cur.execute(remove_fav_sql, {"id": fid})
cur.close()
class RemoveAllTracks:
"""
This migration removes all tracks from the tracks table.
"""
name = "RemoveAllTracks"
@staticmethod
def migrate():
sql = "DELETE FROM tracks"
with SQLiteManager() as cur:
cur.execute(sql)
cur.close()

2
app/models/album.py
View File

@ -191,7 +191,7 @@ class Album:
if self.date:
return
dates = {t.date for t in tracks if t.date}
dates = {int(t.date) for t in tracks if t.date}
if len(dates) == 0:
self.date = 0

5
app/models/playlist.py
View File

@ -11,7 +11,6 @@ class Playlist:
"""Creates playlist objects"""
id: int
artisthashes: str | list[str]
image: str
last_updated: str
name: str
@ -26,11 +25,8 @@ class Playlist:
def __post_init__(self):
self.trackhashes = json.loads(str(self.trackhashes))
# self.artisthashes = json.loads(str(self.artisthashes))
# commentted until we need it 👆
self.artisthashes = []
self.count = len(self.trackhashes)
if isinstance(self.settings, str):
@ -55,4 +51,3 @@ class Playlist:
over the API.
"""
self.trackhashes = []
self.artisthashes = []

2
app/models/track.py
View File

@ -30,10 +30,10 @@ class Track:
filepath: str
folder: str
genre: str | list[str]
last_mod: float
title: str
track: int
trackhash: str
last_mod: float
filetype: str = ""
image: str = ""

30
app/setup/sqlite.py
View File

@ -4,12 +4,17 @@ Applies migrations.
"""
from app.db.sqlite import create_connection, create_tables, queries
from app.migrations import apply_migrations, set_postinit_migration_versions
from app.migrations.__preinit import run_preinit_migrations, set_preinit_migration_versions
from app.migrations import apply_migrations
from app.settings import Db
def run_migrations():
"""
Run migrations and updates migration version.
"""
apply_migrations()
def setup_sqlite():
"""
Create Sqlite databases and tables.
@ -17,25 +22,12 @@ def setup_sqlite():
# if os.path.exists(DB_PATH):
# os.remove(DB_PATH)
run_preinit_migrations()
app_db_conn = create_connection(Db.get_app_db_path())
playlist_db_conn = create_connection(Db.get_userdata_db_path())
user_db_conn = create_connection(Db.get_userdata_db_path())
create_tables(app_db_conn, queries.CREATE_APPDB_TABLES)
create_tables(playlist_db_conn, queries.CREATE_USERDATA_TABLES)
create_tables(user_db_conn, queries.CREATE_USERDATA_TABLES)
create_tables(app_db_conn, queries.CREATE_MIGRATIONS_TABLE)
create_tables(playlist_db_conn, queries.CREATE_MIGRATIONS_TABLE)
app_db_conn.close()
playlist_db_conn.close()
def run_migrations():
"""
Run migrations and updates migration version.
"""
apply_migrations()
set_preinit_migration_versions()
set_postinit_migration_versions()
user_db_conn.close()

1
app/utils/dates.py
View File

@ -25,3 +25,4 @@ def date_string_to_time_passed(prev_date: str) -> str:
now = pendulum.now()
return now.subtract(seconds=seconds).diff_for_humans()

11
app/utils/decorators.py Normal file
View File

@ -0,0 +1,11 @@
def coroutine(func):
"""
Decorator: primes `func` by advancing to first `yield`
"""
def start(*args, **kwargs):
cr = func(*args, **kwargs)
next(cr)
return cr
return start

2
app/utils/parsers.py
View File

@ -211,7 +211,7 @@ def get_base_title_and_versions(
Extracts the base album title and version info from an album title string using regex.
"""
album_title, version_block = get_base_album_title(original_album_title)
if version_block is None:
return original_album_title, []