REFACTOR: MUSIC LIBRARYgit status #91

command.py

@@ -9,8 +9,8 @@ import media.system
 import util
 import variables as var
 from librb import radiobrowser
-from database import Database
+from database import SettingsDatabase
-from media.playlist import PlaylistItemWrapper
+from media.playlist import get_item_wrapper
 from media.file import FileItem
 from media.url_from_playlist import PlaylistURLItem, get_playlist_info
 from media.url import URLItem
@@ -171,11 +171,10 @@ def cmd_play_file(bot, user, text, command, parameter):
         files = util.get_recursive_file_list_sorted(var.music_folder)
         if int(parameter) < len(files):
             filename = files[int(parameter)].replace(var.music_folder, '')
-            music_wrapper = PlaylistItemWrapper(FileItem(bot, filename), user)
+            music_wrapper = get_item_wrapper(bot, type='file', path=filename, user=user)
             var.playlist.append(music_wrapper)
-            music = music_wrapper.item
+            log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
-            log.info("cmd: add to playlist: " + music.format_debug_string())
+            bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
-            bot.send_msg(constants.strings('file_added', item=music.format_song_string(user)), text)
 
     # if parameter is {path}
     else:
@@ -186,11 +185,10 @@ def cmd_play_file(bot, user, text, command, parameter):
             return
 
         if os.path.isfile(path):
-            music_wrapper = PlaylistItemWrapper(FileItem(bot, parameter), user)
+            music_wrapper = get_item_wrapper(bot, type='file', path=parameter, user=user)
             var.playlist.append(music_wrapper)
-            music = music_wrapper.item
+            log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
-            log.info("cmd: add to playlist: " + music.format_debug_string())
+            bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
-            bot.send_msg(constants.strings('file_added', item=music.format_song_string(user)), text)
             return
 
         # if parameter is {folder}
@@ -212,11 +210,10 @@ def cmd_play_file(bot, user, text, command, parameter):
 
             for file in files:
                 count += 1
-                music_wrapper = PlaylistItemWrapper(FileItem(bot, file), user)
+                music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
                 var.playlist.append(music_wrapper)
-                music = music_wrapper.item
+                log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
-                log.info("cmd: add to playlist: " + music.format_debug_string())
+                msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path))
-                msgs.append("{} ({})".format(music.title, music.path))
 
             if count != 0:
                 send_multi_lines(bot, msgs, text)
@@ -231,11 +228,10 @@ def cmd_play_file(bot, user, text, command, parameter):
                 bot.send_msg(constants.strings('no_file'), text)
             elif len(matches) == 1:
                 file = matches[0][1]
-                music_wrapper = PlaylistItemWrapper(FileItem(bot, file), user)
+                music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
                 var.playlist.append(music_wrapper)
-                music = music_wrapper.item
+                log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
-                log.info("cmd: add to playlist: " + music.format_debug_string())
+                bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
-                bot.send_msg(constants.strings('file_added', item=music.format_song_string(user)), text)
             else:
                 msgs = [ constants.strings('multiple_matches')]
                 for match in matches:
@@ -252,17 +248,18 @@ def cmd_play_file_match(bot, user, text, command, parameter):
         msgs = [ constants.strings('multiple_file_added')]
         count = 0
         try:
+            music_wrappers = []
             for file in files:
                 match = re.search(parameter, file)
                 if match:
                     count += 1
-                    music_wrapper = PlaylistItemWrapper(FileItem(bot, file), user)
+                    music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
-                    var.playlist.append(music_wrapper)
+                    music_wrappers.append(music_wrapper)
-                    music = music_wrapper.item
+                    log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
-                    log.info("cmd: add to playlist: " + music.format_debug_string())
+                    msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path))
-                    msgs.append("{} ({})".format(music.title, music.path))
 
             if count != 0:
+                var.playlist.extend(music_wrappers)
                 send_multi_lines(bot, msgs, text)
             else:
                 bot.send_msg(constants.strings('no_file'), text)
@@ -271,14 +268,14 @@ def cmd_play_file_match(bot, user, text, command, parameter):
             msg = constants.strings('wrong_pattern', error=str(e))
             bot.send_msg(msg, text)
     else:
-        bot.send_msg(constants.strings('bad_parameter', command))
+        bot.send_msg(constants.strings('bad_parameter', command=command))
 
 
 def cmd_play_url(bot, user, text, command, parameter):
     global log
 
     url = util.get_url_from_input(parameter)
-    music_wrapper = PlaylistItemWrapper(URLItem(bot, url), user)
+    music_wrapper = get_item_wrapper(bot, type='url', url=url)
     var.playlist.append(music_wrapper)
 
     log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
@@ -326,7 +323,7 @@ def cmd_play_radio(bot, user, text, command, parameter):
             parameter = parameter.split()[0]
         url = util.get_url_from_input(parameter)
         if url:
-            music_wrapper = PlaylistItemWrapper(RadioItem(bot, url), user)
+            music_wrapper = get_item_wrapper(bot, type='radio', url=url)
 
             var.playlist.append(music_wrapper)
             log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
@@ -425,7 +422,7 @@ def cmd_rb_play(bot, user, text, command, parameter):
         url = radiobrowser.geturl_byid(parameter)
         if url != "-1":
             log.info('cmd: Found url: ' + url)
-            music_wrapper = PlaylistItemWrapper(RadioItem(bot, url, stationname), user)
+            music_wrapper = get_item_wrapper(bot, type='radio', url=url, name=stationname)
             var.playlist.append(music_wrapper)
             log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
             bot.async_download_next()
@@ -713,9 +710,8 @@ def cmd_queue(bot, user, text, command, parameter):
         bot.send_msg(msg, text)
     else:
         msgs = [ constants.strings('queue_contents')]
-        for i, value in enumerate(var.playlist):
+        for i, music in enumerate(var.playlist):
             newline = ''
-            music = value.item
             if i == var.playlist.current_index:
                 newline = '<b>{} ▶ ({}) {} ◀</b>'.format(i + 1, music.display_type(),
                                                            music.format_short_string())
@@ -772,7 +768,7 @@ def cmd_drop_database(bot, user, text, command, parameter):
     global log
 
     var.db.drop_table()
-    var.db = Database(var.dbfile)
+    var.db = SettingsDatabase(var.dbfile)
     bot.send_msg(constants.strings('database_dropped'), text)
 
 # Just for debug use
@@ -784,4 +780,4 @@ def cmd_loop_state(bot, user, text, command, parameter):
 
 def cmd_item(bot, user, text, command, parameter):
     print(bot.wait_for_downloading)
-    print(var.playlist.current_item().item.to_dict())
+    print(var.playlist.current_item().to_dict())

database.py

@@ -1,9 +1,12 @@
 import sqlite3
+import json
+import datetime
 
 class DatabaseError(Exception):
     pass
 
-class Database:
+class SettingsDatabase:
+    version = 1
     def __init__(self, db_path):
         self.db_path = db_path
 
@@ -11,12 +14,53 @@ class Database:
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
 
-        # check if table exists, or create one
+        self.db_version_check_and_create()
-        tables = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='botamusique';").fetchall()
-        if len(tables) == 0:
-            cursor.execute("CREATE TABLE botamusique (section text, option text, value text, UNIQUE(section, option))")
-            conn.commit()
 
+        conn.commit()
+        conn.close()
+
+    def has_table(self):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        tables = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='botamusique';").fetchall()
+        conn.close()
+        if len(tables) == 0:
+            return False
+        return True
+
+    def db_version_check_and_create(self):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        if self.has_table():
+            # check version
+            result = cursor.execute("SELECT value FROM botamusique WHERE section=? AND option=?",
+                                    ("bot", "db_version")).fetchall()
+
+            if len(result) == 0 or int(result[0][0]) != self.version:
+                old_name = "botamusique_old_%s" % datetime.datetime.now().strftime("%Y%m%d")
+                cursor.execute("ALTER TABLE botamusique RENAME TO %s" % old_name)
+                conn.commit()
+                self.create_table()
+                self.set("bot", "old_db_name", old_name)
+        else:
+            self.create_table()
+
+        conn.close()
+
+    def create_table(self):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute("CREATE TABLE IF NOT EXISTS botamusique ("
+                       "section TEXT, "
+                       "option TEXT, "
+                       "value TEXT, "
+                       "UNIQUE(section, option))")
+        cursor.execute("INSERT INTO botamusique (section, option, value) "
+                       "VALUES (?, ?, ?)" , ("bot", "db_version", "1"))
+        cursor.execute("INSERT INTO botamusique (section, option, value) "
+                       "VALUES (?, ?, ?)" , ("bot", "music_db_version", "0"))
+        conn.commit()
         conn.close()
 
     def get(self, section, option, **kwargs):
@@ -45,10 +89,8 @@ class Database:
     def set(self, section, option, value):
         conn = sqlite3.connect(self.db_path)
         cursor = conn.cursor()
-        cursor.execute('''
+        cursor.execute("INSERT OR REPLACE INTO botamusique (section, option, value) "
-            INSERT OR REPLACE INTO botamusique (section, option, value)
+                       "VALUES (?, ?, ?)" , (section, option, value))
-            VALUES (?, ?, ?)
-        ''', (section, option, value))
         conn.commit()
         conn.close()
 
@@ -82,7 +124,10 @@ class Database:
         results = cursor.execute("SELECT option, value FROM botamusique WHERE section=?", (section, )).fetchall()
         conn.close()
 
-        return map(lambda v: (v[0], v[1]), results)
+        if len(results) > 0:
+            return list(map(lambda v: (v[0], v[1]), results))
+        else:
+            return []
 
     def drop_table(self):
         conn = sqlite3.connect(self.db_path)
@@ -91,3 +136,98 @@ class Database:
         conn.close()
 
 
+class MusicDatabase:
+    def __init__(self, db_path):
+        self.db_path = db_path
+
+        # connect
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        # check if table exists, or create one
+        cursor.execute("CREATE TABLE IF NOT EXISTS music ("
+                       "id TEXT PRIMARY KEY, "
+                       "type TEXT, "
+                       "title TEXT, "
+                       "metadata TEXT, "
+                       "tags TEXT)")
+        conn.commit()
+        conn.close()
+
+    def insert_music(self, music_dict):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+
+        id = music_dict['id']
+        title = music_dict['title']
+        type = music_dict['type']
+        tags = ",".join(music_dict['tags'])
+        del music_dict['id']
+        del music_dict['title']
+        del music_dict['type']
+        del music_dict['tags']
+
+        cursor.execute("INSERT OR REPLACE INTO music (id, type, title, metadata, tags) VALUES (?, ?, ?, ?, ?)",
+                       (id,
+                        type,
+                        title,
+                        json.dumps(music_dict),
+                        tags))
+
+        conn.commit()
+        conn.close()
+
+    def query_music(self, **kwargs):
+        condition = []
+        filler = []
+
+        for key, value in kwargs.items():
+            if isinstance(value, str):
+                condition.append(key + "=?")
+                filler.append(value)
+            else:
+                condition.append(key + " " + value[0] + " ?")
+                filler.append(value[1])
+
+        condition_str = " AND ".join(condition)
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        results = cursor.execute("SELECT id, type, title, metadata, tags FROM music "
+                                "WHERE %s" % condition_str, filler).fetchall()
+        conn.close()
+
+        if len(results) > 0:
+            music_dicts = []
+            for result in results:
+                music_dict = json.loads(result[3])
+                music_dict['type'] = result[1]
+                music_dict['title'] = result[2]
+                music_dict['tags'] = result[4].split(",")
+                music_dict['id'] = result[0]
+                music_dicts.append(music_dict)
+
+            return music_dicts
+        else:
+            return None
+
+    def delete_music(self, **kwargs):
+        condition = []
+        filler = []
+
+        for key, value in kwargs.items():
+            if isinstance(value, str):
+                condition.append(key + "=?")
+                filler.append(value)
+            else:
+                condition.append(key + " " + value[0] + " ?")
+                filler.append(value[1])
+
+        condition_str = " AND ".join(condition)
+
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()
+        cursor.execute("DELETE FROM music "
+                                 "WHERE %s" % condition_str, filler)
+        conn.commit()
+        conn.close()

interface.py

@@ -12,7 +12,7 @@ import random
 from werkzeug.utils import secure_filename
 import errno
 import media
-from media.playlist import PlaylistItemWrapper
+from media.playlist import get_item_wrapper
 from media.file import FileItem
 from media.url_from_playlist import PlaylistURLItem, get_playlist_info
 from media.url import URLItem
@@ -132,7 +132,7 @@ def playlist():
     for index, item_wrapper in enumerate(var.playlist):
          items.append(render_template('playlist.html',
                                      index=index,
-                                     m=item_wrapper.item,
+                                     m=item_wrapper.item(),
                                      playlist=var.playlist
                                      )
                      )
@@ -164,14 +164,15 @@ def post():
         if 'add_file_bottom' in request.form and ".." not in request.form['add_file_bottom']:
             path = var.music_folder + request.form['add_file_bottom']
             if os.path.isfile(path):
-                music_wrapper = PlaylistItemWrapper(FileItem(var.bot, request.form['add_file_bottom']), user)
+                music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_bottom'], user=user)
+
                 var.playlist.append(music_wrapper)
                 log.info('web: add to playlist(bottom): ' + music_wrapper.format_debug_string())
 
         elif 'add_file_next' in request.form and ".." not in request.form['add_file_next']:
             path = var.music_folder + request.form['add_file_next']
             if os.path.isfile(path):
-                music_wrapper = PlaylistItemWrapper(FileItem(var.bot, request.form['add_file_next']), user)
+                music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_next'], user=user)
                 var.playlist.insert(var.playlist.current_index + 1, music_wrapper)
                 log.info('web: add to playlist(next): ' + music_wrapper.format_debug_string())
 
@@ -197,8 +198,8 @@ def post():
                     files = music_library.get_files(folder)
 
                 music_wrappers = list(map(
-                    lambda file: PlaylistItemWrapper(FileItem(var.bot, folder + file), user),
+                    lambda file: get_item_wrapper(var.bot, type='file', path=file, user=user),
-                    files))
+                files))
 
                 var.playlist.extend(music_wrappers)
 
@@ -207,7 +208,7 @@ def post():
 
 
         elif 'add_url' in request.form:
-            music_wrapper = PlaylistItemWrapper(URLItem(var.bot, request.form['add_url']), user)
+            music_wrapper = get_item_wrapper(var.bot, type='url', url=request.form['url'])
             var.playlist.append(music_wrapper)
 
             log.info("web: add to playlist: " + music_wrapper.format_debug_string())
@@ -217,7 +218,7 @@ def post():
 
         elif 'add_radio' in request.form:
             url = request.form['add_radio']
-            music_wrapper = PlaylistItemWrapper(RadioItem(var.bot, url), user)
+            music_wrapper = get_item_wrapper(var.bot, type='radio', url=url)
             var.playlist.append(music_wrapper)
 
             log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())

media/file.py

@@ -10,7 +10,7 @@ import json
 
 import util
 import variables as var
-from media.item import BaseItem
+from media.item import BaseItem, item_builders, item_loaders, item_id_generators
 import constants
 
 '''
@@ -24,6 +24,20 @@ type : file
     user
 '''
 
+def file_item_builder(bot, **kwargs):
+    return FileItem(bot, kwargs['path'])
+
+def file_item_loader(bot, _dict):
+    return FileItem(bot, "", _dict)
+
+def file_item_id_generator(**kwargs):
+    return hashlib.md5(kwargs['path'].encode()).hexdigest()
+
+item_builders['file'] = file_item_builder
+item_loaders['file'] = file_item_loader
+item_id_generators['file'] = file_item_id_generator
+
+
 class FileItem(BaseItem):
     def __init__(self, bot, path, from_dict=None):
         if not from_dict:
@@ -49,7 +63,7 @@ class FileItem(BaseItem):
         self.type = "file"
 
     def uri(self):
-        return var.music_folder + self.path
+        return var.music_folder + self.path if self.path[0] != "/" else self.path
 
     def is_ready(self):
         return True
@@ -61,6 +75,7 @@ class FileItem(BaseItem):
             self.send_client_message(constants.strings('file_missed', file=self.path))
             return False
 
+        self.version = 1 # 0 -> 1, notify the wrapper to save me when validate() is visited the first time
         self.ready = "yes"
         return True
 

media/item.py

@@ -11,30 +11,22 @@ from PIL import Image
 import util
 import variables as var
 
-"""
+item_builders = {}
-FORMAT OF A MUSIC INTO THE PLAYLIST
+item_loaders = {}
-type : url
+item_id_generators = {}
-    id
-    url
-    title
-    path
-    duration
-    artist
-    thumbnail
-    user
-    ready (validation, no, downloading, yes, failed)
-    from_playlist (yes,no)
-    playlist_title
-    playlist_url
 
-type : radio
+def example_builder(bot, **kwargs):
-    id
+    return BaseItem(bot)
-    url
-    name
-    current_title
-    user
 
-"""
+def example_loader(bot, _dict):
+    return BaseItem(bot, from_dict=_dict)
+
+def example_id_generator(**kwargs):
+    return ""
+
+item_builders['base'] = example_builder
+item_loaders['base'] = example_loader
+item_id_generators['base'] = example_id_generator
 
 class BaseItem:
     def __init__(self, bot, from_dict=None):
@@ -42,6 +34,9 @@ class BaseItem:
         self.log = logging.getLogger("bot")
         self.type = "base"
         self.title = ""
+        self.path = ""
+        self.tags = []
+        self.version = 0 # if version increase, wrapper will re-save this item
 
         if from_dict is None:
             self.id = ""
@@ -62,22 +57,9 @@ class BaseItem:
     def uri(self):
         raise
 
-    def async_prepare(self):
-        th = threading.Thread(
-            target=self.prepare, name="Prepare-" + self.id[:7])
-        self.log.info(
-            "%s: start preparing item in thread: " % self.type + self.format_debug_string())
-        th.daemon = True
-        th.start()
-        #self.download_threads.append(th)
-        return th
-
     def prepare(self):
         return True
 
-    def play(self):
-        pass
-
     def format_song_string(self, user):
         return self.id
 
@@ -97,6 +79,6 @@ class BaseItem:
         self.bot.send_msg(msg)
 
     def to_dict(self):
-        return {"type" : "base", "id": self.id, "ready": self.ready}
+        return {"type" : "base", "id": self.id, "ready": self.ready, "path": self.path, "tags": self.tags}
 
 

media/library.py

@@ -0,0 +1,70 @@
+import logging
+
+from database import MusicDatabase
+from media.item import item_builders, item_loaders, item_id_generators
+from media.file import FileItem
+from media.url import URLItem
+from media.url_from_playlist import PlaylistURLItem
+from media.radio import RadioItem
+from database import MusicDatabase
+import variables as var
+
+
+class MusicLibrary(dict):
+    def __init__(self, db: MusicDatabase):
+        super().__init__()
+        self.db = db
+        self.log = logging.getLogger("bot")
+
+    def get_item_by_id(self, bot, id):
+        if id in self:
+            return self[id]
+
+        # if not cached, query the database
+        item = self.fetch(bot, id)
+        if item is not None:
+            self[id] = item
+            self.log.debug("library: music found in database: %s" % item.format_debug_string())
+            return item
+
+    def get_item(self, bot, **kwargs):
+        # kwargs should provide type and id, and parameters to build the item if not existed in the library.
+        # if cached
+        id = item_id_generators[kwargs['type']](**kwargs)
+        if id in self:
+            return self[id]
+
+        # if not cached, query the database
+        item = self.fetch(bot, id)
+        if item is not None:
+            self[id] = item
+            self.log.debug("library: music found in database: %s" % item.format_debug_string())
+            return item
+
+        # if not in the database, build one
+        self[id] = item_builders[kwargs['type']](bot, **kwargs) # newly built item will not be saved immediately
+        return self[id]
+
+    def fetch(self, bot, id):
+        music_dicts = self.db.query_music(id=id)
+        if music_dicts:
+            music_dict = music_dicts[0]
+            type = music_dict['type']
+            self[id] = item_loaders[type](bot, music_dict)
+            return self[id]
+        else:
+            return None
+
+    def save(self, id):
+        self.log.debug("library: music save into database: %s" % self[id].format_debug_string())
+        self.db.insert_music(self[id].to_dict())
+
+    def delete(self, id):
+        self.db.delete_music(id=id)
+
+    def free(self, id):
+        if id in self:
+            del self[id]
+
+    def free_all(self):
+        self.clear()

media/playlist.py

@@ -8,40 +8,81 @@ from media.file import FileItem
 from media.url import URLItem
 from media.url_from_playlist import PlaylistURLItem
 from media.radio import RadioItem
-
+from database import MusicDatabase
+from media.library import MusicLibrary
 
 class PlaylistItemWrapper:
-    def __init__(self, item, user):
+    def __init__(self, lib, id, type, user):
-        self.item = item
+        self.lib = lib
+        self.id = id
         self.user = user
+        self.type = type
+        self.log = logging.getLogger("bot")
+        self.version = 1
+
+    def item(self):
+        return self.lib[self.id]
 
     def to_dict(self):
-        dict = self.item.to_dict()
+        dict = self.item().to_dict()
         dict['user'] = self.user
         return dict
 
+    def validate(self):
+        ret = self.item().validate()
+        if ret and self.item().version > self.version:
+            self.version = self.item().version
+            self.lib.save(self.id)
+        return ret
+
+    def prepare(self):
+        ret = self.item().prepare()
+        if ret and self.item().version > self.version:
+            self.version = self.item().version
+            self.lib.save(self.id)
+        return ret
+
+    def async_prepare(self):
+        th = threading.Thread(
+            target=self.item().prepare, name="Prepare-" + self.id[:7])
+        self.log.info(
+            "%s: start preparing item in thread: " % self.item().type + self.format_debug_string())
+        th.daemon = True
+        th.start()
+        return th
+
+    def uri(self):
+        return self.item().uri()
+
+    def is_ready(self):
+        return self.item().is_ready()
+
+    def is_failed(self):
+        return self.item().is_failed()
+
     def format_current_playing(self):
-        return self.item.format_current_playing(self.user)
+        return self.item().format_current_playing(self.user)
 
     def format_song_string(self):
-        return self.item.format_song_string(self.user)
+        return self.item().format_song_string(self.user)
 
     def format_short_string(self):
-        return self.item.format_short_string()
+        return self.item().format_short_string()
 
     def format_debug_string(self):
-        return self.item.format_debug_string()
+        return self.item().format_debug_string()
+
+    def display_type(self):
+        return self.item().display_type()
 
 
-def dict_to_item(dict):
+def get_item_wrapper(bot, **kwargs):
-    if dict['type'] == 'file':
+    item = var.library.get_item(bot, **kwargs)
-        return PlaylistItemWrapper(FileItem(var.bot, "", dict), dict['user'])
+    return PlaylistItemWrapper(var.library, item.id, kwargs['type'], kwargs['user'])
-    elif dict['type'] == 'url':
+
-        return PlaylistItemWrapper(URLItem(var.bot, "", dict), dict['user'])
+def get_item_wrapper_by_id(bot, id, user):
-    elif dict['type'] == 'url_from_playlist':
+    item = var.library.get_item_by_id(bot, id)
-        return PlaylistItemWrapper(PlaylistURLItem(var.bot, "", "", "", "", dict), dict['user'])
+    return PlaylistItemWrapper(var.library, item.id, item.type, user)
-    elif dict['type'] == 'radio':
-        return PlaylistItemWrapper(RadioItem(var.bot, "", "", dict), dict['user'])
 
 def get_playlist(mode, _list=None, index=None):
     if _list and index is None:
@@ -61,10 +102,8 @@ def get_playlist(mode, _list=None, index=None):
             return RepeatPlaylist().from_list(_list, index)
         elif mode == "random":
             return RandomPlaylist().from_list(_list, index)
-
     raise
 
-
 class BasePlayList(list):
     def __init__(self):
         super().__init__()
@@ -154,18 +193,21 @@ class BasePlayList(list):
         if self.current_index > index:
             self.current_index -= 1
 
+        var.music_db.free(removed.id)
         return removed
 
     def remove_by_id(self, id):
         self.version += 1
         to_be_removed = []
         for index, wrapper in enumerate(self):
-            if wrapper.item.id == id:
+            if wrapper.id == id:
                 to_be_removed.append(index)
 
         for index in to_be_removed:
             self.remove(index)
 
+        var.music_db.free(id)
+
     def current_item(self):
         if len(self) == 0:
             return False
@@ -198,6 +240,7 @@ class BasePlayList(list):
     def clear(self):
         self.version += 1
         self.current_index = -1
+        var.library.free_all()
         super().clear()
 
     def save(self):
@@ -205,16 +248,23 @@ class BasePlayList(list):
         var.db.set("playlist", "current_index", self.current_index)
 
         for index, music in enumerate(self):
-            var.db.set("playlist_item", str(index), json.dumps(music.to_dict()))
+            var.db.set("playlist_item", str(index), json.dumps({'id': music.id, 'user': music.user }))
 
     def load(self):
         current_index = var.db.getint("playlist", "current_index", fallback=-1)
         if current_index == -1:
             return
 
-        items = list(var.db.items("playlist_item"))
+        items = var.db.items("playlist_item")
-        items.sort(key=lambda v: int(v[0]))
+        if items:
-        self.from_list(list(map(lambda v: dict_to_item(json.loads(v[1])), items)), current_index)
+            music_wrappers = []
+            items.sort(key=lambda v: int(v[0]))
+            for item in items:
+                item = json.loads(item[1])
+                music_wrapper = get_item_wrapper_by_id(var.bot, item['id'], item['user'])
+                if music_wrapper:
+                    music_wrappers.append(music_wrapper)
+            self.from_list(music_wrappers, current_index)
 
     def _debug_print(self):
         print("===== Playlist(%d)=====" % self.current_index)
@@ -235,10 +285,10 @@ class BasePlayList(list):
         self.log.debug("playlist: start validating...")
         self.validating_thread_lock.acquire()
         while len(self.pending_items) > 0:
-            item = self.pending_items.pop().item
+            item = self.pending_items.pop()
             self.log.debug("playlist: validating %s" % item.format_debug_string())
-            if not item.validate() or item.ready == 'failed':
+            if not item.validate() or item.is_failed():
-                # TODO: logging
+                self.log.debug("playlist: validating failed.")
                 self.remove_by_id(item.id)
 
         self.log.debug("playlist: validating finished.")

media/radio.py

@@ -6,6 +6,7 @@ import traceback
 import hashlib
 
 from media.item import BaseItem
+from media.item import item_builders, item_loaders, item_id_generators
 import constants
 
 log = logging.getLogger("bot")
@@ -74,6 +75,24 @@ def get_radio_title(url):
         pass
     return url
 
+
+def radio_item_builder(bot, **kwargs):
+    if 'name' in kwargs:
+        return RadioItem(bot, kwargs['url'], kwargs['name'])
+    else:
+        return RadioItem(bot, kwargs['url'], '')
+
+def radio_item_loader(bot, _dict):
+    return RadioItem(bot, "", "", _dict)
+
+def radio_item_id_generator(**kwargs):
+    return hashlib.md5(kwargs['url'].encode()).hexdigest()
+
+item_builders['radio'] = radio_item_builder
+item_loaders['radio'] = radio_item_loader
+item_id_generators['radio'] = radio_item_id_generator
+
+
 class RadioItem(BaseItem):
     def __init__(self, bot, url, name="", from_dict=None):
         if from_dict is None:
@@ -92,6 +111,7 @@ class RadioItem(BaseItem):
         self.type = "radio"
 
     def validate(self):
+        self.version = 1 # 0 -> 1, notify the wrapper to save me when validate() is visited the first time
         return True
 
     def is_ready(self):

media/url.py

@@ -10,17 +10,31 @@ import glob
 import constants
 import media
 import variables as var
+from media.item import item_builders, item_loaders, item_id_generators
 from media.file import FileItem
 import media.system
 
 log = logging.getLogger("bot")
 
+def url_item_builder(bot, **kwargs):
+    return URLItem(bot, kwargs['url'])
+
+def url_item_loader(bot, _dict):
+    return URLItem(bot, "", _dict)
+
+def url_item_id_generator(**kwargs):
+    return hashlib.md5(kwargs['url'].encode()).hexdigest()
+
+item_builders['url'] = url_item_builder
+item_loaders['url'] = url_item_loader
+item_id_generators['url'] = url_item_id_generator
+
 
 class URLItem(FileItem):
     def __init__(self, bot, url, from_dict=None):
         self.validating_lock = threading.Lock()
         if from_dict is None:
-            self.url = url
+            self.url = url if url[-1] != "/" else url[:-1]
             self.title = ''
             self.duration = 0
             self.ready = 'pending'
@@ -45,7 +59,7 @@ class URLItem(FileItem):
         self.type = "url"
 
     def uri(self):
-        return self.path
+        return var.music_folder + self.path if self.path[0] != "/" else self.path
 
     def is_ready(self):
         if self.downloading or self.ready != 'yes':
@@ -82,6 +96,7 @@ class URLItem(FileItem):
             return False
         else:
             self.ready = "validated"
+            self.version += 1 # notify wrapper to save me
             return True
 
     # Run in a other thread
@@ -165,6 +180,7 @@ class URLItem(FileItem):
                     "bot: finished downloading url (%s) %s, saved to %s." % (self.title, self.url, self.path))
                 self.downloading = False
                 self._read_thumbnail_from_file(base_path + ".jpg")
+                self.version += 1 # notify wrapper to save me
                 return True
             else:
                 for f in glob.glob(base_path + "*"):

media/url_from_playlist.py

@@ -2,7 +2,9 @@ import youtube_dl
 import constants
 import media
 import variables as var
-from media.url import URLItem
+import hashlib
+from media.item import item_builders, item_loaders, item_id_generators
+from media.url import URLItem, url_item_id_generator
 
 def get_playlist_info(bot, url, start_index=0, user=""):
     items = []
@@ -48,6 +50,23 @@ def get_playlist_info(bot, url, start_index=0, user=""):
 
     return items
 
+
+def playlist_url_item_builder(bot, **kwargs):
+    return PlaylistURLItem(bot,
+                           kwargs['url'],
+                           kwargs['title'],
+                           kwargs['playlist_url'],
+                           kwargs['playlist_title'])
+
+
+def playlist_url_item_loader(bot, _dict):
+    return PlaylistURLItem(bot, "", "", "", "", _dict)
+
+item_builders['url_from_playlist'] = playlist_url_item_builder
+item_loaders['url_from_playlist'] = playlist_url_item_loader
+item_id_generators['url_from_playlist'] = url_item_id_generator
+
+
 class PlaylistURLItem(URLItem):
     def __init__(self, bot, url, title, playlist_url, playlist_title, from_dict=None):
         if from_dict is None:

mumbleBot.py

@@ -24,12 +24,13 @@ from packaging import version
 import util
 import command
 import constants
-from database import Database
+from database import SettingsDatabase, MusicDatabase
 import media.url
 import media.file
 import media.radio
 import media.system
 from media.playlist import BasePlayList
+from media.library import MusicLibrary
 
 
 class MumbleBot:
@@ -299,9 +300,9 @@ class MumbleBot:
         assert self.wait_for_downloading == False
 
         music_wrapper = var.playlist.current_item()
-        uri = music_wrapper.item.uri()
+        uri = music_wrapper.uri()
 
-        self.log.info("bot: play music " + music_wrapper.item.format_debug_string())
+        self.log.info("bot: play music " + music_wrapper.format_debug_string())
 
         if var.config.getboolean('bot', 'announce_current_music'):
             self.send_msg(music_wrapper.format_current_playing())
@@ -330,11 +331,11 @@ class MumbleBot:
         # Function start if the next music isn't ready
         # Do nothing in case the next music is already downloaded
         self.log.debug("bot: Async download next asked ")
-        while var.playlist.next_item() and var.playlist.next_item().item.type in ['url', 'url_from_playlist']:
+        while var.playlist.next_item() and var.playlist.next_item().type in ['url', 'url_from_playlist']:
             # usually, all validation will be done when adding to the list.
             # however, for performance consideration, youtube playlist won't be validate when added.
             # the validation has to be done here.
-            next = var.playlist.next_item().item
+            next = var.playlist.next_item()
             if next.validate():
                 if not next.is_ready():
                     next.async_prepare()
@@ -388,7 +389,7 @@ class MumbleBot:
                 # ffmpeg thread has gone. indicate that last song has finished. move to the next song.
                 if not self.wait_for_downloading:
                     if var.playlist.next():
-                        current = var.playlist.current_item().item
+                        current = var.playlist.current_item()
                         if current.validate():
                             if current.is_ready():
                                 self.launch_music()
@@ -403,7 +404,7 @@ class MumbleBot:
                     else:
                         self._loop_status = 'Empty queue'
                 else:
-                    current = var.playlist.current_item().item
+                    current = var.playlist.current_item()
                     if current:
                         if current.is_ready():
                             self.wait_for_downloading = False
@@ -487,7 +488,7 @@ class MumbleBot:
     def pause(self):
         # Kill the ffmpeg thread
         if self.thread:
-            self.pause_at_id = var.playlist.current_item().item.id
+            self.pause_at_id = var.playlist.current_item()
             self.thread.kill()
             self.thread = None
         self.is_pause = True
@@ -502,7 +503,7 @@ class MumbleBot:
 
         music_wrapper = var.playlist.current_item()
 
-        if not music_wrapper or not music_wrapper.item.id == self.pause_at_id or not music_wrapper.item.is_ready():
+        if not music_wrapper or not music_wrapper.id == self.pause_at_id or not music_wrapper.is_ready():
             self.playhead = 0
             return
 
@@ -513,7 +514,7 @@ class MumbleBot:
 
         self.log.info("bot: resume music at %.2f seconds" % self.playhead)
 
-        uri = music_wrapper.item.uri()
+        uri = music_wrapper.uri()
 
         command = ("ffmpeg", '-v', ffmpeg_debug, '-nostdin', '-ss', "%f" % self.playhead, '-i',
                    uri, '-ac', '1', '-f', 's16le', '-ar', '48000', '-')
@@ -607,7 +608,7 @@ if __name__ == '__main__':
         sys.exit()
 
     var.config = config
-    var.db = Database(var.dbfile)
+    var.db = SettingsDatabase(var.dbfile)
 
     # Setup logger
     bot_logger = logging.getLogger("bot")
@@ -625,6 +626,13 @@ if __name__ == '__main__':
     bot_logger.addHandler(handler)
     var.bot_logger = bot_logger
 
+    if var.config.get("bot", "save_music_library", fallback=True):
+        var.music_db = MusicDatabase(var.dbfile)
+    else:
+        var.music_db = MusicDatabase(":memory:")
+
+    var.library = MusicLibrary(var.music_db)
+
     # load playback mode
     playback_mode = None
     if var.db.has_option("playlist", "playback_mode"):

variables.py

@@ -1,11 +1,13 @@
 bot = None
 playlist = None
+library = None
 
 user = ""
 is_proxified = False
 
 dbfile = None
 db = None
+music_db = None
 config = None
 
 bot_logger = None