diff --git a/command.py b/command.py index 4fa43c3..3705fe5 100644 --- a/command.py +++ b/command.py @@ -1,5 +1,7 @@ # coding=utf-8 import logging +import math + import pymumble.pymumble_py3 as pymumble import re @@ -8,10 +10,10 @@ import media.system import util import variables as var from librb import radiobrowser -from database import SettingsDatabase, MusicDatabase +from database import SettingsDatabase, MusicDatabase, Condition from media.item import item_id_generators, dict_to_item, dicts_to_items from media.cache import get_cached_wrapper_from_scrap, get_cached_wrapper_by_id, get_cached_wrappers_by_tags, \ - get_cached_wrapper + get_cached_wrapper, get_cached_wrappers, get_cached_wrapper_from_dict, get_cached_wrappers_from_dicts from media.url_from_playlist import get_playlist_info log = logging.getLogger("bot") @@ -89,6 +91,8 @@ def send_multi_lines(bot, lines, text, linebreak="
"): # ---------------- Variables ----------------- +ITEMS_PER_PAGE = 50 + song_shortlist = [] @@ -206,75 +210,60 @@ def cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=Fals # if parameter is {index} if parameter.isdigit(): - files = var.cache.files - if int(parameter) < len(files): - music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[files[int(parameter)]], user) - var.playlist.append(music_wrapper) - log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string())) + music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition() + .and_equal('type', 'file') + .order_by('path') + .limit(1) + .offset(int(parameter))), user) + + if music_wrappers: + var.playlist.append(music_wrappers[0]) + log.info("cmd: add to playlist: " + music_wrappers[0].format_debug_string()) + bot.send_msg(constants.strings('file_added', item=music_wrappers[0].format_song_string())) return - # if parameter is {path} - else: - # sanitize "../" and so on - # path = os.path.abspath(os.path.join(var.music_folder, parameter)) - # if not path.startswith(os.path.abspath(var.music_folder)): - # bot.send_msg(constants.strings('no_file'), text) - # return + # assume parameter is a path + music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition().and_equal('path', parameter)), user) + if music_wrappers: + var.playlist.append(music_wrappers[0]) + log.info("cmd: add to playlist: " + music_wrappers[0].format_debug_string()) + bot.send_msg(constants.strings('file_added', item=music_wrappers[0].format_song_string())) + return - if parameter in var.cache.files: - music_wrapper = get_cached_wrapper_from_scrap(bot, type='file', path=parameter, user=user) + # assume parameter is a folder + music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition() + .and_equal('type', 'file') + .and_like('path', parameter + '%')), user) + if music_wrappers: + msgs = [constants.strings('multiple_file_added')] + + for music_wrapper in music_wrappers: var.playlist.append(music_wrapper) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string())) - return + msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path)) - # if parameter is {folder} - files = var.cache.dir.get_files(parameter) - if files: - folder = parameter - if not folder.endswith('/'): - folder += '/' + send_multi_lines(bot, msgs, None) + return - msgs = [constants.strings('multiple_file_added')] - count = 0 - - for file in files: - count += 1 - music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[folder + file], user) - var.playlist.append(music_wrapper) - log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path)) - - if count != 0: - send_multi_lines(bot, msgs, None) - return - - else: - # try to do a partial match - files = var.cache.files - matches = [file for file in files if parameter.lower() in file.lower()] - if len(matches) == 1: - file = matches[0] - music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[file], user) - var.playlist.append(music_wrapper) - log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string())) - return - elif len(matches) > 1: - msgs = [constants.strings('multiple_matches')] - song_shortlist = [] - for index, match in enumerate(matches): - id = var.cache.file_id_lookup[match] - music_dict = var.music_db.query_music_by_id(id) - item = dict_to_item(bot, music_dict) - - song_shortlist.append(music_dict) - - msgs.append("{:d} - {:s} ({:s})".format( - index + 1, item.title, match)) - send_multi_lines(bot, msgs, text) - return + # try to do a partial match + matches = var.music_db.query_music(Condition() + .and_equal('type', 'file') + .and_like('path', '%' + parameter + '%', case_sensitive=False)) + if len(matches) == 1: + music_wrapper = get_cached_wrapper_from_dict(bot, matches[0], user) + var.playlist.append(music_wrapper) + log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) + bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string())) + return + elif len(matches) > 1: + song_shortlist = matches + msgs = [constants.strings('multiple_matches')] + for index, match in enumerate(matches): + msgs.append("{:d} - {:s} ({:s})".format( + index + 1, match['title'], match['path'])) + msgs.append(constants.strings("shortlist_instruction")) + send_multi_lines(bot, msgs, text) + return if do_not_refresh_cache: bot.send_msg(constants.strings("no_file"), text) @@ -287,16 +276,17 @@ def cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cach global log if parameter: - files = var.cache.files + file_dicts = var.music_db.query_music(Condition().and_equal('type', 'file')) msgs = [constants.strings('multiple_file_added') + "") + if count > ITEMS_PER_PAGE: + msgs.append(constants.strings("records_omitted")) msgs.append(constants.strings("shortlist_instruction")) send_multi_lines(bot, msgs, text, "") else: @@ -1000,6 +1010,8 @@ def cmd_search_library(bot, user, text, command, parameter): else: for item in items: count += 1 + if count > ITEMS_PER_PAGE: + break if len(item.tags) > 0: msgs.append("
  • {:d} - [{}] {} ({})
    • ".format(count, item.display_type(), item.title, ", ".join(item.tags))) else: @@ -1007,6 +1019,8 @@ def cmd_search_library(bot, user, text, command, parameter): if count != 0: msgs.append("") + if count > ITEMS_PER_PAGE: + msgs.append(constants.strings("records_omitted")) msgs.append(constants.strings("shortlist_instruction")) send_multi_lines(bot, msgs, text, "") else: diff --git a/configuration.default.ini b/configuration.default.ini index 2b67106..79e0a21 100644 --- a/configuration.default.ini +++ b/configuration.default.ini @@ -196,6 +196,8 @@ file_deleted = Deleted {item} from the library. multiple_file_added = Multiple items added: multiple_file_deleted = Multiple items deleted from the library: multiple_file_found = Found: +page_instruction = Page {current}/{total}. Use !{command} {{page}} to navigate. +records_omitted = ... bad_url = Bad URL requested. preconfigurated_radio = Preconfigurated Radio available: unable_download = Error while downloading music... diff --git a/database.py b/database.py index 5577994..f5b6e20 100644 --- a/database.py +++ b/database.py @@ -1,3 +1,4 @@ +import re import sqlite3 import json import datetime @@ -14,23 +15,32 @@ class Condition: self._limit = 0 self._offset = 0 self._order_by = "" + self.has_regex = False pass - def sql(self): + def sql(self, conn: sqlite3.Connection = None): sql = self._sql if not self._sql: sql = "TRUE" + if self._order_by: + sql += f" ORDER BY {self._order_by}" if self._limit: sql += f" LIMIT {self._limit}" if self._offset: sql += f" OFFSET {self._offset}" - if self._order_by: - sql += f" ORDEY BY {self._order_by}" + if self.has_regex and conn: + conn.create_function("REGEXP", 2, self._regexp) print(sql) - print(self.filler) return sql + @staticmethod + def _regexp(expr, item): + if not item: + return False + reg = re.compile(expr) + return reg.search(item) is not None + def or_equal(self, column, equals_to, case_sensitive=True): if not case_sensitive: column = f"LOWER({column})" @@ -87,39 +97,75 @@ class Condition: return self + def and_regexp(self, column, regex): + self.has_regex = True + + if self._sql: + self._sql += f" AND {column} REGEXP ?" + else: + self._sql += f"{column} REGEXP ?" + + self.filler.append(regex) + + return self + + def or_regexp(self, column, regex): + self.has_regex = True + + if self._sql: + self._sql += f" OR {column} REGEXP ?" + else: + self._sql += f"{column} REGEXP ?" + + self.filler.append(regex) + + return self + def or_sub_condition(self, sub_condition): + if sub_condition.has_regex: + self.has_regex = True + self.filler.extend(sub_condition.filler) if self._sql: - self._sql += f"OR ({sub_condition.sql()})" + self._sql += f" OR ({sub_condition.sql(None)})" else: - self._sql += f"({sub_condition.sql()})" + self._sql += f"({sub_condition.sql(None)})" return self def or_not_sub_condition(self, sub_condition): + if sub_condition.has_regex: + self.has_regex = True + self.filler.extend(sub_condition.filler) if self._sql: - self._sql += f"OR NOT ({sub_condition.sql()})" + self._sql += f" OR NOT ({sub_condition.sql(None)})" else: - self._sql += f"NOT ({sub_condition.sql()})" + self._sql += f"NOT ({sub_condition.sql(None)})" return self def and_sub_condition(self, sub_condition): + if sub_condition.has_regex: + self.has_regex = True + self.filler.extend(sub_condition.filler) if self._sql: - self._sql += f"AND ({sub_condition.sql()})" + self._sql += f" AND ({sub_condition.sql(None)})" else: - self._sql += f"({sub_condition.sql()})" + self._sql += f"({sub_condition.sql(None)})" return self def and_not_sub_condition(self, sub_condition): + if sub_condition.has_regex: + self.has_regex = True + self.filler.extend(sub_condition.filler) if self._sql: - self._sql += f"AND NOT({sub_condition.sql()})" + self._sql += f" AND NOT({sub_condition.sql(None)})" else: - self._sql += f"NOT ({sub_condition.sql()})" + self._sql += f"NOT ({sub_condition.sql(None)})" return self @@ -133,6 +179,11 @@ class Condition: return self + def order_by(self, order_by): + self._order_by = order_by + + return self + SETTING_DB_VERSION = 1 MUSIC_DB_VERSION = 1 @@ -369,13 +420,21 @@ class MusicDatabase: conn.commit() conn.close() - def query_all_ids(self): + def query_music_ids(self, condition: Condition): conn = sqlite3.connect(self.db_path) cursor = conn.cursor() - results = cursor.execute("SELECT id FROM music WHERE id != 'info'").fetchall() + results = cursor.execute("SELECT id FROM music WHERE id != 'info' AND %s" % + condition.sql(conn), condition.filler).fetchall() conn.close() return list(map(lambda i: i[0], results)) + def query_all_paths(self): + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + results = cursor.execute("SELECT path FROM music WHERE id != 'info'").fetchall() + conn.close() + return [ result[0] for result in results ] + def query_all_tags(self): conn = sqlite3.connect(self.db_path) cursor = conn.cursor() @@ -390,9 +449,9 @@ class MusicDatabase: def query_music_count(self, condition: Condition): filler = condition.filler - condition_str = condition.sql() conn = sqlite3.connect(self.db_path) + condition_str = condition.sql(conn) cursor = conn.cursor() results = cursor.execute("SELECT COUNT(*) FROM music " "WHERE id != 'info' AND %s" % condition_str, filler).fetchall() @@ -402,9 +461,9 @@ class MusicDatabase: def query_music(self, condition: Condition, _conn=None): filler = condition.filler - condition_str = condition.sql() conn = sqlite3.connect(self.db_path) if _conn is None else _conn + condition_str = condition.sql(conn) cursor = conn.cursor() results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music " "WHERE id != 'info' AND %s" % condition_str, filler).fetchall() @@ -461,7 +520,7 @@ class MusicDatabase: conn = sqlite3.connect(self.db_path) cursor = conn.cursor() results = cursor.execute("SELECT id, tags FROM music " - "WHERE id != 'info' AND %s" % condition.sql(), condition.filler).fetchall() + "WHERE id != 'info' AND %s" % condition.sql(conn), condition.filler).fetchall() conn.close() @@ -484,7 +543,7 @@ class MusicDatabase: results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music " "WHERE id IN (SELECT id FROM music WHERE %s ORDER BY RANDOM() LIMIT ?) ORDER BY RANDOM()" - % condition.sql(), condition.filler + [count]).fetchall() + % condition.sql(conn), condition.filler + [count]).fetchall() conn.close() return self._result_to_dict(results) @@ -514,7 +573,7 @@ class MusicDatabase: conn = sqlite3.connect(self.db_path) cursor = conn.cursor() cursor.execute("DELETE FROM music " - "WHERE %s" % condition.sql(), condition.filler) + "WHERE %s" % condition.sql(conn), condition.filler) conn.commit() conn.close() diff --git a/interface.py b/interface.py index 49b6062..c2afa26 100644 --- a/interface.py +++ b/interface.py @@ -126,24 +126,20 @@ def build_tags_color_lookup(): return color_lookup +def get_all_dirs(): + dirs = [] + paths = var.music_db.query_all_paths() + for path in paths: + pos = 0 + while True: + pos = path.find("/", pos+1) + if pos == -1: + break + folder = path[:pos] + if folder not in dirs: + dirs.append(folder) -def build_path_tags_lookup(): - path_tags_lookup = {} - ids = list(var.cache.file_id_lookup.values()) - if len(ids) > 0: - condition = Condition().and_equal("type", "file") - id_tags_lookup = var.music_db.query_tags(condition) - - for path, id in var.cache.file_id_lookup.items(): - path_tags_lookup[path] = id_tags_lookup[id] - - return path_tags_lookup - - -def recur_dir(dirobj): - for name, dir in dirobj.get_subdirs().items(): - print(dirobj.fullpath + "/" + name) - recur_dir(dir) + return dirs @web.route("/", methods=['GET']) @@ -153,17 +149,10 @@ def index(): time.sleep(0.1) tags_color_lookup = build_tags_color_lookup() - path_tags_lookup = build_path_tags_lookup() return render_template('index.html', - all_files=var.cache.files, - tags_lookup=path_tags_lookup, + dirs=get_all_dirs(), tags_color_lookup=tags_color_lookup, - music_library=var.cache.dir, - os=os, - playlist=var.playlist, - user=var.user, - paused=var.bot.is_pause, ) @@ -177,7 +166,7 @@ def playlist(): )] }) - tags_color_lookup = build_tags_color_lookup() + tags_color_lookup = build_tags_color_lookup() # TODO: cached this? items = [] for index, item_wrapper in enumerate(var.playlist): @@ -384,18 +373,7 @@ def build_library_query_condition(form): folder = form['dir'] if not folder.endswith('/') and folder: folder += '/' - sub_cond = Condition() - count = 0 - for file in var.cache.files: - if file.startswith(folder): - count += 1 - sub_cond.or_equal("id", var.cache.file_id_lookup[file]) - if count > 900: - break - if count > 0: - condition.and_sub_condition(sub_cond) - else: - condition.and_equal("id", None) + condition.and_like('path', folder + '%') tags = form['tags'].split(",") for tag in tags: diff --git a/media/cache.py b/media/cache.py index 4189e59..b16e4eb 100644 --- a/media/cache.py +++ b/media/cache.py @@ -19,9 +19,6 @@ class MusicCache(dict): super().__init__() self.db = db self.log = logging.getLogger("bot") - self.dir = None - self.files = [] - self.file_id_lookup = {} # TODO: Now I see this is silly. Gonna add a column "path" in the database. self.dir_lock = threading.Lock() def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message! @@ -90,12 +87,7 @@ class MusicCache(dict): if item: self.log.debug("library: DELETE item from the database: %s" % item.format_debug_string()) - if item.type == 'file' and item.path in self.file_id_lookup: - if item.path in self.file_id_lookup: - del self.file_id_lookup[item.path] - self.files.remove(item.path) - self.save_dir_cache() - elif item.type == 'url': + if item.type == 'url': if os.path.exists(item.path): os.remove(item.path) @@ -115,9 +107,7 @@ class MusicCache(dict): def build_dir_cache(self, bot): self.dir_lock.acquire() self.log.info("library: rebuild directory cache") - self.files = [] files = util.get_recursive_file_list_sorted(var.music_folder) - self.dir = util.Dir(var.music_folder) for file in files: item = self.fetch(bot, item_id_generators['file'](path=file)) if not item: @@ -125,25 +115,6 @@ class MusicCache(dict): self.log.debug("library: music save into database: %s" % item.format_debug_string()) self.db.insert_music(item.to_dict()) - self.dir.add_file(file) - self.files.append(file) - self.file_id_lookup[file] = item.id - - self.save_dir_cache() - self.dir_lock.release() - - def save_dir_cache(self): - var.db.set("dir_cache", "files", json.dumps(self.file_id_lookup)) - - def load_dir_cache(self, bot): - self.dir_lock.acquire() - self.log.info("library: load directory cache from database") - loaded = json.loads(var.db.get("dir_cache", "files")) - self.files = loaded.keys() - self.file_id_lookup = loaded - self.dir = util.Dir(var.music_folder) - for file, id in loaded.items(): - self.dir.add_file(file) self.dir_lock.release() @@ -223,9 +194,18 @@ class CachedItemWrapper: # Remember!!! Get wrapper functions will automatically add items into the cache! def get_cached_wrapper(item, user): - var.cache[item.id] = item - return CachedItemWrapper(var.cache, item.id, item.type, user) + if item: + var.cache[item.id] = item + return CachedItemWrapper(var.cache, item.id, item.type, user) + return None +def get_cached_wrappers(items, user): + wrappers = [] + for item in items: + if item: + wrappers.append(get_cached_wrapper(item, user)) + + return wrappers def get_cached_wrapper_from_scrap(bot, **kwargs): item = var.cache.get_item(bot, **kwargs) @@ -233,19 +213,24 @@ def get_cached_wrapper_from_scrap(bot, **kwargs): raise KeyError("Which user added this song?") return CachedItemWrapper(var.cache, item.id, kwargs['type'], kwargs['user']) - def get_cached_wrapper_from_dict(bot, dict_from_db, user): - item = dict_to_item(bot, dict_from_db) - return get_cached_wrapper(item, user) + if dict_from_db: + item = dict_to_item(bot, dict_from_db) + return get_cached_wrapper(item, user) + return None +def get_cached_wrappers_from_dicts(bot, dicts_from_db, user): + items = [] + for dict_from_db in dicts_from_db: + if dict_from_db: + items.append(get_cached_wrapper_from_dict(bot, dict_from_db, user)) + + return items def get_cached_wrapper_by_id(bot, id, user): item = var.cache.get_item_by_id(bot, id) if item: return CachedItemWrapper(var.cache, item.id, item.type, user) - else: - return None - def get_cached_wrappers_by_tags(bot, tags, user): items = var.cache.get_items_by_tags(bot, tags) diff --git a/mumbleBot.py b/mumbleBot.py index 417072b..a49be3a 100644 --- a/mumbleBot.py +++ b/mumbleBot.py @@ -680,8 +680,6 @@ if __name__ == '__main__': if var.config.get("bot", "refresh_cache_on_startup", fallback=True)\ or not var.db.has_option("dir_cache", "files"): var.cache.build_dir_cache(var.bot) - else: - var.cache.load_dir_cache(var.bot) # load playlist if var.config.getboolean('bot', 'save_playlist', fallback=True): diff --git a/templates/index.html b/templates/index.html index f98f1d6..de3a2a2 100644 --- a/templates/index.html +++ b/templates/index.html @@ -136,7 +136,7 @@
    @@ -298,7 +298,7 @@ - {% for dir in music_library.get_subdirs_recursively() %} + {% for dir in dirs %}