From 4fce3b956ede3d83d1661ebd04b248d5685fe5f4 Mon Sep 17 00:00:00 2001 From: Terry Geng Date: Sat, 7 Mar 2020 15:12:22 +0800 Subject: [PATCH] feat: directory cache --- command.py | 129 +++++++++++++++++++++----------------- configuration.default.ini | 25 +++++--- configuration.example.ini | 4 ++ database.py | 7 +++ interface.py | 41 ++++-------- media/library.py | 52 ++++++++++++--- media/playlist.py | 5 ++ mumbleBot.py | 18 ++++-- 8 files changed, 174 insertions(+), 107 deletions(-) diff --git a/command.py b/command.py index 305edba..da821db 100644 --- a/command.py +++ b/command.py @@ -9,8 +9,8 @@ import media.system import util import variables as var from librb import radiobrowser -from database import SettingsDatabase -from media.playlist import get_item_wrapper +from database import SettingsDatabase, MusicDatabase +from media.playlist import get_item_wrapper, get_item_wrapper_by_id from media.file import FileItem from media.url_from_playlist import PlaylistURLItem, get_playlist_info from media.url import URLItem @@ -54,21 +54,22 @@ def register_all_commands(bot): bot.register_command(constants.commands('random'), cmd_random) bot.register_command(constants.commands('repeat'), cmd_repeat) bot.register_command(constants.commands('mode'), cmd_mode) - bot.register_command(constants.commands('drop_database'), cmd_drop_database) + bot.register_command(constants.commands('drop_database'), cmd_drop_database, True) + bot.register_command(constants.commands('recache'), cmd_refresh_cache, True) # Just for debug use - bot.register_command('rtrms', cmd_real_time_rms) - bot.register_command('loop', cmd_loop_state) - bot.register_command('item', cmd_item) + bot.register_command('rtrms', cmd_real_time_rms, True) + bot.register_command('loop', cmd_loop_state, True) + bot.register_command('item', cmd_item, True) -def send_multi_lines(bot, lines, text): +def send_multi_lines(bot, lines, text, linebreak="
"): global log msg = "" br = "" for newline in lines: msg += br - br = "
" + br = linebreak if (len(msg) + len(newline)) > (bot.mumble.get_max_message_length() - 4) != 0: # 4 == len("
") bot.send_msg(msg, text) msg = "" @@ -163,106 +164,113 @@ def cmd_pause(bot, user, text, command, parameter): bot.send_msg(constants.strings('paused')) -def cmd_play_file(bot, user, text, command, parameter): +def cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=False): global log # if parameter is {index} if parameter.isdigit(): - files = util.get_recursive_file_list_sorted(var.music_folder) + files = var.library.files if int(parameter) < len(files): - filename = files[int(parameter)].replace(var.music_folder, '') - music_wrapper = get_item_wrapper(bot, type='file', path=filename, user=user) + music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[files[int(parameter)]], user) var.playlist.append(music_wrapper) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text) + bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text) + return # if parameter is {path} else: # sanitize "../" and so on - path = os.path.abspath(os.path.join(var.music_folder, parameter)) - if not path.startswith(os.path.abspath(var.music_folder)): - bot.send_msg(constants.strings('no_file'), text) - return + # path = os.path.abspath(os.path.join(var.music_folder, parameter)) + # if not path.startswith(os.path.abspath(var.music_folder)): + # bot.send_msg(constants.strings('no_file'), text) + # return - if os.path.isfile(path): + if parameter in var.library.files: music_wrapper = get_item_wrapper(bot, type='file', path=parameter, user=user) var.playlist.append(music_wrapper) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text) + bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text) return # if parameter is {folder} - elif os.path.isdir(path): - if parameter != '.' and parameter != './': - if not parameter.endswith("/"): - parameter += "/" - else: - parameter = "" - - files = util.get_recursive_file_list_sorted(var.music_folder) - music_library = util.Dir(var.music_folder) - for file in files: - music_library.add_file(file) - - files = music_library.get_files(parameter) + files = var.library.dir.get_files(parameter) + if files: msgs = [constants.strings('multiple_file_added')] count = 0 for file in files: count += 1 - music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user) + music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[file],user) var.playlist.append(music_wrapper) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path)) if count != 0: send_multi_lines(bot, msgs, text) - else: - bot.send_msg(constants.strings('no_file'), text) + return else: # try to do a partial match - files = util.get_recursive_file_list_sorted(var.music_folder) + files = var.library.files matches = [(index, file) for index, file in enumerate(files) if parameter.lower() in file.lower()] - if len(matches) == 0: - bot.send_msg(constants.strings('no_file'), text) - elif len(matches) == 1: + if len(matches) == 1: file = matches[0][1] - music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user) + music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[file],user) var.playlist.append(music_wrapper) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) - bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text) - else: - msgs = [ constants.strings('multiple_matches')] + bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text) + return + elif len(matches) > 1: + msgs = [ constants.strings('multiple_matches') ] for match in matches: - msgs.append("{:0>3d} - {:s}".format(match[0], match[1])) + music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[match[1]], user) + msgs.append("{:0>3d} - {:s} ({:s})".format( + match[0], music_wrapper.item().title, match[1])) send_multi_lines(bot, msgs, text) + return + + if do_not_refresh_cache: + bot.send_msg(constants.strings("no_file"), text) + else: + var.library.build_dir_cache(bot) + cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=True) -def cmd_play_file_match(bot, user, text, command, parameter): +def cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cache=False): global log music_folder = var.music_folder if parameter: - files = util.get_recursive_file_list_sorted(music_folder) - msgs = [ constants.strings('multiple_file_added')] + files = var.library.files + msgs = [ constants.strings('multiple_file_added') + "") var.playlist.extend(music_wrappers) - send_multi_lines(bot, msgs, text) + send_multi_lines(bot, msgs, text, "") else: - bot.send_msg(constants.strings('no_file'), text) + if do_not_refresh_cache: + bot.send_msg(constants.strings("no_file"), text) + else: + var.library.build_dir_cache(bot) + cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cache=True) except re.error as e: msg = constants.strings('wrong_pattern', error=str(e)) @@ -678,9 +686,7 @@ def cmd_remove(bot, user, text, command, parameter): def cmd_list_file(bot, user, text, command, parameter): global log - folder_path = var.music_folder - - files = util.get_recursive_file_list_sorted(folder_path) + files = var.library.files msgs = [ "
Files available:" if not parameter else "
Matched files:" ] try: count = 0 @@ -714,7 +720,7 @@ def cmd_queue(bot, user, text, command, parameter): for i, music in enumerate(var.playlist): newline = '' if i == var.playlist.current_index: - newline = '{} ▶ ({}) {} ◀'.format(i + 1, music.display_type(), + newline = "{} ({}) {} ".format(i + 1, music.display_type(), music.format_short_string()) else: newline = '{} ({}) {}'.format(i + 1, music.display_type(), @@ -745,7 +751,7 @@ def cmd_repeat(bot, user, text, command, parameter): ) log.info("bot: add to playlist: " + music.format_debug_string) - bot.send_msg(constants.strings("repeat", song=music.format_song_string, n=str(repeat)), text) + bot.send_msg(constants.strings("repeat", song=music.format_song_string(), n=str(repeat)), text) def cmd_mode(bot, user, text, command, parameter): global log @@ -770,8 +776,17 @@ def cmd_drop_database(bot, user, text, command, parameter): var.db.drop_table() var.db = SettingsDatabase(var.dbfile) + var.music_db.drop_table() + var.music_db = MusicDatabase(var.dbfile) + log.info("command: database dropped.") bot.send_msg(constants.strings('database_dropped'), text) +def cmd_refresh_cache(bot, user, text, command, parameter): + global log + var.library.build_dir_cache(bot) + log.info("command: cache refreshed.") + bot.send_msg(constants.strings('cache_refreshed'), text) + # Just for debug use def cmd_real_time_rms(bot, user, text, command, parameter): bot._display_rms = not bot._display_rms diff --git a/configuration.default.ini b/configuration.default.ini index 1d102ec..2feaef3 100644 --- a/configuration.default.ini +++ b/configuration.default.ini @@ -61,6 +61,13 @@ announce_current_music = True allow_other_channel_message = False allow_private_message = True +# 'save_music_library': If this is set True, the bot will save the metadata of music into the database. +save_music_library = True + +# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up. +# But it won't reload metadata from each files. If set to False, it will used the cache last time. +refresh_cache_on_startup = True + # If save_playlist is set True, the bot will save current # playlist before quitting and reload it the next time it start. save_playlist = True @@ -159,6 +166,7 @@ ducking_threshold = duckthres ducking_volume = duckv drop_database = dropdatabase +recache = recache [strings] current_volume = Current volume: {volume}. @@ -178,19 +186,19 @@ bad_url = Bad URL requested. preconfigurated_radio = Preconfigurated Radio available: unable_download = Error while downloading music... which_command = Do you mean
{commands} -multiple_matches = Track not found! Possible candidates: +multiple_matches = File not found! Possible candidates: queue_contents = Items on the playlist: queue_empty = Playlist is empty! invalid_index = Invalid index {index}. Use '!queue' to see your playlist. -now_playing = Playing
{item} +now_playing = Playing {item} radio = Radio file = File url_from_playlist = URL url = URL -radio_item = {title} from {name} added by {user} -file_item = {artist} - {title} added by {user} -url_from_playlist_item = {title} from playlist {playlist} added by {user} -url_item = {title} added by {user} +radio_item = {title} from {name} added by {user} +file_item = {artist} - {title} added by {user} +url_from_playlist_item = {title} from playlist {playlist} added by {user} +url_item = {title} added by {user} not_in_my_channel = You're not in my channel, command refused! pm_not_allowed = Private message aren't allowed. too_long = {song} is too long, removed from playlist! @@ -216,6 +224,7 @@ yt_result = Youtube query result: {result_table} Use !ytplay {{index}} to yt_no_more = No more results! yt_query_error = Unable to query youtube! playlist_fetching_failed = Unable to fetch the playlist! +cache_refreshed = Cache refreshed! help =

Commands

Control @@ -266,7 +275,9 @@ admin_help =

Admin command

  • !userunban {user} - unban a user
  • !urlban {url} - ban an url
  • !urlunban {url} - unban an url
    • -
  • !dropdatabase - clear the entire database, YOU SHOULD KNOW WHAT YOU ARE DOING.
    • +
  • !urlunban {url} - unban an url
    • +
  • !recache {url} - rebuild local music file cache
    • +
  • !dropdatabase - clear the entire database, you will lose all settings and music library.
    • diff --git a/configuration.example.ini b/configuration.example.ini index edccee7..9b45300 100644 --- a/configuration.example.ini +++ b/configuration.example.ini @@ -74,6 +74,10 @@ port = 64738 # 'save_music_library': If this is set True, the bot will save the metadata of music into the database. #save_music_library = True +# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up. +# But it won't reload metadata from each files. If set to False, it will used the cache last time. +#refresh_cache_on_startup = True + # 'save_playlist': If save_playlist is set True, the bot will save current playlist before quitting # and reload it the next time it start. It requires save_music_library to be True to function. #save_playlist = True diff --git a/database.py b/database.py index 13f5ef9..2352f90 100644 --- a/database.py +++ b/database.py @@ -238,3 +238,10 @@ class MusicDatabase: "WHERE %s" % condition_str, filler) conn.commit() conn.close() + + + def drop_table(self): + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute("DROP TABLE music") + conn.close() diff --git a/interface.py b/interface.py index eed0f6e..36b811c 100644 --- a/interface.py +++ b/interface.py @@ -4,22 +4,15 @@ from functools import wraps from flask import Flask, render_template, request, redirect, send_file, Response, jsonify, abort import variables as var import util -from datetime import datetime import os import os.path import shutil -import random from werkzeug.utils import secure_filename import errno import media -from media.playlist import get_item_wrapper -from media.file import FileItem -from media.url_from_playlist import PlaylistURLItem, get_playlist_info -from media.url import URLItem -from media.radio import RadioItem +from media.playlist import get_item_wrapper, get_item_wrapper_by_id import logging import time -import constants class ReverseProxied(object): @@ -101,16 +94,9 @@ def requires_auth(f): @web.route("/", methods=['GET']) @requires_auth def index(): - folder_path = var.music_folder - files = util.get_recursive_file_list_sorted(var.music_folder) - music_library = util.Dir(folder_path) - for file in files: - music_library.add_file(file) - - return render_template('index.html', - all_files=files, - music_library=music_library, + all_files=var.library.files, + music_library=var.library.dir, os=os, playlist=var.playlist, user=var.user, @@ -157,14 +143,13 @@ def status(): def post(): global log - folder_path = var.music_folder if request.method == 'POST': if request.form: log.debug("web: Post request from %s: %s" % ( request.remote_addr, str(request.form))) if 'add_file_bottom' in request.form and ".." not in request.form['add_file_bottom']: path = var.music_folder + request.form['add_file_bottom'] if os.path.isfile(path): - music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_bottom'], user=user) + music_wrapper = get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[request.form['add_file_bottom']], user) var.playlist.append(music_wrapper) log.info('web: add to playlist(bottom): ' + music_wrapper.format_debug_string()) @@ -172,7 +157,7 @@ def post(): elif 'add_file_next' in request.form and ".." not in request.form['add_file_next']: path = var.music_folder + request.form['add_file_next'] if os.path.isfile(path): - music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_next'], user=user) + music_wrapper = get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[request.form['add_file_next']], user) var.playlist.insert(var.playlist.current_index + 1, music_wrapper) log.info('web: add to playlist(next): ' + music_wrapper.format_debug_string()) @@ -186,19 +171,15 @@ def post(): folder += '/' if os.path.isdir(var.music_folder + folder): - - files = util.get_recursive_file_list_sorted(var.music_folder) - music_library = util.Dir(folder_path) - for file in files: - music_library.add_file(file) - + dir = var.library.dir if 'add_folder_recursively' in request.form: - files = music_library.get_files_recursively(folder) + files = dir.get_files_recursively(folder) else: - files = music_library.get_files(folder) + files = dir.get_files(folder) music_wrappers = list(map( - lambda file: get_item_wrapper(var.bot, type='file', path=file, user=user), + lambda file: + get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[folder + file], user), files)) var.playlist.extend(music_wrappers) @@ -370,7 +351,7 @@ def download(): log.info('web: Download of file %s requested from %s:' % (requested_file, request.remote_addr)) if '../' not in requested_file: folder_path = var.music_folder - files = util.get_recursive_file_list_sorted(var.music_folder) + files = var.library.files if requested_file in files: filepath = os.path.join(folder_path, requested_file) diff --git a/media/library.py b/media/library.py index 48049ce..33c1c05 100644 --- a/media/library.py +++ b/media/library.py @@ -1,13 +1,11 @@ import logging - from database import MusicDatabase +import json + from media.item import item_builders, item_loaders, item_id_generators -from media.file import FileItem -from media.url import URLItem -from media.url_from_playlist import PlaylistURLItem -from media.radio import RadioItem from database import MusicDatabase import variables as var +import util class MusicLibrary(dict): @@ -15,8 +13,10 @@ class MusicLibrary(dict): super().__init__() self.db = db self.log = logging.getLogger("bot") + self.dir = None + self.files = [] - def get_item_by_id(self, bot, id): + def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message! if id in self: return self[id] @@ -26,6 +26,9 @@ class MusicLibrary(dict): self[id] = item self.log.debug("library: music found in database: %s" % item.format_debug_string()) return item + else: + raise KeyError("Unable to fetch item from the database! Please try to refresh the cache by !recache.") + def get_item(self, bot, **kwargs): # kwargs should provide type and id, and parameters to build the item if not existed in the library. @@ -59,8 +62,13 @@ class MusicLibrary(dict): self.log.debug("library: music save into database: %s" % self[id].format_debug_string()) self.db.insert_music(self[id].to_dict()) - def delete(self, id): - self.db.delete_music(id=id) + def delete(self, item): + if item.type == 'file' and item.path in self.file_id_lookup: + del self.file_id_lookup[item.path] + self.files.remove(item.path) + self.save_dir_cache() + + self.db.delete_music(id=item.id) def free(self, id): if id in self: @@ -68,3 +76,31 @@ class MusicLibrary(dict): def free_all(self): self.clear() + + def build_dir_cache(self, bot): + self.log.info("library: rebuild directory cache") + self.files = [] + self.file_id_lookup = {} + files = util.get_recursive_file_list_sorted(var.music_folder) + self.dir = util.Dir(var.music_folder) + for file in files: + item = self.get_item(bot, type='file', path=file) + if item.validate(): + self.dir.add_file(file) + self.files.append(file) + self.file_id_lookup[file] = item.id + + self.save_dir_cache() + + def save_dir_cache(self): + var.db.set("dir_cache", "files", json.dumps(self.file_id_lookup)) + + def load_dir_cache(self, bot): + self.log.info("library: load directory cache from database") + loaded = json.loads(var.db.get("dir_cache", "files")) + self.files = loaded.keys() + self.file_id_lookup = loaded + self.dir = util.Dir(var.music_folder) + for file, id in loaded.items(): + self.dir.add_file(file) + diff --git a/media/playlist.py b/media/playlist.py index 6b8d430..3f34af5 100644 --- a/media/playlist.py +++ b/media/playlist.py @@ -298,6 +298,7 @@ class BasePlaylist(list): if not item.validate() or item.is_failed(): self.log.debug("playlist: validating failed.") self.remove_by_id(item.id) + var.library.delete(item.item()) self.log.debug("playlist: validating finished.") self.validating_thread_lock.release() @@ -422,6 +423,10 @@ class AutoPlaylist(BasePlaylist): # self.refresh() # return self + def clear(self): + super().clear() + self.refresh() + def next(self): if len(self) == 0: return False diff --git a/mumbleBot.py b/mumbleBot.py index 058c6cc..7433a48 100644 --- a/mumbleBot.py +++ b/mumbleBot.py @@ -184,12 +184,12 @@ class MumbleBot: else: self.log.debug("update: no new version found.") - def register_command(self, cmd, handle): + def register_command(self, cmd, handle, no_partial_match=False): cmds = cmd.split(",") for command in cmds: command = command.strip() if command: - self.cmd_handle[command] = handle + self.cmd_handle[command] = { 'handle': handle, 'partial_match': not no_partial_match} self.log.debug("bot: command added: " + command) def set_comment(self): @@ -254,19 +254,19 @@ class MumbleBot: try: if command in self.cmd_handle: command_exc = command - self.cmd_handle[command](self, user, text, command, parameter) + self.cmd_handle[command]['handle'](self, user, text, command, parameter) else: # try partial match cmds = self.cmd_handle.keys() matches = [] for cmd in cmds: - if cmd.startswith(command): + if cmd.startswith(command) and self.cmd_handle[cmd]['partial_match']: matches.append(cmd) if len(matches) == 1: self.log.info("bot: {:s} matches {:s}".format(command, matches[0])) command_exc = matches[0] - self.cmd_handle[command_exc](self, user, text, command_exc, parameter) + self.cmd_handle[command_exc]['handle'](self, user, text, command_exc, parameter) elif len(matches) > 1: self.mumble.users[text.actor].send_text_message( constants.strings('which_command', commands="
    ".join(matches))) @@ -347,6 +347,7 @@ class MumbleBot: break else: var.playlist.remove_by_id(next.id) + var.library.delete(next.item()) # ======================= @@ -406,6 +407,7 @@ class MumbleBot: self.send_msg(constants.strings('download_in_progress', item=current.format_short_string())) else: var.playlist.remove_by_id(current.id) + var.library.delete(current.item()) else: self._loop_status = 'Empty queue' else: @@ -654,6 +656,12 @@ if __name__ == '__main__': var.bot = MumbleBot(args) command.register_all_commands(var.bot) + if var.config.get("bot", "refresh_cache_on_startup", fallback=True)\ + or not var.db.has_option("dir_cache", "files"): + var.library.build_dir_cache(var.bot) + else: + var.library.load_dir_cache(var.bot) + # load playlist if var.config.getboolean('bot', 'save_playlist', fallback=True): var.bot_logger.info("bot: load playlist from previous session")