feat: directory cache

This commit is contained in:
Terry Geng 2020-03-07 15:12:22 +08:00
parent 8e6a639e57
commit 4fce3b956e
8 changed files with 174 additions and 107 deletions

View File

@ -9,8 +9,8 @@ import media.system
import util
import variables as var
from librb import radiobrowser
from database import SettingsDatabase
from media.playlist import get_item_wrapper
from database import SettingsDatabase, MusicDatabase
from media.playlist import get_item_wrapper, get_item_wrapper_by_id
from media.file import FileItem
from media.url_from_playlist import PlaylistURLItem, get_playlist_info
from media.url import URLItem
@ -54,21 +54,22 @@ def register_all_commands(bot):
bot.register_command(constants.commands('random'), cmd_random)
bot.register_command(constants.commands('repeat'), cmd_repeat)
bot.register_command(constants.commands('mode'), cmd_mode)
bot.register_command(constants.commands('drop_database'), cmd_drop_database)
bot.register_command(constants.commands('drop_database'), cmd_drop_database, True)
bot.register_command(constants.commands('recache'), cmd_refresh_cache, True)
# Just for debug use
bot.register_command('rtrms', cmd_real_time_rms)
bot.register_command('loop', cmd_loop_state)
bot.register_command('item', cmd_item)
bot.register_command('rtrms', cmd_real_time_rms, True)
bot.register_command('loop', cmd_loop_state, True)
bot.register_command('item', cmd_item, True)
def send_multi_lines(bot, lines, text):
def send_multi_lines(bot, lines, text, linebreak="<br />"):
global log
msg = ""
br = ""
for newline in lines:
msg += br
br = "<br>"
br = linebreak
if (len(msg) + len(newline)) > (bot.mumble.get_max_message_length() - 4) != 0: # 4 == len("<br>")
bot.send_msg(msg, text)
msg = ""
@ -163,106 +164,113 @@ def cmd_pause(bot, user, text, command, parameter):
bot.send_msg(constants.strings('paused'))
def cmd_play_file(bot, user, text, command, parameter):
def cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=False):
global log
# if parameter is {index}
if parameter.isdigit():
files = util.get_recursive_file_list_sorted(var.music_folder)
files = var.library.files
if int(parameter) < len(files):
filename = files[int(parameter)].replace(var.music_folder, '')
music_wrapper = get_item_wrapper(bot, type='file', path=filename, user=user)
music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[files[int(parameter)]], user)
var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text)
return
# if parameter is {path}
else:
# sanitize "../" and so on
path = os.path.abspath(os.path.join(var.music_folder, parameter))
if not path.startswith(os.path.abspath(var.music_folder)):
bot.send_msg(constants.strings('no_file'), text)
return
# path = os.path.abspath(os.path.join(var.music_folder, parameter))
# if not path.startswith(os.path.abspath(var.music_folder)):
# bot.send_msg(constants.strings('no_file'), text)
# return
if os.path.isfile(path):
if parameter in var.library.files:
music_wrapper = get_item_wrapper(bot, type='file', path=parameter, user=user)
var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text)
return
# if parameter is {folder}
elif os.path.isdir(path):
if parameter != '.' and parameter != './':
if not parameter.endswith("/"):
parameter += "/"
else:
parameter = ""
files = util.get_recursive_file_list_sorted(var.music_folder)
music_library = util.Dir(var.music_folder)
for file in files:
music_library.add_file(file)
files = music_library.get_files(parameter)
files = var.library.dir.get_files(parameter)
if files:
msgs = [constants.strings('multiple_file_added')]
count = 0
for file in files:
count += 1
music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[file],user)
var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path))
if count != 0:
send_multi_lines(bot, msgs, text)
else:
bot.send_msg(constants.strings('no_file'), text)
return
else:
# try to do a partial match
files = util.get_recursive_file_list_sorted(var.music_folder)
files = var.library.files
matches = [(index, file) for index, file in enumerate(files) if parameter.lower() in file.lower()]
if len(matches) == 0:
bot.send_msg(constants.strings('no_file'), text)
elif len(matches) == 1:
if len(matches) == 1:
file = matches[0][1]
music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[file],user)
var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string(user)), text)
else:
msgs = [ constants.strings('multiple_matches')]
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()), text)
return
elif len(matches) > 1:
msgs = [ constants.strings('multiple_matches') ]
for match in matches:
msgs.append("<b>{:0>3d}</b> - {:s}".format(match[0], match[1]))
music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[match[1]], user)
msgs.append("<b>{:0>3d}</b> - <b>{:s}</b> ({:s})".format(
match[0], music_wrapper.item().title, match[1]))
send_multi_lines(bot, msgs, text)
return
if do_not_refresh_cache:
bot.send_msg(constants.strings("no_file"), text)
else:
var.library.build_dir_cache(bot)
cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=True)
def cmd_play_file_match(bot, user, text, command, parameter):
def cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cache=False):
global log
music_folder = var.music_folder
if parameter:
files = util.get_recursive_file_list_sorted(music_folder)
msgs = [ constants.strings('multiple_file_added')]
files = var.library.files
msgs = [ constants.strings('multiple_file_added') + "<ul>"]
count = 0
try:
music_wrappers = []
for file in files:
match = re.search(parameter, file)
if match:
if match and match[0]:
count += 1
music_wrapper = get_item_wrapper(bot, type='file', path=file, user=user)
music_wrapper = get_item_wrapper_by_id(bot, var.library.file_id_lookup[file], user)
music_wrappers.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path))
msgs.append("<li><b>{}</b> ({})</li>".format(music_wrapper.item().title,
file[:match.span()[0]]
+ "<b style='color:pink'>"
+ file[match.span()[0]: match.span()[1]]
+ "</b>"
+ file[match.span()[1]:]
))
if count != 0:
msgs.append("</ul>")
var.playlist.extend(music_wrappers)
send_multi_lines(bot, msgs, text)
send_multi_lines(bot, msgs, text, "")
else:
bot.send_msg(constants.strings('no_file'), text)
if do_not_refresh_cache:
bot.send_msg(constants.strings("no_file"), text)
else:
var.library.build_dir_cache(bot)
cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cache=True)
except re.error as e:
msg = constants.strings('wrong_pattern', error=str(e))
@ -678,9 +686,7 @@ def cmd_remove(bot, user, text, command, parameter):
def cmd_list_file(bot, user, text, command, parameter):
global log
folder_path = var.music_folder
files = util.get_recursive_file_list_sorted(folder_path)
files = var.library.files
msgs = [ "<br> <b>Files available:</b>" if not parameter else "<br> <b>Matched files:</b>" ]
try:
count = 0
@ -714,7 +720,7 @@ def cmd_queue(bot, user, text, command, parameter):
for i, music in enumerate(var.playlist):
newline = ''
if i == var.playlist.current_index:
newline = '<b>{} ▶ ({}) {} ◀</b>'.format(i + 1, music.display_type(),
newline = "<b style='color:orange'>{} ({}) {} </b>".format(i + 1, music.display_type(),
music.format_short_string())
else:
newline = '<b>{}</b> ({}) {}'.format(i + 1, music.display_type(),
@ -745,7 +751,7 @@ def cmd_repeat(bot, user, text, command, parameter):
)
log.info("bot: add to playlist: " + music.format_debug_string)
bot.send_msg(constants.strings("repeat", song=music.format_song_string, n=str(repeat)), text)
bot.send_msg(constants.strings("repeat", song=music.format_song_string(), n=str(repeat)), text)
def cmd_mode(bot, user, text, command, parameter):
global log
@ -770,8 +776,17 @@ def cmd_drop_database(bot, user, text, command, parameter):
var.db.drop_table()
var.db = SettingsDatabase(var.dbfile)
var.music_db.drop_table()
var.music_db = MusicDatabase(var.dbfile)
log.info("command: database dropped.")
bot.send_msg(constants.strings('database_dropped'), text)
def cmd_refresh_cache(bot, user, text, command, parameter):
global log
var.library.build_dir_cache(bot)
log.info("command: cache refreshed.")
bot.send_msg(constants.strings('cache_refreshed'), text)
# Just for debug use
def cmd_real_time_rms(bot, user, text, command, parameter):
bot._display_rms = not bot._display_rms

View File

@ -61,6 +61,13 @@ announce_current_music = True
allow_other_channel_message = False
allow_private_message = True
# 'save_music_library': If this is set True, the bot will save the metadata of music into the database.
save_music_library = True
# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up.
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
refresh_cache_on_startup = True
# If save_playlist is set True, the bot will save current
# playlist before quitting and reload it the next time it start.
save_playlist = True
@ -159,6 +166,7 @@ ducking_threshold = duckthres
ducking_volume = duckv
drop_database = dropdatabase
recache = recache
[strings]
current_volume = Current volume: {volume}.
@ -178,19 +186,19 @@ bad_url = Bad URL requested.
preconfigurated_radio = Preconfigurated Radio available:
unable_download = Error while downloading music...
which_command = Do you mean <br /> {commands}
multiple_matches = Track not found! Possible candidates:
multiple_matches = File not found! Possible candidates:
queue_contents = Items on the playlist:
queue_empty = Playlist is empty!
invalid_index = Invalid index <i>{index}</i>. Use '!queue' to see your playlist.
now_playing = Playing <br />{item}
now_playing = Playing {item}
radio = Radio
file = File
url_from_playlist = URL
url = URL
radio_item = <a href="{url}">{title}</a> <i>from</i> {name} <i>added by</i> {user}
file_item = {artist} - {title} <i>added by</i> {user}
url_from_playlist_item = <a href="{url}">{title}</a> <i>from playlist</i> <a href="{playlist_url}">{playlist}</a> <i>added by</i> {user}
url_item = <a href="{url}">{title}</a> <i>added by</i> {user}
radio_item = <a href="{url}"><b>{title}</b></a> <i>from</i> {name} <i>added by</i> {user}
file_item = <b>{artist} - {title}</b> <i>added by</i> {user}
url_from_playlist_item = <a href="{url}"><b>{title}</b></a> <i>from playlist</i> <a href="{playlist_url}">{playlist}</a> <i>added by</i> {user}
url_item = <a href="{url}"><b>{title}</b></a> <i>added by</i> {user}
not_in_my_channel = You're not in my channel, command refused!
pm_not_allowed = Private message aren't allowed.
too_long = {song} is too long, removed from playlist!
@ -216,6 +224,7 @@ yt_result = Youtube query result: {result_table} Use <i>!ytplay</i> {{index}} to
yt_no_more = No more results!
yt_query_error = Unable to query youtube!
playlist_fetching_failed = Unable to fetch the playlist!
cache_refreshed = Cache refreshed!
help = <h3>Commands</h3>
<b>Control</b>
@ -266,7 +275,9 @@ admin_help = <h3>Admin command</h3>
<li><b>!<u>useru</u>nban </b> {user} - unban a user</li>
<li><b>!<u>urlb</u>an </b> {url} - ban an url</li>
<li><b>!<u>urlu</u>nban </b> {url} - unban an url</li>
<li><b>!dropdatabase</b> - clear the entire database, YOU SHOULD KNOW WHAT YOU ARE DOING.</li>
<li><b>!<u>urlu</u>nban </b> {url} - unban an url</li>
<li><b>!recache </b> {url} - rebuild local music file cache</li>
<li><b>!dropdatabase</b> - clear the entire database, you will lose all settings and music library.</li>
</ul>

View File

@ -74,6 +74,10 @@ port = 64738
# 'save_music_library': If this is set True, the bot will save the metadata of music into the database.
#save_music_library = True
# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up.
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
#refresh_cache_on_startup = True
# 'save_playlist': If save_playlist is set True, the bot will save current playlist before quitting
# and reload it the next time it start. It requires save_music_library to be True to function.
#save_playlist = True

View File

@ -238,3 +238,10 @@ class MusicDatabase:
"WHERE %s" % condition_str, filler)
conn.commit()
conn.close()
def drop_table(self):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("DROP TABLE music")
conn.close()

View File

@ -4,22 +4,15 @@ from functools import wraps
from flask import Flask, render_template, request, redirect, send_file, Response, jsonify, abort
import variables as var
import util
from datetime import datetime
import os
import os.path
import shutil
import random
from werkzeug.utils import secure_filename
import errno
import media
from media.playlist import get_item_wrapper
from media.file import FileItem
from media.url_from_playlist import PlaylistURLItem, get_playlist_info
from media.url import URLItem
from media.radio import RadioItem
from media.playlist import get_item_wrapper, get_item_wrapper_by_id
import logging
import time
import constants
class ReverseProxied(object):
@ -101,16 +94,9 @@ def requires_auth(f):
@web.route("/", methods=['GET'])
@requires_auth
def index():
folder_path = var.music_folder
files = util.get_recursive_file_list_sorted(var.music_folder)
music_library = util.Dir(folder_path)
for file in files:
music_library.add_file(file)
return render_template('index.html',
all_files=files,
music_library=music_library,
all_files=var.library.files,
music_library=var.library.dir,
os=os,
playlist=var.playlist,
user=var.user,
@ -157,14 +143,13 @@ def status():
def post():
global log
folder_path = var.music_folder
if request.method == 'POST':
if request.form:
log.debug("web: Post request from %s: %s" % ( request.remote_addr, str(request.form)))
if 'add_file_bottom' in request.form and ".." not in request.form['add_file_bottom']:
path = var.music_folder + request.form['add_file_bottom']
if os.path.isfile(path):
music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_bottom'], user=user)
music_wrapper = get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[request.form['add_file_bottom']], user)
var.playlist.append(music_wrapper)
log.info('web: add to playlist(bottom): ' + music_wrapper.format_debug_string())
@ -172,7 +157,7 @@ def post():
elif 'add_file_next' in request.form and ".." not in request.form['add_file_next']:
path = var.music_folder + request.form['add_file_next']
if os.path.isfile(path):
music_wrapper = get_item_wrapper(var.bot, type='file', path=request.form['add_file_next'], user=user)
music_wrapper = get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[request.form['add_file_next']], user)
var.playlist.insert(var.playlist.current_index + 1, music_wrapper)
log.info('web: add to playlist(next): ' + music_wrapper.format_debug_string())
@ -186,19 +171,15 @@ def post():
folder += '/'
if os.path.isdir(var.music_folder + folder):
files = util.get_recursive_file_list_sorted(var.music_folder)
music_library = util.Dir(folder_path)
for file in files:
music_library.add_file(file)
dir = var.library.dir
if 'add_folder_recursively' in request.form:
files = music_library.get_files_recursively(folder)
files = dir.get_files_recursively(folder)
else:
files = music_library.get_files(folder)
files = dir.get_files(folder)
music_wrappers = list(map(
lambda file: get_item_wrapper(var.bot, type='file', path=file, user=user),
lambda file:
get_item_wrapper_by_id(var.bot, var.library.file_id_lookup[folder + file], user),
files))
var.playlist.extend(music_wrappers)
@ -370,7 +351,7 @@ def download():
log.info('web: Download of file %s requested from %s:' % (requested_file, request.remote_addr))
if '../' not in requested_file:
folder_path = var.music_folder
files = util.get_recursive_file_list_sorted(var.music_folder)
files = var.library.files
if requested_file in files:
filepath = os.path.join(folder_path, requested_file)

View File

@ -1,13 +1,11 @@
import logging
from database import MusicDatabase
import json
from media.item import item_builders, item_loaders, item_id_generators
from media.file import FileItem
from media.url import URLItem
from media.url_from_playlist import PlaylistURLItem
from media.radio import RadioItem
from database import MusicDatabase
import variables as var
import util
class MusicLibrary(dict):
@ -15,8 +13,10 @@ class MusicLibrary(dict):
super().__init__()
self.db = db
self.log = logging.getLogger("bot")
self.dir = None
self.files = []
def get_item_by_id(self, bot, id):
def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message!
if id in self:
return self[id]
@ -26,6 +26,9 @@ class MusicLibrary(dict):
self[id] = item
self.log.debug("library: music found in database: %s" % item.format_debug_string())
return item
else:
raise KeyError("Unable to fetch item from the database! Please try to refresh the cache by !recache.")
def get_item(self, bot, **kwargs):
# kwargs should provide type and id, and parameters to build the item if not existed in the library.
@ -59,8 +62,13 @@ class MusicLibrary(dict):
self.log.debug("library: music save into database: %s" % self[id].format_debug_string())
self.db.insert_music(self[id].to_dict())
def delete(self, id):
self.db.delete_music(id=id)
def delete(self, item):
if item.type == 'file' and item.path in self.file_id_lookup:
del self.file_id_lookup[item.path]
self.files.remove(item.path)
self.save_dir_cache()
self.db.delete_music(id=item.id)
def free(self, id):
if id in self:
@ -68,3 +76,31 @@ class MusicLibrary(dict):
def free_all(self):
self.clear()
def build_dir_cache(self, bot):
self.log.info("library: rebuild directory cache")
self.files = []
self.file_id_lookup = {}
files = util.get_recursive_file_list_sorted(var.music_folder)
self.dir = util.Dir(var.music_folder)
for file in files:
item = self.get_item(bot, type='file', path=file)
if item.validate():
self.dir.add_file(file)
self.files.append(file)
self.file_id_lookup[file] = item.id
self.save_dir_cache()
def save_dir_cache(self):
var.db.set("dir_cache", "files", json.dumps(self.file_id_lookup))
def load_dir_cache(self, bot):
self.log.info("library: load directory cache from database")
loaded = json.loads(var.db.get("dir_cache", "files"))
self.files = loaded.keys()
self.file_id_lookup = loaded
self.dir = util.Dir(var.music_folder)
for file, id in loaded.items():
self.dir.add_file(file)

View File

@ -298,6 +298,7 @@ class BasePlaylist(list):
if not item.validate() or item.is_failed():
self.log.debug("playlist: validating failed.")
self.remove_by_id(item.id)
var.library.delete(item.item())
self.log.debug("playlist: validating finished.")
self.validating_thread_lock.release()
@ -422,6 +423,10 @@ class AutoPlaylist(BasePlaylist):
# self.refresh()
# return self
def clear(self):
super().clear()
self.refresh()
def next(self):
if len(self) == 0:
return False

View File

@ -184,12 +184,12 @@ class MumbleBot:
else:
self.log.debug("update: no new version found.")
def register_command(self, cmd, handle):
def register_command(self, cmd, handle, no_partial_match=False):
cmds = cmd.split(",")
for command in cmds:
command = command.strip()
if command:
self.cmd_handle[command] = handle
self.cmd_handle[command] = { 'handle': handle, 'partial_match': not no_partial_match}
self.log.debug("bot: command added: " + command)
def set_comment(self):
@ -254,19 +254,19 @@ class MumbleBot:
try:
if command in self.cmd_handle:
command_exc = command
self.cmd_handle[command](self, user, text, command, parameter)
self.cmd_handle[command]['handle'](self, user, text, command, parameter)
else:
# try partial match
cmds = self.cmd_handle.keys()
matches = []
for cmd in cmds:
if cmd.startswith(command):
if cmd.startswith(command) and self.cmd_handle[cmd]['partial_match']:
matches.append(cmd)
if len(matches) == 1:
self.log.info("bot: {:s} matches {:s}".format(command, matches[0]))
command_exc = matches[0]
self.cmd_handle[command_exc](self, user, text, command_exc, parameter)
self.cmd_handle[command_exc]['handle'](self, user, text, command_exc, parameter)
elif len(matches) > 1:
self.mumble.users[text.actor].send_text_message(
constants.strings('which_command', commands="<br>".join(matches)))
@ -347,6 +347,7 @@ class MumbleBot:
break
else:
var.playlist.remove_by_id(next.id)
var.library.delete(next.item())
# =======================
@ -406,6 +407,7 @@ class MumbleBot:
self.send_msg(constants.strings('download_in_progress', item=current.format_short_string()))
else:
var.playlist.remove_by_id(current.id)
var.library.delete(current.item())
else:
self._loop_status = 'Empty queue'
else:
@ -654,6 +656,12 @@ if __name__ == '__main__':
var.bot = MumbleBot(args)
command.register_all_commands(var.bot)
if var.config.get("bot", "refresh_cache_on_startup", fallback=True)\
or not var.db.has_option("dir_cache", "files"):
var.library.build_dir_cache(var.bot)
else:
var.library.load_dir_cache(var.bot)
# load playlist
if var.config.getboolean('bot', 'save_playlist', fallback=True):
var.bot_logger.info("bot: load playlist from previous session")