storm/bragi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: removed useless files path cache

Browse Source
This commit is contained in:
Terry Geng 2020-03-22 21:58:39 +08:00
parent 91f603c15e
commit e256d50c5c
7 changed files with 220 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
command.py
View File

@ -1,5 +1,7 @@
# coding=utf-8 # coding=utf-8
import logging import logging
import math
import pymumble.pymumble_py3 as pymumble import pymumble.pymumble_py3 as pymumble
import re import re
@ -8,10 +10,10 @@ import media.system
import util import util
import variables as var import variables as var
from librb import radiobrowser from librb import radiobrowser
from database import SettingsDatabase, MusicDatabase from database import SettingsDatabase, MusicDatabase, Condition
from media.item import item_id_generators, dict_to_item, dicts_to_items from media.item import item_id_generators, dict_to_item, dicts_to_items
from media.cache import get_cached_wrapper_from_scrap, get_cached_wrapper_by_id, get_cached_wrappers_by_tags, \ from media.cache import get_cached_wrapper_from_scrap, get_cached_wrapper_by_id, get_cached_wrappers_by_tags, \
get_cached_wrapper get_cached_wrapper, get_cached_wrappers, get_cached_wrapper_from_dict, get_cached_wrappers_from_dicts
from media.url_from_playlist import get_playlist_info from media.url_from_playlist import get_playlist_info
log = logging.getLogger("bot") log = logging.getLogger("bot")
@ -89,6 +91,8 @@ def send_multi_lines(bot, lines, text, linebreak="<br />"):
# ---------------- Variables ----------------- # ---------------- Variables -----------------
ITEMS_PER_PAGE = 50
song_shortlist = [] song_shortlist = []
@ -206,75 +210,60 @@ def cmd_play_file(bot, user, text, command, parameter, do_not_refresh_cache=Fals
# if parameter is {index} # if parameter is {index}
if parameter.isdigit(): if parameter.isdigit():
files = var.cache.files music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition()
if int(parameter) < len(files): .and_equal('type', 'file')
music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[files[int(parameter)]], user) .order_by('path')
var.playlist.append(music_wrapper) .limit(1)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) .offset(int(parameter))), user)
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()))
if music_wrappers:
var.playlist.append(music_wrappers[0])
log.info("cmd: add to playlist: " + music_wrappers[0].format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrappers[0].format_song_string()))
return return
# if parameter is {path} # assume parameter is a path
else: music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition().and_equal('path', parameter)), user)
# sanitize "../" and so on if music_wrappers:
# path = os.path.abspath(os.path.join(var.music_folder, parameter)) var.playlist.append(music_wrappers[0])
# if not path.startswith(os.path.abspath(var.music_folder)): log.info("cmd: add to playlist: " + music_wrappers[0].format_debug_string())
# bot.send_msg(constants.strings('no_file'), text) bot.send_msg(constants.strings('file_added', item=music_wrappers[0].format_song_string()))
# return return
if parameter in var.cache.files: # assume parameter is a folder
music_wrapper = get_cached_wrapper_from_scrap(bot, type='file', path=parameter, user=user) music_wrappers = get_cached_wrappers_from_dicts(bot, var.music_db.query_music(Condition()
.and_equal('type', 'file')
.and_like('path', parameter + '%')), user)
if music_wrappers:
msgs = [constants.strings('multiple_file_added')]
for music_wrapper in music_wrappers:
var.playlist.append(music_wrapper) var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string())) msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path))
return
# if parameter is {folder} send_multi_lines(bot, msgs, None)
files = var.cache.dir.get_files(parameter) return
if files:
folder = parameter
if not folder.endswith('/'):
folder += '/'
msgs = [constants.strings('multiple_file_added')] # try to do a partial match
count = 0 matches = var.music_db.query_music(Condition()
.and_equal('type', 'file')
for file in files: .and_like('path', '%' + parameter + '%', case_sensitive=False))
count += 1 if len(matches) == 1:
music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[folder + file], user) music_wrapper = get_cached_wrapper_from_dict(bot, matches[0], user)
var.playlist.append(music_wrapper) var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
msgs.append("{} ({})".format(music_wrapper.item().title, music_wrapper.item().path)) bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()))
return
if count != 0: elif len(matches) > 1:
send_multi_lines(bot, msgs, None) song_shortlist = matches
return msgs = [constants.strings('multiple_matches')]
for index, match in enumerate(matches):
else: msgs.append("<b>{:d}</b> - <b>{:s}</b> ({:s})".format(
# try to do a partial match index + 1, match['title'], match['path']))
files = var.cache.files msgs.append(constants.strings("shortlist_instruction"))
matches = [file for file in files if parameter.lower() in file.lower()] send_multi_lines(bot, msgs, text)
if len(matches) == 1: return
file = matches[0]
music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[file], user)
var.playlist.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
bot.send_msg(constants.strings('file_added', item=music_wrapper.format_song_string()))
return
elif len(matches) > 1:
msgs = [constants.strings('multiple_matches')]
song_shortlist = []
for index, match in enumerate(matches):
id = var.cache.file_id_lookup[match]
music_dict = var.music_db.query_music_by_id(id)
item = dict_to_item(bot, music_dict)
song_shortlist.append(music_dict)
msgs.append("<b>{:d}</b> - <b>{:s}</b> ({:s})".format(
index + 1, item.title, match))
send_multi_lines(bot, msgs, text)
return
if do_not_refresh_cache: if do_not_refresh_cache:
bot.send_msg(constants.strings("no_file"), text) bot.send_msg(constants.strings("no_file"), text)
@ -287,16 +276,17 @@ def cmd_play_file_match(bot, user, text, command, parameter, do_not_refresh_cach
global log global log
if parameter: if parameter:
files = var.cache.files file_dicts = var.music_db.query_music(Condition().and_equal('type', 'file'))
msgs = [constants.strings('multiple_file_added') + "<ul>"] msgs = [constants.strings('multiple_file_added') + "<ul>"]
count = 0
try: try:
count = 0
music_wrappers = [] music_wrappers = []
for file in files: for file_dict in file_dicts:
file = file_dict['title']
match = re.search(parameter, file) match = re.search(parameter, file)
if match and match[0]: if match and match[0]:
count += 1 count += 1
music_wrapper = get_cached_wrapper_by_id(bot, var.cache.file_id_lookup[file], user) music_wrapper = get_cached_wrapper(dict_to_item(bot, file_dict), user)
music_wrappers.append(music_wrapper) music_wrappers.append(music_wrapper)
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string()) log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
msgs.append("<li><b>{}</b> ({})</li>".format(music_wrapper.item().title, msgs.append("<li><b>{}</b> ({})</li>".format(music_wrapper.item().title,
@ -743,7 +733,14 @@ def cmd_remove(bot, user, text, command, parameter):
def cmd_list_file(bot, user, text, command, parameter): def cmd_list_file(bot, user, text, command, parameter):
global log global log
files = var.cache.files page = 0
files = [ file['path'] for file in var.music_db.query_music(Condition()
.and_equal('type', 'file')
.order_by('path')
.limit(ITEMS_PER_PAGE)
.offset(page * ITEMS_PER_PAGE)) ]
msgs = [constants.strings("multiple_file_found")] msgs = [constants.strings("multiple_file_found")]
try: try:
count = 0 count = 0
@ -754,9 +751,13 @@ def cmd_list_file(bot, user, text, command, parameter):
continue continue
count += 1 count += 1
if count > ITEMS_PER_PAGE:
break
msgs.append("<b>{:0>3d}</b> - {:s}".format(index, file)) msgs.append("<b>{:0>3d}</b> - {:s}".format(index, file))
if count != 0: if count != 0:
if count > ITEMS_PER_PAGE:
msgs.append(constants.strings("records_omitted"))
send_multi_lines(bot, msgs, text) send_multi_lines(bot, msgs, text)
else: else:
bot.send_msg(constants.strings('no_file'), text) bot.send_msg(constants.strings('no_file'), text)
@ -861,16 +862,19 @@ def cmd_play_tags(bot, user, text, command, parameter):
def cmd_add_tag(bot, user, text, command, parameter): def cmd_add_tag(bot, user, text, command, parameter):
global log global log
params = parameter.split() params = parameter.split(" ", 1)
if len(params) == 2: index = 0
tags = []
if len(params) == 2 and params[0].isdigit():
index = params[0] index = params[0]
tags = list(map(lambda t: t.strip(), params[1].split(","))) tags = list(map(lambda t: t.strip(), params[1].split(",")))
elif len(params) == 1: elif len(params) == 2 and params[0] == "*":
index = str(var.playlist.current_index + 1) index = "*"
tags = list(map(lambda t: t.strip(), params[0].split(","))) tags = list(map(lambda t: t.strip(), params[1].split(",")))
else: else:
bot.send_msg(constants.strings('bad_parameter', command=command), text) index = str(var.playlist.current_index + 1)
return tags = list(map(lambda t: t.strip(), parameter.split(",")))
if tags[0]: if tags[0]:
if index.isdigit() and 1 <= int(index) <= len(var.playlist): if index.isdigit() and 1 <= int(index) <= len(var.playlist):
@ -896,17 +900,19 @@ def cmd_add_tag(bot, user, text, command, parameter):
def cmd_remove_tag(bot, user, text, command, parameter): def cmd_remove_tag(bot, user, text, command, parameter):
global log global log
params = parameter.split() params = parameter.split(" ", 1)
index = 0
tags = []
if len(params) == 2: if len(params) == 2 and params[0].isdigit():
index = params[0] index = params[0]
tags = list(map(lambda t: t.strip(), params[1].split(","))) tags = list(map(lambda t: t.strip(), params[1].split(",")))
elif len(params) == 1: elif len(params) == 2 and params[0] == "*":
index = str(var.playlist.current_index + 1) index = "*"
tags = list(map(lambda t: t.strip(), params[0].split(","))) tags = list(map(lambda t: t.strip(), params[1].split(",")))
else: else:
bot.send_msg(constants.strings('bad_parameter', command=command), text) index = str(var.playlist.current_index + 1)
return tags = list(map(lambda t: t.strip(), parameter.split(",")))
if tags[0]: if tags[0]:
if index.isdigit() and 1 <= int(index) <= len(var.playlist): if index.isdigit() and 1 <= int(index) <= len(var.playlist):
@ -958,14 +964,18 @@ def cmd_find_tagged(bot, user, text, command, parameter):
music_dicts = var.music_db.query_music_by_tags(tags) music_dicts = var.music_db.query_music_by_tags(tags)
song_shortlist = music_dicts song_shortlist = music_dicts
items = dicts_to_items(bot, music_dicts)
for i, item in enumerate(items): for i, music_dict in enumerate(music_dicts):
item = dict_to_item(bot, music_dict)
count += 1 count += 1
if count > ITEMS_PER_PAGE:
break
msgs.append("<li><b>{:d}</b> - <b>{}</b> (<i>{}</i>)</li>".format(i+1, item.title, ", ".join(item.tags))) msgs.append("<li><b>{:d}</b> - <b>{}</b> (<i>{}</i>)</li>".format(i+1, item.title, ", ".join(item.tags)))
if count != 0: if count != 0:
msgs.append("</ul>") msgs.append("</ul>")
if count > ITEMS_PER_PAGE:
msgs.append(constants.strings("records_omitted"))
msgs.append(constants.strings("shortlist_instruction")) msgs.append(constants.strings("shortlist_instruction"))
send_multi_lines(bot, msgs, text, "") send_multi_lines(bot, msgs, text, "")
else: else:
@ -1000,6 +1010,8 @@ def cmd_search_library(bot, user, text, command, parameter):
else: else:
for item in items: for item in items:
count += 1 count += 1
if count > ITEMS_PER_PAGE:
break
if len(item.tags) > 0: if len(item.tags) > 0:
msgs.append("<li><b>{:d}</b> - [{}] <b>{}</b> (<i>{}</i>)</li>".format(count, item.display_type(), item.title, ", ".join(item.tags))) msgs.append("<li><b>{:d}</b> - [{}] <b>{}</b> (<i>{}</i>)</li>".format(count, item.display_type(), item.title, ", ".join(item.tags)))
else: else:
@ -1007,6 +1019,8 @@ def cmd_search_library(bot, user, text, command, parameter):
if count != 0: if count != 0:
msgs.append("</ul>") msgs.append("</ul>")
if count > ITEMS_PER_PAGE:
msgs.append(constants.strings("records_omitted"))
msgs.append(constants.strings("shortlist_instruction")) msgs.append(constants.strings("shortlist_instruction"))
send_multi_lines(bot, msgs, text, "") send_multi_lines(bot, msgs, text, "")
else: else:

2
configuration.default.ini
View File

@ -196,6 +196,8 @@ file_deleted = Deleted {item} from the library.
multiple_file_added = Multiple items added: multiple_file_added = Multiple items added:
multiple_file_deleted = Multiple items deleted from the library: multiple_file_deleted = Multiple items deleted from the library:
multiple_file_found = Found: multiple_file_found = Found:
page_instruction = Page {current}/{total}. Use <i>!{command} {{page}}</i> to navigate.
records_omitted = ...
bad_url = Bad URL requested. bad_url = Bad URL requested.
preconfigurated_radio = Preconfigurated Radio available: preconfigurated_radio = Preconfigurated Radio available:
unable_download = Error while downloading music... unable_download = Error while downloading music...

97
database.py
View File

@ -1,3 +1,4 @@
import re
import sqlite3 import sqlite3
import json import json
import datetime import datetime
@ -14,23 +15,32 @@ class Condition:
self._limit = 0 self._limit = 0
self._offset = 0 self._offset = 0
self._order_by = "" self._order_by = ""
self.has_regex = False
pass pass
def sql(self): def sql(self, conn: sqlite3.Connection = None):
sql = self._sql sql = self._sql
if not self._sql: if not self._sql:
sql = "TRUE" sql = "TRUE"
if self._order_by:
sql += f" ORDER BY {self._order_by}"
if self._limit: if self._limit:
sql += f" LIMIT {self._limit}" sql += f" LIMIT {self._limit}"
if self._offset: if self._offset:
sql += f" OFFSET {self._offset}" sql += f" OFFSET {self._offset}"
if self._order_by: if self.has_regex and conn:
sql += f" ORDEY BY {self._order_by}" conn.create_function("REGEXP", 2, self._regexp)
print(sql) print(sql)
print(self.filler)
return sql return sql
@staticmethod
def _regexp(expr, item):
if not item:
return False
reg = re.compile(expr)
return reg.search(item) is not None
def or_equal(self, column, equals_to, case_sensitive=True): def or_equal(self, column, equals_to, case_sensitive=True):
if not case_sensitive: if not case_sensitive:
column = f"LOWER({column})" column = f"LOWER({column})"
@ -87,39 +97,75 @@ class Condition:
return self return self
def and_regexp(self, column, regex):
self.has_regex = True
if self._sql:
self._sql += f" AND {column} REGEXP ?"
else:
self._sql += f"{column} REGEXP ?"
self.filler.append(regex)
return self
def or_regexp(self, column, regex):
self.has_regex = True
if self._sql:
self._sql += f" OR {column} REGEXP ?"
else:
self._sql += f"{column} REGEXP ?"
self.filler.append(regex)
return self
def or_sub_condition(self, sub_condition): def or_sub_condition(self, sub_condition):
if sub_condition.has_regex:
self.has_regex = True
self.filler.extend(sub_condition.filler) self.filler.extend(sub_condition.filler)
if self._sql: if self._sql:
self._sql += f"OR ({sub_condition.sql()})" self._sql += f" OR ({sub_condition.sql(None)})"
else: else:
self._sql += f"({sub_condition.sql()})" self._sql += f"({sub_condition.sql(None)})"
return self return self
def or_not_sub_condition(self, sub_condition): def or_not_sub_condition(self, sub_condition):
if sub_condition.has_regex:
self.has_regex = True
self.filler.extend(sub_condition.filler) self.filler.extend(sub_condition.filler)
if self._sql: if self._sql:
self._sql += f"OR NOT ({sub_condition.sql()})" self._sql += f" OR NOT ({sub_condition.sql(None)})"
else: else:
self._sql += f"NOT ({sub_condition.sql()})" self._sql += f"NOT ({sub_condition.sql(None)})"
return self return self
def and_sub_condition(self, sub_condition): def and_sub_condition(self, sub_condition):
if sub_condition.has_regex:
self.has_regex = True
self.filler.extend(sub_condition.filler) self.filler.extend(sub_condition.filler)
if self._sql: if self._sql:
self._sql += f"AND ({sub_condition.sql()})" self._sql += f" AND ({sub_condition.sql(None)})"
else: else:
self._sql += f"({sub_condition.sql()})" self._sql += f"({sub_condition.sql(None)})"
return self return self
def and_not_sub_condition(self, sub_condition): def and_not_sub_condition(self, sub_condition):
if sub_condition.has_regex:
self.has_regex = True
self.filler.extend(sub_condition.filler) self.filler.extend(sub_condition.filler)
if self._sql: if self._sql:
self._sql += f"AND NOT({sub_condition.sql()})" self._sql += f" AND NOT({sub_condition.sql(None)})"
else: else:
self._sql += f"NOT ({sub_condition.sql()})" self._sql += f"NOT ({sub_condition.sql(None)})"
return self return self
@ -133,6 +179,11 @@ class Condition:
return self return self
def order_by(self, order_by):
self._order_by = order_by
return self
SETTING_DB_VERSION = 1 SETTING_DB_VERSION = 1
MUSIC_DB_VERSION = 1 MUSIC_DB_VERSION = 1
@ -369,13 +420,21 @@ class MusicDatabase:
conn.commit() conn.commit()
conn.close() conn.close()
def query_all_ids(self): def query_music_ids(self, condition: Condition):
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
cursor = conn.cursor() cursor = conn.cursor()
results = cursor.execute("SELECT id FROM music WHERE id != 'info'").fetchall() results = cursor.execute("SELECT id FROM music WHERE id != 'info' AND %s" %
condition.sql(conn), condition.filler).fetchall()
conn.close() conn.close()
return list(map(lambda i: i[0], results)) return list(map(lambda i: i[0], results))
def query_all_paths(self):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
results = cursor.execute("SELECT path FROM music WHERE id != 'info'").fetchall()
conn.close()
return [ result[0] for result in results ]
def query_all_tags(self): def query_all_tags(self):
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
cursor = conn.cursor() cursor = conn.cursor()
@ -390,9 +449,9 @@ class MusicDatabase:
def query_music_count(self, condition: Condition): def query_music_count(self, condition: Condition):
filler = condition.filler filler = condition.filler
condition_str = condition.sql()
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
condition_str = condition.sql(conn)
cursor = conn.cursor() cursor = conn.cursor()
results = cursor.execute("SELECT COUNT(*) FROM music " results = cursor.execute("SELECT COUNT(*) FROM music "
"WHERE id != 'info' AND %s" % condition_str, filler).fetchall() "WHERE id != 'info' AND %s" % condition_str, filler).fetchall()
@ -402,9 +461,9 @@ class MusicDatabase:
def query_music(self, condition: Condition, _conn=None): def query_music(self, condition: Condition, _conn=None):
filler = condition.filler filler = condition.filler
condition_str = condition.sql()
conn = sqlite3.connect(self.db_path) if _conn is None else _conn conn = sqlite3.connect(self.db_path) if _conn is None else _conn
condition_str = condition.sql(conn)
cursor = conn.cursor() cursor = conn.cursor()
results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music " results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music "
"WHERE id != 'info' AND %s" % condition_str, filler).fetchall() "WHERE id != 'info' AND %s" % condition_str, filler).fetchall()
@ -461,7 +520,7 @@ class MusicDatabase:
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
cursor = conn.cursor() cursor = conn.cursor()
results = cursor.execute("SELECT id, tags FROM music " results = cursor.execute("SELECT id, tags FROM music "
"WHERE id != 'info' AND %s" % condition.sql(), condition.filler).fetchall() "WHERE id != 'info' AND %s" % condition.sql(conn), condition.filler).fetchall()
conn.close() conn.close()
@ -484,7 +543,7 @@ class MusicDatabase:
results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music " results = cursor.execute("SELECT id, type, title, metadata, tags, path, keywords FROM music "
"WHERE id IN (SELECT id FROM music WHERE %s ORDER BY RANDOM() LIMIT ?) ORDER BY RANDOM()" "WHERE id IN (SELECT id FROM music WHERE %s ORDER BY RANDOM() LIMIT ?) ORDER BY RANDOM()"
% condition.sql(), condition.filler + [count]).fetchall() % condition.sql(conn), condition.filler + [count]).fetchall()
conn.close() conn.close()
return self._result_to_dict(results) return self._result_to_dict(results)
@ -514,7 +573,7 @@ class MusicDatabase:
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
cursor = conn.cursor() cursor = conn.cursor()
cursor.execute("DELETE FROM music " cursor.execute("DELETE FROM music "
"WHERE %s" % condition.sql(), condition.filler) "WHERE %s" % condition.sql(conn), condition.filler)
conn.commit() conn.commit()
conn.close() conn.close()

54
interface.py
View File

@ -126,24 +126,20 @@ def build_tags_color_lookup():
return color_lookup return color_lookup
def get_all_dirs():
dirs = []
paths = var.music_db.query_all_paths()
for path in paths:
pos = 0
while True:
pos = path.find("/", pos+1)
if pos == -1:
break
folder = path[:pos]
if folder not in dirs:
dirs.append(folder)
def build_path_tags_lookup(): return dirs
path_tags_lookup = {}
ids = list(var.cache.file_id_lookup.values())
if len(ids) > 0:
condition = Condition().and_equal("type", "file")
id_tags_lookup = var.music_db.query_tags(condition)
for path, id in var.cache.file_id_lookup.items():
path_tags_lookup[path] = id_tags_lookup[id]
return path_tags_lookup
def recur_dir(dirobj):
for name, dir in dirobj.get_subdirs().items():
print(dirobj.fullpath + "/" + name)
recur_dir(dir)
@web.route("/", methods=['GET']) @web.route("/", methods=['GET'])
@ -153,17 +149,10 @@ def index():
time.sleep(0.1) time.sleep(0.1)
tags_color_lookup = build_tags_color_lookup() tags_color_lookup = build_tags_color_lookup()
path_tags_lookup = build_path_tags_lookup()
return render_template('index.html', return render_template('index.html',
all_files=var.cache.files, dirs=get_all_dirs(),
tags_lookup=path_tags_lookup,
tags_color_lookup=tags_color_lookup, tags_color_lookup=tags_color_lookup,
music_library=var.cache.dir,
os=os,
playlist=var.playlist,
user=var.user,
paused=var.bot.is_pause,
) )
@ -177,7 +166,7 @@ def playlist():
)] )]
}) })
tags_color_lookup = build_tags_color_lookup() tags_color_lookup = build_tags_color_lookup() # TODO: cached this?
items = [] items = []
for index, item_wrapper in enumerate(var.playlist): for index, item_wrapper in enumerate(var.playlist):
@ -384,18 +373,7 @@ def build_library_query_condition(form):
folder = form['dir'] folder = form['dir']
if not folder.endswith('/') and folder: if not folder.endswith('/') and folder:
folder += '/' folder += '/'
sub_cond = Condition() condition.and_like('path', folder + '%')
count = 0
for file in var.cache.files:
if file.startswith(folder):
count += 1
sub_cond.or_equal("id", var.cache.file_id_lookup[file])
if count > 900:
break
if count > 0:
condition.and_sub_condition(sub_cond)
else:
condition.and_equal("id", None)
tags = form['tags'].split(",") tags = form['tags'].split(",")
for tag in tags: for tag in tags:

61
media/cache.py
View File

@ -19,9 +19,6 @@ class MusicCache(dict):
super().__init__() super().__init__()
self.db = db self.db = db
self.log = logging.getLogger("bot") self.log = logging.getLogger("bot")
self.dir = None
self.files = []
self.file_id_lookup = {} # TODO: Now I see this is silly. Gonna add a column "path" in the database.
self.dir_lock = threading.Lock() self.dir_lock = threading.Lock()
def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message! def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message!
@ -90,12 +87,7 @@ class MusicCache(dict):
if item: if item:
self.log.debug("library: DELETE item from the database: %s" % item.format_debug_string()) self.log.debug("library: DELETE item from the database: %s" % item.format_debug_string())
if item.type == 'file' and item.path in self.file_id_lookup: if item.type == 'url':
if item.path in self.file_id_lookup:
del self.file_id_lookup[item.path]
self.files.remove(item.path)
self.save_dir_cache()
elif item.type == 'url':
if os.path.exists(item.path): if os.path.exists(item.path):
os.remove(item.path) os.remove(item.path)
@ -115,9 +107,7 @@ class MusicCache(dict):
def build_dir_cache(self, bot): def build_dir_cache(self, bot):
self.dir_lock.acquire() self.dir_lock.acquire()
self.log.info("library: rebuild directory cache") self.log.info("library: rebuild directory cache")
self.files = []
files = util.get_recursive_file_list_sorted(var.music_folder) files = util.get_recursive_file_list_sorted(var.music_folder)
self.dir = util.Dir(var.music_folder)
for file in files: for file in files:
item = self.fetch(bot, item_id_generators['file'](path=file)) item = self.fetch(bot, item_id_generators['file'](path=file))
if not item: if not item:
@ -125,25 +115,6 @@ class MusicCache(dict):
self.log.debug("library: music save into database: %s" % item.format_debug_string()) self.log.debug("library: music save into database: %s" % item.format_debug_string())
self.db.insert_music(item.to_dict()) self.db.insert_music(item.to_dict())
self.dir.add_file(file)
self.files.append(file)
self.file_id_lookup[file] = item.id
self.save_dir_cache()
self.dir_lock.release()
def save_dir_cache(self):
var.db.set("dir_cache", "files", json.dumps(self.file_id_lookup))
def load_dir_cache(self, bot):
self.dir_lock.acquire()
self.log.info("library: load directory cache from database")
loaded = json.loads(var.db.get("dir_cache", "files"))
self.files = loaded.keys()
self.file_id_lookup = loaded
self.dir = util.Dir(var.music_folder)
for file, id in loaded.items():
self.dir.add_file(file)
self.dir_lock.release() self.dir_lock.release()
@ -223,9 +194,18 @@ class CachedItemWrapper:
# Remember!!! Get wrapper functions will automatically add items into the cache! # Remember!!! Get wrapper functions will automatically add items into the cache!
def get_cached_wrapper(item, user): def get_cached_wrapper(item, user):
var.cache[item.id] = item if item:
return CachedItemWrapper(var.cache, item.id, item.type, user) var.cache[item.id] = item
return CachedItemWrapper(var.cache, item.id, item.type, user)
return None
def get_cached_wrappers(items, user):
wrappers = []
for item in items:
if item:
wrappers.append(get_cached_wrapper(item, user))
return wrappers
def get_cached_wrapper_from_scrap(bot, **kwargs): def get_cached_wrapper_from_scrap(bot, **kwargs):
item = var.cache.get_item(bot, **kwargs) item = var.cache.get_item(bot, **kwargs)
@ -233,19 +213,24 @@ def get_cached_wrapper_from_scrap(bot, **kwargs):
raise KeyError("Which user added this song?") raise KeyError("Which user added this song?")
return CachedItemWrapper(var.cache, item.id, kwargs['type'], kwargs['user']) return CachedItemWrapper(var.cache, item.id, kwargs['type'], kwargs['user'])
def get_cached_wrapper_from_dict(bot, dict_from_db, user): def get_cached_wrapper_from_dict(bot, dict_from_db, user):
item = dict_to_item(bot, dict_from_db) if dict_from_db:
return get_cached_wrapper(item, user) item = dict_to_item(bot, dict_from_db)
return get_cached_wrapper(item, user)
return None
def get_cached_wrappers_from_dicts(bot, dicts_from_db, user):
items = []
for dict_from_db in dicts_from_db:
if dict_from_db:
items.append(get_cached_wrapper_from_dict(bot, dict_from_db, user))
return items
def get_cached_wrapper_by_id(bot, id, user): def get_cached_wrapper_by_id(bot, id, user):
item = var.cache.get_item_by_id(bot, id) item = var.cache.get_item_by_id(bot, id)
if item: if item:
return CachedItemWrapper(var.cache, item.id, item.type, user) return CachedItemWrapper(var.cache, item.id, item.type, user)
else:
return None
def get_cached_wrappers_by_tags(bot, tags, user): def get_cached_wrappers_by_tags(bot, tags, user):
items = var.cache.get_items_by_tags(bot, tags) items = var.cache.get_items_by_tags(bot, tags)

2
mumbleBot.py
View File

@ -680,8 +680,6 @@ if __name__ == '__main__':
if var.config.get("bot", "refresh_cache_on_startup", fallback=True)\ if var.config.get("bot", "refresh_cache_on_startup", fallback=True)\
or not var.db.has_option("dir_cache", "files"): or not var.db.has_option("dir_cache", "files"):
var.cache.build_dir_cache(var.bot) var.cache.build_dir_cache(var.bot)
else:
var.cache.load_dir_cache(var.bot)
# load playlist # load playlist
if var.config.getboolean('bot', 'save_playlist', fallback=True): if var.config.getboolean('bot', 'save_playlist', fallback=True):

4
templates/index.html
View File

@ -136,7 +136,7 @@
<div class="col-sm-8"> <div class="col-sm-8">
<select class="form-control form-control-sm" id="filter-dir" style="margin-top:5px;" disabled> <select class="form-control form-control-sm" id="filter-dir" style="margin-top:5px;" disabled>
<option value="">.</option> <option value="">.</option>
{% for dir in music_library.get_subdirs_recursively() %} {% for dir in dirs %}
<option value="{{ dir }}">{{ dir }}</option> <option value="{{ dir }}">{{ dir }}</option>
{% endfor %} {% endfor %}
</select> </select>
@ -298,7 +298,7 @@
<input class="form-control btn-space" list="targetdirs" id="targetdir" name="targetdir" <input class="form-control btn-space" list="targetdirs" id="targetdir" name="targetdir"
placeholder="uploads" /> placeholder="uploads" />
<datalist id="targetdirs"> <datalist id="targetdirs">
{% for dir in music_library.get_subdirs_recursively() %} {% for dir in dirs %}
<option value="{{ dir }}" /> <option value="{{ dir }}" />
{% endfor %} {% endfor %}
</datalist> </datalist>