fix: util failed on some strange encoded files

interface.py

@@ -119,16 +119,26 @@ def build_tags_color_lookup():
 
 def build_path_tags_lookup():
     path_lookup = {}
-    for path, id in var.library.file_id_lookup.items():
+    items = var.library.file_id_lookup.items()
+    for path, id in items:
         path_lookup[path] = var.music_db.query_tags_by_id(id)
 
     return path_lookup
 
+def recur_dir(dirobj):
+    for name, dir in dirobj.get_subdirs().items():
+        print(dirobj.fullpath + "/" + name)
+        recur_dir(dir)
+
 @web.route("/", methods=['GET'])
 @requires_auth
 def index():
     tags_color_lookup = build_tags_color_lookup()
     path_tags_lookup = build_path_tags_lookup()
+
+    while var.library.dir_lock.locked():
+        time.sleep(0.1)
+
     return render_template('index.html',
                            all_files=var.library.files,
                            tags_lookup=path_tags_lookup,
@@ -137,7 +147,7 @@ def index():
                            os=os,
                            playlist=var.playlist,
                            user=var.user,
-                           paused=var.bot.is_pause
+                           paused=var.bot.is_pause,
                            )
 
 @web.route("/playlist", methods=['GET'])

media/library.py

@@ -1,6 +1,7 @@
 import logging
 from database import MusicDatabase
 import json
+import threading
 
 from media.item import item_builders, item_loaders, item_id_generators
 from database import MusicDatabase
@@ -15,6 +16,7 @@ class MusicLibrary(dict):
         self.log = logging.getLogger("bot")
         self.dir = None
         self.files = []
+        self.dir_lock = threading.Lock()
 
     def get_item_by_id(self, bot, id): # Why all these functions need a bot? Because it need the bot to send message!
         if id in self:
@@ -27,6 +29,7 @@ class MusicLibrary(dict):
             self.log.debug("library: music found in database: %s" % item.format_debug_string())
             return item
         else:
+            print(id)
             raise KeyError("Unable to fetch item from the database! Please try to refresh the cache by !recache.")
 
 
@@ -101,6 +104,7 @@ class MusicLibrary(dict):
         self.clear()
 
     def build_dir_cache(self, bot):
+        self.dir_lock.acquire()
         self.log.info("library: rebuild directory cache")
         self.files = []
         self.file_id_lookup = {}
@@ -114,11 +118,13 @@ class MusicLibrary(dict):
                 self.file_id_lookup[file] = item.id
 
         self.save_dir_cache()
+        self.dir_lock.release()
 
     def save_dir_cache(self):
         var.db.set("dir_cache", "files", json.dumps(self.file_id_lookup))
 
     def load_dir_cache(self, bot):
+        self.dir_lock.acquire()
         self.log.info("library: load directory cache from database")
         loaded = json.loads(var.db.get("dir_cache", "files"))
         self.files = loaded.keys()
@@ -126,4 +132,5 @@ class MusicLibrary(dict):
         self.dir = util.Dir(var.music_folder)
         for file, id in loaded.items():
             self.dir.add_file(file)
+        self.dir_lock.release()
 

templates/index.html

@@ -1,5 +1,6 @@
 {% macro dirlisting(dir, path='') -%}
 <ul class="list-group">
+    {% if dir and dir.get_subdirs().items() %}
     {% for subdirname, subdirobj in dir.get_subdirs().items() %}
     {%- set subdirpath = os.path.relpath(subdirobj.fullpath, music_library.fullpath) %}
     {%- set subdirid = subdirpath.replace("/","-") %}
@@ -47,6 +48,7 @@
         {{- dirlisting(subdirobj, subdirpath) -}}
     </div>
     {% endfor %}
+    {% endif %}
     {%- set files = dir.get_files() %}
     {%- if files %}
     {% for file in files %}

util.py

@@ -41,7 +41,7 @@ def solve_filepath(path):
 
 def get_recursive_file_list_sorted(path):
     filelist = []
-    for root, dirs, files in os.walk(path):
+    for root, dirs, files in os.walk(path, topdown=True, onerror=None, followlinks=True):
         relroot = root.replace(path, '', 1)
         if relroot != '' and relroot in var.config.get('bot', 'ignored_folders'):
             continue
@@ -55,9 +55,12 @@ def get_recursive_file_list_sorted(path):
             if not os.access(fullpath, os.R_OK):
                 continue
 
+            try:
                 mime = magic.from_file(fullpath, mime=True)
                 if 'audio' in mime or 'audio' in magic.from_file(fullpath).lower() or 'video' in mime:
                     filelist.append(relroot + file)
+            except:
+                pass
 
     filelist.sort()
     return filelist