fix: youtube playlist related CRAZY issues.

command.py

@@ -57,6 +57,8 @@ def register_all_commands(bot):
 
     # Just for debug use
     bot.register_command('rtrms', cmd_real_time_rms)
+    # bot.register_command('loop', cmd_loop_state)
+    # bot.register_command('item', cmd_item)
 
 def send_multi_lines(bot, lines, text):
     global log
@@ -799,3 +801,10 @@ def cmd_drop_database(bot, user, text, command, parameter):
 # Just for debug use
 def cmd_real_time_rms(bot, user, text, command, parameter):
     bot._display_rms = not bot._display_rms
+
+def cmd_loop_state(bot, user, text, command, parameter):
+    print(bot._loop_status)
+
+def cmd_item(bot, user, text, command, parameter):
+    print(bot.wait_for_downloading)
+    print(var.playlist.current_item())

media/playlist.py

@@ -1,190 +1,5 @@
 import youtube_dl
 import variables as var
-import util
-import random
-import json
-import logging
-
-class PlayList(list):
-    current_index = -1
-    version = 0 # increase by one after each change
-    mode = "one-shot" # "repeat", "random"
-
-    def __init__(self, *args):
-        super().__init__(*args)
-
-    def set_mode(self, mode):
-        # modes are "one-shot", "repeat", "random"
-        self.mode = mode
-
-        if mode == "random":
-            self.randomize()
-
-        elif mode == "one-shot" and self.current_index > 0:
-            # remove items before current item
-            self.version += 1
-            for i in range(self.current_index):
-                super().__delitem__(0)
-            self.current_index = 0
-
-    def append(self, item):
-        self.version += 1
-        item = util.get_music_tag_info(item)
-        super().append(item)
-
-        return item
-
-    def insert(self, index, item):
-        self.version += 1
-
-        if index == -1:
-            index = self.current_index
-
-        item = util.get_music_tag_info(item)
-        super().insert(index, item)
-
-        if index <= self.current_index:
-            self.current_index += 1
-
-        return item
-
-    def length(self):
-        return len(self)
-
-    def extend(self, items):
-        self.version += 1
-        items = list(map(
-            lambda item: util.get_music_tag_info(item),
-            items))
-        super().extend(items)
-        return items
-
-    def next(self):
-        if len(self) == 0:
-            return False
-
-        self.version += 1
-        #logging.debug("playlist: Next into the queue")
-
-        if self.current_index < len(self) - 1:
-            if self.mode == "one-shot" and self.current_index != -1:
-                super().__delitem__(self.current_index)
-            else:
-                self.current_index += 1
-
-            return self[self.current_index]
-        else:
-            self.current_index = 0
-            if self.mode == "one-shot":
-                self.clear()
-                return False
-            elif self.mode == "repeat":
-                return self[0]
-            elif self.mode == "random":
-                self.randomize()
-                return self[0]
-            else:
-                raise TypeError("Unknown playlist mode '%s'." % self.mode)
-
-    def update(self, item, index=-1):
-        self.version += 1
-        if index == -1:
-            index = self.current_index
-        self[index] = item
-
-    def __delitem__(self, key):
-        return self.remove(key)
-
-    def remove(self, index=-1):
-        self.version += 1
-        if index > len(self) - 1:
-            return False
-
-        if index == -1:
-            index = self.current_index
-
-        removed = self[index]
-        super().__delitem__(index)
-
-        if self.current_index > index:
-            self.current_index -= 1
-
-        return removed
-
-    def current_item(self):
-        return self[self.current_index]
-
-    def current_item_downloading(self):
-        if self[self.current_index]['type'] == 'url' and self[self.current_index]['ready'] == 'downloading':
-            return True
-        return False
-
-    def next_index(self):
-        if len(self) == 0:
-            return False
-
-        if self.current_index < len(self) - 1:
-            return self.current_index + 1
-        else:
-            return 0
-
-    def next_item(self):
-        if len(self) == 0:
-            return False
-
-        return self[self.next_index()]
-
-    def jump(self, index):
-        if self.mode == "one-shot":
-            for i in range(index):
-                super().__delitem__(0)
-            self.current_index = 0
-        else:
-            self.current_index = index
-
-        self.version += 1
-        return self[self.current_index]
-
-    def randomize(self):
-        # current_index will lose track after shuffling, thus we take current music out before shuffling
-        #current = self.current_item()
-        #del self[self.current_index]
-
-        random.shuffle(self)
-
-        #self.insert(0, current)
-        self.current_index = -1
-        self.version += 1
-
-    def clear(self):
-        self.version += 1
-        self.current_index = -1
-        super().clear()
-
-    def save(self):
-        var.db.remove_section("playlist_item")
-        var.db.set("playlist", "current_index", self.current_index)
-        for index, item in enumerate(self):
-            var.db.set("playlist_item", str(index), json.dumps(item))
-
-    def load(self):
-        current_index = var.db.getint("playlist", "current_index", fallback=-1)
-        if current_index == -1:
-            return
-
-        items = list(var.db.items("playlist_item"))
-        items.sort(key=lambda v: int(v[0]))
-        self.extend(list(map(lambda v: json.loads(v[1]), items)))
-        self.current_index = current_index
-
-    def _debug_print(self):
-        print("===== Playlist(%d) ====" % self.current_index)
-        for index, item in enumerate(self):
-            if index == self.current_index:
-                print("-> %d %s" % (index, item['title']))
-            else:
-                print("%d %s" % (index, item['title']))
-        print("=====      End     ====")
 
 
 def get_playlist_info(url, start_index=0, user=""):
@@ -209,6 +24,7 @@ def get_playlist_info(url, start_index=0, user=""):
 
                 playlist_title = info['title']
                 for j in range(start_index, min(len(info['entries']), start_index + var.config.getint('bot', 'max_track_playlist'))):
+                    print(info['entries'][j])
                     # Unknow String if No title into the json
                     title = info['entries'][j]['title'] if 'title' in info['entries'][j] else "Unknown Title"
                     # Add youtube url if the url in the json isn't a full url
@@ -227,25 +43,3 @@ def get_playlist_info(url, start_index=0, user=""):
                 pass
 
     return items
-
-# def get_music_info(index=0):
-#     ydl_opts = {
-#         'playlist_items': str(index)
-#     }
-#     with youtube_dl.YoutubeDL(ydl_opts) as ydl:
-#         for i in range(2):
-#             try:
-#                 info = ydl.extract_info(var.playlist.playlist[index]['url'], download=False)
-#                 # Check if the Duration is longer than the config
-#                 if var.playlist[index]['current_index'] == index:
-#                     var.playlist[index]['current_duration'] = info['entries'][0]['duration'] / 60
-#                     var.playlist[index]['current_title'] = info['entries'][0]['title']
-#                 # Check if the Duration of the next music is longer than the config (async download)
-#                 elif var.playlist[index]['current_index'] == index - 1:
-#                     var.playlist[index]['next_duration'] = info['entries'][0]['duration'] / 60
-#                     var.playlist[index]['next_title'] = info['entries'][0]['title']
-#             except youtube_dl.utils.DownloadError:
-#                 pass
-#             else:
-#                 return True
-#     return False

mumbleBot.py

@@ -32,38 +32,7 @@ import media.playlist
 import media.radio
 import media.system
 from librb import radiobrowser
-from media.playlist import PlayList
+from playlist import PlayList
-
-
-"""
-FORMAT OF A MUSIC INTO THE PLAYLIST
-type : url
-    url
-    title
-    path
-    duration
-    artist
-    thumbnail
-    user
-    ready (validation, no, downloading, yes, failed)
-    from_playlist (yes,no)
-    playlist_title
-    playlist_url
-
-type : radio
-    url
-    name
-    current_title
-    user
-
-type : file
-    path
-    title
-    artist
-    duration
-    thumbnail
-    user
-"""
 
 
 class MumbleBot:
@@ -104,6 +73,8 @@ class MumbleBot:
         self.is_pause = False
         self.playhead = -1
         self.song_start_at = -1
+        self.download_in_progress = False
+        self.wait_for_downloading = False # flag for the loop are waiting for download to complete in the other thread
 
         if var.config.getboolean("webinterface", "enabled"):
             wi_addr = var.config.get("webinterface", "listening_addr")
@@ -184,6 +155,7 @@ class MumbleBot:
             self.mumble.set_receive_sound(True)
 
         # Debug use
+        self._loop_status = 'Idle'
         self._display_rms = False
         self._max_rms = 0
 
@@ -325,7 +297,7 @@ class MumbleBot:
     def launch_music(self, index=-1):
         uri = ""
         music = None
-        if var.playlist.length() == 0:
+        if var.playlist.is_empty():
             return
 
         if index == -1:
@@ -333,19 +305,20 @@ class MumbleBot:
         else:
             music = var.playlist.jump(index)
 
+        self.wait_for_downloading = False
+
         self.log.info("bot: play music " + util.format_debug_song_string(music))
         if music["type"] == "url":
             # Delete older music is the tmp folder is too big
             media.system.clear_tmp_folder(var.tmp_folder, var.config.getint('bot', 'tmp_folder_max_size'))
 
-            # Check if the music is ready to be played
+            if music['ready'] == 'downloading':
-            if music["ready"] == "downloading":
+                self.wait_for_downloading = True
-                self.log.info("bot: current music isn't ready, downloading in progress.")
+                self.log.info("bot: current music isn't ready, other thread is downloading.")
-                while var.playlist.current_item_downloading():
+                return
-                    time.sleep(0.5)
-                music = var.playlist.current_item()
 
-            elif music["ready"] != "yes" or not os.path.exists(music['path']):
+            # Check if the music is ready to be played
+            if music["ready"] != "yes" or not os.path.exists(music['path']):
                 self.log.info("bot: current music isn't ready, start to download.")
                 music = self.download_music()
 
@@ -413,7 +386,7 @@ class MumbleBot:
             if music['duration'] > var.config.getint('bot', 'max_track_duration'):
                 # Check the length, useful in case of playlist, it wasn't checked before)
                 self.log.info(
-                    "the music " + music["url"] + " has a duration of " + music['duration'] + "s -- too long")
+                    "the music " + music["url"] + " has a duration of " + str(music['duration']) + "s -- too long")
                 self.send_msg(constants.strings('too_long'))
                 return False
             else:
@@ -446,7 +419,7 @@ class MumbleBot:
         if not os.path.isfile(mp3):
             # download the music
             music['ready'] = "downloading"
-
+            var.playlist.update(music, index)
 
             self.log.info("bot: downloading url (%s) %s " % (music['title'], url))
             ydl_opts = ""
@@ -502,16 +475,25 @@ class MumbleBot:
         # Function start if the next music isn't ready
         # Do nothing in case the next music is already downloaded
         self.log.debug("bot: Async download next asked ")
-        if var.playlist.length() > 1 and var.playlist.next_item()['type'] == 'url' \
+        if var.playlist.next_item() and var.playlist.next_item()['type'] == 'url':
-                and (var.playlist.next_item()['ready'] in ["no", "validation"]):
+            # usually, all validation will be done when adding to the list.
-            th = threading.Thread(
+            # however, for performance consideration, youtube playlist won't be validate when added.
-                target=self.download_music, name="DownloadThread", args=(var.playlist.next_index(),))
+            # the validation has to be done here.
-            self.log.info(
+            while var.playlist.next_item() and var.playlist.next_item()['ready'] == "validation":
-                "bot: start downloading item in thread: " + util.format_debug_song_string(var.playlist.next_item()))
+                music = self.validate_music(var.playlist.next_item())
-            th.daemon = True
+                if music:
-            th.start()
+                    var.playlist.update(music, var.playlist.next_index())
-        else:
+                    break
-            return
+                else:
+                    var.playlist.remove(var.playlist.next_index())
+
+            if var.playlist.next_item() and var.playlist.next_item()['ready'] == "no":
+                th = threading.Thread(
+                    target=self.download_music, name="DownloadThread", args=(var.playlist.next_index(),))
+                self.log.info(
+                    "bot: start downloading item in thread: " + util.format_debug_song_string(var.playlist.next_item()))
+                th.daemon = True
+                th.start()
 
     def check_item_path_or_remove(self, index = -1):
         if index == -1:
@@ -549,12 +531,15 @@ class MumbleBot:
         raw_music = ""
         while not self.exit and self.mumble.is_alive():
 
-            while self.mumble.sound_output.get_buffer_size() > 0.5 and not self.exit:
+            while self.thread and self.mumble.sound_output.get_buffer_size() > 0.5 and not self.exit:
                 # If the buffer isn't empty, I cannot send new music part, so I wait
+                self._loop_status = 'Wait for buffer %.3f' % self.mumble.sound_output.get_buffer_size()
                 time.sleep(0.01)
+
             if self.thread:
                 # I get raw from ffmpeg thread
                 # move playhead forward
+                self._loop_status = 'Reading raw'
                 if self.song_start_at == -1:
                     self.song_start_at = time.time() - self.playhead
                 self.playhead = time.time() - self.song_start_at
@@ -578,12 +563,23 @@ class MumbleBot:
             else:
                 time.sleep(0.1)
 
-            if self.thread is None or not raw_music:
+            if not self.is_pause and (self.thread is None or not raw_music):
-                # Not music into the buffet
+                # ffmpeg thread has gone. indicate that last song has finished. move to the next song.
-                if not self.is_pause:
+                if not self.wait_for_downloading:
-                    if len(var.playlist) > 0 and var.playlist.next():
+                    if var.playlist.next():
+                    # if downloading in the other thread
                         self.launch_music()
                         self.async_download_next()
+                    else:
+                        self._loop_status = 'Empty queue'
+                else:
+                    print(var.playlist.current_item()["ready"])
+                    if var.playlist.current_item()["ready"] != "downloading":
+                        self.wait_for_downloading = False
+                        self.launch_music()
+                        self.async_download_next()
+                    else:
+                        self._loop_status = 'Wait for downloading'
 
         while self.mumble.sound_output.get_buffer_size() > 0:
             # Empty the buffer before exit
@@ -591,6 +587,7 @@ class MumbleBot:
         time.sleep(0.5)
 
         if self.exit:
+            self._loop_status = "exited"
             if var.config.getboolean('bot', 'save_playlist', fallback=True):
                 self.log.info("bot: save playlist into database")
                 var.playlist.save()