From c1c3e475821614a32eaf06c674edfd1dd71b899e Mon Sep 17 00:00:00 2001 From: Terry Geng Date: Tue, 3 Mar 2020 16:12:45 +0800 Subject: [PATCH] fix: youtube playlist related CRAZY issues. --- command.py | 9 ++ media/playlist.py | 208 +--------------------------------------------- mumbleBot.py | 111 ++++++++++++------------- 3 files changed, 64 insertions(+), 264 deletions(-) diff --git a/command.py b/command.py index 92581cf..ca56a72 100644 --- a/command.py +++ b/command.py @@ -57,6 +57,8 @@ def register_all_commands(bot): # Just for debug use bot.register_command('rtrms', cmd_real_time_rms) + # bot.register_command('loop', cmd_loop_state) + # bot.register_command('item', cmd_item) def send_multi_lines(bot, lines, text): global log @@ -799,3 +801,10 @@ def cmd_drop_database(bot, user, text, command, parameter): # Just for debug use def cmd_real_time_rms(bot, user, text, command, parameter): bot._display_rms = not bot._display_rms + +def cmd_loop_state(bot, user, text, command, parameter): + print(bot._loop_status) + +def cmd_item(bot, user, text, command, parameter): + print(bot.wait_for_downloading) + print(var.playlist.current_item()) diff --git a/media/playlist.py b/media/playlist.py index 576485a..5b6d17b 100644 --- a/media/playlist.py +++ b/media/playlist.py @@ -1,190 +1,5 @@ import youtube_dl import variables as var -import util -import random -import json -import logging - -class PlayList(list): - current_index = -1 - version = 0 # increase by one after each change - mode = "one-shot" # "repeat", "random" - - def __init__(self, *args): - super().__init__(*args) - - def set_mode(self, mode): - # modes are "one-shot", "repeat", "random" - self.mode = mode - - if mode == "random": - self.randomize() - - elif mode == "one-shot" and self.current_index > 0: - # remove items before current item - self.version += 1 - for i in range(self.current_index): - super().__delitem__(0) - self.current_index = 0 - - def append(self, item): - self.version += 1 - item = util.get_music_tag_info(item) - super().append(item) - - return item - - def insert(self, index, item): - self.version += 1 - - if index == -1: - index = self.current_index - - item = util.get_music_tag_info(item) - super().insert(index, item) - - if index <= self.current_index: - self.current_index += 1 - - return item - - def length(self): - return len(self) - - def extend(self, items): - self.version += 1 - items = list(map( - lambda item: util.get_music_tag_info(item), - items)) - super().extend(items) - return items - - def next(self): - if len(self) == 0: - return False - - self.version += 1 - #logging.debug("playlist: Next into the queue") - - if self.current_index < len(self) - 1: - if self.mode == "one-shot" and self.current_index != -1: - super().__delitem__(self.current_index) - else: - self.current_index += 1 - - return self[self.current_index] - else: - self.current_index = 0 - if self.mode == "one-shot": - self.clear() - return False - elif self.mode == "repeat": - return self[0] - elif self.mode == "random": - self.randomize() - return self[0] - else: - raise TypeError("Unknown playlist mode '%s'." % self.mode) - - def update(self, item, index=-1): - self.version += 1 - if index == -1: - index = self.current_index - self[index] = item - - def __delitem__(self, key): - return self.remove(key) - - def remove(self, index=-1): - self.version += 1 - if index > len(self) - 1: - return False - - if index == -1: - index = self.current_index - - removed = self[index] - super().__delitem__(index) - - if self.current_index > index: - self.current_index -= 1 - - return removed - - def current_item(self): - return self[self.current_index] - - def current_item_downloading(self): - if self[self.current_index]['type'] == 'url' and self[self.current_index]['ready'] == 'downloading': - return True - return False - - def next_index(self): - if len(self) == 0: - return False - - if self.current_index < len(self) - 1: - return self.current_index + 1 - else: - return 0 - - def next_item(self): - if len(self) == 0: - return False - - return self[self.next_index()] - - def jump(self, index): - if self.mode == "one-shot": - for i in range(index): - super().__delitem__(0) - self.current_index = 0 - else: - self.current_index = index - - self.version += 1 - return self[self.current_index] - - def randomize(self): - # current_index will lose track after shuffling, thus we take current music out before shuffling - #current = self.current_item() - #del self[self.current_index] - - random.shuffle(self) - - #self.insert(0, current) - self.current_index = -1 - self.version += 1 - - def clear(self): - self.version += 1 - self.current_index = -1 - super().clear() - - def save(self): - var.db.remove_section("playlist_item") - var.db.set("playlist", "current_index", self.current_index) - for index, item in enumerate(self): - var.db.set("playlist_item", str(index), json.dumps(item)) - - def load(self): - current_index = var.db.getint("playlist", "current_index", fallback=-1) - if current_index == -1: - return - - items = list(var.db.items("playlist_item")) - items.sort(key=lambda v: int(v[0])) - self.extend(list(map(lambda v: json.loads(v[1]), items))) - self.current_index = current_index - - def _debug_print(self): - print("===== Playlist(%d) ====" % self.current_index) - for index, item in enumerate(self): - if index == self.current_index: - print("-> %d %s" % (index, item['title'])) - else: - print("%d %s" % (index, item['title'])) - print("===== End ====") def get_playlist_info(url, start_index=0, user=""): @@ -209,6 +24,7 @@ def get_playlist_info(url, start_index=0, user=""): playlist_title = info['title'] for j in range(start_index, min(len(info['entries']), start_index + var.config.getint('bot', 'max_track_playlist'))): + print(info['entries'][j]) # Unknow String if No title into the json title = info['entries'][j]['title'] if 'title' in info['entries'][j] else "Unknown Title" # Add youtube url if the url in the json isn't a full url @@ -227,25 +43,3 @@ def get_playlist_info(url, start_index=0, user=""): pass return items - -# def get_music_info(index=0): -# ydl_opts = { -# 'playlist_items': str(index) -# } -# with youtube_dl.YoutubeDL(ydl_opts) as ydl: -# for i in range(2): -# try: -# info = ydl.extract_info(var.playlist.playlist[index]['url'], download=False) -# # Check if the Duration is longer than the config -# if var.playlist[index]['current_index'] == index: -# var.playlist[index]['current_duration'] = info['entries'][0]['duration'] / 60 -# var.playlist[index]['current_title'] = info['entries'][0]['title'] -# # Check if the Duration of the next music is longer than the config (async download) -# elif var.playlist[index]['current_index'] == index - 1: -# var.playlist[index]['next_duration'] = info['entries'][0]['duration'] / 60 -# var.playlist[index]['next_title'] = info['entries'][0]['title'] -# except youtube_dl.utils.DownloadError: -# pass -# else: -# return True -# return False diff --git a/mumbleBot.py b/mumbleBot.py index 827272a..b187cb5 100644 --- a/mumbleBot.py +++ b/mumbleBot.py @@ -32,38 +32,7 @@ import media.playlist import media.radio import media.system from librb import radiobrowser -from media.playlist import PlayList - - -""" -FORMAT OF A MUSIC INTO THE PLAYLIST -type : url - url - title - path - duration - artist - thumbnail - user - ready (validation, no, downloading, yes, failed) - from_playlist (yes,no) - playlist_title - playlist_url - -type : radio - url - name - current_title - user - -type : file - path - title - artist - duration - thumbnail - user -""" +from playlist import PlayList class MumbleBot: @@ -104,6 +73,8 @@ class MumbleBot: self.is_pause = False self.playhead = -1 self.song_start_at = -1 + self.download_in_progress = False + self.wait_for_downloading = False # flag for the loop are waiting for download to complete in the other thread if var.config.getboolean("webinterface", "enabled"): wi_addr = var.config.get("webinterface", "listening_addr") @@ -184,6 +155,7 @@ class MumbleBot: self.mumble.set_receive_sound(True) # Debug use + self._loop_status = 'Idle' self._display_rms = False self._max_rms = 0 @@ -325,7 +297,7 @@ class MumbleBot: def launch_music(self, index=-1): uri = "" music = None - if var.playlist.length() == 0: + if var.playlist.is_empty(): return if index == -1: @@ -333,19 +305,20 @@ class MumbleBot: else: music = var.playlist.jump(index) + self.wait_for_downloading = False + self.log.info("bot: play music " + util.format_debug_song_string(music)) if music["type"] == "url": # Delete older music is the tmp folder is too big media.system.clear_tmp_folder(var.tmp_folder, var.config.getint('bot', 'tmp_folder_max_size')) - # Check if the music is ready to be played - if music["ready"] == "downloading": - self.log.info("bot: current music isn't ready, downloading in progress.") - while var.playlist.current_item_downloading(): - time.sleep(0.5) - music = var.playlist.current_item() + if music['ready'] == 'downloading': + self.wait_for_downloading = True + self.log.info("bot: current music isn't ready, other thread is downloading.") + return - elif music["ready"] != "yes" or not os.path.exists(music['path']): + # Check if the music is ready to be played + if music["ready"] != "yes" or not os.path.exists(music['path']): self.log.info("bot: current music isn't ready, start to download.") music = self.download_music() @@ -413,7 +386,7 @@ class MumbleBot: if music['duration'] > var.config.getint('bot', 'max_track_duration'): # Check the length, useful in case of playlist, it wasn't checked before) self.log.info( - "the music " + music["url"] + " has a duration of " + music['duration'] + "s -- too long") + "the music " + music["url"] + " has a duration of " + str(music['duration']) + "s -- too long") self.send_msg(constants.strings('too_long')) return False else: @@ -446,7 +419,7 @@ class MumbleBot: if not os.path.isfile(mp3): # download the music music['ready'] = "downloading" - + var.playlist.update(music, index) self.log.info("bot: downloading url (%s) %s " % (music['title'], url)) ydl_opts = "" @@ -502,16 +475,25 @@ class MumbleBot: # Function start if the next music isn't ready # Do nothing in case the next music is already downloaded self.log.debug("bot: Async download next asked ") - if var.playlist.length() > 1 and var.playlist.next_item()['type'] == 'url' \ - and (var.playlist.next_item()['ready'] in ["no", "validation"]): - th = threading.Thread( - target=self.download_music, name="DownloadThread", args=(var.playlist.next_index(),)) - self.log.info( - "bot: start downloading item in thread: " + util.format_debug_song_string(var.playlist.next_item())) - th.daemon = True - th.start() - else: - return + if var.playlist.next_item() and var.playlist.next_item()['type'] == 'url': + # usually, all validation will be done when adding to the list. + # however, for performance consideration, youtube playlist won't be validate when added. + # the validation has to be done here. + while var.playlist.next_item() and var.playlist.next_item()['ready'] == "validation": + music = self.validate_music(var.playlist.next_item()) + if music: + var.playlist.update(music, var.playlist.next_index()) + break + else: + var.playlist.remove(var.playlist.next_index()) + + if var.playlist.next_item() and var.playlist.next_item()['ready'] == "no": + th = threading.Thread( + target=self.download_music, name="DownloadThread", args=(var.playlist.next_index(),)) + self.log.info( + "bot: start downloading item in thread: " + util.format_debug_song_string(var.playlist.next_item())) + th.daemon = True + th.start() def check_item_path_or_remove(self, index = -1): if index == -1: @@ -549,12 +531,15 @@ class MumbleBot: raw_music = "" while not self.exit and self.mumble.is_alive(): - while self.mumble.sound_output.get_buffer_size() > 0.5 and not self.exit: + while self.thread and self.mumble.sound_output.get_buffer_size() > 0.5 and not self.exit: # If the buffer isn't empty, I cannot send new music part, so I wait + self._loop_status = 'Wait for buffer %.3f' % self.mumble.sound_output.get_buffer_size() time.sleep(0.01) + if self.thread: # I get raw from ffmpeg thread # move playhead forward + self._loop_status = 'Reading raw' if self.song_start_at == -1: self.song_start_at = time.time() - self.playhead self.playhead = time.time() - self.song_start_at @@ -578,12 +563,23 @@ class MumbleBot: else: time.sleep(0.1) - if self.thread is None or not raw_music: - # Not music into the buffet - if not self.is_pause: - if len(var.playlist) > 0 and var.playlist.next(): + if not self.is_pause and (self.thread is None or not raw_music): + # ffmpeg thread has gone. indicate that last song has finished. move to the next song. + if not self.wait_for_downloading: + if var.playlist.next(): + # if downloading in the other thread self.launch_music() self.async_download_next() + else: + self._loop_status = 'Empty queue' + else: + print(var.playlist.current_item()["ready"]) + if var.playlist.current_item()["ready"] != "downloading": + self.wait_for_downloading = False + self.launch_music() + self.async_download_next() + else: + self._loop_status = 'Wait for downloading' while self.mumble.sound_output.get_buffer_size() > 0: # Empty the buffer before exit @@ -591,6 +587,7 @@ class MumbleBot: time.sleep(0.5) if self.exit: + self._loop_status = "exited" if var.config.getboolean('bot', 'save_playlist', fallback=True): self.log.info("bot: save playlist into database") var.playlist.save()