REFACTOR: ITEM REVOLUTION #91

This commit is contained in:
Terry Geng
2020-03-05 16:28:08 +08:00
parent c32c6c5d9b
commit 6ab8a7958a
15 changed files with 1079 additions and 899 deletions

View File

@@ -0,0 +1,158 @@
import logging
import os
import re
from io import BytesIO
import base64
import hashlib
import mutagen
from PIL import Image
import json
import util
import variables as var
from media.item import BaseItem
import constants
'''
type : file
id
path
title
artist
duration
thumbnail
user
'''
class FileItem(BaseItem):
def __init__(self, bot, path, from_dict=None):
if not from_dict:
super().__init__(bot)
self.path = path
self.title = ""
self.artist = "??"
self.thumbnail = None
if self.path:
self.id = hashlib.md5(path.encode()).hexdigest()
if os.path.exists(self.uri()):
self._get_info_from_tag()
self.ready = "yes"
else:
super().__init__(bot, from_dict)
self.path = from_dict['path']
self.title = from_dict['title']
self.artist = from_dict['artist']
self.thumbnail = from_dict['thumbnail']
if not self.validate():
self.ready = "failed"
self.type = "file"
def uri(self):
return var.music_folder + self.path
def is_ready(self):
return True
def validate(self):
if not os.path.exists(self.uri()):
self.log.info(
"file: music file missed for %s" % self.format_debug_string())
self.send_client_message(constants.strings('file_missed', file=self.path))
return False
self.ready = "yes"
return True
def _get_info_from_tag(self):
match = re.search("(.+)\.(.+)", self.uri())
assert match is not None
file_no_ext = match[1]
ext = match[2]
try:
im = None
path_thumbnail = file_no_ext + ".jpg"
if os.path.isfile(path_thumbnail):
im = Image.open(path_thumbnail)
if ext == "mp3":
# title: TIT2
# artist: TPE1, TPE2
# album: TALB
# cover artwork: APIC:
tags = mutagen.File(self.uri())
if 'TIT2' in tags:
self.title = tags['TIT2'].text[0]
if 'TPE1' in tags: # artist
self.artist = tags['TPE1'].text[0]
if im is None:
if "APIC:" in tags:
im = Image.open(BytesIO(tags["APIC:"].data))
elif ext == "m4a" or ext == "m4b" or ext == "mp4" or ext == "m4p":
# title: ©nam (\xa9nam)
# artist: ©ART
# album: ©alb
# cover artwork: covr
tags = mutagen.File(self.uri())
if '©nam' in tags:
self.title = tags['©nam'][0]
if '©ART' in tags: # artist
self.artist = tags['©ART'][0]
if im is None:
if "covr" in tags:
im = Image.open(BytesIO(tags["covr"][0]))
if im:
self.thumbnail = self._prepare_thumbnail(im)
except:
pass
if not self.title:
self.title = os.path.basename(file_no_ext)
def _prepare_thumbnail(self, im):
im.thumbnail((100, 100), Image.ANTIALIAS)
buffer = BytesIO()
im = im.convert('RGB')
im.save(buffer, format="JPEG")
return base64.b64encode(buffer.getvalue()).decode('utf-8')
def to_dict(self):
dict = super().to_dict()
dict['type'] = 'file'
dict['path'] = self.path
dict['title'] = self.title
dict['artist'] = self.artist
dict['thumbnail'] = self.thumbnail
return dict
def format_debug_string(self):
return "[file] {artist} - {title} ({path})".format(
title=self.title,
artist=self.artist,
path=self.path
)
def format_song_string(self, user):
return constants.strings("file_item",
title=self.title,
artist=self.artist,
user=user
)
def format_current_playing(self, user):
display = constants.strings("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
def display_type(self):
return constants.strings("file")

98
media/item.py Normal file
View File

@@ -0,0 +1,98 @@
import logging
import threading
import os
import re
from io import BytesIO
import base64
import hashlib
import mutagen
from PIL import Image
import util
import variables as var
"""
FORMAT OF A MUSIC INTO THE PLAYLIST
type : url
id
url
title
path
duration
artist
thumbnail
user
ready (validation, no, downloading, yes, failed)
from_playlist (yes,no)
playlist_title
playlist_url
type : radio
id
url
name
current_title
user
"""
class BaseItem:
def __init__(self, bot, from_dict=None):
self.bot = bot
self.log = logging.getLogger("bot")
self.type = "base"
if from_dict is None:
self.id = ""
self.ready = "pending" # pending - is_valid() -> validated - prepare() -> yes, failed
else:
self.id = from_dict['id']
self.ready = from_dict['ready']
def is_ready(self):
return True if self.ready == "yes" else False
def is_failed(self):
return True if self.ready == "failed" else False
def validate(self):
return False
def uri(self):
raise
def async_prepare(self):
th = threading.Thread(
target=self.prepare, name="Prepare-" + self.id[:7])
self.log.info(
"%s: start preparing item in thread: " % self.type + self.format_debug_string())
th.daemon = True
th.start()
#self.download_threads.append(th)
return th
def prepare(self):
return True
def play(self):
pass
def format_song_string(self, user):
return self.id
def format_current_playing(self, user):
return self.id
def format_debug_string(self):
return self.id
def display_type(self):
return ""
def send_client_message(self, msg):
self.bot.send_msg(msg)
def to_dict(self):
return {"type" : "base", "id": self.id, "ready": self.ready}

View File

@@ -1,44 +1,271 @@
import youtube_dl
import json
import random
import hashlib
import threading
import logging
import util
import variables as var
from media.item import BaseItem
from media.file import FileItem
from media.url import URLItem
def get_playlist_info(url, start_index=0, user=""):
items = []
ydl_opts = {
'extract_flat': 'in_playlist'
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
attempts = var.config.getint('bot', 'download_attempts', fallback=2)
for i in range(attempts):
try:
info = ydl.extract_info(url, download=False)
# # if url is not a playlist but a video
# if 'entries' not in info and 'webpage_url' in info:
# music = {'type': 'url',
# 'title': info['title'],
# 'url': info['webpage_url'],
# 'user': user,
# 'ready': 'validation'}
# items.append(music)
# return items
class PlaylistItemWrapper:
def __init__(self, item, user):
self.item = item
self.user = user
playlist_title = info['title']
for j in range(start_index, min(len(info['entries']), start_index + var.config.getint('bot', 'max_track_playlist'))):
# Unknow String if No title into the json
title = info['entries'][j]['title'] if 'title' in info['entries'][j] else "Unknown Title"
# Add youtube url if the url in the json isn't a full url
url = info['entries'][j]['url'] if info['entries'][j]['url'][0:4] == 'http' else "https://www.youtube.com/watch?v=" + info['entries'][j]['url']
def to_dict(self):
dict = self.item.to_dict()
dict['user'] = self.user
return dict
music = {'type': 'url',
'title': title,
'url': url,
'user': user,
'from_playlist': True,
'playlist_title': playlist_title,
'playlist_url': url,
'ready': 'validation'}
items.append(music)
except:
pass
def format_current_playing(self):
return self.item.format_current_playing(self.user)
return items
def format_song_string(self):
return self.item.format_song_string(self.user)
def format_debug_string(self):
return self.item.format_debug_string()
def dict_to_item(dict):
if dict['type'] == 'file':
return PlaylistItemWrapper(FileItem(var.bot, "", dict), dict['user'])
elif dict['type'] == 'url':
return PlaylistItemWrapper(URLItem(var.bot, "", dict), dict['user'])
class PlayList(list):
def __init__(self, *args):
super().__init__(*args)
self.current_index = -1
self.version = 0 # increase by one after each change
self.mode = "one-shot" # "repeat", "random"
self.pending_items = []
self.log = logging.getLogger("bot")
self.validating_thread_lock = threading.Lock()
def is_empty(self):
return True if len(self) == 0 else False
def set_mode(self, mode):
# modes are "one-shot", "repeat", "random"
self.mode = mode
if mode == "random":
self.randomize()
elif mode == "one-shot" and self.current_index > 0:
# remove items before current item
self.version += 1
for i in range(self.current_index):
super().__delitem__(0)
self.current_index = 0
def append(self, item: PlaylistItemWrapper):
self.version += 1
super().append(item)
self.pending_items.append(item)
self.start_async_validating()
return item
def insert(self, index, item):
self.version += 1
if index == -1:
index = self.current_index
item = util.attach_music_tag_info(item)
super().insert(index, item)
if index <= self.current_index:
self.current_index += 1
self.pending_items.append(item)
self.start_async_validating()
return item
def length(self):
return len(self)
def extend(self, items):
self.version += 1
items = list(map(
lambda item: item,
items))
super().extend(items)
self.pending_items.extend(items)
self.start_async_validating()
return items
def next(self):
if len(self) == 0:
return False
self.version += 1
#logging.debug("playlist: Next into the queue")
if self.current_index < len(self) - 1:
if self.mode == "one-shot" and self.current_index != -1:
super().__delitem__(self.current_index)
else:
self.current_index += 1
return self[self.current_index]
else:
self.current_index = 0
if self.mode == "one-shot":
self.clear()
return False
elif self.mode == "repeat":
return self[0]
elif self.mode == "random":
self.randomize()
return self[0]
else:
raise TypeError("Unknown playlist mode '%s'." % self.mode)
def point_to(self, index):
if -1 <= index < len(self):
self.current_index = index
def find(self, id):
for index, wrapper in enumerate(self):
if wrapper.item.id == id:
return index
return None
def update(self, item, id):
self.version += 1
index = self.find(id)
if index:
self[index] = item
return True
return False
def __delitem__(self, key):
return self.remove(key)
def remove(self, index=-1):
self.version += 1
if index > len(self) - 1:
return False
if index == -1:
index = self.current_index
removed = self[index]
super().__delitem__(index)
if self.current_index > index:
self.current_index -= 1
return removed
def remove_by_id(self, id):
to_be_removed = []
for index, item in enumerate(self):
if item.id == id:
to_be_removed.append(index)
for index in to_be_removed:
self.remove(index)
def current_item(self):
if len(self) == 0:
return False
return self[self.current_index]
def next_index(self):
if len(self) == 0 or (len(self) == 1 and self.mode == 'one_shot'):
return False
if self.current_index < len(self) - 1:
return self.current_index + 1
else:
return 0
def next_item(self):
if len(self) == 0 or (len(self) == 1 and self.mode == 'one_shot'):
return False
return self[self.next_index()]
def jump(self, index):
if self.mode == "one-shot":
for i in range(index):
super().__delitem__(0)
self.current_index = 0
else:
self.current_index = index
self.version += 1
return self[self.current_index]
def randomize(self):
# current_index will lose track after shuffling, thus we take current music out before shuffling
#current = self.current_item()
#del self[self.current_index]
random.shuffle(self)
#self.insert(0, current)
self.current_index = -1
self.version += 1
def clear(self):
self.version += 1
self.current_index = -1
super().clear()
def save(self):
var.db.remove_section("playlist_item")
var.db.set("playlist", "current_index", self.current_index)
for index, music in enumerate(self):
var.db.set("playlist_item", str(index), json.dumps(music.to_dict()))
def load(self):
current_index = var.db.getint("playlist", "current_index", fallback=-1)
if current_index == -1:
return
items = list(var.db.items("playlist_item"))
items.sort(key=lambda v: int(v[0]))
self.extend(list(map(lambda v: dict_to_item(json.loads(v[1])), items)))
self.current_index = current_index
def _debug_print(self):
print("===== Playlist(%d)=====" % self.current_index)
for index, item_wrapper in enumerate(self):
if index == self.current_index:
print("-> %d %s" % (index, item_wrapper.item.title))
else:
print("%d %s" % (index, item_wrapper.item.title))
print("===== End =====")
def start_async_validating(self):
if not self.validating_thread_lock.locked():
th = threading.Thread(target=self._check_valid, name="Validating")
th.daemon = True
th.start()
def _check_valid(self):
self.log.debug("playlist: start validating...")
self.validating_thread_lock.acquire()
while len(self.pending_items) > 0:
item = self.pending_items.pop().item
self.log.debug("playlist: validating %s" % item.format_debug_string())
if not item.validate() or item.ready == 'failed':
# TODO: logging
self.remove_by_id(item.id)
self.log.debug("playlist: validating finished.")
self.validating_thread_lock.release()

View File

@@ -1,16 +1,19 @@
import re
import logging
import json
import http.client
import struct
import requests
import traceback
import hashlib
from media.item import BaseItem
import constants
log = logging.getLogger("bot")
def get_radio_server_description(url):
global log
log.debug("radio: fetching radio server description")
p = re.compile('(https?\:\/\/[^\/]*)', re.IGNORECASE)
res = re.search(p, url)
base_url = res.group(1)
@@ -50,6 +53,9 @@ def get_radio_server_description(url):
def get_radio_title(url):
global log
log.debug("radio: fetching radio server description")
try:
r = requests.get(url, headers={'Icy-MetaData': '1'}, stream=True, timeout=5)
icy_metaint_header = int(r.headers['icy-metaint'])
@@ -67,3 +73,57 @@ def get_radio_title(url):
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
pass
return url
class RadioItem(BaseItem):
def __init__(self, bot, url, name="", from_dict=None):
if from_dict is None:
super().__init__(bot)
self.url = url
if not name:
self.title = get_radio_server_description(self.url) # The title of the radio station
else:
self.title = name
self.id = hashlib.md5(url.encode()).hexdigest()
else:
super().__init__(bot, from_dict)
self.url = from_dict['url']
self.title = from_dict['title']
self.type = "radio"
def validate(self):
return True
def is_ready(self):
return True
def uri(self):
return self.url
def to_dict(self):
dict = super().to_dict()
dict['url'] = self.url
dict['title'] = self.title
def format_debug_string(self):
return "[radio] {name} ({url})".format(
name=self.title,
url=self.url
)
def format_song_string(self, user):
return constants.strings("radio_item",
url=self.url,
title=get_radio_title(self.url), # the title of current song
name=self.title, # the title of radio station
user=user
)
def format_current_playing(self, user):
return constants.strings("now_playing", item=self.format_song_string(user))
def display_type(self):
return constants.strings("radio")

View File

@@ -1,22 +1,215 @@
import threading
import logging
import os
import hashlib
import traceback
from PIL import Image
import youtube_dl
import glob
import constants
import media
import variables as var
from media.file import FileItem
import media.system
log = logging.getLogger("bot")
def get_url_info(music):
ydl_opts = {
'noplaylist': True
}
music['duration'] = 0
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
for i in range(2):
try:
info = ydl.extract_info(music['url'], download=False)
music['duration'] = info['duration'] / 60
music['title'] = info['title']
except youtube_dl.utils.DownloadError:
pass
except KeyError:
return music
class URLItem(FileItem):
def __init__(self, bot, url, from_dict=None):
self.validating_lock = threading.Lock()
if from_dict is None:
self.url = url
self.title = ''
self.duration = 0
self.ready = 'pending'
super().__init__(bot, "")
self.id = hashlib.md5(url.encode()).hexdigest()
path = var.tmp_folder + self.id + ".mp3"
if os.path.isfile(path):
self.log.info("url: file existed for url %s " % self.url)
self.ready = 'yes'
self.path = path
self._get_info_from_tag()
else:
return music
return False
# self._get_info_from_url()
pass
else:
super().__init__(bot, "", from_dict)
self.url = from_dict['url']
self.duration = from_dict['duration']
self.downloading = False
self.type = "url"
def uri(self):
return self.path
def is_ready(self):
if self.downloading or self.ready != 'yes':
return False
if self.ready == 'yes' and not os.path.exists(self.path):
self.log.info(
"url: music file missed for %s" % self.format_debug_string())
self.ready = 'validated'
return False
return True
def validate(self):
if self.ready in ['yes', 'validated']:
return True
if os.path.exists(self.path):
self.ready = "yes"
return True
# avoid multiple process validating in the meantime
self.validating_lock.acquire()
info = self._get_info_from_url()
self.validating_lock.release()
if self.duration == 0 and not info:
return False
if self.duration > var.config.getint('bot', 'max_track_duration') != 0:
# Check the length, useful in case of playlist, it wasn't checked before)
log.info(
"url: " + self.url + " has a duration of " + str(self.duration) + " min -- too long")
self.send_client_message(constants.strings('too_long'))
return False
else:
self.ready = "validated"
return True
# Run in a other thread
def prepare(self):
if not self.downloading:
assert self.ready == 'validated'
return self._download()
else:
assert self.ready == 'yes'
return True
def _get_info_from_url(self):
self.log.info("url: fetching metadata of url %s " % self.url)
ydl_opts = {
'noplaylist': True
}
succeed = False
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
attempts = var.config.getint('bot', 'download_attempts', fallback=2)
for i in range(attempts):
try:
info = ydl.extract_info(self.url, download=False)
self.duration = info['duration'] / 60
self.title = info['title']
succeed = True
return True
except youtube_dl.utils.DownloadError:
pass
if not succeed:
self.ready = 'failed'
self.log.error("url: error while fetching info from the URL")
self.send_client_message(constants.strings('unable_download'))
return False
def _download(self):
media.system.clear_tmp_folder(var.tmp_folder, var.config.getint('bot', 'tmp_folder_max_size'))
self.downloading = True
base_path = var.tmp_folder + self.id
save_path = base_path + ".%(ext)s"
mp3_path = base_path + ".mp3"
# Download only if music is not existed
self.ready = "preparing"
self.log.info("bot: downloading url (%s) %s " % (self.title, self.url))
ydl_opts = ""
ydl_opts = {
'format': 'bestaudio/best',
'outtmpl': save_path,
'noplaylist': True,
'writethumbnail': True,
'updatetime': False,
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192'},
{'key': 'FFmpegMetadata'}]
}
# TODO
self.send_client_message(constants.strings('download_in_progress', item=self.url))
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
attempts = var.config.getint('bot', 'download_attempts', fallback=2)
download_succeed = False
for i in range(attempts):
self.log.info("bot: download attempts %d / %d" % (i+1, attempts))
try:
info = ydl.extract_info(self.url)
download_succeed = True
break
except:
error_traceback = traceback.format_exc().split("During")[0]
error = error_traceback.rstrip().split("\n")[-1]
self.log.error("bot: download failed with error:\n %s" % error)
if download_succeed:
self.path = mp3_path
self.ready = "yes"
self.log.info(
"bot: finished downloading url (%s) %s, saved to %s." % (self.title, self.url, self.path))
self.downloading = False
return True
else:
for f in glob.glob(base_path + "*"):
os.remove(f)
self.send_client_message(constants.strings('unable_download'))
self.ready = "failed"
self.downloading = False
return False
def _read_thumbnail_from_file(self, path_thumbnail):
if os.path.isfile(path_thumbnail):
im = Image.open(path_thumbnail)
self.thumbnail = self._prepare_thumbnail(im)
def to_dict(self):
dict = super().to_dict()
dict['type'] = 'url'
dict['url'] = self.url
dict['duration'] = self.duration
return dict
def format_debug_string(self):
return "[url] {title} ({url})".format(
title=self.title,
url=self.url
)
def format_song_string(self, user):
return constants.strings("url_item",
title=self.title,
url=self.url,
user=user)
def format_current_playing(self, user):
display = constants.strings("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
def display_type(self):
return constants.strings("url")

View File

@@ -0,0 +1,99 @@
import youtube_dl
import constants
import media
import variables as var
from media.url import URLItem
from media.playlist import PlaylistItemWrapper
def get_playlist_info(bot, url, start_index=0, user=""):
items = []
ydl_opts = {
'extract_flat': 'in_playlist'
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
attempts = var.config.getint('bot', 'download_attempts', fallback=2)
for i in range(attempts):
try:
info = ydl.extract_info(url, download=False)
# # if url is not a playlist but a video
# if 'entries' not in info and 'webpage_url' in info:
# music = {'type': 'url',
# 'title': info['title'],
# 'url': info['webpage_url'],
# 'user': user,
# 'ready': 'validation'}
# items.append(music)
# return items
playlist_title = info['title']
for j in range(start_index, min(len(info['entries']),
start_index + var.config.getint('bot', 'max_track_playlist'))):
# Unknow String if No title into the json
title = info['entries'][j]['title'] if 'title' in info['entries'][j] else "Unknown Title"
# Add youtube url if the url in the json isn't a full url
item_url = info['entries'][j]['url'] if info['entries'][j]['url'][0:4] == 'http' \
else "https://www.youtube.com/watch?v=" + info['entries'][j]['url']
music = PlaylistItemWrapper(
URLFromPlaylistItem(
bot,
item_url,
title,
url,
playlist_title
), user)
items.append(music)
except:
pass
return items
class URLFromPlaylistItem(URLItem):
def __init__(self, bot, url, title, playlist_url, playlist_title, from_dict=None):
if from_dict is None:
super().__init__(bot, url)
self.title = title
self.playlist_url = playlist_url
self.playlist_title = playlist_title
else:
super().__init__(bot, "", from_dict)
self.playlist_title = from_dict['playlist_title']
self.playlist_url = from_dict['playlist_url']
self.type = "url_from_playlist"
def to_dict(self):
dict = super().to_dict()
dict['playlist_url'] = self.playlist_url
dict['playlist_title'] = self.playlist_title
return dict
def format_debug_string(self):
return "[url] {title} ({url}) from playlist {playlist}".format(
title=self.title,
url=self.url,
playlist=self.playlist_title
)
def format_song_string(self, user):
return constants.strings("url_from_playlist_item",
title=self.title,
url=self.url,
playlist_url=self.playlist_url,
playlist=self.playlist_title,
user=user)
def format_current_playing(self, user):
display = constants.strings("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
def display_type(self):
return constants.strings("url_from_playlist")