feat: Whitelist URL feature.

In the configuration, `max_duration` can be set to prevent
long song(URL item) being downloaded and added to the
playlist.
This whitelist feature provided a way to override this
duration check: songs being whitelisted will be added to
the playlist no matter how long they are.
Three admin commands are introduced:
 - !urlwhitelist (!urlw)
 - !urlunwhitelist, (!urlunw)
 - !urlwhitelistlist, (!urlwls).

 Also, if one song fails due to its length, the bot will
 show the length of this song and the max length limit in
 the reply message.

 Implement #173, #196.
This commit is contained in:
Terry Geng 2021-02-17 12:55:11 +08:00
parent f59062ec1f
commit 1f9573b1d5
11 changed files with 176 additions and 111 deletions

View File

@ -14,7 +14,7 @@ import util
import variables as var
from pyradios import RadioBrowser
from database import SettingsDatabase, MusicDatabase, Condition
from media.item import item_id_generators, dict_to_item, dicts_to_items
from media.item import item_id_generators, dict_to_item, dicts_to_items, ValidationFailedError
from media.cache import get_cached_wrapper_from_scrap, get_cached_wrapper_by_id, get_cached_wrappers_by_tags, \
get_cached_wrapper, get_cached_wrappers, get_cached_wrapper_from_dict, get_cached_wrappers_from_dicts
from media.url_from_playlist import get_playlist_info
@ -74,6 +74,9 @@ def register_all_commands(bot):
bot.register_command(commands('url_ban'), cmd_url_ban, no_partial_match=True, admin=True)
bot.register_command(commands('url_ban_list'), cmd_url_ban_list, no_partial_match=True, admin=True)
bot.register_command(commands('url_unban'), cmd_url_unban, no_partial_match=True, admin=True)
bot.register_command(commands('url_unwhitelist'), cmd_url_unwhitelist, no_partial_match=True, admin=True)
bot.register_command(commands('url_whitelist'), cmd_url_whitelist, no_partial_match=True, admin=True)
bot.register_command(commands('url_whitelist_list'), cmd_url_whitelist_list, no_partial_match=True, admin=True)
bot.register_command(commands('user_ban'), cmd_user_ban, no_partial_match=True, admin=True)
bot.register_command(commands('user_unban'), cmd_user_unban, no_partial_match=True, admin=True)
@ -149,46 +152,102 @@ def cmd_user_ban(bot, user, text, command, parameter):
global log
if parameter:
bot.mumble.users[text.actor].send_text_message(util.user_ban(parameter))
var.db.set("user_ban", parameter, None)
bot.send_msg(tr("user_ban_success", user=parameter), text)
else:
bot.mumble.users[text.actor].send_text_message(util.get_user_ban())
ban_list = "<ul>"
for i in var.db.items("url_ban"):
ban_list += "<li>" + i[0] + "</li>"
ban_list += "</ul>"
bot.send_msg(tr("user_ban_list", list=ban_list), text)
def cmd_user_unban(bot, user, text, command, parameter):
global log
if parameter:
bot.mumble.users[text.actor].send_text_message(util.user_unban(parameter))
if parameter and var.db.has_option("user_ban", parameter):
var.db.remove_option("user_ban", parameter)
bot.send_msg(tr("user_unban_success", user=parameter), text)
def cmd_url_ban(bot, user, text, command, parameter):
global log
if parameter:
bot.mumble.users[text.actor].send_text_message(util.url_ban(util.get_url_from_input(parameter)))
id = item_id_generators['url'](url=parameter)
var.cache.free_and_delete(id)
var.playlist.remove_by_id(id)
url = util.get_url_from_input(parameter)
if url:
_id = item_id_generators['url'](url=url)
var.cache.free_and_delete(_id)
var.playlist.remove_by_id(_id)
else:
if var.playlist.current_item() and var.playlist.current_item().type == 'url':
item = var.playlist.current_item().item()
bot.mumble.users[text.actor].send_text_message(util.url_ban(util.get_url_from_input(item.url)))
url = item.url
var.cache.free_and_delete(item.id)
var.playlist.remove_by_id(item.id)
else:
bot.send_msg(tr('bad_parameter', command=command), text)
return
# Remove from the whitelist first
if var.db.has_option('url_whitelist', url):
var.db.remove_option("url_whitelist", url)
bot.send_msg(tr("url_unwhitelist_success", url=url), text)
if not var.db.has_option('url_ban', url):
var.db.set("url_ban", url, None)
bot.send_msg(tr("url_ban_success", url=url), text)
def cmd_url_ban_list(bot, user, text, command, parameter):
bot.mumble.users[text.actor].send_text_message(util.get_url_ban())
ban_list = "<ul>"
for i in var.db.items("url_ban"):
ban_list += "<li>" + i[0] + "</li>"
ban_list += "</ul>"
bot.send_msg(tr("url_ban_list", list=ban_list), text)
def cmd_url_unban(bot, user, text, command, parameter):
global log
url = util.get_url_from_input(parameter)
if url:
var.db.remove_option("url_ban", url)
bot.send_msg(tr("url_unban_success", url=url), text)
else:
bot.send_msg(tr('bad_parameter', command=command), text)
if parameter:
bot.mumble.users[text.actor].send_text_message(util.url_unban(util.get_url_from_input(parameter)))
def cmd_url_whitelist(bot, user, text, command, parameter):
url = util.get_url_from_input(parameter)
if url:
# Unban first
if var.db.has_option('url_ban', url):
var.db.remove_option("url_ban", url)
bot.send_msg(tr("url_unban_success"), text)
# Then add to whitelist
if not var.db.has_option('url_whitelist', url):
var.db.set("url_whitelist", url, None)
bot.send_msg(tr("url_whitelist_success", url=url), text)
else:
bot.send_msg(tr('bad_parameter', command=command), text)
def cmd_url_whitelist_list(bot, user, text, command, parameter):
ban_list = "<ul>"
for i in var.db.items("url_whitelist"):
ban_list += "<li>" + i[0] + "</li>"
ban_list += "</ul>"
bot.send_msg(tr("url_whitelist_list", list=ban_list), text)
def cmd_url_unwhitelist(bot, user, text, command, parameter):
url = util.get_url_from_input(parameter)
if url:
var.db.remove_option("url_whitelist", url)
bot.send_msg(tr("url_unwhitelist_success"), text)
else:
bot.send_msg(tr('bad_parameter', command=command), text)
def cmd_play(bot, user, text, command, parameter):
@ -338,6 +397,7 @@ def cmd_play_url(bot, user, text, command, parameter):
log.info("cmd: add to playlist: " + music_wrapper.format_debug_string())
send_item_added_message(bot, music_wrapper, len(var.playlist) - 1, text)
if len(var.playlist) == 2:
# If I am the second item on the playlist. (I am the next one!)
bot.async_download_next()

View File

@ -183,6 +183,9 @@ user_unban = userunban
url_ban = urlban
url_ban_list = urlbanlist
url_unban = urlunban
url_whitelist = urlwhitelist, urlw
url_unwhitelist = urlunwhitelist, urlunw
url_whitelist_list = urlwhitelistlist, urlwls
ducking = duck
ducking_threshold = duckthres

View File

@ -61,17 +61,26 @@
"shortlist_instruction": "Use <i>!sl {indexes}</i> to play the item you want.",
"start_updating": "Start updating...",
"stopped": "Music stopped.",
"too_long": "<b>{song}</b> is too long, removed from playlist!",
"too_long": "<b>{song}</b> is too long ({duration} > {max_duration}), removed from playlist!",
"unable_download": "Unable to download <b>{item}</b>. Removed from the library.",
"unable_play": "Unable to play <b>{item}</b>. Removed from the library.",
"unknown_mode": "Unknown playback mode '{mode}'. It should be one of <i>one-shot</i>, <i>repeat</i>, <i>random</i>.",
"update_successful": "<h2>botamusique v{version} Installed!</h2><hr />\n<h3>Changelog</h3> {changelog} <hr /> Visit <a href=\"https://github.com/azlux/botamusique\">our github repo</a> for more details!",
"url": "URL",
"url_ban": "This URL is banned!",
"url_ban": "The URL {url} is banned! Removed from playlist!",
"url_ban_success": "The following URL is banned: {url}.",
"url_ban_list": "List of banned URL: <br>{list}",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a> <i>from playlist</i> <a href=\"{playlist_url}\">{playlist}</a> <i>added by</i> {user}",
"url_item": "<a href=\"{url}\"><b>{title}</b></a> <i>added by</i> {user}",
"url_unban_success": "The following URL is unbanned: {url}.",
"url_unwhitelist_success": "The following URL is un-whitelisted: {url}.",
"url_whitelist_success": "The following URL is whitelisted: {url}.",
"url_whitelist_list": "List of whitelisted URL: <br>{list}",
"user_ban": "You are banned, not allowed to do that!",
"user_ban_success": "User {user} is banned.",
"user_ban_list": "List of banned user: <br>{list}",
"user_unban_success": "User {user} is unbanned.",
"user_password_set": "Your password has been updated.",
"web_user_list": "Following users have the privilege to access the web interface: <br /> {users}",
"webpage_address": "Your own address to access the web interface is <a href=\"{address}\">{address}</a>",

View File

@ -61,13 +61,13 @@
"shortlist_instruction": "Use <i>!sl {índices}</i> para reproducir los elementos que usted desea.",
"start_updating": "Empezando la actualización...",
"stopped": "Música fue detenida.",
"too_long": "<b>{song}</b> es muy larga. Eliminada de la lista de reproducción!",
"too_long": "<b>{song}</b> es muy larga ({duration} > {max_duration}). Eliminada de la lista de reproducción!",
"unable_download": "No fue posible descargar <b>{item}</b>. Eliminado de la biblioteca.",
"unable_play": "No fue posible reproducir <b>{item}</b>. Eliminado de la biblioteca.",
"unknown_mode": "Modo de reproducción '{mode}' desconocido. Debiera ser o bien <i>one-shot</i>, <i>repeat</i> o <i>random</i>.",
"update_successful": "<h2>botamusique v{version} instalado!</h2><hr />\n<h3>Lista de cambios</h3> {changelog} <hr /> Visite <a href=\"https://github.com/azlux/botamusique\">nuestro repositorio en Github</a> para más detalles!",
"url": "URL",
"url_ban": "Esta URL está baneada!",
"url_ban": "URL {url} está baneada! Eliminada de la lista de reproducción!",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a> <i>de lista de reproducción</i> <a href=\"{playlist_url}\">{playlist}</a> <i>añadido por</i> {user}",
"url_item": "<a href=\"{url}\"><b>{title}</b></a> <i>añadido por</i> {user}",

View File

@ -61,13 +61,13 @@
"shortlist_instruction": "Utilisez <i>!sl {indexes}</i> pour jouer l'élément que vous voulez.",
"start_updating": "Début de la mise à jour...",
"stopped": "Musique arrêté.",
"too_long": "<b>{song}</b> est trop long, supprimé de la playlist !",
"too_long": "<b>{song}</b> est trop long ({duration} > {max_duration}), supprimé de la playlist !",
"unable_download": "Impossible de télécharger <b>{item}</b>. Retiré de la bibliothèque.",
"unable_play": "Impossible de jouer <b>{item}</b>. Retiré de la bibliothèque.",
"unknown_mode": "Mode de lecture \"{mode}\" inconnu. Il devrait s'agir d'un des modes suivants : <i>one-shot</i>, <i>repeat</i>, <i>random</i>.",
"update_successful": "<h2>botamusique v{version} Installé ! </h2><hr />\n<h3>Changelog</h3> {changelog} <hr /> Visitez <a href=\"https://github.com/azlux/botamusique\">notre repo github</a> pour plus de détails !",
"url": "URL",
"url_ban": "Cette URL est interdite !",
"url_ban": "URL {url} est interdite !",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a> <i>depuis la playlist</i> <a href=\"{playlist_url}\">{playlist}</a> <i>ajouté par</i> {user}",
"url_item": "<a href=\"{url}\"><b>{title}</b></a> <i>ajouté par</i> {user}",

View File

@ -61,13 +61,13 @@
"shortlist_instruction": "Usa <i>!sl {indexes}</i> per riprodurre l'elemento desiderato.",
"start_updating": "Inizio aggiornamento...",
"stopped": "Riproduzione interrotta.",
"too_long": "<b>{song}</b> è troppo lunga, rimossa dalla playlist!",
"too_long": "<b>{song}</b> è troppo lunga ({duration} > {max_duration}), rimossa dalla playlist!",
"unable_download": "Impossibile scaricare <b>{item}</b>. Rimosso dalla libreria.",
"unable_play": "Impossibile riprodurre <b>{item}</b>. Rimosso dalla libreria.",
"unknown_mode": "Modalità di riproduzione '{mode}' sconosciuta. Dovrebbe essere <i>one-shot</i>, <i>ripeti</i>, <i>casuale</i>.",
"update_successful": "<h2>botamusique v{version} installato!</h2><hr />\n<h3>Changelog</h3> {changelog} <hr /> Visita <a href=\"https://github.com/azlux/botamusique\">la nostra repository GitHub</a> per ulteriori dettagli!",
"url": "URL",
"url_ban": "Questo URL è vietato!",
"url_ban": "URL {url} è vietato!",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a> <i>dalla playlist</i> <a href=\"{playlist_url}\">{playlist}</a> <i>aggiunto da</i> {user}",
"url_item": "<a href=\"{url}\"><b>{title}</b></a> <i>aggiunto da</i> {user}",

View File

@ -53,21 +53,21 @@
"rb_play_empty": "ラジオIDを提供してください。",
"rb_query_result": "検索の結果(<i> !rbplay {ID} </i>を送信して再生する)",
"records_omitted": "…",
"removed_tags": "もう<b>{song}</b>からタグ「 <i>{tags}</i>」を削除しました。",
"removed_tags_from_all": "もう再生リストの全ての曲にタグ「<i>{tags}</i> 」を削除しました。",
"removing_item": "もう再生リストに「{item}」を削除しました。",
"removed_tags": "<b>{song}</b>からタグ「 <i>{tags}</i>」を削除しました。",
"removed_tags_from_all": "再生リストの全ての曲にタグ「<i>{tags}</i> 」を削除しました。",
"removing_item": "再生リストに「{item}」を削除しました。",
"repeat": "「{song}」を{n}回リピートするになります。",
"report_version": "現在のbotamusiqueバージョンは<b>{version}</b>です。",
"shortlist_instruction": "<i>!sl {indexes}</i>を使ってこのリストの曲を再生する。",
"start_updating": "更新しています…",
"stopped": "再生停止。",
"too_long": "「{song}」が長さ制限を超えました。削除されました。",
"too_long": "「{song}」が長さ制限を超えました{duration} > {max_duration}。削除されました。",
"unable_download": "「{item}」がダウンロードできません。削除されました。",
"unable_play": "「{item}」が再生できません。削除されました。",
"unknown_mode": "不正な再生モード「{mode}」。 <i>one-shot</i>, <i>repeat</i>, <i>random</i>, <i>autoplay</i>の中の一つを使ってください。",
"update_successful": "<h2>botamusique v{version} インストール完成!</h2><hr />\n<h3>更新履歴</h3> {changelog} <hr /> このプロジェクトの <a href=\"https://github.com/azlux/botamusique\">githubページ</a> をご覧ください!",
"url": "URL",
"url_ban": "このURLが禁止されています。",
"url_ban": "URL {url} が禁止されています。",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a>、(<a href=\"{playlist_url}\">{playlist}</a>から)、 <i>{user} </i>に追加されました。",
"url_item": "<a href=\"{url}\"><b>{title}</b></a><i> {user} </i>に追加されました。",

View File

@ -61,13 +61,13 @@
"shortlist_instruction": "使用<i>!sl {indexes}</i>播放列表中的曲目。",
"start_updating": "开始更新……",
"stopped": "音乐停止。",
"too_long": "<b>{song}</b>超出长度限制!已被移出播放列表。",
"too_long": "<b>{song}</b>超出长度限制{duration} > {max_duration}!已被移出播放列表。",
"unable_download": "无法下载<b>{item}</b>。已移出播放列表。",
"unable_play": "无法播放<b>{item}</b>。已移出播放列表。",
"unknown_mode": "未知播放模式\"{mode}\"。播放模式应为 <i>one-shot</i>, <i>repeat</i>, <i>random</i>中的一个。",
"update_successful": "<h2>botamusique v{version} 安装完毕!</h2><hr />\n<h3>更新日志</h3> {changelog} <hr /> 请访问我们的 <a href=\"https://github.com/azlux/botamusique\">github页面</a> 获取更多信息!",
"url": "URL",
"url_ban": "链接被列入黑名单了!",
"url_ban": "链接{url}被列入黑名单了!",
"url_from_playlist": "URL",
"url_from_playlist_item": "<a href=\"{url}\"><b>{title}</b></a>,来自播放列表 <a href=\"{playlist_url}\">{playlist}</a>,由<i> {user} </i>添加。",
"url_item": "<a href=\"{url}\"><b>{title}</b></a><i>由</i> {user} <i>添加</i>。",

View File

@ -15,6 +15,7 @@ import variables as var
from media.item import BaseItem, item_builders, item_loaders, item_id_generators, ValidationFailedError, \
PreparationFailedError
import media.system
from util import format_time
log = logging.getLogger("bot")
@ -74,36 +75,45 @@ class URLItem(BaseItem):
return True
def validate(self):
self.validating_lock.acquire()
if self.ready in ['yes', 'validated']:
try:
self.validating_lock.acquire()
if self.ready in ['yes', 'validated']:
return True
# if self.ready == 'failed':
# self.validating_lock.release()
# return False
#
if os.path.exists(self.path):
self.ready = "yes"
return True
# Check if this url is banned
if var.db.has_option('url_ban', self.url):
raise ValidationFailedError(tr('url_ban', url=self.url))
# avoid multiple process validating in the meantime
info = self._get_info_from_url()
if not info:
return False
# Check if the song is too long and is not whitelisted
max_duration = var.config.getint('bot', 'max_track_duration') * 60
if max_duration and \
not var.db.has_option('url_whitelist', self.url) and \
self.duration > max_duration:
log.info(
"url: " + self.url + " has a duration of " + str(self.duration / 60) + " min -- too long")
raise ValidationFailedError(tr('too_long', song=self.format_title(),
duration=format_time(self.duration),
max_duration=format_time(max_duration)))
else:
self.ready = "validated"
self.version += 1 # notify wrapper to save me
return True
finally:
self.validating_lock.release()
return True
# if self.ready == 'failed':
# self.validating_lock.release()
# return False
#
if os.path.exists(self.path):
self.validating_lock.release()
self.ready = "yes"
return True
# avoid multiple process validating in the meantime
info = self._get_info_from_url()
self.validating_lock.release()
if not info:
return False
if self.duration > var.config.getint('bot', 'max_track_duration') * 60 != 0:
# Check the length, useful in case of playlist, it wasn't checked before)
log.info(
"url: " + self.url + " has a duration of " + str(self.duration / 60) + " min -- too long")
raise ValidationFailedError(tr('too_long', song=self.title))
else:
self.ready = "validated"
self.version += 1 # notify wrapper to save me
return True
# Run in a other thread
def prepare(self):
@ -126,8 +136,8 @@ class URLItem(BaseItem):
try:
info = ydl.extract_info(self.url, download=False)
self.duration = info['duration']
self.title = info['title']
self.keywords = info['title']
self.title = info['title'].strip()
self.keywords = self.title
succeed = True
return True
except youtube_dl.utils.DownloadError:
@ -237,7 +247,7 @@ class URLItem(BaseItem):
return display
def format_title(self):
return self.title if self.title.strip() else self.url
return self.title if self.title else self.url
def display_type(self):
return tr("url")

View File

@ -276,12 +276,10 @@ class MumbleBot:
if not self.is_admin(user) and parameter:
input_url = util.get_url_from_input(parameter)
if input_url:
for i in var.db.items("url_ban"):
if input_url == i[0]:
self.mumble.users[text.actor].send_text_message(
tr('url_ban'))
return
if input_url and var.db.has_option('url_ban', input_url):
self.mumble.users[text.actor].send_text_message(
tr('url_ban'))
return
command_exc = ""
try:
@ -408,13 +406,12 @@ class MumbleBot:
# Function start if the next music isn't ready
# Do nothing in case the next music is already downloaded
self.log.debug("bot: Async download next asked ")
while var.playlist.next_item() and var.playlist.next_item().type in ['url', 'url_from_playlist']:
while var.playlist.next_item():
# usually, all validation will be done when adding to the list.
# however, for performance consideration, youtube playlist won't be validate when added.
# the validation has to be done here.
next = var.playlist.next_item()
try:
next.validate()
if not next.is_ready():
self.async_download(next)
@ -432,8 +429,7 @@ class MumbleBot:
th.start()
return th
def validate_and_start_download(self, item):
item.validate()
def start_download(self, item):
if not item.is_ready():
self.log.info("bot: current music isn't ready, start downloading.")
self.async_download(item)
@ -442,15 +438,25 @@ class MumbleBot:
def _download(self, item):
ver = item.version
try:
item.validate()
if item.is_ready():
return True
except ValidationFailedError as e:
self.send_channel_msg(e.msg)
var.playlist.remove_by_id(item.id)
var.cache.free_and_delete(item.id)
return False
try:
item.prepare()
if item.version > ver:
var.playlist.version += 1
return True
except PreparationFailedError as e:
self.send_channel_msg(e.msg)
return False
if item.version > ver:
var.playlist.version += 1
# =======================
# Loop
# =======================
@ -528,7 +534,7 @@ class MumbleBot:
current = var.playlist.current_item()
self.log.debug(f"bot: next into the song: {current.format_debug_string()}")
try:
self.validate_and_start_download(current)
self.start_download(current)
self.wait_for_ready = True
self.song_start_at = -1
@ -640,7 +646,7 @@ class MumbleBot:
current = var.playlist.current_item()
self.validate_and_start_download(current)
self.start_download(current)
self.is_pause = False
self.wait_for_ready = True
self.song_start_at = -1

43
util.py
View File

@ -159,37 +159,6 @@ def update(current_version):
return msg
def user_ban(user):
var.db.set("user_ban", user, None)
res = "User " + user + " banned"
return res
def user_unban(user):
var.db.remove_option("user_ban", user)
res = "Done"
return res
def get_url_ban():
res = "List of ban:"
for i in var.db.items("url_ban"):
res += "<br/>" + i[0]
return res
def url_ban(url):
var.db.set("url_ban", url, None)
res = "url " + url + " banned"
return res
def url_unban(url):
var.db.remove_option("url_ban", url)
res = "Done"
return res
def pipe_no_wait():
""" Generate a non-block pipe used to fetch the STDERR of ffmpeg.
"""
@ -337,14 +306,14 @@ def get_url_from_input(string):
if res:
string = res.group(1)
else:
return False
return ""
match = re.search("(http|https)://(\S*)?/(\S*)", string, flags=re.IGNORECASE)
if match:
url = match[1].lower() + "://" + match[2].lower() + "/" + match[3]
return url
else:
return False
return ""
def youtube_search(query):
@ -412,6 +381,14 @@ def parse_time(human):
raise ValueError("Invalid time string given.")
def format_time(seconds):
hours = seconds // 3600
seconds = seconds % 3600
minutes = seconds // 60
seconds = seconds % 60
return f"{hours:d}:{minutes:02d}:{seconds:02d}"
def parse_file_size(human):
units = {"B": 1, "KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024, "TB": 1024 * 1024 * 1024 * 1024,
"K": 1024, "M": 1024 * 1024, "G": 1024 * 1024 * 1024, "T": 1024 * 1024 * 1024 * 1024}