storm/bragi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor: move music db into a separated file (#151)

Browse Source 
Merge pull request #152 from TerryGeng/separate-music-db
This commit is contained in:
azlux
2020-05-14 00:14:05 +02:00
committed by GitHub
parent 4c27bb28a1 5fe8b266c2
commit 140c523a39
8 changed files with 167 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -114,7 +114,7 @@ configuration.ini
music_folder/
tmp/
database.db
*.db
# Pycharm
.idea/

4
command.py
View File

@ -1151,9 +1151,9 @@ def cmd_drop_database(bot, user, text, command, parameter):
if bot.is_admin(user):
var.db.drop_table()
var.db = SettingsDatabase(var.dbfile)
var.db = SettingsDatabase(var.settings_db_path)
var.music_db.drop_table()
var.music_db = MusicDatabase(var.dbfile)
var.music_db = MusicDatabase(var.settings_db_path)
log.info("command: database dropped.")
bot.send_msg(constants.strings('database_dropped'), text)
else:

1
configuration.default.ini
View File

@ -45,7 +45,6 @@ admin = User1;User2;
music_folder = music_folder/
# Folder that stores the downloaded music.
tmp_folder = /tmp/
database_path = database.db
pip3_path = venv/bin/pip
auto_check_update = True
logfile =

55
configuration.example.ini
View File

@ -24,14 +24,30 @@ port = 64738
#username = botamusique
#comment = Hi, I'm here to play radio, local music or youtube/soundcloud music. Have fun!
# 'music_folder': Folder that stores your local songs.
#music_folder = music_folder/
# 'database_path': The path of the database. The database will store things like your volume
# set by command !volume, your playback mode and your playlist, banned URLs, etc.
# This option will be overridden by command line arguments.
# 'music_database_path': The path of database that stores the music library. Can be disabled by
# setting 'save_music_library=False'
#database_path=settings.db
#music_database_path=music.db
# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';'
#admin = User1;User2;
# 'volume' is default volume from 0 to 1.
# This option will be overridden by value in the database.
#volume = 0.1
# 'playback_mode' defined the playback mode of the bot.
# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist),
# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library).
# This option will be overridden by value in the database.
# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist),
# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library).
# This option will be overridden by value in the database.
# 'autoplay_length': how many songs the autoplay mode fills the playlist
# 'clear_when_stop_in_oneshot': clear the playlist when stop the bot in one-shot mode.
#playback_mode = one-shot
@ -42,17 +58,6 @@ port = 64738
# stable will use simple bash with curl command to get releases, testing will follow github master branch with git commands
#target_version = stable
# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';'
#admin = User1;User2;
# 'music_folder': Folder that stores your local songs.
#music_folder = music_folder/
# 'database_path': The path of the database. The database will store things like your volume
# set by command !volume, your playback mode and your playlist, etc.
# This option will be overridden by command line arguments.
#database_path = database.db
# 'tmp_folder': Folder that stores the downloaded music.
# 'tmp_folder_max_size': in MB, 0 for no cache, -1 for unlimited size
# 'ignored_folders', 'ignored_files': files and folders that would be ignored during scanning.
@ -79,11 +84,11 @@ port = 64738
#save_music_library = True
# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up.
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
#refresh_cache_on_startup = True
# 'save_playlist': If save_playlist is set True, the bot will save current playlist before quitting
# and reload it the next time it start. It requires save_music_library to be True to function.
# and reload it the next time it start. It requires save_music_library to be True to function.
#save_playlist = True
# 'max_track_playlist': Maximum track played when a playlist is added.
@ -93,24 +98,24 @@ port = 64738
#max_track_duration = 60
# 'ducking': If ducking is enabled, the bot will automatically attenuate its
# volume when someone is talking.
# volume when someone is talking.
#ducking = False
#ducking_volume = 0.05
#ducking_threshold = 3000
# 'when_nobody_in_channel': Specify what the bot should do if nobody is in the channel.
# Possible value of this options are:
# - "pause",
# - "pause_resume" (pause and resume once somebody re-enters the channel)
# - "stop" (also clears playlist)
# - "nothing" (do nothing)
# Possible value of this options are:
# - "pause",
# - "pause_resume" (pause and resume once somebody re-enters the channel)
# - "stop" (also clears playlist)
# - "nothing" (do nothing)
#when_nobody_in_channel = nothing
# [webinterface] stores settings related to the web interface.
[webinterface]
# 'enable': Set 'enabled' to True if you'd like to use the web interface to manage
# your playlist, upload files, etc.
# The web interface is disable by default for security and performance reason.
# your playlist, upload files, etc.
# The web interface is disable by default for security and performance reason.
#enabled = False
#listening_addr = 127.0.0.1
#listening_port = 8181
@ -148,7 +153,7 @@ port = 64738
# You may also customize commands recognized by the bot. For a full list of commands,
# see configuration.default.ini. Copy options you want to edit into this file.
# see configuration.default.ini. Copy options you want to edit into this file.
#play_file = file, f
#play_file_match = filematch, fm

158
database.py
View File

@ -1,3 +1,4 @@
import os
import re
import sqlite3
import json
@ -188,62 +189,13 @@ class Condition:
return self
SETTING_DB_VERSION = 1
SETTING_DB_VERSION = 2
MUSIC_DB_VERSION = 2
class SettingsDatabase:
def __init__(self, db_path):
self.db_path = db_path
# connect
conn = sqlite3.connect(self.db_path)
self.db_version_check_and_create()
conn.commit()
conn.close()
def has_table(self, table):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
tables = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?;", (table,)).fetchall()
conn.close()
if len(tables) == 0:
return False
return True
def db_version_check_and_create(self):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
if self.has_table('botamusique'):
# check version
ver = self.getint("bot", "db_version", fallback=None)
if ver is None or ver != SETTING_DB_VERSION:
# old_name = "botamusique_old_%s" % datetime.datetime.now().strftime("%Y%m%d")
# cursor.execute("ALTER TABLE botamusique RENAME TO %s" % old_name)
cursor.execute("DROP TABLE botamusique")
conn.commit()
self.create_table()
else:
self.create_table()
conn.close()
def create_table(self):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS botamusique ("
"section TEXT, "
"option TEXT, "
"value TEXT, "
"UNIQUE(section, option))")
cursor.execute("INSERT INTO botamusique (section, option, value) "
"VALUES (?, ?, ?)", ("bot", "db_version", SETTING_DB_VERSION))
conn.commit()
conn.close()
def get(self, section, option, **kwargs):
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
@ -321,9 +273,6 @@ class MusicDatabase:
def __init__(self, db_path):
self.db_path = db_path
MusicDatabaseMigration(self).migrate()
self.manage_special_tags()
def insert_music(self, music_dict, _conn=None):
conn = sqlite3.connect(self.db_path) if _conn is None else _conn
cursor = conn.cursor()
@ -539,15 +488,53 @@ class MusicDatabase:
conn.close()
class MusicDatabaseMigration:
def __init__(self, db: MusicDatabase):
self.db = db
self.migrate_func = {}
self.migrate_func[0] = self.migrate_from_0_to_1
self.migrate_func[1] = self.migrate_from_1_to_2
class DatabaseMigration:
def __init__(self, settings_db: SettingsDatabase, music_db: MusicDatabase):
self.settings_db = settings_db
self.music_db = music_db
self.settings_table_migrate_func = {}
self.settings_table_migrate_func[0] = self.settings_table_migrate_from_0_to_2
self.settings_table_migrate_func[1] = self.settings_table_migrate_from_0_to_2
self.music_table_migrate_func = {}
self.music_table_migrate_func[0] = self.music_table_migrate_from_0_to_1
self.music_table_migrate_func[1] = self.music_table_migrate_from_1_to_2
def migrate(self):
conn = sqlite3.connect(self.db.db_path)
self.settings_database_migrate()
self.music_database_migrate()
def settings_database_migrate(self):
conn = sqlite3.connect(self.settings_db.db_path)
cursor = conn.cursor()
if self.has_table('botamusique', conn):
current_version = 0
ver = cursor.execute("SELECT value FROM botamusique WHERE section='bot' "
"AND option='db_version'").fetchone()
if ver:
current_version = int(ver[0])
if current_version == SETTING_DB_VERSION:
conn.close()
return
else:
log.info(f"database: migrating from settings table version {current_version} to {SETTING_DB_VERSION}...")
while current_version < SETTING_DB_VERSION:
log.debug(f"database: migrate step {current_version}/{SETTING_DB_VERSION - 1}")
current_version = self.settings_table_migrate_func[current_version](conn)
log.info(f"database: migration done.")
cursor.execute("UPDATE botamusique SET value=? "
"WHERE section='bot' AND option='db_version'", (SETTING_DB_VERSION,))
else:
log.info(f"database: no settings table found. Creating settings table version {SETTING_DB_VERSION}.")
self.create_settings_table_version_2(conn)
conn.commit()
conn.close()
def music_database_migrate(self):
conn = sqlite3.connect(self.music_db.db_path)
cursor = conn.cursor()
if self.has_table('music', conn):
current_version = 0
@ -562,7 +549,7 @@ class MusicDatabaseMigration:
log.info(f"database: migrating from music table version {current_version} to {MUSIC_DB_VERSION}...")
while current_version < MUSIC_DB_VERSION:
log.debug(f"database: migrate step {current_version}/{MUSIC_DB_VERSION - 1}")
current_version = self.migrate_func[current_version](conn)
current_version = self.music_table_migrate_func[current_version](conn)
log.info(f"database: migration done.")
cursor.execute("UPDATE music SET title=? "
@ -570,7 +557,7 @@ class MusicDatabaseMigration:
else:
log.info(f"database: no music table found. Creating music table version {MUSIC_DB_VERSION}.")
self.create_table_version_2(conn)
self.create_music_table_version_2(conn)
conn.commit()
conn.close()
@ -582,7 +569,20 @@ class MusicDatabaseMigration:
return False
return True
def create_table_version_1(self, conn):
def create_settings_table_version_2(self, conn):
cursor = conn.cursor()
cursor.execute("CREATE TABLE IF NOT EXISTS botamusique ("
"section TEXT, "
"option TEXT, "
"value TEXT, "
"UNIQUE(section, option))")
cursor.execute("INSERT INTO botamusique (section, option, value) "
"VALUES (?, ?, ?)", ("bot", "db_version", 2))
conn.commit()
return 1
def create_music_table_version_1(self, conn):
cursor = conn.cursor()
cursor.execute("CREATE TABLE music ("
@ -600,15 +600,37 @@ class MusicDatabaseMigration:
conn.commit()
def create_table_version_2(self, conn):
self.create_table_version_1(conn)
def create_music_table_version_2(self, conn):
self.create_music_table_version_1(conn)
def migrate_from_0_to_1(self, conn):
def settings_table_migrate_from_0_to_2(self, conn):
cursor = conn.cursor()
cursor.execute("DROP TABLE botamusique")
conn.commit()
self.create_settings_table_version_2(conn)
# move music database into a separated file
if self.has_table('music', conn) and not os.path.exists(self.music_db.db_path):
log.info(f"database: move music db into separated file.")
cursor.execute(f"ATTACH DATABASE '{self.music_db.db_path}' AS music_db")
cursor.execute(f"SELECT sql FROM sqlite_master "
f"WHERE type='table' AND name='music'")
sql_create_table = cursor.fetchone()[0]
sql_create_table = sql_create_table.replace("music", "music_db.music")
cursor.execute(sql_create_table)
cursor.execute("INSERT INTO music_db.music SELECT * FROM music")
conn.commit()
cursor.execute("DETACH DATABASE music_db")
cursor.execute("DROP TABLE music")
return 2
def music_table_migrate_from_0_to_1(self, conn):
cursor = conn.cursor()
cursor.execute("ALTER TABLE music RENAME TO music_old")
conn.commit()
self.create_table_version_1(conn)
self.create_music_table_version_1(conn)
cursor.execute("INSERT INTO music (id, type, title, metadata, tags)"
"SELECT id, type, title, metadata, tags FROM music_old")
@ -617,7 +639,7 @@ class MusicDatabaseMigration:
return 1 # return new version number
def migrate_from_1_to_2(self, conn):
def music_table_migrate_from_1_to_2(self, conn):
items_to_update = self.db.query_music(Condition(), conn)
for item in items_to_update:
item['keywords'] = item['title']

1
media/cache.py
View File

@ -115,6 +115,7 @@ class MusicCache(dict):
self.log.debug("library: music save into database: %s" % item.format_debug_string())
self.db.insert_music(item.to_dict())
self.db.manage_special_tags()
self.dir_lock.release()

55
mumbleBot.py
View File

@ -23,7 +23,7 @@ from packaging import version
import util
import command
import constants
from database import SettingsDatabase, MusicDatabase
from database import SettingsDatabase, MusicDatabase, DatabaseMigration
import media.system
from media.item import ValidationFailedError, PreparationFailedError
from media.playlist import BasePlaylist
@ -666,7 +666,9 @@ if __name__ == '__main__':
parser.add_argument("--config", dest='config', type=str, default='configuration.ini',
help='Load configuration from this file. Default: configuration.ini')
parser.add_argument("--db", dest='db', type=str,
default=None, help='database file. Default: database.db')
default=None, help='settings database file. Default: settings-{username_of_the_bot}.db')
parser.add_argument("--music-db", dest='music_db', type=str,
default=None, help='music library database file. Default: music.db')
parser.add_argument("-q", "--quiet", dest="quiet",
action="store_true", help="Only Error logs")
@ -691,20 +693,21 @@ if __name__ == '__main__':
args = parser.parse_args()
# ======================
# Load Config
# ======================
config = configparser.ConfigParser(interpolation=None, allow_no_value=True)
var.config = config
parsed_configs = config.read([util.solve_filepath('configuration.default.ini'), util.solve_filepath(args.config)],
encoding='utf-8')
var.dbfile = args.db if args.db is not None else util.solve_filepath(
config.get("bot", "database_path", fallback="database.db"))
if len(parsed_configs) == 0:
logging.error('Could not read configuration from file \"{}\"'.format(args.config))
sys.exit()
var.config = config
var.db = SettingsDatabase(var.dbfile)
# Setup logger
# ======================
# Setup Logger
# ======================
bot_logger = logging.getLogger("bot")
formatter = logging.Formatter('[%(asctime)s %(levelname)s %(threadName)s] %(message)s', "%b %d %H:%M:%S")
@ -732,14 +735,38 @@ if __name__ == '__main__':
logging.getLogger("root").addHandler(handler)
var.bot_logger = bot_logger
# ======================
# Load Database
# ======================
if args.user:
username = args.user
else:
username = var.config.get("bot", "username")
sanitized_username = "".join([x if x.isalnum() else "_" for x in username])
var.settings_db_path = args.db if args.db is not None else util.solve_filepath(
config.get("bot", "database_path", fallback=f"settings-{sanitized_username}.db"))
var.music_db_path = args.music_db if args.music_db is not None else util.solve_filepath(
config.get("bot", "music_database_path", fallback="music.db"))
var.db = SettingsDatabase(var.settings_db_path)
if var.config.get("bot", "save_music_library", fallback=True):
var.music_db = MusicDatabase(var.dbfile)
var.music_db = MusicDatabase(var.music_db_path)
else:
var.music_db = MusicDatabase(":memory:")
DatabaseMigration(var.db, var.music_db).migrate()
var.cache = MusicCache(var.music_db)
# load playback mode
if var.config.get("bot", "refresh_cache_on_startup", fallback=True):
var.cache.build_dir_cache()
# ======================
# Load playback mode
# ======================
playback_mode = None
if var.db.has_option("playlist", "playback_mode"):
playback_mode = var.db.get('playlist', 'playback_mode')
@ -751,12 +778,12 @@ if __name__ == '__main__':
else:
raise KeyError("Unknown playback mode '%s'" % playback_mode)
# ======================
# Create bot instance
# ======================
var.bot = MumbleBot(args)
command.register_all_commands(var.bot)
if var.config.get("bot", "refresh_cache_on_startup", fallback=True):
var.cache.build_dir_cache()
# load playlist
if var.config.getboolean('bot', 'save_playlist', fallback=True):
var.bot_logger.info("bot: load playlist from previous session")

3
variables.py
View File

@ -13,7 +13,8 @@ cache: 'media.cache.MusicCache' = None
user = ""
is_proxified = False
dbfile = None
settings_db_path = None
music_db_path = None
db = None
music_db: 'database.MusicDatabase' = None
config: 'database.SettingsDatabase' = None