diff --git a/command.py b/command.py index d9a390e..cbdfd25 100644 --- a/command.py +++ b/command.py @@ -1151,9 +1151,9 @@ def cmd_drop_database(bot, user, text, command, parameter): if bot.is_admin(user): var.db.drop_table() - var.db = SettingsDatabase(var.dbfile) + var.db = SettingsDatabase(var.settings_db_path) var.music_db.drop_table() - var.music_db = MusicDatabase(var.dbfile) + var.music_db = MusicDatabase(var.settings_db_path) log.info("command: database dropped.") bot.send_msg(constants.strings('database_dropped'), text) else: diff --git a/configuration.default.ini b/configuration.default.ini index ff15036..75c299d 100644 --- a/configuration.default.ini +++ b/configuration.default.ini @@ -45,7 +45,6 @@ admin = User1;User2; music_folder = music_folder/ # Folder that stores the downloaded music. tmp_folder = /tmp/ -database_path = database.db pip3_path = venv/bin/pip auto_check_update = True logfile = diff --git a/configuration.example.ini b/configuration.example.ini index 64af744..1a16509 100644 --- a/configuration.example.ini +++ b/configuration.example.ini @@ -24,14 +24,30 @@ port = 64738 #username = botamusique #comment = Hi, I'm here to play radio, local music or youtube/soundcloud music. Have fun! + +# 'music_folder': Folder that stores your local songs. +#music_folder = music_folder/ + +# 'database_path': The path of the database. The database will store things like your volume +# set by command !volume, your playback mode and your playlist, banned URLs, etc. +# This option will be overridden by command line arguments. +# 'music_database_path': The path of database that stores the music library. Can be disabled by +# setting 'save_music_library=False' +#database_path=settings.db +#music_database_path=music.db + +# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';' +#admin = User1;User2; + + # 'volume' is default volume from 0 to 1. # This option will be overridden by value in the database. #volume = 0.1 # 'playback_mode' defined the playback mode of the bot. -# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist), -# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library). -# This option will be overridden by value in the database. +# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist), +# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library). +# This option will be overridden by value in the database. # 'autoplay_length': how many songs the autoplay mode fills the playlist # 'clear_when_stop_in_oneshot': clear the playlist when stop the bot in one-shot mode. #playback_mode = one-shot @@ -42,17 +58,6 @@ port = 64738 # stable will use simple bash with curl command to get releases, testing will follow github master branch with git commands #target_version = stable -# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';' -#admin = User1;User2; - -# 'music_folder': Folder that stores your local songs. -#music_folder = music_folder/ - -# 'database_path': The path of the database. The database will store things like your volume -# set by command !volume, your playback mode and your playlist, etc. -# This option will be overridden by command line arguments. -#database_path = database.db - # 'tmp_folder': Folder that stores the downloaded music. # 'tmp_folder_max_size': in MB, 0 for no cache, -1 for unlimited size # 'ignored_folders', 'ignored_files': files and folders that would be ignored during scanning. @@ -79,11 +84,11 @@ port = 64738 #save_music_library = True # 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up. -# But it won't reload metadata from each files. If set to False, it will used the cache last time. +# But it won't reload metadata from each files. If set to False, it will used the cache last time. #refresh_cache_on_startup = True # 'save_playlist': If save_playlist is set True, the bot will save current playlist before quitting -# and reload it the next time it start. It requires save_music_library to be True to function. +# and reload it the next time it start. It requires save_music_library to be True to function. #save_playlist = True # 'max_track_playlist': Maximum track played when a playlist is added. @@ -93,24 +98,24 @@ port = 64738 #max_track_duration = 60 # 'ducking': If ducking is enabled, the bot will automatically attenuate its -# volume when someone is talking. +# volume when someone is talking. #ducking = False #ducking_volume = 0.05 #ducking_threshold = 3000 # 'when_nobody_in_channel': Specify what the bot should do if nobody is in the channel. -# Possible value of this options are: -# - "pause", -# - "pause_resume" (pause and resume once somebody re-enters the channel) -# - "stop" (also clears playlist) -# - "nothing" (do nothing) +# Possible value of this options are: +# - "pause", +# - "pause_resume" (pause and resume once somebody re-enters the channel) +# - "stop" (also clears playlist) +# - "nothing" (do nothing) #when_nobody_in_channel = nothing # [webinterface] stores settings related to the web interface. [webinterface] # 'enable': Set 'enabled' to True if you'd like to use the web interface to manage -# your playlist, upload files, etc. -# The web interface is disable by default for security and performance reason. +# your playlist, upload files, etc. +# The web interface is disable by default for security and performance reason. #enabled = False #listening_addr = 127.0.0.1 #listening_port = 8181 @@ -148,7 +153,7 @@ port = 64738 # You may also customize commands recognized by the bot. For a full list of commands, -# see configuration.default.ini. Copy options you want to edit into this file. +# see configuration.default.ini. Copy options you want to edit into this file. #play_file = file, f #play_file_match = filematch, fm diff --git a/database.py b/database.py index db7d5fc..bd20af1 100644 --- a/database.py +++ b/database.py @@ -1,3 +1,4 @@ +import os import re import sqlite3 import json @@ -188,62 +189,13 @@ class Condition: return self -SETTING_DB_VERSION = 1 +SETTING_DB_VERSION = 2 MUSIC_DB_VERSION = 2 class SettingsDatabase: def __init__(self, db_path): self.db_path = db_path - # connect - conn = sqlite3.connect(self.db_path) - - self.db_version_check_and_create() - - conn.commit() - conn.close() - - def has_table(self, table): - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - tables = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?;", (table,)).fetchall() - conn.close() - if len(tables) == 0: - return False - return True - - def db_version_check_and_create(self): - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - - if self.has_table('botamusique'): - # check version - ver = self.getint("bot", "db_version", fallback=None) - - if ver is None or ver != SETTING_DB_VERSION: - # old_name = "botamusique_old_%s" % datetime.datetime.now().strftime("%Y%m%d") - # cursor.execute("ALTER TABLE botamusique RENAME TO %s" % old_name) - cursor.execute("DROP TABLE botamusique") - conn.commit() - self.create_table() - else: - self.create_table() - - conn.close() - - def create_table(self): - conn = sqlite3.connect(self.db_path) - cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS botamusique (" - "section TEXT, " - "option TEXT, " - "value TEXT, " - "UNIQUE(section, option))") - cursor.execute("INSERT INTO botamusique (section, option, value) " - "VALUES (?, ?, ?)", ("bot", "db_version", SETTING_DB_VERSION)) - conn.commit() - conn.close() - def get(self, section, option, **kwargs): conn = sqlite3.connect(self.db_path) cursor = conn.cursor() @@ -321,9 +273,6 @@ class MusicDatabase: def __init__(self, db_path): self.db_path = db_path - MusicDatabaseMigration(self).migrate() - self.manage_special_tags() - def insert_music(self, music_dict, _conn=None): conn = sqlite3.connect(self.db_path) if _conn is None else _conn cursor = conn.cursor() @@ -539,15 +488,53 @@ class MusicDatabase: conn.close() -class MusicDatabaseMigration: - def __init__(self, db: MusicDatabase): - self.db = db - self.migrate_func = {} - self.migrate_func[0] = self.migrate_from_0_to_1 - self.migrate_func[1] = self.migrate_from_1_to_2 +class DatabaseMigration: + def __init__(self, settings_db: SettingsDatabase, music_db: MusicDatabase): + self.settings_db = settings_db + self.music_db = music_db + self.settings_table_migrate_func = {} + self.settings_table_migrate_func[0] = self.settings_table_migrate_from_0_to_2 + self.settings_table_migrate_func[1] = self.settings_table_migrate_from_0_to_2 + self.music_table_migrate_func = {} + self.music_table_migrate_func[0] = self.music_table_migrate_from_0_to_1 + self.music_table_migrate_func[1] = self.music_table_migrate_from_1_to_2 def migrate(self): - conn = sqlite3.connect(self.db.db_path) + self.settings_database_migrate() + self.music_database_migrate() + + def settings_database_migrate(self): + conn = sqlite3.connect(self.settings_db.db_path) + cursor = conn.cursor() + if self.has_table('botamusique', conn): + current_version = 0 + ver = cursor.execute("SELECT value FROM botamusique WHERE section='bot' " + "AND option='db_version'").fetchone() + if ver: + current_version = int(ver[0]) + + if current_version == SETTING_DB_VERSION: + conn.close() + return + else: + log.info(f"database: migrating from settings table version {current_version} to {SETTING_DB_VERSION}...") + while current_version < SETTING_DB_VERSION: + log.debug(f"database: migrate step {current_version}/{SETTING_DB_VERSION - 1}") + current_version = self.settings_table_migrate_func[current_version](conn) + log.info(f"database: migration done.") + + cursor.execute("UPDATE botamusique SET value=? " + "WHERE section='bot' AND option='db_version'", (SETTING_DB_VERSION,)) + + else: + log.info(f"database: no settings table found. Creating settings table version {SETTING_DB_VERSION}.") + self.create_settings_table_version_2(conn) + + conn.commit() + conn.close() + + def music_database_migrate(self): + conn = sqlite3.connect(self.music_db.db_path) cursor = conn.cursor() if self.has_table('music', conn): current_version = 0 @@ -562,7 +549,7 @@ class MusicDatabaseMigration: log.info(f"database: migrating from music table version {current_version} to {MUSIC_DB_VERSION}...") while current_version < MUSIC_DB_VERSION: log.debug(f"database: migrate step {current_version}/{MUSIC_DB_VERSION - 1}") - current_version = self.migrate_func[current_version](conn) + current_version = self.music_table_migrate_func[current_version](conn) log.info(f"database: migration done.") cursor.execute("UPDATE music SET title=? " @@ -570,7 +557,7 @@ class MusicDatabaseMigration: else: log.info(f"database: no music table found. Creating music table version {MUSIC_DB_VERSION}.") - self.create_table_version_2(conn) + self.create_music_table_version_2(conn) conn.commit() conn.close() @@ -582,7 +569,20 @@ class MusicDatabaseMigration: return False return True - def create_table_version_1(self, conn): + def create_settings_table_version_2(self, conn): + cursor = conn.cursor() + cursor.execute("CREATE TABLE IF NOT EXISTS botamusique (" + "section TEXT, " + "option TEXT, " + "value TEXT, " + "UNIQUE(section, option))") + cursor.execute("INSERT INTO botamusique (section, option, value) " + "VALUES (?, ?, ?)", ("bot", "db_version", 2)) + conn.commit() + + return 1 + + def create_music_table_version_1(self, conn): cursor = conn.cursor() cursor.execute("CREATE TABLE music (" @@ -600,15 +600,37 @@ class MusicDatabaseMigration: conn.commit() - def create_table_version_2(self, conn): - self.create_table_version_1(conn) + def create_music_table_version_2(self, conn): + self.create_music_table_version_1(conn) - def migrate_from_0_to_1(self, conn): + def settings_table_migrate_from_0_to_2(self, conn): + cursor = conn.cursor() + cursor.execute("DROP TABLE botamusique") + conn.commit() + self.create_settings_table_version_2(conn) + + # move music database into a separated file + if self.has_table('music', conn) and not os.path.exists(self.music_db.db_path): + log.info(f"database: move music db into separated file.") + cursor.execute(f"ATTACH DATABASE '{self.music_db.db_path}' AS music_db") + cursor.execute(f"SELECT sql FROM sqlite_master " + f"WHERE type='table' AND name='music'") + sql_create_table = cursor.fetchone()[0] + sql_create_table = sql_create_table.replace("music", "music_db.music") + cursor.execute(sql_create_table) + cursor.execute("INSERT INTO music_db.music SELECT * FROM music") + conn.commit() + cursor.execute("DETACH DATABASE music_db") + + cursor.execute("DROP TABLE music") + return 2 + + def music_table_migrate_from_0_to_1(self, conn): cursor = conn.cursor() cursor.execute("ALTER TABLE music RENAME TO music_old") conn.commit() - self.create_table_version_1(conn) + self.create_music_table_version_1(conn) cursor.execute("INSERT INTO music (id, type, title, metadata, tags)" "SELECT id, type, title, metadata, tags FROM music_old") @@ -617,7 +639,7 @@ class MusicDatabaseMigration: return 1 # return new version number - def migrate_from_1_to_2(self, conn): + def music_table_migrate_from_1_to_2(self, conn): items_to_update = self.db.query_music(Condition(), conn) for item in items_to_update: item['keywords'] = item['title'] diff --git a/media/cache.py b/media/cache.py index 68551a5..518dd5b 100644 --- a/media/cache.py +++ b/media/cache.py @@ -115,6 +115,7 @@ class MusicCache(dict): self.log.debug("library: music save into database: %s" % item.format_debug_string()) self.db.insert_music(item.to_dict()) + self.db.manage_special_tags() self.dir_lock.release() diff --git a/mumbleBot.py b/mumbleBot.py index 884d785..9109b8f 100644 --- a/mumbleBot.py +++ b/mumbleBot.py @@ -23,7 +23,7 @@ from packaging import version import util import command import constants -from database import SettingsDatabase, MusicDatabase +from database import SettingsDatabase, MusicDatabase, DatabaseMigration import media.system from media.item import ValidationFailedError, PreparationFailedError from media.playlist import BasePlaylist @@ -666,7 +666,9 @@ if __name__ == '__main__': parser.add_argument("--config", dest='config', type=str, default='configuration.ini', help='Load configuration from this file. Default: configuration.ini') parser.add_argument("--db", dest='db', type=str, - default=None, help='database file. Default: database.db') + default=None, help='settings database file. Default: settings.db') + parser.add_argument("--music-db", dest='music_db', type=str, + default=None, help='music library database file. Default: music.db') parser.add_argument("-q", "--quiet", dest="quiet", action="store_true", help="Only Error logs") @@ -691,20 +693,21 @@ if __name__ == '__main__': args = parser.parse_args() + # ====================== + # Load Config + # ====================== + config = configparser.ConfigParser(interpolation=None, allow_no_value=True) + var.config = config parsed_configs = config.read([util.solve_filepath('configuration.default.ini'), util.solve_filepath(args.config)], encoding='utf-8') - var.dbfile = args.db if args.db is not None else util.solve_filepath( - config.get("bot", "database_path", fallback="database.db")) - if len(parsed_configs) == 0: logging.error('Could not read configuration from file \"{}\"'.format(args.config)) sys.exit() - var.config = config - var.db = SettingsDatabase(var.dbfile) - - # Setup logger + # ====================== + # Setup Logger + # ====================== bot_logger = logging.getLogger("bot") formatter = logging.Formatter('[%(asctime)s %(levelname)s %(threadName)s] %(message)s', "%b %d %H:%M:%S") @@ -732,14 +735,32 @@ if __name__ == '__main__': logging.getLogger("root").addHandler(handler) var.bot_logger = bot_logger + + # ====================== + # Load Database + # ====================== + var.settings_db_path = args.db if args.db is not None else util.solve_filepath( + config.get("bot", "database_path", fallback="settings.db")) + var.music_db_path = args.music_db if args.music_db is not None else util.solve_filepath( + config.get("bot", "music_database_path", fallback="music.db")) + + var.db = SettingsDatabase(var.settings_db_path) + if var.config.get("bot", "save_music_library", fallback=True): - var.music_db = MusicDatabase(var.dbfile) + var.music_db = MusicDatabase(var.music_db_path) else: var.music_db = MusicDatabase(":memory:") + DatabaseMigration(var.db, var.music_db).migrate() + var.cache = MusicCache(var.music_db) - # load playback mode + if var.config.get("bot", "refresh_cache_on_startup", fallback=True): + var.cache.build_dir_cache() + + # ====================== + # Load playback mode + # ====================== playback_mode = None if var.db.has_option("playlist", "playback_mode"): playback_mode = var.db.get('playlist', 'playback_mode') @@ -751,12 +772,13 @@ if __name__ == '__main__': else: raise KeyError("Unknown playback mode '%s'" % playback_mode) + # ====================== + # Create bot instance + # ====================== + var.bot = MumbleBot(args) command.register_all_commands(var.bot) - if var.config.get("bot", "refresh_cache_on_startup", fallback=True): - var.cache.build_dir_cache() - # load playlist if var.config.getboolean('bot', 'save_playlist', fallback=True): var.bot_logger.info("bot: load playlist from previous session") diff --git a/variables.py b/variables.py index 68b928d..271414c 100644 --- a/variables.py +++ b/variables.py @@ -13,7 +13,8 @@ cache: 'media.cache.MusicCache' = None user = "" is_proxified = False -dbfile = None +settings_db_path = None +music_db_path = None db = None music_db: 'database.MusicDatabase' = None config: 'database.SettingsDatabase' = None