refactor: move music db into a separated file.
IMPORTANT NOTE:
The default database path has changed. It was database.db, but
now they ARE settings.db and music.db.
Database migration will detect music table in settings.db, and
move it out of settings.db.
To update from old version, you need to
1. if you use your own db path settings in the command
line or in configuration.ini, you don't have to change
anything;
1. or,if you use default database path, rename database.db
into settings.db.
This commit is contained in:
Terry Geng
parent
4c27bb28a1
commit
40ea744e7f
@ -1151,9 +1151,9 @@ def cmd_drop_database(bot, user, text, command, parameter):
|
||||
|
||||
if bot.is_admin(user):
|
||||
var.db.drop_table()
|
||||
var.db = SettingsDatabase(var.dbfile)
|
||||
var.db = SettingsDatabase(var.settings_db_path)
|
||||
var.music_db.drop_table()
|
||||
var.music_db = MusicDatabase(var.dbfile)
|
||||
var.music_db = MusicDatabase(var.settings_db_path)
|
||||
log.info("command: database dropped.")
|
||||
bot.send_msg(constants.strings('database_dropped'), text)
|
||||
else:
|
||||
|
||||
@ -45,7 +45,6 @@ admin = User1;User2;
|
||||
music_folder = music_folder/
|
||||
# Folder that stores the downloaded music.
|
||||
tmp_folder = /tmp/
|
||||
database_path = database.db
|
||||
pip3_path = venv/bin/pip
|
||||
auto_check_update = True
|
||||
logfile =
|
||||
|
||||
@ -24,14 +24,30 @@ port = 64738
|
||||
#username = botamusique
|
||||
#comment = Hi, I'm here to play radio, local music or youtube/soundcloud music. Have fun!
|
||||
|
||||
|
||||
# 'music_folder': Folder that stores your local songs.
|
||||
#music_folder = music_folder/
|
||||
|
||||
# 'database_path': The path of the database. The database will store things like your volume
|
||||
# set by command !volume, your playback mode and your playlist, banned URLs, etc.
|
||||
# This option will be overridden by command line arguments.
|
||||
# 'music_database_path': The path of database that stores the music library. Can be disabled by
|
||||
# setting 'save_music_library=False'
|
||||
#database_path=settings.db
|
||||
#music_database_path=music.db
|
||||
|
||||
# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';'
|
||||
#admin = User1;User2;
|
||||
|
||||
|
||||
# 'volume' is default volume from 0 to 1.
|
||||
# This option will be overridden by value in the database.
|
||||
#volume = 0.1
|
||||
|
||||
# 'playback_mode' defined the playback mode of the bot.
|
||||
# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist),
|
||||
# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library).
|
||||
# This option will be overridden by value in the database.
|
||||
# it should be one of "one-shot" (remove item once played), "repeat" (looping through the playlist),
|
||||
# or "random" (randomize the playlist), "autoplay" (randomly grab something from the music library).
|
||||
# This option will be overridden by value in the database.
|
||||
# 'autoplay_length': how many songs the autoplay mode fills the playlist
|
||||
# 'clear_when_stop_in_oneshot': clear the playlist when stop the bot in one-shot mode.
|
||||
#playback_mode = one-shot
|
||||
@ -42,17 +58,6 @@ port = 64738
|
||||
# stable will use simple bash with curl command to get releases, testing will follow github master branch with git commands
|
||||
#target_version = stable
|
||||
|
||||
# 'admin': Users allowed to kill the bot, or ban URLs. Separated by ';'
|
||||
#admin = User1;User2;
|
||||
|
||||
# 'music_folder': Folder that stores your local songs.
|
||||
#music_folder = music_folder/
|
||||
|
||||
# 'database_path': The path of the database. The database will store things like your volume
|
||||
# set by command !volume, your playback mode and your playlist, etc.
|
||||
# This option will be overridden by command line arguments.
|
||||
#database_path = database.db
|
||||
|
||||
# 'tmp_folder': Folder that stores the downloaded music.
|
||||
# 'tmp_folder_max_size': in MB, 0 for no cache, -1 for unlimited size
|
||||
# 'ignored_folders', 'ignored_files': files and folders that would be ignored during scanning.
|
||||
@ -79,11 +84,11 @@ port = 64738
|
||||
#save_music_library = True
|
||||
|
||||
# 'refresh_cache_on_startup': If this is set true, the bot will refresh its music directory cache when starting up.
|
||||
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
|
||||
# But it won't reload metadata from each files. If set to False, it will used the cache last time.
|
||||
#refresh_cache_on_startup = True
|
||||
|
||||
# 'save_playlist': If save_playlist is set True, the bot will save current playlist before quitting
|
||||
# and reload it the next time it start. It requires save_music_library to be True to function.
|
||||
# and reload it the next time it start. It requires save_music_library to be True to function.
|
||||
#save_playlist = True
|
||||
|
||||
# 'max_track_playlist': Maximum track played when a playlist is added.
|
||||
@ -93,24 +98,24 @@ port = 64738
|
||||
#max_track_duration = 60
|
||||
|
||||
# 'ducking': If ducking is enabled, the bot will automatically attenuate its
|
||||
# volume when someone is talking.
|
||||
# volume when someone is talking.
|
||||
#ducking = False
|
||||
#ducking_volume = 0.05
|
||||
#ducking_threshold = 3000
|
||||
|
||||
# 'when_nobody_in_channel': Specify what the bot should do if nobody is in the channel.
|
||||
# Possible value of this options are:
|
||||
# - "pause",
|
||||
# - "pause_resume" (pause and resume once somebody re-enters the channel)
|
||||
# - "stop" (also clears playlist)
|
||||
# - "nothing" (do nothing)
|
||||
# Possible value of this options are:
|
||||
# - "pause",
|
||||
# - "pause_resume" (pause and resume once somebody re-enters the channel)
|
||||
# - "stop" (also clears playlist)
|
||||
# - "nothing" (do nothing)
|
||||
#when_nobody_in_channel = nothing
|
||||
|
||||
# [webinterface] stores settings related to the web interface.
|
||||
[webinterface]
|
||||
# 'enable': Set 'enabled' to True if you'd like to use the web interface to manage
|
||||
# your playlist, upload files, etc.
|
||||
# The web interface is disable by default for security and performance reason.
|
||||
# your playlist, upload files, etc.
|
||||
# The web interface is disable by default for security and performance reason.
|
||||
#enabled = False
|
||||
#listening_addr = 127.0.0.1
|
||||
#listening_port = 8181
|
||||
@ -148,7 +153,7 @@ port = 64738
|
||||
|
||||
|
||||
# You may also customize commands recognized by the bot. For a full list of commands,
|
||||
# see configuration.default.ini. Copy options you want to edit into this file.
|
||||
# see configuration.default.ini. Copy options you want to edit into this file.
|
||||
#play_file = file, f
|
||||
#play_file_match = filematch, fm
|
||||
|
||||
|
||||
158
database.py
158
database.py
@ -1,3 +1,4 @@
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import json
|
||||
@ -188,62 +189,13 @@ class Condition:
|
||||
|
||||
return self
|
||||
|
||||
SETTING_DB_VERSION = 1
|
||||
SETTING_DB_VERSION = 2
|
||||
MUSIC_DB_VERSION = 2
|
||||
|
||||
class SettingsDatabase:
|
||||
def __init__(self, db_path):
|
||||
self.db_path = db_path
|
||||
|
||||
# connect
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
|
||||
self.db_version_check_and_create()
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def has_table(self, table):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
tables = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name=?;", (table,)).fetchall()
|
||||
conn.close()
|
||||
if len(tables) == 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
def db_version_check_and_create(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
|
||||
if self.has_table('botamusique'):
|
||||
# check version
|
||||
ver = self.getint("bot", "db_version", fallback=None)
|
||||
|
||||
if ver is None or ver != SETTING_DB_VERSION:
|
||||
# old_name = "botamusique_old_%s" % datetime.datetime.now().strftime("%Y%m%d")
|
||||
# cursor.execute("ALTER TABLE botamusique RENAME TO %s" % old_name)
|
||||
cursor.execute("DROP TABLE botamusique")
|
||||
conn.commit()
|
||||
self.create_table()
|
||||
else:
|
||||
self.create_table()
|
||||
|
||||
conn.close()
|
||||
|
||||
def create_table(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS botamusique ("
|
||||
"section TEXT, "
|
||||
"option TEXT, "
|
||||
"value TEXT, "
|
||||
"UNIQUE(section, option))")
|
||||
cursor.execute("INSERT INTO botamusique (section, option, value) "
|
||||
"VALUES (?, ?, ?)", ("bot", "db_version", SETTING_DB_VERSION))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def get(self, section, option, **kwargs):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
cursor = conn.cursor()
|
||||
@ -321,9 +273,6 @@ class MusicDatabase:
|
||||
def __init__(self, db_path):
|
||||
self.db_path = db_path
|
||||
|
||||
MusicDatabaseMigration(self).migrate()
|
||||
self.manage_special_tags()
|
||||
|
||||
def insert_music(self, music_dict, _conn=None):
|
||||
conn = sqlite3.connect(self.db_path) if _conn is None else _conn
|
||||
cursor = conn.cursor()
|
||||
@ -539,15 +488,53 @@ class MusicDatabase:
|
||||
conn.close()
|
||||
|
||||
|
||||
class MusicDatabaseMigration:
|
||||
def __init__(self, db: MusicDatabase):
|
||||
self.db = db
|
||||
self.migrate_func = {}
|
||||
self.migrate_func[0] = self.migrate_from_0_to_1
|
||||
self.migrate_func[1] = self.migrate_from_1_to_2
|
||||
class DatabaseMigration:
|
||||
def __init__(self, settings_db: SettingsDatabase, music_db: MusicDatabase):
|
||||
self.settings_db = settings_db
|
||||
self.music_db = music_db
|
||||
self.settings_table_migrate_func = {}
|
||||
self.settings_table_migrate_func[0] = self.settings_table_migrate_from_0_to_2
|
||||
self.settings_table_migrate_func[1] = self.settings_table_migrate_from_0_to_2
|
||||
self.music_table_migrate_func = {}
|
||||
self.music_table_migrate_func[0] = self.music_table_migrate_from_0_to_1
|
||||
self.music_table_migrate_func[1] = self.music_table_migrate_from_1_to_2
|
||||
|
||||
def migrate(self):
|
||||
conn = sqlite3.connect(self.db.db_path)
|
||||
self.settings_database_migrate()
|
||||
self.music_database_migrate()
|
||||
|
||||
def settings_database_migrate(self):
|
||||
conn = sqlite3.connect(self.settings_db.db_path)
|
||||
cursor = conn.cursor()
|
||||
if self.has_table('botamusique', conn):
|
||||
current_version = 0
|
||||
ver = cursor.execute("SELECT value FROM botamusique WHERE section='bot' "
|
||||
"AND option='db_version'").fetchone()
|
||||
if ver:
|
||||
current_version = int(ver[0])
|
||||
|
||||
if current_version == SETTING_DB_VERSION:
|
||||
conn.close()
|
||||
return
|
||||
else:
|
||||
log.info(f"database: migrating from settings table version {current_version} to {SETTING_DB_VERSION}...")
|
||||
while current_version < SETTING_DB_VERSION:
|
||||
log.debug(f"database: migrate step {current_version}/{SETTING_DB_VERSION - 1}")
|
||||
current_version = self.settings_table_migrate_func[current_version](conn)
|
||||
log.info(f"database: migration done.")
|
||||
|
||||
cursor.execute("UPDATE botamusique SET value=? "
|
||||
"WHERE section='bot' AND option='db_version'", (SETTING_DB_VERSION,))
|
||||
|
||||
else:
|
||||
log.info(f"database: no settings table found. Creating settings table version {SETTING_DB_VERSION}.")
|
||||
self.create_settings_table_version_2(conn)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
def music_database_migrate(self):
|
||||
conn = sqlite3.connect(self.music_db.db_path)
|
||||
cursor = conn.cursor()
|
||||
if self.has_table('music', conn):
|
||||
current_version = 0
|
||||
@ -562,7 +549,7 @@ class MusicDatabaseMigration:
|
||||
log.info(f"database: migrating from music table version {current_version} to {MUSIC_DB_VERSION}...")
|
||||
while current_version < MUSIC_DB_VERSION:
|
||||
log.debug(f"database: migrate step {current_version}/{MUSIC_DB_VERSION - 1}")
|
||||
current_version = self.migrate_func[current_version](conn)
|
||||
current_version = self.music_table_migrate_func[current_version](conn)
|
||||
log.info(f"database: migration done.")
|
||||
|
||||
cursor.execute("UPDATE music SET title=? "
|
||||
@ -570,7 +557,7 @@ class MusicDatabaseMigration:
|
||||
|
||||
else:
|
||||
log.info(f"database: no music table found. Creating music table version {MUSIC_DB_VERSION}.")
|
||||
self.create_table_version_2(conn)
|
||||
self.create_music_table_version_2(conn)
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
@ -582,7 +569,20 @@ class MusicDatabaseMigration:
|
||||
return False
|
||||
return True
|
||||
|
||||
def create_table_version_1(self, conn):
|
||||
def create_settings_table_version_2(self, conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("CREATE TABLE IF NOT EXISTS botamusique ("
|
||||
"section TEXT, "
|
||||
"option TEXT, "
|
||||
"value TEXT, "
|
||||
"UNIQUE(section, option))")
|
||||
cursor.execute("INSERT INTO botamusique (section, option, value) "
|
||||
"VALUES (?, ?, ?)", ("bot", "db_version", 2))
|
||||
conn.commit()
|
||||
|
||||
return 1
|
||||
|
||||
def create_music_table_version_1(self, conn):
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute("CREATE TABLE music ("
|
||||
@ -600,15 +600,37 @@ class MusicDatabaseMigration:
|
||||
|
||||
conn.commit()
|
||||
|
||||
def create_table_version_2(self, conn):
|
||||
self.create_table_version_1(conn)
|
||||
def create_music_table_version_2(self, conn):
|
||||
self.create_music_table_version_1(conn)
|
||||
|
||||
def migrate_from_0_to_1(self, conn):
|
||||
def settings_table_migrate_from_0_to_2(self, conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("DROP TABLE botamusique")
|
||||
conn.commit()
|
||||
self.create_settings_table_version_2(conn)
|
||||
|
||||
# move music database into a separated file
|
||||
if self.has_table('music', conn) and not os.path.exists(self.music_db.db_path):
|
||||
log.info(f"database: move music db into separated file.")
|
||||
cursor.execute(f"ATTACH DATABASE '{self.music_db.db_path}' AS music_db")
|
||||
cursor.execute(f"SELECT sql FROM sqlite_master "
|
||||
f"WHERE type='table' AND name='music'")
|
||||
sql_create_table = cursor.fetchone()[0]
|
||||
sql_create_table = sql_create_table.replace("music", "music_db.music")
|
||||
cursor.execute(sql_create_table)
|
||||
cursor.execute("INSERT INTO music_db.music SELECT * FROM music")
|
||||
conn.commit()
|
||||
cursor.execute("DETACH DATABASE music_db")
|
||||
|
||||
cursor.execute("DROP TABLE music")
|
||||
return 2
|
||||
|
||||
def music_table_migrate_from_0_to_1(self, conn):
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("ALTER TABLE music RENAME TO music_old")
|
||||
conn.commit()
|
||||
|
||||
self.create_table_version_1(conn)
|
||||
self.create_music_table_version_1(conn)
|
||||
|
||||
cursor.execute("INSERT INTO music (id, type, title, metadata, tags)"
|
||||
"SELECT id, type, title, metadata, tags FROM music_old")
|
||||
@ -617,7 +639,7 @@ class MusicDatabaseMigration:
|
||||
|
||||
return 1 # return new version number
|
||||
|
||||
def migrate_from_1_to_2(self, conn):
|
||||
def music_table_migrate_from_1_to_2(self, conn):
|
||||
items_to_update = self.db.query_music(Condition(), conn)
|
||||
for item in items_to_update:
|
||||
item['keywords'] = item['title']
|
||||
|
||||
@ -115,6 +115,7 @@ class MusicCache(dict):
|
||||
self.log.debug("library: music save into database: %s" % item.format_debug_string())
|
||||
self.db.insert_music(item.to_dict())
|
||||
|
||||
self.db.manage_special_tags()
|
||||
self.dir_lock.release()
|
||||
|
||||
|
||||
|
||||
50
mumbleBot.py
50
mumbleBot.py
@ -23,7 +23,7 @@ from packaging import version
|
||||
import util
|
||||
import command
|
||||
import constants
|
||||
from database import SettingsDatabase, MusicDatabase
|
||||
from database import SettingsDatabase, MusicDatabase, DatabaseMigration
|
||||
import media.system
|
||||
from media.item import ValidationFailedError, PreparationFailedError
|
||||
from media.playlist import BasePlaylist
|
||||
@ -666,7 +666,9 @@ if __name__ == '__main__':
|
||||
parser.add_argument("--config", dest='config', type=str, default='configuration.ini',
|
||||
help='Load configuration from this file. Default: configuration.ini')
|
||||
parser.add_argument("--db", dest='db', type=str,
|
||||
default=None, help='database file. Default: database.db')
|
||||
default=None, help='settings database file. Default: settings.db')
|
||||
parser.add_argument("--music-db", dest='music_db', type=str,
|
||||
default=None, help='music library database file. Default: music.db')
|
||||
|
||||
parser.add_argument("-q", "--quiet", dest="quiet",
|
||||
action="store_true", help="Only Error logs")
|
||||
@ -691,20 +693,21 @@ if __name__ == '__main__':
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# ======================
|
||||
# Load Config
|
||||
# ======================
|
||||
|
||||
config = configparser.ConfigParser(interpolation=None, allow_no_value=True)
|
||||
var.config = config
|
||||
parsed_configs = config.read([util.solve_filepath('configuration.default.ini'), util.solve_filepath(args.config)],
|
||||
encoding='utf-8')
|
||||
var.dbfile = args.db if args.db is not None else util.solve_filepath(
|
||||
config.get("bot", "database_path", fallback="database.db"))
|
||||
|
||||
if len(parsed_configs) == 0:
|
||||
logging.error('Could not read configuration from file \"{}\"'.format(args.config))
|
||||
sys.exit()
|
||||
|
||||
var.config = config
|
||||
var.db = SettingsDatabase(var.dbfile)
|
||||
|
||||
# Setup logger
|
||||
# ======================
|
||||
# Setup Logger
|
||||
# ======================
|
||||
|
||||
bot_logger = logging.getLogger("bot")
|
||||
formatter = logging.Formatter('[%(asctime)s %(levelname)s %(threadName)s] %(message)s', "%b %d %H:%M:%S")
|
||||
@ -732,14 +735,32 @@ if __name__ == '__main__':
|
||||
logging.getLogger("root").addHandler(handler)
|
||||
var.bot_logger = bot_logger
|
||||
|
||||
|
||||
# ======================
|
||||
# Load Database
|
||||
# ======================
|
||||
var.settings_db_path = args.db if args.db is not None else util.solve_filepath(
|
||||
config.get("bot", "database_path", fallback="settings.db"))
|
||||
var.music_db_path = args.music_db if args.music_db is not None else util.solve_filepath(
|
||||
config.get("bot", "music_database_path", fallback="music.db"))
|
||||
|
||||
var.db = SettingsDatabase(var.settings_db_path)
|
||||
|
||||
if var.config.get("bot", "save_music_library", fallback=True):
|
||||
var.music_db = MusicDatabase(var.dbfile)
|
||||
var.music_db = MusicDatabase(var.music_db_path)
|
||||
else:
|
||||
var.music_db = MusicDatabase(":memory:")
|
||||
|
||||
DatabaseMigration(var.db, var.music_db).migrate()
|
||||
|
||||
var.cache = MusicCache(var.music_db)
|
||||
|
||||
# load playback mode
|
||||
if var.config.get("bot", "refresh_cache_on_startup", fallback=True):
|
||||
var.cache.build_dir_cache()
|
||||
|
||||
# ======================
|
||||
# Load playback mode
|
||||
# ======================
|
||||
playback_mode = None
|
||||
if var.db.has_option("playlist", "playback_mode"):
|
||||
playback_mode = var.db.get('playlist', 'playback_mode')
|
||||
@ -751,12 +772,13 @@ if __name__ == '__main__':
|
||||
else:
|
||||
raise KeyError("Unknown playback mode '%s'" % playback_mode)
|
||||
|
||||
# ======================
|
||||
# Create bot instance
|
||||
# ======================
|
||||
|
||||
var.bot = MumbleBot(args)
|
||||
command.register_all_commands(var.bot)
|
||||
|
||||
if var.config.get("bot", "refresh_cache_on_startup", fallback=True):
|
||||
var.cache.build_dir_cache()
|
||||
|
||||
# load playlist
|
||||
if var.config.getboolean('bot', 'save_playlist', fallback=True):
|
||||
var.bot_logger.info("bot: load playlist from previous session")
|
||||
|
||||
@ -13,7 +13,8 @@ cache: 'media.cache.MusicCache' = None
|
||||
user = ""
|
||||
is_proxified = False
|
||||
|
||||
dbfile = None
|
||||
settings_db_path = None
|
||||
music_db_path = None
|
||||
db = None
|
||||
music_db: 'database.MusicDatabase' = None
|
||||
config: 'database.SettingsDatabase' = None
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user