storm/bragi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Speed updatabase generation. Generate and use a certificate by default.

Browse Source
This commit is contained in:
Storm Dragon
2025-12-12 22:34:11 -05:00
parent 8ededf6408
commit c310a1c318
9 changed files with 246 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
media/cache.py
View File

@@ -1,10 +1,12 @@
#
#
# Bragi - A Mumble music bot
# Forked from botamusique by azlux (https://github.com/azlux/botamusque)
#
import logging
import os
import multiprocessing
from concurrent.futures import ProcessPoolExecutor, as_completed
import json
import threading
@@ -23,6 +25,29 @@ class ItemNotCachedError(Exception):
pass
def _process_file_for_cache(file_path):
"""Worker function to process a single file for the cache.
This must be a module-level function for multiprocessing to work.
Args:
file_path: Relative path to the audio file
Returns:
dict: Music item dictionary ready for database insertion, or None on error
"""
try:
# Import inside function to avoid pickling issues
import variables as var
from media.item import item_builders
item = item_builders['file'](path=file_path)
return item.to_dict()
except Exception as e:
# Log errors but don't fail the whole process
logging.getLogger("bot").warning(f"library: failed to process file {file_path}: {e}")
return None
class MusicCache(dict):
def __init__(self, db: MusicDatabase):
super().__init__()
@@ -115,27 +140,90 @@ class MusicCache(dict):
def build_dir_cache(self):
self.dir_lock.acquire()
self.log.info("library: rebuild directory cache")
files = util.get_recursive_file_list_sorted(var.music_folder)
try:
self.log.info("library: rebuild directory cache")
files_list = util.get_recursive_file_list_sorted(var.music_folder)
files_on_disk = set(files_list) # Convert to set for O(1) lookup
# remove deleted files
results = self.db.query_music(Condition().or_equal('type', 'file'))
for result in results:
if result['path'] not in files:
self.log.debug("library: music file missed: %s, delete from library." % result['path'])
self.db.delete_music(Condition().and_equal('id', result['id']))
self.log.info(f"library: found {len(files_on_disk)} audio files on disk")
# Get all existing file paths from database as a set
db_paths = set(self.db.query_all_paths())
self.log.info(f"library: found {len(db_paths)} files in database")
# Find files to delete (in DB but not on disk)
files_to_delete = db_paths - files_on_disk
if files_to_delete:
self.log.info(f"library: removing {len(files_to_delete)} deleted files from database")
for path in files_to_delete:
self.log.debug(f"library: music file missed: {path}, delete from library.")
self.db.delete_music(Condition().and_equal('path', path))
# Find new files to add (on disk but not in DB)
new_files = files_on_disk - db_paths
if not new_files:
self.log.info("library: no new files to add")
self.db.manage_special_tags()
return
self.log.info(f"library: processing {len(new_files)} new files with parallel workers")
# Determine number of worker processes from config
# 0 = auto (cpu_count - 1), N = use N workers
configured_workers = var.config.getint('bot', 'rebuild_workers', fallback=0)
if configured_workers == 0:
# Auto mode: use all cores minus one (leave one free for audio/system)
num_workers = max(1, multiprocessing.cpu_count() - 1)
self.log.info(f"library: auto-detected {multiprocessing.cpu_count()} cores, using {num_workers} workers")
else:
files.remove(result['path'])
# User specified: validate minimum of 1
num_workers = max(1, configured_workers)
if num_workers == 1:
self.log.info("library: using 1 worker (sequential processing)")
else:
self.log.info(f"library: using {num_workers} workers (configured)")
for file in files:
results = self.db.query_music(Condition().and_equal('path', file))
if not results:
item = item_builders['file'](path=file)
self.log.debug("library: music save into database: %s" % item.format_debug_string())
self.db.insert_music(item.to_dict())
self.db.manage_special_tags()
self.dir_lock.release()
# Process files in parallel
processed_items = []
with ProcessPoolExecutor(max_workers=num_workers) as executor:
# Submit all files for processing
future_to_file = {executor.submit(_process_file_for_cache, file_path): file_path
for file_path in new_files}
# Collect results as they complete
completed = 0
for future in as_completed(future_to_file):
file_path = future_to_file[future]
try:
result = future.result()
if result:
processed_items.append(result)
completed += 1
if completed % 100 == 0:
self.log.info(f"library: processed {completed}/{len(new_files)} files")
except Exception as e:
self.log.warning(f"library: failed to process {file_path}: {e}")
self.log.info(f"library: successfully processed {len(processed_items)} files")
# Batch insert all new items into database
if processed_items:
self.log.info(f"library: inserting {len(processed_items)} items into database")
import sqlite3
conn = sqlite3.connect(self.db.db_path)
try:
for item in processed_items:
self.db.insert_music(item, _conn=conn)
conn.commit()
self.log.info("library: database batch insert completed")
finally:
conn.close()
self.db.manage_special_tags()
self.log.info("library: directory cache rebuild complete")
finally:
self.dir_lock.release()
class CachedItemWrapper:

70
media/file.py
View File

@@ -1,15 +1,12 @@
#
#
# Bragi - A Mumble music bot
# Forked from botamusique by azlux (https://github.com/azlux/botamusque)
#
import os
import re
from io import BytesIO
import base64
import hashlib
import mutagen
from PIL import Image
import util
import variables as var
@@ -23,7 +20,6 @@ type : file
title
artist
duration
thumbnail
user
'''
@@ -52,7 +48,6 @@ class FileItem(BaseItem):
self.path = path
self.title = ""
self.artist = ""
self.thumbnail = None
self.id = hashlib.md5(path.encode()).hexdigest()
if os.path.exists(self.uri()):
self._get_info_from_tag()
@@ -62,7 +57,6 @@ class FileItem(BaseItem):
else:
super().__init__(from_dict)
self.artist = from_dict['artist']
self.thumbnail = from_dict['thumbnail']
try:
self.validate()
except ValidationFailedError:
@@ -95,112 +89,58 @@ class FileItem(BaseItem):
assert path is not None and file_name is not None
try:
im = None
path_thumbnail = os.path.join(path, file_name + ".jpg")
if os.path.isfile(path_thumbnail):
im = Image.open(path_thumbnail)
else:
path_thumbnail = os.path.join(path, "cover.jpg")
if os.path.isfile(path_thumbnail):
im = Image.open(path_thumbnail)
if ext == ".mp3":
# title: TIT2
# artist: TPE1, TPE2
# album: TALB
# cover artwork: APIC:
tags = mutagen.File(self.uri())
if 'TIT2' in tags:
self.title = tags['TIT2'].text[0]
if 'TPE1' in tags: # artist
self.artist = tags['TPE1'].text[0]
if im is None:
if "APIC:" in tags:
im = Image.open(BytesIO(tags["APIC:"].data))
elif ext == ".m4a" or ext == ".m4b" or ext == ".mp4" or ext == ".m4p":
# title: ©nam (\xa9nam)
# artist: ©ART
# album: ©alb
# cover artwork: covr
tags = mutagen.File(self.uri())
if '©nam' in tags:
self.title = tags['©nam'][0]
if '©ART' in tags: # artist
self.artist = tags['©ART'][0]
if im is None:
if "covr" in tags:
im = Image.open(BytesIO(tags["covr"][0]))
elif ext == ".opus":
# title: 'title'
# artist: 'artist'
# album: 'album'
# cover artwork: 'metadata_block_picture', and then:
## |
## |
## v
## Decode string as base64 binary
## |
## v
## Open that binary as a mutagen.flac.Picture
## |
## v
## Extract binary image data
tags = mutagen.File(self.uri())
if 'title' in tags:
self.title = tags['title'][0]
if 'artist' in tags:
self.artist = tags['artist'][0]
if im is None:
if 'metadata_block_picture' in tags:
pic_as_base64 = tags['metadata_block_picture'][0]
as_flac_picture = mutagen.flac.Picture(base64.b64decode(pic_as_base64))
im = Image.open(BytesIO(as_flac_picture.data))
elif ext == ".flac":
# title: 'title'
# artist: 'artist'
# album: 'album'
# cover artwork: tags.pictures
tags = mutagen.File(self.uri())
if 'title' in tags:
self.title = tags['title'][0]
if 'artist' in tags:
self.artist = tags['artist'][0]
if im is None:
for flac_picture in tags.pictures:
if flac_picture.type == 3:
im = Image.open(BytesIO(flac_picture.data))
if im:
self.thumbnail = self._prepare_thumbnail(im)
except:
pass
if not self.title:
self.title = file_name
@staticmethod
def _prepare_thumbnail(im):
im.thumbnail((100, 100), Image.LANCZOS)
buffer = BytesIO()
im = im.convert('RGB')
im.save(buffer, format="JPEG")
return base64.b64encode(buffer.getvalue()).decode('utf-8')
def to_dict(self):
dict = super().to_dict()
dict['type'] = 'file'
dict['path'] = self.path
dict['title'] = self.title
dict['artist'] = self.artist
dict['thumbnail'] = self.thumbnail
return dict
def format_debug_string(self):
@@ -217,13 +157,7 @@ class FileItem(BaseItem):
)
def format_current_playing(self, user):
display = tr("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
return tr("now_playing", item=self.format_song_string(user))
def format_title(self):
title = self.title if self.title else self.path

31
media/url.py
View File

@@ -1,4 +1,4 @@
#
#
# Bragi - A Mumble music bot
# Forked from botamusique by azlux (https://github.com/azlux/botamusque)
#
@@ -8,11 +8,8 @@ import logging
import os
import hashlib
import traceback
from PIL import Image
import yt_dlp as youtube_dl
import glob
from io import BytesIO
import base64
import util
from constants import tr_cli as tr
@@ -52,7 +49,6 @@ class URLItem(BaseItem):
self.duration = 0
self.id = hashlib.md5(url.encode()).hexdigest()
self.path = var.tmp_folder + self.id
self.thumbnail = ""
self.keywords = ""
else:
super().__init__(from_dict)
@@ -60,7 +56,6 @@ class URLItem(BaseItem):
self.duration = from_dict['duration']
self.path = from_dict['path']
self.title = from_dict['title']
self.thumbnail = from_dict['thumbnail']
self.downloading = False
self.type = "url"
@@ -194,7 +189,6 @@ class URLItem(BaseItem):
'format': 'bestaudio/best',
'outtmpl': base_path,
'noplaylist': True,
'writethumbnail': True,
'updatetime': False,
'verbose': var.config.getboolean('debug', 'youtube_dl'),
'postprocessors': [{
@@ -232,7 +226,6 @@ class URLItem(BaseItem):
self.log.info(
"bot: finished downloading url (%s) %s, saved to %s." % (self.title, self.url, self.path))
self.downloading = False
self._read_thumbnail_from_file(base_path + ".jpg")
self.version += 1 # notify wrapper to save me
return True
else:
@@ -242,18 +235,6 @@ class URLItem(BaseItem):
self.downloading = False
raise PreparationFailedError(tr('unable_download', item=self.format_title()))
def _read_thumbnail_from_file(self, path_thumbnail):
if os.path.isfile(path_thumbnail):
im = Image.open(path_thumbnail)
self.thumbnail = self._prepare_thumbnail(im)
def _prepare_thumbnail(self, im):
im.thumbnail((100, 100), Image.LANCZOS)
buffer = BytesIO()
im = im.convert('RGB')
im.save(buffer, format="JPEG")
return base64.b64encode(buffer.getvalue()).decode('utf-8')
def to_dict(self):
dict = super().to_dict()
dict['type'] = 'url'
@@ -261,7 +242,6 @@ class URLItem(BaseItem):
dict['duration'] = self.duration
dict['path'] = self.path
dict['title'] = self.title
dict['thumbnail'] = self.thumbnail
return dict
@@ -280,14 +260,7 @@ class URLItem(BaseItem):
return self.url
def format_current_playing(self, user):
display = tr("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
return tr("now_playing", item=self.format_song_string(user))
def format_title(self):
return self.title if self.title else self.url

9
media/url_from_playlist.py
View File

@@ -125,14 +125,7 @@ class PlaylistURLItem(URLItem):
user=user)
def format_current_playing(self, user):
display = tr("now_playing", item=self.format_song_string(user))
if self.thumbnail:
thumbnail_html = '<img width="80" src="data:image/jpge;base64,' + \
self.thumbnail + '"/>'
display += "<br />" + thumbnail_html
return display
return tr("now_playing", item=self.format_song_string(user))
def display_type(self):
return tr("url_from_playlist")