storm/minidlna
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scanner: Add non-destructive update rescan

Browse Source 
New functionality, based on Shrimpkin's patch #145 on SF.
This commit is contained in:
Justin Maggard 2017-05-04 23:29:59 -07:00
parent 5450ac486e
commit 49aa42d893
8 changed files with 177 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile.am
View File

@ -25,7 +25,7 @@ minidlnad_SOURCES = minidlna.c upnphttp.c upnpdescgen.c upnpsoap.c \
upnpreplyparse.c minixml.c clients.c \
getifaddr.c process.c upnpglobalvars.c \
options.c minissdp.c uuid.c upnpevents.c \
sql.c utils.c metadata.c scanner.c inotify.c \
sql.c utils.c metadata.c scanner.c monitor.c \
tivo_utils.c tivo_beacon.c tivo_commands.c \
playlist.c image_utils.c albumart.c log.c \
containers.c avahi.c tagutils/tagutils.c

4
inotify.h
View File

@ -1,4 +0,0 @@
#ifdef HAVE_INOTIFY
void *
start_inotify();
#endif

20
minidlna.c
View File

@ -90,7 +90,7 @@
#include "process.h"
#include "upnpevents.h"
#include "scanner.h"
#include "inotify.h"
#include "monitor.h"
#include "log.h"
#include "tivo_beacon.h"
#include "tivo_utils.h"
@ -328,12 +328,13 @@ check_db(sqlite3 *db, int new_db, pid_t *scanner_pid)
if (ret != 0)
{
rescan:
rescan_db = 0;
if (ret < 0)
DPRINTF(E_WARN, L_GENERAL, "Creating new database at %s/files.db\n", db_path);
else if (ret == 1)
DPRINTF(E_WARN, L_GENERAL, "New media_dir detected; rescanning...\n");
DPRINTF(E_WARN, L_GENERAL, "New media_dir detected; rebuilding...\n");
else if (ret == 2)
DPRINTF(E_WARN, L_GENERAL, "Removed media_dir detected; rescanning...\n");
DPRINTF(E_WARN, L_GENERAL, "Removed media_dir detected; rebuilding...\n");
else
DPRINTF(E_WARN, L_GENERAL, "Database version mismatch (%d=>%d); need to recreate...\n",
ret, DB_VERSION);
@ -346,6 +347,9 @@ rescan:
open_db(&db);
if (CreateDatabase() != 0)
DPRINTF(E_FATAL, L_GENERAL, "ERROR: Failed to create sqlite database! Exiting...\n");
}
if (ret || rescan_db)
{
#if USE_FORK
scanning = 1;
sqlite3_close(db);
@ -834,6 +838,9 @@ init(int argc, char **argv)
case 'h':
runtime_vars.port = -1; // triggers help display
break;
case 'r':
rescan_db = 1;
break;
case 'R':
snprintf(buf, sizeof(buf), "rm -rf %s/files.db %s/art_cache", db_path, db_path);
if (system(buf) != 0)
@ -880,9 +887,9 @@ init(int argc, char **argv)
"\t\t[-t notify_interval] [-P pid_filename]\n"
"\t\t[-s serial] [-m model_number]\n"
#ifdef __linux__
"\t\t[-w url] [-R] [-L] [-S] [-V] [-h]\n"
"\t\t[-w url] [-r] [-R] [-L] [-S] [-V] [-h]\n"
#else
"\t\t[-w url] [-R] [-L] [-V] [-h]\n"
"\t\t[-w url] [-r] [-R] [-L] [-V] [-h]\n"
#endif
"\nNotes:\n\tNotify interval is in seconds. Default is 895 seconds.\n"
"\tDefault pid file is %s.\n"
@ -890,7 +897,8 @@ init(int argc, char **argv)
"\t-w sets the presentation url. Default is http address on port 80\n"
"\t-v enables verbose output\n"
"\t-h displays this text\n"
"\t-R forces a full rescan\n"
"\t-r forces a rescan\n"
"\t-R forces a rebuild\n"
"\t-L do not create playlists\n"
#ifdef __linux__
"\t-S changes behaviour for systemd\n"

124
inotify.c → monitor.c
View File

@ -17,7 +17,6 @@
*/
#include "config.h"
#ifdef HAVE_INOTIFY
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
@ -30,6 +29,7 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/time.h>
#ifdef HAVE_INOTIFY
#include <sys/resource.h>
#include <poll.h>
#ifdef HAVE_SYS_INOTIFY_H
@ -38,10 +38,11 @@
#include "linux/inotify.h"
#include "linux/inotify-syscalls.h"
#endif
#endif
#include "libav.h"
#include "upnpglobalvars.h"
#include "inotify.h"
#include "monitor.h"
#include "utils.h"
#include "sql.h"
#include "scanner.h"
@ -50,6 +51,9 @@
#include "playlist.h"
#include "log.h"
static time_t next_pl_fill = 0;
#ifdef HAVE_INOTIFY
#define EVENT_SIZE ( sizeof (struct inotify_event) )
#define BUF_LEN ( 1024 * ( EVENT_SIZE + 16 ) )
#define DESIRED_WATCH_LIMIT 65536
@ -65,7 +69,6 @@ struct watch
static struct watch *watches;
static struct watch *lastwatch = NULL;
static time_t next_pl_fill = 0;
static char *
get_path_from_wd(int wd)
@ -244,9 +247,10 @@ inotify_remove_watches(int fd)
return rm_watches;
}
#endif
static int
inotify_remove_file(const char * path)
int
monitor_remove_file(const char * path)
{
char sql[128];
char art_cache[PATH_MAX];
@ -323,8 +327,8 @@ inotify_remove_file(const char * path)
return 0;
}
static int
inotify_insert_file(char * name, const char * path)
int
monitor_insert_file(char * name, const char * path)
{
int len;
char * last_dir;
@ -406,14 +410,24 @@ inotify_insert_file(char * name, const char * path)
if( !ts && is_playlist(path) && (sql_get_int_field(db, "SELECT ID from PLAYLISTS where PATH = '%q'", path) > 0) )
{
DPRINTF(E_DEBUG, L_INOTIFY, "Re-reading modified playlist (%s).\n", path);
inotify_remove_file(path);
monitor_remove_file(path);
next_pl_fill = 1;
}
else if( ts < st.st_mtime )
else if( !ts )
{
if( ts > 0 )
DPRINTF(E_DEBUG, L_INOTIFY, "%s is newer than the last db entry.\n", path);
inotify_remove_file(path);
DPRINTF(E_DEBUG, L_INOTIFY, "Adding: %s\n", path);
}
else if( ts != st.st_mtime )
{
DPRINTF(E_DEBUG, L_INOTIFY, "%s is %s than the last db entry.\n",
path, (ts < st.st_mtime) ? "older" : "newer");
monitor_remove_file(path);
}
else
{
if( ts == st.st_mtime )
DPRINTF(E_DEBUG, L_INOTIFY, "%s already exists\n", path);
return 0;
}
/* Find the parentID. If it's not found, create all necessary parents. */
@ -478,14 +492,13 @@ inotify_insert_file(char * name, const char * path)
return depth;
}
static int
inotify_insert_directory(int fd, char *name, const char * path)
int
monitor_insert_directory(int fd, char *name, const char * path)
{
DIR * ds;
struct dirent * e;
char *id, *parent_buf, *esc_name;
char path_buf[PATH_MAX];
int wd;
enum file_types type = TYPE_UNKNOWN;
media_types dir_types = ALL_MEDIA;
struct media_dir_s* media_path;
@ -498,27 +511,34 @@ inotify_insert_directory(int fd, char *name, const char * path)
}
if( sql_get_int_field(db, "SELECT ID from DETAILS where PATH = '%q'", path) > 0 )
{
fd = 0;
DPRINTF(E_DEBUG, L_INOTIFY, "%s already exists\n", path);
return 0;
}
parent_buf = strdup(path);
id = sql_get_text_field(db, "SELECT OBJECT_ID from OBJECTS o left join DETAILS d on (d.ID = o.DETAIL_ID)"
" where d.PATH = '%q' and REF_ID is NULL", dirname(parent_buf));
if( !id )
id = sqlite3_mprintf("%s", BROWSEDIR_ID);
insert_directory(name, path, BROWSEDIR_ID, id+2, get_next_available_id("OBJECTS", id));
sqlite3_free(id);
free(parent_buf);
wd = add_watch(fd, path);
if( wd == -1 )
{
DPRINTF(E_ERROR, L_INOTIFY, "add_watch() failed\n");
}
else
{
DPRINTF(E_INFO, L_INOTIFY, "Added watch to %s [%d]\n", path, wd);
parent_buf = strdup(path);
id = sql_get_text_field(db, "SELECT OBJECT_ID from OBJECTS o left join DETAILS d on (d.ID = o.DETAIL_ID)"
" WHERE d.PATH = '%q' and REF_ID is NULL", dirname(parent_buf));
if( !id )
id = sqlite3_mprintf("%s", BROWSEDIR_ID);
insert_directory(name, path, BROWSEDIR_ID, id+2, get_next_available_id("OBJECTS", id));
sqlite3_free(id);
free(parent_buf);
}
if( fd > 0 )
{
#ifdef HAVE_INOTIFY
int wd = add_watch(fd, path);
if( wd == -1 )
{
DPRINTF(E_ERROR, L_INOTIFY, "add_watch() failed\n");
}
else
{
DPRINTF(E_INFO, L_INOTIFY, "Added watch to %s [%d]\n", path, wd);
}
#endif
}
media_path = media_dirs;
@ -556,13 +576,13 @@ inotify_insert_directory(int fd, char *name, const char * path)
}
if( type == TYPE_DIR )
{
inotify_insert_directory(fd, esc_name, path_buf);
monitor_insert_directory(fd, esc_name, path_buf);
}
else if( type == TYPE_FILE )
{
if( (stat(path_buf, &st) == 0) && (st.st_blocks<<9 >= st.st_size) )
{
inotify_insert_file(esc_name, path_buf);
monitor_insert_file(esc_name, path_buf);
}
}
free(esc_name);
@ -572,8 +592,8 @@ inotify_insert_directory(int fd, char *name, const char * path)
return 0;
}
static int
inotify_remove_directory(int fd, const char * path)
int
monitor_remove_directory(int fd, const char * path)
{
char * sql;
char **result;
@ -582,7 +602,12 @@ inotify_remove_directory(int fd, const char * path)
/* Invalidate the scanner cache so we don't insert files into non-existent containers */
valid_cache = 0;
remove_watch(fd, path);
if( fd > 0 )
{
#ifdef HAVE_INOTIFY
remove_watch(fd, path);
#endif
}
sql = sqlite3_mprintf("SELECT ID from DETAILS where (PATH > '%q/' and PATH <= '%q/%c')"
" or PATH = '%q'", path, path, 0xFF, path);
if( (sql_get_table(db, sql, &result, &rows, NULL) == SQLITE_OK) )
@ -606,6 +631,7 @@ inotify_remove_directory(int fd, const char * path)
return ret;
}
#ifdef HAVE_INOTIFY
void *
start_inotify(void)
{
@ -620,7 +646,7 @@ start_inotify(void)
sigfillset(&set);
pthread_sigmask(SIG_BLOCK, &set, NULL);
pollfds[0].fd = inotify_init();
pollfds[0].events = POLLIN;
@ -638,10 +664,10 @@ start_inotify(void)
DPRINTF(E_WARN, L_INOTIFY, "Failed to reduce inotify thread priority\n");
sqlite3_release_memory(1<<31);
av_register_all();
while( !quitting )
{
length = poll(pollfds, 1, timeout);
length = poll(pollfds, 1, timeout);
if( !length )
{
if( next_pl_fill && (time(NULL) >= next_pl_fill) )
@ -653,9 +679,9 @@ start_inotify(void)
}
else if( length < 0 )
{
if( (errno == EINTR) || (errno == EAGAIN) )
continue;
else
if( (errno == EINTR) || (errno == EAGAIN) )
continue;
else
DPRINTF(E_ERROR, L_INOTIFY, "read failed!\n");
}
else
@ -681,7 +707,7 @@ start_inotify(void)
{
DPRINTF(E_DEBUG, L_INOTIFY, "The directory %s was %s.\n",
path_buf, (event->mask & IN_MOVED_TO ? "moved here" : "created"));
inotify_insert_directory(pollfds[0].fd, esc_name, path_buf);
monitor_insert_directory(pollfds[0].fd, esc_name, path_buf);
}
else if ( (event->mask & (IN_CLOSE_WRITE|IN_MOVED_TO|IN_CREATE)) &&
(lstat(path_buf, &st) == 0) )
@ -692,9 +718,9 @@ start_inotify(void)
(S_ISLNK(st.st_mode) ? "symbolic" : "hard"),
path_buf, (event->mask & IN_MOVED_TO ? "moved here" : "created"));
if( stat(path_buf, &st) == 0 && S_ISDIR(st.st_mode) )
inotify_insert_directory(pollfds[0].fd, esc_name, path_buf);
monitor_insert_directory(pollfds[0].fd, esc_name, path_buf);
else
inotify_insert_file(esc_name, path_buf);
monitor_insert_file(esc_name, path_buf);
}
else if( event->mask & (IN_CLOSE_WRITE|IN_MOVED_TO) && st.st_size > 0 )
{
@ -703,7 +729,7 @@ start_inotify(void)
{
DPRINTF(E_DEBUG, L_INOTIFY, "The file %s was %s.\n",
path_buf, (event->mask & IN_MOVED_TO ? "moved here" : "changed"));
inotify_insert_file(esc_name, path_buf);
monitor_insert_file(esc_name, path_buf);
}
}
}
@ -713,9 +739,9 @@ start_inotify(void)
(event->mask & IN_ISDIR ? "directory" : "file"),
path_buf, (event->mask & IN_MOVED_FROM ? "moved away" : "deleted"));
if ( event->mask & IN_ISDIR )
inotify_remove_directory(pollfds[0].fd, path_buf);
monitor_remove_directory(pollfds[0].fd, path_buf);
else
inotify_remove_file(path_buf);
monitor_remove_file(path_buf);
}
free(esc_name);
}

16
monitor.h Normal file
View File

@ -0,0 +1,16 @@
int
monitor_insert_file(char * name, const char * path);
int
monitor_insert_directory(int fd, char *name, const char * path);
int
monitor_remove_file(const char * path);
int
monitor_remove_directory(int fd, const char * path);
#ifdef HAVE_INOTIFY
void *
start_inotify();
#endif

70
scanner.c
View File

@ -46,6 +46,7 @@
#include "albumart.h"
#include "containers.h"
#include "log.h"
#include "monitor.h"
#if SCANDIR_CONST
typedef const struct dirent scan_filter;
@ -837,6 +838,68 @@ _notify_stop(void)
#endif
}
/* rescan functions added by shrimpkin@sourceforge.net */
static int
cb_orphans(void *args, int argc, char **argv, char **azColName)
{
const char *path = argv[0];
const char *mime = argv[1];
/* If we can't access the path, remove it */
if (access(path, R_OK) != 0)
{
DPRINTF(E_DEBUG, L_SCANNER, "Removing %s [%s]\n", path, mime ? "file" : "dir");
if (mime)
monitor_remove_file(path);
else
monitor_remove_directory(0, path);
}
return 0;
}
void
start_rescan(void)
{
struct media_dir_s *media_path;
char *esc_name = NULL;
char *zErrMsg;
const char *sql_files = "SELECT path, mime FROM details WHERE path NOT NULL AND mime IS NOT NULL;";
const char *sql_dir = "SELECT path, mime FROM details WHERE path NOT NULL AND mime IS NULL;";
int ret;
DPRINTF(E_INFO, L_SCANNER, "Starting rescan\n");
/* Find and remove any dead directory links */
ret = sqlite3_exec(db, sql_dir, cb_orphans, NULL, &zErrMsg);
if (ret != SQLITE_OK)
{
DPRINTF(E_MAXDEBUG, L_SCANNER, "SQL error: %s\nBAD SQL: %s\n", zErrMsg, sql_dir);
sqlite3_free(zErrMsg);
}
/* Find and remove any dead file links */
ret = sqlite3_exec(db, sql_files, cb_orphans, NULL, &zErrMsg);
if (ret != SQLITE_OK)
{
DPRINTF(E_MAXDEBUG, L_SCANNER, "SQL error: %s\nBAD SQL: %s\n", zErrMsg, sql_files);
sqlite3_free(zErrMsg);
}
/* Rescan media_paths for new and/or modified files */
for (media_path = media_dirs; media_path != NULL; media_path = media_path->next)
{
char path[MAXPATHLEN], buf[MAXPATHLEN];
strncpyt(path, media_path->path, sizeof(path));
strncpyt(buf, media_path->path, sizeof(buf));
esc_name = escape_tag(basename(buf), 1);
monitor_insert_directory(0, esc_name, path);
free(esc_name);
}
DPRINTF(E_INFO, L_SCANNER, "Rescan completed\n");
}
/* end rescan functions */
void
start_scanner()
{
@ -845,12 +908,17 @@ start_scanner()
if (setpriority(PRIO_PROCESS, 0, 15) == -1)
DPRINTF(E_WARN, L_INOTIFY, "Failed to reduce scanner thread priority\n");
_notify_start();
setlocale(LC_COLLATE, "");
av_register_all();
av_log_set_level(AV_LOG_PANIC);
if( rescan_db )
{
start_rescan();
return;
}
_notify_start();
for( media_path = media_dirs; media_path != NULL; media_path = media_path->next )
{
int64_t id;

1
upnpglobalvars.c
View File

@ -89,3 +89,4 @@ short int scanning = 0;
volatile short int quitting = 0;
volatile uint32_t updateID = 0;
const char *force_sort_criteria = NULL;
short int rescan_db = 0;

1
upnpglobalvars.h
View File

@ -233,5 +233,6 @@ extern short int scanning;
extern volatile short int quitting;
extern volatile uint32_t updateID;
extern const char *force_sort_criteria;
extern short int rescan_db;
#endif