Implementation of Sync Optimization for Movies, Series and Episodes

bazarr/app/config.py

@@ -161,6 +161,8 @@ class Validator(OriginalValidator):
     Validator('sonarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('sonarr.exclude_season_zero', must_exist=True, default=False, is_type_of=bool),
     Validator('sonarr.defer_search_signalr', must_exist=True, default=False, is_type_of=bool),
+    Validator('sonarr.sync_only_monitored_series', must_exist=True, default=False, is_type_of=bool),
+    Validator('sonarr.sync_only_monitored_episodes', must_exist=True, default=False, is_type_of=bool),
 
     # radarr section
     Validator('radarr.ip', must_exist=True, default='127.0.0.1', is_type_of=str),
@@ -180,6 +182,7 @@ class Validator(OriginalValidator):
     Validator('radarr.excluded_tags', must_exist=True, default=[], is_type_of=list),
     Validator('radarr.use_ffprobe_cache', must_exist=True, default=True, is_type_of=bool),
     Validator('radarr.defer_search_signalr', must_exist=True, default=False, is_type_of=bool),
+    Validator('radarr.sync_only_monitored_movies', must_exist=True, default=False, is_type_of=bool),
 
     # proxy section
     Validator('proxy.type', must_exist=True, default=None, is_type_of=(NoneType, str),

bazarr/radarr/sync/movies.py

@@ -16,6 +16,13 @@
 from .utils import get_profile_list, get_tags, get_movies_from_radarr_api
 from .parser import movieParser
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
+
+FEATURE_PREFIX = "SYNC_MOVIES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
 
 def update_all_movies():
     movies_full_scan_subtitles()
@@ -45,6 +52,16 @@ def update_movie(updated_movie, send_event):
             event_stream(type='movie', action='update', payload=updated_movie['radarrId'])
 
 
+def get_movie_monitored_status(movie_id):
+    existing_movie_monitored = database.execute(
+        select(TableMovies.monitored)
+        .where(TableMovies.tmdbId == movie_id))\
+        .first()
+    if existing_movie_monitored == None:
+        return True
+    else:
+        return bool_map[existing_movie_monitored[0]]
+
 # Insert new movies in DB
 def add_movie(added_movie, send_event):
     try:
@@ -107,51 +124,82 @@ def update_movies(send_event=True):
                                      (movie['movieFile']['size'] > 20480 or
                                       get_movie_file_size_from_db(movie['movieFile']['path']) > 20480)]
 
-            # Remove old movies from DB
+            # Remove movies from DB that either no longer exist in Radarr or exist and Radarr says do not have a movie file
             movies_to_delete = list(set(current_movies_id_db) - set(current_movies_radarr))
-
+            movies_deleted = []
             if len(movies_to_delete):
                 try:
                     database.execute(delete(TableMovies).where(TableMovies.tmdbId.in_(movies_to_delete)))
                 except IntegrityError as e:
                     logging.error(f"BAZARR cannot delete movies because of {e}")
                 else:
                     for removed_movie in movies_to_delete:
+                        movies_deleted.append(removed_movie['title'])
                         if send_event:
                             event_stream(type='movie', action='delete', payload=removed_movie)
 
-            # Build new and updated movies
+            # Add new movies and update movies that Radarr says have media files
+            # Any new movies added to Radarr that don't have media files yet will not be added to DB
             movies_count = len(movies)
+            sync_monitored = settings.radarr.sync_only_monitored_movies
+            if sync_monitored:
+                skipped_count = 0
+            files_missing = 0
+            movies_added = []
+            movies_updated = []
             for i, movie in enumerate(movies):
                 if send_event:
                     show_progress(id='movies_progress',
                                   header='Syncing movies...',
                                   name=movie['title'],
                                   value=i,
                                   count=movies_count)
-
+                # Only movies that Radarr says have files downloaded will be kept up to date in the DB
                 if movie['hasFile'] is True:
                     if 'movieFile' in movie:
+                        if sync_monitored:   
+                            if get_movie_monitored_status(movie['tmdbId']) != movie['monitored']:
+                                # monitored status is not the same as our DB
+                                trace(f"{i}: (Monitor Status Mismatch) {movie['title']}")
+                            elif not movie['monitored']:
+                                trace(f"{i}: (Skipped Unmonitored) {movie['title']}")
+                                skipped_count += 1
+                                continue
+
                         if (movie['movieFile']['size'] > 20480 or
                                 get_movie_file_size_from_db(movie['movieFile']['path']) > 20480):
-                            # Add movies in radarr to current movies list
+                            # Add/update movies from Radarr that have a movie file to current movies list
+                            trace(f"{i}: (Processing) {movie['title']}")
                             if str(movie['tmdbId']) in current_movies_id_db:
                                 parsed_movie = movieParser(movie, action='update',
                                                            tags_dict=tagsDict,
                                                            movie_default_profile=movie_default_profile,
                                                            audio_profiles=audio_profiles)
                                 if not any([parsed_movie.items() <= x for x in current_movies_db_kv]):
                                     update_movie(parsed_movie, send_event)
+                                    movies_updated.append(parsed_movie['title'])
                             else:
                                 parsed_movie = movieParser(movie, action='insert',
                                                            tags_dict=tagsDict,
                                                            movie_default_profile=movie_default_profile,
                                                            audio_profiles=audio_profiles)
                                 add_movie(parsed_movie, send_event)
+                                movies_added.append(parsed_movie['title'])
+                else:
+                     trace(f"{i}: (Skipped File Missing) {movie['title']}")
+                     files_missing += 1
 
             if send_event:
                 hide_progress(id='movies_progress')
 
+            trace(f"Skipped {files_missing} file missing movies out of {i}")
+            if sync_monitored:
+                trace(f"Skipped {skipped_count} unmonitored movies out of {i}")
+                trace(f"Processed {i - files_missing - skipped_count} movies out of {i} " +
+                              f"with {len(movies_added)} added, {len(movies_updated)} updated and {len(movies_deleted)} deleted")
+            else:
+                trace(f"Processed {i - files_missing} movies out of {i} with {len(movies_added)} added and {len(movies_updated)} updated")
+
             logging.debug('BAZARR All movies synced from Radarr into database.')
 
 

bazarr/radarr/sync/parser.py

@@ -13,12 +13,6 @@
 
 def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles):
     if 'movieFile' in movie:
-        # Detect file separator
-        if movie['path'][0] == "/":
-            separator = "/"
-        else:
-            separator = "\\"
-
         try:
             overview = str(movie['overview'])
         except Exception:
@@ -120,10 +114,9 @@ def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles)
 
         tags = [d['label'] for d in tags_dict if d['id'] in movie['tags']]
 
-        if action == 'update':
-            return {'radarrId': int(movie["id"]),
+        parsed_movie = {'radarrId': int(movie["id"]),
                     'title': movie["title"],
-                    'path': movie["path"] + separator + movie['movieFile']['relativePath'],
+                    'path': os.path.join(movie["path"], movie['movieFile']['relativePath']),                    
                     'tmdbId': str(movie["tmdbId"]),
                     'poster': poster,
                     'fanart': fanart,
@@ -142,30 +135,12 @@ def movieParser(movie, action, tags_dict, movie_default_profile, audio_profiles)
                     'movie_file_id': int(movie['movieFile']['id']),
                     'tags': str(tags),
                     'file_size': movie['movieFile']['size']}
-        else:
-            return {'radarrId': int(movie["id"]),
-                    'title': movie["title"],
-                    'path': movie["path"] + separator + movie['movieFile']['relativePath'],
-                    'tmdbId': str(movie["tmdbId"]),
-                    'subtitles': '[]',
-                    'overview': overview,
-                    'poster': poster,
-                    'fanart': fanart,
-                    'audio_language': str(audio_language),
-                    'sceneName': sceneName,
-                    'monitored': str(bool(movie['monitored'])),
-                    'sortTitle': movie['sortTitle'],
-                    'year': str(movie['year']),
-                    'alternativeTitles': alternativeTitles,
-                    'format': format,
-                    'resolution': resolution,
-                    'video_codec': videoCodec,
-                    'audio_codec': audioCodec,
-                    'imdbId': imdbId,
-                    'movie_file_id': int(movie['movieFile']['id']),
-                    'tags': str(tags),
-                    'profileId': movie_default_profile,
-                    'file_size': movie['movieFile']['size']}
+
+        if action == 'insert':
+            parsed_movie['subtitles'] = '[]'
+            parsed_movie['profileId'] = movie_default_profile
+
+        return parsed_movie
 
 
 def profile_id_to_language(id, profiles):

bazarr/sonarr/sync/episodes.py

@@ -5,7 +5,7 @@
 
 from sqlalchemy.exc import IntegrityError
 
-from app.database import database, TableEpisodes, delete, update, insert, select
+from app.database import database, TableShows, TableEpisodes, delete, update, insert, select
 from app.config import settings
 from utilities.path_mappings import path_mappings
 from subtitles.indexer.series import store_subtitles, series_full_scan_subtitles
@@ -16,14 +16,29 @@
 from .parser import episodeParser
 from .utils import get_episodes_from_sonarr_api, get_episodesFiles_from_sonarr_api
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
 
+FEATURE_PREFIX = "SYNC_EPISODES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
+
+def get_episodes_monitored_table(series_id):
+    episodes_monitored = database.execute(
+        select(TableEpisodes.episode_file_id, TableEpisodes.monitored)
+        .where(TableEpisodes.sonarrSeriesId == series_id))\
+        .all()
+    episode_dict = dict((x, y) for x, y in episodes_monitored)
+    return episode_dict
+
 def update_all_episodes():
     series_full_scan_subtitles()
     logging.info('BAZARR All existing episode subtitles indexed from disk.')
 
 
 def sync_episodes(series_id, send_event=True):
-    logging.debug('BAZARR Starting episodes sync from Sonarr.')
+    logging.debug(f'BAZARR Starting episodes sync from Sonarr for series ID {series_id}.')
     apikey_sonarr = settings.sonarr.apikey
 
     # Get current episodes id in DB
@@ -58,10 +73,36 @@ def sync_episodes(series_id, send_event=True):
                     if item:
                         episode['episodeFile'] = item[0]
 
+
+        sync_monitored = settings.sonarr.sync_only_monitored_series and settings.sonarr.sync_only_monitored_episodes
+        if sync_monitored:
+            episodes_monitored = get_episodes_monitored_table(series_id)
+            skipped_count = 0
+
         for episode in episodes:
             if 'hasFile' in episode:
                 if episode['hasFile'] is True:
                     if 'episodeFile' in episode:
+                        # monitored_status_db = get_episodes_monitored_status(episode['episodeFileId'])
+                        if sync_monitored:
+                            try:
+                                monitored_status_db = bool_map[episodes_monitored[episode['episodeFileId']]]
+                            except KeyError:
+                                monitored_status_db = None
+
+                            if monitored_status_db == None:
+                                # not in db, might need to add, if we have a file on disk
+                                pass
+                            elif monitored_status_db != episode['monitored']:
+                                # monitored status changed and we don't know about it until now
+                                trace(f"(Monitor Status Mismatch) {episode['title']}")
+                                # pass
+                            elif not episode['monitored']:
+                                # Add unmonitored episode in sonarr to current episode list, otherwise it will be deleted from db
+                                current_episodes_sonarr.append(episode['id'])
+                                skipped_count += 1
+                                continue
+
                         try:
                             bazarr_file_size = \
                                 os.path.getsize(path_mappings.path_replace(episode['episodeFile']['path']))
@@ -80,6 +121,12 @@ def sync_episodes(series_id, send_event=True):
                                 episodes_to_add.append(episodeParser(episode))
     else:
         return
+
+    if sync_monitored:
+        # try to avoid unnecessary database calls
+        if settings.general.debug:
+            series_title = database.execute(select(TableShows.title).where(TableShows.sonarrSeriesId == series_id)).first()[0]
+            trace(f"Skipped {skipped_count} unmonitored episodes out of {len(episodes)} for {series_title}")
 
     # Remove old episodes from DB
     episodes_to_delete = list(set(current_episodes_id_db_list) - set(current_episodes_sonarr))

bazarr/sonarr/sync/series.py

@@ -16,6 +16,20 @@
 from .parser import seriesParser
 from .utils import get_profile_list, get_tags, get_series_from_sonarr_api
 
+# map between booleans and strings in DB
+bool_map = {"True": True, "False": False}
+
+FEATURE_PREFIX = "SYNC_SERIES "
+def trace(message):
+    if settings.general.debug:
+        logging.debug(FEATURE_PREFIX + message)
+
+def get_series_monitored_table():
+    series_monitored = database.execute(
+        select(TableShows.tvdbId, TableShows.monitored))\
+        .all()
+    series_dict = dict((x, y) for x, y in series_monitored)
+    return series_dict
 
 def update_series(send_event=True):
     check_sonarr_rootfolder()
@@ -55,6 +69,12 @@ def update_series(send_event=True):
         current_shows_sonarr = []
 
         series_count = len(series)
+        sync_monitored = settings.sonarr.sync_only_monitored_series
+        if sync_monitored:
+            series_monitored = get_series_monitored_table()
+            skipped_count = 0
+        trace(f"Starting sync for {series_count} shows")
+
         for i, show in enumerate(series):
             if send_event:
                 show_progress(id='series_progress',
@@ -63,6 +83,26 @@ def update_series(send_event=True):
                               value=i,
                               count=series_count)
 
+            if sync_monitored:
+                try:
+                    monitored_status_db = bool_map[series_monitored[show['tvdbId']]]
+                except KeyError:
+                    monitored_status_db = None
+                if monitored_status_db == None:
+                    # not in db, need to add
+                    pass
+                elif monitored_status_db != show['monitored']:
+                    # monitored status changed and we don't know about it until now
+                    trace(f"{i}: (Monitor Status Mismatch) {show['title']}")
+                    # pass
+                elif not show['monitored']:
+                    # Add unmonitored series in sonarr to current series list, otherwise it will be deleted from db
+                    trace(f"{i}: (Skipped Unmonitored) {show['title']}")
+                    current_shows_sonarr.append(show['id'])
+                    skipped_count += 1
+                    continue
+
+            trace(f"{i}: (Processing) {show['title']}")
             # Add shows in Sonarr to current shows list
             current_shows_sonarr.append(show['id'])
 
@@ -76,6 +116,7 @@ def update_series(send_event=True):
                         .filter_by(**updated_series))\
                         .first():
                     try:
+                        trace(f"Updating {show['title']}")
                         database.execute(
                             update(TableShows)
                             .values(updated_series)
@@ -92,6 +133,7 @@ def update_series(send_event=True):
                                             audio_profiles=audio_profiles)
 
                 try:
+                    trace(f"Inserting {show['title']}")
                     database.execute(
                         insert(TableShows)
                         .values(added_series))
@@ -110,6 +152,10 @@ def update_series(send_event=True):
         removed_series = list(set(current_shows_db) - set(current_shows_sonarr))
 
         for series in removed_series:
+             # try to avoid unnecessary database calls
+            if settings.general.debug:
+                series_title = database.execute(select(TableShows.title).where(TableShows.sonarrSeriesId == series)).first()[0]
+                trace(f"Deleting {series_title}")
             database.execute(
                 delete(TableShows)
                 .where(TableShows.sonarrSeriesId == series))
@@ -120,6 +166,8 @@ def update_series(send_event=True):
         if send_event:
             hide_progress(id='series_progress')
 
+        if sync_monitored:
+            trace(f"skipped {skipped_count} unmonitored series out of {i}")
         logging.debug('BAZARR All series synced from Sonarr into database.')
 
 

bazarr/utilities/video_analyzer.py

@@ -266,6 +266,11 @@ def parse_video_metadata(file, file_size, episode_file_id=None, movie_file_id=No
     elif embedded_subs_parser == 'mediainfo':
         mediainfo_path = get_binary("mediainfo")
 
+    # see if file exists (perhaps offline)
+    if not os.path.exists(file):
+        logging.error(f'Video file "{file}" cannot be found for analysis')
+        return None
+
     # if we have ffprobe available
     if ffprobe_path:
         try: