From 9113d42fbf12fa444f326e5412dfb69eadc94588 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Fri, 27 Jan 2023 22:39:44 -0500 Subject: [PATCH 1/2] improve error handling and logging (#87) This commit improves error handling and logging when theme song does not exist in ThemerrDB. It also logs the url for where the user should go to have the item added. Additionally, add a retry loop for uploading theme songs. --- Contents/Code/__init__.py | 71 +++++++++++---- Contents/Code/default_prefs.py | 1 + Contents/Code/helpers.py | 31 +++++++ Contents/Code/plex_api_helper.py | 151 +++++++++++++++++++++++++------ Contents/DefaultPrefs.json | 7 ++ docs/source/about/usage.rst | 14 +++ 6 files changed, 227 insertions(+), 48 deletions(-) diff --git a/Contents/Code/__init__.py b/Contents/Code/__init__.py index 22e709ff..0ec9426d 100644 --- a/Contents/Code/__init__.py +++ b/Contents/Code/__init__.py @@ -29,11 +29,13 @@ # local imports if sys.version_info.major < 3: from default_prefs import default_prefs - from plex_api_helper import add_themes, plex_listener + from helpers import issue_url_games, issue_url_movies + from plex_api_helper import add_themes, get_plex_item, plex_listener from youtube_dl_helper import process_youtube else: from .default_prefs import default_prefs - from .plex_api_helper import add_themes, plex_listener + from .helpers import issue_url_games, issue_url_movies + from .plex_api_helper import add_themes, get_plex_item, plex_listener from .youtube_dl_helper import process_youtube @@ -95,7 +97,7 @@ def ValidatePrefs(): return MessageContainer(header='Error', message=error_message) else: Log.Info("DefaultPrefs.json is valid") - return MessageContainer(header='Success', message='RetroArcher - Provided preference values are ok') + return MessageContainer(header='Success', message='Themerr-plex - Provided preference values are ok') def Start(): @@ -245,7 +247,7 @@ def search(results, media, lang, manual): Log.Debug('media.primary_metadata.id: %s' % media.primary_metadata.id) # the media_id will be used to create the url path, replacing `-` with `/` - if media.primary_metadata == 'dev.lizardbyte.retroarcher-plex': + if media.primary_agent == 'dev.lizardbyte.retroarcher-plex': media_id = 'games-%s' % re.search(r'((igdb)-(\d+))', media.primary_metadata.id).group(1) else: media_id = 'movies-%s-%s' % (media.primary_agent.rsplit('.', 1)[-1], media.primary_metadata.id) @@ -297,26 +299,57 @@ def update(metadata, media, lang, force): >>> Themerr().update(metadata=..., media=..., lang='en', force=True) ... """ - Log.Debug('Updating with arguments: {metadata=%s, media=%s, lang=%s, force=%s' % - (metadata, media, lang, force)) - rating_key = int(media.id) # rating key of plex item - Log.Info('Rating key: %s' % rating_key) - Log.Info('metadata.id: %s' % metadata.id) - url = 'https://app.lizardbyte.dev/ThemerrDB/%s.json' % metadata.id.replace('-', '/') + Log.Debug('%s: Updating with arguments: {metadata=%s, media=%s, lang=%s, force=%s' % + (rating_key, metadata, media, lang, force)) - data = JSON.ObjectFromURL(url=url, errors='ignore') + Log.Info('%s: metadata.id: %s' % (rating_key, metadata.id)) + split_id = metadata.id.split('-') + item_type = split_id[0] + database = split_id[1] + database_id = split_id[2] + url = 'https://app.lizardbyte.dev/ThemerrDB/%s/%s/%s.json' % (item_type, database, database_id) try: - yt_video_url = data['youtube_theme_url'] - except KeyError: - Log.Info('No theme song found for %s (%s)' % (metadata.title, metadata.year)) - return + data = JSON.ObjectFromURL(url=url, errors='ignore') + except Exception as e: + Log.Error('%s: Error retrieving data from ThemerrDB: %s' % (rating_key, e)) + if database == 'themoviedb': # movies + # need to use python-plexapi to get the movie year + plex_item = get_plex_item(rating_key=rating_key) + + # create the url to add the theme song to ThemerrDB + try: + issue_title = '%s (%s)' % (plex_item.title, plex_item.year) + issue_url = issue_url_movies % (issue_title, database_id) + Log.Info('%s: Theme song missing in ThemerrDB. Add it here -> "%s"' % (rating_key, issue_url)) + except Exception as e: + Log.Error('%s: Error creating the url to add the theme song to ThemerrDB: %s' % (rating_key, e)) + elif database == 'igdb': # games + try: + game_data = JSON.ObjectFromURL(url='https://db.lizardbyte.dev/games/%s.json' % database_id) + except Exception as e: + Log.Error('%s: Error retrieving data from LizardByteDB: %s' % (rating_key, e)) + else: + try: + issue_year = game_data['release_date'][0]['y'] + except (KeyError, IndexError): + issue_year = None + issue_url_suffix = game_data['slug'] + issue_title = '%s (%s)' % (game_data['name'], issue_year) + issue_url = issue_url_games % (issue_title, issue_url_suffix) + Log.Info('%s: Theme song missing in ThemerrDB. Add it here -> %s' % (rating_key, issue_url)) else: - theme_url = process_youtube(url=yt_video_url) - - if theme_url: - add_themes(rating_key=rating_key, theme_urls=[theme_url]) + try: + yt_video_url = data['youtube_theme_url'] + except KeyError: + Log.Info('%s: No theme song found for %s (%s)' % (rating_key, metadata.title, metadata.year)) + return + else: + theme_url = process_youtube(url=yt_video_url) + + if theme_url: + add_themes(rating_key=rating_key, theme_urls=[theme_url]) return metadata diff --git a/Contents/Code/default_prefs.py b/Contents/Code/default_prefs.py index 3b793461..c94ea887 100644 --- a/Contents/Code/default_prefs.py +++ b/Contents/Code/default_prefs.py @@ -1,5 +1,6 @@ default_prefs = dict( int_plexapi_plexapi_timeout='180', + int_plexapi_upload_retries_max='3', int_plexapi_upload_threads='3', str_youtube_user='', str_youtube_passwd='' diff --git a/Contents/Code/helpers.py b/Contents/Code/helpers.py index 076886eb..55b28439 100644 --- a/Contents/Code/helpers.py +++ b/Contents/Code/helpers.py @@ -3,3 +3,34 @@ tmdb='themoviedb', tvdb='thetvdb' ) + + +base_url = 'https://github.com/LizardByte/ThemerrDB/issues/new?assignees=' +issue_labels = dict( + game='request-game', + movie='request-movie', +) +issue_template = dict( + game='game-theme.yml', + movie='movie-theme.yml', +) +title_prefix = dict( + game='[GAME]: ', + movie='[MOVIE]: ', +) +url_name = dict( + game='igdb_url', + movie='themoviedb_url', +) +url_prefix = dict( + game='https://www.igdb.com/games/', + movie='https://www.themoviedb.org/movie/', +) + +# two additional strings to fill in later, item title and item url +issue_url_movies = '%s&labels=%s&template=%s&title=%s%s&%s=%s%s' % (base_url, issue_labels['movie'], + issue_template['movie'], title_prefix['movie'], + '%s', url_name['movie'], url_prefix['movie'], '%s') +issue_url_games = '%s&labels=%s&template=%s&title=%s%s&%s=%s%s' % (base_url, issue_labels['game'], + issue_template['game'], title_prefix['game'], + '%s', url_name['game'], url_prefix['game'], '%s') diff --git a/Contents/Code/plex_api_helper.py b/Contents/Code/plex_api_helper.py index 06bbbfb9..1cac76da 100644 --- a/Contents/Code/plex_api_helper.py +++ b/Contents/Code/plex_api_helper.py @@ -29,10 +29,10 @@ # local imports if sys.version_info.major < 3: - from helpers import guid_map + from helpers import guid_map, issue_url_movies from youtube_dl_helper import process_youtube else: - from .helpers import guid_map + from .helpers import guid_map, issue_url_movies from .youtube_dl_helper import process_youtube plex = None @@ -80,7 +80,7 @@ def setup_plexapi(): def add_themes(rating_key, theme_files=None, theme_urls=None): - # type: (int, Optional[list], Optional[list]) -> None + # type: (int, Optional[list], Optional[list]) -> bool """ Apply themes to the specified item. @@ -95,10 +95,17 @@ def add_themes(rating_key, theme_files=None, theme_urls=None): theme_urls : Optional[list] A list of urls to theme songs. + Returns + ------- + bool + True if the themes were added successfully, False otherwise. + Examples -------- >>> add_themes(theme_list=[...], rating_key=...) """ + uploaded = False + if theme_files or theme_urls: global plex if not plex: @@ -112,14 +119,90 @@ def add_themes(rating_key, theme_files=None, theme_urls=None): if theme_files: for theme_file in theme_files: Log.Info('Attempting to upload theme file: %s' % theme_file) - plex_item.uploadTheme(filepath=theme_file) + uploaded = upload_theme(plex_item=plex_item, filepath=theme_file) if theme_urls: for theme_url in theme_urls: Log.Info('Attempting to upload theme file: %s' % theme_url) - plex_item.uploadTheme(url=theme_url) + uploaded = upload_theme(plex_item=plex_item, url=theme_url) else: Log.Info('No theme songs provided for rating key: %s' % rating_key) + return uploaded + + +def upload_theme(plex_item, filepath=None, url=None): + # type: (any, Optional[str], Optional[str]) -> bool + """ + Upload a theme to the specified item. + + Uploads a theme to the item specified by the ``plex_item``. + + Parameters + ---------- + plex_item : any + The item to upload the theme to. + filepath : Optional[str] + The path to the theme song. + url : Optional[str] + The url to the theme song. + + Returns + ------- + bool + True if the theme was uploaded successfully, False otherwise. + + Examples + -------- + >>> upload_theme(plex_item=..., url=...) + ... + """ + count = 0 + while count <= int(Prefs['int_plexapi_upload_retries_max']): + try: + if filepath: + plex_item.uploadTheme(filepath=filepath) + elif url: + plex_item.uploadTheme(url=url) + except BadRequest as e: + sleep_time = 2**count + Log.Error('%s: Error uploading theme: %s' % (plex_item.ratingKey, e)) + Log.Error('%s: Trying again in : %s' % (plex_item.ratingKey, sleep_time)) + time.sleep(sleep_time) + count += 1 + else: + return True + return False + + +def get_plex_item(rating_key): + # type: (int) -> any + """ + Get any item from the Plex Server. + + This function is used to get an item from the Plex Server. It can then be used to get the metadata for the item. + + Parameters + ---------- + rating_key : int + The ``rating_key`` of the item to upload a theme for. + + Returns + ------- + any + The item from the Plex Server. + + Examples + -------- + >>> get_plex_item(rating_key=1) + ... + """ + global plex, processing_completed + if not plex: + plex = setup_plexapi() + plex_item = plex.fetchItem(ekey=rating_key) + + return plex_item + def process_queue(): # type: () -> None @@ -240,6 +323,8 @@ def update_plex_movie_item(rating_key): plex = setup_plexapi() plex_item = plex.fetchItem(ekey=rating_key) + themerr_db_logs = [] + # guids does not appear to exist for legacy agents or plugins # therefore, we don't need to filter those out for guid in plex_item.guids: @@ -247,33 +332,41 @@ def update_plex_movie_item(rating_key): database = guid_map[split_guid[0]] database_id = split_guid[1] - Log.Debug('Attempting update for: {title=%s, rating_key=%s, database=%s, database_id=%s}' % - (plex_item.title, plex_item.ratingKey, database, database_id)) + Log.Debug('%s: Attempting update for: {title=%s, rating_key=%s, database=%s, database_id=%s}' % + (rating_key, plex_item.title, plex_item.ratingKey, database, database_id)) url = 'https://app.lizardbyte.dev/ThemerrDB/movies/%s/%s.json' % (database, database_id) - data = JSON.ObjectFromURL(url=url, errors='ignore') - try: - yt_video_url = data['youtube_theme_url'] - except KeyError: - Log.Info('No theme song found for %s (%s)' % (plex_item.title, plex_item.year)) - return + data = JSON.ObjectFromURL(url=url, errors='ignore') + except Exception: + themerr_db_logs.append('%s: Could not retrieve data from ThemerrDB using %s' % (rating_key, database)) + if database == 'themoviedb': + issue_title = '%s (%s)' % (plex_item.title, plex_item.year) + issue_url = issue_url_movies % (issue_title, database_id) + themerr_db_logs.insert( + 0, + '%s: Theme song missing in ThemerrDB. Add it here -> "%s"' % (rating_key, issue_url) + ) else: - theme_url = process_youtube(url=yt_video_url) - - if theme_url: - try: - add_themes(rating_key=plex_item.ratingKey, theme_urls=[theme_url]) - except BadRequest as e: - # log it and try again in 30 seconds - Log.Error('Error uploading theme: %s' % e) - Log.Info('Trying again in 30 seconds.') - time.sleep(30) - add_themes(rating_key=plex_item.ratingKey, theme_urls=[theme_url]) - - # add the item to processing_completed list - processing_completed.append(rating_key) - - # theme found and uploaded using this database, so return + try: + yt_video_url = data['youtube_theme_url'] + except KeyError: + Log.Info('%s: No theme song found for %s (%s)' % (rating_key, plex_item.title, plex_item.year)) return + else: + theme_url = process_youtube(url=yt_video_url) + + if theme_url: + theme_added = add_themes(rating_key=plex_item.ratingKey, theme_urls=[theme_url]) + + # add the item to processing_completed list + if theme_added: + processing_completed.append(rating_key) + + # theme found and uploaded using this database, so return + return + + # could not upload theme using any database, so log the errors + for log in themerr_db_logs: + Log.Error(log) diff --git a/Contents/DefaultPrefs.json b/Contents/DefaultPrefs.json index be617905..f217bcc9 100644 --- a/Contents/DefaultPrefs.json +++ b/Contents/DefaultPrefs.json @@ -6,6 +6,13 @@ "default": "180", "secure": "false" }, + { + "id": "int_plexapi_upload_retries_max", + "type": "text", + "label": "Max Retries, integer (min: 0)", + "default": "6", + "secure": "false" + }, { "id": "int_plexapi_upload_threads", "type": "text", diff --git a/docs/source/about/usage.rst b/docs/source/about/usage.rst index 64f22501..94a90572 100644 --- a/docs/source/about/usage.rst +++ b/docs/source/about/usage.rst @@ -33,6 +33,20 @@ Default Minimum ``1`` +Max Retries +^^^^^^^^^^^ + +Description + The number of times to retry uploading theme audio to the Plex server. The time between retries will increase + exponentially. The time between is calculated as ``2 ^ retry_number``. For example, the first retry will occur + after 2 seconds, the second retry will occur after 4 seconds, and the third retry will occur after 8 seconds. + +Default + ``6`` + +Minimum + ``0`` + Multiprocessing Threads ^^^^^^^^^^^^^^^^^^^^^^^ From 58182595cb3b5d570e6b4c4311b8cb6dcbb69298 Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sat, 28 Jan 2023 10:03:48 -0500 Subject: [PATCH 2/2] changelog: v0.1.3 (#89) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bddd7001..27e7d3fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.1.3] - 2023-01-28 +### Added +- Max Retries setting added, allowing you to specify how many times to retry a failed upload +### Fixed +- Improve error handling and logging when theme song does not exist in ThemerrDB + ## [0.1.2] - 2023-01-23 ### Added - Process items from Plex Movie agent with a queue @@ -68,3 +74,4 @@ [0.1.0]: https://github.com/lizardbyte/themerr-plex/releases/tag/v0.1.0 [0.1.1]: https://github.com/lizardbyte/themerr-plex/releases/tag/v0.1.1 [0.1.2]: https://github.com/lizardbyte/themerr-plex/releases/tag/v0.1.2 +[0.1.3]: https://github.com/lizardbyte/themerr-plex/releases/tag/v0.1.3