From 54b192a0e9b2360880e98b82d4f3242e67fbb8a1 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 8 May 2024 19:19:38 +1000 Subject: [PATCH 01/53] Remove BaseItem.set_context_menu - Use more flexible add_context_menu method instead - Also filter out invalid context menu items --- .../youtube_plugin/kodion/items/base_item.py | 10 ++++------ .../kodion/items/next_page_item.py | 2 +- .../kodion/items/search_history_item.py | 2 +- .../lib/youtube_plugin/youtube/helper/utils.py | 18 +++++++----------- .../lib/youtube_plugin/youtube/provider.py | 12 ++++++------ 5 files changed, 19 insertions(+), 25 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/base_item.py b/resources/lib/youtube_plugin/kodion/items/base_item.py index d0d69e0e7..f3dadfc22 100644 --- a/resources/lib/youtube_plugin/kodion/items/base_item.py +++ b/resources/lib/youtube_plugin/kodion/items/base_item.py @@ -119,12 +119,10 @@ def set_fanart(self, fanart): def get_fanart(self): return self._fanart - def set_context_menu(self, context_menu): - self._context_menu = context_menu - - def add_context_menu(self, context_menu, position=0): - if self._context_menu is None: - self._context_menu = context_menu + def add_context_menu(self, context_menu, position='end', replace=False): + context_menu = (item for item in context_menu if item) + if replace or not self._context_menu: + self._context_menu = list(context_menu) elif position == 'end': self._context_menu.extend(context_menu) else: diff --git a/resources/lib/youtube_plugin/kodion/items/next_page_item.py b/resources/lib/youtube_plugin/kodion/items/next_page_item.py index 55886c9f7..713b37b2f 100644 --- a/resources/lib/youtube_plugin/kodion/items/next_page_item.py +++ b/resources/lib/youtube_plugin/kodion/items/next_page_item.py @@ -44,7 +44,7 @@ def __init__(self, context, params, image=None, fanart=None): menu_items.goto_quick_search(context), menu_items.separator(), ] - self.set_context_menu(context_menu) + self.add_context_menu(context_menu) @classmethod def create_page_token(cls, page, items_per_page=50): diff --git a/resources/lib/youtube_plugin/kodion/items/search_history_item.py b/resources/lib/youtube_plugin/kodion/items/search_history_item.py index 93f2e3ba9..7c648506c 100644 --- a/resources/lib/youtube_plugin/kodion/items/search_history_item.py +++ b/resources/lib/youtube_plugin/kodion/items/search_history_item.py @@ -39,4 +39,4 @@ def __init__(self, context, query, image=None, fanart=None, location=False): menu_items.search_rename(context, query), menu_items.search_clear(context), ] - self.set_context_menu(context_menu) + self.add_context_menu(context_menu) diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index 7aa6d30c1..184fa90ca 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -233,7 +233,7 @@ def update_channel_infos(provider, context, channel_id_dict, ) if context_menu: - channel_item.set_context_menu(context_menu) + channel_item.add_context_menu(context_menu, replace=True) fanart_images = yt_item.get('brandingSettings', {}).get('image', {}) for banner in banners: @@ -297,16 +297,12 @@ def update_playlist_infos(provider, context, playlist_id_dict, context_menu = [ menu_items.play_all_from_playlist( context, playlist_id - ) + ), + menu_items.bookmarks_add( + context, playlist_item + ) if not in_bookmarks_list and channel_id != 'mine' else None, ] - if not in_bookmarks_list and channel_id != 'mine': - context_menu.append( - menu_items.bookmarks_add( - context, playlist_item - ) - ) - if logged_in: if channel_id != 'mine': # subscribe to the channel via the playlist item @@ -352,7 +348,7 @@ def update_playlist_infos(provider, context, playlist_id_dict, ) if context_menu: - playlist_item.set_context_menu(context_menu) + playlist_item.add_context_menu(context_menu, replace=True) # update channel mapping if channel_items_dict is not None: @@ -782,7 +778,7 @@ def update_video_infos(provider, context, video_id_dict, context_menu.append( menu_items.separator(), ) - video_item.set_context_menu(context_menu) + video_item.add_context_menu(context_menu, replace=True) def update_play_info(provider, context, video_id, video_item, video_stream, diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index b26424989..04ba64eb4 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -1033,7 +1033,7 @@ def on_playback_history(self, context, re_match): ), menu_items.separator(), ] - video_item.add_context_menu(context_menu) + video_item.add_context_menu(context_menu, position=0) return video_items @@ -1215,7 +1215,7 @@ def on_root(self, context, re_match): context, watch_later_id ) ] - watch_later_item.set_context_menu(context_menu) + watch_later_item.add_context_menu(context_menu, replace=True) result.append(watch_later_item) else: watch_history_item = DirectoryItem( @@ -1240,7 +1240,7 @@ def on_root(self, context, re_match): context, playlists['likes'] ) ] - liked_videos_item.set_context_menu(context_menu) + liked_videos_item.add_context_menu(context_menu, replace=True) result.append(liked_videos_item) # disliked videos @@ -1265,7 +1265,7 @@ def on_root(self, context, re_match): context, history_id ) ] - watch_history_item.set_context_menu(context_menu) + watch_history_item.add_context_menu(context_menu, replace=True) result.append(watch_history_item) elif local_history: watch_history_item = DirectoryItem( @@ -1436,7 +1436,7 @@ def on_bookmarks(self, context, re_match): ), menu_items.separator(), ] - item.add_context_menu(context_menu) + item.add_context_menu(context_menu, position=0) bookmarks.append(item) return bookmarks @@ -1500,7 +1500,7 @@ def on_watch_later(self, context, re_match): ), menu_items.separator(), ] - video_item.add_context_menu(context_menu) + video_item.add_context_menu(context_menu, position=0) return video_items From 7e24e8d6e61cb2a6179a09cfb336539f85c1f555 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 8 May 2024 19:21:24 +1000 Subject: [PATCH 02/53] Don't add page_token and jump context menu item to Next page item in unsupported listings --- .../lib/youtube_plugin/kodion/items/next_page_item.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/next_page_item.py b/resources/lib/youtube_plugin/kodion/items/next_page_item.py index 713b37b2f..f685bcd81 100644 --- a/resources/lib/youtube_plugin/kodion/items/next_page_item.py +++ b/resources/lib/youtube_plugin/kodion/items/next_page_item.py @@ -19,14 +19,18 @@ def __init__(self, context, params, image=None, fanart=None): if 'refresh' in params: del params['refresh'] + path = context.get_path() page = params.get('page', 2) items_per_page = params.get('items_per_page', 50) - if 'page_token' not in params: + can_jump = ('next_page_token' not in params + and not path.startswith(('/channel', + '/special/recommendations'))) + if 'page_token' not in params and can_jump: params['page_token'] = self.create_page_token(page, items_per_page) super(NextPageItem, self).__init__( context.localize('page.next') % page, - context.create_uri(context.get_path(), params), + context.create_uri(path, params), image=image, category_label='__inherit__', ) @@ -39,7 +43,7 @@ def __init__(self, context, params, image=None, fanart=None): context_menu = [ menu_items.refresh(context), - menu_items.goto_page(context), + menu_items.goto_page(context) if can_jump else None, menu_items.goto_home(context), menu_items.goto_quick_search(context), menu_items.separator(), From 43f1ba43cfcd65be888a4c8b380a27c1f29f1982 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 8 May 2024 19:27:29 +1000 Subject: [PATCH 03/53] Update XbmcContext.is_plugin_path for now optional trailing slash --- .../youtube_plugin/kodion/context/xbmc/xbmc_context.py | 10 ++++++++-- .../youtube_plugin/kodion/monitors/player_monitor.py | 2 +- .../lib/youtube_plugin/youtube/helper/yt_playlist.py | 4 ++-- .../lib/youtube_plugin/youtube/helper/yt_video.py | 2 +- resources/lib/youtube_plugin/youtube/provider.py | 2 +- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py index eb12c24c1..893f3bb3d 100644 --- a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py +++ b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py @@ -351,8 +351,14 @@ def get_region(self): def addon(self): return self._addon - def is_plugin_path(self, uri, uri_path=''): - return uri.startswith('plugin://%s/%s' % (self.get_id(), uri_path)) + def is_plugin_path(self, uri, uri_path='', partial=False): + uri_path = ('plugin://%s/%s' % (self.get_id(), uri_path)).rstrip('/') + if not partial: + uri_path = ( + uri_path + '/', + uri_path + '?' + ) + return uri.startswith(uri_path) @staticmethod def format_date_short(date_obj, str_format=None): diff --git a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py index 0604c10f7..148108b4e 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py @@ -114,7 +114,7 @@ def run(self): break if (not current_file.startswith(playing_file) and not ( - self._context.is_plugin_path(current_file, 'play/') + self._context.is_plugin_path(current_file, 'play') and video_id_param in current_file )) or total_time <= 0: self.stop() diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py index 8dc657abc..3ca9fce23 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py @@ -34,7 +34,7 @@ def _process_add_video(provider, context, keymap_action=False): video_id = context.get_param('video_id', '') if not video_id: - if context.is_plugin_path(path, 'play/'): + if context.is_plugin_path(path, 'play'): video_id = find_video_id(path) keymap_action = True if not video_id: @@ -160,7 +160,7 @@ def _process_select_playlist(provider, context): video_id = params.get('video_id', '') if not video_id: - if context.is_plugin_path(path, 'play/'): + if context.is_plugin_path(path, 'play'): video_id = find_video_id(path) if video_id: context.set_param('video_id', video_id) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_video.py b/resources/lib/youtube_plugin/youtube/helper/yt_video.py index e2c6b2db2..4f88e9ac6 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_video.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_video.py @@ -28,7 +28,7 @@ def _process_rate_video(provider, context, re_match): try: video_id = re_match.group('video_id') except IndexError: - if context.is_plugin_path(listitem_path, 'play/'): + if context.is_plugin_path(listitem_path, 'play'): video_id = find_video_id(listitem_path) if not video_id: diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 04ba64eb4..54d2a77e2 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -633,7 +633,7 @@ def on_play(self, context, re_match): if ({'channel_id', 'live', 'playlist_id', 'playlist_ids', 'video_id'} .isdisjoint(params.keys())): path = context.get_listitem_detail('FileNameAndPath', attr=True) - if context.is_plugin_path(path, 'play/'): + if context.is_plugin_path(path, 'play'): video_id = find_video_id(path) if video_id: context.set_param('video_id', video_id) From 53a703b323920179777c15dadbe11d732ef320fe Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 8 May 2024 19:30:49 +1000 Subject: [PATCH 04/53] Version bump v7.0.7+beta.2 --- addon.xml | 2 +- changelog.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 7f0abffd2..2028da74c 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + diff --git a/changelog.txt b/changelog.txt index 65cda863e..21d95efe9 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,8 @@ +## v7.0.7+beta.2 +### Fixed +- Fix invalid pageToken error by removing jump from unsupported listings #715 + + ## v7.0.7+beta.1 ### Fixed - Fixed not being able to re-refresh a directory listing that has already been refreshed From fe251ded11e7268ae890fe9401be8cd494c01436 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 8 May 2024 21:34:28 +1000 Subject: [PATCH 05/53] Respect internal Kodi resume parameter for all playback actions #693 --- .../youtube_plugin/kodion/items/xbmc/xbmc_items.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index b05649603..269b6e450 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -18,7 +18,7 @@ from ...utils import current_system_version, datetime_parser -def set_info(list_item, item, properties): +def set_info(list_item, item, properties, resume=True): is_video = False if not current_system_version.compatible(20, 0): if isinstance(item, VideoItem): @@ -161,7 +161,7 @@ def set_info(list_item, item, properties): list_item.setProperties(properties) return - resume_time = item.get_start_time() + resume_time = resume and item.get_start_time() if resume_time: properties['ResumeTime'] = str(resume_time) duration = item.get_duration() @@ -289,7 +289,7 @@ def set_info(list_item, item, properties): if value is not None: info_tag.setAlbum(value) - resume_time = item.get_start_time() + resume_time = resume and item.get_start_time() duration = item.get_duration() if resume_time and duration: info_tag.setResumePoint(resume_time, float(duration)) @@ -432,7 +432,8 @@ def video_playback_item(context, video_item, show_fanart=None, **_kwargs): if video_item.subtitles: list_item.setSubtitles(video_item.subtitles) - set_info(list_item, video_item, props) + resume = context.get_param('resume') + set_info(list_item, video_item, props, resume=resume) return list_item @@ -463,7 +464,8 @@ def audio_listitem(context, audio_item, show_fanart=None, for_playback=False): 'thumb': image, }) - set_info(list_item, audio_item, props) + resume = context.get_param('resume') or not for_playback + set_info(list_item, audio_item, props, resume=resume) context_menu = audio_item.get_context_menu() if context_menu: From 272adbd59fceec7a15f0f8444528049076b0e6b8 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 13:37:09 +1000 Subject: [PATCH 06/53] Further improvements to player monitoring when seeking #746 --- .../youtube_plugin/kodion/monitors/player_monitor.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py index 148108b4e..11dc3c351 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py @@ -107,8 +107,9 @@ def run(self): current_file = player.getPlayingFile() played_time = player.getTime() total_time = player.getTotalTime() - player.current_time = played_time - player.total_time = total_time + if not player.seeking: + player.current_time = played_time + player.total_time = total_time except RuntimeError: self.stop() break @@ -299,6 +300,7 @@ def __init__(self, provider, context, monitor): self._monitor = monitor self._ui = self._context.get_ui() self.threads = [] + self.seeking = False self.seek_time = None self.start_time = None self.end_time = None @@ -403,9 +405,13 @@ def onPlayBackError(self): def onPlayBackSeek(self, time, seekOffset): time_s = time / 1000 + self.seeking = True self.current_time = time_s self.seek_time = None if ((self.end_time and time_s > self.end_time + 1) or (self.start_time and time_s < self.start_time - 1)): self.start_time = None self.end_time = None + + def onAVChange(self): + self.seeking = False From bc56e1018c62744e284180aeaefc3537e4f6d05b Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 14:51:47 +1000 Subject: [PATCH 07/53] Improve post play refresh --- .../kodion/constants/__init__.py | 2 ++ .../kodion/monitors/player_monitor.py | 19 ++++++++++--------- .../kodion/monitors/service_monitor.py | 18 +++++++++++++----- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/constants/__init__.py b/resources/lib/youtube_plugin/kodion/constants/__init__.py index d9eca4807..bd4511db8 100644 --- a/resources/lib/youtube_plugin/kodion/constants/__init__.py +++ b/resources/lib/youtube_plugin/kodion/constants/__init__.py @@ -38,6 +38,7 @@ PLAYER_DATA = 'player_json' PLAYLIST_PATH = 'playlist_path' PLAYLIST_POSITION = 'playlist_position' +REFRESH_CONTAINER = 'refresh_container' REROUTE = 'reroute' SLEEPING = 'sleeping' SWITCH_PLAYER_FLAG = 'switch_player' @@ -55,6 +56,7 @@ 'PLAYER_DATA', 'PLAYLIST_PATH', 'PLAYLIST_POSITION', + 'REFRESH_CONTAINER', 'RESOURCE_PATH', 'REROUTE', 'SLEEPING', diff --git a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py index 11dc3c351..a74aa6b95 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py @@ -14,7 +14,12 @@ import threading from ..compatibility import xbmc -from ..constants import BUSY_FLAG, PLAYER_DATA, SWITCH_PLAYER_FLAG +from ..constants import ( + BUSY_FLAG, + PLAYER_DATA, + REFRESH_CONTAINER, + SWITCH_PLAYER_FLAG, +) class PlayerMonitorThread(threading.Thread): @@ -243,9 +248,6 @@ def run(self): else: self._context.get_watch_later_list().remove(self.video_id) - playlist = xbmc.PlayList(xbmc.PLAYLIST_VIDEO) - in_playlist = playlist.size() >= 2 - if logged_in and not refresh_only: history_id = access_manager.get_watch_history_id() if history_id: @@ -253,8 +255,8 @@ def run(self): # rate video if (settings.get_bool('youtube.post.play.rate') and - (not in_playlist or - settings.get_bool('youtube.post.play.rate.playlists'))): + (settings.get_bool('youtube.post.play.rate.playlists') + or xbmc.PlayList(xbmc.PLAYLIST_VIDEO).size() < 2)): json_data = client.get_video_rating(self.video_id) if json_data: items = json_data.get('items', [{'rating': 'none'}]) @@ -269,9 +271,8 @@ def run(self): self._context, rating_match) - if ((not in_playlist or playlist.getposition() == -1) - and settings.get_bool('youtube.post.play.refresh', False)): - self._context.get_ui().refresh_container() + if settings.get_bool('youtube.post.play.refresh', False): + self._context.send_notification(REFRESH_CONTAINER, True) self.end() diff --git a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py index e6441de2b..b5d998b22 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py @@ -13,7 +13,7 @@ import threading from ..compatibility import xbmc, xbmcaddon, xbmcgui -from ..constants import ADDON_ID, CHECK_SETTINGS, WAKEUP +from ..constants import ADDON_ID, CHECK_SETTINGS, REFRESH_CONTAINER, WAKEUP from ..logger import log_debug from ..network import get_connect_address, get_http_server, httpd_status from ..settings import XbmcPluginSettings @@ -42,6 +42,14 @@ def __init__(self): super(ServiceMonitor, self).__init__() + @staticmethod + def _refresh_allowed(): + return (not xbmc.getCondVisibility('Container.IsUpdating') + and not xbmc.getCondVisibility('System.HasActiveModalDialog') + and xbmc.getInfoLabel('Container.FolderPath').startswith( + 'plugin://{0}/'.format(ADDON_ID) + )) + def onNotification(self, sender, method, data): if sender != ADDON_ID: return @@ -63,6 +71,9 @@ def onNotification(self, sender, method, data): elif event == WAKEUP: if not self.httpd and self.httpd_required(): self.start_httpd() + elif event == REFRESH_CONTAINER: + if self._refresh_allowed(): + xbmc.executebuiltin('Container.Refresh') else: log_debug('onNotification: |unhandled method| -> |{method}|' .format(method=method)) @@ -86,10 +97,7 @@ def onSettingsChanged(self): '-'.join((ADDON_ID, CHECK_SETTINGS)), 'true' ) - if (not xbmc.getCondVisibility('Container.IsUpdating') - and not xbmc.getCondVisibility('System.HasActiveModalDialog') - and xbmc.getInfoLabel('Container.FolderPath').startswith( - 'plugin://{0}/'.format(ADDON_ID))): + if self._refresh_allowed(): xbmc.executebuiltin('Container.Refresh') use_httpd = (settings.use_isa() From d9122005d314987b12626a6eff32bcff6e460181 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 15:11:36 +1000 Subject: [PATCH 08/53] Improve and simplify player monitor logging --- .../kodion/monitors/player_monitor.py | 78 +++++++++---------- .../youtube_plugin/youtube/helper/yt_play.py | 12 +-- 2 files changed, 39 insertions(+), 51 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py index a74aa6b95..37e18f6fe 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/player_monitor.py @@ -23,7 +23,7 @@ class PlayerMonitorThread(threading.Thread): - def __init__(self, player, provider, context, monitor, playback_json): + def __init__(self, player, provider, context, monitor, playback_data): super(PlayerMonitorThread, self).__init__() self._stopped = threading.Event() @@ -34,10 +34,10 @@ def __init__(self, player, provider, context, monitor, playback_json): self._context = context self._monitor = monitor - self.playback_json = playback_json - self.video_id = self.playback_json.get('video_id') - self.channel_id = self.playback_json.get('channel_id') - self.video_status = self.playback_json.get('video_status') + self.playback_data = playback_data + self.video_id = playback_data.get('video_id') + self.channel_id = playback_data.get('channel_id') + self.video_status = playback_data.get('video_status') self.current_time = 0.0 self.total_time = 0.0 @@ -52,13 +52,13 @@ def abort_now(self): or self.stopped()) def run(self): - playing_file = self.playback_json.get('playing_file') - play_count = self.playback_json.get('play_count', 0) - use_remote_history = self.playback_json.get('use_remote_history', False) - use_local_history = self.playback_json.get('use_local_history', False) - playback_stats = self.playback_json.get('playback_stats', {}) - refresh_only = self.playback_json.get('refresh_only', False) - clip = self.playback_json.get('clip', False) + playing_file = self.playback_data.get('playing_file') + play_count = self.playback_data.get('play_count', 0) + use_remote_history = self.playback_data.get('use_remote_history', False) + use_local_history = self.playback_data.get('use_local_history', False) + playback_stats = self.playback_data.get('playback_stats', {}) + refresh_only = self.playback_data.get('refresh_only', False) + clip = self.playback_data.get('clip', False) self._context.log_debug('PlayerMonitorThread[{0}]: Starting' .format(self.video_id)) @@ -187,20 +187,6 @@ def run(self): if self.total_time > 0: self.progress = int(100 * self.current_time / self.total_time) - state = 'stopped' - self._context.send_notification('PlaybackStopped', { - 'video_id': self.video_id, - 'channel_id': self.channel_id, - 'status': self.video_status, - }) - self._context.log_debug('Playback stopped [{video_id}]:' - ' {current:.3f} secs of {total:.3f}' - ' @ {percent}%' - .format(video_id=self.video_id, - current=self.current_time, - total=self.total_time, - percent=self.progress)) - if logged_in: client = self._provider.get_client(self._context) logged_in = self._provider.is_logged_in() @@ -213,26 +199,32 @@ def run(self): segment_end = self.current_time refresh_only = True + play_data = { + 'play_count': play_count, + 'total_time': self.total_time, + 'played_time': self.current_time, + 'played_percent': self.progress, + } + self.playback_data['play_data'] = play_data + if logged_in and report_url: client.update_watch_history( self._context, self.video_id, report_url, - status=(segment_end, - segment_end, - segment_end, - state), + status=(segment_end, segment_end, segment_end, 'stopped'), ) if use_local_history: - play_data = { - 'play_count': play_count, - 'total_time': self.total_time, - 'played_time': self.current_time, - 'played_percent': self.progress, - } self._context.get_playback_history().update(self.video_id, play_data) + self._context.send_notification('PlaybackStopped', self.playback_data) + self._context.log_debug('Playback stopped [{video_id}]:' + ' {played_time:.3f} secs of {total_time:.3f}' + ' @ {played_percent}%,' + ' played {play_count} time(s)' + .format(video_id=self.video_id, **play_data)) + if refresh_only: pass elif settings.get_bool('youtube.playlist.watchlater.autoremove', True): @@ -362,17 +354,17 @@ def onAVStarted(self): if not self._ui.busy_dialog_active(): self._ui.clear_property(BUSY_FLAG) - playback_json = self._ui.get_property(PLAYER_DATA) - if not playback_json: + playback_data = self._ui.get_property(PLAYER_DATA) + if not playback_data: return self._ui.clear_property(PLAYER_DATA) self.cleanup_threads() - playback_json = json.loads(playback_json) + playback_data = json.loads(playback_data) try: - self.seek_time = float(playback_json.get('seek_time')) - self.start_time = float(playback_json.get('start_time')) - self.end_time = float(playback_json.get('end_time')) + self.seek_time = float(playback_data.get('seek_time')) + self.start_time = float(playback_data.get('start_time')) + self.end_time = float(playback_data.get('end_time')) self.current_time = max(0.0, self.getTime()) self.total_time = max(0.0, self.getTotalTime()) except (ValueError, TypeError, RuntimeError): @@ -386,7 +378,7 @@ def onAVStarted(self): self._provider, self._context, self._monitor, - playback_json)) + playback_data)) def onPlayBackEnded(self): if not self._ui.busy_dialog_active(): diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_play.py b/resources/lib/youtube_plugin/youtube/helper/yt_play.py index d09675bee..787f69113 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_play.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_play.py @@ -131,7 +131,7 @@ def play_video(provider, context): play_count = use_play_data and video_item.get_play_count() or 0 playback_stats = video_stream.get('playback_stats') - playback_json = { + playback_data = { 'video_id': video_id, 'channel_id': metadata.get('channel', {}).get('id', ''), 'video_status': video_details.get('status', {}), @@ -143,16 +143,12 @@ def play_video(provider, context): 'seek_time': seek_time, 'start_time': start_time, 'end_time': end_time, - 'clip': params.get('clip'), + 'clip': params.get('clip', False), 'refresh_only': screensaver } - ui.set_property(PLAYER_DATA, json.dumps(playback_json, ensure_ascii=False)) - context.send_notification('PlaybackInit', { - 'video_id': video_id, - 'channel_id': playback_json.get('channel_id', ''), - 'status': playback_json.get('video_status', {}) - }) + ui.set_property(PLAYER_DATA, json.dumps(playback_data, ensure_ascii=False)) + context.send_notification('PlaybackInit', playback_data) return video_item From c312169aabea0104888e36a22270fbd3379a8059 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 15:38:48 +1000 Subject: [PATCH 09/53] Remove container refresh via script --- resources/lib/youtube_plugin/kodion/script_actions.py | 4 ---- .../youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py | 11 +++-------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/script_actions.py b/resources/lib/youtube_plugin/kodion/script_actions.py index af32894cb..f091c6481 100644 --- a/resources/lib/youtube_plugin/kodion/script_actions.py +++ b/resources/lib/youtube_plugin/kodion/script_actions.py @@ -333,10 +333,6 @@ def run(argv): xbmcaddon.Addon().openSettings() return - if action == 'refresh': - xbmc.executebuiltin('Container.Refresh') - return - if action == 'play_with': ui.set_property(SWITCH_PLAYER_FLAG, 'true') xbmc.executebuiltin('Action(Play)') diff --git a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py index 93c8c6ebe..18e63037b 100644 --- a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py +++ b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py @@ -13,7 +13,7 @@ from .xbmc_progress_dialog import XbmcProgressDialog, XbmcProgressDialogBG from ..abstract_context_ui import AbstractContextUI from ...compatibility import xbmc, xbmcgui -from ...constants import ADDON_ID +from ...constants import ADDON_ID, REFRESH_CONTAINER from ...utils import to_unicode @@ -138,13 +138,8 @@ def show_notification(self, def open_settings(self): self._xbmc_addon.openSettings() - @staticmethod - def refresh_container(): - # TODO: find out why the RunScript call is required - # xbmc.executebuiltin("Container.Refresh") - xbmc.executebuiltin('RunScript({addon_id},action/refresh)'.format( - addon_id=ADDON_ID - )) + def refresh_container(self): + self._context.send_notification(REFRESH_CONTAINER, True) @staticmethod def set_property(property_id, value): From 8a8b7d65758129de74bf9347f6ecb21d96fd2f7c Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 22:27:59 +1000 Subject: [PATCH 10/53] Set default value for XbmcContextUI.set_property --- .../lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py | 2 +- .../lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py | 2 +- resources/lib/youtube_plugin/kodion/script_actions.py | 4 ++-- resources/lib/youtube_plugin/kodion/service_runner.py | 4 ++-- .../lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py index 893f3bb3d..bc65f3e06 100644 --- a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py +++ b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py @@ -713,5 +713,5 @@ def tear_down(self): self._audio_player = None def wakeup(self): - self.get_ui().set_property(WAKEUP, 'true') + self.get_ui().set_property(WAKEUP) self.send_notification(WAKEUP, True) diff --git a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py index 5cb82e1f0..a95f35a35 100644 --- a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py +++ b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py @@ -202,7 +202,7 @@ def _set_resolved_url(self, context, base_item): if base_item.playable: ui = context.get_ui() if not context.is_plugin_path(uri) and ui.busy_dialog_active(): - ui.set_property(BUSY_FLAG, 'true') + ui.set_property(BUSY_FLAG) playlist = XbmcPlaylist('auto', context) position, _ = playlist.get_position() items = playlist.get_items() diff --git a/resources/lib/youtube_plugin/kodion/script_actions.py b/resources/lib/youtube_plugin/kodion/script_actions.py index f091c6481..136e72e7b 100644 --- a/resources/lib/youtube_plugin/kodion/script_actions.py +++ b/resources/lib/youtube_plugin/kodion/script_actions.py @@ -311,7 +311,7 @@ def switch_to_user(user): def run(argv): context = XbmcContext() ui = context.get_ui() - ui.set_property(WAIT_FLAG, 'true') + ui.set_property(WAIT_FLAG) try: category = action = params = None args = argv[1:] @@ -334,7 +334,7 @@ def run(argv): return if action == 'play_with': - ui.set_property(SWITCH_PLAYER_FLAG, 'true') + ui.set_property(SWITCH_PLAYER_FLAG) xbmc.executebuiltin('Action(Play)') return diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index aadb2f17b..ec715923c 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -49,7 +49,7 @@ def run(): waited = 0 if waited >= 30: monitor.shutdown_httpd() - ui.set_property(SLEEPING, 'true') + ui.set_property(SLEEPING) elif waited >= ping_period: waited = 0 if monitor.ping_httpd(): @@ -69,7 +69,7 @@ def run(): break waited += wait_interval - ui.set_property(ABORT_FLAG, 'true') + ui.set_property(ABORT_FLAG) # clean up any/all playback monitoring threads player.cleanup_threads(only_ended=False) diff --git a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py index 18e63037b..9def04591 100644 --- a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py +++ b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py @@ -142,7 +142,7 @@ def refresh_container(self): self._context.send_notification(REFRESH_CONTAINER, True) @staticmethod - def set_property(property_id, value): + def set_property(property_id, value='true'): property_id = '-'.join((ADDON_ID, property_id)) xbmcgui.Window(10000).setProperty(property_id, value) From 8d10c1a0b9d3b04f5b0d2a294cd61f4cf31e9c4f Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 22:33:30 +1000 Subject: [PATCH 11/53] Fix setting infolabels for listitems created from a DirectoryItem or ImageItem --- .../kodion/items/xbmc/xbmc_items.py | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index 269b6e450..2efe09b08 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -110,9 +110,18 @@ def set_info(list_item, item, properties, resume=True): list_item.setInfo('video', info_labels) elif isinstance(item, DirectoryItem): + info_labels = {} + + value = item.get_name() + if value is not None: + info_labels['title'] = value + value = item.get_plot() if value is not None: - list_item.setInfo('picture', {'plot': value}) + info_labels['plot'] = value + + if info_labels: + list_item.setInfo('video', info_labels) if properties: list_item.setProperties(properties) @@ -169,6 +178,7 @@ def set_info(list_item, item, properties, resume=True): properties['TotalTime'] = str(duration) if is_video: list_item.addStreamInfo('video', {'duration': duration}) + if properties: list_item.setProperties(properties) return @@ -269,15 +279,21 @@ def set_info(list_item, item, properties, resume=True): elif isinstance(item, DirectoryItem): info_tag = list_item.getVideoInfoTag() + value = item.get_name() + if value is not None: + info_tag.setTitle(value) + value = item.get_plot() if value is not None: info_tag.setPlot(value) return elif isinstance(item, ImageItem): + info_tag = list_item.getPictureInfoTag() + value = item.get_title() if value is not None: - list_item.setInfo('picture', {'title': value}) + info_tag.setTitle(value) return elif isinstance(item, AudioItem): From ef0493fa2b078fb92137d22ea9f97d95f0c9ae86 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 22:38:50 +1000 Subject: [PATCH 12/53] Misc tidy ups --- .../youtube_plugin/kodion/plugin_runner.py | 7 +++--- .../youtube_plugin/kodion/service_runner.py | 25 +++++++++++++------ .../kodion/ui/xbmc/xbmc_context_ui.py | 1 - 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/plugin_runner.py b/resources/lib/youtube_plugin/kodion/plugin_runner.py index 4f3eabbf7..e0edf7e7b 100644 --- a/resources/lib/youtube_plugin/kodion/plugin_runner.py +++ b/resources/lib/youtube_plugin/kodion/plugin_runner.py @@ -14,8 +14,8 @@ from copy import deepcopy from platform import python_version -from .plugin import XbmcPlugin from .context import XbmcContext +from .plugin import XbmcPlugin from ..youtube import Provider @@ -50,13 +50,14 @@ def run(context=_context, if key in params: params[key] = '' - context.log_notice('Running: {plugin} ({version}) on {kodi} with {python}\n' + context.log_notice('Running: {plugin} ({version})' + ' on {kodi} with Python {python}\n' 'Path: {path}\n' 'Params: {params}' .format(plugin=context.get_name(), version=context.get_version(), kodi=context.get_system_version(), - python='Python {0}'.format(python_version()), + python=python_version(), path=context.get_path(), params=params)) diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index ec715923c..ff1c07842 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -23,8 +23,17 @@ def run(): context = XbmcContext() context.log_debug('YouTube service initialization...') + + get_infobool = context.get_infobool + get_infolabel = context.get_infolabel + get_listitem_detail = context.get_listitem_detail + ui = context.get_ui() - ui.clear_property(ABORT_FLAG) + clear_property = ui.clear_property + get_property = ui.get_property + set_property = ui.set_property + + clear_property(ABORT_FLAG) monitor = ServiceMonitor() player = PlayerMonitor(provider=Provider(), @@ -40,16 +49,16 @@ def run(): while not monitor.abortRequested(): if not monitor.httpd: if (monitor.httpd_required() - and not context.get_infobool('System.IdleTime(10)')): + and not get_infobool('System.IdleTime(10)')): monitor.start_httpd() waited = 0 - elif context.get_infobool('System.IdleTime(10)'): - if ui.get_property(WAKEUP): - ui.clear_property(WAKEUP) + elif get_infobool('System.IdleTime(10)'): + if get_property(WAKEUP): + clear_property(WAKEUP) waited = 0 if waited >= 30: monitor.shutdown_httpd() - ui.set_property(SLEEPING) + set_property(SLEEPING) elif waited >= ping_period: waited = 0 if monitor.ping_httpd(): @@ -60,7 +69,7 @@ def run(): else: monitor.shutdown_httpd() - if context.get_infolabel('Container.FolderPath').startswith(plugin_url): + if get_infolabel('Container.FolderPath').startswith(plugin_url): wait_interval = 1 else: wait_interval = 10 @@ -69,7 +78,7 @@ def run(): break waited += wait_interval - ui.set_property(ABORT_FLAG) + set_property(ABORT_FLAG) # clean up any/all playback monitoring threads player.cleanup_threads(only_ended=False) diff --git a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py index 9def04591..eb98b8cb1 100644 --- a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py +++ b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py @@ -31,7 +31,6 @@ def create_progress_dialog(self, heading, text=None, background=False): return XbmcProgressDialog(heading, text) - def on_keyboard_input(self, title, default='', hidden=False): # Starting with Gotham (13.X > ...) dialog = xbmcgui.Dialog() From 41cbeeb9e297e6bde3a60e19d03e2f639493e64e Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 9 May 2024 22:43:35 +1000 Subject: [PATCH 13/53] Enable support for Kodi internal mark as watched #709 --- .../kodion/constants/__init__.py | 4 ++ .../kodion/items/xbmc/xbmc_items.py | 47 ++++++++++++++----- .../kodion/plugin/xbmc/xbmc_plugin.py | 9 +++- .../youtube_plugin/kodion/plugin_runner.py | 5 +- .../youtube_plugin/kodion/service_runner.py | 39 ++++++++++++++- 5 files changed, 88 insertions(+), 16 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/constants/__init__.py b/resources/lib/youtube_plugin/kodion/constants/__init__.py index bd4511db8..1d30c094f 100644 --- a/resources/lib/youtube_plugin/kodion/constants/__init__.py +++ b/resources/lib/youtube_plugin/kodion/constants/__init__.py @@ -35,6 +35,7 @@ ABORT_FLAG = 'abort_requested' BUSY_FLAG = 'busy' CHECK_SETTINGS = 'check_settings' +PLAY_COUNT = 'video_play_count' PLAYER_DATA = 'player_json' PLAYLIST_PATH = 'playlist_path' PLAYLIST_POSITION = 'playlist_position' @@ -42,6 +43,7 @@ REROUTE = 'reroute' SLEEPING = 'sleeping' SWITCH_PLAYER_FLAG = 'switch_player' +VIDEO_ID = 'video_id' WAIT_FLAG = 'builtin_running' WAKEUP = 'wakeup' @@ -53,6 +55,7 @@ 'CHECK_SETTINGS', 'DATA_PATH', 'MEDIA_PATH', + 'PLAY_COUNT', 'PLAYER_DATA', 'PLAYLIST_PATH', 'PLAYLIST_POSITION', @@ -63,6 +66,7 @@ 'SWITCH_PLAYER_FLAG', 'TEMP_PATH', 'VALUE_FROM_STR', + 'VIDEO_ID', 'WAIT_FLAG', 'WAKEUP', 'content', diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index 2efe09b08..1df50cd1f 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -18,7 +18,7 @@ from ...utils import current_system_version, datetime_parser -def set_info(list_item, item, properties, resume=True): +def set_info(list_item, item, properties, set_play_count=True, resume=True): is_video = False if not current_system_version.compatible(20, 0): if isinstance(item, VideoItem): @@ -72,7 +72,9 @@ def set_info(list_item, item, properties, resume=True): value = item.get_play_count() if value is not None: - info_labels['playcount'] = value + if set_play_count: + info_labels['playcount'] = value + properties['play_count'] = value value = item.get_plot() if value is not None: @@ -183,9 +185,6 @@ def set_info(list_item, item, properties, resume=True): list_item.setProperties(properties) return - if properties: - list_item.setProperties(properties) - value = item.get_date(as_info_label=True) if value is not None: list_item.setDateTime(value) @@ -258,7 +257,9 @@ def set_info(list_item, item, properties, resume=True): # playcount: int value = item.get_play_count() if value is not None: - info_tag.setPlaycount(value) + if set_play_count: + info_tag.setPlaycount(value) + properties['play_count'] = value # plot: str value = item.get_plot() @@ -286,6 +287,9 @@ def set_info(list_item, item, properties, resume=True): value = item.get_plot() if value is not None: info_tag.setPlot(value) + + if properties: + list_item.setProperties(properties) return elif isinstance(item, ImageItem): @@ -294,6 +298,9 @@ def set_info(list_item, item, properties, resume=True): value = item.get_title() if value is not None: info_tag.setTitle(value) + + if properties: + list_item.setProperties(properties) return elif isinstance(item, AudioItem): @@ -354,6 +361,9 @@ def set_info(list_item, item, properties, resume=True): if value is not None: info_tag.setYear(value) + if properties: + list_item.setProperties(properties) + def video_playback_item(context, video_item, show_fanart=None, **_kwargs): uri = video_item.get_uri() @@ -454,7 +464,11 @@ def video_playback_item(context, video_item, show_fanart=None, **_kwargs): return list_item -def audio_listitem(context, audio_item, show_fanart=None, for_playback=False): +def audio_listitem(context, + audio_item, + show_fanart=None, + for_playback=False, + **_kwargs): uri = audio_item.get_uri() context.log_debug('Converting AudioItem |%s|' % uri) @@ -492,7 +506,7 @@ def audio_listitem(context, audio_item, show_fanart=None, for_playback=False): return uri, list_item, False -def directory_listitem(context, directory_item, show_fanart=None): +def directory_listitem(context, directory_item, show_fanart=None, **_kwargs): uri = directory_item.get_uri() context.log_debug('Converting DirectoryItem |%s|' % uri) @@ -549,7 +563,7 @@ def directory_listitem(context, directory_item, show_fanart=None): return uri, list_item, is_folder -def image_listitem(context, image_item, show_fanart=None): +def image_listitem(context, image_item, show_fanart=None, **_kwargs): uri = image_item.get_uri() context.log_debug('Converting ImageItem |%s|' % uri) @@ -602,7 +616,11 @@ def uri_listitem(context, uri_item, **_kwargs): return list_item -def video_listitem(context, video_item, show_fanart=None): +def video_listitem(context, + video_item, + show_fanart=None, + focused=None, + **_kwargs): uri = video_item.get_uri() context.log_debug('Converting VideoItem |%s|' % uri) @@ -633,6 +651,13 @@ def video_listitem(context, video_item, show_fanart=None): context, local_datetime )) + set_play_count = True + prop_value = video_item.video_id + if prop_value: + if focused and focused == prop_value: + set_play_count = False + props['video_id'] = prop_value + # make channel_id property available for keymapping prop_value = video_item.get_channel_id() if prop_value: @@ -665,7 +690,7 @@ def video_listitem(context, video_item, show_fanart=None): if video_item.subtitles: list_item.setSubtitles(video_item.subtitles) - set_info(list_item, video_item, props) + set_info(list_item, video_item, props, set_play_count=set_play_count) context_menu = video_item.get_context_menu() if context_menu: diff --git a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py index a95f35a35..c264f0b81 100644 --- a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py +++ b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py @@ -21,6 +21,7 @@ PLAYLIST_POSITION, REROUTE, SLEEPING, + VIDEO_ID, ) from ...exceptions import KodionException from ...items import ( @@ -57,7 +58,7 @@ def __init__(self): super(XbmcPlugin, self).__init__() self.handle = None - def run(self, provider, context): + def run(self, provider, context, refresh=False): self.handle = context.get_handle() ui = context.get_ui() @@ -162,12 +163,16 @@ def run(self, provider, context): )) ui.on_ok("Error in ContentProvider", exc.__str__()) + focused = ui.get_property(VIDEO_ID) if refresh else None item_count = 0 if isinstance(result, (list, tuple)): show_fanart = settings.fanart_selection() result = [ self._LIST_ITEM_MAP[item.__class__.__name__]( - context, item, show_fanart=show_fanart + context, + item, + show_fanart=show_fanart, + focused=focused, ) for item in result if item.__class__.__name__ in self._LIST_ITEM_MAP diff --git a/resources/lib/youtube_plugin/kodion/plugin_runner.py b/resources/lib/youtube_plugin/kodion/plugin_runner.py index e0edf7e7b..0df9b4bc9 100644 --- a/resources/lib/youtube_plugin/kodion/plugin_runner.py +++ b/resources/lib/youtube_plugin/kodion/plugin_runner.py @@ -43,7 +43,10 @@ def run(context=_context, profiler.enable(flush=True) context.log_debug('Starting Kodion framework by bromix...') + + current_uri = context.get_uri() context.init() + new_uri = context.get_uri() params = deepcopy(context.get_params()) for key in ('api_key', 'client_id', 'client_secret'): @@ -61,7 +64,7 @@ def run(context=_context, path=context.get_path(), params=params)) - plugin.run(provider, context) + plugin.run(provider, context, new_uri == current_uri) if profiler: profiler.print_stats() diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index ff1c07842..a5a2e2176 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -10,7 +10,15 @@ from __future__ import absolute_import, division, unicode_literals -from .constants import ABORT_FLAG, ADDON_ID, SLEEPING, TEMP_PATH, WAKEUP +from .constants import ( + ABORT_FLAG, + ADDON_ID, + PLAY_COUNT, + SLEEPING, + TEMP_PATH, + VIDEO_ID, + WAKEUP, +) from .context import XbmcContext from .monitors import PlayerMonitor, ServiceMonitor from .utils import rm_dir @@ -46,6 +54,7 @@ def run(): ping_period = waited = 60 restart_attempts = 0 plugin_url = 'plugin://{0}/'.format(ADDON_ID) + video_id = None while not monitor.abortRequested(): if not monitor.httpd: if (monitor.httpd_required() @@ -70,7 +79,33 @@ def run(): monitor.shutdown_httpd() if get_infolabel('Container.FolderPath').startswith(plugin_url): - wait_interval = 1 + new_video_id = get_listitem_detail('video_id') + if not new_video_id: + video_id = None + if get_listitem_detail('Label', True): + clear_property(VIDEO_ID) + clear_property(PLAY_COUNT) + elif video_id != new_video_id: + video_id = new_video_id + set_property(VIDEO_ID, video_id) + plugin_play_count = get_listitem_detail('play_count') + set_property(PLAY_COUNT, plugin_play_count) + else: + kodi_play_count = get_listitem_detail('PlayCount', True) + kodi_play_count = int(kodi_play_count or 0) + plugin_play_count = get_property(PLAY_COUNT) + plugin_play_count = int(plugin_play_count or 0) + if kodi_play_count != plugin_play_count: + playback_history = context.get_playback_history() + play_data = playback_history.get_item(video_id) + if not play_data: + play_data = {'play_count': kodi_play_count} + playback_history.update(video_id, play_data) + elif play_data.get('play_count') != kodi_play_count: + play_data['play_count'] = kodi_play_count + playback_history.update(video_id, play_data) + set_property(PLAY_COUNT, str(kodi_play_count)) + wait_interval = 0.5 else: wait_interval = 10 From d76e74dd6b20933e2bba10a27d810a3545888dfa Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 10 May 2024 10:07:41 +1000 Subject: [PATCH 14/53] Fix bug that could lead to plugin continuously reloading #608 --- resources/lib/youtube_plugin/youtube/client/__config__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/youtube/client/__config__.py b/resources/lib/youtube_plugin/youtube/client/__config__.py index 618b005ab..3e954298a 100644 --- a/resources/lib/youtube_plugin/youtube/client/__config__.py +++ b/resources/lib/youtube_plugin/youtube/client/__config__.py @@ -59,7 +59,7 @@ def __init__(self, context): j_id = self._json_api['keys']['personal'].get('client_id', '') j_secret = self._json_api['keys']['personal'].get('client_secret', '') - if (not original_key or not original_id or not original_secret + if ((not original_key or not original_id or not original_secret) and j_key and j_secret and j_id): settings.api_key(j_key) settings.api_id(j_id) From 38454fa063261bee16448a0a72c02e3a207d08ed Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 10 May 2024 10:20:16 +1000 Subject: [PATCH 15/53] Reset resume point when using Kodi internal mark as watched #709 --- .../lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py | 8 +++++++- resources/lib/youtube_plugin/kodion/service_runner.py | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index 1df50cd1f..d0b3a6bcf 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -652,10 +652,12 @@ def video_listitem(context, )) set_play_count = True + resume = True prop_value = video_item.video_id if prop_value: if focused and focused == prop_value: set_play_count = False + resume = False props['video_id'] = prop_value # make channel_id property available for keymapping @@ -690,7 +692,11 @@ def video_listitem(context, if video_item.subtitles: list_item.setSubtitles(video_item.subtitles) - set_info(list_item, video_item, props, set_play_count=set_play_count) + set_info(list_item, + video_item, + props, + set_play_count=set_play_count, + resume=resume) context_menu = video_item.get_context_menu() if context_menu: diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index a5a2e2176..1743928af 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -103,6 +103,8 @@ def run(): playback_history.update(video_id, play_data) elif play_data.get('play_count') != kodi_play_count: play_data['play_count'] = kodi_play_count + play_data['played_time'] = 0.0 + play_data['played_percent'] = 0 playback_history.update(video_id, play_data) set_property(PLAY_COUNT, str(kodi_play_count)) wait_interval = 0.5 From 90536d9c8d1f1676e25bfa3a0989611c981acdde Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sat, 11 May 2024 07:12:04 +1000 Subject: [PATCH 16/53] Add website link to video descriptions #721 --- resources/lib/youtube_plugin/youtube/helper/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index 184fa90ca..320e7a0ea 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -588,6 +588,7 @@ def update_video_infos(provider, context, video_id_dict, (ui.italic(start_at, cr_after=1) if video_item.upcoming else ui.new_line(start_at, cr_after=1)) if start_at else '', description, + ui.new_line('https://youtu.be/' + video_id, cr_before=1) )) video_item.set_plot(description) From fa5e4e13c6b0342d6a600457ef7833a15309db51 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sat, 11 May 2024 09:23:10 +1000 Subject: [PATCH 17/53] Update changelog --- changelog.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/changelog.txt b/changelog.txt index 21d95efe9..07d3a395c 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,7 +1,17 @@ ## v7.0.7+beta.2 ### Fixed - Fix invalid pageToken error by removing jump from unsupported listings #715 +- Fix issues with post play refresh possibly causing loss of window history +- Fix bug that could lead to plugin continuously reloading #608 +### Changed +- Use internal Kodi resume enable/disable for all playback #693 + - For playback where Kodi does not prompt to resume, the plugin will also no longer resume + - For playback where Kodi does prompt to resume, the plugin will follow whatever is selected + +### New +- Enable syncing of plugin watched state with Kodi watched state when using Kodi mark as (un)watched #709 +- Add website link to video descriptions #721 ## v7.0.7+beta.1 ### Fixed From 155a801bc91535d263141d1dd661d27bbf54f809 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sat, 11 May 2024 21:42:51 +1000 Subject: [PATCH 18/53] Attempted fix for socket error 98 on Linux #746 - May help with error 10013 on Windows, otherwise try either of the following: - Start-Process powershell -Verb runAs -ArgumentList "net stop winnat; net start winnat" - Start-Process powershell -Verb runAs -ArgumentList "net stop hns; net start hns" --- .../kodion/compatibility/__init__.py | 9 ++++++--- .../kodion/monitors/service_monitor.py | 1 - .../youtube_plugin/kodion/network/http_server.py | 15 ++++++++++++--- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/compatibility/__init__.py b/resources/lib/youtube_plugin/kodion/compatibility/__init__.py index b63f5ea75..7306563e3 100644 --- a/resources/lib/youtube_plugin/kodion/compatibility/__init__.py +++ b/resources/lib/youtube_plugin/kodion/compatibility/__init__.py @@ -8,7 +8,8 @@ """ __all__ = ( - 'BaseHTTPServer', + 'BaseHTTPRequestHandler', + 'TCPServer', 'byte_string_type', 'datetime_infolabel', 'parse_qs', @@ -32,7 +33,8 @@ # Kodi v19+ and Python v3.x try: from html import unescape - from http import server as BaseHTTPServer + from http.server import BaseHTTPRequestHandler + from socketserver import TCPServer from urllib.parse import ( parse_qs, parse_qsl, @@ -58,8 +60,9 @@ to_str = str # Compatibility shims for Kodi v18 and Python v2.7 except ImportError: - import BaseHTTPServer + from BaseHTTPServer import BaseHTTPRequestHandler from contextlib import contextmanager as _contextmanager + from SocketServer import TCPServer from urllib import ( quote as _quote, unquote as _unquote, diff --git a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py index b5d998b22..6d65e86e0 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py @@ -152,7 +152,6 @@ def start_httpd(self): return self.httpd_thread = threading.Thread(target=self.httpd.serve_forever) - self.httpd_thread.daemon = True self.httpd_thread.start() address = self.httpd.socket.getsockname() diff --git a/resources/lib/youtube_plugin/kodion/network/http_server.py b/resources/lib/youtube_plugin/kodion/network/http_server.py index 903660ed3..3bc77a64d 100644 --- a/resources/lib/youtube_plugin/kodion/network/http_server.py +++ b/resources/lib/youtube_plugin/kodion/network/http_server.py @@ -18,7 +18,8 @@ from .requests import BaseRequestsClass from ..compatibility import ( - BaseHTTPServer, + BaseHTTPRequestHandler, + TCPServer, parse_qs, urlsplit, xbmc, @@ -42,7 +43,7 @@ _server_requests = BaseRequestsClass() -class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler, object): +class RequestHandler(BaseHTTPRequestHandler, object): BASE_PATH = xbmcvfs.translatePath(TEMP_PATH) chunk_size = 1024 * 64 local_ranges = ( @@ -548,7 +549,11 @@ class Pages(object): def get_http_server(address, port): try: - server = BaseHTTPServer.HTTPServer((address, port), RequestHandler) + server = TCPServer((address, port), RequestHandler, False) + server.allow_reuse_address = True + server.allow_reuse_port = True + server.server_bind() + server.server_activate() return server except socket.error as exc: log_error('HTTPServer: Failed to start |{address}:{port}| |{response}|' @@ -601,6 +606,10 @@ def get_connect_address(as_netloc=False): sock = None try: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + if hasattr(socket, "SO_REUSEADDR"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + if hasattr(socket, "SO_REUSEPORT"): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) except socket.error: address = xbmc.getIPAddress() From 60fdd74978ad682ba51231d73ed316e12ddd6b63 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:00:15 +1000 Subject: [PATCH 19/53] Identify json response data that is not part of API response with underscore prefix --- .../youtube_plugin/youtube/client/youtube.py | 50 +++++++++---------- .../youtube/helper/resource_manager.py | 6 +-- .../lib/youtube_plugin/youtube/provider.py | 6 +-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 85b3890b6..654b6d4d3 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -573,7 +573,7 @@ def get_recommended_for_home(self, { 'kind': 'youtube#video', 'id': video['videoId'], - 'partial': True, + '_partial': True, 'snippet': { 'title': self.json_traverse(video, ( ('title', 'runs', 0, 'text'), @@ -652,7 +652,7 @@ def get_related_for_home(self, page_token=''): # Fetch existing list of items, if any cache = self._context.get_data_cache() - cache_items_key = 'get-activities-home-items' + cache_items_key = 'get-activities-home-items-v2' cached = cache.get_item(cache_items_key, None) or [] # Increase value to recursively retrieve recommendations for the first @@ -682,13 +682,13 @@ def index_items(items, index, for idx, item in enumerate(items): if original_related is not None: - related = item['related_video_id'] = original_related + related = item['_related_video_id'] = original_related else: - related = item['related_video_id'] + related = item['_related_video_id'] if original_channel is not None: - channel = item['related_channel_id'] = original_channel + channel = item['_related_channel_id'] = original_channel else: - channel = item['related_channel_id'] + channel = item['_related_channel_id'] video_id = item['id'] index['_related'].setdefault(related, 0) @@ -696,15 +696,15 @@ def index_items(items, index, if video_id in index: item_count = index[video_id] - item_count['related'].setdefault(related, 0) - item_count['related'][related] += 1 - item_count['channels'].setdefault(channel, 0) - item_count['channels'][channel] += 1 + item_count['_related'].setdefault(related, 0) + item_count['_related'][related] += 1 + item_count['_channels'].setdefault(channel, 0) + item_count['_channels'][channel] += 1 continue index[video_id] = { - 'related': {related: 1}, - 'channels': {channel: 1} + '_related': {related: 1}, + '_channels': {channel: 1} } if item_store is None: @@ -720,7 +720,7 @@ def index_items(items, index, group = 0 num_stored = len(item_store[group]) - item['order'] = items_per_page * group + num_stored + item['_order'] = items_per_page * group + num_stored item_store[group].append(item) if num_stored or depth <= 1: @@ -793,18 +793,18 @@ def threaded_get_related(video_id, func, *args, **kwargs): # Finally sort items per page by rank and date for a better distribution def rank_and_sort(item): - if 'order' not in item: + if '_order' not in item: counts['_counter'] += 1 - item['order'] = counts['_counter'] + item['_order'] = counts['_counter'] - page = 1 + item['order'] // (items_per_page * max_depth) + page = 1 + item['_order'] // (items_per_page * max_depth) page_count = counts['_pages'].setdefault(page, {'_counter': 0}) while page_count['_counter'] < items_per_page and page > 1: page -= 1 page_count = counts['_pages'].setdefault(page, {'_counter': 0}) - related_video = item['related_video_id'] - related_channel = item['related_channel_id'] + related_video = item['_related_video_id'] + related_channel = item['_related_channel_id'] channel_id = item.get('snippet', {}).get('channelId') """ # Video channel and related channel can be the same which can double @@ -833,16 +833,16 @@ def rank_and_sort(item): page_count.setdefault(channel_id, 0) page_count[channel_id] += 1 page_count['_counter'] += 1 - item['page'] = page + item['_page'] = page item_count = counts[item['id']] - item['rank'] = (2 * sum(item_count['channels'].values()) - + sum(item_count['related'].values())) + item['_rank'] = (2 * sum(item_count['_channels'].values()) + + sum(item_count['_related'].values())) return ( - -item['page'], - item['rank'], - -randint(0, item['order']) + -item['_page'], + item['_rank'], + -randint(0, item['_order']) ) items.sort(key=rank_and_sort, reverse=True) @@ -1208,7 +1208,7 @@ def get_related_videos(self, 'id': video['videoId'], 'related_video_id': video_id, 'related_channel_id': channel_id, - 'partial': True, + '_partial': True, 'snippet': { 'title': self.json_traverse(video, path=( 'title', diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index f1e1a9afe..af1e0c12b 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -63,7 +63,7 @@ def get_channels(self, ids, defer_cache=False): else: result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) to_update = [id_ for id_ in ids - if id_ not in result or result[id_].get('partial')] + if id_ not in result or result[id_].get('_partial')] if result: self._context.log_debug('Found cached data for channels:\n|{ids}|' @@ -134,7 +134,7 @@ def get_playlists(self, ids, defer_cache=False): else: result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) to_update = [id_ for id_ in ids - if id_ not in result or result[id_].get('partial')] + if id_ not in result or result[id_].get('_partial')] if result: self._context.log_debug('Found cached data for playlists:\n|{ids}|' @@ -279,7 +279,7 @@ def get_videos(self, else: result = self._data_cache.get_items(ids, self._data_cache.ONE_MONTH) to_update = [id_ for id_ in ids - if id_ not in result or result[id_].get('partial')] + if id_ not in result or result[id_].get('_partial')] if result: self._context.log_debug('Found cached data for videos:\n|{ids}|' diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 54d2a77e2..7ff1b1057 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -1016,7 +1016,7 @@ def on_playback_history(self, context, re_match): { 'kind': 'youtube#video', 'id': video_id, - 'partial': True, + '_partial': True, } for video_id in items.keys() ] @@ -1407,7 +1407,7 @@ def on_bookmarks(self, context, re_match): { 'kind': 'youtube#channel', 'id': item_id, - 'partial': True, + '_partial': True, } for item_id, item in items.items() if isinstance(item, float) @@ -1483,7 +1483,7 @@ def on_watch_later(self, context, re_match): { 'kind': 'youtube#video', 'id': video_id, - 'partial': True, + '_partial': True, } for video_id in items.keys() ] From 7d43631fe68120943c728bda4e5582757ff2b84a Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:02:04 +1000 Subject: [PATCH 20/53] Attempt to generate thumbnail urls for sources that don't provide them #751 --- resources/lib/youtube_plugin/youtube/helper/v3.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/v3.py b/resources/lib/youtube_plugin/youtube/helper/v3.py index 22a92e827..67937e626 100644 --- a/resources/lib/youtube_plugin/youtube/helper/v3.py +++ b/resources/lib/youtube_plugin/youtube/helper/v3.py @@ -13,6 +13,7 @@ from threading import Thread from .utils import ( + THUMB_TYPES, filter_short_videos, get_thumbnail, make_comment_item, @@ -71,7 +72,16 @@ def _process_list_response(provider, context, json_data): snippet = yt_item.get('snippet', {}) title = snippet.get('title', context.localize('untitled')) - thumbnails = snippet.get('thumbnails', {}) + thumbnails = snippet.get('thumbnails') + if not thumbnails and yt_item.get('_partial'): + thumbnails = { + thumb_type: { + 'url': thumb['url'].format(item_id, ''), + 'size': thumb['size'], + 'ratio': thumb['ratio'], + } + for thumb_type, thumb in THUMB_TYPES.items() + } image = get_thumbnail(thumb_size, thumbnails) fanart = get_thumbnail(fanart_type, thumbnails) if fanart_type else None From 89e3040567b627e125921528f30fadbf84b30cd0 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:07:48 +1000 Subject: [PATCH 21/53] Use v3 API processing for My Subscriptions listings #751 --- .../youtube_plugin/youtube/client/youtube.py | 284 ++++++++++-------- .../lib/youtube_plugin/youtube/helper/tv.py | 67 ----- .../youtube/helper/yt_specials.py | 9 +- 3 files changed, 154 insertions(+), 206 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 654b6d4d3..2d9584ea3 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -1425,167 +1425,185 @@ def search(self, **kwargs) def get_my_subscriptions(self, - page_token=None, - offset=0, + page_token=1, logged_in=False, + do_filter=False, **kwargs): """ modified by PureHemp, using YouTube RSS for fetching latest videos """ - result = { + v3_response = { + 'kind': 'youtube#videoListResponse', 'items': [], } - def _perform(_page_token, _offset, _result): + cache = self._context.get_data_cache() + settings = self._context.get_settings() - if not _result: - _result = { - 'items': [] + filter_list = [] + black_list = False + if do_filter: + black_list = settings.get_bool( + 'youtube.filter.my_subscriptions_filtered.blacklist', False + ) + filter_list = settings.get_string( + 'youtube.filter.my_subscriptions_filtered.list', '' + ).replace(', ', ',').split(',') + filter_list = {filter_item.lower() for filter_item in filter_list} + + # if new uploads is cached + cache_items_key = 'my-subscriptions-items-v2' + cached = cache.get_item(cache_items_key, cache.ONE_HOUR) or [] + if cached: + items = cached + # no cache, get uploads data from web + else: + sub_channel_ids = [] + + if logged_in: + params = { + 'part': 'snippet', + 'maxResults': '50', + 'order': 'alphabetical', + 'mine': 'true' } - cache = self._context.get_data_cache() - - # if new uploads is cached - cache_items_key = 'my-subscriptions-items' - cached = cache.get_item(cache_items_key, cache.ONE_HOUR) or [] - if cached: - _result['items'] = cached + while 1: + json_data = self.api_request(method='GET', + path='subscriptions', + params=params, + **kwargs) + if not json_data: + break - """ no cache, get uploads data from web """ - if not _result['items']: - sub_channel_ids = [] - - if logged_in: - params = { - 'part': 'snippet', - 'maxResults': '50', - 'order': 'alphabetical', - 'mine': 'true' - } - - while 1: - json_data = self.api_request(method='GET', - path='subscriptions', - params=params, - **kwargs) - if not json_data: - break - - sub_channel_ids.extend([ - item['snippet']['resourceId']['channelId'] - for item in json_data.get('items', []) - ]) - - # get next token if exists - sub_page_token = json_data.get('nextPageToken') - if sub_page_token: - params['pageToken'] = sub_page_token - # terminate loop when last page - else: - break - - items = self._context.get_bookmarks_list().get_items() - if items: sub_channel_ids.extend([ - item_id - for item_id, item in items.items() - if (item_id not in sub_channel_ids - and (isinstance(item, float) - or getattr(item, 'get_channel_id', bool)())) + item['snippet']['resourceId']['channelId'] + for item in json_data.get('items', []) ]) - headers = { - 'Host': 'www.youtube.com', - 'Connection': 'keep-alive', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'DNT': '1', - 'Accept-Encoding': 'gzip, deflate', - 'Accept-Language': 'en-US,en;q=0.7,de;q=0.3' - } - - responses = [] - - def fetch_xml(_url, _responses): - _response = self.request(_url, headers=headers) - if _response: - _responses.append(_response) - - threads = [] - for channel_id in sub_channel_ids: - thread = threading.Thread( - target=fetch_xml, - args=('https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id, - responses) - ) - threads.append(thread) - thread.start() - - for thread in threads: - thread.join(30) + # get next token if exists + sub_page_token = json_data.get('nextPageToken') + if sub_page_token: + params['pageToken'] = sub_page_token + # terminate loop when last page + else: + break + + items = self._context.get_bookmarks_list().get_items() + if items: + sub_channel_ids.extend([ + item_id + for item_id, item in items.items() + if (item_id not in sub_channel_ids + and (isinstance(item, float) + or getattr(item, 'get_channel_id', bool)())) + ]) + + headers = { + 'Host': 'www.youtube.com', + 'Connection': 'keep-alive', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'DNT': '1', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-US,en;q=0.7,de;q=0.3' + } - do_encode = not current_system_version.compatible(19, 0) + def fetch_xml(_channel_id, _responses): + _response = self.request( + 'https://www.youtube.com/feeds/videos.xml?channel_id=' + + _channel_id, + headers=headers, + ) + if _response: + _responses.append(_response) - ns = { - 'atom': 'http://www.w3.org/2005/Atom', - 'yt': 'http://www.youtube.com/xml/schemas/2015', - 'media': 'http://search.yahoo.com/mrss/', - } + responses = [] + threads = [] + for channel_id in sub_channel_ids: + thread = threading.Thread( + target=fetch_xml, + args=(channel_id, responses) + ) + threads.append(thread) + thread.start() - for response in responses: - if not response: - continue - - response.encoding = 'utf-8' - xml_data = to_unicode(response.content) - xml_data = xml_data.replace('\n', '') - if do_encode: - xml_data = to_str(xml_data) - - root = ET.fromstring(xml_data) - _result['items'].extend([{ - 'id': entry.find('yt:videoId', ns).text, - 'title': entry.find('media:group/media:title', ns).text, - 'channel': entry.find('atom:author/atom:name', ns).text, - 'published': entry.find('atom:published', ns).text, - } for entry in root.findall('atom:entry', ns)]) - - # sorting by publish date - def _sort_by_date_time(item): - return datetime_parser.since_epoch( - datetime_parser.strptime(item['published']) - ) + for thread in threads: + thread.join(30) - _result['items'].sort(reverse=True, key=_sort_by_date_time) + do_encode = not current_system_version.compatible(19, 0) - # Update cache - cache.set_item(cache_items_key, _result['items']) - """ no cache, get uploads data from web """ + ns = { + 'atom': 'http://www.w3.org/2005/Atom', + 'yt': 'http://www.youtube.com/xml/schemas/2015', + 'media': 'http://search.yahoo.com/mrss/', + } - # trim result - if not _page_token: - _page_token = 0 + items = [] + for response in responses: + if not response: + continue - if len(_result['items']) > self._max_results: - start = _page_token * self._max_results - end = start + self._max_results - _result['items'] = _result['items'][start:end] - _result['next_page_token'] = _page_token + 1 + response.encoding = 'utf-8' + xml_data = to_unicode(response.content) + xml_data = xml_data.replace('\n', '') + if do_encode: + xml_data = to_str(xml_data) - if len(_result['items']) < self._max_results: - if 'continue' in _result: - del _result['continue'] + root = ET.fromstring(xml_data) + items.extend([{ + 'kind': 'youtube#video', + 'id': entry.find('yt:videoId', ns).text, + 'snippet': { + 'title': entry.find('atom:title', ns).text, + 'channelId': entry.find('yt:channelId', ns).text, + }, + '_channel': (entry.find('atom:author/atom:name', ns).text + .lower().replace(',', '')), + '_timestamp': datetime_parser.since_epoch( + datetime_parser.strptime( + entry.find('atom:published', ns).text + ) + ), + '_partial': True, + } for entry in root.findall('atom:entry', ns)]) - if 'next_page_token' in _result: - del _result['next_page_token'] + # Update cache + cache.set_item(cache_items_key, items) - if 'offset' in _result: - del _result['offset'] + # filter, sorting by publish date and trim + page = page_token or 1 - return _result + limits = { + 'num': 0, + 'start': -self._max_results, + 'end': page * self._max_results, + } + limits['start'] += limits['end'] + + def _sort_by_date_time(item, limits=limits): + if do_filter: + filtered = item['_channel'] in filter_list + if black_list: + if filtered: + return -1 + elif not filtered: + return -1 + limits['num'] += 1 + return item['_timestamp'] + + items.sort(reverse=True, key=_sort_by_date_time) + + if limits['num'] > limits['end']: + v3_response['nextPageToken'] = page + 1 + if limits['num'] > limits['start']: + items = items[limits['start']:min(limits['num'], limits['end'])] + else: + items = [] - return _perform(_page_token=page_token, _offset=offset, _result=result) + v3_response['items'] = items + return v3_response def get_saved_playlists(self, page_token, offset): if not page_token: diff --git a/resources/lib/youtube_plugin/youtube/helper/tv.py b/resources/lib/youtube_plugin/youtube/helper/tv.py index bc101beb7..e12ca60a5 100644 --- a/resources/lib/youtube_plugin/youtube/helper/tv.py +++ b/resources/lib/youtube_plugin/youtube/helper/tv.py @@ -14,73 +14,6 @@ from ...kodion.items import DirectoryItem, NextPageItem, VideoItem -def my_subscriptions_to_items(provider, context, json_data, do_filter=False): - settings = context.get_settings() - result = [] - video_id_dict = {} - - filter_list = [] - black_list = False - if do_filter: - black_list = settings.get_bool( - 'youtube.filter.my_subscriptions_filtered.blacklist', False - ) - filter_list = settings.get_string( - 'youtube.filter.my_subscriptions_filtered.list', '' - ) - filter_list = { - filter_item.lower() - for filter_item in filter_list.replace(', ', ',').split(',') - } - - item_params = {'video_id': None} - incognito = context.get_param('incognito', False) - if incognito: - item_params['incognito'] = incognito - - items = json_data.get('items', []) - for item in items: - channel = item['channel'].lower().replace(',', '') - if (not do_filter - or (black_list and channel not in filter_list) - or (not black_list and channel in filter_list)): - video_id = item['id'] - item_params['video_id'] = video_id - item_uri = context.create_uri(('play',), item_params) - video_item = VideoItem(item['title'], uri=item_uri) - if incognito: - video_item.set_play_count(0) - result.append(video_item) - - video_id_dict[video_id] = video_item - - use_play_data = not incognito and settings.use_local_history() - - channel_item_dict = {} - utils.update_video_infos(provider, - context, - video_id_dict, - channel_items_dict=channel_item_dict, - use_play_data=use_play_data) - utils.update_fanarts(provider, context, channel_item_dict) - - if context.get_settings().hide_short_videos(): - result = utils.filter_short_videos(result) - - # next page - next_page_token = json_data.get('next_page_token') - if next_page_token or json_data.get('continue'): - params = context.get_params() - new_params = dict(params, - next_page_token=next_page_token, - offset=json_data.get('offset', 0), - page=params.get('page', 1) + 1) - next_page_item = NextPageItem(context, new_params) - result.append(next_page_item) - - return result - - def tv_videos_to_items(provider, context, json_data): result = [] video_id_dict = {} diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py index 8705b2fac..94ef3a27b 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py @@ -296,17 +296,14 @@ def _process_new_uploaded_videos_tv(provider, context, filtered=False): context.set_content(content.VIDEO_CONTENT) json_data = provider.get_client(context).get_my_subscriptions( - page_token=context.get_param('next_page_token', 0), - offset=context.get_param('offset', 0), + page_token=context.get_param('page'), logged_in=provider.is_logged_in(), + do_filter=filtered, ) if not json_data: return False - return tv.my_subscriptions_to_items(provider, - context, - json_data, - do_filter=filtered) + return v3.response_to_items(provider, context, json_data) def process(category, provider, context): From 3eb347f49db83cd76f012972824e108a8929a6a1 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:09:00 +1000 Subject: [PATCH 22/53] Misc tidy ups --- .../kodion/plugin/xbmc/xbmc_plugin.py | 2 +- .../youtube_plugin/youtube/client/youtube.py | 2 +- .../youtube/helper/yt_specials.py | 2 +- .../youtube/helper/yt_subscriptions.py | 25 ++++++------------- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py index c264f0b81..3e76f863a 100644 --- a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py +++ b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py @@ -161,7 +161,7 @@ def run(self, provider, context, refresh=False): context.log_error('XbmcRunner.run - {exc}:\n{details}'.format( exc=exc, details=''.join(format_stack()) )) - ui.on_ok("Error in ContentProvider", exc.__str__()) + ui.on_ok('Error in ContentProvider', exc.__str__()) focused = ui.get_property(VIDEO_ID) if refresh else None item_count = 0 diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 2d9584ea3..b0f9e91e7 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -1204,7 +1204,7 @@ def get_related_videos(self, related_videos = chain.from_iterable(related_videos) items = [{ - 'kind': "youtube#video", + 'kind': 'youtube#video', 'id': video['videoId'], 'related_video_id': video_id, 'related_channel_id': channel_id, diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py index 94ef3a27b..31ead8c03 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_specials.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_specials.py @@ -167,7 +167,7 @@ def _extract_urls(video_id): url_resolver = UrlResolver(context) with context.get_ui().create_progress_dialog( - heading=context.localize('please_wait'), background=False + heading=context.localize('please_wait'), background=False ) as progress_dialog: resource_manager = provider.get_resource_manager(context) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_subscriptions.py b/resources/lib/youtube_plugin/youtube/helper/yt_subscriptions.py index 348ab5217..d5a638448 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_subscriptions.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_subscriptions.py @@ -16,16 +16,12 @@ def _process_list(provider, context): - result = [] - - page_token = context.get_param('page_token', '') - # no caching - json_data = provider.get_client(context).get_subscription('mine', page_token=page_token) + json_data = provider.get_client(context).get_subscription( + 'mine', page_token=context.get_param('page_token', '') + ) if not json_data: return [] - result.extend(v3.response_to_items(provider, context, json_data)) - - return result + return v3.response_to_items(provider, context, json_data) def _process_add(provider, context): @@ -84,20 +80,15 @@ def _process_remove(provider, context): def process(method, provider, context): - result = [] - # we need a login _ = provider.get_client(context) if not provider.is_logged_in(): return UriItem(context.create_uri(('sign', 'in'))) if method == 'list': - result.extend(_process_list(provider, context)) - elif method == 'add': + return _process_list(provider, context) + if method == 'add': return _process_add(provider, context) - elif method == 'remove': + if method == 'remove': return _process_remove(provider, context) - else: - raise KodionException("Unknown subscriptions method '%s'" % method) - - return result + raise KodionException("Unknown subscriptions method '%s'" % method) From ac71f4925aa6d16a31218d35f7d3fd9074be38d4 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:18:32 +1000 Subject: [PATCH 23/53] Fix not deleting unused api key from client params --- resources/lib/youtube_plugin/youtube/client/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index b0f9e91e7..8d16ea98b 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -1856,7 +1856,7 @@ def api_request(self, if key: client['params']['key'] = key else: - client['params']['key'] + del client['params']['key'] if method != 'POST' and 'json' in client: del client['json'] From fc0b944b65d0db4e9eefd415fc9e398bb4e154b5 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 12 May 2024 08:39:24 +1000 Subject: [PATCH 24/53] Update changelog --- changelog.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/changelog.txt b/changelog.txt index 07d3a395c..20ded5779 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,11 +3,17 @@ - Fix invalid pageToken error by removing jump from unsupported listings #715 - Fix issues with post play refresh possibly causing loss of window history - Fix bug that could lead to plugin continuously reloading #608 +- Attempted fix for socket error 98 on Linux #746 + - For error 10013 on Windows try either of the following: + - Start-Process powershell -Verb runAs -ArgumentList "net stop winnat; net start winnat" + - Start-Process powershell -Verb runAs -ArgumentList "net stop hns; net start hns" +- Fix changing fanart type on My Subscriptions #751 ### Changed - Use internal Kodi resume enable/disable for all playback #693 - For playback where Kodi does not prompt to resume, the plugin will also no longer resume - For playback where Kodi does prompt to resume, the plugin will follow whatever is selected +- Cached data for My Subscriptions and Related Videos will be replaced due to changes in cached data ### New - Enable syncing of plugin watched state with Kodi watched state when using Kodi mark as (un)watched #709 From 39c3a9c083ea444254a99e581b7525b59b1038d9 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 00:16:26 +1000 Subject: [PATCH 25/53] Allow checking whether non-default fanart has been set for an item --- resources/lib/youtube_plugin/kodion/items/base_item.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/base_item.py b/resources/lib/youtube_plugin/kodion/items/base_item.py index f3dadfc22..4ba56b601 100644 --- a/resources/lib/youtube_plugin/kodion/items/base_item.py +++ b/resources/lib/youtube_plugin/kodion/items/base_item.py @@ -108,7 +108,6 @@ def get_image(self): def set_fanart(self, fanart): if not fanart: - self._fanart = '{0}/fanart.jpg'.format(MEDIA_PATH) return if '{media}/' in fanart: @@ -116,8 +115,10 @@ def set_fanart(self, fanart): else: self._fanart = fanart - def get_fanart(self): - return self._fanart + def get_fanart(self, default=True): + if self._fanart or not default: + return self._fanart + return '{0}/fanart.jpg'.format(MEDIA_PATH) def add_context_menu(self, context_menu, position='end', replace=False): context_menu = (item for item in context_menu if item) From 6958849353aa05cf37508da640c10f8009d53fa9 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 00:27:27 +1000 Subject: [PATCH 26/53] Optimise and simplify setting channel fanart --- .../youtube/helper/resource_manager.py | 7 +++--- .../youtube_plugin/youtube/helper/utils.py | 23 +++++-------------- .../lib/youtube_plugin/youtube/provider.py | 5 ---- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index af1e0c12b..2cff49cf9 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -116,10 +116,9 @@ def get_fanarts(self, channel_ids, defer_cache=False): images = item.get('brandingSettings', {}).get('image', {}) for banner in banners: image = images.get(banner) - if not image: - continue - result[key] = image - break + if image: + result[key] = image + break else: # set an empty url result[key] = '' diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index 320e7a0ea..ea808b20c 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -173,11 +173,6 @@ def update_channel_infos(provider, context, channel_id_dict, in_subscription_list = False thumb_size = settings.get_thumbnail_size() - banners = [ - 'bannerTvMediumImageUrl', - 'bannerTvLowImageUrl', - 'bannerTvImageUrl' - ] for channel_id, yt_item in data.items(): channel_item = channel_id_dict[channel_id] @@ -235,13 +230,6 @@ def update_channel_infos(provider, context, channel_id_dict, if context_menu: channel_item.add_context_menu(context_menu, replace=True) - fanart_images = yt_item.get('brandingSettings', {}).get('image', {}) - for banner in banners: - fanart = fanart_images.get(banner) - if fanart: - channel_item.set_fanart(fanart) - break - # update channel mapping if channel_items_dict is not None: if channel_id not in channel_items_dict: @@ -842,11 +830,12 @@ def update_fanarts(provider, context, channel_items_dict, data=None): return for channel_id, channel_items in channel_items_dict.items(): - for channel_item in channel_items: - # only set not empty fanarts - fanart = data.get(channel_id, '') - if fanart: - channel_item.set_fanart(fanart) + # only set not empty fanarts + fanart = data.get(channel_id) + if not fanart: + continue + for item in channel_items: + item.set_fanart(fanart) THUMB_TYPES = { diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 7ff1b1057..81a37b1b7 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -717,11 +717,6 @@ def _on_subscriptions(self, context, re_match): if method == 'list': context.set_content(content.LIST_CONTENT) - channel_ids = {subscription.get_channel_id(): subscription - for subscription in subscriptions} - channel_fanarts = resource_manager.get_fanarts(channel_ids) - for channel_id, fanart in channel_fanarts.items(): - channel_ids[channel_id].set_fanart(fanart) return subscriptions From ae35e019cf4d3bcf6bcdd373fed1e388f975425e Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 00:35:39 +1000 Subject: [PATCH 27/53] Fix not using thumbnail fanart for channels/subscriptions/playlists when enabled #751 --- resources/lib/youtube_plugin/youtube/helper/v3.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resources/lib/youtube_plugin/youtube/helper/v3.py b/resources/lib/youtube_plugin/youtube/helper/v3.py index 67937e626..4d29e8e08 100644 --- a/resources/lib/youtube_plugin/youtube/helper/v3.py +++ b/resources/lib/youtube_plugin/youtube/helper/v3.py @@ -110,6 +110,7 @@ def _process_list_response(provider, context, json_data): item = DirectoryItem(title, item_uri, image=image, + fanart=fanart, channel_id=item_id) channel_id_dict[item_id] = item @@ -133,6 +134,7 @@ def _process_list_response(provider, context, json_data): item = DirectoryItem(title, item_uri, image=image, + fanart=fanart, channel_id=item_id, subscription_id=subscription_id) channel_id_dict[item_id] = item @@ -150,6 +152,7 @@ def _process_list_response(provider, context, json_data): item = DirectoryItem(title, item_uri, image=image, + fanart=fanart, playlist_id=item_id) playlist_id_dict[item_id] = item From 26df4b128a999d2682227fa4388cbc885238ae64 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 07:25:26 +1000 Subject: [PATCH 28/53] Use channel fanart as backup to thumb fanart for channel items without thumbs #751 --- .../youtube/helper/resource_manager.py | 6 ++- .../youtube_plugin/youtube/helper/utils.py | 13 +++++- .../lib/youtube_plugin/youtube/helper/v3.py | 5 ++- .../lib/youtube_plugin/youtube/provider.py | 40 ++++++++++++------- 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index 2cff49cf9..cb4963087 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -100,8 +100,10 @@ def get_channels(self, ids, defer_cache=False): return result - def get_fanarts(self, channel_ids, defer_cache=False): - if self._fanart_type != self._context.get_settings().FANART_CHANNEL: + def get_fanarts(self, channel_ids, force=False, defer_cache=False): + if force: + pass + elif self._fanart_type != self._context.get_settings().FANART_CHANNEL: return {} result = self.get_channels(channel_ids, defer_cache=defer_cache) diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index ea808b20c..ebf7ac28e 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -824,18 +824,27 @@ def update_fanarts(provider, context, channel_items_dict, data=None): if not data: resource_manager = provider.get_resource_manager(context) - data = resource_manager.get_fanarts(channel_ids) + data = resource_manager.get_fanarts(channel_ids, force=True) if not data: return + settings = context.get_settings() + fanart_type = context.get_param('fanart_type') + if fanart_type is None: + fanart_type = settings.fanart_selection() + use_channel_fanart = fanart_type == settings.FANART_CHANNEL + use_thumb_fanart = fanart_type == settings.FANART_THUMBNAIL + for channel_id, channel_items in channel_items_dict.items(): # only set not empty fanarts fanart = data.get(channel_id) if not fanart: continue for item in channel_items: - item.set_fanart(fanart) + if (use_channel_fanart + or use_thumb_fanart and not item.get_fanart(default=False)): + item.set_fanart(fanart) THUMB_TYPES = { diff --git a/resources/lib/youtube_plugin/youtube/helper/v3.py b/resources/lib/youtube_plugin/youtube/helper/v3.py index 4d29e8e08..b48204223 100644 --- a/resources/lib/youtube_plugin/youtube/helper/v3.py +++ b/resources/lib/youtube_plugin/youtube/helper/v3.py @@ -283,7 +283,10 @@ def _process_list_response(provider, context, json_data): 4: { 'fetcher': resource_manager.get_fanarts, 'args': (channel_items_dict,), - 'kwargs': {'defer_cache': True}, + 'kwargs': { + 'force': bool(channel_id_dict or playlist_id_dict), + 'defer_cache': True, + }, 'thread': None, 'updater': update_fanarts, 'upd_args': ( diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 81a37b1b7..1f5bbe2c7 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -352,12 +352,9 @@ def _on_playlist(self, context, re_match): @RegisterProviderPath('^/channel/(?P[^/]+)/playlists/?$') def _on_channel_playlists(self, context, re_match): context.set_content(content.LIST_CONTENT) - result = [] channel_id = re_match.group('channel_id') - resource_manager = self.get_resource_manager(context) - params = context.get_params() page_token = params.get('page_token', '') incognito = params.get('incognito') @@ -369,27 +366,38 @@ def _on_channel_playlists(self, context, re_match): if addon_id: new_params['addon_id'] = addon_id + resource_manager = self.get_resource_manager(context) + fanart = resource_manager.get_fanarts( + (channel_id,), force=True + ).get(channel_id) playlists = resource_manager.get_related_playlists(channel_id) - uploads_playlist = playlists.get('uploads', '') - if uploads_playlist: + + uploads = playlists.get('uploads') + if uploads: item_label = context.localize('uploads') - uploads_item = DirectoryItem( + uploads = DirectoryItem( context.get_ui().bold(item_label), context.create_uri( - ('channel', channel_id, 'playlist', uploads_playlist), + ('channel', channel_id, 'playlist', uploads), new_params, ), image='{media}/playlist.png', + fanart=fanart, category_label=item_label, ) - result.append(uploads_item) + result = [uploads] + else: + result = False - # no caching - json_data = self.get_client(context).get_playlists_of_channel(channel_id, page_token) + json_data = self.get_client(context).get_playlists_of_channel( + channel_id, page_token + ) if not json_data: - return False - result.extend(v3.response_to_items(self, context, json_data)) + return result + if not result: + result = [] + result.extend(v3.response_to_items(self, context, json_data)) return result """ @@ -484,7 +492,9 @@ def _on_channel(self, context, re_match): if method == 'user': return False - channel_fanarts = resource_manager.get_fanarts((channel_id,)) + fanart = resource_manager.get_fanarts( + (channel_id,), force=True + ).get(channel_id) page = params.get('page', 1) page_token = params.get('page_token', '') @@ -513,7 +523,7 @@ def _on_channel(self, context, re_match): new_params, ), image='{media}/playlist.png', - fanart=channel_fanarts.get(channel_id), + fanart=fanart, category_label=item_label, ) result.append(playlists_item) @@ -523,6 +533,7 @@ def _on_channel(self, context, re_match): search_item = NewSearchItem( context, name=ui.bold(localize('search')), image='{media}/search.png', + fanart=fanart, channel_id=search_live_id, incognito=incognito, addon_id=addon_id, @@ -535,6 +546,7 @@ def _on_channel(self, context, re_match): ui.bold(item_label), create_uri(('channel', search_live_id, 'live'), new_params), image='{media}/live.png', + fanart=fanart, category_label=item_label, ) result.append(live_item) From 76b29546f91fe778a8d7cc0ed1929a54ea212d6c Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 12:33:44 +1000 Subject: [PATCH 29/53] Fix My Subscriptions threading issues #529 --- .../youtube_plugin/youtube/client/youtube.py | 144 ++++++++++++------ 1 file changed, 97 insertions(+), 47 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 8d16ea98b..084fe850b 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -14,6 +14,7 @@ import xml.etree.ElementTree as ET from copy import deepcopy from itertools import chain, islice +from os import cpu_count from random import randint from .login_client import LoginClient @@ -1478,7 +1479,7 @@ def get_my_subscriptions(self, break sub_channel_ids.extend([ - item['snippet']['resourceId']['channelId'] + (item['snippet']['resourceId']['channelId'],) for item in json_data.get('items', []) ]) @@ -1493,81 +1494,125 @@ def get_my_subscriptions(self, items = self._context.get_bookmarks_list().get_items() if items: sub_channel_ids.extend([ - item_id + (item_id,) for item_id, item in items.items() - if (item_id not in sub_channel_ids - and (isinstance(item, float) - or getattr(item, 'get_channel_id', bool)())) + if (isinstance(item, float) + or getattr(item, 'get_channel_id', bool)()) ]) headers = { 'Host': 'www.youtube.com', 'Connection': 'keep-alive', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' + ' AppleWebKit/537.36 (KHTML, like Gecko)' + ' Chrome/87.0.4280.66 Safari/537.36', + 'Accept': 'text/html,' + 'application/xhtml+xml,' + 'application/xml;q=0.9,' + 'image/webp,*/*;q=0.8', 'DNT': '1', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US,en;q=0.7,de;q=0.3' } - - def fetch_xml(_channel_id, _responses): - _response = self.request( - 'https://www.youtube.com/feeds/videos.xml?channel_id=' - + _channel_id, - headers=headers, - ) - if _response: - _responses.append(_response) - - responses = [] - threads = [] - for channel_id in sub_channel_ids: - thread = threading.Thread( - target=fetch_xml, - args=(channel_id, responses) - ) - threads.append(thread) - thread.start() - - for thread in threads: - thread.join(30) - - do_encode = not current_system_version.compatible(19, 0) - - ns = { + namespaces = { 'atom': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015', 'media': 'http://search.yahoo.com/mrss/', } - items = [] - for response in responses: + def _feed_items(channel_id, + encode=not current_system_version.compatible(19, 0), + headers=headers, + ns=namespaces): + response = self.request( + 'https://www.youtube.com/feeds/videos.xml?channel_id=' + + channel_id, + headers=headers, + ) if not response: - continue + return None response.encoding = 'utf-8' xml_data = to_unicode(response.content) xml_data = xml_data.replace('\n', '') - if do_encode: + if encode: xml_data = to_str(xml_data) root = ET.fromstring(xml_data) - items.extend([{ + channel_name = (root.findtext('atom:title', '', ns) + .lower().replace(',', '')) + return [{ 'kind': 'youtube#video', - 'id': entry.find('yt:videoId', ns).text, + 'id': item.findtext('yt:videoId', '', ns), 'snippet': { - 'title': entry.find('atom:title', ns).text, - 'channelId': entry.find('yt:channelId', ns).text, + 'title': item.findtext('atom:title', '', ns), + 'channelId': channel_id, }, - '_channel': (entry.find('atom:author/atom:name', ns).text - .lower().replace(',', '')), + '_channel': channel_name, '_timestamp': datetime_parser.since_epoch( datetime_parser.strptime( - entry.find('atom:published', ns).text + item.findtext('atom:published', '', ns) ) ), '_partial': True, - } for entry in root.findall('atom:entry', ns)]) + } for item in root.findall('atom:entry', ns)] + + def _threaded_fetch(args, + kwargs, + output, + worker, + input_lock, + output_lock): + while 1: + if input_lock.acquire(blocking=False): + _args = args.pop() if args else [] + _kwargs = kwargs.pop() if kwargs else {} + if not _args and not _kwargs: + break + input_lock.release() + else: + continue + + try: + _output = worker(*_args, **_kwargs) + except Exception as exc: + self._context.log_error('threaded_fetch error: |{exc}|' + .format(exc=exc)) + continue + if _output and output_lock.acquire(blocking=True): + output.extend(_output) + output_lock.release() + input_lock.release() + + payload = { + 'args': list(set(sub_channel_ids)), + 'kwargs': None, + 'output': [], + 'worker': _feed_items, + 'input_lock': threading.Lock(), + 'output_lock': threading.Lock(), + } + + threads = [] + num_threads = 0 + max_threads = min(32, (cpu_count() or 1) + 4) + while (payload['args'] + or payload['kwargs'] + or payload['output_lock'].locked()): + if num_threads >= max_threads: + continue + thread = threading.Thread( + target=_threaded_fetch, + kwargs=payload, + ) + threads.append(thread) + num_threads += 1 + thread.start() + + for thread in threads: + thread.join(30) + + items = payload['output'] # Update cache cache.set_item(cache_items_key, items) @@ -1579,10 +1624,11 @@ def fetch_xml(_channel_id, _responses): 'num': 0, 'start': -self._max_results, 'end': page * self._max_results, + 'video_ids': set(), } limits['start'] += limits['end'] - def _sort_by_date_time(item, limits=limits): + def _sort_by_date_time(item, _limits=limits): if do_filter: filtered = item['_channel'] in filter_list if black_list: @@ -1590,7 +1636,11 @@ def _sort_by_date_time(item, limits=limits): return -1 elif not filtered: return -1 - limits['num'] += 1 + video_id = item['id'] + if video_id in _limits['video_ids']: + return -1 + _limits['num'] += 1 + _limits['video_ids'].add(video_id) return item['_timestamp'] items.sort(reverse=True, key=_sort_by_date_time) From f71ca6d422fbf768846a4e523ea8c739e75b7c12 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 07:40:47 +1000 Subject: [PATCH 30/53] Update changelog --- changelog.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/changelog.txt b/changelog.txt index 20ded5779..f6b85cfd8 100644 --- a/changelog.txt +++ b/changelog.txt @@ -8,12 +8,15 @@ - Start-Process powershell -Verb runAs -ArgumentList "net stop winnat; net start winnat" - Start-Process powershell -Verb runAs -ArgumentList "net stop hns; net start hns" - Fix changing fanart type on My Subscriptions #751 +- Fix not using thumbnail fanart for channels/subscriptions/playlists when enabled #751 +- Fix My Subscriptions threading issues #529 ### Changed - Use internal Kodi resume enable/disable for all playback #693 - For playback where Kodi does not prompt to resume, the plugin will also no longer resume - For playback where Kodi does prompt to resume, the plugin will follow whatever is selected - Cached data for My Subscriptions and Related Videos will be replaced due to changes in cached data +- Use channel fanart as backup to thumbnail fanart for channel items without thumbnails #751 ### New - Enable syncing of plugin watched state with Kodi watched state when using Kodi mark as (un)watched #709 From e46bf5de5e81c125125b7b04cacecb34d7e8b2fc Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 16:35:32 +1000 Subject: [PATCH 31/53] Attempt to make Python strptime bug workaround thread safe --- .../kodion/utils/datetime_parser.py | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/utils/datetime_parser.py b/resources/lib/youtube_plugin/kodion/utils/datetime_parser.py index f8d62dc84..402af6d2c 100644 --- a/resources/lib/youtube_plugin/kodion/utils/datetime_parser.py +++ b/resources/lib/youtube_plugin/kodion/utils/datetime_parser.py @@ -14,6 +14,7 @@ from datetime import date, datetime, time as dt_time, timedelta from importlib import import_module from sys import modules +from threading import Condition, Lock from ..exceptions import KodionException from ..logger import log_error @@ -278,18 +279,32 @@ def strptime(datetime_str, fmt=None): try: return datetime.strptime(datetime_str, fmt) except TypeError: - log_error('Python strptime bug workaround.\n' - 'Refer to https://github.com/python/cpython/issues/71587') - - if '_strptime' not in modules: - modules['_strptime'] = import_module('_strptime') - _strptime = modules['_strptime'] + if '_strptime' not in modules or strptime.reloading.locked(): + if strptime.reloaded.acquire(blocking=False): + _strptime = import_module('_strptime') + modules['_strptime'] = _strptime + log_error('Python strptime bug workaround - ' + 'https://github.com/python/cpython/issues/71587') + strptime.reloaded.notify_all() + strptime.reloaded.release() + else: + strptime.reloaded.acquire() + while '_strptime' not in modules: + strptime.reloaded.wait() + _strptime = modules['_strptime'] + strptime.reloaded.release() + else: + _strptime = modules['_strptime'] if timezone: return _strptime._strptime_datetime(datetime, datetime_str, fmt) return datetime(*(_strptime._strptime(datetime_str, fmt)[0][0:6])) +strptime.reloading = Lock() +strptime.reloaded = Condition(lock=strptime.reloading) + + def since_epoch(dt_object=None): if dt_object is None: dt_object = now(tz=timezone.utc) if timezone else datetime.utcnow() From 855ed0777deafe726e283c4fb2e4a47763e8b962 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Mon, 13 May 2024 21:15:42 +1000 Subject: [PATCH 32/53] Fix building incorrect client details --- .../lib/youtube_plugin/youtube/helper/video_info.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/video_info.py b/resources/lib/youtube_plugin/youtube/helper/video_info.py index 89482d022..107692584 100644 --- a/resources/lib/youtube_plugin/youtube/helper/video_info.py +++ b/resources/lib/youtube_plugin/youtube/helper/video_info.py @@ -757,7 +757,8 @@ def _get_player_page(self, client_name='web', embed=False): # Manually configured cookies to avoid cookie consent redirect cookies = {'SOCS': 'CAISAiAD'} - client = self.build_client(client_name) + client_data = {'json': {'videoId': self.video_id}} + client = self.build_client(client_name, client_data) result = self.request( url, @@ -840,7 +841,8 @@ def _get_player_js(self): return cached client_name = 'web' - client = self.build_client(client_name) + client_data = {'json': {'videoId': self.video_id}} + client = self.build_client(client_name, client_data) result = self.request( js_url, @@ -897,7 +899,8 @@ def _load_hls_manifest(self, url, live_type=None, meta_info=None, del headers['Authorization'] else: client_name = 'web' - headers = self.build_client(client_name)['headers'] + client_data = {'json': {'videoId': self.video_id}} + headers = self.build_client(client_name, client_data)['headers'] curl_headers = self._make_curl_headers(headers, cookies=None) result = self.request( @@ -977,7 +980,9 @@ def _create_stream_list(self, if 'Authorization' in headers: del headers['Authorization'] else: - headers = self.build_client('web')['headers'] + client_name = 'web' + client_data = {'json': {'videoId': self.video_id}} + headers = self.build_client(client_name, client_data)['headers'] curl_headers = self._make_curl_headers(headers, cookies=None) if meta_info is None: From 4a60437d323266074f73dece75c3397997728245 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Tue, 14 May 2024 21:10:51 +1000 Subject: [PATCH 33/53] Split XbmcContext.get_listitem_detail out into separate get_listitem_info method --- .../kodion/context/abstract_context.py | 6 +++++- .../kodion/context/xbmc/xbmc_context.py | 13 +++++++------ .../lib/youtube_plugin/kodion/service_runner.py | 5 +++-- .../youtube_plugin/youtube/helper/yt_playlist.py | 6 +++--- .../lib/youtube_plugin/youtube/helper/yt_video.py | 2 +- resources/lib/youtube_plugin/youtube/provider.py | 2 +- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/context/abstract_context.py b/resources/lib/youtube_plugin/kodion/context/abstract_context.py index c3855547b..e48964a1d 100644 --- a/resources/lib/youtube_plugin/kodion/context/abstract_context.py +++ b/resources/lib/youtube_plugin/kodion/context/abstract_context.py @@ -420,7 +420,11 @@ def get_infolabel(name): raise NotImplementedError() @staticmethod - def get_listitem_detail(detail_name, attr=False): + def get_listitem_detail(detail_name): + raise NotImplementedError() + + @staticmethod + def get_listitem_info(detail_name): raise NotImplementedError() def tear_down(self): diff --git a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py index bc65f3e06..fbcec130c 100644 --- a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py +++ b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py @@ -680,12 +680,13 @@ def get_infolabel(name): return xbmc.getInfoLabel(name) @staticmethod - def get_listitem_detail(detail_name, attr=False): - return xbmc.getInfoLabel( - 'Container.ListItem(0).{0}'.format(detail_name) - if attr else - 'Container.ListItem(0).Property({0})'.format(detail_name) - ) + def get_listitem_detail(detail_name): + return xbmc.getInfoLabel('Container.ListItem(0).Property({0})' + .format(detail_name)) + + @staticmethod + def get_listitem_info(detail_name): + return xbmc.getInfoLabel('Container.ListItem(0).' + detail_name) def tear_down(self): self._settings.flush() diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index 1743928af..533c5c26d 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -35,6 +35,7 @@ def run(): get_infobool = context.get_infobool get_infolabel = context.get_infolabel get_listitem_detail = context.get_listitem_detail + get_listitem_info = context.get_listitem_info ui = context.get_ui() clear_property = ui.clear_property @@ -82,7 +83,7 @@ def run(): new_video_id = get_listitem_detail('video_id') if not new_video_id: video_id = None - if get_listitem_detail('Label', True): + if get_listitem_info('Label'): clear_property(VIDEO_ID) clear_property(PLAY_COUNT) elif video_id != new_video_id: @@ -91,7 +92,7 @@ def run(): plugin_play_count = get_listitem_detail('play_count') set_property(PLAY_COUNT, plugin_play_count) else: - kodi_play_count = get_listitem_detail('PlayCount', True) + kodi_play_count = get_listitem_info('PlayCount') kodi_play_count = int(kodi_play_count or 0) plugin_play_count = get_property(PLAY_COUNT) plugin_play_count = int(plugin_play_count or 0) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py index 3ca9fce23..fb9bbe3ab 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py @@ -16,7 +16,7 @@ def _process_add_video(provider, context, keymap_action=False): - path = context.get_listitem_detail('FileNameAndPath', attr=True) + path = context.get_listitem_info('FileNameAndPath') client = provider.get_client(context) logged_in = provider.is_logged_in() @@ -67,7 +67,7 @@ def _process_add_video(provider, context, keymap_action=False): def _process_remove_video(provider, context): listitem_playlist_id = context.get_listitem_detail('playlist_id') listitem_playlist_item_id = context.get_listitem_detail('playlist_item_id') - listitem_title = context.get_listitem_detail('Title', attr=True) + listitem_title = context.get_listitem_info('Title') keymap_action = False params = context.get_params() @@ -150,7 +150,7 @@ def _process_remove_playlist(provider, context): def _process_select_playlist(provider, context): # Get listitem path asap, relies on listitems focus - path = context.get_listitem_detail('FileNameAndPath', attr=True) + path = context.get_listitem_info('FileNameAndPath') params = context.get_params() ui = context.get_ui() diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_video.py b/resources/lib/youtube_plugin/youtube/helper/yt_video.py index 4f88e9ac6..8560463a1 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_video.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_video.py @@ -16,7 +16,7 @@ def _process_rate_video(provider, context, re_match): - listitem_path = context.get_listitem_detail('FileNameAndPath', attr=True) + listitem_path = context.get_listitem_info('FileNameAndPath') ratings = ['like', 'dislike', 'none'] rating_param = context.get_param('rating', '') diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 1f5bbe2c7..25514775a 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -644,7 +644,7 @@ def on_play(self, context, re_match): if ({'channel_id', 'live', 'playlist_id', 'playlist_ids', 'video_id'} .isdisjoint(params.keys())): - path = context.get_listitem_detail('FileNameAndPath', attr=True) + path = context.get_listitem_info('FileNameAndPath') if context.is_plugin_path(path, 'play'): video_id = find_video_id(path) if video_id: From 83ef672712edc7c3dd0d64d3dfe6ac954a175056 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Tue, 14 May 2024 21:12:24 +1000 Subject: [PATCH 34/53] Fix YouTube.get_related_videos after change to underscore prefix on non-API response data --- resources/lib/youtube_plugin/youtube/client/youtube.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 084fe850b..2abb72a36 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -1207,8 +1207,8 @@ def get_related_videos(self, items = [{ 'kind': 'youtube#video', 'id': video['videoId'], - 'related_video_id': video_id, - 'related_channel_id': channel_id, + '_related_video_id': video_id, + '_related_channel_id': channel_id, '_partial': True, 'snippet': { 'title': self.json_traverse(video, path=( From 0dd7588c1b8c6f701520a8dd3fc46bb3baa4bb7d Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Tue, 14 May 2024 21:14:28 +1000 Subject: [PATCH 35/53] Improve responsiveness of using Kodi internal mark as watched #709 --- .../lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py | 6 +++--- resources/lib/youtube_plugin/kodion/service_runner.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index d0b3a6bcf..081d15e55 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -14,7 +14,7 @@ from .. import AudioItem, DirectoryItem, ImageItem, VideoItem from ...compatibility import xbmc, xbmcgui -from ...constants import SWITCH_PLAYER_FLAG +from ...constants import PLAY_COUNT, SWITCH_PLAYER_FLAG from ...utils import current_system_version, datetime_parser @@ -74,7 +74,7 @@ def set_info(list_item, item, properties, set_play_count=True, resume=True): if value is not None: if set_play_count: info_labels['playcount'] = value - properties['play_count'] = value + properties[PLAY_COUNT] = value value = item.get_plot() if value is not None: @@ -259,7 +259,7 @@ def set_info(list_item, item, properties, set_play_count=True, resume=True): if value is not None: if set_play_count: info_tag.setPlaycount(value) - properties['play_count'] = value + properties[PLAY_COUNT] = value # plot: str value = item.get_plot() diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index 533c5c26d..df2445cc7 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -89,7 +89,7 @@ def run(): elif video_id != new_video_id: video_id = new_video_id set_property(VIDEO_ID, video_id) - plugin_play_count = get_listitem_detail('play_count') + plugin_play_count = get_listitem_detail(PLAY_COUNT) set_property(PLAY_COUNT, plugin_play_count) else: kodi_play_count = get_listitem_info('PlayCount') @@ -108,7 +108,7 @@ def run(): play_data['played_percent'] = 0 playback_history.update(video_id, play_data) set_property(PLAY_COUNT, str(kodi_play_count)) - wait_interval = 0.5 + wait_interval = 0.1 else: wait_interval = 10 From 513a9f400748708b6480df18790b1c45f81fa6b2 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Tue, 14 May 2024 22:16:44 +1000 Subject: [PATCH 36/53] Optimise operations performed in _threaded_fetch for YouTube.get_my_subscriptions #529 --- .../youtube_plugin/youtube/client/youtube.py | 90 +++++++++---------- 1 file changed, 40 insertions(+), 50 deletions(-) diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 2abb72a36..3de53a9e6 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -26,6 +26,7 @@ datetime_parser, strip_html_from_text, to_unicode, + wait, ) @@ -1514,55 +1515,19 @@ def get_my_subscriptions(self, 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US,en;q=0.7,de;q=0.3' } - namespaces = { - 'atom': 'http://www.w3.org/2005/Atom', - 'yt': 'http://www.youtube.com/xml/schemas/2015', - 'media': 'http://search.yahoo.com/mrss/', - } - def _feed_items(channel_id, - encode=not current_system_version.compatible(19, 0), - headers=headers, - ns=namespaces): - response = self.request( + def _get_feed(channel_id, headers=headers): + return channel_id, self.request( 'https://www.youtube.com/feeds/videos.xml?channel_id=' + channel_id, headers=headers, ) - if not response: - return None - - response.encoding = 'utf-8' - xml_data = to_unicode(response.content) - xml_data = xml_data.replace('\n', '') - if encode: - xml_data = to_str(xml_data) - - root = ET.fromstring(xml_data) - channel_name = (root.findtext('atom:title', '', ns) - .lower().replace(',', '')) - return [{ - 'kind': 'youtube#video', - 'id': item.findtext('yt:videoId', '', ns), - 'snippet': { - 'title': item.findtext('atom:title', '', ns), - 'channelId': channel_id, - }, - '_channel': channel_name, - '_timestamp': datetime_parser.since_epoch( - datetime_parser.strptime( - item.findtext('atom:published', '', ns) - ) - ), - '_partial': True, - } for item in root.findall('atom:entry', ns)] def _threaded_fetch(args, kwargs, output, worker, - input_lock, - output_lock): + input_lock): while 1: if input_lock.acquire(blocking=False): _args = args.pop() if args else [] @@ -1571,34 +1536,31 @@ def _threaded_fetch(args, break input_lock.release() else: + wait(0.1) continue try: - _output = worker(*_args, **_kwargs) + key, _output = worker(*_args, **_kwargs) except Exception as exc: self._context.log_error('threaded_fetch error: |{exc}|' .format(exc=exc)) continue - if _output and output_lock.acquire(blocking=True): - output.extend(_output) - output_lock.release() + if _output: + output[key] = _output input_lock.release() payload = { 'args': list(set(sub_channel_ids)), 'kwargs': None, - 'output': [], - 'worker': _feed_items, + 'output': {}, + 'worker': _get_feed, 'input_lock': threading.Lock(), - 'output_lock': threading.Lock(), } threads = [] num_threads = 0 max_threads = min(32, (cpu_count() or 1) + 4) - while (payload['args'] - or payload['kwargs'] - or payload['output_lock'].locked()): + while payload['args'] or payload['kwargs']: if num_threads >= max_threads: continue thread = threading.Thread( @@ -1612,7 +1574,35 @@ def _threaded_fetch(args, for thread in threads: thread.join(30) - items = payload['output'] + namespaces = { + 'atom': 'http://www.w3.org/2005/Atom', + 'yt': 'http://www.youtube.com/xml/schemas/2015', + 'media': 'http://search.yahoo.com/mrss/', + } + encode = not current_system_version.compatible(19, 0) + items = [] + for channel_id, feed in payload['output'].items(): + feed.encoding = 'utf-8' + feed = to_unicode(feed.content).replace('\n', '') + + root = ET.fromstring(to_str(feed) if encode else feed) + channel_name = (root.findtext('atom:title', '', namespaces) + .lower().replace(',', '')) + items.extend([{ + 'kind': 'youtube#video', + 'id': item.findtext('yt:videoId', '', namespaces), + 'snippet': { + 'title': item.findtext('atom:title', '', namespaces), + 'channelId': channel_id, + }, + '_channel': channel_name, + '_timestamp': datetime_parser.since_epoch( + datetime_parser.strptime( + item.findtext('atom:published', '', namespaces) + ) + ), + '_partial': True, + } for item in root.findall('atom:entry', namespaces)]) # Update cache cache.set_item(cache_items_key, items) From 7c114e0e91362ee076a1b01806370d9ad38ad1d5 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Tue, 14 May 2024 22:51:45 +1000 Subject: [PATCH 37/53] Fix setting unicode localised dates as listiitem properties in Kodi 18 #754 --- .../lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py index 081d15e55..daa04372c 100644 --- a/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py +++ b/resources/lib/youtube_plugin/kodion/items/xbmc/xbmc_items.py @@ -13,7 +13,7 @@ from json import dumps from .. import AudioItem, DirectoryItem, ImageItem, VideoItem -from ...compatibility import xbmc, xbmcgui +from ...compatibility import to_str, xbmc, xbmcgui from ...constants import PLAY_COUNT, SWITCH_PLAYER_FLAG from ...utils import current_system_version, datetime_parser @@ -643,11 +643,11 @@ def video_listitem(context, local_datetime = None if datetime: local_datetime = datetime_parser.utc_to_local(datetime) - props['PublishedLocal'] = str(local_datetime) + props['PublishedLocal'] = to_str(local_datetime) if video_item.live: props['PublishedSince'] = context.localize('live') elif local_datetime: - props['PublishedSince'] = str(datetime_parser.datetime_to_since( + props['PublishedSince'] = to_str(datetime_parser.datetime_to_since( context, local_datetime )) From 534fe667832687109401060cc227d99c6a0ff398 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Wed, 15 May 2024 08:11:14 +1000 Subject: [PATCH 38/53] Further optimisations for YouTube.get_my_subscriptions #529 --- .../kodion/compatibility/__init__.py | 3 + .../youtube_plugin/youtube/client/youtube.py | 294 ++++++++++++------ 2 files changed, 196 insertions(+), 101 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/compatibility/__init__.py b/resources/lib/youtube_plugin/kodion/compatibility/__init__.py index 7306563e3..9e62f6373 100644 --- a/resources/lib/youtube_plugin/kodion/compatibility/__init__.py +++ b/resources/lib/youtube_plugin/kodion/compatibility/__init__.py @@ -11,6 +11,7 @@ 'BaseHTTPRequestHandler', 'TCPServer', 'byte_string_type', + 'cpu_count', 'datetime_infolabel', 'parse_qs', 'parse_qsl', @@ -35,6 +36,7 @@ from html import unescape from http.server import BaseHTTPRequestHandler from socketserver import TCPServer + from os import cpu_count from urllib.parse import ( parse_qs, parse_qsl, @@ -62,6 +64,7 @@ except ImportError: from BaseHTTPServer import BaseHTTPRequestHandler from contextlib import contextmanager as _contextmanager + from multiprocessing import cpu_count from SocketServer import TCPServer from urllib import ( quote as _quote, diff --git a/resources/lib/youtube_plugin/youtube/client/youtube.py b/resources/lib/youtube_plugin/youtube/client/youtube.py index 3de53a9e6..2a324e909 100644 --- a/resources/lib/youtube_plugin/youtube/client/youtube.py +++ b/resources/lib/youtube_plugin/youtube/client/youtube.py @@ -14,19 +14,17 @@ import xml.etree.ElementTree as ET from copy import deepcopy from itertools import chain, islice -from os import cpu_count from random import randint from .login_client import LoginClient from ..helper.video_info import VideoInfo from ..youtube_exceptions import InvalidJSON, YouTubeException -from ...kodion.compatibility import string_type, to_str +from ...kodion.compatibility import cpu_count, string_type, to_str from ...kodion.utils import ( current_system_version, datetime_parser, strip_html_from_text, to_unicode, - wait, ) @@ -1461,46 +1459,44 @@ def get_my_subscriptions(self, items = cached # no cache, get uploads data from web else: - sub_channel_ids = [] - - if logged_in: - params = { - 'part': 'snippet', - 'maxResults': '50', - 'order': 'alphabetical', - 'mine': 'true' - } - - while 1: - json_data = self.api_request(method='GET', - path='subscriptions', - params=params, - **kwargs) - if not json_data: - break + channel_ids = [] + params = { + 'part': 'snippet', + 'maxResults': '50', + 'order': 'alphabetical', + 'mine': 'true' + } - sub_channel_ids.extend([ - (item['snippet']['resourceId']['channelId'],) - for item in json_data.get('items', []) - ]) + def _get_channels(params=params): + if not params or 'complete' in params: + return None, None + json_data = self.api_request(method='GET', + path='subscriptions', + params=params, + **kwargs) + if not json_data: + return None, None + + page_token = json_data.get('nextPageToken') + if page_token: + params['pageToken'] = page_token + else: + params['complete'] = True - # get next token if exists - sub_page_token = json_data.get('nextPageToken') - if sub_page_token: - params['pageToken'] = sub_page_token - # terminate loop when last page - else: - break + return 'list_list', [{ + 'channel_id': item['snippet']['resourceId']['channelId'] + } for item in json_data.get('items', [])] - items = self._context.get_bookmarks_list().get_items() - if items: - sub_channel_ids.extend([ - (item_id,) - for item_id, item in items.items() + bookmarks = self._context.get_bookmarks_list().get_items() + if bookmarks: + channel_ids.extend([ + {'channel_id': item_id} + for item_id, item in bookmarks.items() if (isinstance(item, float) or getattr(item, 'get_channel_id', bool)()) ]) + feeds = [] headers = { 'Host': 'www.youtube.com', 'Connection': 'keep-alive', @@ -1517,78 +1513,34 @@ def get_my_subscriptions(self, } def _get_feed(channel_id, headers=headers): - return channel_id, self.request( - 'https://www.youtube.com/feeds/videos.xml?channel_id=' - + channel_id, - headers=headers, - ) - - def _threaded_fetch(args, - kwargs, - output, - worker, - input_lock): - while 1: - if input_lock.acquire(blocking=False): - _args = args.pop() if args else [] - _kwargs = kwargs.pop() if kwargs else {} - if not _args and not _kwargs: - break - input_lock.release() - else: - wait(0.1) - continue - - try: - key, _output = worker(*_args, **_kwargs) - except Exception as exc: - self._context.log_error('threaded_fetch error: |{exc}|' - .format(exc=exc)) - continue - if _output: - output[key] = _output - input_lock.release() - - payload = { - 'args': list(set(sub_channel_ids)), - 'kwargs': None, - 'output': {}, - 'worker': _get_feed, - 'input_lock': threading.Lock(), - } - - threads = [] - num_threads = 0 - max_threads = min(32, (cpu_count() or 1) + 4) - while payload['args'] or payload['kwargs']: - if num_threads >= max_threads: - continue - thread = threading.Thread( - target=_threaded_fetch, - kwargs=payload, - ) - threads.append(thread) - num_threads += 1 - thread.start() - - for thread in threads: - thread.join(30) + return 'value_list', { + 'channel_id': channel_id, + 'feed': self.request( + 'https://www.youtube.com/feeds/videos.xml?channel_id=' + + channel_id, + headers=headers, + ), + } + items = [] namespaces = { 'atom': 'http://www.w3.org/2005/Atom', 'yt': 'http://www.youtube.com/xml/schemas/2015', 'media': 'http://search.yahoo.com/mrss/', } - encode = not current_system_version.compatible(19, 0) - items = [] - for channel_id, feed in payload['output'].items(): + + def _parse_feed(channel_id, + feed, + encode=not current_system_version.compatible(19, 0), + namespaces=namespaces, + as_list=False): feed.encoding = 'utf-8' feed = to_unicode(feed.content).replace('\n', '') root = ET.fromstring(to_str(feed) if encode else feed) channel_name = (root.findtext('atom:title', '', namespaces) .lower().replace(',', '')) - items.extend([{ + feed_items = [{ 'kind': 'youtube#video', 'id': item.findtext('yt:videoId', '', namespaces), 'snippet': { @@ -1602,7 +1554,147 @@ def _threaded_fetch(args, ) ), '_partial': True, - } for item in root.findall('atom:entry', namespaces)]) + } for item in root.findall('atom:entry', namespaces)] + if as_list: + return feed_items + return 'list_list', feed_items + + def _threaded_fetch(kwargs, + output, + worker, + threads, + pool_id, + dynamic, + input_wait, + **_kwargs): + while not threads['balance'].is_set(): + if kwargs is True: + _kwargs = {} + elif kwargs: + _kwargs = kwargs.pop() + elif input_wait: + input_wait.acquire(blocking=True) + input_wait.release() + if kwargs: + continue + break + else: + break + + try: + output_type, _output = worker(**_kwargs) + except Exception as exc: + self._context.log_error('threaded_fetch error: |{exc}|' + .format(exc=exc)) + continue + + if not output_type: + break + if output_type == 'value_dict': + output[_output[0]] = _output[1] + elif output_type == 'dict_dict': + output.update(_output) + elif output_type == 'value_list': + output.append(_output) + elif output_type == 'list_list': + output.extend(_output) + else: + threads['balance'].clear() + + thread = threading.current_thread() + threads['available'].release() + if dynamic: + threads['pool_counts'][pool_id] -= 1 + threads['current'].discard(thread) + + try: + num_cores = cpu_count() or 1 + except NotImplementedError: + num_cores = 1 + max_threads = min(32, 2 * (num_cores + 4)) + threads = { + 'max': max_threads, + 'available': threading.Semaphore(max_threads), + 'current': set(), + 'pool_counts': {}, + 'balance': threading.Event(), + } + payloads = [ + { + 'pool_id': 1, + 'kwargs': True, + 'output': channel_ids, + 'worker': _get_channels, + 'threads': threads, + 'limit': 1, + 'dynamic': False, + 'input_wait': None, + }, + ] if logged_in else [] + payloads.extend(( + { + 'pool_id': 2, + 'kwargs': channel_ids, + 'output': feeds, + 'worker': _get_feed, + 'threads': threads, + 'limit': None, + 'dynamic': True, + 'input_wait': threading.Lock(), + }, + { + 'pool_id': 3, + 'kwargs': feeds, + 'output': items, + 'worker': _parse_feed, + 'threads': threads, + 'limit': 1, + 'dynamic': True, + 'input_wait': threading.Lock(), + }, + )) + while 1: + for payload in payloads: + pool_id = payload['pool_id'] + if pool_id in threads['pool_counts']: + current_num = threads['pool_counts'][pool_id] + else: + current_num = threads['pool_counts'][pool_id] = 0 + + input_wait = payload['input_wait'] + if payload['kwargs']: + if input_wait and input_wait.locked(): + input_wait.release() + else: + continue + + spots_available = threads['available']._value + limit = payload['limit'] + if limit: + if current_num >= limit: + continue + if not spots_available: + threads['balance'].set() + elif not spots_available: + continue + + thread = threading.Thread( + target=_threaded_fetch, + kwargs=payload, + ) + thread.daemon = True + threads['current'].add(thread) + threads['pool_counts'][pool_id] += 1 + threads['available'].acquire(blocking=True) + thread.start() + + if not threads['current']: + break + + for thread in threads['current']: + if thread and thread.is_alive(): + thread.join(30) + # Update cache cache.set_item(cache_items_key, items) @@ -1618,7 +1710,7 @@ def _threaded_fetch(args, } limits['start'] += limits['end'] - def _sort_by_date_time(item, _limits=limits): + def _sort_by_date_time(item, limits=limits): if do_filter: filtered = item['_channel'] in filter_list if black_list: @@ -1627,10 +1719,10 @@ def _sort_by_date_time(item, _limits=limits): elif not filtered: return -1 video_id = item['id'] - if video_id in _limits['video_ids']: + if video_id in limits['video_ids']: return -1 - _limits['num'] += 1 - _limits['video_ids'].add(video_id) + limits['num'] += 1 + limits['video_ids'].add(video_id) return item['_timestamp'] items.sort(reverse=True, key=_sort_by_date_time) From 1b6868fd29dac0c2449be062222f7e3a467aa77c Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Fri, 17 May 2024 08:11:53 +1000 Subject: [PATCH 39/53] Fix failing to login #759 - Also remove a bunch of duplicated code - Further de-duplication to follow --- resources/lib/youtube_authentication.py | 2 +- .../kodion/json_store/access_manager.py | 149 +++++------------- .../youtube/client/__config__.py | 11 +- .../youtube_plugin/youtube/helper/yt_login.py | 113 +++++-------- .../lib/youtube_plugin/youtube/provider.py | 53 +++---- 5 files changed, 107 insertions(+), 221 deletions(-) diff --git a/resources/lib/youtube_authentication.py b/resources/lib/youtube_authentication.py index 5156eee9a..d32ca5b17 100644 --- a/resources/lib/youtube_authentication.py +++ b/resources/lib/youtube_authentication.py @@ -166,6 +166,6 @@ def reset_access_tokens(addon_id): .format(addon_id)) return context = XbmcContext(params={'addon_id': addon_id}) - context.get_access_manager().update_dev_access_token( + context.get_access_manager().update_access_token( addon_id, access_token='', refresh_token='' ) diff --git a/resources/lib/youtube_plugin/kodion/json_store/access_manager.py b/resources/lib/youtube_plugin/kodion/json_store/access_manager.py index c49efd55b..b39d98175 100644 --- a/resources/lib/youtube_plugin/kodion/json_store/access_manager.py +++ b/resources/lib/youtube_plugin/kodion/json_store/access_manager.py @@ -127,10 +127,12 @@ def load(self, process=_process_data.__func__): def save(self, data, update=False, process=_process_data.__func__): return super(AccessManager, self).save(data, update, process) - def get_current_user_details(self): + def get_current_user_details(self, addon_id=None): """ :return: current user """ + if addon_id: + return self.get_developers().get(addon_id, {}) return self.get_users()[self._user] def get_current_user_id(self): @@ -373,38 +375,39 @@ def get_last_origin(self): """ return self._last_origin - def get_access_token(self): + def get_access_token(self, addon_id=None): """ Returns the access token for some API :return: access_token """ - token = self.get_current_user_details().get('access_token', '') - return token.split('|') + details = self.get_current_user_details(addon_id) + return details.get('access_token', '').split('|') - def get_refresh_token(self): + def get_refresh_token(self, addon_id=None): """ Returns the refresh token :return: refresh token """ - token = self.get_current_user_details().get('refresh_token', '') - return token.split('|') + details = self.get_current_user_details(addon_id) + return details.get('refresh_token', '').split('|') - def is_access_token_expired(self): + def is_access_token_expired(self, addon_id=None): """ Returns True if the access_token is expired otherwise False. If no expiration date was provided and an access_token exists this method will always return True :return: """ - current_user = self.get_current_user_details() - access_token = current_user.get('access_token', '') - expires = int(current_user.get('token_expires', -1)) + details = self.get_current_user_details(addon_id) + access_token = details.get('access_token', '') + expires = int(details.get('token_expires', -1)) if access_token and expires <= int(time.time()): return True return False def update_access_token(self, + addon_id, access_token=None, unix_timestamp=None, refresh_token=None): @@ -415,7 +418,7 @@ def update_access_token(self, :param refresh_token: :return: """ - current_user = { + details = { 'access_token': ( '|'.join(access_token) if isinstance(access_token, (list, tuple)) else @@ -426,10 +429,14 @@ def update_access_token(self, } if unix_timestamp is not None: - current_user['token_expires'] = int(unix_timestamp) + details['token_expires'] = ( + min(map(int, unix_timestamp)) + if isinstance(unix_timestamp, (list, tuple)) else + int(unix_timestamp) + ) if refresh_token is not None: - current_user['refresh_token'] = ( + details['refresh_token'] = ( '|'.join(refresh_token) if isinstance(refresh_token, (list, tuple)) else refresh_token @@ -437,16 +444,30 @@ def update_access_token(self, data = { 'access_manager': { + 'developers': { + addon_id: details, + }, + } if addon_id else { 'users': { - self._user: current_user, + self._user: details, }, }, } self.save(data, update=True) - def set_last_key_hash(self, key_hash): + def get_last_key_hash(self, addon_id=None): + details = self.get_current_user_details(addon_id) + return details.get('last_key_hash', '') + + def set_last_key_hash(self, key_hash, addon_id=None): data = { 'access_manager': { + 'developers': { + addon_id: { + 'last_key_hash': key_hash, + }, + }, + } if addon_id else { 'users': { self._user: { 'last_key_hash': key_hash, @@ -475,9 +496,6 @@ def get_developers(self): """ return self._data['access_manager'].get('developers', {}) - def get_developer(self, addon_id): - return self.get_developers().get(addon_id, {}) - def set_developers(self, developers): """ Updates the users @@ -488,103 +506,16 @@ def set_developers(self, developers): data['access_manager']['developers'] = developers self.save(data) - def get_dev_access_token(self, addon_id): - """ - Returns the access token for some API - :param addon_id: addon id - :return: access_token - """ - return self.get_developer(addon_id).get('access_token', '').split('|') - - def get_dev_refresh_token(self, addon_id): - """ - Returns the refresh token - :return: refresh token - """ - return self.get_developer(addon_id).get('refresh_token', '').split('|') - - def is_dev_access_token_expired(self, addon_id): - """ - Returns True if the access_token is expired otherwise False. - If no expiration date was provided and an access_token exists - this method will always return True - :return: - """ - developer = self.get_developer(addon_id) - access_token = developer.get('access_token', '') - expires = int(developer.get('token_expires', -1)) - - if access_token and expires <= int(time.time()): - return True - return False - - def update_dev_access_token(self, - addon_id, - access_token=None, - unix_timestamp=None, - refresh_token=None): - """ - Updates the old access token with the new one. - :param addon_id: - :param access_token: - :param unix_timestamp: - :param refresh_token: - :return: - """ - developer = { - 'access_token': ( - '|'.join(access_token) - if isinstance(access_token, (list, tuple)) else - access_token - if access_token else - '' - ) - } - - if unix_timestamp is not None: - developer['token_expires'] = int(unix_timestamp) - - if refresh_token is not None: - developer['refresh_token'] = ( - '|'.join(refresh_token) - if isinstance(refresh_token, (list, tuple)) else - refresh_token - ) - - data = { - 'access_manager': { - 'developers': { - addon_id: developer, - }, - }, - } - self.save(data, update=True) - - def get_dev_last_key_hash(self, addon_id): - return self.get_developer(addon_id).get('last_key_hash', '') - - def set_dev_last_key_hash(self, addon_id, key_hash): - data = { - 'access_manager': { - 'developers': { - addon_id: { - 'last_key_hash': key_hash, - }, - }, - }, - } - self.save(data, update=True) - def dev_keys_changed(self, addon_id, api_key, client_id, client_secret): - last_hash = self.get_dev_last_key_hash(addon_id) + last_hash = self.get_last_key_hash(addon_id) current_hash = self.calc_key_hash(api_key, client_id, client_secret) if not last_hash and current_hash: - self.set_dev_last_key_hash(addon_id, current_hash) + self.set_last_key_hash(current_hash, addon_id) return False if last_hash != current_hash: - self.set_dev_last_key_hash(addon_id, current_hash) + self.set_last_key_hash(current_hash, addon_id) return True return False diff --git a/resources/lib/youtube_plugin/youtube/client/__config__.py b/resources/lib/youtube_plugin/youtube/client/__config__.py index 3e954298a..fdec14976 100644 --- a/resources/lib/youtube_plugin/youtube/client/__config__.py +++ b/resources/lib/youtube_plugin/youtube/client/__config__.py @@ -66,15 +66,14 @@ def __init__(self, context): settings.api_secret(j_secret) switch = self.get_current_switch() - user_details = self._access_manager.get_current_user_details() - last_hash = user_details.get('last_key_hash', '') - current_set_hash = self._get_key_set_hash(switch) + last_hash = self._access_manager.get_last_key_hash() + current_hash = self._get_key_set_hash(switch) - changed = current_set_hash != last_hash + changed = current_hash != last_hash if changed and switch == 'own': changed = self._get_key_set_hash('own_old') != last_hash if not changed: - self._access_manager.set_last_key_hash(current_set_hash) + self._access_manager.set_last_key_hash(current_hash) self.changed = changed self._context.log_debug('User: |{user}|, ' @@ -91,7 +90,7 @@ def __init__(self, context): } ) )) - self._access_manager.set_last_key_hash(current_set_hash) + self._access_manager.set_last_key_hash(current_hash) @staticmethod def get_current_switch(): diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_login.py b/resources/lib/youtube_plugin/youtube/helper/yt_login.py index 799f93178..15dffb72a 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_login.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_login.py @@ -23,37 +23,25 @@ def process(mode, provider, context, sign_out_refresh=True): ui = context.get_ui() def _do_logout(): - if addon_id: - refresh_tokens = access_manager.get_dev_refresh_token(addon_id) - client = provider.get_client(context) - if refresh_tokens: - for _refresh_token in set(refresh_tokens): - try: - client.revoke(_refresh_token) - except LoginException: - pass - access_manager.update_dev_access_token( - addon_id, access_token='', refresh_token='' - ) - else: - refresh_tokens = access_manager.get_refresh_token() - client = provider.get_client(context) - if refresh_tokens: - for _refresh_token in set(refresh_tokens): - try: - client.revoke(_refresh_token) - except LoginException: - pass - access_manager.update_access_token( - access_token='', refresh_token='' - ) + refresh_tokens = access_manager.get_refresh_token() + client = provider.get_client(context) + if refresh_tokens: + for _refresh_token in set(refresh_tokens): + try: + client.revoke(_refresh_token) + except LoginException: + pass + access_manager.update_access_token( + addon_id, access_token='', refresh_token='', + ) provider.reset_client() - def _do_login(_for_tv=False): + def _do_login(login_type): + for_tv = login_type == 'tv' _client = provider.get_client(context) try: - if _for_tv: + if for_tv: json_data = _client.request_device_and_user_code_tv() else: json_data = _client.request_device_and_user_code() @@ -85,7 +73,7 @@ def _do_login(_for_tv=False): for _ in range(steps): dialog.update() try: - if _for_tv: + if for_tv: json_data = _client.request_access_token_tv(device_code) else: json_data = _client.request_access_token(device_code) @@ -133,53 +121,30 @@ def _do_login(_for_tv=False): elif mode == 'in': ui.on_ok(localize('sign.twice.title'), localize('sign.twice.text')) - tv_token = _do_login(_for_tv=True) - access_token, expires_in, refresh_token = tv_token - # abort tv login - context.log_debug('YouTube-TV Login:' - ' Access Token |{0}|,' - ' Refresh Token |{1}|,' - ' Expires |{2}|' - .format(access_token != '', - refresh_token != '', - expires_in)) - if not access_token and not refresh_token: - provider.reset_client() - if addon_id: - access_manager.update_dev_access_token(addon_id) - else: - access_manager.update_access_token('') - ui.refresh_container() - return - - kodi_token = _do_login(_for_tv=False) - access_token, expires_in, refresh_token = kodi_token - # abort kodi login - context.log_debug('YouTube-Kodi Login:' - ' Access Token |{0}|,' - ' Refresh Token |{1}|,' - ' Expires |{2}|' - .format(access_token != '', - refresh_token != '', - expires_in)) - if not access_token and not refresh_token: - provider.reset_client() - if addon_id: - access_manager.update_dev_access_token(addon_id) - else: - access_manager.update_access_token('') - ui.refresh_container() - return + tokens = { + 'tv': None, + 'kodi': None, + } + for token in tokens: + new_token = _do_login(login_type=token) + access_token, expires_in, refresh_token = new_token + context.log_debug('YouTube Login:' + ' Type |{0}|,' + ' Access Token |{1}|,' + ' Refresh Token |{2}|,' + ' Expires |{3}|' + .format(token, + access_token != '', + refresh_token != '', + expires_in)) + # abort login + if not access_token and not refresh_token: + provider.reset_client() + access_manager.update_access_token(addon_id, '') + ui.refresh_container() + return + tokens[token] = new_token provider.reset_client() - - if addon_id: - access_manager.update_dev_access_token( - addon_id, *list(zip(tv_token, kodi_token)) - ) - else: - access_manager.update_access_token( - *list(zip(tv_token, kodi_token)) - ) - + access_manager.update_access_token(addon_id, *zip(*tokens.values())) ui.refresh_container() diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 25514775a..3e910cee5 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -168,11 +168,11 @@ def get_client(self, context): self.reset_client() if dev_id: - access_tokens = access_manager.get_dev_access_token(dev_id) - if access_manager.is_dev_access_token_expired(dev_id): + access_tokens = access_manager.get_access_token(dev_id) + if access_manager.is_access_token_expired(dev_id): # reset access_token access_tokens = [] - access_manager.update_dev_access_token(dev_id, access_tokens) + access_manager.update_access_token(dev_id, access_tokens) elif self._client: return self._client @@ -186,7 +186,7 @@ def get_client(self, context): ' w/ developer access tokens' .format(dev_keys['system'])) - refresh_tokens = access_manager.get_dev_refresh_token(dev_id) + refresh_tokens = access_manager.get_refresh_token(dev_id) if refresh_tokens: keys_changed = access_manager.dev_keys_changed( dev_id, dev_keys['key'], dev_keys['id'], dev_keys['secret'] @@ -197,7 +197,7 @@ def get_client(self, context): self.reset_client() access_tokens = [] refresh_tokens = [] - access_manager.update_dev_access_token( + access_manager.update_access_token( dev_id, access_tokens, -1, refresh_tokens ) @@ -206,18 +206,18 @@ def get_client(self, context): .format(len(access_tokens), len(refresh_tokens)) ) else: - access_tokens = access_manager.get_access_token() - if access_manager.is_access_token_expired(): + access_tokens = access_manager.get_access_token(dev_id) + if access_manager.is_access_token_expired(dev_id): # reset access_token access_tokens = [] - access_manager.update_access_token(access_tokens) + access_manager.update_access_token(dev_id, access_tokens) elif self._client: return self._client context.log_debug('Selecting YouTube config "{0}"' .format(configs['main']['system'])) - refresh_tokens = access_manager.get_refresh_token() + refresh_tokens = access_manager.get_refresh_token(dev_id) if refresh_tokens: if self._api_check.changed: context.log_warning('API key set changed: Resetting client' @@ -226,7 +226,7 @@ def get_client(self, context): access_tokens = [] refresh_tokens = [] access_manager.update_access_token( - access_tokens, -1, refresh_tokens + dev_id, access_tokens, -1, refresh_tokens, ) context.log_debug( @@ -252,30 +252,18 @@ def get_client(self, context): tv_token = client.refresh_token_tv(refresh_tokens[0]) access_tokens = (tv_token[0], kodi_token[0]) expires_in = min(tv_token[1], kodi_token[1]) - if dev_id: - access_manager.update_dev_access_token( - dev_id, access_tokens, expires_in - ) - else: - access_manager.update_access_token( - access_tokens, expires_in - ) + access_manager.update_access_token( + dev_id, access_tokens, expires_in, + ) except (InvalidGrant, LoginException) as exc: self.handle_exception(context, exc) # reset access_token if isinstance(exc, InvalidGrant): - if dev_id: - access_manager.update_dev_access_token( - dev_id, access_token='', refresh_token='' - ) - else: - access_manager.update_access_token( - access_token='', refresh_token='' - ) - elif dev_id: - access_manager.update_dev_access_token(dev_id) + access_manager.update_access_token( + dev_id, access_token='', refresh_token='', + ) else: - access_manager.update_access_token() + access_manager.update_access_token(dev_id) # in debug log the login status self._logged_in = len(access_tokens) == 2 @@ -931,6 +919,7 @@ def maintenance_actions(self, context, re_match): if target == 'access_manager' and ui.on_yes_no_input( context.get_name(), localize('reset.access_manager.confirm') ): + addon_id = context.get_param('addon_id', None) access_manager = context.get_access_manager() client = self.get_client(context) refresh_tokens = access_manager.get_refresh_token() @@ -943,7 +932,7 @@ def maintenance_actions(self, context, re_match): success = False self.reset_client() access_manager.update_access_token( - access_token='', refresh_token='' + addon_id, access_token='', refresh_token='', ) ui.refresh_container() ui.show_notification(localize('succeeded' if success else 'failed')) @@ -1576,7 +1565,9 @@ def handle_exception(self, context, exception_to_handle): if error == 'deleted_client': message = context.localize('key.requirement') context.get_access_manager().update_access_token( - access_token='', refresh_token='' + context.get_param('addon_id', None), + access_token='', + refresh_token='', ) ok_dialog = True From f0e2c614564d311f28065ad324b8e13788fef71d Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:17:13 +1000 Subject: [PATCH 40/53] Remove XbmcContextUI.open_settings --- .../youtube_plugin/kodion/context/xbmc/xbmc_context.py | 2 +- .../lib/youtube_plugin/kodion/ui/abstract_context_ui.py | 2 -- .../lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py | 8 +------- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py index fbcec130c..026666912 100644 --- a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py +++ b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py @@ -426,7 +426,7 @@ def get_audio_player(self): def get_ui(self): if not self._ui: - self._ui = XbmcContextUI(self._addon, proxy(self)) + self._ui = XbmcContextUI(proxy(self)) return self._ui def get_data_path(self): diff --git a/resources/lib/youtube_plugin/kodion/ui/abstract_context_ui.py b/resources/lib/youtube_plugin/kodion/ui/abstract_context_ui.py index 85aa74eec..967fc7d8a 100644 --- a/resources/lib/youtube_plugin/kodion/ui/abstract_context_ui.py +++ b/resources/lib/youtube_plugin/kodion/ui/abstract_context_ui.py @@ -42,8 +42,6 @@ def on_clear_content(self, name): def on_select(self, title, items=None, preselect=-1, use_details=False): raise NotImplementedError() - def open_settings(self): - raise NotImplementedError() def show_notification(self, message, header='', image_uri='', time_ms=5000, audible=True): diff --git a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py index eb98b8cb1..59c1a80bd 100644 --- a/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py +++ b/resources/lib/youtube_plugin/kodion/ui/xbmc/xbmc_context_ui.py @@ -18,11 +18,8 @@ class XbmcContextUI(AbstractContextUI): - def __init__(self, xbmc_addon, context): + def __init__(self, context): super(XbmcContextUI, self).__init__() - - self._xbmc_addon = xbmc_addon - self._context = context def create_progress_dialog(self, heading, text=None, background=False): @@ -134,9 +131,6 @@ def show_notification(self, time_ms, audible) - def open_settings(self): - self._xbmc_addon.openSettings() - def refresh_container(self): self._context.send_notification(REFRESH_CONTAINER, True) From 1d92d46054736808c8c29292578a6fa93f2ccef4 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:18:46 +1000 Subject: [PATCH 41/53] Improve provider tear down --- .../youtube_plugin/kodion/plugin_runner.py | 1 - .../lib/youtube_plugin/youtube/provider.py | 23 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/plugin_runner.py b/resources/lib/youtube_plugin/kodion/plugin_runner.py index 0df9b4bc9..886d02722 100644 --- a/resources/lib/youtube_plugin/kodion/plugin_runner.py +++ b/resources/lib/youtube_plugin/kodion/plugin_runner.py @@ -31,7 +31,6 @@ _profiler = Profiler(enabled=False) -atexit.register(_provider.tear_down) atexit.register(_context.tear_down) diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 3e910cee5..1295425bc 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, division, unicode_literals +import atexit import json import re from base64 import b64decode @@ -56,6 +57,8 @@ def __init__(self): self._logged_in = False self.yt_video = yt_video + atexit.register(self.tear_down) + def get_wizard_steps(self, context): steps = [ yt_setup_wizard.process_default_settings, @@ -1591,11 +1594,15 @@ def handle_exception(self, context, exception_to_handle): return True def tear_down(self): - del self._resource_manager - self._resource_manager = None - del self._client - self._client = None - del self._api_check - self._api_check = None - del self.yt_video - self.yt_video = None + attrs = ( + '_resource_manager', + '_client', + '_api_check', + 'yt_video', + ) + for attr in attrs: + try: + delattr(self, attr) + setattr(self, attr, None) + except (AttributeError, TypeError): + pass From 936a23b167838d6f49bffc18b95172a87dda423b Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:28:04 +1000 Subject: [PATCH 42/53] Don't create unnecessary additional context, settings and Addon instances in service --- .../kodion/monitors/service_monitor.py | 24 ++-- .../kodion/network/http_server.py | 120 +++++++++--------- .../youtube_plugin/kodion/script_actions.py | 4 +- .../youtube_plugin/kodion/service_runner.py | 8 +- .../youtube/helper/video_info.py | 6 +- .../youtube_plugin/youtube/helper/yt_play.py | 2 +- .../youtube/helper/yt_setup_wizard.py | 2 +- 7 files changed, 82 insertions(+), 84 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py index 6d65e86e0..31aab8421 100644 --- a/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py +++ b/resources/lib/youtube_plugin/kodion/monitors/service_monitor.py @@ -12,24 +12,23 @@ import json import threading -from ..compatibility import xbmc, xbmcaddon, xbmcgui +from ..compatibility import xbmc, xbmcgui from ..constants import ADDON_ID, CHECK_SETTINGS, REFRESH_CONTAINER, WAKEUP from ..logger import log_debug from ..network import get_connect_address, get_http_server, httpd_status -from ..settings import XbmcPluginSettings class ServiceMonitor(xbmc.Monitor): - _settings = XbmcPluginSettings(xbmcaddon.Addon(ADDON_ID)) _settings_changes = 0 _settings_state = None - def __init__(self): - settings = self._settings + def __init__(self, context): + self._context = context + settings = context.get_settings() self._use_httpd = (settings.use_isa() or settings.api_config_page() or settings.support_alternative_player()) - address, port = get_connect_address() + address, port = get_connect_address(self._context) self._old_httpd_address = self._httpd_address = address self._old_httpd_port = self._httpd_port = port self._whitelist = settings.httpd_whitelist() @@ -90,8 +89,7 @@ def onSettingsChanged(self): log_debug('onSettingsChanged: {0} change(s)'.format(changes)) self._settings_changes = 0 - settings = self._settings - settings.flush(xbmcaddon.Addon(ADDON_ID)) + settings = self._context.get_settings(refresh=True) xbmcgui.Window(10000).setProperty( '-'.join((ADDON_ID, CHECK_SETTINGS)), 'true' @@ -103,7 +101,7 @@ def onSettingsChanged(self): use_httpd = (settings.use_isa() or settings.api_config_page() or settings.support_alternative_player()) - address, port = get_connect_address() + address, port = get_connect_address(self._context) whitelist = settings.httpd_whitelist() whitelist_changed = whitelist != self._whitelist @@ -147,7 +145,8 @@ def start_httpd(self): .format(ip=self._httpd_address, port=self._httpd_port)) self.httpd_address_sync() self.httpd = get_http_server(address=self._httpd_address, - port=self._httpd_port) + port=self._httpd_port, + context=self._context) if not self.httpd: return @@ -180,10 +179,7 @@ def restart_httpd(self): self.start_httpd() def ping_httpd(self): - return self.httpd and httpd_status() + return self.httpd and httpd_status(self._context) def httpd_required(self): return self._use_httpd - - def tear_down(self): - self._settings.flush() diff --git a/resources/lib/youtube_plugin/kodion/network/http_server.py b/resources/lib/youtube_plugin/kodion/network/http_server.py index 3bc77a64d..f54a49b8e 100644 --- a/resources/lib/youtube_plugin/kodion/network/http_server.py +++ b/resources/lib/youtube_plugin/kodion/network/http_server.py @@ -23,27 +23,17 @@ parse_qs, urlsplit, xbmc, - xbmcaddon, xbmcgui, xbmcvfs, ) from ..constants import ADDON_ID, TEMP_PATH, paths from ..logger import log_debug, log_error -from ..settings import XbmcPluginSettings from ..utils import validate_ip_address, wait -_addon = xbmcaddon.Addon(ADDON_ID) -_settings = XbmcPluginSettings(_addon) -_i18n = _addon.getLocalizedString -_addon_name = _addon.getAddonInfo('name') -_addon_icon = _addon.getAddonInfo('icon') -del _addon - -_server_requests = BaseRequestsClass() - - class RequestHandler(BaseHTTPRequestHandler, object): + _context = None + requests = BaseRequestsClass() BASE_PATH = xbmcvfs.translatePath(TEMP_PATH) chunk_size = 1024 * 64 local_ranges = ( @@ -56,7 +46,7 @@ class RequestHandler(BaseHTTPRequestHandler, object): ) def __init__(self, *args, **kwargs): - self.whitelist_ips = _settings.httpd_whitelist() + self.whitelist_ips = self._context.get_settings().httpd_whitelist() super(RequestHandler, self).__init__(*args, **kwargs) def connection_allowed(self): @@ -85,7 +75,9 @@ def connection_allowed(self): # noinspection PyPep8Naming def do_GET(self): - api_config_enabled = _settings.api_config_page() + settings = self._context.get_settings() + localize = self._context.localize + api_config_enabled = settings.api_config_page() # Strip trailing slash if present stripped_path = self.path.rstrip('/') @@ -146,7 +138,10 @@ def do_GET(self): api_id = params.get('api_id', [None])[0] api_secret = params.get('api_secret', [None])[0] # Bookmark this page - footer = _i18n(30638) if api_key and api_id and api_secret else '' + if api_key and api_id and api_secret: + footer = localize(30638) + else: + footer = '' if re.search(r'api_key=(?:&|$)', query): api_key = '' @@ -155,29 +150,29 @@ def do_GET(self): if re.search(r'api_secret=(?:&|$)', query): api_secret = '' - if api_key is not None and api_key != _settings.api_key(): - _settings.api_key(new_key=api_key) - updated.append(_i18n(30201)) # API Key + if api_key is not None and api_key != settings.api_key(): + settings.api_key(new_key=api_key) + updated.append(localize(30201)) # API Key - if api_id is not None and api_id != _settings.api_id(): - _settings.api_id(new_id=api_id) - updated.append(_i18n(30202)) # API ID + if api_id is not None and api_id != settings.api_id(): + settings.api_id(new_id=api_id) + updated.append(localize(30202)) # API ID - if api_secret is not None and api_secret != _settings.api_secret(): - _settings.api_secret(new_secret=api_secret) - updated.append(_i18n(30203)) # API Secret + if api_secret is not None and api_secret != settings.api_secret(): + settings.api_secret(new_secret=api_secret) + updated.append(localize(30203)) # API Secret if api_key and api_id and api_secret: - enabled = _i18n(30636) # Personal keys enabled + enabled = localize(30636) # Personal keys enabled else: - enabled = _i18n(30637) # Personal keys disabled + enabled = localize(30637) # Personal keys disabled if updated: # Successfully updated - updated = _i18n(30631) % ', '.join(updated) + updated = localize(30631) % ', '.join(updated) else: # No changes, not updated - updated = _i18n(30635) + updated = localize(30635) html = self.api_submit_page(updated, enabled, footer) html = html.encode('utf-8') @@ -262,11 +257,11 @@ def do_POST(self): 'Authorization': 'Bearer %s' % lic_token } - response = _server_requests.request(lic_url, - method='POST', - headers=li_headers, - data=post_data, - stream=True) + response = self.requests.request(lic_url, + method='POST', + headers=li_headers, + data=post_data, + stream=True) if not response or not response.ok: self.send_error(response and response.status_code or 500) return @@ -328,38 +323,41 @@ def get_chunks(self, data): for i in range(0, len(data), self.chunk_size): yield data[i:i + self.chunk_size] - @staticmethod - def api_config_page(): - api_key = _settings.api_key() - api_id = _settings.api_id() - api_secret = _settings.api_secret() + @classmethod + def api_config_page(cls): + settings = cls._context.get_settings() + localize = cls._context.localize + api_key = settings.api_key() + api_id = settings.api_id() + api_secret = settings.api_secret() html = Pages.api_configuration.get('html') css = Pages.api_configuration.get('css') html = html.format( css=css, - title=_i18n(30634), # YouTube Add-on API Configuration - api_key_head=_i18n(30201), # API Key - api_id_head=_i18n(30202), # API ID - api_secret_head=_i18n(30203), # API Secret + title=localize(30634), # YouTube Add-on API Configuration + api_key_head=localize(30201), # API Key + api_id_head=localize(30202), # API ID + api_secret_head=localize(30203), # API Secret api_id_value=api_id, api_key_value=api_key, api_secret_value=api_secret, - submit=_i18n(30630), # Save - header=_i18n(30634), # YouTube Add-on API Configuration + submit=localize(30630), # Save + header=localize(30634), # YouTube Add-on API Configuration ) return html - @staticmethod - def api_submit_page(updated_keys, enabled, footer): + @classmethod + def api_submit_page(cls, updated_keys, enabled, footer): + localize = cls._context.localize html = Pages.api_submit.get('html') css = Pages.api_submit.get('css') html = html.format( css=css, - title=_i18n(30634), # YouTube Add-on API Configuration + title=localize(30634), # YouTube Add-on API Configuration updated=updated_keys, enabled=enabled, footer=footer, - header=_i18n(30634), # YouTube Add-on API Configuration + header=localize(30634), # YouTube Add-on API Configuration ) return html @@ -547,7 +545,8 @@ class Pages(object): } -def get_http_server(address, port): +def get_http_server(address, port, context): + RequestHandler._context = context try: server = TCPServer((address, port), RequestHandler, False) server.allow_reuse_address = True @@ -558,20 +557,20 @@ def get_http_server(address, port): except socket.error as exc: log_error('HTTPServer: Failed to start |{address}:{port}| |{response}|' .format(address=address, port=port, response=exc)) - xbmcgui.Dialog().notification(_addon_name, + xbmcgui.Dialog().notification(context.get_name(), str(exc), - _addon_icon, + context.get_icon(), time=5000, sound=False) return None -def httpd_status(): - address, port = get_connect_address() +def httpd_status(context): + address, port = get_connect_address(context) url = 'http://{address}:{port}{path}'.format(address=address, port=port, path=paths.PING) - response = _server_requests.request(url) + response = RequestHandler.requests.request(url) result = response and response.status_code if result == 204: return True @@ -583,13 +582,13 @@ def httpd_status(): return False -def get_client_ip_address(): +def get_client_ip_address(context): ip_address = None - address, port = get_connect_address() + address, port = get_connect_address(context) url = 'http://{address}:{port}{path}'.format(address=address, port=port, path=paths.IP) - response = _server_requests.request(url) + response = RequestHandler.requests.request(url) if response and response.status_code == 200: response_json = response.json() if response_json: @@ -597,9 +596,10 @@ def get_client_ip_address(): return ip_address -def get_connect_address(as_netloc=False): - address = _settings.httpd_listen() - port = _settings.httpd_port() +def get_connect_address(context, as_netloc=False): + settings = context.get_settings() + address = settings.httpd_listen() + port = settings.httpd_port() if address == '0.0.0.0': address = '127.0.0.1' diff --git a/resources/lib/youtube_plugin/kodion/script_actions.py b/resources/lib/youtube_plugin/kodion/script_actions.py index 136e72e7b..d8e7bca7f 100644 --- a/resources/lib/youtube_plugin/kodion/script_actions.py +++ b/resources/lib/youtube_plugin/kodion/script_actions.py @@ -111,8 +111,8 @@ def _config_actions(context, action, *_args): settings.httpd_listen(addresses[selected_address]) elif action == 'show_client_ip': - if httpd_status(): - client_ip = get_client_ip_address() + if httpd_status(context): + client_ip = get_client_ip_address(context) if client_ip: ui.on_ok(context.get_name(), context.localize('client.ip') % client_ip) diff --git a/resources/lib/youtube_plugin/kodion/service_runner.py b/resources/lib/youtube_plugin/kodion/service_runner.py index df2445cc7..662de4589 100644 --- a/resources/lib/youtube_plugin/kodion/service_runner.py +++ b/resources/lib/youtube_plugin/kodion/service_runner.py @@ -32,6 +32,8 @@ def run(): context = XbmcContext() context.log_debug('YouTube service initialization...') + provider = Provider() + get_infobool = context.get_infobool get_infolabel = context.get_infolabel get_listitem_detail = context.get_listitem_detail @@ -44,8 +46,8 @@ def run(): clear_property(ABORT_FLAG) - monitor = ServiceMonitor() - player = PlayerMonitor(provider=Provider(), + monitor = ServiceMonitor(context=context) + player = PlayerMonitor(provider=provider, context=context, monitor=monitor) @@ -124,5 +126,5 @@ def run(): if monitor.httpd: monitor.shutdown_httpd() # shutdown http server - monitor.tear_down() + provider.tear_down() context.tear_down() diff --git a/resources/lib/youtube_plugin/youtube/helper/video_info.py b/resources/lib/youtube_plugin/youtube/helper/video_info.py index 107692584..15610bbf1 100644 --- a/resources/lib/youtube_plugin/youtube/helper/video_info.py +++ b/resources/lib/youtube_plugin/youtube/helper/video_info.py @@ -1369,7 +1369,7 @@ def _get_video_info(self): } use_mpd_vod = _settings.use_mpd_videos() - httpd_running = _settings.use_isa() and httpd_status() + httpd_running = _settings.use_isa() and httpd_status(self._context) pa_li_info = streaming_data.get('licenseInfos', []) if any(pa_li_info) and not httpd_running: @@ -1382,7 +1382,7 @@ def _get_video_info(self): continue self._context.log_debug('Found widevine license url: {0}' .format(url)) - address, port = get_connect_address() + address, port = get_connect_address(self._context) license_info = { 'url': url, 'proxy': 'http://{address}:{port}{path}||R{{SSM}}|'.format( @@ -2129,7 +2129,7 @@ def _filter_group(previous_group, previous_stream, item): .format(file=filepath)) success = False if success: - address, port = get_connect_address() + address, port = get_connect_address(self._context) return 'http://{address}:{port}{path}{file}'.format( address=address, port=port, diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_play.py b/resources/lib/youtube_plugin/youtube/helper/yt_play.py index 787f69113..bb85e75f9 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_play.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_play.py @@ -102,7 +102,7 @@ def play_video(provider, context): if is_external: url = urlunsplit(( 'http', - get_connect_address(as_netloc=True), + get_connect_address(context=context, as_netloc=True), paths.REDIRECT, urlencode({'url': video_stream['url']}), '', diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py b/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py index a7301fc38..c2b64339f 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py @@ -329,7 +329,7 @@ def process_default_settings(_provider, context, step, steps): settings.default_player_web_urls(False) if settings.cache_size() < 20: settings.cache_size(20) - if settings.use_isa() and not httpd_status(): + if settings.use_isa() and not httpd_status(context): settings.httpd_listen('0.0.0.0') return step From 6926bfec2f04325203f788a41cd62a2d88e66686 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:33:49 +1000 Subject: [PATCH 43/53] Partial workaround for Kodi not properly garbage collecting instances of classes from xbmc* Python modules --- .../youtube_plugin/kodion/network/requests.py | 11 +- .../kodion/settings/abstract_settings.py | 1 - .../settings/xbmc/xbmc_plugin_settings.py | 155 +++++++++++------- 3 files changed, 97 insertions(+), 70 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/network/requests.py b/resources/lib/youtube_plugin/kodion/network/requests.py index 4241edf10..8fb22a543 100644 --- a/resources/lib/youtube_plugin/kodion/network/requests.py +++ b/resources/lib/youtube_plugin/kodion/network/requests.py @@ -16,8 +16,6 @@ from requests.adapters import HTTPAdapter, Retry from requests.exceptions import InvalidJSONError, RequestException -from ..compatibility import xbmcaddon -from ..constants import ADDON_ID from ..logger import log_error from ..settings import XbmcPluginSettings @@ -27,8 +25,6 @@ 'InvalidJSONError' ) -_settings = XbmcPluginSettings(xbmcaddon.Addon(ADDON_ID)) - class BaseRequestsClass(object): _http_adapter = HTTPAdapter( @@ -47,8 +43,11 @@ class BaseRequestsClass(object): atexit.register(_session.close) def __init__(self, exc_type=None): - self._verify = _settings.verify_ssl() - self._timeout = _settings.get_timeout() + settings = XbmcPluginSettings() + self._verify = settings.verify_ssl() + self._timeout = settings.get_timeout() + settings.flush(flush_all=False) + if isinstance(exc_type, tuple): self._default_exc = (RequestException,) + exc_type elif exc_type: diff --git a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py index 94017f517..d43341fdf 100644 --- a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py +++ b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py @@ -25,7 +25,6 @@ class AbstractSettings(object): _echo = False _cache = {} _check_set = True - _instance = None @classmethod def flush(cls, xbmc_addon): diff --git a/resources/lib/youtube_plugin/kodion/settings/xbmc/xbmc_plugin_settings.py b/resources/lib/youtube_plugin/kodion/settings/xbmc/xbmc_plugin_settings.py index f188bbca0..c7dee4809 100644 --- a/resources/lib/youtube_plugin/kodion/settings/xbmc/xbmc_plugin_settings.py +++ b/resources/lib/youtube_plugin/kodion/settings/xbmc/xbmc_plugin_settings.py @@ -10,76 +10,108 @@ from __future__ import absolute_import, division, unicode_literals -import atexit +from weakref import ref from ..abstract_settings import AbstractSettings from ...compatibility import xbmcaddon -from ...constants import VALUE_FROM_STR +from ...constants import ADDON_ID, VALUE_FROM_STR from ...logger import log_debug from ...utils.methods import get_kodi_setting_bool from ...utils.system_version import current_system_version +class SettingsProxy(object): + def __init__(self, instance): + self.ref = instance + + if current_system_version.compatible(21, 0): + def get_bool(self, *args, **kwargs): + return xbmcaddon.Settings.getBool(self.ref, *args, **kwargs) + + def set_bool(self, *args, **kwargs): + return xbmcaddon.Settings.setBool(self.ref, *args, **kwargs) + + def get_int(self, *args, **kwargs): + return xbmcaddon.Settings.getInt(self.ref, *args, **kwargs) + + def set_int(self, *args, **kwargs): + return xbmcaddon.Settings.setInt(self.ref, *args, **kwargs) + + def get_str(self, *args, **kwargs): + return xbmcaddon.Settings.getString(self.ref, *args, **kwargs) + + def set_str(self, *args, **kwargs): + return xbmcaddon.Settings.setString(self.ref, *args, **kwargs) + + def get_str_list(self, *args, **kwargs): + return xbmcaddon.Settings.getStringList(self.ref, *args, **kwargs) + + def set_str_list(self, *args, **kwargs): + return xbmcaddon.Settings.setStringList(self.ref, *args, **kwargs) + else: + def get_bool(self, *args, **kwargs): + return xbmcaddon.Addon.getSettingBool(self.ref(), *args, **kwargs) + + def set_bool(self, *args, **kwargs): + return xbmcaddon.Addon.setSettingBool(self.ref(), *args, **kwargs) + + def get_int(self, *args, **kwargs): + return xbmcaddon.Addon.getSettingInt(self.ref(), *args, **kwargs) + + def set_int(self, *args, **kwargs): + return xbmcaddon.Addon.setSettingInt(self.ref(), *args, **kwargs) + + def get_str(self, *args, **kwargs): + return xbmcaddon.Addon.getSettingString(self.ref(), *args, **kwargs) + + def set_str(self, *args, **kwargs): + return xbmcaddon.Addon.setSettingString(self.ref(), *args, **kwargs) + + def get_str_list(self, setting): + return xbmcaddon.Addon.getSetting(self.ref(), setting).split(',') + + def set_str_list(self, setting, value): + value = ','.join(value) + return xbmcaddon.Addon.setSetting(self.ref(), setting, value) + + class XbmcPluginSettings(AbstractSettings): - def __init__(self, xbmc_addon): - super(XbmcPluginSettings, self).__init__() + _instances = set() + _proxy = None - self.flush(xbmc_addon) + def __init__(self, xbmc_addon=None): + self.flush(xbmc_addon, fill=True) - if current_system_version.compatible(21, 0): - _class = xbmcaddon.Settings + def flush(self, xbmc_addon=None, fill=False, flush_all=True): + if not xbmc_addon: + if fill: + xbmc_addon = xbmcaddon.Addon(ADDON_ID) + else: + if self.__class__._instances: + if not flush_all: + self.__class__._instances.discard(self._proxy.ref()) + else: + self.__class__._instances.clear() + del self._proxy + self._proxy = None + return + else: + fill = False + self._echo = get_kodi_setting_bool('debug.showloginfo') + self._cache = {} + if current_system_version.compatible(21, 0): + self._proxy = SettingsProxy(xbmc_addon.getSettings()) # set methods in new Settings class are documented as returning a # bool, True if value was set, False otherwise, similar to how the # old set setting methods of the Addon class function. Except they # don't actually return anything... # Ignore return value until bug is fixed in Kodi - XbmcPluginSettings._check_set = False - - self.__dict__.update({ - '_get_bool': _class.getBool, - '_set_bool': _class.setBool, - '_get_int': _class.getInt, - '_set_int': _class.setInt, - '_get_str': _class.getString, - '_set_str': _class.setString, - '_get_str_list': _class.getStringList, - '_set_str_list': _class.setStringList, - }) + self._check_set = False else: - _class = xbmcaddon.Addon - - def _get_string_list(store, setting): - return _class.getSetting(store, setting).split(',') - - def _set_string_list(store, setting, value): - value = ','.join(value) - return _class.setSetting(store, setting, value) - - self.__dict__.update({ - '_get_bool': _class.getSettingBool, - '_set_bool': _class.setSettingBool, - '_get_int': _class.getSettingInt, - '_set_int': _class.setSettingInt, - '_get_str': _class.getSettingString, - '_set_str': _class.setSettingString, - '_get_str_list': _get_string_list, - '_set_str_list': _set_string_list, - }) - - @classmethod - def flush(cls, xbmc_addon=None): - if not xbmc_addon: - del cls._instance - cls._instance = None - return - - cls._echo = get_kodi_setting_bool('debug.showloginfo') - cls._cache = {} - if current_system_version.compatible(21, 0): - cls._instance = xbmc_addon.getSettings() - else: - cls._instance = xbmcaddon.Addon() + if fill: + self.__class__._instances.add(xbmc_addon) + self._proxy = SettingsProxy(ref(xbmc_addon)) def get_bool(self, setting, default=None, echo=None): if setting in self._cache: @@ -87,7 +119,7 @@ def get_bool(self, setting, default=None, echo=None): error = False try: - value = bool(self._get_bool(self._instance, setting)) + value = bool(self._proxy.get_bool(setting)) except (TypeError, ValueError) as exc: error = exc try: @@ -111,7 +143,7 @@ def get_bool(self, setting, default=None, echo=None): def set_bool(self, setting, value, echo=None): try: - error = not self._set_bool(self._instance, setting, value) + error = not self._proxy.set_bool(setting, value) if error and self._check_set: error = 'failed' else: @@ -134,7 +166,7 @@ def get_int(self, setting, default=-1, process=None, echo=None): error = False try: - value = int(self._get_int(self._instance, setting)) + value = int(self._proxy.get_int(setting)) if process: value = process(value) except (TypeError, ValueError) as exc: @@ -160,7 +192,7 @@ def get_int(self, setting, default=-1, process=None, echo=None): def set_int(self, setting, value, echo=None): try: - error = not self._set_int(self._instance, setting, value) + error = not self._proxy.set_int(setting, value) if error and self._check_set: error = 'failed' else: @@ -183,7 +215,7 @@ def get_string(self, setting, default='', echo=None): error = False try: - value = self._get_str(self._instance, setting) or default + value = self._proxy.get_str(setting) or default except (RuntimeError, TypeError) as exc: error = exc value = default @@ -207,7 +239,7 @@ def get_string(self, setting, default='', echo=None): def set_string(self, setting, value, echo=None): try: - error = not self._set_str(self._instance, setting, value) + error = not self._proxy.set_str(setting, value) if error and self._check_set: error = 'failed' else: @@ -238,7 +270,7 @@ def get_string_list(self, setting, default=None, echo=None): error = False try: - value = self._get_str_list(self._instance, setting) + value = self._proxy.get_str_list(setting) if not value: value = [] if default is None else default except (RuntimeError, TypeError) as exc: @@ -256,7 +288,7 @@ def get_string_list(self, setting, default=None, echo=None): def set_string_list(self, setting, value, echo=None): try: - error = not self._set_str_list(self._instance, setting, value) + error = not self._proxy.set_str_list(setting, value) if error and self._check_set: error = 'failed' else: @@ -272,6 +304,3 @@ def set_string_list(self, setting, value, echo=None): status=error if error else 'success' )) return not error - - -atexit.register(XbmcPluginSettings.flush) From eb8e104f52376d5538c078e1152b93a9453cbadd Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:36:43 +1000 Subject: [PATCH 44/53] Improve creation and tear down of XbmcContext instances --- .../kodion/context/abstract_context.py | 7 +- .../kodion/context/xbmc/xbmc_context.py | 122 ++++++++++-------- .../kodion/plugin/xbmc/xbmc_plugin.py | 2 +- .../youtube_plugin/kodion/plugin_runner.py | 3 - .../youtube_plugin/kodion/script_actions.py | 1 - .../youtube/helper/yt_setup_wizard.py | 2 +- 6 files changed, 75 insertions(+), 62 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/context/abstract_context.py b/resources/lib/youtube_plugin/kodion/context/abstract_context.py index e48964a1d..5c2aefaee 100644 --- a/resources/lib/youtube_plugin/kodion/context/abstract_context.py +++ b/resources/lib/youtube_plugin/kodion/context/abstract_context.py @@ -28,6 +28,10 @@ class AbstractContext(object): + _initialized = False + _addon = None + _settings = None + _BOOL_PARAMS = { 'ask_for_quality', 'audio_only', @@ -114,6 +118,7 @@ def __init__(self, path='/', params=None, plugin_id=''): self._plugin_handle = -1 self._plugin_id = plugin_id self._plugin_name = None + self._plugin_icon = None self._version = 'UNKNOWN' self._path = self.create_path(path) @@ -343,7 +348,7 @@ def get_addon_path(self): raise NotImplementedError() def get_icon(self): - return self.create_resource_path('media/icon.png') + return self._plugin_icon def get_fanart(self): return self.create_resource_path('media/fanart.jpg') diff --git a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py index 026666912..f9cdcf741 100644 --- a/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py +++ b/resources/lib/youtube_plugin/kodion/context/xbmc/xbmc_context.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, division, unicode_literals +import atexit import sys from weakref import proxy @@ -38,9 +39,6 @@ class XbmcContext(AbstractContext): - _addon = None - _settings = None - _KODI_UI_SUBTITLE_OPTIONS = None LOCAL_MAP = { @@ -276,20 +274,23 @@ class XbmcContext(AbstractContext): } def __new__(cls, *args, **kwargs): - if not cls._addon: - cls._addon = xbmcaddon.Addon(ADDON_ID) - cls._settings = XbmcPluginSettings(cls._addon) + self = super(XbmcContext, cls).__new__(cls) + + if not cls._initialized: + addon = xbmcaddon.Addon(ADDON_ID) + cls._addon = addon + cls._settings = XbmcPluginSettings(addon) - if not cls._KODI_UI_SUBTITLE_OPTIONS: cls._KODI_UI_SUBTITLE_OPTIONS = { None, # No setting value - cls.localize(231), # None - cls.localize(13207), # Forced only - cls.localize(308), # Original language - cls.localize(309), # UI language + self.localize(231), # None + self.localize(13207), # Forced only + self.localize(308), # Original language + self.localize(309), # UI language } - self = super(XbmcContext, cls).__new__(cls) + cls._initialized = True + return self def __init__(self, @@ -300,8 +301,15 @@ def __init__(self, self._plugin_id = plugin_id or ADDON_ID if self._plugin_id != ADDON_ID: - self._addon = xbmcaddon.Addon(self._plugin_id) - self._settings = XbmcPluginSettings(self._addon) + addon = xbmcaddon.Addon(ADDON_ID) + self._addon = addon + self._settings = XbmcPluginSettings(addon) + + self._addon_path = make_dirs(self._addon.getAddonInfo('path')) + self._data_path = make_dirs(self._addon.getAddonInfo('profile')) + self._plugin_name = self._addon.getAddonInfo('name') + self._plugin_icon = self._addon.getAddonInfo('icon') + self._version = self._addon.getAddonInfo('version') self._ui = None self._video_playlist = None @@ -309,11 +317,7 @@ def __init__(self, self._video_player = None self._audio_player = None - self._plugin_name = self._addon.getAddonInfo('name') - self._version = self._addon.getAddonInfo('version') - - self._addon_path = make_dirs(self._addon.getAddonInfo('path')) - self._data_path = make_dirs(self._addon.getAddonInfo('profile')) + atexit.register(self.tear_down) def init(self): num_args = len(sys.argv) @@ -348,9 +352,6 @@ def init(self): def get_region(self): pass # implement from abstract - def addon(self): - return self._addon - def is_plugin_path(self, uri, uri_path='', partial=False): uri_path = ('plugin://%s/%s' % (self.get_id(), uri_path)).rstrip('/') if not partial: @@ -435,24 +436,31 @@ def get_data_path(self): def get_addon_path(self): return self._addon_path - def get_settings(self, flush=False): - if flush or not self._settings: + def clear_settings(self): + if self._plugin_id != ADDON_ID and self._settings: + self._settings.flush() + if self.__class__._settings: + self.__class__._settings.flush() + + def get_settings(self, refresh=False): + if refresh or not self._settings: if self._plugin_id != ADDON_ID: - self._addon = xbmcaddon.Addon(self._plugin_id) - self._settings = XbmcPluginSettings(self._addon) + addon = xbmcaddon.Addon(self._plugin_id) + self._addon = addon + self._settings = XbmcPluginSettings(addon) else: - self.__class__._addon = xbmcaddon.Addon(ADDON_ID) - self.__class__._settings = XbmcPluginSettings(self._addon) + addon = xbmcaddon.Addon(ADDON_ID) + self.__class__._addon = addon + self.__class__._settings = XbmcPluginSettings(addon) return self._settings - @classmethod - def localize(cls, text_id, default_text=None): + def localize(self, text_id, default_text=None): if default_text is None: default_text = 'Undefined string ID: |{0}|'.format(text_id) if not isinstance(text_id, int): try: - text_id = cls.LOCAL_MAP[text_id] + text_id = self.LOCAL_MAP[text_id] except KeyError: try: text_id = int(text_id) @@ -467,7 +475,7 @@ def localize(cls, text_id, default_text=None): (see: http://kodi.wiki/view/Language_support) but we do it anyway. I want some of the localized strings for the views of a skin. """ - source = cls._addon if 30000 <= text_id < 31000 else xbmc + source = self._addon if 30000 <= text_id < 31000 else xbmc result = source.getLocalizedString(text_id) result = to_unicode(result) if result else default_text return result @@ -689,29 +697,33 @@ def get_listitem_info(detail_name): return xbmc.getInfoLabel('Container.ListItem(0).' + detail_name) def tear_down(self): - self._settings.flush() - try: - del self._addon - del self._settings - except AttributeError: - pass - try: - del self.__class__._addon - self.__class__._addon = None - del self.__class__._settings - self.__class__._settings = None - except AttributeError: - pass - del self._ui - self._ui = None - del self._video_playlist - self._video_playlist = None - del self._audio_playlist - self._audio_playlist = None - del self._video_player - self._video_player = None - del self._audio_player - self._audio_player = None + self.clear_settings() + attrs = ( + '_addon', + '_settings', + ) + for attr in attrs: + try: + if self._plugin_id != ADDON_ID: + delattr(self, attr) + delattr(self.__class__, attr) + setattr(self.__class__, attr, None) + except AttributeError: + pass + + attrs = ( + '_ui', + '_video_playlist', + '_audio_playlist', + '_video_player', + '_audio_player', + ) + for attr in attrs: + try: + delattr(self, attr) + setattr(self, attr, None) + except AttributeError: + pass def wakeup(self): self.get_ui().set_property(WAKEUP) diff --git a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py index 3e76f863a..024692253 100644 --- a/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py +++ b/resources/lib/youtube_plugin/kodion/plugin/xbmc/xbmc_plugin.py @@ -133,7 +133,7 @@ def run(self, provider, context, refresh=False): if ui.get_property(CHECK_SETTINGS): provider.reset_client() - settings = context.get_settings(flush=True) + settings = context.get_settings(refresh=True) ui.clear_property(CHECK_SETTINGS) else: settings = context.get_settings() diff --git a/resources/lib/youtube_plugin/kodion/plugin_runner.py b/resources/lib/youtube_plugin/kodion/plugin_runner.py index 886d02722..35f5aceca 100644 --- a/resources/lib/youtube_plugin/kodion/plugin_runner.py +++ b/resources/lib/youtube_plugin/kodion/plugin_runner.py @@ -10,7 +10,6 @@ from __future__ import absolute_import, division, unicode_literals -import atexit from copy import deepcopy from platform import python_version @@ -31,8 +30,6 @@ _profiler = Profiler(enabled=False) -atexit.register(_context.tear_down) - def run(context=_context, plugin=_plugin, diff --git a/resources/lib/youtube_plugin/kodion/script_actions.py b/resources/lib/youtube_plugin/kodion/script_actions.py index d8e7bca7f..4b3763179 100644 --- a/resources/lib/youtube_plugin/kodion/script_actions.py +++ b/resources/lib/youtube_plugin/kodion/script_actions.py @@ -350,5 +350,4 @@ def run(argv): _user_actions(context, action, params) return finally: - context.tear_down() ui.clear_property(WAIT_FLAG) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py b/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py index c2b64339f..951331dff 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_setup_wizard.py @@ -446,7 +446,7 @@ def process_subtitles(_provider, context, step, steps): context.execute('RunScript({addon_id},config/subtitles)'.format( addon_id=ADDON_ID ), wait_for=WAIT_FLAG) - context.get_settings(flush=True) + context.get_settings(refresh=True) return step From bb8c2f4146ecc5b344d9a09a205d633763a9dd69 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:37:46 +1000 Subject: [PATCH 45/53] Improve tear down of Profiler instances --- resources/lib/youtube_plugin/kodion/debug.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/debug.py b/resources/lib/youtube_plugin/kodion/debug.py index 8736296b4..6898b32ec 100644 --- a/resources/lib/youtube_plugin/kodion/debug.py +++ b/resources/lib/youtube_plugin/kodion/debug.py @@ -10,6 +10,7 @@ from __future__ import absolute_import, division, unicode_literals +import atexit import os from .logger import log_debug @@ -95,6 +96,11 @@ def print_stats(self, *args, **kwargs): *args, **kwargs ) + def tear_down(self, *args, **kwargs): + return super(Profiler.Proxy, self).__call__().tear_down( + *args, **kwargs + ) + _instances = set() def __new__(cls, *args, **kwargs): @@ -120,9 +126,7 @@ def __init__(self, if enabled and not lazy: self._create_profiler() - def __del__(self): - # pylint: disable=protected-access - self.__class__._instances.discard(self) + atexit.register(self.tear_down) def __enter__(self): if not self._enabled: @@ -139,7 +143,7 @@ def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): reuse=self._reuse ))) if not self._reuse: - self.__del__() + self.tear_down() def __call__(self, func=None, name=__name__, reuse=False): """Decorator used to profile function calls""" @@ -187,7 +191,7 @@ def wrapper(*args, **kwargs): return result if not self._enabled: - self.__del__() + self.tear_down() return func return wrapper @@ -243,3 +247,6 @@ def print_stats(self): log_debug('Profiling stats: {0}'.format(self.get_stats( reuse=self._reuse ))) + + def tear_down(self): + self.__class__._instances.discard(self) From 4f92799db41aae5fd307fb557e9b672596610c6a Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:39:04 +1000 Subject: [PATCH 46/53] Improve tear down of BaseRequestsClass instances - __del__ is unnecessary and could prevent GC in older Python versions --- resources/lib/youtube_plugin/kodion/network/requests.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/network/requests.py b/resources/lib/youtube_plugin/kodion/network/requests.py index 8fb22a543..4bb0e4644 100644 --- a/resources/lib/youtube_plugin/kodion/network/requests.py +++ b/resources/lib/youtube_plugin/kodion/network/requests.py @@ -55,9 +55,6 @@ def __init__(self, exc_type=None): else: self._default_exc = (RequestException,) - def __del__(self): - self._session.close() - def __enter__(self): return self From 1c1fbf34b0b81a947c9ee6955a53e28f3bd98a47 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Sun, 19 May 2024 14:41:37 +1000 Subject: [PATCH 47/53] Add dummy end of settings marker - TODO: Improve settings maintenance to remove any unused user setting after this marker --- .../youtube_plugin/kodion/constants/const_settings.py | 1 + .../youtube_plugin/kodion/settings/abstract_settings.py | 3 ++- resources/settings.xml | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/kodion/constants/const_settings.py b/resources/lib/youtube_plugin/kodion/constants/const_settings.py index 8b80d4a38..b011235b0 100644 --- a/resources/lib/youtube_plugin/kodion/constants/const_settings.py +++ b/resources/lib/youtube_plugin/kodion/constants/const_settings.py @@ -13,6 +13,7 @@ SETUP_WIZARD = 'kodion.setup_wizard' # (bool) SETUP_WIZARD_RUNS = 'kodion.setup_wizard.forced_runs' # (int) +SETTINGS_END = '|end_settings_marker|' # (bool) MPD_VIDEOS = 'kodion.mpd.videos' # (bool) MPD_STREAM_SELECT = 'kodion.mpd.stream.select' # (int) diff --git a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py index d43341fdf..bc7a60a57 100644 --- a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py +++ b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py @@ -95,7 +95,7 @@ def get_search_history_size(self): def setup_wizard_enabled(self, value=None): # Increment min_required on new release to enable oneshot on first run - min_required = 3 + min_required = 4 if value is False: self.set_int(settings.SETUP_WIZARD_RUNS, min_required) @@ -107,6 +107,7 @@ def setup_wizard_enabled(self, value=None): forced_runs = self.get_int(settings.SETUP_WIZARD_RUNS, 0) if forced_runs < min_required: self.set_int(settings.SETUP_WIZARD_RUNS, min_required) + self.set_bool(settings.SETTINGS_END, True) return True return self.get_bool(settings.SETUP_WIZARD, False) diff --git a/resources/settings.xml b/resources/settings.xml index 07f5385e1..affc627ca 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -1130,5 +1130,14 @@ + + + + 4 + false + + + + From 2a506c047cf526aa3bf104af5d1a45d952500d01 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 11:18:58 +1000 Subject: [PATCH 48/53] Fix adding video to playlist #764 --- resources/lib/youtube_plugin/youtube/helper/yt_playlist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py index fb9bbe3ab..26cdd2a5f 100644 --- a/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py +++ b/resources/lib/youtube_plugin/youtube/helper/yt_playlist.py @@ -178,7 +178,7 @@ def _process_select_playlist(provider, context): else: watch_later_id = None - thumb_size = context.get_settings().get_thumb_size() + thumb_size = context.get_settings().get_thumbnail_size() default_thumb = context.create_resource_path('media', 'playlist.png') while True: From 8052d653ce6de1dcff8aaa71f078bf265f8dc41a Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 11:20:16 +1000 Subject: [PATCH 49/53] Fix possible error when failing to get channelId for username --- resources/lib/youtube_plugin/youtube/helper/resource_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py index cb4963087..4bd74b2eb 100644 --- a/resources/lib/youtube_plugin/youtube/helper/resource_manager.py +++ b/resources/lib/youtube_plugin/youtube/helper/resource_manager.py @@ -47,7 +47,7 @@ def get_channels(self, ids, defer_cache=False): self._function_cache.ONE_DAY, _refresh=refresh, username=channel_id - ) + ) or {} items = data.get('items', [{'id': 'mine'}]) try: From e8300db0ac02e6488f17857ba335e6e38cde0b46 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 11:23:41 +1000 Subject: [PATCH 50/53] Add addition properties for VideoItem - completed (any scheduled stream that has completed, may have been live or a premiere) - short (less than 60s in duration) - vod (regular video) --- .../youtube_plugin/kodion/items/video_item.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/resources/lib/youtube_plugin/kodion/items/video_item.py b/resources/lib/youtube_plugin/kodion/items/video_item.py index cb2cadc4d..430430482 100644 --- a/resources/lib/youtube_plugin/kodion/items/video_item.py +++ b/resources/lib/youtube_plugin/kodion/items/video_item.py @@ -49,8 +49,13 @@ def __init__(self, name, uri, image='', fanart=''): self._last_played = None self._start_percent = None self._start_time = None + + self._completed = False self._live = False + self._short = False self._upcoming = False + self._vod = False + self.subtitles = None self._headers = None self.license_key = None @@ -240,6 +245,14 @@ def set_scheduled_start_utc(self, date_time): def get_scheduled_start_utc(self): return self._scheduled_start_utc + @property + def completed(self): + return self._completed + + @completed.setter + def completed(self, value): + self._completed = value + @property def live(self): return self._live @@ -248,6 +261,14 @@ def live(self): def live(self, value): self._live = value + @property + def short(self): + return self._short + + @short.setter + def short(self, value): + self._short = value + @property def upcoming(self): return self._upcoming @@ -256,6 +277,14 @@ def upcoming(self): def upcoming(self, value): self._upcoming = value + @property + def vod(self): + return self._vod + + @vod.setter + def vod(self, value): + self._vod = value + def add_genre(self, genre): if self._genres is None: self._genres = [] From 576b95133ed09d15c54ed3c3f18e7120dc3c8026 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 11:34:17 +1000 Subject: [PATCH 51/53] Update video filtering for live streams #755 - Add options to filter/hide the following types of videos - shorts - upcoming videos (either live streams or premieres) - upcoming live streams - currently live streams - upcoming premieres - completed videos (previously either live or premieres) - No currently live streams will be shown in searches, only in the Live subfolder - No currently live or upcoming live streams will be shown in channels, only in the Live subfolder - Completed live stream will continue to be shown in settings - Should also be much faster - Live subfolders will not be filtered by default - Filter selection dialog has option to also filter the Live subfolder - Most other listings will be filtered using whatever options have been set --- .../resource.language.en_gb/strings.po | 18 +++- .../kodion/constants/const_settings.py | 2 +- .../kodion/settings/abstract_settings.py | 31 +++++- .../lib/youtube_plugin/youtube/helper/tv.py | 41 ++++---- .../youtube_plugin/youtube/helper/utils.py | 98 +++++++++++++++---- .../lib/youtube_plugin/youtube/helper/v3.py | 21 ++-- .../lib/youtube_plugin/youtube/provider.py | 65 +++++++++--- resources/settings.xml | 21 +++- 8 files changed, 226 insertions(+), 71 deletions(-) diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 6ba6eac82..d4bce3261 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -1250,7 +1250,7 @@ msgid "Edited" msgstr "" msgctxt "#30736" -msgid "Hide short videos (1 minute or less)" +msgid "Shorts (1 minute or less)" msgstr "" msgctxt "#30737" @@ -1536,3 +1536,19 @@ msgstr "" msgctxt "#30807" msgid "Use channel name as" msgstr "" + +msgctxt "#30808" +msgid "Hide videos from listings" +msgstr "" + +msgctxt "#30809" +msgid "All upcoming videos" +msgstr "" + +msgctxt "#30810" +msgid "All previously streamed (completed) videos" +msgstr "" + +msgctxt "#30811" +msgid "Filter Live folders" +msgstr "" diff --git a/resources/lib/youtube_plugin/kodion/constants/const_settings.py b/resources/lib/youtube_plugin/kodion/constants/const_settings.py index b011235b0..a2099a811 100644 --- a/resources/lib/youtube_plugin/kodion/constants/const_settings.py +++ b/resources/lib/youtube_plugin/kodion/constants/const_settings.py @@ -27,7 +27,7 @@ SUBTITLE_DOWNLOAD = 'kodion.subtitle.download' # (bool) ITEMS_PER_PAGE = 'kodion.content.max_per_page' # (int) -HIDE_SHORT_VIDEOS = 'youtube.hide_shorts' # (bool) +HIDE_VIDEOS = 'youtube.view.hide_videos' # (list[string]) SAFE_SEARCH = 'kodion.safe.search' # (int) AGE_GATE = 'kodion.age.gate' # (bool) diff --git a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py index bc7a60a57..881d56d5e 100644 --- a/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py +++ b/resources/lib/youtube_plugin/kodion/settings/abstract_settings.py @@ -385,8 +385,35 @@ def stream_select(self, value=None): return self._STREAM_SELECT[value] return self._STREAM_SELECT[default] - def hide_short_videos(self): - return self.get_bool(settings.HIDE_SHORT_VIDEOS, False) + _DEFAULT_FILTER = { + 'shorts': True, + 'upcoming': True, + 'upcoming_live': True, + 'live': True, + 'premieres': True, + 'completed': True, + 'vod': True, + } + + def item_filter(self, update=None): + types = dict.fromkeys(self.get_string_list(settings.HIDE_VIDEOS), False) + types = dict(self._DEFAULT_FILTER, **types) + if update: + if 'live_folder' in update: + if 'live_folder' in types: + types.update(update) + else: + types.update({ + 'upcoming': True, + 'upcoming_live': True, + 'live': True, + 'premieres': True, + 'completed': True, + }) + types['vod'] = False + else: + types.update(update) + return types def client_selection(self, value=None): if value is not None: diff --git a/resources/lib/youtube_plugin/youtube/helper/tv.py b/resources/lib/youtube_plugin/youtube/helper/tv.py index e12ca60a5..155670d9f 100644 --- a/resources/lib/youtube_plugin/youtube/helper/tv.py +++ b/resources/lib/youtube_plugin/youtube/helper/tv.py @@ -15,39 +15,40 @@ def tv_videos_to_items(provider, context, json_data): - result = [] - video_id_dict = {} - - item_params = {'video_id': None} - incognito = context.get_param('incognito', False) + incognito = context.get_param('incognito') + settings = context.get_settings() + use_play_data = not incognito and settings.use_local_history() + item_filter = settings.item_filter() + + item_params = { + 'video_id': None, + } if incognito: - item_params['incognito'] = incognito - - items = json_data.get('items', []) - for item in items: + item_params['incognito'] = True + video_id_dict = {} + channel_item_dict = {} + for item in json_data.get('items', []): video_id = item['id'] item_params['video_id'] = video_id - item_uri = context.create_uri(('play',), item_params) - video_item = VideoItem(item['title'], uri=item_uri) + video_item = VideoItem( + item['title'], context.create_uri(('play',), item_params) + ) if incognito: video_item.set_play_count(0) - - result.append(video_item) - video_id_dict[video_id] = video_item - use_play_data = not incognito and context.get_settings().use_local_history() - - channel_item_dict = {} utils.update_video_infos(provider, context, video_id_dict, channel_items_dict=channel_item_dict, - use_play_data=use_play_data) + use_play_data=use_play_data, + item_filter=item_filter) utils.update_fanarts(provider, context, channel_item_dict) - if context.get_settings().hide_short_videos(): - result = utils.filter_short_videos(result) + if item_filter: + result = utils.filter_videos(video_id_dict.values(), **item_filter) + else: + result = list(video_id_dict.values()) # next page next_page_token = json_data.get('next_page_token') diff --git a/resources/lib/youtube_plugin/youtube/helper/utils.py b/resources/lib/youtube_plugin/youtube/helper/utils.py index ebf7ac28e..daaa468f1 100644 --- a/resources/lib/youtube_plugin/youtube/helper/utils.py +++ b/resources/lib/youtube_plugin/youtube/helper/utils.py @@ -350,6 +350,7 @@ def update_video_infos(provider, context, video_id_dict, channel_items_dict=None, live_details=True, use_play_data=True, + item_filter=None, data=None): video_ids = list(video_id_dict) if not video_ids and not data: @@ -380,7 +381,6 @@ def update_video_infos(provider, context, video_id_dict, ask_quality = not default_web_urls and settings.ask_for_video_quality() audio_only = settings.audio_only() channel_name_aliases = settings.get_channel_name_aliases() - hide_shorts = settings.hide_short_videos() show_details = settings.show_detailed_description() subtitles_prompt = settings.get_subtitle_selection() == 1 thumb_size = settings.get_thumbnail_size() @@ -421,26 +421,66 @@ def update_video_infos(provider, context, video_id_dict, if not yt_item or 'snippet' not in yt_item: continue - snippet = yt_item['snippet'] - play_data = use_play_data and yt_item.get('play_data') - broadcast_type = snippet.get('liveBroadcastContent') - video_item.live = broadcast_type == 'live' - video_item.upcoming = broadcast_type == 'upcoming' - # duration - if (not (video_item.live or video_item.upcoming) - and play_data and 'total_time' in play_data): + play_data = use_play_data and yt_item.get('play_data') + if play_data and 'total_time' in play_data: duration = play_data['total_time'] else: duration = yt_item.get('contentDetails', {}).get('duration') if duration: duration = datetime_parser.parse(duration) - # subtract 1s because YouTube duration is +1s too long - duration = (duration.seconds - 1) if duration.seconds else None + if duration.seconds: + # subtract 1s because YouTube duration is +1s too long + duration = duration.seconds - 1 if duration: video_item.set_duration_from_seconds(duration) - if hide_shorts and duration <= 60: + if duration <= 60: + video_item.short = True + + broadcast_type = snippet.get('liveBroadcastContent') + video_item.live = broadcast_type == 'live' + video_item.upcoming = broadcast_type == 'upcoming' + + upload_status = yt_item.get('status', {}).get('uploadStatus') + if upload_status == 'processed' and duration: + video_item.live = False + elif upload_status == 'uploaded' and not duration: + video_item.live = True + + if 'liveStreamingDetails' in yt_item: + streaming_details = yt_item['liveStreamingDetails'] + if 'actualStartTime' in streaming_details: + start_at = streaming_details['actualStartTime'] + video_item.upcoming = False + if 'actualEndTime' in streaming_details: + video_item.completed = True + else: + start_at = streaming_details.get('scheduledStartTime') + video_item.upcoming = True + else: + video_item.completed = False + video_item.live = False + video_item.upcoming = False + video_item.vod = True + start_at = None + + if item_filter and ( + (not item_filter['shorts'] + and video_item.short) + or (not item_filter['completed'] + and video_item.completed) + or (not item_filter['live'] + and video_item.live and not video_item.upcoming) + or (not item_filter['upcoming'] + and video_item.upcoming) + or (not item_filter['premieres'] + and video_item.upcoming and not video_item.live) + or (not item_filter['upcoming_live'] + and video_item.upcoming and video_item.live) + or (not item_filter['vod'] + and video_item.vod) + ): continue if not video_item.live and play_data: @@ -458,11 +498,6 @@ def update_video_infos(provider, context, video_id_dict, elif video_item.live: video_item.set_play_count(0) - if ((video_item.live or video_item.upcoming) - and 'liveStreamingDetails' in yt_item): - start_at = yt_item['liveStreamingDetails'].get('scheduledStartTime') - else: - start_at = None if start_at: datetime = datetime_parser.parse(start_at) video_item.set_scheduled_start_utc(datetime) @@ -471,7 +506,15 @@ def update_video_infos(provider, context, video_id_dict, video_item.set_aired_from_datetime(local_datetime) video_item.set_premiered_from_datetime(local_datetime) video_item.set_date_from_datetime(local_datetime) - type_label = localize('live' if video_item.live else 'upcoming') + if video_item.upcoming: + if video_item.live: + type_label = localize('live.upcoming') + else: + type_label = localize('upcoming') + elif video_item.live: + type_label = localize('live') + else: + type_label = localize(335) # "Start" start_at = '{type_label} {start_at}'.format( type_label=type_label, start_at=datetime_parser.get_scheduled_start( @@ -1004,9 +1047,24 @@ def add_related_video_to_playlist(provider, context, client, v3, video_id): break -def filter_short_videos(items): +def filter_videos(items, + shorts=True, + live=True, + upcoming_live=True, + premieres=True, + upcoming=True, + completed=True, + vod=True, + **_kwargs): return [ item for item in items - if not item.playable or not 0 <= item.get_duration() <= 60 + if (not item.playable or ( + (completed and item.completed) + or (live and item.live and not item.upcoming) + or (premieres and upcoming and item.upcoming and not item.live) + or (upcoming_live and upcoming and item.upcoming and item.live) + or (vod and shorts and item.vod) + or (vod and not shorts and item.vod and not item.short) + )) ] diff --git a/resources/lib/youtube_plugin/youtube/helper/v3.py b/resources/lib/youtube_plugin/youtube/helper/v3.py index b48204223..e8f08e96d 100644 --- a/resources/lib/youtube_plugin/youtube/helper/v3.py +++ b/resources/lib/youtube_plugin/youtube/helper/v3.py @@ -14,7 +14,7 @@ from .utils import ( THUMB_TYPES, - filter_short_videos, + filter_videos, get_thumbnail, make_comment_item, update_channel_infos, @@ -27,7 +27,7 @@ from ...kodion.items import DirectoryItem, NextPageItem, VideoItem -def _process_list_response(provider, context, json_data): +def _process_list_response(provider, context, json_data, item_filter): yt_items = json_data.get('items', []) if not yt_items: context.log_warning('v3 response: Items list is empty') @@ -242,7 +242,8 @@ def _process_list_response(provider, context, json_data): 'upd_kwargs': { 'data': None, 'live_details': True, - 'use_play_data': use_play_data + 'use_play_data': use_play_data, + 'item_filter': item_filter, }, 'complete': False, 'defer': False, @@ -382,23 +383,27 @@ def response_to_items(provider, json_data, sort=None, reverse=False, - process_next_page=True): + process_next_page=True, + item_filter=None): is_youtube, kind = _parse_kind(json_data) if not is_youtube: context.log_debug('v3 response: Response discarded, is_youtube=False') return [] if kind in _KNOWN_RESPONSE_KINDS: - result = _process_list_response(provider, context, json_data) + item_filter = context.get_settings().item_filter(item_filter) + result = _process_list_response( + provider, context, json_data, item_filter + ) else: raise KodionException("Unknown kind '%s'" % kind) + if item_filter: + result = filter_videos(result, **item_filter) + if sort is not None: result.sort(key=sort, reverse=reverse) - if context.get_settings().hide_short_videos(): - result = filter_short_videos(result) - # no processing of next page item if not result or not process_next_page: return result diff --git a/resources/lib/youtube_plugin/youtube/provider.py b/resources/lib/youtube_plugin/youtube/provider.py index 1295425bc..3943dcd56 100644 --- a/resources/lib/youtube_plugin/youtube/provider.py +++ b/resources/lib/youtube_plugin/youtube/provider.py @@ -403,19 +403,32 @@ def _on_channel_live(self, context, re_match): result = [] channel_id = re_match.group('channel_id') - page_token = context.get_param('page_token', '') - safe_search = context.get_settings().safe_search() + params = context.get_params() + page_token = params.get('page_token', '') - # no caching - json_data = self.get_client(context).search(q='', - search_type='video', - event_type='live', - channel_id=channel_id, - page_token=page_token, - safe_search=safe_search) - if not json_data: - return False - result.extend(v3.response_to_items(self, context, json_data)) + client = self.get_client(context) + function_cache = context.get_function_cache() + resource_manager = self.get_resource_manager(context) + + playlists = function_cache.run(resource_manager.get_related_playlists, + function_cache.ONE_DAY, + channel_id=channel_id) + upload_playlist = playlists.get('uploads', '') + if upload_playlist: + json_data = function_cache.run(client.get_playlist_items, + function_cache.ONE_MINUTE * 5, + _refresh=params.get('refresh'), + playlist_id=upload_playlist, + page_token=page_token) + if not json_data: + return result + + result.extend(v3.response_to_items( + self, context, json_data, + item_filter={ + 'live_folder': True, + }, + )) return result @@ -542,7 +555,9 @@ def _on_channel(self, context, re_match): ) result.append(live_item) - playlists = resource_manager.get_related_playlists(channel_id) + playlists = function_cache.run(resource_manager.get_related_playlists, + function_cache.ONE_DAY, + channel_id=channel_id) upload_playlist = playlists.get('uploads', '') if upload_playlist: json_data = function_cache.run(client.get_playlist_items, @@ -553,7 +568,13 @@ def _on_channel(self, context, re_match): if not json_data: return result - result.extend(v3.response_to_items(self, context, json_data)) + result.extend(v3.response_to_items( + self, context, json_data, + item_filter={ + 'live': False, + 'upcoming_live': False, + }, + )) return result @@ -788,6 +809,7 @@ def on_search(self, search_text, context, re_match): location = params.get('location') page = params.get('page', 1) page_token = params.get('page_token', '') + order = params.get('order', 'relevance') search_type = params.get('search_type', 'video') safe_search = context.get_settings().safe_search() @@ -796,7 +818,10 @@ def on_search(self, search_text, context, re_match): else: context.set_content(content.LIST_CONTENT) - if page == 1 and search_type == 'video' and not event_type and not hide_folders: + if (page == 1 + and search_type == 'video' + and not event_type + and not hide_folders): if not channel_id and not location: channel_params = dict(params, search_type='channel') item_label = context.localize('channels') @@ -846,10 +871,18 @@ def on_search(self, search_text, context, re_match): safe_search=safe_search, page_token=page_token, channel_id=channel_id, + order=order, location=location) if not json_data: return False - result.extend(v3.response_to_items(self, context, json_data)) + result.extend(v3.response_to_items( + self, context, json_data, + item_filter={ + 'live_folder': True, + } if event_type else { + 'live': False, + }, + )) return result @RegisterProviderPath('^/config/(?P[^/]+)/?$') diff --git a/resources/settings.xml b/resources/settings.xml index affc627ca..5fb17f92a 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -214,10 +214,25 @@ false - + 0 - false - + + + + + + + + + + + + , + + + true + true + From 59a4e6027abd79ae6f0effcf4e7027c89b2dbd24 Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 11:34:50 +1000 Subject: [PATCH 52/53] Misc tidy ups --- resources/lib/youtube_plugin/kodion/items/video_item.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/resources/lib/youtube_plugin/kodion/items/video_item.py b/resources/lib/youtube_plugin/kodion/items/video_item.py index 430430482..f6d7e8590 100644 --- a/resources/lib/youtube_plugin/kodion/items/video_item.py +++ b/resources/lib/youtube_plugin/kodion/items/video_item.py @@ -43,9 +43,10 @@ def __init__(self, name, uri, image='', fanart=''): self._track_number = None self._studios = None self._artists = None - self._play_count = None - self._uses_isa = None + self._production_code = None self._mediatype = None + + self._play_count = None self._last_played = None self._start_percent = None self._start_time = None @@ -56,15 +57,16 @@ def __init__(self, name, uri, image='', fanart=''): self._upcoming = False self._vod = False + self._uses_isa = None self.subtitles = None self._headers = None self.license_key = None + self._video_id = None self._channel_id = None self._subscription_id = None self._playlist_id = None self._playlist_item_id = None - self._production_code = None def set_play_count(self, play_count): self._play_count = int(play_count or 0) From 02a954f5c8327826b25056385eda29068dfcc16c Mon Sep 17 00:00:00 2001 From: MoojMidge <56883549+MoojMidge@users.noreply.github.com> Date: Thu, 23 May 2024 12:00:49 +1000 Subject: [PATCH 53/53] Update changelog --- changelog.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog.txt b/changelog.txt index f6b85cfd8..e59d45666 100644 --- a/changelog.txt +++ b/changelog.txt @@ -10,6 +10,10 @@ - Fix changing fanart type on My Subscriptions #751 - Fix not using thumbnail fanart for channels/subscriptions/playlists when enabled #751 - Fix My Subscriptions threading issues #529 +- Fix failing to login #759 +- Improve resource usage +- Fix adding video to playlist #764 +- Update video filtering for live streams #755 ### Changed - Use internal Kodi resume enable/disable for all playback #693