Merge pull request #466 from MoojMidge/master

New client fixes

resources/language/resource.language.en_au/strings.po

@@ -1238,3 +1238,11 @@ msgstr ""
 msgctxt "#30737"
 msgid "Use alternate client details"
 msgstr ""
+
+msgctxt "#30738"
+msgid "Alternate #1"
+msgstr ""
+
+msgctxt "#30739"
+msgid "Alternate #2"
+msgstr ""

resources/language/resource.language.en_gb/strings.po

@@ -1246,3 +1246,11 @@ msgstr ""
 msgctxt "#30737"
 msgid "Use alternate client details"
 msgstr ""
+
+msgctxt "#30738"
+msgid "Alternate #1"
+msgstr ""
+
+msgctxt "#30739"
+msgid "Alternate #2"
+msgstr ""

resources/language/resource.language.en_nz/strings.po

@@ -1238,3 +1238,11 @@ msgstr ""
 msgctxt "#30737"
 msgid "Use alternate client details"
 msgstr ""
+
+msgctxt "#30738"
+msgid "Alternate #1"
+msgstr ""
+
+msgctxt "#30739"
+msgid "Alternate #2"
+msgstr ""

resources/language/resource.language.en_us/strings.po

@@ -1239,3 +1239,11 @@ msgstr ""
 msgctxt "#30737"
 msgid "Use alternate client details"
 msgstr ""
+
+msgctxt "#30738"
+msgid "Alternate #1"
+msgstr ""
+
+msgctxt "#30739"
+msgid "Alternate #2"
+msgstr ""

resources/lib/youtube_plugin/kodion/constants/const_settings.py

@@ -47,4 +47,4 @@
 
 API_CONFIG_PAGE = 'youtube.api.config.page'  # (bool)
 
-ALTERNATIVE_CLIENT = 'youtube.client.alternative'  # (bool)
+CLIENT_SELECTION = 'youtube.client.selection'  # (int)

resources/lib/youtube_plugin/kodion/impl/abstract_settings.py

@@ -250,5 +250,5 @@ def remote_friendly_search(self):
     def hide_short_videos(self):
         return self.get_bool(constants.setting.HIDE_SHORT_VIDEOS, False)
 
-    def use_alternative_client(self):
-        return self.get_bool(constants.setting.ALTERNATIVE_CLIENT, False)
+    def client_selection(self):
+        return self.get_int(constants.setting.CLIENT_SELECTION, 0)

resources/lib/youtube_plugin/youtube/helper/subtitles.py

@@ -7,10 +7,7 @@
 """
 
 from html import unescape
-from urllib.parse import parse_qs
-from urllib.parse import urlencode
-from urllib.parse import urlsplit
-from urllib.parse import urlunsplit
+from urllib.parse import (parse_qs, urlsplit, urlunsplit, urlencode, urljoin)
 
 import xbmcvfs
 import requests
@@ -120,6 +117,7 @@ def get_subtitles(self):
             return list(set(list_of_subs))
         else:
             self.context.log_debug('Unknown language_enum: %s for subtitles' % str(languages))
+        return []
 
     def _get_all(self):
         list_of_subs = []
@@ -179,12 +177,12 @@ def _get(self, language='en', language_name=None, no_asr=False):
 
         subtitle_url = None
         if (caption_track is None) and has_translation:
-            base_url = self.caption_track.get('baseUrl')
+            base_url = self._normalize_url(self.caption_track.get('baseUrl'))
             if base_url:
                 subtitle_url = self.set_query_param(base_url, 'type', 'track')
                 subtitle_url = self.set_query_param(subtitle_url, 'tlang', language)
         elif caption_track is not None:
-            base_url = caption_track.get('baseUrl')
+            base_url = self._normalize_url(caption_track.get('baseUrl'))
             if base_url:
                 subtitle_url = self.set_query_param(base_url, 'type', 'track')
 
@@ -236,3 +234,15 @@ def set_query_param(url, name, value):
             new_query_string = new_query_string.encode('utf-8')
 
         return urlunsplit((scheme, netloc, path, new_query_string, fragment))
+
+    @staticmethod
+    def _normalize_url(url):
+        if not url:
+            url = ''
+        elif url.startswith(('http://', 'https://')):
+            pass
+        elif url.startswith('//'):
+            url = urljoin('https:', url)
+        elif url.startswith('/'):
+            url = urljoin('https://www.youtube.com', url)
+        return url

resources/lib/youtube_plugin/youtube/helper/video_info.py

@@ -485,27 +485,35 @@ class VideoInfo(object):
     }
 
     CLIENTS = {
+        # 4k no VP9 HDR
         'android_testsuite': {
             'id': 30,
+            'api_key': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
             'details': {
                 'clientName': 'ANDROID_TESTSUITE',
                 'clientVersion': '1.9',
-                'androidSdkVersion': '31',
+                'androidSdkVersion': '29',
                 'osName': 'Android',
-                'osVersion': '12',
+                'osVersion': '10',
                 'platform': 'MOBILE',
             },
+            'headers': {
+                'User-Agent': 'com.google.android.youtube/{details[clientVersion]} (Linux; U; Android {details[osVersion]}; US) gzip',
+                'X-YouTube-Client-Name': '{id}',
+                'X-YouTube-Client-Version': '{details[clientVersion]}',
+            },
         },
         # Connection to stream URL closes after 30s
         # Subsequent attempts to connect result in 403 Forbidden error
         'android': {
             'id': 3,
+            'api_key': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
             'details': {
                 'clientName': 'ANDROID',
-                'clientVersion': '17.36.4',
-                'androidSdkVersion': '31',
+                'clientVersion': '17.31.35',
+                'androidSdkVersion': '29',
                 'osName': 'Android',
-                'osVersion': '12',
+                'osVersion': '10',
                 'platform': 'MOBILE',
             },
             'headers': {
@@ -518,13 +526,14 @@ class VideoInfo(object):
         # Limited to 720p on some videos
         'android_embedded': {
             'id': 55,
+            'api_key': 'AIzaSyCjc_pVEDi4qsv5MtC2dMXzpIaDoRFLsxw',
             'details': {
                 'clientName': 'ANDROID_EMBEDDED_PLAYER',
                 'clientVersion': '17.36.4',
                 'clientScreen': 'EMBED',
-                'androidSdkVersion': '31',
+                'androidSdkVersion': '29',
                 'osName': 'Android',
-                'osVersion': '12',
+                'osVersion': '10',
                 'platform': 'MOBILE',
             },
             'headers': {
@@ -533,27 +542,33 @@ class VideoInfo(object):
                 'X-YouTube-Client-Version': '{details[clientVersion]}',
             },
         },
-        # Fallback for videos requiring age verification
-        # Requires handling of nsig to overcome throttling (TODO)
-        'tv': {
-            'id': 85,
+        # 4k with HDR
+        # Some videos block this client, may also require embedding enabled
+        'android_youtube_tv': {
+            'id': 29,
+            'api_key': 'AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w',
             'details': {
-                'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER',
-                'clientVersion': '2.0',
-                'platform': 'TV',
+                'clientName': 'ANDROID_UNPLUGGED',
+                'clientVersion': '6.36',
+                'androidSdkVersion': '29',
+                'osName': 'Android',
+                'osVersion': '10',
+                'platform': 'MOBILE',
             },
             'headers': {
+                'User-Agent': 'com.google.android.apps.youtube.unplugged/{details[clientVersion]} (Linux; U; Android {details[osVersion]}; US) gzip',
                 'X-YouTube-Client-Name': '{id}',
                 'X-YouTube-Client-Version': '{details[clientVersion]}',
             },
         },
-        # Second fallback for restricted videos
-        # Requires handling of signatureCipher and nsig (TODD)
+        # Used for misc api requests by default
+        # Requires handling of nsig to overcome throttling (TODO)
         'web': {
-            'id': 62,
+            'id': 1,
+            'api_key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
             'details': {
-                'clientName': 'WEB_CREATOR',
-                'clientVersion': '1.20210909.07.00',
+                'clientName': 'WEB',
+                'clientVersion': '2.20220801.00.00',
             },
             # Headers from the "Galaxy S20 Ultra" from Chrome dev tools device emulation
             'headers': {
@@ -579,28 +594,38 @@ class VideoInfo(object):
     PRIORITISED_CLIENTS = None
 
     def __init__(self, context, access_token='', api_key='', language='en-US'):
+        settings = context.get_settings()
+
         self.video_id = None
         self._context = context
         self._data_cache = self._context.get_data_cache()
-        self._verify = context.get_settings().verify_ssl()
+        self._verify = settings.verify_ssl()
         self._language = language.replace('-', '_')
         self._access_token = access_token
         self._api_key = api_key
         self._player_js = None
         self._calculate_n = True
         self._cipher = None
 
-        if self._context.get_settings().use_alternative_client():
+        client_selection = settings.client_selection()
+        # Alternate #1
+        if client_selection == 1:
             self.PRIORITISED_CLIENTS = (self.CLIENTS['android_embedded'],
-                                        self.CLIENTS['android_testsuite'],
-                                        self.CLIENTS['web'])
+                                        self.CLIENTS['android_youtube_tv'],
+                                        self.CLIENTS['android_testsuite'])
+        # Alternate #2
+        elif client_selection == 2:
+            self.PRIORITISED_CLIENTS = (self.CLIENTS['android'],
+                                        self.CLIENTS['android_youtube_tv'],
+                                        self.CLIENTS['android_testsuite'])
+        # Default
         else:
-            self.PRIORITISED_CLIENTS = (self.CLIENTS['android_testsuite'],
-                                        self.CLIENTS['android_embedded'],
-                                        self.CLIENTS['web'])
+            self.PRIORITISED_CLIENTS = (self.CLIENTS['android_youtube_tv'],
+                                        self.CLIENTS['android_testsuite'],
+                                        self.CLIENTS['android_embedded'])
 
-        self.CLIENTS['_common']['hl'] = context.get_settings().get_string('youtube.language', 'en_US').replace('-', '_')
-        self.CLIENTS['_common']['gl'] = context.get_settings().get_string('youtube.region', 'US')
+        self.CLIENTS['_common']['hl'] = settings.get_string('youtube.language', 'en_US').replace('-', '_')
+        self.CLIENTS['_common']['gl'] = settings.get_string('youtube.region', 'US')
 
     @staticmethod
     def generate_cpn():
@@ -836,7 +861,7 @@ def _process_url_params(self, url):
                 new_query['n'] = new_n
                 new_query['ratebypass'] = 'yes'
             else:
-                self._context.log_debug('nsig handling failed')
+                self._context.log_error('nsig handling failed')
                 self._calculate_n = False
 
         if 'range' not in query:
@@ -890,10 +915,8 @@ def _get_error_details(playability_status, details=None):
     def _method_get_video_info(self):
         if self._access_token:
             auth_header = 'Bearer %s' % self._access_token
-            params = None
         else:
             auth_header = None
-            params = {'key': self._api_key}
 
         video_info_url = 'https://www.youtube.com/youtubei/v1/player'
 
@@ -926,6 +949,9 @@ def _method_get_video_info(self):
                     headers[name] = value.format(**client)
                 if auth_header:
                     headers['Authorization'] = auth_header
+                    params = None
+                else:
+                    params = {'key': client['api_key'] or self._api_key}
                 headers.update(self.CLIENTS['_headers'])
 
                 try:
@@ -962,10 +988,11 @@ def _method_get_video_info(self):
             else:
                 if auth_header:
                     auth_header = None
-                    params = {'key': self._api_key}
                     continue
             # Otherwise skip retrying clients without Authorization header
             break
+        self._context.log_debug('Requested video info with client: {0} (logged {1})'.format(
+            client['details']['clientName'], 'in' if auth_header else 'out'))
 
         # Make a set of URL-quoted headers to be sent to Kodi when requesting
         # the stream during playback. The YT player doesn't seem to use any
@@ -1049,11 +1076,14 @@ def _method_get_video_info(self):
 
             raise YouTubeException(reason)
 
-        captions = player_response.get('captions', {})
-        headers = self.CLIENTS['web']['headers'].copy()
-        headers.update(self.CLIENTS['_headers'])
-        meta_info['subtitles'] = Subtitles(self._context, headers,
-                                           self.video_id, captions).get_subtitles()
+        captions = player_response.get('captions')
+        if captions:
+            headers = self.CLIENTS['web']['headers'].copy()
+            headers.update(self.CLIENTS['_headers'])
+            meta_info['subtitles'] = Subtitles(self._context, headers,
+                                               self.video_id, captions).get_subtitles()
+        else:
+            meta_info['subtitles'] = []
 
         playback_stats = {
             'playback_url': '',

resources/settings.xml

@@ -564,10 +564,17 @@
         </category>
         <category help="" id="advanced" label="30031">
             <group id="1">
-                <setting help="" id="youtube.client.alternative" label="30737" type="boolean">
+                <setting help="" id="youtube.client.selection" label="30737" type="integer">
                     <level>0</level>
-                    <default>false</default>
-                    <control type="toggle"/>
+                    <default>0</default>
+                    <constraints>
+                        <options>
+                            <option label="30532">0</option>
+                            <option label="30738">1</option>
+                            <option label="30739">2</option>
+                        </options>
+                    </constraints>
+                    <control format="string" type="spinner"/>
                 </setting>
                 <setting help="" id="simple.requests.ssl.verify" label="30578" type="boolean">
                     <level>0</level>