Merge pull request #2460 from emilsvennesson/script.module.inputstrea…

…mhelper@matrix

[script.module.inputstreamhelper@matrix] 0.6.1+matrix.1

script.module.inputstreamhelper/README.md

@@ -91,6 +91,10 @@ Please report any issues or bug reports on the [GitHub Issues](https://github.co
 This module is licensed under the **The MIT License**. Please see the [LICENSE.txt](LICENSE.txt) file for details.
 
 ## Releases
+### v0.6.1 (2023-05-30)
+- Performance improvements on Linux ARM (@horstle)
+- This will be the last release for Python 2 i.e. Kodi 18 (Leia) and below. The next release will require Python 3 and Kodi 19 (Matrix) or higher.
+
 ### v0.6.0 (2023-05-03)
 - Initial support for AARCH64 Linux (@horstle)
 - Initial support for AARCH64 Macs (@mediaminister)

script.module.inputstreamhelper/addon.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<addon id="script.module.inputstreamhelper" name="InputStream Helper" version="0.6.0+matrix.1" provider-name="emilsvennesson, dagwieers, mediaminister, horstle">
+<addon id="script.module.inputstreamhelper" name="InputStream Helper" version="0.6.1+matrix.1" provider-name="emilsvennesson, dagwieers, mediaminister, horstle">
   <requires>
     <!--py3 compliant-->
     <import addon="xbmc.python" version="3.0.0"/>
@@ -25,10 +25,14 @@
     <description lang="hr_HR">Jednostavan Kodi modul koji olakšava razvijanje dodataka koji se temelje na InputStream dodatku i reprodukciji DRM zaštićenog sadržaja.</description>
     <description lang="ru_RU">Простой модуль для Kodi, который облегчает жизнь разработчикам дополнений, с использованием InputStream дополнений и воспроизведения DRM контента.</description>
     <news>
+v0.6.1 (2023-05-30)
+- Performance improvements on Linux ARM
+- This will be the last release for Python 2 i.e. Kodi 18 (Leia) and below. The next release will require Python 3 and Kodi 19 (Matrix) or higher.
+
 v0.6.0 (2023-05-03)
-- Initial support for AARCH64 Linux (@horstle)
-- Initial support for AARCH64 Macs (@mediaminister)
-- New option to install a specific version on most platforms (@horstle)
+- Initial support for AARCH64 Linux
+- Initial support for AARCH64 Macs
+- New option to install a specific version on most platforms
 
 v0.5.10 (2022-04-18)
 - Fix automatic submission of release

script.module.inputstreamhelper/lib/inputstreamhelper/config.py

@@ -84,53 +84,22 @@
 # https://www.chromium.org/chromium-os/developer-information-for-chrome-os-devices
 # https://chromiumdash.appspot.com/serving-builds?deviceCategory=Chrome%20OS
 # Last updated: 2023-03-24
-CHROMEOS_RECOVERY_ARM_HWIDS = [
-    'BOB',
-    'BURNET',
-    'COACHZ',
-    'COZMO',
-    'DAMU',
-    'DOJO-EJPG',
-    'DRUWL',
-    'DUMO',
-    'ELM',
-    'ESCHE',
-    'FENNEL',
-    'FENNEL14',
-    'HANA',
-    'HAYATO-YLRO',
-    'HOMESTAR-MBLE',
-    'JUNIPER-HVPU',
-    'KAKADU-WFIQ',
-    'KAPPA',
-    'KAPPA-EWFK',
-    'KATSU',
-    'KENZO-IGRW',
-    'KEVIN',
-    'KODAMA',
-    'KRANE-ZDKS',
-    'MAKOMO-UTTX',
-    'PICO-EXEM',
-    'QUACKINGSTICK',
-    'SCARLET',
-    'SPHERION',
-    'TOMATO-LYVN',
-    'WILLOW-TFIY',
-    'WORMDINGLER-JQAO',
+CHROMEOS_RECOVERY_ARM_BNAMES = [
+    'asurada',
+    'bob',
+    'cherry',
+    'elm',
+    'hana',
+    'jacuzzi',
+    'kevin',
+    'kukui',
+    'scarlet',
+    'strongbad',
 ]
 
-CHROMEOS_RECOVERY_ARM64_HWIDS = [
-    'KINGOFTOWN-KDDA',
-    'LAZOR',
-    'LIMOZEEN',
-    'MAGNETON-LCKC',
-    'PAZQUEL-HGNV',
-    'PAZQUEL-OPNA',
-    'POMPOM-MZVS',
-    'RUSTY-ZNCE',
-    'STEELIX-VZSZ',
-    'TENTACOOL-ZLJE',
-    'TENTACRUEL-VAFH',
+CHROMEOS_RECOVERY_ARM64_BNAMES = [
+    'corsola',
+    'trogdor',
 ]
 
 MINIMUM_INPUTSTREAM_VERSION_ARM64 = {

script.module.inputstreamhelper/lib/inputstreamhelper/utils.py

@@ -164,21 +164,26 @@ def http_download(url, message=None, checksum=None, hash_alg='sha1', dl_size=Non
 
             progress.update(percent, prog_message)
 
-    if checksum and calc_checksum.hexdigest() != checksum:
-        progress.close()
-        req.close()
-        log(4, 'Download failed, checksums do not match!')
-        return False
-
-    if dl_size and stat_file(dl_path).st_size() != dl_size:
-        progress.close()
-        req.close()
-        free_space = sizeof_fmt(diskspace())
-        log(4, 'Download failed, filesize does not match! Filesystem full? Remaining diskspace in temp: {}.'.format(free_space))
-        return False
-
     progress.close()
     req.close()
+
+    checksum_ok = (not checksum or calc_checksum.hexdigest() == checksum)
+    size_ok = (not dl_size or stat_file(dl_path).st_size() == dl_size)
+
+    if not all((checksum_ok, size_ok)):
+        free_space = sizeof_fmt(diskspace())
+        log(4, 'Something may be wrong with the downloaded file.')
+        if not checksum_ok:
+            log(4, 'Provided checksum: {}\nCalculated checksum: {}'.format(checksum, calc_checksum.hexdigest()))
+        if not size_ok:
+            free_space = sizeof_fmt(diskspace())
+            log(4, 'Expected filesize: {}\nReal filesize: {}\nRemaining diskspace: {}'.format(dl_size, stat_file(dl_path).st_size(), free_space))
+
+        if yesno_dialog(localize(30003), localize(30070, filename=filename)):  # file maybe broken. Continue anyway?
+            log(4, 'Continuing despite possibly corrupt file!')
+        else:
+            return False
+
     store('download_path', dl_path)
     return True
 

script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm.py

@@ -17,47 +17,44 @@ def select_best_chromeos_image(devices):
     log(0, 'Find best ARM image to use from the Chrome OS recovery.json')
 
     if userspace64():
-        arm_hwids = config.CHROMEOS_RECOVERY_ARM64_HWIDS
+        arm_bnames = config.CHROMEOS_RECOVERY_ARM64_BNAMES
     else:
-        arm_hwids = config.CHROMEOS_RECOVERY_ARM_HWIDS
+        arm_bnames = config.CHROMEOS_RECOVERY_ARM_BNAMES
 
-    arm_hwids = [h for arm_hwid in arm_hwids for h in ['^{} '.format(arm_hwid), '^{}-.*'.format(arm_hwid), '^{}.*'.format(arm_hwid)]]
     best = None
     for device in devices:
         # Select ARM hardware only
-        for arm_hwid in arm_hwids:
-            if arm_hwid in device['hwidmatch']:
-                hwid = arm_hwid
+        for arm_bname in arm_bnames:
+            if arm_bname == device['file'].split('_')[2]:
+                device['boardname'] = arm_bname  # add this new entry to avoid extracting it from the filename all the time
                 break  # We found an ARM device, rejoice !
         else:
             continue  # Not ARM, skip this device
 
-        device['hwid'] = hwid
-
         # Select the first ARM device
         if best is None:
             best = device
             continue  # Go to the next device
 
-        # Skip identical hwid
-        if hwid == best['hwid']:
+        # Skip identical boardname
+        if device['boardname'] == best['boardname']:
             continue
 
         # Select the newest version
         if parse_version(device['version']) > parse_version(best['version']):
-            log(0, '{device[hwid]} ({device[version]}) is newer than {best[hwid]} ({best[version]})',
+            log(0, '{device[boardname]} ({device[version]}) is newer than {best[boardname]} ({best[version]})',
                 device=device,
                 best=best)
             best = device
 
         # Select the smallest image (disk space requirement)
         elif parse_version(device['version']) == parse_version(best['version']):
-            if int(device['filesize']) + int(device['zipfilesize']) < int(best['filesize']) + int(best['zipfilesize']):
-                log(0, '{device[hwid]} ({device_size}) is smaller than {best[hwid]} ({best_size})',
+            if int(device['zipfilesize']) < int(best['zipfilesize']):
+                log(0, '{device[boardname]} ({device_size}) is smaller than {best[boardname]} ({best_size})',
                     device=device,
+                    device_size=int(device['zipfilesize']),
                     best=best,
-                    device_size=int(device['filesize']) + int(device['zipfilesize']),
-                    best_size=int(best['filesize']) + int(best['zipfilesize']))
+                    best_size=int(best['zipfilesize']))
                 best = device
 
     return best
@@ -96,7 +93,7 @@ def install_widevine_arm(backup_path):
                       localize(30018, diskspace=sizeof_fmt(required_diskspace)))
             return False
 
-        log(2, 'Downloading ChromeOS image for Widevine: {hwid} ({version})'.format(**arm_device))
+        log(2, 'Downloading ChromeOS image for Widevine: {boardname} ({version})'.format(**arm_device))
         url = arm_device['url']
 
         extracted = dl_extract_widevine(url, backup_path, arm_device)

script.module.inputstreamhelper/lib/inputstreamhelper/widevine/arm_chromeos.py

@@ -73,12 +73,67 @@ def chromeos_offset(self):
 
         return offset
 
-    def find_file(self, filename, path_to_file=("opt", "google", "chrome", "WidevineCdm", "_platform_specific", "cros_arm")):
+    def _find_file_in_chunk(self, bfname, chunk):
         """
-        Finds a file at a given path, or search upwards if not found.
+        Checks if the filename is found in the given chunk.
+        Then makes some plausibility checks, to see if it is in fact a proper dentry.
+
+        returns the dentry if found, else False.
+        """
+
+        if bfname in chunk:
+            i_index_pos = chunk.index(bfname) - 8  # the filename is the last element of the dentry, the elements before are 8 bytes total
+            file_entry = self.dir_entry(chunk[i_index_pos:i_index_pos + len(bfname) + 8])  # 8 because see above
+            if file_entry['inode'] < self.sb_dict['s_inodes_count'] and file_entry['name_len'] == len(bfname):
+                return file_entry
+
+            log(0, 'Found filename, but checks did not pass:')
+            log(0, 'inode number: {inode} < {count}, name_len: {name_len} == {len_fname}'.format(inode=file_entry['inode'],
+                                                                                                 count=self.sb_dict['s_inodes_count'],
+                                                                                                 name_len=file_entry['name_len'],
+                                                                                                 len_fname=len(bfname)))
+        return False
+
+    def _find_file_naive(self, fname):
+        """
+        Finds a file by basically searching for the filename as bytes in the bytestream.
+        Searches through the whole image only once, making it fast, but may be unreliable at times.
+
         Returns a directory entry.
+        """
+
+        fname_alt = fname + '#new'  # Sometimes the filename has "#new" at the end
+        bfname = fname.encode('ascii')
+        bfname_alt = fname_alt.encode('ascii')
+        chunksize = 4 * 1024**2
+        chunk1 = self.read_stream(chunksize)
+        while True:
+            chunk2 = self.read_stream(chunksize)
+            if not chunk2:
+                raise ChromeOSError('File {fname} not found in the ChromeOS image'.format(fname=fname))
+
+            chunk = chunk1 + chunk2
+
+            file_entry = self._find_file_in_chunk(bfname, chunk)
+            if file_entry:
+                break
+
+            file_entry = self._find_file_in_chunk(bfname_alt, chunk)
+            if file_entry:
+                break
+
+            chunk1 = chunk2
+
+        return file_entry
+
+    def _find_file_properly(self, filename, path_to_file=("opt", "google", "chrome", "WidevineCdm", "_platform_specific", "cros_arm")):
+        """
+        Finds a file at a given path, or searches upwards if not found.
 
         Assumes the path is roughly correct, else it might take long.
+        It also might take long for ZIP files, since it might have to jump back and forth while traversing down the given path.
+
+        Returns a directory entry.
         """
         root_inode_pos = self._calc_inode_pos(2)
         root_inode_dict = self._inode_table(root_inode_pos)
@@ -107,10 +162,9 @@ def find_file(self, filename, path_to_file=("opt", "google", "chrome", "Widevine
     def _find_file_in_dir(self, filename, dentries):
         """
         Finds a file in a directory or recursively in its subdirectories.
-        Returns a directory entry.
-
         Can take long for deep searches.
-        Returns the first result.
+
+        Returns the first result as a directory entry.
         """
         try:
             return dentries[filename]
@@ -128,6 +182,23 @@ def _find_file_in_dir(self, filename, dentries):
 
             return None
 
+    def find_file(self, filename, path_to_file=None):
+        """
+        Finds a file. Supplying a path could take longer for ZIP files!
+
+        Returns a directory entry.
+        """
+
+        if path_to_file:
+            return self._find_file_properly(filename, path_to_file)
+
+        try:
+            return self._find_file_naive(filename)
+        except ChromeOSError:
+            if self.progress:
+                self.progress.update(5, localize(30071))  # Could not find file, doing proper search
+            return self._find_file_properly(filename)
+
     def _calc_inode_pos(self, inode_num):
         """Calculate the byte position of an inode from its index"""
         blk_group_num = (inode_num - 1) // self.sb_dict['s_inodes_per_group']
@@ -223,10 +294,10 @@ def dir_entry(chunk):
         dir_names = ('inode', 'rec_len', 'name_len', 'file_type', 'name')
         dir_fmt = '<IHBB' + str(len(chunk) - 8) + 's'
 
-        dir_dict = dict(list(zip(dir_names, unpack(dir_fmt, chunk))))
-        dir_dict["name"] = dir_dict["name"][:dir_dict["name_len"]]
+        entry = dict(list(zip(dir_names, unpack(dir_fmt, chunk))))
+        entry["name"] = entry["name"][:entry["name_len"]]
 
-        return dir_dict
+        return entry
 
     def dir_entries(self, dir_file):
         """Returns all directory entries of a directory file as dict of dicts with name as key"""
@@ -371,11 +442,11 @@ def extract_file(self, filename, extract_path):
         try:
             if self.progress:
                 self.progress.update(5, localize(30061))
-            dir_dict = self.find_file(filename)
+            file_entry = self.find_file(filename)
 
             if self.progress:
                 self.progress.update(32, localize(30062))
-            inode_pos = self._calc_inode_pos(dir_dict["inode"])
+            inode_pos = self._calc_inode_pos(file_entry["inode"])
             inode_dict = self._inode_table(inode_pos)
 
             self.write_file(inode_dict, os.path.join(extract_path, filename))

script.module.inputstreamhelper/resources/language/resource.language.en_gb/strings.po

@@ -21,6 +21,10 @@ msgctxt "#30002"
 msgid "This add-on relies on the proprietary decryption module [B]Widevine CDM[/B] for playback."
 msgstr ""
 
+msgctxt "#30003"
+msgid "Warning"
+msgstr ""
+
 msgctxt "#30004"
 msgid "Error"
 msgstr ""
@@ -269,6 +273,14 @@ msgctxt "#30069"
 msgid "Choose which version to install"
 msgstr ""
 
+msgctxt "#30070"
+msgid "Something seems wrong with the downloaded {filename}. Shall we try to continue anyway?"
+msgstr ""
+
+msgctxt "#30071"
+msgid "Could not find the Widevine CDM by a quick scan. Now doing a proper search, but this might take very long..."
+msgstr ""
+
 
 ### INFORMATION DIALOG
 msgctxt "#30800"