diff --git a/.github/workflows/release-development-repository.yml b/.github/workflows/release-development-repository.yml new file mode 100644 index 000000000..eff9c9dc2 --- /dev/null +++ b/.github/workflows/release-development-repository.yml @@ -0,0 +1,131 @@ +name: Generate Development Repository +on: + workflow_dispatch: + inputs: + prerelease: + description: 'Is this a pre-release? [true|false]' + required: false + default: 'true' + + push: + tags: + - 'v*' + - '*-dev' + +jobs: + repo-gen: + if: github.repository == 'anxdpanic/plugin.video.youtube' || github.event_name == 'workflow_dispatch' + name: Generate development repository and upload to mirrors + runs-on: ubuntu-22.04 + permissions: + contents: write + + steps: + - name: Release Status + id: release + run: | + version=${GITHUB_REF/refs\/tags\//} + if [[ ! -z ${{ github.event.inputs.prerelease }} ]] ; + then + echo "pre-release=${{ github.event.inputs.prerelease }}" >> $GITHUB_OUTPUT + elif [[ $version == *[-+]@(alpha|beta|dev)*([.0-9a-z]) ]] ; + then + echo "pre-release=true" >> $GITHUB_OUTPUT + else + echo "pre-release=false" >> $GITHUB_OUTPUT + fi + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + sudo apt update + sudo apt install lftp + python -m pip install --upgrade pip + pip install lxml + + - name: Checkout Official Add-on + uses: actions/checkout@v4 + with: + path: ./official_testing/${{ github.event.repository.name }} + ref: "master" + + - name: Checkout Unofficial Add-on + uses: actions/checkout@v4 + with: + path: ./unofficial_testing/${{ github.event.repository.name }} + ref: "nexus-unofficial" + + - name: Checkout Unofficial Add-on + if: steps.release.outputs.pre-release == 'false' + uses: actions/checkout@v4 + with: + path: ./unofficial/${{ github.event.repository.name }} + ref: "nexus-unofficial" + + - name: Checkout Add-on + uses: actions/checkout@v4 + with: + path: ${{ github.event.repository.name }} + + - name: Download Repository Add-on Files from FTP + run: | + lftp -c "\ + set ssl:verify-certificate false; \ + set sftp:auto-confirm yes; \ + open ${{ secrets.SFTP }}; \ + cd /root/anxdpanic/kodi/youtube/; \ + mirror --verbose --use-pget-n=8 -c --verbose repository.yt.testing_official ./official_testing/repository.yt.testing_official; \ + mirror --verbose --use-pget-n=8 -c --verbose repository.yt.testing_unofficial ./unofficial_testing/repository.yt.testing_unofficial; \ + bye \ + " + + - name: Download Unofficial Repository Add-on Files from FTP + if: steps.release.outputs.pre-release == 'false' + run: | + lftp -c "\ + set ssl:verify-certificate false; \ + set sftp:auto-confirm yes; \ + open ${{ secrets.SFTP }}; \ + cd /root/anxdpanic/kodi/youtube/; \ + mirror --verbose --use-pget-n=8 -c --verbose repository.yt.testing_unofficial ./unofficial/repository.yt.unofficial; \ + bye \ + " + + - name: Copy Generator Script + run: | + cp ./official/${{ github.event.repository.name }}/.scripts/.prepare_repository.py . + cp ./official/${{ github.event.repository.name }}/.scripts/.config.json . + + - name: Generate Repository + run: | + python .prepare_repository.py --prerelease=${{ steps.release.outputs.pre-release }} + + - name: Mirror Repository to FTP + run: | + lftp -c "\ + set ssl:verify-certificate false; \ + set sftp:auto-confirm yes; \ + open ${{ secrets.SFTP }}; \ + cd /root/anxdpanic/kodi/youtube/; \ + lcd repository/; \ + mirror --verbose --use-pget-n=8 -c --reverse --verbose official_testing official_testing; \ + mirror --verbose --use-pget-n=8 -c --reverse --verbose unofficial_testing unofficial_testing; \ + bye \ + " + + - name: Mirror Unofficial Repository to FTP + if: steps.release.outputs.pre-release == 'false' + run: | + lftp -c "\ + set ssl:verify-certificate false; \ + set sftp:auto-confirm yes; \ + open ${{ secrets.SFTP }}; \ + cd /root/anxdpanic/kodi/youtube/; \ + lcd repository/; \ + mirror --verbose --use-pget-n=8 -c --reverse --verbose unofficial unofficial; \ + bye \ + " diff --git a/.scripts/.config.json b/.scripts/.config.json new file mode 100644 index 000000000..27fca1c75 --- /dev/null +++ b/.scripts/.config.json @@ -0,0 +1,38 @@ +{ + "compress_addons": true, + "path": { + "source": "./official_testing", + "source-unofficial-testing": "./unofficial_testing", + "source-unofficial": "./unofficial", + "repository": "./repository/official_testing", + "repository-unofficial-testing": "./repository/unofficial_testing", + "repository-unofficial": "./repository/unofficial" + }, + "development_versions": true, + "ignored": { + "addons": [ + ], + "files": [ + "README.md", + "append_to_languages.py", + "new_strings.txt", + "convert_settings.py", + "core.po", + "changelog.txt" + ], + "file_starts_with": [ + "." + ], + "file_ends_with": [ + ".pyo", + ".pyc", + "template.md" + ], + "directories": [ + ".git", + ".idea", + "test_youtube", + "mock" + ] + } +} diff --git a/.scripts/.prepare_repository.py b/.scripts/.prepare_repository.py new file mode 100644 index 000000000..ba8710a16 --- /dev/null +++ b/.scripts/.prepare_repository.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python3 +""" +This is an: +- addons.xml generator +- addons.xml.md5 generator +- optional auto-compressor (including handling of icons, fanart and changelog) + +Python 2.7 -> 3.7 + +Compression of addons in repositories has many benefits, including: + - Protects addon downloads from corruption. + - Smaller addon file size resulting in faster downloads and + less space / bandwidth used on the repository. + - Ability to 'roll back' addon updates in Kodi to previous versions. + +To enable the auto-compressor, set the compress_addons setting to True +If you do this you must make sure the 'datadir zip' parameter in the addon.xml +of your repository file is set to 'true'. + +Please bump __revision__ one decimal point and add your name to credits when making changes +""" +import sys + +from lxml import etree +import hashlib +import json +import os +import shutil +import zipfile + +__script__ = '.prepare_repository.py' +__revision__ = '7.0' +__homepage__ = 'https://forum.kodi.tv/showthread.php?tid=129401' +__credits__ = 'Unobtanium, anxdpanic' +__license__ = 'GPL-3.0-only' + + +class Generator: + """ + Generates a new addons.xml file from each addons addon.xml file + and a new addons.xml.md5 hash file. Must be run from the root of + the checked-out repo. Only handles single depth folder structure. + """ + + def __init__(self): + + # paths + self.addons_xml = os.path.join(REPOSITORY_PATH, 'addons.xml') + self.addons_xml_md5 = os.path.join(REPOSITORY_PATH, 'addons.xml.md5') + + # call master function + self.generate_addons_files() + + def generate_addons_files(self): + # addon list + addons = os.listdir(ZIPS_PATH) + # final addons text + addons_xml = u'\n\n' + + found_an_addon = False + path = '' + + # loop through and add each addons addon.xml file + for addon in addons: + if addon in IGNORED_ADDONS: + continue + try: + addon = os.path.join(ZIPS_PATH, addon) + # skip any file or .svn folder + if is_addon_dir(addon): + + # create path + path = os.path.join(addon, 'addon.xml') + + if os.path.exists(path): + found_an_addon = True + + # split lines for stripping + xml_lines = read_file(path) + xml_lines = xml_lines.splitlines() + + # new addon + addon_xml = '' + + # loop through cleaning each line + for line in xml_lines: + if isinstance(line, bytes): + line.decode('utf-8') + line = str(line) + # skip encoding format line + if line.find('= 0: + continue + # add line + addon_xml += line.rstrip() + '\n' + + # we succeeded so add to our final addons.xml text + addons_xml += addon_xml.rstrip() + '\n\n' + + except Exception as error: # pylint: disable=broad-except + # missing or poorly formatted addon.xml + print('Excluding %s for %s' % (path, error)) + + # clean and add closing tag + addons_xml = addons_xml.strip() + u'\n\n' + + # only generate files if we found an addon.xml + if found_an_addon: + # save files + save_file(self.addons_xml, addons_xml.encode('UTF-8')) + self.generate_md5_file() + + # notify user + print('Updated addons xml and addons.xml.md5 files') + print(' ') + else: + print('Could not find any addons, so script has done nothing.') + + def generate_md5_file(self): + try: + # create a new md5 hash + contents = read_file(self.addons_xml) + md5 = hashlib.md5(contents.encode('utf-8')).hexdigest() + # save file + save_file(self.addons_xml_md5, md5) + + except Exception as error: # pylint: disable=broad-except + # oops + print('An error occurred creating addons.xml.md5 file!\n%s' % error) + + +class Compressor: + def __init__(self): + # variables used later on + self.addon_name = None + self.addon_path = None + self.addon_folder_contents = None + self.xml_file = None + self.addon_xml = None + self.addon_version_number = None + self.addon_zip_path = None + self.addon_path_zips = None + + # run the master method of the class, when class is initialised. + # only do so if we want addons compressed. + if COMPRESS_ADDONS: + self.compress_addons() + + def compress_addons(self): + source_directory = os.listdir(SOURCE_PATH) + for addon in source_directory: + if addon in IGNORED_ADDONS: + continue + # set variables + self.addon_name = str(addon) + self.addon_path = os.path.join(SOURCE_PATH, addon) + self.addon_path_zips = os.path.join(ZIPS_PATH, addon) + # skip any file or .svn folder. + if is_addon_dir(self.addon_path): + # set another variable + self.addon_folder_contents = os.listdir(self.addon_path) + + # check if addon has a current zipped release in it. + addon_zip_exists = self._get_zipped_addon_path() + + # checking for addon.xml and try reading it. + addon_xml_exists = self._read_addon_xml() + if addon_xml_exists: + self._get_artwork() + + # now addon.xml has been read, scrape version number from it. + # we need this when naming the zip (and if it exists the changelog) + self._read_version_number() + + if not addon_zip_exists: + addon_zip_exists = self._get_zipped_addon_path() + if not addon_zip_exists: + tags = '' + + print('Create compressed %s%s release for [%s] v%s' % + (tags, self._get_release_type(), self.addon_name, + self.addon_version_number)) + self.create_compressed_addon_release() + + def _get_zip_name(self): + if not self.addon_version_number: + self._read_version_number() + + zip_name = self.addon_name + zip_name += '-' + self.addon_version_number + '.zip' + + return zip_name + + def _get_zipped_addon_path(self): + # get name of addon zip file. returns False if not found. + addon_xml_exists = self._read_addon_xml() + if addon_xml_exists: + self._read_version_number() + + zip_name = self._get_zip_name() + + if not os.path.exists(self.addon_path_zips): + if not self.addon_path_zips.endswith('zips'): + make_path = self.addon_path_zips + os.makedirs(make_path) + + folder_contents = os.listdir(self.addon_path_zips) + + for potential_zip in folder_contents: + if zip_name == potential_zip: + self.addon_zip_path = os.path.join(self.addon_path_zips, zip_name) + return True + # if loop is not broken by returning the addon path, zip was not found so return False + self.addon_zip_path = None + return False + + def _extract_addon_xml_to_release_folder(self): + with zipfile.ZipFile(self.addon_path_zips, 'r') as zip_file: + for filename in zip_file.namelist(): + if filename.find('addon.xml'): + zip_file.extract(filename, self.addon_path_zips) + break + + @staticmethod + def recursive_zipper(directory, zip_file): + # initialize zipping module + ignored_files = IGNORED_FILES + + with zipfile.ZipFile(zip_file, 'w', compression=zipfile.ZIP_DEFLATED) as zipped_file: + # get length of characters of what we will use as the root path + root_len = len(os.path.dirname(os.path.abspath(directory))) + + # recursive writer + for root, _, files in os.walk(directory): + # subtract the source file's root from the archive root - + # ie. make /Users/me/desktop/zip_me.txt into just /zip_me.txt + archive_root = os.path.abspath(root)[root_len:] + + for repo_file in files: + + if repo_file.endswith(IGNORED_FILES_END): + continue + if repo_file.startswith(IGNORED_FILES_START): + continue + if any(match for match in ignored_files if repo_file == match): + continue + + full_path = str(os.path.join(root, repo_file)) + if any(bl in full_path for bl in IGNORED_DIRECTORIES): + continue + + archive_name = str(os.path.join(archive_root, repo_file)) + zipped_file.write(full_path, archive_name, zipfile.ZIP_DEFLATED) + + def create_compressed_addon_release(self): + # create a zip of the addon into repo root directory, + # tagging it with '-x.x.x' release number scraped from addon.xml + + def _copy_asset(asset): + if not asset: + return + if asset == 'changelog.txt': + asset = 'changelog-' + self.addon_version_number + '.txt' + + asset_path = os.path.join(self.addon_path_zips, asset) + try: + try: + os.makedirs(os.path.dirname(asset_path)) + except(IOError, OSError) as _: + pass + + shutil.copyfile(os.path.join(self.addon_path, asset), asset_path) + except (shutil.Error, IOError) as _: + pass + + zip_name = self._get_zip_name() + zip_path = os.path.join(ZIPS_PATH, zip_name) + + # zip full directories + self.recursive_zipper(self.addon_path, zip_path) + + # now move the zip into the addon folder, + # which we will now treat as the 'addon release directory' + + os.rename(zip_path, os.path.join(self.addon_path_zips, zip_name)) + + art = self._get_artwork() + + _copy_asset('addon.xml') + _copy_asset('changelog.txt') + _copy_asset(art.get('icon', 'icon.png')) + _copy_asset(art.get('fanart', 'fanart.jpg')) + _copy_asset(art.get('banner', '')) + _copy_asset(art.get('clearlogo', '')) + + for screenshot in art.get('screenshot', []): + _copy_asset(screenshot) + + def _read_addon_xml(self): + # check for addon.xml and try and read it. + addon_xml_path = os.path.join(self.addon_path, 'addon.xml') + if os.path.exists(addon_xml_path): + # load whole text into string + self.xml_file = read_file(addon_xml_path) + self.addon_xml = etree.fromstring(self.xml_file.encode('utf-8')) + # return True if we found and read the addon.xml + return True + # return False if we couldn't find the addon.xml + return False + + def _read_version_number(self): + # find the header of the addon. + version = self.addon_xml.get('version') + self.addon_version_number = version + + def _get_release_type(self): + + if not self.addon_version_number: + self._read_version_number() + + if 'alpha' in self.addon_version_number: + tag = 'alpha' + elif 'beta' in self.addon_version_number: + tag = 'beta' + else: + tag = 'stable' + + return tag + + def _get_artwork(self): + art = { + 'banner': '', + 'clearlogo': '', + 'fanart': 'fanart.jpg', + 'icon': 'icon.png', + 'screenshot': [], + } + + def _set_asset(metadata, asset_type, findall=False): + if findall: + asset = metadata.findall('./assets/' + asset_type) + else: + asset = metadata.find('./assets/' + asset_type) + if asset is not None: + if findall: + if (isinstance(asset, list) and + len(asset) > 0 and isinstance(asset[0].text, str)): + art[asset_type] = [a.text for a in asset] + else: + if isinstance(asset.text, str): + art[asset_type] = asset.text + + extensions = self.addon_xml.findall('./extension') + for extension in extensions: + if extension.get('point') == 'xbmc.addon.metadata': + _set_asset(extension, 'icon') + _set_asset(extension, 'fanart') + _set_asset(extension, 'banner') + _set_asset(extension, 'clearlogo') + _set_asset(extension, 'screenshot', findall=True) + + break + + return art + + +def is_addon_dir(addon): + # this function is used by both classes. + # very simple and weak check that it is an addon dir. + # intended to be fast, not totally accurate. + # skip any file or .svn folder + if not os.path.isdir(addon) or addon == '.git' or addon.endswith('zips') or addon == 'zips': + return False + + return True + + +def read_file(filename, is_json=False): + if is_json: + with open(filename, 'r', encoding='utf-8') as open_file: + return json.load(open_file) + else: + with open(filename, 'r', encoding='utf-8') as open_file: + return open_file.read() + + +def save_file(filename_and_path, contents): + if isinstance(contents, bytes): + contents = contents.decode('utf-8') + try: + with open(filename_and_path, 'w', encoding='utf-8') as open_file: + open_file.write(contents) + except Exception as error: # pylint: disable=broad-except + # oops + print('An error occurred saving %s file!\n%s' % (filename_and_path, error)) + + +def loose_version(v): + filled = [] + for point in v.split('.'): + filled.append(point.zfill(8)) + return tuple(filled) + + +def split_version(v): + filled = [] + for point in v.split('.'): + if '~' in point: + for p in point.split('~'): + if 'alpha' in p or 'beta' in p: + p = '~' + p + filled.append(p) + else: + filled.append(point) + return filled + + +if __name__ == '__main__': + CONFIG = read_file('.config.json', is_json=True) + + COMPRESS_ADDONS = bool(CONFIG.get('compress_addons')) + + IGNORED_ADDONS = list(set(CONFIG.get('ignored', {}).get('addons', []))) + IGNORED_FILES = list(set(CONFIG.get('ignored', {}).get('files', []))) + IGNORED_FILES_START = tuple(set(CONFIG.get('ignored', {}).get('file_starts_with', []))) + IGNORED_FILES_END = tuple(set(CONFIG.get('ignored', {}).get('file_ends_with', []))) + IGNORED_DIRECTORIES = list(set(CONFIG.get('ignored', {}).get('directories', []))) + + SOURCE_PATH = CONFIG.get('path', {}).get('source', '') + REPOSITORY_PATH = CONFIG.get('path', {}).get('repository', '') + ZIPS_PATH = os.path.join(REPOSITORY_PATH, 'zips') + SOURCE_PATH_UNOFFICIAL = CONFIG.get('path', {}).get('source-unofficial', '') + REPOSITORY_PATH_UNOFFICIAL = CONFIG.get('path', {}).get('repository-unofficial', '') + ZIPS_PATH_UNOFFICIAL = os.path.join(REPOSITORY_PATH_UNOFFICIAL, 'zips') + SOURCE_PATH_UNOFFICIAL_TESTING = CONFIG.get('path', {}).get('source-unofficial-testing', '') + REPOSITORY_PATH_UNOFFICIAL_TESTING = CONFIG.get('path', {}).get('repository-unofficial-testing', '') + ZIPS_PATH_UNOFFICIAL_TESTING = os.path.join(REPOSITORY_PATH_UNOFFICIAL_TESTING, 'zips') + + print(__script__) + print('Version: v' + str(__revision__)) + print('License: ' + __license__) + print('Credits: ' + __credits__) + print('Homepage: ' + __homepage__) + print(' ') + + print('Paths:') + print(' Source: ' + SOURCE_PATH) + print(' Repository: ' + REPOSITORY_PATH) + print(' Zips: ' + ZIPS_PATH) + print(' Unofficial Development Source: ' + SOURCE_PATH_UNOFFICIAL) + print(' Unofficial Development Repository: ' + REPOSITORY_PATH_UNOFFICIAL) + print(' Unofficial Development Zips: ' + ZIPS_PATH_UNOFFICIAL) + print(' Unofficial Source: ' + SOURCE_PATH_UNOFFICIAL) + print(' Unofficial Repository: ' + REPOSITORY_PATH_UNOFFICIAL) + print(' Unofficial Zips: ' + ZIPS_PATH_UNOFFICIAL) + print(' ') + + print('Compress Add-ons: ' + str(COMPRESS_ADDONS)) + print(' ') + + print(' ') + + print('Generating official testing repository') + Compressor() + Generator() + + if os.path.isdir(SOURCE_PATH_UNOFFICIAL_TESTING): + print('Generating unofficial testing repository') + SOURCE_PATH = SOURCE_PATH_UNOFFICIAL_TESTING + REPOSITORY_PATH = REPOSITORY_PATH_UNOFFICIAL_TESTING + ZIPS_PATH = ZIPS_PATH_UNOFFICIAL_TESTING + + Compressor() + Generator() + + if len(sys.argv) > 1 and sys.argv[1] == '--prerelease=false': + if os.path.isdir(SOURCE_PATH_UNOFFICIAL): + print('Generating unofficial repository') + SOURCE_PATH = SOURCE_PATH_UNOFFICIAL + REPOSITORY_PATH = REPOSITORY_PATH_UNOFFICIAL + ZIPS_PATH = ZIPS_PATH_UNOFFICIAL + + Compressor() + Generator()