From 3f39f8ea20b1fd4e08cfcd0d9ac936f34efa3c7d Mon Sep 17 00:00:00 2001 From: Anton Soroko Date: Fri, 24 Jan 2025 22:10:18 +0300 Subject: [PATCH] Sync with upstream v0.0.10 --- .gitattributes | 2 + README.md | 45 +-- addon.xml | 10 +- lib/cache.py | 130 +++++--- lib/compatibility.py | 61 +++- lib/entries.py | 61 ++-- lib/github.py | 86 +++++ lib/httpserver.py | 56 +++- lib/kodi.py | 19 +- lib/platform/__init__.py | 0 lib/platform/core.py | 26 ++ lib/platform/definitions.py | 35 ++ lib/platform/kodi_platform.py | 97 ++++++ lib/platform/os_platform.py | 44 +++ lib/repository.py | 308 +++++++++++++----- lib/routes.py | 74 ++--- lib/service.py | 9 +- lib/utils.py | 110 +++++++ lib/version.py | 110 +++++++ requirements.txt | 1 + .../resource.language.en_gb/strings.po | 10 +- .../resource.language.es_es/strings.po | 67 ++++ .../resource.language.pt_br/strings.po | 67 ++++ .../resource.language.pt_pt/strings.po | 67 ++++ resources/repository-schema.json | 28 +- resources/settings.xml | 1 + setup.cfg | 5 +- standalone.py | 30 ++ 28 files changed, 1280 insertions(+), 279 deletions(-) create mode 100644 lib/github.py create mode 100644 lib/platform/__init__.py create mode 100644 lib/platform/core.py create mode 100644 lib/platform/definitions.py create mode 100644 lib/platform/kodi_platform.py create mode 100644 lib/platform/os_platform.py create mode 100644 lib/utils.py create mode 100644 lib/version.py create mode 100644 requirements.txt create mode 100644 resources/language/resource.language.es_es/strings.po create mode 100644 resources/language/resource.language.pt_br/strings.po create mode 100644 resources/language/resource.language.pt_pt/strings.po create mode 100644 standalone.py diff --git a/.gitattributes b/.gitattributes index 45751f9..eceac70 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,6 @@ .github export-ignore .gitignore export-ignore .gitattributes export-ignore +requirements.txt export-ignore setup.cfg export-ignore +standalone.py export-ignore diff --git a/README.md b/README.md index a1bf1b1..7fb1e15 100644 --- a/README.md +++ b/README.md @@ -2,34 +2,39 @@ This repository is a fork of a [i96751414's](https://github.com/i96751414) https://github.com/i96751414/repository.github project with minor changes. Many thanks for this project. -This add-on creates a virtual repository for Kodi. This way, one does not need to use a GitHub repository for storing add-ons zips when all that information is already accessible from each add-on repository. +This add-on creates a virtual repository for Kodi. This way, one does not need to use a GitHub repository for storing +add-ons zips when all that information is already accessible from each add-on repository. -It works by setting a HTTP server which has the following endpoints: +It works by setting an HTTP server which has the following endpoints: -|Endpoint|Description| -|--------|-----------| -|http://127.0.0.1:{port}/addons.xml|Main xml file containing all add-ons information| -|http://127.0.0.1:{port}/addons.xml.md5|Checksum of the main xml file| -|http://127.0.0.1:{port}/{addon_id}/{asset_path}|Endpoint for serving add-ons assets/zips| -|http://127.0.0.1:{port}/update|Endpoint for updating repository entries and clearing caches| +| Endpoint | Description | +|-------------------------------------------------|--------------------------------------------------------------| +| http://127.0.0.1:{port}/addons.xml | Main xml file containing all add-ons information | +| http://127.0.0.1:{port}/addons.xml.md5 | Checksum of the main xml file | +| http://127.0.0.1:{port}/{addon_id}/{asset_path} | Endpoint for serving add-ons assets/zips | +| http://127.0.0.1:{port}/update | Endpoint for updating repository entries and clearing caches | ## Installation -Get the [latest release](https://github.com/elementumorg/repository.elementumorg/releases/latest) from github. -Then, [install from zip](https://kodi.wiki/view/Add-on_manager#How_to_install_from_a_ZIP_file) within [Kodi](https://kodi.tv/). +Get the [latest release](https://github.com/elementumorg/repository.elementumorg/releases/latest) from GitHub. +Then, [install from zip](https://kodi.wiki/view/Add-on_manager#How_to_install_from_a_ZIP_file) +within [Kodi](https://kodi.tv/). ## Add-on entries -In order to know which repositories to proxy, the virtual repository needs to be provided with a _list of add-on entries_. +In order to know which repositories to proxy, the virtual repository needs to be provided with a _list of add-on +entries_. An example can be found [here](resources/repository.json). Each entry must follow the following schema (also available as [json schema](resources/repository-schema.json)): -|Property|Required|Description| -|--------|--------|-----------| -|id|true|Add-on id.| -|username|true|GitHub repository username.| -|branch|false|GitHub repository branch. If not defined, it will be the commit of the latest release or, in case there are no releases, master branch.| -|assets|false|Dictionary containing string/string key-value pairs, where the key corresponds to the relative asset location and the value corresponds to the real asset location. One can also set "zip" asset, which is a special case for the add-on zip. If an asset is not defined, its location will be automatically evaluated.
Note: assets are treated as "new style" format strings with the following keywords - _id_, _username_, _repository_, _branch_.| -|asset_prefix|false|Prefix to use on the real asset location when it is automatically evaluated.| -|repository|false|GitHub repository name. If not set, it is assumed to be the same as the add-on id.| -|platforms|false|Platforms where the add-on is supported. If not set, it is assumed all platforms are supported.| +| Property | Required | Description | +|--------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| id | true | Add-on id. | +| username | true | GitHub repository username. | +| branch | false | The github repository branch. If not defined, it will be either
1) the commit of the latest release,
2) the respective tag,
3) the repository default branch, or
4) if all the previous are unable to fetch, "main" branch. | +| assets | false | Dictionary containing string/string key-value pairs, where the key corresponds to the relative asset location and the value corresponds to the real asset location. One can also set "zip" asset, which is a special case for the add-on zip. If an asset is not defined, its location will be automatically evaluated.

Note: assets are treated as "new style" format strings with the following keywords - _id_, _username_, _repository_, _ref_, _system_, _arch_ and _version_ (_version_ is available for zip assets only). | +| asset_prefix | false | Prefix to use on the real asset location when it is automatically evaluated. | +| repository | false | GitHub repository name. If not set, it is assumed to be the same as the add-on id. | +| tag_pattern | false | The pattern for matching eligible tags. If not set, all tags are considered. | +| token | false | The token to use for accessing the repository. If not provided, the repository must have public access. | +| platforms | false | Platforms where the add-on is supported. If not set, it is assumed all platforms are supported. | diff --git a/addon.xml b/addon.xml index e8f08fa..735d779 100644 --- a/addon.xml +++ b/addon.xml @@ -1,5 +1,5 @@ - + @@ -18,9 +18,15 @@ GitHub virtual Add-on repository Customizable repository which acts as a proxy for defined GitHub users' add-ons updates. +- Allow running in standalone mode +- Migrate to GitHub API +- Add support for "tag_pattern" field +- Add support for private repositories by supplying a "token" +- Try to automatically determine zip ref from version +- Multiple minor improvements icon.png - \ No newline at end of file + diff --git a/lib/cache.py b/lib/cache.py index 541d013..710700a 100644 --- a/lib/cache.py +++ b/lib/cache.py @@ -1,20 +1,44 @@ import time -from functools import wraps +from operator import attrgetter +from threading import Lock -class _HashedSeq(list): - __slots__ = 'hash_value' +class _CacheValue(object): + __slots__ = ["_value", "_modified"] - # noinspection PyMissingConstructor - def __init__(self, tup): - self[:] = tup - self.hash_value = hash(tup) + def __init__(self, value): + self._value = value + self._modified = time.time() + + @property + def modified(self): + return self._modified + + @property + def value(self): + return self._value + + def expired(self, ttl): + return time.time() - self._modified > ttl + + def update(self): + self._modified = time.time() + + +class _HashedTuple(tuple): + __hash_value = None def __hash__(self): - return self.hash_value + hash_value = self.__hash_value + if hash_value is None: + self.__hash_value = hash_value = super(_HashedTuple, self).__hash__() + return hash_value + + def __getstate__(self): + return {} -def _make_key(args, kwds, typed, kwd_mark=(object(),), fast_types=(int, str)): +def _make_key(args, kwargs, typed, kwd_mark=(_HashedTuple,), fast_types=(int, str)): """ Make a cache key from optionally typed positional and keyword arguments @@ -26,50 +50,52 @@ def _make_key(args, kwds, typed, kwd_mark=(object(),), fast_types=(int, str)): saves space and improves lookup speed. """ key = args - if kwds: - key += kwd_mark - for item in kwds.items(): - key += item + sorted_kwargs = tuple(sorted(kwargs.items())) + if sorted_kwargs: + key += kwd_mark + sorted_kwargs if typed: key += tuple(type(v) for v in args) - if kwds: - key += tuple(type(v) for v in kwds.values()) + if sorted_kwargs: + key += tuple(type(v) for _, v in sorted_kwargs) elif len(key) == 1 and type(key[0]) in fast_types: return key[0] - return _HashedSeq(key) - - -def cached(seconds=60 * 60, max_size=128, typed=False, lru=False): - def wrapper(func): - cache = {} - - if max_size == 0: - # no caching - new_func = func - else: - @wraps(func) - def new_func(*args, **kwargs): - key = _make_key(args, kwargs, typed) - cache_entry = cache.get(key) - if cache_entry is None or time.time() - cache_entry[0] > seconds: - # Check cache size first and clean if necessary - if len(cache) >= max_size: - min_key = min(cache, key=lambda k: cache[k][0]) - del cache[min_key] - - result = func(*args, **kwargs) - cache[key] = [time.time(), result] - else: - if lru: - cache_entry[0] = time.time() - result = cache_entry[1] - - return result - - def cache_clear(): - cache.clear() - - new_func.cache_clear = cache_clear - return new_func - - return wrapper + return _HashedTuple(key) + + +class LoadingCache(object): + __slots__ = ["_func", "_store", "_ttl", "_max_size", "_typed", "_lru", "_get_modifier", "_lock"] + + def __init__(self, func, ttl_seconds=60 * 60, max_size=128, typed=False, lru=False): + self._func = func + self._store = {} + self._ttl = ttl_seconds + self._max_size = max_size + self._typed = typed + self._lru = lru + self._get_modifier = attrgetter("modified") + self._lock = Lock() + + def get(self, *args, **kwargs): + key = _make_key(args, kwargs, self._typed) + with self._lock: + cache_entry = self._store.get(key) # type: _CacheValue + if cache_entry is None: + # Check cache size first and clean if necessary + if len(self._store) >= self._max_size: + min_key = min(self._store, key=self._get_modifier) + del self._store[min_key] + result = self._func(*args, **kwargs) + self._store[key] = _CacheValue(result) + elif cache_entry.expired(self._ttl): + result = self._func(*args, **kwargs) + self._store[key] = _CacheValue(result) + else: + if self._lru: + cache_entry.update() + result = cache_entry.value + + return result + + def clear(self): + with self._lock: + self._store.clear() diff --git a/lib/compatibility.py b/lib/compatibility.py index 5b97f57..6cc5263 100644 --- a/lib/compatibility.py +++ b/lib/compatibility.py @@ -1,45 +1,76 @@ import importlib import os +import re import sys from xml.etree import ElementTree # nosec import xbmc import xbmcaddon +from lib.version import DebianVersion + PY3 = sys.version_info.major >= 3 +_digits_re = re.compile(r"(\d+)") + class UndefinedModuleError(ImportError): pass -def register_module(name, py2_module=None, py3_module=None): +class InvalidModuleVersionError(ImportError): + pass + + +def register_module(name, module=None, py2_module=None, py3_module=None, version=None): try: importlib.import_module(name) xbmc.log("{} module is already installed".format(name), xbmc.LOGDEBUG) except ImportError: xbmc.log("Failed to import module. Going to register it.", xbmc.LOGDEBUG) - module = py3_module if PY3 else py2_module + module = module or (py3_module if PY3 else py2_module) if module is None: raise UndefinedModuleError("No module was defined") - try: - import_module(module) - xbmc.log("{} module was already installed, but missing on addon.xml".format(name), xbmc.LOGDEBUG) - except RuntimeError: - install_and_import_module(name, module) + if has_addon(module): + xbmc.log("{} module is already installed, but missing on addon.xml".format(name), xbmc.LOGDEBUG) + import_module(module, version=version) + else: + install_and_import_module(name, module, version=version) -def import_module(module): - path = xbmcaddon.Addon(module).getAddonInfo("path") +def import_module(module, version=None): + addon = xbmcaddon.Addon(module) + addon_path = addon.getAddonInfo("path") + addon_version = addon.getAddonInfo("version") if not PY3: # noinspection PyUnresolvedReferences - path = path.decode("utf-8") - tree = ElementTree.parse(os.path.join(path, "addon.xml")) + addon_path = addon_path.decode("utf-8") + # noinspection PyUnresolvedReferences + addon_version = addon_version.decode("utf-8") + if version is not None and DebianVersion(addon_version) < DebianVersion(version): + raise InvalidModuleVersionError("No valid version for module {}: {} < {}".format( + module, addon_version, version)) + tree = ElementTree.parse(os.path.join(addon_path, "addon.xml")) + # Check for dependencies + for dependency in tree.findall("./requires//import"): + dependency_module = dependency.attrib["addon"] + if dependency_module.startswith("script.module."): + xbmc.log("{} module depends on {}. Going to import it.".format(module, dependency_module), xbmc.LOGDEBUG) + import_module(dependency_module, dependency.attrib.get("version")) + # Install the actual module library_path = tree.find("./extension[@point='xbmc.python.module']").attrib["library"] - sys.path.append(os.path.join(path, library_path)) + sys.path.append(os.path.join(addon_path, library_path)) -def install_and_import_module(name, module): +def install_and_import_module(name, module, version=None): xbmc.log("Installing and registering module {}:{}".format(name, module), xbmc.LOGINFO) - xbmc.executebuiltin("InstallAddon(" + module + ")", wait=True) - import_module(module) + install_addon(module) + import_module(module, version=version) + + +def install_addon(addon): + xbmc.executebuiltin("InstallAddon(" + addon + ")", wait=True) + + +def has_addon(addon): + return xbmc.getCondVisibility("System.HasAddon(" + addon + ")") diff --git a/lib/entries.py b/lib/entries.py index 4821a21..c90c6ad 100644 --- a/lib/entries.py +++ b/lib/entries.py @@ -1,12 +1,15 @@ import json import os import sys +from collections import OrderedDict from zipfile import ZipFile import xbmcgui -from lib.kodi import ADDON_DATA, ADDON_NAME, translate, notification, get_repository_port, str_to_unicode, translatePath -from lib.repository import validate_json_schema, get_request +from lib.kodi import ADDON_DATA, ADDON_NAME, translate, notification, get_repository_port, translatePath +from lib.platform.core import PLATFORM, dump_platform +from lib.repository import validate_schema +from lib.utils import str_to_unicode, request if not os.path.exists(ADDON_DATA): os.makedirs(ADDON_DATA) @@ -20,63 +23,53 @@ class Entries(object): def __init__(self, path=ENTRIES_PATH): self._path = path - self._data = [] - self._ids = [] + self._data = OrderedDict() if os.path.exists(self._path): self.load() def clear(self): - self._data = [] - self._ids = [] + self._data.clear() def length(self): - return len(self._ids) + return len(self._data) @property def ids(self): - return list(self._ids) + return list(self._data) - def remove(self, index): - self._data.pop(index) - self._ids.pop(index) + def remove(self, addon_id): + self._data.pop(addon_id) def load(self): with open(self._path) as f: - self._data = json.load(f) - self._ids = [addon["id"] for addon in self._data] + self.add_entries_from_data(json.load(f)) def save(self): with open(self._path, "w") as f: - json.dump(self._data, f) + json.dump(list(self._data.values()), f) def add_entries_from_file(self, path): if path.endswith(".zip"): with ZipFile(path) as zip_file: for name in zip_file.namelist(): if name.endswith(".json"): - self._add_entries_from_data(json.loads(zip_file.read(name))) + self.add_entries_from_data(json.loads(zip_file.read(name))) elif path.endswith(".json"): with open(path) as f: - self._add_entries_from_data(json.load(f)) + self.add_entries_from_data(json.load(f)) else: raise ValueError("Unknown file extension. Supported extensions are .json and .zip") - def _add_entries_from_data(self, data): - validate_json_schema(data) + def add_entries_from_data(self, data): + validate_schema(data) for entry in data: - addon_id = entry["id"] - try: - index = self._ids.index(addon_id) - self._data[index] = entry - except ValueError: - self._data.append(entry) - self._ids.append(addon_id) + self._data[entry["id"]] = entry def update_repository(notify=False): - get_request("http://127.0.0.1:{}/update".format(get_repository_port()), timeout=2) - if notify: - notification(translate(30013)) + with request("http://127.0.0.1:{}/update".format(get_repository_port()), timeout=2) as r: + if notify: + notification(translate(30013 if r.status_code == 200 else 30014)) def import_entries(): @@ -114,13 +107,19 @@ def clear_entries(): notification(translate(30011)) +def about(): + xbmcgui.Dialog().textviewer(translate(30006), "[B]{}[/B]\n\nDetected platform: {}\n\n{}".format( + ADDON_NAME, PLATFORM.name(), dump_platform())) + + def run(): + methods = ("import_entries", "delete_entries", "clear_entries", "update_repository", "about") if len(sys.argv) == 1: - selected = xbmcgui.Dialog().select(ADDON_NAME, [translate(30002 + i) for i in range(4)]) + selected = xbmcgui.Dialog().select(ADDON_NAME, [translate(30002 + i) for i in range(len(methods))]) elif len(sys.argv) == 2: method = sys.argv[1] try: - selected = ("import_entries", "delete_entries", "clear_entries", "update_repository").index(method) + selected = methods.index(method) except ValueError: raise NotImplementedError("Unknown method '{}'".format(method)) else: @@ -134,3 +133,5 @@ def run(): clear_entries() elif selected == 3: update_repository(True) + elif selected == 4: + about() diff --git a/lib/github.py b/lib/github.py new file mode 100644 index 0000000..2347ed2 --- /dev/null +++ b/lib/github.py @@ -0,0 +1,86 @@ +from lib.utils import request + + +class _Dict(dict): + def __getattr__(self, name): + return self[name] + + +class GitHubApiError(Exception): + pass + + +class GitHubRepositoryApi(object): + def __init__(self, username, repository, base_url="https://api.github.com", version="2022-11-28", token=None): + self._base_url = "{}/repos/{username}/{repository}".format( + base_url, username=username, repository=repository) + self._version = version + self._token = token + + def get_repository_info(self): + return self._request_json("") + + def get_refs_tags(self): + return self._request_json("/git/refs/tags") + + def get_release(self, release): + return self._request_json("/releases/{}".format(release)) + + def get_latest_release(self): + return self.get_release("latest") + + def get_release_by_tag(self, tag_name): + return self.get_release("tags/{}".format(tag_name)) + + def get_release_asset(self, asset_id): + return self._request( + "/releases/assets/{}".format(asset_id), + headers={"Accept": "application/octet-stream"}) + + def get_zip(self, ref=None): + # One could also use "https://github.com/{username}/{repository}/archive/{branch}.zip" + # to avoid GitHub API rate limiting + return self._request( + "/zipball/{}".format(ref) if ref else "/zipball", + headers={"Accept": "application/vnd.github.raw"}) + + def get_contents(self, path, ref=None): + # One could also use "https://raw.githubusercontent.com/{username}/{repository}/{branch}/{path}" + # to avoid GitHub API rate limiting + return self._request( + "/contents/{}".format(path), + params=dict(ref=ref) if ref else None, + headers={"Accept": "application/vnd.github.raw"}) + + def _request_json(self, url, params=None): + with self._request(url, params=params, headers={"Accept": "application/vnd.github+json"}) as response: + return response.json(object_pairs_hook=_Dict) + + def _request(self, url, params=None, headers=None): + full_url = self._base_url + url + response = request(full_url, params=params, headers=self._headers(headers)) + if response.status_code >= 400: + try: + response.close() + finally: + raise GitHubApiError("Call to {} failed with HTTP {}".format(full_url, response.status_code)) + return response + + def _headers(self, headers): + if headers is None: + headers = {} + if self._token: + headers["Authorization"] = "Bearer {}".format(self._token) + headers["X-GitHub-Api-Version"] = self._version + return headers + + def __eq__(self, other): + if isinstance(other, self.__class__): + return self._base_url == other._base_url and self._version == other._version and self._token == other._token + return NotImplemented + + def __ne__(self, other): + return not self == other + + def __hash__(self): + return hash((self._base_url, self._version, self._token)) diff --git a/lib/httpserver.py b/lib/httpserver.py index d5f397e..04066c1 100644 --- a/lib/httpserver.py +++ b/lib/httpserver.py @@ -2,6 +2,7 @@ import logging import re +from shutil import copyfileobj try: import urlparse @@ -12,8 +13,10 @@ from socketserver import ThreadingMixIn from http.server import BaseHTTPRequestHandler, HTTPServer +from lib.utils import str_to_bytes -class ServerHandler(BaseHTTPRequestHandler): + +class HTTPRequestHandler(BaseHTTPRequestHandler, object): protocol_version = "HTTP/1.1" get_routes = [] @@ -39,6 +42,7 @@ def do_GET(self): self._handle_request(self.get_routes) def _handle_request(self, routes): + self._response_started = False try: self.url = urlparse.urlparse(self.path) self.query = dict(urlparse.parse_qsl(self.url.query)) @@ -55,8 +59,16 @@ def _handle_request(self, routes): else: self.send_response_and_end(404) except Exception as e: - logging.error(e, exc_info=True) - self.send_response_and_end(500) + if self._response_started: + raise e + else: + logging.error(e, exc_info=True) + self.send_response_and_end(500) + + def send_response(self, *args, **kwargs): + # noinspection PyAttributeOutsideInit + self._response_started = True + super(HTTPRequestHandler, self).send_response(*args, **kwargs) def log_message(self, fmt, *args): logging.debug(fmt, *args) @@ -79,6 +91,40 @@ def send_redirect(self, url, code=301): self.send_header("Content-Length", "0") self.end_headers() + def send_file_contents(self, fp, code, length=None, content_type=None, + content_disposition=None, chunked=True): + self.send_response(code) + + if content_type: + self.send_header("Content-Type", content_type) + if content_disposition: + self.send_header("Content-Disposition", content_disposition) + if length: + self.send_header("Content-Length", length) + chunked = False + else: + if chunked: + self.send_header("Transfer-Encoding", "chunked") + self.send_header("Connection", "close") + + self.end_headers() + + if chunked: + self._send_chunked(fp) + else: + copyfileobj(fp, self.wfile) + + def _send_chunked(self, fp, chunk_size=16 * 1024): + while True: + buf = fp.read(chunk_size) + if not buf: + self.wfile.write(b"0\r\n\r\n") + break + self.wfile.write(str_to_bytes(format(len(buf), "x"))) + self.wfile.write(b"\r\n") + self.wfile.write(buf) + self.wfile.write(b"\r\n") + class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): """ @@ -88,12 +134,12 @@ class ThreadedHTTPServer(ThreadingMixIn, HTTPServer): def threaded_http_server(host, port): - return ThreadedHTTPServer((host, port), ServerHandler) + return ThreadedHTTPServer((host, port), HTTPRequestHandler) def add_get_route(pattern): def wrapper(func): - ServerHandler.add_get_route(pattern, func) + HTTPRequestHandler.add_get_route(pattern, func) return func return wrapper diff --git a/lib/kodi.py b/lib/kodi.py index 73b9d7c..cbd0bb6 100644 --- a/lib/kodi.py +++ b/lib/kodi.py @@ -1,33 +1,23 @@ import logging -import sys import xbmc import xbmcaddon import xbmcgui -PY3 = sys.version_info.major >= 3 +from lib.utils import PY3, str_to_unicode + ADDON = xbmcaddon.Addon() if PY3: from xbmcvfs import translatePath translate = ADDON.getLocalizedString - string_types = str - - def str_to_unicode(s): - return s else: from xbmc import translatePath - # noinspection PyUnresolvedReferences - string_types = basestring # noqa - def translate(*args, **kwargs): return ADDON.getLocalizedString(*args, **kwargs).encode("utf-8") - def str_to_unicode(s): - return s.decode("utf-8") - ADDON_ID = ADDON.getAddonInfo("id") ADDON_NAME = ADDON.getAddonInfo("name") ADDON_PATH = str_to_unicode(ADDON.getAddonInfo("path")) @@ -43,7 +33,7 @@ def get_repository_port(): return int(ADDON.getSetting("repository_port")) -class KodiLogHandler(logging.StreamHandler): +class KodiLogHandler(logging.Handler): levels = { logging.CRITICAL: xbmc.LOGFATAL, logging.ERROR: xbmc.LOGERROR, @@ -60,9 +50,6 @@ def __init__(self): def emit(self, record): xbmc.log(self.format(record), self.levels[record.levelno]) - def flush(self): - pass - def set_logger(name=None, level=logging.NOTSET): logger = logging.getLogger(name) diff --git a/lib/platform/__init__.py b/lib/platform/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lib/platform/core.py b/lib/platform/core.py new file mode 100644 index 0000000..5b052ea --- /dev/null +++ b/lib/platform/core.py @@ -0,0 +1,26 @@ +import logging + +from . import kodi_platform +from . import os_platform +from .definitions import PlatformError, SHARED_LIB_EXTENSIONS, EXECUTABLE_EXTENSIONS + + +def dump_platform(): + return "Kodi platform: " + kodi_platform.dump_platform() + "\n" + os_platform.dump_platform() + + +def get_platform(): + try: + return kodi_platform.get_platform() + except PlatformError: + return os_platform.get_platform() + + +try: + PLATFORM = get_platform() + SHARED_LIB_EXTENSION = SHARED_LIB_EXTENSIONS.get(PLATFORM.system, "") + EXECUTABLE_EXTENSION = EXECUTABLE_EXTENSIONS.get(PLATFORM.system, "") +except Exception as _e: + logging.fatal(_e, exc_info=True) + logging.fatal(dump_platform()) + raise _e diff --git a/lib/platform/definitions.py b/lib/platform/definitions.py new file mode 100644 index 0000000..70afcf7 --- /dev/null +++ b/lib/platform/definitions.py @@ -0,0 +1,35 @@ +from collections import namedtuple + + +class PlatformError(Exception): + pass + + +class Enum: + @classmethod + def values(cls): + return [value for name, value in vars(cls).items() if not name.startswith("_")] + + +class System(Enum): + linux = "linux" + android = "android" + darwin = "darwin" + windows = "windows" + + +class Arch(Enum): + x64 = "x64" + x86 = "x86" + arm = "arm" + arm64 = "arm64" + armv7 = "armv7" + + +class Platform(namedtuple("Platform", ["system", "version", "arch"])): + def name(self, sep="-"): + return self.system + sep + self.arch + + +SHARED_LIB_EXTENSIONS = {System.linux: ".so", System.android: ".so", System.darwin: ".dylib", System.windows: ".dll"} +EXECUTABLE_EXTENSIONS = {System.linux: "", System.android: "", System.darwin: "", System.windows: ".exe"} diff --git a/lib/platform/kodi_platform.py b/lib/platform/kodi_platform.py new file mode 100644 index 0000000..5ca90dc --- /dev/null +++ b/lib/platform/kodi_platform.py @@ -0,0 +1,97 @@ +import json +import logging +import os +import re +from platform import release + +import xbmc + +try: + from xbmcvfs import translatePath +except ImportError: + from xbmc import translatePath + +from .definitions import PlatformError, System, Arch, Platform + +DARWIN_PLATFORMS = ("macOS", "iOS", "tvOS") +LINUX_PLATFORMS = ("Linux", "webOS") +ANDROID_PLATFORMS = ("Android",) +WINDOWS_PLATFORMS = ("Windows NT", "unknown Win32 platform") +SUPPORTED_PLATFORMS = DARWIN_PLATFORMS + ANDROID_PLATFORMS + LINUX_PLATFORMS + WINDOWS_PLATFORMS + ( + "FreeBSD", "unknown platform") + +SUPPORTED_CPUS = ["ARM (Thumb)", "ARM", "LoongArch", "MIPS", "x86", "s390", "PowerPC", "RISC-V", "unknown CPU family"] + +_PLATFORM_RE = re.compile(r"^({}) ({}) (\d+)-bit$".format( + "|".join(map(re.escape, SUPPORTED_PLATFORMS)), "|".join(map(re.escape, SUPPORTED_CPUS)))) + + +def get_application_name(): + cmd = '{"jsonrpc":"2.0", "method":"Application.GetProperties","params": {"properties": ["name"]}, "id":1}' + data = json.loads(xbmc.executeJSONRPC(cmd)) + return data["result"]["name"] + + +def get_kodi_log_path(): + log_name = os.path.join(translatePath("special://logpath"), get_application_name().lower()) + return log_name + ".log", log_name + ".old.log" + + +def get_kodi_platform_from_log(): + # GetBuildTargetPlatformName, GetBuildTargetCpuFamily and GetXbmcBitness from 2nd log line + # (tree/master -> blob/6d5b46ba127eacd706610a91a32167abfbf8ac8e) + # https://github.com/xbmc/xbmc/tree/master/xbmc/utils/SystemInfo.cpp + # https://github.com/xbmc/xbmc/tree/master/xbmc/application/Application.cpp#L3673 + new_log_path, old_log_path = get_kodi_log_path() + with open(old_log_path if os.path.exists(old_log_path) else new_log_path, encoding='utf-8') as f: + # Ignore first line + next(f) + # Second line ends with the platform + kodi_platform = next(f).split("Platform: ")[-1].rstrip() + + return kodi_platform + + +def dump_platform(): + try: + return get_kodi_platform_from_log() + except Exception as e: + logging.warning("Failed getting kodi platform: %s", e, exc_info=True) + return "unknown" + + +def get_platform(): + raw_platform = dump_platform() + logging.debug("Resolving platform - %s", raw_platform) + match = _PLATFORM_RE.match(raw_platform) + if not match: + raise PlatformError("Unable to parse Kodi platform") + + platform_name = match.group(1) + cpu_family = match.group(2) + bitness = int(match.group(3)) + + if platform_name in ANDROID_PLATFORMS: + system = System.android + elif platform_name in LINUX_PLATFORMS: + system = System.linux + elif platform_name in WINDOWS_PLATFORMS: + system = System.windows + elif platform_name in DARWIN_PLATFORMS: + system = System.darwin + else: + raise PlatformError("Unknown platform: {}".format(platform_name)) + + if cpu_family == "ARM": + if system == System.android: + arch = Arch.arm64 if bitness == 64 else Arch.arm + elif system == System.linux: + arch = Arch.arm64 if bitness == 64 else Arch.armv7 + else: + raise PlatformError("Unknown arch {} for platform: {}".format(cpu_family, system)) + elif cpu_family == "x86": + arch = Arch.x64 if bitness == 64 else Arch.x86 + else: + raise PlatformError("Unknown platform: {}".format(cpu_family)) + + return Platform(system, release(), arch) diff --git a/lib/platform/os_platform.py b/lib/platform/os_platform.py new file mode 100644 index 0000000..9177c3e --- /dev/null +++ b/lib/platform/os_platform.py @@ -0,0 +1,44 @@ +import logging +import os +import platform +import sys + +from .definitions import Arch, System, Platform + + +def get_platform(): + system = platform.system().lower() + version = platform.release() + arch = Arch.x64 if sys.maxsize > 2 ** 32 else Arch.x86 + machine = platform.machine().lower() + is_arch64 = "64" in machine and arch == Arch.x64 + + logging.debug("Resolving platform - system=%s, version=%s, arch=%s, machine=%s", system, version, arch, machine) + + if "ANDROID_STORAGE" in os.environ: + system = System.android + if "arm" in machine or "aarch" in machine: + arch = Arch.arm64 if is_arch64 else Arch.arm + elif system == System.linux: + if "armv7" in machine: + arch = Arch.armv7 + elif "aarch" in machine: + arch = Arch.arm64 if is_arch64 else Arch.armv7 + elif "arm" in machine: + arch = Arch.arm64 if is_arch64 else Arch.arm + elif system == System.windows: + if machine.endswith("64"): + arch = Arch.x64 + elif system == System.darwin: + arch = Arch.x64 + + if system not in System.values() or arch not in Arch.values(): + logging.warning("Unknown system (%s) and/or arch (%s) values", system, arch) + + return Platform(system, version, arch) + + +def dump_platform(): + return "system: {}\nrelease: {}\nmachine: {}\narchitecture: {}\nmax_size: {} ({:x} {})\nplatform: {}".format( + platform.system(), platform.release(), platform.machine(), platform.architecture(), sys.maxsize, + sys.maxsize, ">32b" if sys.maxsize > 2 ** 32 else "<=32b", platform.platform()) diff --git a/lib/repository.py b/lib/repository.py index 8ae7eb1..69a29b7 100644 --- a/lib/repository.py +++ b/lib/repository.py @@ -2,87 +2,135 @@ import logging import re from collections import namedtuple, OrderedDict +from concurrent.futures import ThreadPoolExecutor from hashlib import md5 from xml.etree import ElementTree # nosec -try: - from urllib import request as ul -except ImportError: - # noinspection PyUnresolvedReferences - import urllib2 as ul +from lib.cache import LoadingCache +from lib.github import GitHubRepositoryApi, GitHubApiError +from lib.utils import string_types, is_http_like, request, remove_prefix +from lib.version import try_parse_version -from concurrent.futures import ThreadPoolExecutor +Addon = namedtuple("Addon", ( + "id", "username", "branch", "assets", "asset_prefix", "repository", "tag_pattern", "token", "platforms")) +EntrySchema = namedtuple("EntrySchema", ("required", "validators")) -from lib.cache import cached -from lib.kodi import string_types -ADDON = namedtuple("ADDON", "id username branch assets asset_prefix repository") +class InvalidSchemaError(Exception): + pass -GITHUB_CONTENT_URL = "https://raw.githubusercontent.com/{username}/{repository}/{branch}" -GITHUB_RELEASES_URL = "https://api.github.com/repos/{username}/{repository}/releases" -GITHUB_LATEST_RELEASE_URL = GITHUB_RELEASES_URL + "/latest" -GITHUB_RELEASE_URL = GITHUB_RELEASES_URL + "/{release}" -GITHUB_ZIP_URL = "https://github.com/{username}/{repository}/archive/{branch}.zip" -ENTRY_SCHEMA = { - "required": ["id", "username"], - "properties": { - "id": {"type": string_types}, - "username": {"type": string_types}, - "branch": {"type": string_types}, - "assets": {"type": dict}, - "asset_prefix": {"type": string_types}, - "repository": {"type": string_types}, - } -} +class InvalidFormatException(Exception): + pass -class InvalidSchemaError(Exception): +class NotFoundException(Exception): + pass + + +class AddonNotFound(NotFoundException): pass +class ReleaseAssetNotFound(NotFoundException): + pass + + +def validate_string(key, value): + if not isinstance(value, string_types): + raise InvalidSchemaError("Expected str for '{}'".format(key)) + + +def validate_string_map(key, value): + if not (isinstance(value, dict) + and all(isinstance(k, string_types) and isinstance(v, string_types) for k, v in value.items())): + raise InvalidSchemaError("Expected dict[str, str] for '{}'".format(key)) + + +def validate_string_list(key, value): + if not (isinstance(value, list) and all(isinstance(v, string_types) for v in value)): + raise InvalidSchemaError("Expected list[str] for '{}'".format(key)) + + +_entry_schema = EntrySchema(required=("id", "username"), validators=dict( + id=validate_string, + username=validate_string, + branch=validate_string, + assets=validate_string_map, + asset_prefix=validate_string, + repository=validate_string, + tag_pattern=validate_string, + token=validate_string, + platforms=validate_string_list, +)) + + def validate_entry_schema(entry): if not isinstance(entry, dict): raise InvalidSchemaError("Expecting dictionary for entry") - for key in ENTRY_SCHEMA["required"]: + for key in _entry_schema.required: if key not in entry: raise InvalidSchemaError("Key '{}' is required".format(key)) for key, value in entry.items(): - if key not in ENTRY_SCHEMA["properties"]: + validator = _entry_schema.validators.get(key) + if not validator: raise InvalidSchemaError("Key '{}' is not valid".format(key)) - value_type = ENTRY_SCHEMA["properties"][key]["type"] - if not isinstance(value, value_type): - raise InvalidSchemaError("Expected type {} for '{}'".format(value_type.__name__, key)) - if value_type is dict: - for k, v in value.items(): - if not (isinstance(k, string_types) and isinstance(v, string_types)): - raise InvalidSchemaError("Expected dict[str, str] for '{}'".format(key)) - elif value_type is list: - for v in value: - if not isinstance(v, string_types): - raise InvalidSchemaError("Expected list[str] for '{}'".format(key)) - - -def validate_json_schema(data): + validator(key, value) + + +def validate_schema(data): if not isinstance(data, (list, tuple)): raise InvalidSchemaError("Expecting list/tuple for data") for entry in data: validate_entry_schema(entry) -def get_request(url, **kwargs): - return ul.urlopen(url, **kwargs).read() +class TagMatchPredicate(object): + def __init__(self, version, tag_pattern=None): + self._version = version + self._tag_pattern = tag_pattern + self._tag_group = tag_pattern.groupindex.get("version", 1) if tag_pattern and tag_pattern.groups else None + self._parsed_version = try_parse_version(version) + + def __call__(self, value): + if self._tag_pattern: + match = self._tag_pattern.match(value) + if not match: + return False + elif self._tag_group: + value = match.group(self._tag_group) + + return value == self._version or ( + self._parsed_version and self._parsed_version == try_parse_version(value)) class Repository(object): - def __init__(self, **kwargs): - self.files = kwargs.get("files", []) - self.urls = kwargs.get("urls", []) - self._max_threads = kwargs.get("max_threads", 5) + ZIP_EXTENSION = ".zip" + VERSION_SEPARATOR = "-" + RELEASE_ASSET_PREFIX = "release_asset://" + + def __init__(self, files=(), urls=(), max_threads=5, platform=None, + cache_ttl=60 * 60, default_branch="main", token=None): + self.files = files + self.urls = urls + self._max_threads = max_threads + self._default_branch = default_branch + self._token = token self._addons = OrderedDict() + + if platform is None: + from lib.platform.core import PLATFORM + self._platform = PLATFORM + else: + self._platform = platform + + self._addons_xml_cache = LoadingCache(self._get_addons_xml, cache_ttl) + self._fallback_ref_cache = LoadingCache(self._get_fallback_ref, cache_ttl) + self._refs_tags_cache = LoadingCache(self._get_refs_tags, cache_ttl) self.update() def update(self, clear=False): + logging.debug("Updating repository (clear=%s)", clear) if clear: self._addons.clear() for u in self.urls: @@ -95,55 +143,60 @@ def _load_file(self, path): self._load_data(json.load(f)) def _load_url(self, url): - self._load_data(json.loads(get_request(url))) + with request(url) as r: + r.raise_for_status() + self._load_data(r.json()) def _load_data(self, data): + validate_schema(data) + platform_name = self._platform.name() for addon_data in data: addon_id = addon_data["id"] - - self._addons[addon_id] = ADDON( - id=addon_id, username=addon_data["username"], branch=addon_data.get("branch"), - assets=addon_data.get("assets", {}), asset_prefix=addon_data.get("asset_prefix", ""), - repository=addon_data.get("repository", addon_id)) + platforms = addon_data.get("platforms") + tag_pattern = addon_data.get("tag_pattern") + + if platforms and platform_name not in platforms: + logging.debug("Skipping addon %s as it does not support platform %s", addon_id, platform_name) + continue + + self._addons[addon_id] = Addon( + id=addon_id, + username=addon_data["username"], + branch=addon_data.get("branch"), + assets=addon_data.get("assets", {}), + asset_prefix=addon_data.get("asset_prefix", ""), + repository=addon_data.get("repository", addon_id), + tag_pattern=re.compile(tag_pattern) if tag_pattern else None, + token=addon_data.get("token"), + platforms=platforms, + ) def clear_cache(self): - self.get_addons_xml.cache_clear() - self.get_latest_release.cache_clear() - - @cached(seconds=60 * 60) - def get_latest_release(self, username, repository, default="master"): - data = json.loads(get_request(GITHUB_LATEST_RELEASE_URL.format(username=username, repository=repository))) - try: - return data["tag_name"] - except KeyError: - return default - - def _get_addon_branch(self, addon): - return addon.branch or self.get_latest_release(addon.username, addon.repository) + logging.debug("Clearing repository cache") + self._addons_xml_cache.clear() + self._fallback_ref_cache.clear() + self._refs_tags_cache.clear() def _get_addon_xml(self, addon): - try: - addon_xml_url = addon.assets["addon.xml"] - except KeyError: - addon_xml_url = GITHUB_CONTENT_URL.format( - username=addon.username, repository=addon.repository, - branch=self._get_addon_branch(addon)) + "/addon.xml" + with self._get_asset(addon, "addon.xml") as r: + r.raise_for_status() + addon_xml = r.content try: - return ElementTree.fromstring(get_request(addon_xml_url)) + return ElementTree.fromstring(addon_xml) except Exception as e: - logging.error("failed getting '%s': %s", addon.id, e, exc_info=True) + logging.error("Failed getting '%s' addon XML: %s", addon.id, e, exc_info=True) return None - @cached(seconds=60 * 60) - def get_addons_xml(self): + def _get_addons_xml(self): root = ElementTree.Element("addons") num_threads = min(self._max_threads, len(self._addons)) if num_threads <= 1: - results = [self._get_addon_xml(a) for a in self._addons.values()] + results = map(self._get_addon_xml, self._addons.values()) else: with ThreadPoolExecutor(num_threads) as pool: - results = list(pool.map(self._get_addon_xml, self._addons.values())) + futures = [pool.submit(self._get_addon_xml, addon) for addon in self._addons.values()] + results = map(lambda f: f.result(), futures) for result in results: if result is not None: @@ -151,23 +204,100 @@ def get_addons_xml(self): return ElementTree.tostring(root, encoding="utf-8", method="xml") + def get_addons_xml(self): + return self._addons_xml_cache.get() + def get_addons_xml_md5(self): m = md5() m.update(self.get_addons_xml()) return m.hexdigest().encode("utf-8") - def get_asset_url(self, addon_id, asset): + def get_asset(self, addon_id, asset): addon = self._addons.get(addon_id) if addon is None: - return None - formats = {"id": addon.id, "username": addon.username, "repository": addon.repository, - "branch": self._get_addon_branch(addon)} - match = re.match(addon_id + r"-(.+?)\.zip$", asset) - if match: - formats["version"] = match.group(1) + raise AddonNotFound("No such addon: {}".format(addon_id)) + return self._get_asset(addon, asset) + + def _get_asset(self, addon, asset): + logging.debug("Getting asset for addon %s: %s", addon.id, asset) + repo = GitHubRepositoryApi(addon.username, addon.repository, token=addon.token or self._token) + ref = addon.branch or self._fallback_ref_cache.get(repo, tag_pattern=addon.tag_pattern) + logging.debug("Using ref %s for addon %s", ref, addon.id) + formats = dict( + id=addon.id, username=addon.username, repository=addon.repository, + ref=ref, system=self._platform.system, arch=self._platform.arch) + + is_zip = asset.startswith(addon.id + self.VERSION_SEPARATOR) and asset.endswith(self.ZIP_EXTENSION) + if is_zip: + formats["version"] = asset[len(addon.id) + len(self.VERSION_SEPARATOR):-len(self.ZIP_EXTENSION)] asset = "zip" - default_asset_url = GITHUB_ZIP_URL + + try: + asset_path = self._format(addon.assets[asset], **formats) + except KeyError: + if is_zip: + version = formats["version"] + zip_ref = self._get_version_tag(repo, version, tag_pattern=addon.tag_pattern, default=ref) + logging.debug("Automatically detected zip ref. Wanted %s, detected %s", version, zip_ref) + response = repo.get_zip(zip_ref) + else: + response = repo.get_contents(self._format(addon.asset_prefix, **formats) + asset, ref) + else: + if asset_path.startswith(self.RELEASE_ASSET_PREFIX): + release_tag, asset_name = asset_path[len(self.RELEASE_ASSET_PREFIX):].rsplit("/", maxsplit=1) + release = repo.get_release_by_tag(release_tag) + for release_asset in release.assets: + if release_asset.name == asset_name: + response = repo.get_release_asset(release_asset.id) + break + else: + raise ReleaseAssetNotFound("Unable to find release asset: {}".format(asset_path)) + elif is_http_like(asset_path): + response = request(asset_path) + else: + response = repo.get_contents(asset_path, ref) + + return response + + def _get_fallback_ref(self, repo, tag_pattern=None): + if tag_pattern is None: + ref = self._get_latest_release_tag(repo) or self._get_matching_tag(repo, lambda _: True) else: - default_asset_url = GITHUB_CONTENT_URL + "/" + addon.asset_prefix + asset + ref = self._get_matching_tag(repo, tag_pattern.match) or self._get_latest_release_tag(repo) + return ref or self._get_repository_default_branch(repo) or self._default_branch + + def _get_matching_tag(self, repo, predicate, default=None): + return next((tag_name for tag_name in ( + remove_prefix(tag.ref, "refs/tags/") for tag in reversed(self._refs_tags_cache.get(repo)) + ) if predicate(tag_name)), default) + + def _get_version_tag(self, repo, version, tag_pattern=None, default=None): + return self._get_matching_tag(repo, TagMatchPredicate(version, tag_pattern=tag_pattern), default=default) - return addon.assets.get(asset, default_asset_url).format(**formats) + @staticmethod + def _get_refs_tags(repo): + try: + return repo.get_refs_tags() + except GitHubApiError: + return [] + + @staticmethod + def _get_latest_release_tag(repo): + try: + return repo.get_latest_release().tag_name + except GitHubApiError: + return None + + @staticmethod + def _get_repository_default_branch(repo): + try: + return repo.get_repository_info().default_branch + except GitHubApiError: + return None + + @staticmethod + def _format(value, **formats): + try: + return value.format(**formats) + except KeyError as e: + raise InvalidSchemaError("Format contains an invalid parameter: {}".format(e)) diff --git a/lib/routes.py b/lib/routes.py index 97d5d5f..25289bd 100644 --- a/lib/routes.py +++ b/lib/routes.py @@ -1,38 +1,36 @@ -import os - -from lib.entries import ENTRIES_PATH -from lib.httpserver import add_get_route # pylint: disable=unused-import -from lib.kodi import ADDON_PATH -from lib.repository import Repository - -repository = Repository(files=(os.path.join(ADDON_PATH, "resources", "repository.json"), ENTRIES_PATH)) - - -@add_get_route("/addons.xml") -def route_get_addons(ctx): - # type: (ServerHandler) -> None - ctx.send_response_with_data(repository.get_addons_xml(), "application/xml") - - -@add_get_route("/addons.xml.md5") -def route_get_addons_md5(ctx): - # type: (ServerHandler) -> None - ctx.send_response_with_data(repository.get_addons_xml_md5(), "text/plain") - - -@add_get_route("/{w}/{p}") -def route_get_assets(ctx, addon_id, asset): - # type: (ServerHandler, str, str) -> None - url = repository.get_asset_url(addon_id, asset) - if url is None: - ctx.send_response_and_end(404) - else: - ctx.send_redirect(url) - - -@add_get_route("/update") -def route_update(ctx): - # type: (ServerHandler) -> None - repository.update() - repository.clear_cache() - ctx.send_response_and_end(200) +from lib.httpserver import HTTPRequestHandler, add_get_route +from lib.repository import Repository, NotFoundException + + +def add_repository_routes(repository): + # type: (Repository) -> None + + @add_get_route("/addons.xml") + def route_get_addons(ctx): + # type: (HTTPRequestHandler) -> None + ctx.send_response_with_data(repository.get_addons_xml(), "application/xml") + + @add_get_route("/addons.xml.md5") + def route_get_addons_md5(ctx): + # type: (HTTPRequestHandler) -> None + ctx.send_response_with_data(repository.get_addons_xml_md5(), "text/plain") + + @add_get_route("/{w}/{p}") + def route_get_assets(ctx, addon_id, asset): + # type: (HTTPRequestHandler, str, str) -> None + try: + with repository.get_asset(addon_id, asset) as response: + ctx.send_file_contents( + response.raw, response.status_code, + length=response.headers.get("Content-Length"), + content_type=response.headers.get("Content-Type"), + content_disposition=response.headers.get("Content-Disposition")) + except NotFoundException: + ctx.send_response_and_end(404) + + @add_get_route("/update") + def route_update(ctx): + # type: (HTTPRequestHandler) -> None + repository.update() + repository.clear_cache() + ctx.send_response_and_end(200) diff --git a/lib/service.py b/lib/service.py index 5caa909..4c93df3 100644 --- a/lib/service.py +++ b/lib/service.py @@ -5,9 +5,15 @@ import xbmc -from lib import routes # noqa +from lib.entries import ENTRIES_PATH from lib.httpserver import threaded_http_server from lib.kodi import ADDON_PATH, get_repository_port, set_logger +from lib.repository import Repository +from lib.routes import add_repository_routes + +set_logger() +add_repository_routes(Repository( + files=(os.path.join(ADDON_PATH, "resources", "repository.json"), ENTRIES_PATH))) def update_repository_port(port, xml_path=os.path.join(ADDON_PATH, "addon.xml")): @@ -61,7 +67,6 @@ def __exit__(self, exc_type, exc_val, exc_tb): def run(): - set_logger() port = get_repository_port() with HTTPServerRunner(port): ServiceMonitor(port).waitForAbort() diff --git a/lib/utils.py b/lib/utils.py new file mode 100644 index 0000000..8872ab7 --- /dev/null +++ b/lib/utils.py @@ -0,0 +1,110 @@ +import json +import logging +import sys + +try: + from urllib.request import urlopen, Request + from urllib.parse import urlparse, urlencode + from urllib.error import HTTPError +except ImportError: + # noinspection PyUnresolvedReferences + from urllib2 import urlopen, Request, HTTPError + # noinspection PyUnresolvedReferences + from urlparse import urlparse + # noinspection PyUnresolvedReferences + from urllib import urlencode + +PY3 = sys.version_info.major >= 3 + +if PY3: + string_types = str + + def str_to_unicode(s): + return s + + def str_to_bytes(s): + return s.encode() +else: + # noinspection PyUnresolvedReferences + string_types = basestring # noqa + + def str_to_unicode(s): + return s.decode("utf-8") + + def str_to_bytes(s): + return s + + +def remove_prefix(text, prefix): + return text[len(prefix):] if text.startswith(prefix) else text + + +def is_http_like(s): + try: + result = urlparse(s) + return result.netloc and result.scheme in ("http", "https") + except ValueError: + return False + + +def request(url, params=None, data=None, headers=None, **kwargs): + if params: + url += "?" + urlencode(params) + request_params = Request(url, data=data, headers=headers if headers else {}) + logging.debug("Doing a HTTP %s request to %s", request_params.get_method(), url) + try: + response = urlopen(request_params, **kwargs) + except HTTPError as e: + response = e + logging.debug( + "HTTP %s response from %s received with status %s", + request_params.get_method(), url, response.getcode()) + return Response(response) + + +class HTTPResponseError(Exception): + def __init__(self, message, response): + super(HTTPResponseError, self).__init__(message) + self.response = response + + +class Response(object): + def __init__(self, response): + self._response = response + self._content = None + + @property + def content(self): + if self._content is None: + self._content = self._response.read() + return self._content + + @property + def raw(self): + return self._response + + @property + def headers(self): + return self._response.info() + + @property + def status_code(self): + return self._response.getcode() + + def json(self, **kwargs): + return json.loads(self.content, **kwargs) + + def raise_for_status(self): + if 400 <= self.status_code < 500: + raise HTTPResponseError("Client Error: {}".format(self.status_code), self) + elif 500 <= self.status_code < 600: + raise HTTPResponseError("Server Error: {}".format(self.status_code), self) + + def close(self): + self._response.close() + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + self.close() diff --git a/lib/version.py b/lib/version.py new file mode 100644 index 0000000..6466ee0 --- /dev/null +++ b/lib/version.py @@ -0,0 +1,110 @@ +import re +from functools import total_ordering + +_digits_re = re.compile(r"(\d+)") + + +@total_ordering +class InfinityType(object): + def __repr__(self): + return "Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __ge__(self, other): + return True + + def __neg__(self): + return NegativeInfinity + + +@total_ordering +class NegativeInfinityType(object): + def __repr__(self): + return "-Infinity" + + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + return isinstance(other, self.__class__) + + def __le__(self, other): + return True + + def __neg__(self): + return Infinity + + +@total_ordering +class _BaseVersion(object): + _key = None + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return None + return self._key == other._key + + def __lt__(self, other): + if not isinstance(other, self.__class__): + return None + return self._key < other._key + + +class Version(_BaseVersion): + _version_re = re.compile(r""" + ^v? + (?P[0-9]+(?:\.[0-9]+)*) + (?P[^+]*) + (?:\+(?P.*))? + $ + """, re.VERBOSE | re.IGNORECASE) + + def __init__(self, value, case_insensitive=True): + if case_insensitive: + value = value.lower() + + match = self._version_re.match(value) + if match is None: + raise ValueError("Invalid version {}".format(repr(value))) + + self._release = tuple(int(i) for i in match.group("release").split(".")) + self._extra = match.group("extra") + self._build = match.group("build") + self._key = self._make_key() + + def _make_key(self): + for i, v in enumerate(reversed(self._release)): + if v != 0: + release = self._release[:len(self._release) - i] + break + else: + release = () + + extra = _nat_tuple(self._extra + "0") if self._extra else Infinity + + return release, extra + + +class DebianVersion(_BaseVersion): + def __init__(self, version): + self._key = _nat_tuple(version, lambda v: v.split("~") + [Infinity]) + + +Infinity = InfinityType() +NegativeInfinity = NegativeInfinityType() + + +def try_parse_version(version, default=None): + try: + return Version(version) + except ValueError: + return default + + +def _nat_tuple(value, converter=lambda a: a): + return tuple(converter(c) if i % 2 == 0 else int(c) for i, c in enumerate(_digits_re.split(value))) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5d3f264 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +Kodistubs \ No newline at end of file diff --git a/resources/language/resource.language.en_gb/strings.po b/resources/language/resource.language.en_gb/strings.po index 92731e7..2775bc7 100644 --- a/resources/language/resource.language.en_gb/strings.po +++ b/resources/language/resource.language.en_gb/strings.po @@ -42,6 +42,10 @@ msgctxt "#30005" msgid "Update repository" msgstr "" +msgctxt "#30006" +msgid "About" +msgstr "" + msgctxt "#30010" msgid "No entries to delete" msgstr "" @@ -56,4 +60,8 @@ msgstr "" msgctxt "#30013" msgid "Repository updated" -msgstr "" \ No newline at end of file +msgstr "" + +msgctxt "#30014" +msgid "Failed updating repository" +msgstr "" diff --git a/resources/language/resource.language.es_es/strings.po b/resources/language/resource.language.es_es/strings.po new file mode 100644 index 0000000..e39ac19 --- /dev/null +++ b/resources/language/resource.language.es_es/strings.po @@ -0,0 +1,67 @@ +# Kodi Media Center language file +# Addon Name: i96751414 Repository +# Addon id: repository.github +# Addon Provider: i96751414 +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: \n" +"Language-Team: \n" +"Language: es_ES\\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +# Settings +msgctxt "#30000" +msgid "General" +msgstr "General" + +msgctxt "#30001" +msgid "Repository port" +msgstr "Puerto del repositorio" + +msgctxt "#30002" +msgid "Import entries" +msgstr "Importar entradas" + +msgctxt "#30003" +msgid "Delete entries" +msgstr "Eliminar entradas" + +msgctxt "#30004" +msgid "Delete all entries" +msgstr "Eliminar todas las entradas" + +msgctxt "#30005" +msgid "Update repository" +msgstr "Actualizar repositorio" + +msgctxt "#30006" +msgid "About" +msgstr "Sobre" + +msgctxt "#30010" +msgid "No entries to delete" +msgstr "No hay entradas que eliminar" + +msgctxt "#30011" +msgid "Entries deleted" +msgstr "Entradas eliminadas" + +msgctxt "#30012" +msgid "Entries imported" +msgstr "Entradas importadas" + +msgctxt "#30013" +msgid "Repository updated" +msgstr "Repositorio actualizado" + +msgctxt "#30014" +msgid "Failed updating repository" +msgstr "Error al actualizar el repositorio" diff --git a/resources/language/resource.language.pt_br/strings.po b/resources/language/resource.language.pt_br/strings.po new file mode 100644 index 0000000..7a84bb5 --- /dev/null +++ b/resources/language/resource.language.pt_br/strings.po @@ -0,0 +1,67 @@ +# Kodi Media Center language file +# Addon Name: i96751414 Repository +# Addon id: repository.github +# Addon Provider: i96751414 +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: \n" +"Language-Team: \n" +"Language: pt_BR\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +# Settings +msgctxt "#30000" +msgid "General" +msgstr "Geral" + +msgctxt "#30001" +msgid "Repository port" +msgstr "Porto do repositório" + +msgctxt "#30002" +msgid "Import entries" +msgstr "Importar entradas" + +msgctxt "#30003" +msgid "Delete entries" +msgstr "Apagar entradas" + +msgctxt "#30004" +msgid "Delete all entries" +msgstr "Apagar todas as entradas" + +msgctxt "#30005" +msgid "Update repository" +msgstr "Atualizar repositório" + +msgctxt "#30006" +msgid "About" +msgstr "Sobre" + +msgctxt "#30010" +msgid "No entries to delete" +msgstr "Sem entradas para apagar" + +msgctxt "#30011" +msgid "Entries deleted" +msgstr "Entradas apagadas" + +msgctxt "#30012" +msgid "Entries imported" +msgstr "Entradas importadas" + +msgctxt "#30013" +msgid "Repository updated" +msgstr "Repositório atualizado" + +msgctxt "#30014" +msgid "Failed updating repository" +msgstr "Erro a atualizar repositório" diff --git a/resources/language/resource.language.pt_pt/strings.po b/resources/language/resource.language.pt_pt/strings.po new file mode 100644 index 0000000..9ebebb1 --- /dev/null +++ b/resources/language/resource.language.pt_pt/strings.po @@ -0,0 +1,67 @@ +# Kodi Media Center language file +# Addon Name: i96751414 Repository +# Addon id: repository.github +# Addon Provider: i96751414 +# +msgid "" +msgstr "" +"Project-Id-Version: \n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: YEAR-MO-DA HO:MI+ZONE\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: \n" +"Language-Team: \n" +"Language: pt\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" + +# Settings +msgctxt "#30000" +msgid "General" +msgstr "Geral" + +msgctxt "#30001" +msgid "Repository port" +msgstr "Porto do repositório" + +msgctxt "#30002" +msgid "Import entries" +msgstr "Importar entradas" + +msgctxt "#30003" +msgid "Delete entries" +msgstr "Apagar entradas" + +msgctxt "#30004" +msgid "Delete all entries" +msgstr "Apagar todas as entradas" + +msgctxt "#30005" +msgid "Update repository" +msgstr "Atualizar repositório" + +msgctxt "#30006" +msgid "About" +msgstr "Sobre" + +msgctxt "#30010" +msgid "No entries to delete" +msgstr "Sem entradas para apagar" + +msgctxt "#30011" +msgid "Entries deleted" +msgstr "Entradas apagadas" + +msgctxt "#30012" +msgid "Entries imported" +msgstr "Entradas importadas" + +msgctxt "#30013" +msgid "Repository updated" +msgstr "Repositório atualizado" + +msgctxt "#30014" +msgid "Failed updating repository" +msgstr "Erro a atualizar repositório" diff --git a/resources/repository-schema.json b/resources/repository-schema.json index 92984fe..8cf493b 100644 --- a/resources/repository-schema.json +++ b/resources/repository-schema.json @@ -9,26 +9,26 @@ "id": { "type": "string", "title": "Add-on ID", - "description": "The add-on identifier as in addon.xml" + "description": "The add-on identifier as in addon.xml." }, "username": { "type": "string", "title": "Repository username", - "description": "The github repository username" + "description": "The github repository username." }, "branch": { "type": "string", "title": "Repository branch", - "description": "The github repository branch. If not defined, it will be the commit of the latest release or, in case there are no releases, master branch." + "description": "The github repository branch. If not defined, it will be either 1) the commit of the latest release, 2) the respective tag, 3) the repository default branch, or 4) if all the previous are unable to fetch, \"main\" branch." }, "assets": { "type": "object", "title": "Repository assets", - "description": "Dictionary containing string/string key-value pairs, where the key corresponds to the relative asset location and the value corresponds to the real asset location. One can also set \"zip\" asset, which is a special case for the add-on zip. If an asset is not defined, its location will be automatically evaluated.\nNote: assets are treated as \"new style\" format strings with the following keywords - id, username, repository, branch, system and arch.", + "description": "Dictionary containing string/string key-value pairs, where the key corresponds to the relative asset location and the value corresponds to the real asset location. One can also set \"zip\" asset, which is a special case for the add-on zip. If an asset is not defined, its location will be automatically evaluated.\nNote: assets are treated as \"new style\" format strings with the following keywords - id, username, repository, ref, system, arch and version (version is available for zip assets only).", "additionalProperties": { "type": "string", "title": "Asset location", - "description": "The real asset location" + "description": "The real asset location. Can be either 1) a relative path, a HTTP/HTTPS URL, or a release asset with the following format - \"release_asset:///\"." } }, "asset_prefix": { @@ -40,6 +40,24 @@ "type": "string", "title": "Github repository", "description": "GitHub repository name. If not set, it is assumed to be the same as the add-on id." + }, + "tag_pattern": { + "type": "string", + "title": "Tag RegEx pattern", + "description": "The pattern for matching eligible tags. If not set, all tags are considered." + }, + "token": { + "type": "string", + "title": "Access Token", + "description": "The token to use for accessing the repository. If not provided, the repository must have public access." + }, + "platforms": { + "type": "array", + "title": "Supported platforms", + "description": "Platforms where the add-on is supported. If not set, it is assumed all platforms are supported.", + "items": { + "type": "string" + } } }, "required": [ diff --git a/resources/settings.xml b/resources/settings.xml index 4802953..1e8ca5a 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -7,5 +7,6 @@ + diff --git a/setup.cfg b/setup.cfg index d93cdc4..6deafc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,2 @@ [flake8] -max-line-length = 370 - -[pytest] -norecursedirs = .git +max-line-length = 120 diff --git a/standalone.py b/standalone.py new file mode 100644 index 0000000..6c487d0 --- /dev/null +++ b/standalone.py @@ -0,0 +1,30 @@ +import logging +import os + +from lib.httpserver import threaded_http_server +from lib.platform.os_platform import get_platform +from lib.repository import Repository +from lib.routes import add_repository_routes + +addon_path = os.path.dirname(__file__) +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(levelname)s %(message)s") +add_repository_routes(Repository( + files=(os.path.join(addon_path, "resources", "repository.json"),), + platform=get_platform())) + + +def run(port): + server = threaded_http_server("", port) + logging.debug("Server started at port %d", port) + + try: + server.serve_forever() + except KeyboardInterrupt: + logging.debug("Execution interrupted") + finally: + logging.debug("Closing server") + server.server_close() + + +if __name__ == "__main__": + run(8080)