From b18c36b3999aa55d2e9fc8a3ca47d20092977839 Mon Sep 17 00:00:00 2001 From: games647 Date: Wed, 18 Sep 2024 15:44:55 +0200 Subject: [PATCH 1/4] Document capturing version numbers from changing URLs Provide an example for using non-capturing regex if there are changing URLs that could be used for load balancing. The provided regex doesn't interfere with the version extraction, because it doesn't capture the group, but matches multiple prefixes. --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index b80c1a41..5425a251 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,22 @@ The version number for the latest version can be detected in two ways: URL for the latest version, and the first match group is taken to be the version. (This follows the convention used by [`debian/watch`](https://wiki.debian.org/debian/watch) files.) + * If the application download is behind load-balanced URL that changes regularly + (e.g. `https://stable.dl2.example.com` and `dl.example.com`), + the regex needs to be adjusted to extract the version number in all cases. + Otherwise this project assumes a new version was published. + * You can use a non-capturing group for this use-case. For the use case above, use + the pattern below to allow downloads from: + * `https://dl.example.com/foo-v1.9.tar.gz` + * `https://stable.dl1.example.com/foo-v1.9.tar.gz` + * ... +```json +"x-checker-data": { + "type": "rotating-url", + "url": "http://example.com/last-version", + "pattern": "https://(?:dl|stable.dl\\d).example.com/foo-v([0-9.]+).tar.gz" +} +``` Some upstream vendors may add unwanted GET query parameters to the download URL, such as identifiers for counting unique downloads. From 676093e5b634b149cf62f21d6fc8675e5638bc78 Mon Sep 17 00:00:00 2001 From: games647 Date: Wed, 18 Sep 2024 15:46:48 +0200 Subject: [PATCH 2/4] Report same version strings that no new update is available For projects that uses pattern rotating urls, compare extracted version numbers if possible. Notably, this doesn't work for AppImages, because it require us to download the old/current version to extract the version number. --- src/checkers/urlchecker.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/checkers/urlchecker.py b/src/checkers/urlchecker.py index 767eb5af..1ba0d6bd 100644 --- a/src/checkers/urlchecker.py +++ b/src/checkers/urlchecker.py @@ -60,6 +60,22 @@ def extract_version(checker_data, url): return m.group(1) +def is_same_version(checker_data, current_url, new_version): + """ + Check if the new application version is the same with the current one. If the version number could be extracted, + those strings are compared with each other to be resilient against load-balanced urls pointing to the same file. + """ + if new_version is None or new_version.version is None: + # No pattern given or failed parsing the new version, so check only if the url is different + return current_url == new_version.url + + current_version_string = extract_version(checker_data, current_url) + if current_version_string is None: + # If the pattern failed to apply to the old/current version, check again only the url + # Otherwise we would compare None values + return current_url == new_version.url + + return current_version_string == new_version.version class URLChecker(Checker): PRIORITY = 99 @@ -138,9 +154,8 @@ async def check(self, external_data: ExternalBase): if not is_rotating: new_version = new_version._replace(url=url) # pylint: disable=no-member + same_version = is_same_version(external_data.checker_data, external_data.current_version.url, new_version) external_data.set_new_version( new_version, - is_update=( - is_rotating and external_data.current_version.url != new_version.url - ), + is_update=is_rotating and not same_version, ) From 43eee1d297a0c488e90a231f2714ec935f38778a Mon Sep 17 00:00:00 2001 From: games647 Date: Wed, 18 Sep 2024 15:47:16 +0200 Subject: [PATCH 3/4] Treat same checksum and size as the same version This can happen if the url for downloading changed (i.e. load-balancing), but the actual application is still the same. --- src/lib/externaldata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lib/externaldata.py b/src/lib/externaldata.py index 4d069746..b962aa6d 100644 --- a/src/lib/externaldata.py +++ b/src/lib/externaldata.py @@ -320,8 +320,7 @@ def matches(self, other: ExternalFile): for i in (self, other): assert i.checksum is None or isinstance(i.checksum, MultiDigest), i.checksum return ( - self.url == other.url - and self.checksum == other.checksum + self.checksum == other.checksum and (self.size is None or other.size is None or self.size == other.size) ) From babb21eb661ab729f4fb3b972abc2788b499cd21 Mon Sep 17 00:00:00 2001 From: games647 Date: Wed, 18 Sep 2024 16:56:36 +0200 Subject: [PATCH 4/4] Fix formatting --- src/checkers/urlchecker.py | 20 +++++++++++++------- src/lib/externaldata.py | 5 ++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/checkers/urlchecker.py b/src/checkers/urlchecker.py index 1ba0d6bd..6b383149 100644 --- a/src/checkers/urlchecker.py +++ b/src/checkers/urlchecker.py @@ -60,23 +60,27 @@ def extract_version(checker_data, url): return m.group(1) + def is_same_version(checker_data, current_url, new_version): """ - Check if the new application version is the same with the current one. If the version number could be extracted, - those strings are compared with each other to be resilient against load-balanced urls pointing to the same file. + Check if the new application version is the same with the current one. If the + version number could be extracted, those strings are compared with each other + to be resilient against load-balanced urls pointing to the same file. """ - if new_version is None or new_version.version is None: - # No pattern given or failed parsing the new version, so check only if the url is different + if new_version.version is None: + # No pattern given or failed parsing the new version, so check only + # if the url is different return current_url == new_version.url current_version_string = extract_version(checker_data, current_url) if current_version_string is None: - # If the pattern failed to apply to the old/current version, check again only the url - # Otherwise we would compare None values + # If the pattern failed to apply to the old/current version, + # check again only the url. Otherwise we would compare None values return current_url == new_version.url return current_version_string == new_version.version + class URLChecker(Checker): PRIORITY = 99 CHECKER_DATA_TYPE = "rotating-url" @@ -154,7 +158,9 @@ async def check(self, external_data: ExternalBase): if not is_rotating: new_version = new_version._replace(url=url) # pylint: disable=no-member - same_version = is_same_version(external_data.checker_data, external_data.current_version.url, new_version) + same_version = is_same_version( + external_data.checker_data, external_data.current_version.url, new_version + ) external_data.set_new_version( new_version, is_update=is_rotating and not same_version, diff --git a/src/lib/externaldata.py b/src/lib/externaldata.py index b962aa6d..825d11fa 100644 --- a/src/lib/externaldata.py +++ b/src/lib/externaldata.py @@ -319,9 +319,8 @@ def json(self) -> t.Dict[str, t.Any]: def matches(self, other: ExternalFile): for i in (self, other): assert i.checksum is None or isinstance(i.checksum, MultiDigest), i.checksum - return ( - self.checksum == other.checksum - and (self.size is None or other.size is None or self.size == other.size) + return self.checksum == other.checksum and ( + self.size is None or other.size is None or self.size == other.size ) def is_same_version(self, other: ExternalFile):