From 4159c3ada378c03496d23cd21d28be079f7a4684 Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Wed, 18 Dec 2024 02:40:41 +0100 Subject: [PATCH 1/5] Add ROS support --- repology-schemacheck.py | 1 + repology/packagemaker/names.py | 2 + repology/parsers/parsers/ros.py | 83 +++++++++++++++++++++++++++++++++ repos.d/ros.yaml | 56 ++++++++++++++++++++++ 4 files changed, 142 insertions(+) create mode 100644 repology/parsers/parsers/ros.py create mode 100644 repos.d/ros.yaml diff --git a/repology-schemacheck.py b/repology-schemacheck.py index 7ca35f90..2fee0ee3 100755 --- a/repology-schemacheck.py +++ b/repology-schemacheck.py @@ -107,6 +107,7 @@ 'pypi', 'ravenports', 'reactos', + 'ros', 'rosa', 'rubygems', 'rudix', diff --git a/repology/packagemaker/names.py b/repology/packagemaker/names.py index 9ec4ca99..49fc63e9 100644 --- a/repology/packagemaker/names.py +++ b/repology/packagemaker/names.py @@ -246,6 +246,8 @@ class NameType: OPAM_NAME: ClassVar[int] = GENERIC_SRC_NAME + ROS_NAME: ClassVar[int] = GENERIC_SRC_NAME + @dataclass class _NameMapping: diff --git a/repology/parsers/parsers/ros.py b/repology/parsers/parsers/ros.py new file mode 100644 index 00000000..397f8cb9 --- /dev/null +++ b/repology/parsers/parsers/ros.py @@ -0,0 +1,83 @@ +# Copyright (C) 2024 Guilhem Saurel +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +from typing import Iterable + +from repology.logger import Logger +from repology.package import LinkType +from repology.packagemaker import NameType, PackageFactory, PackageMaker +from repology.parsers import Parser +from yaml import load + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +def ros_extract_recipe_url(key, /, *, url, tags, version, **kwargs): + release = tags["release"].format( + package=key, + version=version, + upstream_version=version, + ) + + if "github.com" in url: + return url.removesuffix(".git") + f"/blob/{release}/package.xml" + if "gitlab" in url: + return url.removesuffix(".git") + f"/-/blob/{release}/package.xml" + + +class RosYamlParser(Parser): + def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: + with open(path) as f: + data = load(f, Loader=Loader) + for key, packagedata in data["repositories"].items(): + with factory.begin(key) as pkg: + # Some included packages are not yet released, + # and only available as source + if "release" not in packagedata: + pkg.log(f"dropping {pkg}: no release", severity=Logger.ERROR) + continue + + release = packagedata["release"] + + if "version" not in release: + pkg.log(f"dropping {pkg}: has no version.", severity=Logger.ERROR) + continue + + if "bitbucket" in release["url"]: + pkg.log(f"dropping {pkg}: RIP bitbucket", severity=Logger.ERROR) + continue + + pkg.add_name(key, NameType.ROS_NAME) + pkg.set_version(release["version"].split("-")[0]) + + if recipe_url := ros_extract_recipe_url(key, **release): + pkg.add_links(LinkType.PACKAGE_RECIPE, recipe_url) + else: + pkg.log(f"{pkg} has no known recipe url", severity=Logger.WARNING) + + if source := packagedata.get("source"): + pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, source["url"]) + else: + pkg.log(f"{pkg} has no source", severity=Logger.WARNING) + + if doc := packagedata.get("doc"): + pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, doc["url"]) + + yield pkg diff --git a/repos.d/ros.yaml b/repos.d/ros.yaml new file mode 100644 index 00000000..f30e8ca8 --- /dev/null +++ b/repos.d/ros.yaml @@ -0,0 +1,56 @@ +########################################################################### +# ROS +########################################################################### +{% macro ros(version, name, minpackages, valid_till) %} +- name: ros_{{version}}_{{name}} + type: repository + desc: ROS {{name}} {% if version > 1 %}(ROS {{version}}){% endif %} + statsgroup: ros + family: ros + ruleset: ros + color: '22314E' + minpackages: {{minpackages}} + {% if valid_till %} + valid_till: {{valid_till}} + {% endif %} + default_maintainer: fallback-mnt-ros@repology + sources: + - name: distribution.yaml + fetcher: + class: FileFetcher + url: https://raw.githubusercontent.com/ros/rosdistro/refs/heads/master/{{name}}/distribution.yaml + parser: + class: RosYamlParser + repolinks: + - desc: ROS home + url: https://ros.org/ + - desc: ROS packages + url: https://index.ros.org/ + - desc: {{name}} GitHub repository + url: https://github.com/ros/rosdistro/tree/master/{{name}} + - desc: {{name}} home + url: https://{% if version > 1 %}docs.ros.org/en{% else %}wiki.ros.org{% endif %}/{{name}} + groups: [ all, production, ros, ros{{version}} ] +{% endmacro %} + +{{ ros(1, 'groovy', minpackages=200, valid_till='2014-07-31') }} +{{ ros(1, 'hydro', minpackages=550, valid_till='2015-06-30') }} +{{ ros(1, 'indigo', minpackages=950, valid_till='2019-04-30') }} +{{ ros(1, 'jade', minpackages=450, valid_till='2017-05-31') }} +{{ ros(1, 'kinetic', minpackages=950, valid_till='2021-04-30') }} +{{ ros(1, 'lunar', minpackages=350, valid_till='2019-05-31') }} +{{ ros(1, 'melodic', minpackages=850, valid_till='2023-06-27') }} +{{ ros(1, 'noetic', minpackages=750, valid_till='2025-05-31') }} + +{{ ros(2, 'ardent', minpackages=50, valid_till='2018-12-31') }} +{{ ros(2, 'bouncy', minpackages=50, valid_till='2019-06-30') }} +{{ ros(2, 'crystal', minpackages=100, valid_till='2019-12-31') }} +{{ ros(2, 'dashing', minpackages=200, valid_till='2021-05-31') }} +{{ ros(2, 'eloquent', minpackages=200, valid_till='2020-11-30') }} +{{ ros(2, 'foxy', minpackages=450, valid_till='2023-05-31') }} +{{ ros(2, 'galactic', minpackages=350, valid_till='2022-11-30') }} +{{ ros(2, 'humble', minpackages=650, valid_till='2027-05-31') }} +{{ ros(2, 'iron', minpackages=500, valid_till='2024-11-30') }} +{{ ros(2, 'jazzy', minpackages=550, valid_till='2029-05-31') }} + +{{ ros(2, 'rolling', minpackages=500) }} From 74321641cb6df425a3427cfb5a9755fa764a5a4b Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Thu, 19 Dec 2024 13:55:55 +0100 Subject: [PATCH 2/5] ros.py: use single quotes --- repology/parsers/parsers/ros.py | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/repology/parsers/parsers/ros.py b/repology/parsers/parsers/ros.py index 397f8cb9..2583a8b1 100644 --- a/repology/parsers/parsers/ros.py +++ b/repology/parsers/parsers/ros.py @@ -30,54 +30,54 @@ def ros_extract_recipe_url(key, /, *, url, tags, version, **kwargs): - release = tags["release"].format( + release = tags['release'].format( package=key, version=version, upstream_version=version, ) - if "github.com" in url: - return url.removesuffix(".git") + f"/blob/{release}/package.xml" - if "gitlab" in url: - return url.removesuffix(".git") + f"/-/blob/{release}/package.xml" + if 'github.com' in url: + return url.removesuffix('.git') + f'/blob/{release}/package.xml' + if 'gitlab' in url: + return url.removesuffix('.git') + f'/-/blob/{release}/package.xml' class RosYamlParser(Parser): def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: with open(path) as f: data = load(f, Loader=Loader) - for key, packagedata in data["repositories"].items(): + for key, packagedata in data['repositories'].items(): with factory.begin(key) as pkg: # Some included packages are not yet released, # and only available as source - if "release" not in packagedata: - pkg.log(f"dropping {pkg}: no release", severity=Logger.ERROR) + if 'release' not in packagedata: + pkg.log(f'dropping {pkg}: no release', severity=Logger.ERROR) continue - release = packagedata["release"] + release = packagedata['release'] - if "version" not in release: - pkg.log(f"dropping {pkg}: has no version.", severity=Logger.ERROR) + if 'version' not in release: + pkg.log(f'dropping {pkg}: has no version.', severity=Logger.ERROR) continue - if "bitbucket" in release["url"]: - pkg.log(f"dropping {pkg}: RIP bitbucket", severity=Logger.ERROR) + if 'bitbucket' in release['url']: + pkg.log(f'dropping {pkg}: RIP bitbucket', severity=Logger.ERROR) continue pkg.add_name(key, NameType.ROS_NAME) - pkg.set_version(release["version"].split("-")[0]) + pkg.set_version(release['version'].split('-')[0]) if recipe_url := ros_extract_recipe_url(key, **release): pkg.add_links(LinkType.PACKAGE_RECIPE, recipe_url) else: - pkg.log(f"{pkg} has no known recipe url", severity=Logger.WARNING) + pkg.log(f'{pkg} has no known recipe url', severity=Logger.WARNING) - if source := packagedata.get("source"): - pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, source["url"]) + if source := packagedata.get('source'): + pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, source['url']) else: - pkg.log(f"{pkg} has no source", severity=Logger.WARNING) + pkg.log(f'{pkg} has no source', severity=Logger.WARNING) - if doc := packagedata.get("doc"): - pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, doc["url"]) + if doc := packagedata.get('doc'): + pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, doc['url']) yield pkg From d1ea1a2db32f60dd79f6ed1fb39bffff69890bbd Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Thu, 19 Dec 2024 15:28:15 +0100 Subject: [PATCH 3/5] ros: follow review --- repology/parsers/parsers/ros.py | 19 +++++++------------ repos.d/ros.yaml | 4 ++-- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/repology/parsers/parsers/ros.py b/repology/parsers/parsers/ros.py index 2583a8b1..36cbe5d9 100644 --- a/repology/parsers/parsers/ros.py +++ b/repology/parsers/parsers/ros.py @@ -21,12 +21,7 @@ from repology.package import LinkType from repology.packagemaker import NameType, PackageFactory, PackageMaker from repology.parsers import Parser -from yaml import load - -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader +from yaml import safe_load def ros_extract_recipe_url(key, /, *, url, tags, version, **kwargs): @@ -45,23 +40,23 @@ def ros_extract_recipe_url(key, /, *, url, tags, version, **kwargs): class RosYamlParser(Parser): def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: with open(path) as f: - data = load(f, Loader=Loader) + data = safe_load(f) for key, packagedata in data['repositories'].items(): with factory.begin(key) as pkg: # Some included packages are not yet released, # and only available as source if 'release' not in packagedata: - pkg.log(f'dropping {pkg}: no release', severity=Logger.ERROR) + pkg.log(f'dropping {key}: no release', severity=Logger.ERROR) continue release = packagedata['release'] if 'version' not in release: - pkg.log(f'dropping {pkg}: has no version.', severity=Logger.ERROR) + pkg.log(f'dropping {key}: has no version.', severity=Logger.ERROR) continue if 'bitbucket' in release['url']: - pkg.log(f'dropping {pkg}: RIP bitbucket', severity=Logger.ERROR) + pkg.log(f'dropping {key}: RIP bitbucket', severity=Logger.ERROR) continue pkg.add_name(key, NameType.ROS_NAME) @@ -70,12 +65,12 @@ def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMake if recipe_url := ros_extract_recipe_url(key, **release): pkg.add_links(LinkType.PACKAGE_RECIPE, recipe_url) else: - pkg.log(f'{pkg} has no known recipe url', severity=Logger.WARNING) + pkg.log(f'{key} has no known recipe url', severity=Logger.WARNING) if source := packagedata.get('source'): pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, source['url']) else: - pkg.log(f'{pkg} has no source', severity=Logger.WARNING) + pkg.log(f'{key} has no source', severity=Logger.WARNING) if doc := packagedata.get('doc'): pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, doc['url']) diff --git a/repos.d/ros.yaml b/repos.d/ros.yaml index f30e8ca8..ce65bd76 100644 --- a/repos.d/ros.yaml +++ b/repos.d/ros.yaml @@ -4,7 +4,7 @@ {% macro ros(version, name, minpackages, valid_till) %} - name: ros_{{version}}_{{name}} type: repository - desc: ROS {{name}} {% if version > 1 %}(ROS {{version}}){% endif %} + desc: ROS {% if version > 1 %}{{ version }} {% endif %}{{name}} statsgroup: ros family: ros ruleset: ros @@ -40,7 +40,7 @@ {{ ros(1, 'kinetic', minpackages=950, valid_till='2021-04-30') }} {{ ros(1, 'lunar', minpackages=350, valid_till='2019-05-31') }} {{ ros(1, 'melodic', minpackages=850, valid_till='2023-06-27') }} -{{ ros(1, 'noetic', minpackages=750, valid_till='2025-05-31') }} +{{ ros(1, 'noetic', minpackages=700, valid_till='2025-05-31') }} {{ ros(2, 'ardent', minpackages=50, valid_till='2018-12-31') }} {{ ros(2, 'bouncy', minpackages=50, valid_till='2019-06-30') }} From 5c1c2ce8cc497e97d5be7322355205ba15416ed8 Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Thu, 19 Dec 2024 20:00:59 +0100 Subject: [PATCH 4/5] ros: dont drop bitbucket packages for a missing link --- repology/parsers/parsers/ros.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repology/parsers/parsers/ros.py b/repology/parsers/parsers/ros.py index 36cbe5d9..2670703a 100644 --- a/repology/parsers/parsers/ros.py +++ b/repology/parsers/parsers/ros.py @@ -35,6 +35,12 @@ def ros_extract_recipe_url(key, /, *, url, tags, version, **kwargs): return url.removesuffix('.git') + f'/blob/{release}/package.xml' if 'gitlab' in url: return url.removesuffix('.git') + f'/-/blob/{release}/package.xml' + if 'bitbucket' in url: + # It is not possible to construct a bitbucket url for a file + # on a specific tag + return + err = f'ROS package {key} is neither on github, gitlab, or bitbucket' + raise RuntimeError(err) class RosYamlParser(Parser): @@ -55,10 +61,6 @@ def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMake pkg.log(f'dropping {key}: has no version.', severity=Logger.ERROR) continue - if 'bitbucket' in release['url']: - pkg.log(f'dropping {key}: RIP bitbucket', severity=Logger.ERROR) - continue - pkg.add_name(key, NameType.ROS_NAME) pkg.set_version(release['version'].split('-')[0]) From 98a55caca8b16d6389a98261d5fbf3104caf8e67 Mon Sep 17 00:00:00 2001 From: Guilhem Saurel Date: Fri, 20 Dec 2024 14:35:10 +0100 Subject: [PATCH 5/5] ros: dont include rolling --- repos.d/ros.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/repos.d/ros.yaml b/repos.d/ros.yaml index ce65bd76..fcb830ad 100644 --- a/repos.d/ros.yaml +++ b/repos.d/ros.yaml @@ -52,5 +52,3 @@ {{ ros(2, 'humble', minpackages=650, valid_till='2027-05-31') }} {{ ros(2, 'iron', minpackages=500, valid_till='2024-11-30') }} {{ ros(2, 'jazzy', minpackages=550, valid_till='2029-05-31') }} - -{{ ros(2, 'rolling', minpackages=500) }}