From 64e05316008130829e3f7af98a9c4bf140d7f9d9 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 02:45:17 +1000 Subject: [PATCH 1/7] Use snapshot file with -z flag --- vinca/distro.py | 12 +++++++++++- vinca/main.py | 27 ++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index 666e420..e545d9c 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -6,10 +6,11 @@ class Distro(object): - def __init__(self, distro_name, python_version=None): + def __init__(self, distro_name, python_version=None, snapshot=None): index = get_index(get_index_url()) self._distro = get_cached_distribution(index, distro_name) self.distro_name = distro_name + self.snapshot = snapshot # set up ROS environments if python_version is None: python_version = index.distributions[distro_name]["python_version"] @@ -60,6 +61,12 @@ def get_depends(self, pkg, ignore_pkgs=None): return dependencies def get_released_repo(self, pkg_name): + if pkg_name in self.snapshot: + return ( + self.snapshot[pkg_name].get("url", None), + self.snapshot[pkg_name].get("tag", None), + ) + pkg = self._distro.release_packages[pkg_name] repo = self._distro.repositories[pkg.repository_name].release_repository release_tag = get_release_tag(repo, pkg_name) @@ -74,6 +81,9 @@ def check_package(self, pkg_name): return False def get_version(self, pkg_name): + if pkg_name in self.snapshot: + return self.snapshot[pkg_name].get("version", None) + pkg = self._distro.release_packages[pkg_name] repo = self._distro.repositories[pkg.repository_name].release_repository return repo.version.split("-")[0] diff --git a/vinca/main.py b/vinca/main.py index 0be6391..1b5111e 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -126,6 +126,13 @@ def parse_command_line(argv): default=None, help="The conda platform to check existing recipes for.", ) + parser.add_argument( + "-z", + "--snapshot", + dest="snapshot", + default=None, + help="The version snapshot file (default: None)." + ) arguments = parser.parse_args(argv[1:]) global selected_platform config.parsed_args = arguments @@ -208,6 +215,15 @@ def read_vinca_yaml(filepath): return vinca_conf +def read_snapshot(filepath): + if not filepath: + return None + + yaml = ruamel.yaml.YAML() + snapshot = yaml.load(open(filepath, "r")) + return snapshot + + def generate_output(pkg_shortname, vinca_conf, distro, version, all_pkgs=None): if not all_pkgs: all_pkgs = [] @@ -534,7 +550,7 @@ def get_version(distro, vinca_conf, pkg_shortname): vinca_conf.get("package_version") and vinca_conf["package_version"][pkg_shortname] ): - version = vinca_conf["package_version"][pkg_shortname]["version"] + version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? return version @@ -607,7 +623,7 @@ def generate_source_version(distro, vinca_conf): and vinca_conf["package_version"][pkg_shortname] ): url = vinca_conf["package_version"][pkg_shortname]["url"] - version = vinca_conf["package_version"][pkg_shortname]["version"] + version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? entry = {} entry["git_url"] = url @@ -865,6 +881,7 @@ def main(): base_dir = os.path.abspath(arguments.dir) vinca_yaml = os.path.join(base_dir, "vinca.yaml") vinca_conf = read_vinca_yaml(vinca_yaml) + snapshot = read_snapshot(arguments.snapshot) from .template import generate_bld_ament_cmake from .template import generate_bld_ament_python @@ -879,7 +896,7 @@ def main(): generate_bld_colcon_merge() generate_bld_catkin_merge() generate_activate_hook() - + if arguments.trigger_new_versions: vinca_conf["trigger_new_versions"] = True else: @@ -892,7 +909,7 @@ def main(): if "python_version" in vinca_conf: python_version = vinca_conf["python_version"] - distro = Distro(vinca_conf["ros_distro"], python_version) + distro = Distro(vinca_conf["ros_distro"], python_version, snapshot) additional_pkgs, parsed_pkgs = [], [] for f in pkg_files: parsed_pkg = catkin_pkg.package.parse_package(f) @@ -994,7 +1011,7 @@ def main(): if "python_version" in vinca_conf: python_version = vinca_conf["python_version"] - distro = Distro(vinca_conf["ros_distro"], python_version) + distro = Distro(vinca_conf["ros_distro"], python_version, snapshot) selected_pkgs = get_selected_packages(distro, vinca_conf) From 686f05a8168d4e5dd2754f6ba62e90422736419e Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 12:26:36 +1000 Subject: [PATCH 2/7] Remove old package_version code --- vinca/main.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/vinca/main.py b/vinca/main.py index 1b5111e..7081251 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -544,21 +544,10 @@ def get_pkg(pkg_name): return outputs -def get_version(distro, vinca_conf, pkg_shortname): - version = distro.get_version(pkg_shortname) - if ( - vinca_conf.get("package_version") - and vinca_conf["package_version"][pkg_shortname] - ): - version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? - - return version - - def generate_outputs_version(distro, vinca_conf): outputs = [] for pkg_shortname in vinca_conf["_selected_pkgs"]: - version = get_version(distro, vinca_conf, pkg_shortname) + version = distro.get_version(pkg_shortname) output = generate_output(pkg_shortname, vinca_conf, distro, version) if output is not None: outputs.append(output) @@ -578,7 +567,7 @@ def generate_source(distro, vinca_conf): entry["git_url"] = url entry["git_rev"] = version pkg_names = resolve_pkgname(pkg_shortname, vinca_conf, distro) - pkg_version = get_version(distro, vinca_conf, pkg_shortname) + pkg_version = distro.get_version(pkg_shortname) print("Checking ", pkg_shortname, pkg_version) if not pkg_names: continue @@ -618,12 +607,6 @@ def generate_source_version(distro, vinca_conf): continue url, version = distro.get_released_repo(pkg_shortname) - if ( - vinca_conf["package_version"] - and vinca_conf["package_version"][pkg_shortname] - ): - url = vinca_conf["package_version"][pkg_shortname]["url"] - version = vinca_conf["package_version"][pkg_shortname]["version"] # TODO: What is this for? entry = {} entry["git_url"] = url From e89a9e8662a56e7e11412e18a708c42a652239ff Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Tue, 3 Sep 2024 14:44:40 +1000 Subject: [PATCH 3/7] Add snapshot system under vinca-snapshot --- setup.cfg | 1 + vinca/snapshot.py | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 vinca/snapshot.py diff --git a/setup.cfg b/setup.cfg index 237cb80..befac41 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ console_scripts = vinca-gha = vinca.generate_gha:main vinca-azure = vinca.generate_azure:main vinca-migrate = vinca.migrate:main + vinca-snapshot = vinca.snapshot:main [flake8] import-order-style = google diff --git a/vinca/snapshot.py b/vinca/snapshot.py new file mode 100644 index 0000000..164962d --- /dev/null +++ b/vinca/snapshot.py @@ -0,0 +1,117 @@ +import argparse +import os +import yaml + +from rosdistro import get_index, get_index_url, get_cached_distribution +from rosdistro.dependency_walker import DependencyWalker +from rosdistro.manifest_provider import get_release_tag + + +def main(): + parser = argparse.ArgumentParser( + description="Dependency snapshotting tool for ROS packages" + ) + parser.add_argument( + "-d", + "--distro", + type=str, + dest="distro", + default="humble", + help="ROS distribution to use (default: humble)", + required=False, + ) + parser.add_argument( + "-p", + "--package", + type=str, + dest="package", + default="ros_base", + help="ROS package to get dependencies for (default: ros_base)", + required=False, + ) + parser.add_argument( + "-t", + "--type", + type=str, + dest="types", + nargs="+", + default=[ + "buildtool", + "buildtool_export", + "build", + "build_export", + "run", + "test", + "exec", + ], + help="Dependency type to get (default: all)", + required=False, + ) + parser.add_argument( + "-s", + "--system", + type=str, + dest="system", + choices=["linux", "osx", "win64"], + default="linux", + help="System to get dependencies for (default: linux)", + required=False, + ) + parser.add_argument( + "-o", + "--output", + type=str, + dest="output", + default="snapshot.yaml", + help="Output file to write dependencies to", + required=False, + ) + parser.add_argument( + "-q", + "--quiet", + dest="quiet", + action="store_true", + help="Suppress output to stdout", + required=False, + ) + args = parser.parse_args() + + index = get_index(get_index_url()) + distro = get_cached_distribution(index, args.distro) + + python_version = index.distributions[args.distro]["python_version"] + os.environ["ROS_PYTHON_VERSION"] = "{0}".format(python_version) + os.environ["ROS_DISTRO"] = "{0}".format(args.distro) + if "ROS_ROOT" in os.environ: + os.environ.pop("ROS_ROOT") + if "ROS_PACKAGE_PATH" in os.environ: + os.environ.pop("ROS_PACKAGE_PATH") + + walker = DependencyWalker(distro, os.environ) + + deps = walker.get_recursive_depends( + pkg_name=args.package, + depend_types=args.types, + ros_packages_only=True, + ignore_pkgs=None, + ) + deps.add(args.package) + + if not args.quiet: + max_len = max([len(dep) for dep in deps]) + print("\033[1m{0:{2}} {1}\033[0m".format("Package", "Version", max_len + 2)) + + output = {} + + for dep in deps: + pkg = distro.release_packages[dep] + repo = distro.repositories[pkg.repository_name].release_repository + tag = get_release_tag(repo, dep) + version = repo.version.split("-")[0] + + if not args.quiet: + print("{0:{2}} {1}".format(dep, version, max_len + 2)) + output[dep] = {"url": repo.url, "version": version, "tag": tag} + + with open(args.output, "w") as f: + yaml.dump(output, f) From 0eb80d30110eaba4e7e227570626a2ddbdae7f79 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Sat, 7 Sep 2024 12:01:22 +1000 Subject: [PATCH 4/7] Fix Distro when snapshot is null --- vinca/distro.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index e545d9c..acf48c5 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -61,7 +61,7 @@ def get_depends(self, pkg, ignore_pkgs=None): return dependencies def get_released_repo(self, pkg_name): - if pkg_name in self.snapshot: + if self.snapshot and pkg_name in self.snapshot: return ( self.snapshot[pkg_name].get("url", None), self.snapshot[pkg_name].get("tag", None), @@ -81,7 +81,7 @@ def check_package(self, pkg_name): return False def get_version(self, pkg_name): - if pkg_name in self.snapshot: + if self.snapshot and pkg_name in self.snapshot: return self.snapshot[pkg_name].get("version", None) pkg = self._distro.release_packages[pkg_name] From 0c3783a3ba0d717bf3fb14640c42041ef8a95930 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Sat, 7 Sep 2024 12:02:26 +1000 Subject: [PATCH 5/7] Use Vinca Distro instead of disros --- vinca/snapshot.py | 63 +++++------------------------------------------ 1 file changed, 6 insertions(+), 57 deletions(-) diff --git a/vinca/snapshot.py b/vinca/snapshot.py index 164962d..7d20f62 100644 --- a/vinca/snapshot.py +++ b/vinca/snapshot.py @@ -1,10 +1,6 @@ import argparse -import os import yaml - -from rosdistro import get_index, get_index_url, get_cached_distribution -from rosdistro.dependency_walker import DependencyWalker -from rosdistro.manifest_provider import get_release_tag +from .distro import Distro def main(): @@ -29,34 +25,6 @@ def main(): help="ROS package to get dependencies for (default: ros_base)", required=False, ) - parser.add_argument( - "-t", - "--type", - type=str, - dest="types", - nargs="+", - default=[ - "buildtool", - "buildtool_export", - "build", - "build_export", - "run", - "test", - "exec", - ], - help="Dependency type to get (default: all)", - required=False, - ) - parser.add_argument( - "-s", - "--system", - type=str, - dest="system", - choices=["linux", "osx", "win64"], - default="linux", - help="System to get dependencies for (default: linux)", - required=False, - ) parser.add_argument( "-o", "--output", @@ -76,25 +44,8 @@ def main(): ) args = parser.parse_args() - index = get_index(get_index_url()) - distro = get_cached_distribution(index, args.distro) - - python_version = index.distributions[args.distro]["python_version"] - os.environ["ROS_PYTHON_VERSION"] = "{0}".format(python_version) - os.environ["ROS_DISTRO"] = "{0}".format(args.distro) - if "ROS_ROOT" in os.environ: - os.environ.pop("ROS_ROOT") - if "ROS_PACKAGE_PATH" in os.environ: - os.environ.pop("ROS_PACKAGE_PATH") - - walker = DependencyWalker(distro, os.environ) - - deps = walker.get_recursive_depends( - pkg_name=args.package, - depend_types=args.types, - ros_packages_only=True, - ignore_pkgs=None, - ) + distro = Distro(args.distro) + deps = distro.get_depends(args.package) deps.add(args.package) if not args.quiet: @@ -104,14 +55,12 @@ def main(): output = {} for dep in deps: - pkg = distro.release_packages[dep] - repo = distro.repositories[pkg.repository_name].release_repository - tag = get_release_tag(repo, dep) - version = repo.version.split("-")[0] + url, tag = distro.get_released_repo(dep) + version = distro.get_version(dep) + output[dep] = {"url": url, "version": version, "tag": tag} if not args.quiet: print("{0:{2}} {1}".format(dep, version, max_len + 2)) - output[dep] = {"url": repo.url, "version": version, "tag": tag} with open(args.output, "w") as f: yaml.dump(output, f) From 1f89b955d99946a18226bbb9399ec9def3bffc09 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Thu, 12 Sep 2024 21:43:08 +1000 Subject: [PATCH 6/7] Default vinca-snapshot package to ALL --- vinca/distro.py | 3 +++ vinca/snapshot.py | 21 +++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/vinca/distro.py b/vinca/distro.py index acf48c5..de7d445 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -96,3 +96,6 @@ def check_ros1(self): def get_python_version(self): return self._python_version + + def get_package_names(self): + return self._distro.release_packages.keys() diff --git a/vinca/snapshot.py b/vinca/snapshot.py index 7d20f62..74ab2fd 100644 --- a/vinca/snapshot.py +++ b/vinca/snapshot.py @@ -21,8 +21,8 @@ def main(): "--package", type=str, dest="package", - default="ros_base", - help="ROS package to get dependencies for (default: ros_base)", + default=None, + help="ROS package to get dependencies for (default: ALL)", required=False, ) parser.add_argument( @@ -45,8 +45,12 @@ def main(): args = parser.parse_args() distro = Distro(args.distro) - deps = distro.get_depends(args.package) - deps.add(args.package) + + if args.package is None: + deps = distro.get_package_names() + else: + deps = distro.get_depends(args.package) + deps.add(args.package) if not args.quiet: max_len = max([len(dep) for dep in deps]) @@ -55,8 +59,13 @@ def main(): output = {} for dep in deps: - url, tag = distro.get_released_repo(dep) - version = distro.get_version(dep) + try: + url, tag = distro.get_released_repo(dep) + version = distro.get_version(dep) + except AttributeError: + print("\033[93mPackage '{}' has no version set, skipping...\033[0m".format(dep)) + continue + output[dep] = {"url": url, "version": version, "tag": tag} if not args.quiet: From 200ed5e26b08207eafc4961e92ce5b9ec3c1dd27 Mon Sep 17 00:00:00 2001 From: Jayden Grubb Date: Fri, 18 Oct 2024 19:38:44 +1000 Subject: [PATCH 7/7] Skip packages not in snapshot when snapshot specified --- vinca/distro.py | 2 +- vinca/main.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vinca/distro.py b/vinca/distro.py index de7d445..aa6bf8f 100644 --- a/vinca/distro.py +++ b/vinca/distro.py @@ -74,7 +74,7 @@ def get_released_repo(self, pkg_name): def check_package(self, pkg_name): if pkg_name in self._distro.release_packages: - return True + return self.snapshot is None or pkg_name in self.snapshot elif pkg_name in self.build_packages: return True else: diff --git a/vinca/main.py b/vinca/main.py index 7081251..215e65b 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -547,6 +547,10 @@ def get_pkg(pkg_name): def generate_outputs_version(distro, vinca_conf): outputs = [] for pkg_shortname in vinca_conf["_selected_pkgs"]: + if not distro.check_package(pkg_shortname): + print(f"Could not generate output for {pkg_shortname}") + continue + version = distro.get_version(pkg_shortname) output = generate_output(pkg_shortname, vinca_conf, distro, version) if output is not None: