From d5cc842f6813c42546e377e16ab09be01c34b59a Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Mon, 2 Jan 2023 13:50:32 +0100 Subject: [PATCH 1/7] Support --sources-cache-dir option for the update command. Signed-off-by: Martin Pecka --- src/rosdep2/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 63c080543..59ca82e65 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -653,7 +653,7 @@ def update_error_handler(data_source, exc): try: if not options.quiet: print('reading in sources list data from %s' % (sources_list_dir)) - sources_cache_dir = get_sources_cache_dir() + sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() try: if os.geteuid() == 0: print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr) @@ -661,7 +661,8 @@ def update_error_handler(data_source, exc): except AttributeError: # nothing we wanna do under Windows pass - update_sources_list(success_handler=update_success_handler, + update_sources_list(sources_cache_dir=sources_cache_dir, + success_handler=update_success_handler, error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, ros_distro=options.ros_distro, From fd7bb5f2a79ee30141062a3326173715210cd388 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Mon, 2 Jan 2023 14:33:56 +0100 Subject: [PATCH 2/7] Add --meta-cache-dir option and ROSDEP_CACHE_PATH env variable. Signed-off-by: Martin Pecka --- src/rosdep2/cache_tools.py | 1 + src/rosdep2/main.py | 15 ++++++++++----- src/rosdep2/meta.py | 3 +++ src/rosdep2/sources_list.py | 10 +++++++--- test/test_rosdep_sources_list.py | 12 +++++++++--- 5 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/rosdep2/cache_tools.py b/src/rosdep2/cache_tools.py index 2930e88cd..44ede7d11 100644 --- a/src/rosdep2/cache_tools.py +++ b/src/rosdep2/cache_tools.py @@ -37,6 +37,7 @@ import pickle PICKLE_CACHE_EXT = '.pickle' +CACHE_PATH_ENV = 'ROSDEP_CACHE_PATH' def compute_filename_hash(key_filenames): diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 59ca82e65..7deaa34ca 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -63,7 +63,7 @@ from .installers import normalize_uninstalled_to_list from .installers import RosdepInstaller from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages -from .meta import MetaDatabase +from .meta import MetaDatabase, get_meta_cache_dir from .rospkg_loader import DEFAULT_VIEW_KEY from .sources_list import update_sources_list, get_sources_cache_dir,\ download_default_sources_list, SourcesListLoader, CACHE_INDEX,\ @@ -270,11 +270,12 @@ def setup_proxy_opener(): install_opener(opener) -def setup_environment_variables(ros_distro): +def setup_environment_variables(ros_distro, meta_cache_dir=None): """ Set environment variables needed to find ROS packages and evaluate conditional dependencies. :param ros_distro: The requested ROS distro passed on the CLI, or None + :param meta_cache_dir: Path to the cache directory of meta information """ if ros_distro is not None: if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != ros_distro: @@ -288,7 +289,7 @@ def setup_environment_variables(ros_distro): if 'ROS_PYTHON_VERSION' not in os.environ and 'ROS_DISTRO' in os.environ: # Set python version to version used by ROS distro - python_versions = MetaDatabase().get('ROS_PYTHON_VERSION', default=[]) + python_versions = MetaDatabase(meta_cache_dir).get('ROS_PYTHON_VERSION', default=[]) if os.environ['ROS_DISTRO'] in python_versions: os.environ['ROS_PYTHON_VERSION'] = str(python_versions[os.environ['ROS_DISTRO']]) @@ -301,12 +302,15 @@ def setup_environment_variables(ros_distro): def _rosdep_main(args): # sources cache dir is our local database. default_sources_cache = get_sources_cache_dir() + default_meta_cache = get_meta_cache_dir() parser = OptionParser(usage=_usage, prog='rosdep') parser.add_option('--os', dest='os_override', default=None, metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid') parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache, metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache)) + parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache, + metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache)) parser.add_option('--verbose', '-v', dest='verbose', default=False, action='store_true', help='verbose display') parser.add_option('--version', dest='print_version', default=False, @@ -434,7 +438,7 @@ def _rosdep_main(args): if command not in ['init', 'update', 'fix-permissions']: check_for_sources_list_init(options.sources_cache_dir) # _package_args_handler uses `ROS_DISTRO`, so environment variables must be set before - setup_environment_variables(options.ros_distro) + setup_environment_variables(options.ros_distro, options.meta_cache_dir) elif command not in ['fix-permissions']: setup_proxy_opener() @@ -666,7 +670,8 @@ def update_error_handler(data_source, exc): error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, ros_distro=options.ros_distro, - quiet=options.quiet) + quiet=options.quiet, + meta_cache_dir=options.meta_cache_dir) if not options.quiet: print('updated cache in %s' % (sources_cache_dir)) except InvalidData as e: diff --git a/src/rosdep2/meta.py b/src/rosdep2/meta.py index b8cd9006b..3ff574229 100644 --- a/src/rosdep2/meta.py +++ b/src/rosdep2/meta.py @@ -45,6 +45,7 @@ from .cache_tools import compute_filename_hash from .cache_tools import write_cache_file from .cache_tools import PICKLE_CACHE_EXT +from .cache_tools import CACHE_PATH_ENV """ Rosdep needs to store data that isn't used to resolve rosdep keys, but needs to be cached during @@ -56,6 +57,8 @@ def get_meta_cache_dir(): """Return storage location for cached meta data.""" + if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]: + return os.path.join(os.environ[CACHE_PATH_ENV], META_CACHE_DIR) ros_home = rospkg.get_ros_home() return os.path.join(ros_home, 'rosdep', META_CACHE_DIR) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index 04e1b6383..e9833a31e 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -37,7 +37,7 @@ except ImportError: import pickle -from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file +from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file, CACHE_PATH_ENV from .core import InvalidData, DownloadFailure, CachePermissionError from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data from .meta import MetaDatabase @@ -109,6 +109,8 @@ def get_default_sources_list_file(): def get_sources_cache_dir(): + if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]: + return os.path.join(os.environ[CACHE_PATH_ENV], SOURCES_CACHE_DIR) ros_home = rospkg.get_ros_home() return os.path.join(ros_home, 'rosdep', SOURCES_CACHE_DIR) @@ -435,7 +437,8 @@ def _generate_key_from_urls(urls): def update_sources_list(sources_list_dir=None, sources_cache_dir=None, success_handler=None, error_handler=None, skip_eol_distros=False, ros_distro=None, - quiet=False): + quiet=False, + meta_cache_dir=None): """ Re-downloaded data from remote sources and store in cache. Also update the cache index based on current sources. @@ -449,6 +452,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, if a particular source fails. This hook is mainly for printing errors to console. :param skip_eol_distros: skip downloading sources for EOL distros + :param meta_cache_dir: override meta cache directory :returns: list of (`DataSource`, cache_file_path) pairs for cache files that were updated, ``[str]`` @@ -516,7 +520,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, sources.append(rds) # cache metadata that isn't a source list - MetaDatabase().set('ROS_PYTHON_VERSION', python_versions) + MetaDatabase(meta_cache_dir).set('ROS_PYTHON_VERSION', python_versions) # Create a combined index of *all* the sources. We do all the # sources regardless of failures because a cache from a previous diff --git a/test/test_rosdep_sources_list.py b/test/test_rosdep_sources_list.py index dff036a4d..960bf10b0 100644 --- a/test/test_rosdep_sources_list.py +++ b/test/test_rosdep_sources_list.py @@ -214,13 +214,15 @@ def test_update_sources_list(): tempdir = tempfile.mkdtemp() # use a subdirectory of test dir to make sure rosdep creates the necessary substructure tempdir = os.path.join(tempdir, 'newdir') + metadir = os.path.join(tempdir, 'meta') errors = [] def error_handler(loc, e): errors.append((loc, e)) retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=error_handler) + sources_cache_dir=tempdir, error_handler=error_handler, + meta_cache_dir=metadir) assert retval assert len(retval) == 2, retval # one of our sources is intentionally bad, this should be a softfail @@ -253,6 +255,7 @@ def error_handler(loc, e): def test_load_cached_sources_list(): from rosdep2.sources_list import load_cached_sources_list, update_sources_list tempdir = tempfile.mkdtemp() + metadir = tempfile.mkdtemp() # test behavior on empty cache assert [] == load_cached_sources_list(sources_cache_dir=tempdir) @@ -260,7 +263,8 @@ def test_load_cached_sources_list(): # pull in cache data sources_list_dir = get_test_dir() retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=None) + sources_cache_dir=tempdir, error_handler=None, + meta_cache_dir=metadir) assert retval # now test with cached data @@ -422,11 +426,13 @@ def test_SourcesListLoader_create_default(): from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher # create temp dir for holding sources cache tempdir = tempfile.mkdtemp() + metadir = tempfile.mkdtemp() # pull in cache data sources_list_dir = get_test_dir() retval = update_sources_list(sources_list_dir=sources_list_dir, - sources_cache_dir=tempdir, error_handler=None) + sources_cache_dir=tempdir, error_handler=None, + meta_cache_dir=metadir) assert retval # now test with cached data From 01412331a453ae0eb489285cd3cc1bd8d4e100a4 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Mon, 2 Jan 2023 15:04:11 +0100 Subject: [PATCH 3/7] Documented ROSDEP_CACHE_PATH and ROSDEP_SOURCE_PATH. Signed-off-by: Martin Pecka --- doc/overview.rst | 21 +++++++++++++++++++++ doc/sources_list.rst | 20 ++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/doc/overview.rst b/doc/overview.rst index 60d98e65b..f0afa05b6 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -29,6 +29,17 @@ rosdep gets its data from. ``rosdep update`` reads through this sources list to initialize your local database. +You can override the location by setting environment variable +``ROSDEP_SOURCE_PATH``. The custom path has to exist prior to calling +``rosdep init``, otherwise the default one will be used. + +Please note that when using ``sudo``, environment +variables from the user are not passed to the command. To specify the variable +for initializing the database, call:: + + sudo mkdir -p /usr/rosdep.sources + sudo env ROSDEP_SOURCE_PATH=/usr/rosdep.sources rosdep init + Updating rosdep --------------- @@ -36,6 +47,12 @@ You can update your rosdep database by running:: rosdep update +If you have specified a custom ``ROSDEP_SOURCE_PATH``, do not forget to set it +also for this command. + +Default location of the local rosdep database is in ``$HOME/.ros/rosdep``. +To change it, set environment variable ``ROSDEP_CACHE_PATH``, or pass +command-line arguments ``--sources-cache-dir`` and ``--meta-cache-dir``. Installating rosdeps -------------------- @@ -95,5 +112,9 @@ dependencies:: +If you specified a custom ``ROSDEP_CACHE_PATH`` or used command-line arguments +``--sources-cache-dir`` and ``--meta-cache-dir``, you have to pass these to +all rosdep commands used afterwards, including ``rosdep install``. + For more information, please see the :ref:`command reference `. diff --git a/doc/sources_list.rst b/doc/sources_list.rst index b083a4389..461016855 100644 --- a/doc/sources_list.rst +++ b/doc/sources_list.rst @@ -30,6 +30,26 @@ Sources list file format ``/etc/ros/rosdep/sources.list.d``. ``sudo rosdep init`` will create a default configuration for you. +You can override the location by setting environment variable +``ROSDEP_SOURCE_PATH``. The custom path has to exist prior to calling +``rosdep init``, otherwise the default one will be used. + +Please note that when using ``sudo``, environment +variables from the user are not passed to the command. To specify the variable +for initializing the database, call:: + + sudo mkdir -p /usr/rosdep.sources + sudo env ROSDEP_SOURCE_PATH=/usr/rosdep.sources rosdep init + +If you point ``ROSDEP_SOURCE_PATH`` to a user-writable directory, you can avoid +obtaining root privileges, so you can just call:: + + mkdir -p $HOME/my_rosdep_sources + ROSDEP_SOURCE_PATH=$HOME/my_rosdep_sources rosdep init + +Be aware that ``~`` is not expanded in ``ROSDEP_SOURCE_PATH``, so you should +specify an absolute path. + rosdep processes the files in this directory, sorted by filename in ascending order. Precedence is assigned to the files in the order they are processed, with the first file having the highest From efd7094a361effdf5c86d72125b4bceed64d6808 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Mon, 2 Jan 2023 19:11:55 +0100 Subject: [PATCH 4/7] Deprecate `-c` and `-m` command line options and use `ROSDEP_CACHE_PATH` instead. Signed-off-by: Martin Pecka --- src/rosdep2/main.py | 29 ++++--- ...06d8ea9d8eec9db440c6e9937664e0159b5.pickle | 4 + .../0a12d6e7b0d47be9b76e7726720e4cb79528cbaa | 0 .../82cbc7008b5117bcc6ce794832659e4f7763d2db | 0 .../f6f4ef95664e373cd4754501337fa217f5b55d91 | 0 test/{sources_cache => sources.cache}/index | 0 test/sources.list.d.good/20-default.list | 4 + test/test_rosdep_catkin_support.py | 32 ++++++++ test/test_rosdep_installers.py | 2 +- test/test_rosdep_lookup.py | 2 +- test/test_rosdep_main.py | 27 ++++--- test/test_rosdep_sources_list.py | 80 +++++++++++++++++++ 12 files changed, 154 insertions(+), 26 deletions(-) create mode 100644 test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle rename test/{sources_cache => sources.cache}/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa (100%) rename test/{sources_cache => sources.cache}/82cbc7008b5117bcc6ce794832659e4f7763d2db (100%) rename test/{sources_cache => sources.cache}/f6f4ef95664e373cd4754501337fa217f5b55d91 (100%) rename test/{sources_cache => sources.cache}/index (100%) create mode 100644 test/sources.list.d.good/20-default.list diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 7deaa34ca..a20f54afb 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -53,7 +53,7 @@ from urllib2 import URLError import warnings -from optparse import OptionParser +from optparse import OptionParser, SUPPRESS_HELP import rospkg @@ -63,7 +63,7 @@ from .installers import normalize_uninstalled_to_list from .installers import RosdepInstaller from .lookup import RosdepLookup, ResolutionError, prune_catkin_packages -from .meta import MetaDatabase, get_meta_cache_dir +from .meta import MetaDatabase from .rospkg_loader import DEFAULT_VIEW_KEY from .sources_list import update_sources_list, get_sources_cache_dir,\ download_default_sources_list, SourcesListLoader, CACHE_INDEX,\ @@ -122,6 +122,16 @@ class UsageError(Exception): Recursively change the permissions of the user's ros home directory. May require sudo. Can be useful to fix permissions after calling "rosdep update" with sudo accidentally. + +Environment variables: + +ROSDEP_SOURCE_PATH + Overrides path to the sources list directory (by default /etc/ros/rosdep/sources.list.d). + Applies to init and update commands. + +ROSDEP_CACHE_PATH + Overrides path to the cache directory (by default $HOME/.ros/rosdep). + Applies to all commands except init. """ @@ -301,16 +311,13 @@ def setup_environment_variables(ros_distro, meta_cache_dir=None): def _rosdep_main(args): # sources cache dir is our local database. - default_sources_cache = get_sources_cache_dir() - default_meta_cache = get_meta_cache_dir() - parser = OptionParser(usage=_usage, prog='rosdep') parser.add_option('--os', dest='os_override', default=None, metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid') - parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache, - metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache)) - parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache, - metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache)) + parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=None, + metavar='SOURCES_CACHE_DIR', help=SUPPRESS_HELP) # deprecated + parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=None, + metavar='META_CACHE_DIR', help=SUPPRESS_HELP) # deprecated parser.add_option('--verbose', '-v', dest='verbose', default=False, action='store_true', help='verbose display') parser.add_option('--version', dest='print_version', default=False, @@ -657,7 +664,6 @@ def update_error_handler(data_source, exc): try: if not options.quiet: print('reading in sources list data from %s' % (sources_list_dir)) - sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() try: if os.geteuid() == 0: print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr) @@ -665,7 +671,7 @@ def update_error_handler(data_source, exc): except AttributeError: # nothing we wanna do under Windows pass - update_sources_list(sources_cache_dir=sources_cache_dir, + update_sources_list(sources_cache_dir=options.sources_cache_dir, success_handler=update_success_handler, error_handler=update_error_handler, skip_eol_distros=not options.include_eol_distros, @@ -673,6 +679,7 @@ def update_error_handler(data_source, exc): quiet=options.quiet, meta_cache_dir=options.meta_cache_dir) if not options.quiet: + sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() print('updated cache in %s' % (sources_cache_dir)) except InvalidData as e: print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr) diff --git a/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle b/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle new file mode 100644 index 000000000..3c37def2d --- /dev/null +++ b/test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle @@ -0,0 +1,4 @@ +€crosdep2.meta +CacheWrapper +q)q}q(Urosdep_versionqU0.22.1qU_CacheWrapper__dataq}q(UfoxyqKUmelodicq KUhumbleq +KUrollingq KUnoeticq KuU category_nameq UROS_PYTHON_VERSIONqub. \ No newline at end of file diff --git a/test/sources_cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa b/test/sources.cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa similarity index 100% rename from test/sources_cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa rename to test/sources.cache/0a12d6e7b0d47be9b76e7726720e4cb79528cbaa diff --git a/test/sources_cache/82cbc7008b5117bcc6ce794832659e4f7763d2db b/test/sources.cache/82cbc7008b5117bcc6ce794832659e4f7763d2db similarity index 100% rename from test/sources_cache/82cbc7008b5117bcc6ce794832659e4f7763d2db rename to test/sources.cache/82cbc7008b5117bcc6ce794832659e4f7763d2db diff --git a/test/sources_cache/f6f4ef95664e373cd4754501337fa217f5b55d91 b/test/sources.cache/f6f4ef95664e373cd4754501337fa217f5b55d91 similarity index 100% rename from test/sources_cache/f6f4ef95664e373cd4754501337fa217f5b55d91 rename to test/sources.cache/f6f4ef95664e373cd4754501337fa217f5b55d91 diff --git a/test/sources_cache/index b/test/sources.cache/index similarity index 100% rename from test/sources_cache/index rename to test/sources.cache/index diff --git a/test/sources.list.d.good/20-default.list b/test/sources.list.d.good/20-default.list new file mode 100644 index 000000000..1ac6041d1 --- /dev/null +++ b/test/sources.list.d.good/20-default.list @@ -0,0 +1,4 @@ +#autogenerated by rosdep, do not edit. use 'rosdep update' instead +yaml https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml +yaml https://github.com/ros/rosdistro/raw/master/rosdep/python.yaml +gbpdistro https://github.com/ros/rosdistro/raw/master/releases/fuerte.yaml fuerte diff --git a/test/test_rosdep_catkin_support.py b/test/test_rosdep_catkin_support.py index d117c4445..3c9ec4918 100644 --- a/test/test_rosdep_catkin_support.py +++ b/test/test_rosdep_catkin_support.py @@ -1,12 +1,35 @@ from rosdep2.catkin_support import get_installer, get_catkin_view, ValidationFailed, resolve_for_os from rosdep2.platforms.debian import APT_INSTALLER +from rosdep2.cache_tools import CACHE_PATH_ENV +from rosdep2.sources_list import SOURCE_PATH_ENV import pytest +import os +from tempfile import mkdtemp + + +def get_test_dir(): + return os.path.abspath(os.path.dirname(__file__)) + + +def get_cache_dir(): + # get_catkin_view calls update(), so we need a writable location + return mkdtemp() + + +def get_source_list_dir(): + p = os.path.join(get_test_dir(), "sources.list.d.good") + assert os.path.isdir(p) + return p @pytest.mark.online def test_workflow(): + old_cpe = os.getenv(CACHE_PATH_ENV, None) + old_spe = os.getenv(SOURCE_PATH_ENV, None) try: + os.environ[CACHE_PATH_ENV] = get_cache_dir() + os.environ[SOURCE_PATH_ENV] = get_source_list_dir() installer = get_installer(APT_INSTALLER) view = get_catkin_view('fuerte', 'ubuntu', 'lucid') resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'lucid') @@ -16,3 +39,12 @@ def test_workflow(): except ValidationFailed: # tests fail on the server because 'rosdep init' has not been run pass + finally: + if old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = old_cpe + if old_spe is None: + del os.environ[SOURCE_PATH_ENV] + else: + os.environ[SOURCE_PATH_ENV] = old_spe diff --git a/test/test_rosdep_installers.py b/test/test_rosdep_installers.py index 4b91a3f22..9eb9825e6 100644 --- a/test/test_rosdep_installers.py +++ b/test/test_rosdep_installers.py @@ -47,7 +47,7 @@ def get_test_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = os.path.join(get_test_dir(), 'sources.cache') assert os.path.isdir(p) return p diff --git a/test/test_rosdep_lookup.py b/test/test_rosdep_lookup.py index 176abd0d0..6179ea74e 100644 --- a/test/test_rosdep_lookup.py +++ b/test/test_rosdep_lookup.py @@ -45,7 +45,7 @@ def get_test_tree_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = os.path.join(get_test_dir(), 'sources.cache') assert os.path.isdir(p) return p diff --git a/test/test_rosdep_main.py b/test/test_rosdep_main.py index c94143c24..532dee5d2 100644 --- a/test/test_rosdep_main.py +++ b/test/test_rosdep_main.py @@ -47,6 +47,7 @@ from rosdep2.ament_packages import AMENT_PREFIX_PATH_ENV_VAR from rosdep2.main import rosdep_main from rosdep2.main import setup_proxy_opener +from rosdep2.cache_tools import CACHE_PATH_ENV GITHUB_BASE_URL = 'https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml' @@ -66,7 +67,7 @@ def get_test_catkin_tree_dir(): def get_cache_dir(): - p = os.path.join(get_test_dir(), 'sources_cache') + p = get_test_dir() assert os.path.isdir(p) return p @@ -95,6 +96,7 @@ def setUp(self): self.old_rr = rospkg.get_ros_root() self.old_rpp = rospkg.get_ros_package_path() self.old_app = os.getenv(AMENT_PREFIX_PATH_ENV_VAR, None) + self.old_cpe = os.getenv(CACHE_PATH_ENV, None) if 'ROS_ROOT' in os.environ: del os.environ['ROS_ROOT'] os.environ['ROS_PACKAGE_PATH'] = os.path.join(get_test_tree_dir()) @@ -102,6 +104,7 @@ def setUp(self): if 'ROS_PYTHON_VERSION' not in os.environ: # avoid `test_check` failure due to warning on stderr os.environ['ROS_PYTHON_VERSION'] = sys.version[0] + os.environ[CACHE_PATH_ENV] = get_cache_dir() def tearDown(self): if self.old_rr is not None: @@ -110,10 +113,13 @@ def tearDown(self): os.environ['ROS_PACKAGE_PATH'] = self.old_rpp if self.old_app is not None: os.environ[AMENT_PREFIX_PATH_ENV_VAR] = self.old_app + if self.old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = self.old_cpe def test_bad_commands(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] for commands in [[], ['fake', 'something'], ['check'], ['install', '-a', 'rospack_fake'], ['check', 'rospack_fake', '--os', 'ubuntulucid'], ]: @@ -124,8 +130,7 @@ def test_bad_commands(self): pass def test_check(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] with fakeout() as b: try: @@ -163,8 +168,7 @@ def test_check(self): @patch('rosdep2.platforms.debian.read_stdout') @patch('rosdep2.installers.os.geteuid', return_value=1) def test_install(self, mock_geteuid, mock_read_stdout): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] catkin_tree = get_test_catkin_tree_dir() def read_stdout(cmd, capture_stderr=False): @@ -221,12 +225,11 @@ def read_stdout(cmd, capture_stderr=False): def test_where_defined(self): try: - sources_cache = get_cache_dir() expected = GITHUB_PYTHON_URL for command in (['where_defined', 'testpython'], ['where_defined', 'testpython']): with fakeout() as b: # set os to ubuntu so this test works on different platforms - rosdep_main(command + ['-c', sources_cache, '--os=ubuntu:lucid']) + rosdep_main(command + ['--os=ubuntu:lucid']) stdout, stderr = b output = stdout.getvalue().strip() assert output == expected, output @@ -235,8 +238,7 @@ def test_where_defined(self): def test_what_needs(self): try: - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] expected = ['python_dep'] with fakeout() as b: rosdep_main(['what-needs', 'testpython'] + cmd_extras) @@ -253,8 +255,7 @@ def test_what_needs(self): assert False, 'system exit occurred' def test_keys(self): - sources_cache = get_cache_dir() - cmd_extras = ['-c', sources_cache] + cmd_extras = [] try: with fakeout() as b: diff --git a/test/test_rosdep_sources_list.py b/test/test_rosdep_sources_list.py index 960bf10b0..a270b96d6 100644 --- a/test/test_rosdep_sources_list.py +++ b/test/test_rosdep_sources_list.py @@ -38,6 +38,9 @@ import pytest import rospkg.distro import rosdep2.sources_list +from rosdep2.sources_list import SOURCES_CACHE_DIR +from rosdep2.cache_tools import CACHE_PATH_ENV +from rosdep2.meta import META_CACHE_DIR GITHUB_BASE_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/base.yaml' @@ -499,6 +502,83 @@ def test_SourcesListLoader_create_default(): loader.load_view(GITHUB_URL, rosdep_db) +@pytest.mark.online +def test_SourcesListLoader_cache_dir_from_env_var(): + from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher + # create temp dir for holding sources cache + tempdir = tempfile.mkdtemp() + old_cpe = os.getenv(CACHE_PATH_ENV, None) + os.environ[CACHE_PATH_ENV] = tempdir + + try: + # pull in cache data + sources_list_dir = get_test_dir() + retval = update_sources_list(sources_list_dir=sources_list_dir, error_handler=None) + assert retval + sources_cache_dir = os.path.join(tempdir, SOURCES_CACHE_DIR) + assert os.path.exists(sources_cache_dir) + assert len(os.listdir(sources_cache_dir)) > 0 + meta_cache_dir = os.path.join(tempdir, META_CACHE_DIR) + assert os.path.exists(meta_cache_dir) + assert len(os.listdir(meta_cache_dir)) > 0 + + # now test with cached data + matcher = rosdep2.sources_list.DataSourceMatcher(['ubuntu', 'lucid']) + loader = SourcesListLoader.create_default(matcher) + assert loader.sources + assert not any([s for s in loader.sources if not matcher.matches(s)]) + + # test API + + # very simple, always raises RNF + try: + loader.get_rosdeps('foo') + except rospkg.ResourceNotFound: + pass + try: + loader.get_view_key('foo') + except rospkg.ResourceNotFound: + pass + + assert [] == loader.get_loadable_resources() + all_sources = [x.url for x in loader.sources] + assert all_sources == loader.get_loadable_views() + + # test get_source early to make sure model matches expected + try: + loader.get_source('foo') + assert False, 'should have raised' + except rospkg.ResourceNotFound: + pass + s = loader.get_source(GITHUB_URL) + assert s.url == GITHUB_URL + + # get_view_dependencies + # - loader doesn't new view name, so assume everything + assert all_sources == loader.get_view_dependencies('foo') + # - actual views don't depend on anything + assert [] == loader.get_view_dependencies(GITHUB_URL) + + # load_view + from rosdep2.model import RosdepDatabase + for verbose in [True, False]: + rosdep_db = RosdepDatabase() + loader.load_view(GITHUB_URL, rosdep_db, verbose=verbose) + assert rosdep_db.is_loaded(GITHUB_URL) + assert [] == rosdep_db.get_view_dependencies(GITHUB_URL) + entry = rosdep_db.get_view_data(GITHUB_URL) + assert 'cmake' in entry.rosdep_data + assert GITHUB_URL == entry.origin + + # - coverage, repeat loader, should noop + loader.load_view(GITHUB_URL, rosdep_db) + finally: + if old_cpe is None: + del os.environ[CACHE_PATH_ENV] + else: + os.environ[CACHE_PATH_ENV] = old_cpe + + def test_unpickle_same_results(): try: import cPickle as pickle From da134a7f31a184e2b7b825fff1e372c6ef8a1556 Mon Sep 17 00:00:00 2001 From: SubaruArai Date: Thu, 17 Aug 2023 11:56:00 +0900 Subject: [PATCH 5/7] fix ci: test_rosdep_catkin_support.py Change old test target (ubuntu lucid: 10.04) to ubuntu focal: 20.04 [python: python-dev] -> [python3: python3-dev] --- test/test_rosdep_catkin_support.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_rosdep_catkin_support.py b/test/test_rosdep_catkin_support.py index d117c4445..1bbe9d853 100644 --- a/test/test_rosdep_catkin_support.py +++ b/test/test_rosdep_catkin_support.py @@ -8,11 +8,11 @@ def test_workflow(): try: installer = get_installer(APT_INSTALLER) - view = get_catkin_view('fuerte', 'ubuntu', 'lucid') - resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'lucid') + view = get_catkin_view('noetic', 'ubuntu', 'focal') + resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'focal') assert ['cmake'] == resolved - resolved = resolve_for_os('python', view, installer, 'ubuntu', 'lucid') - assert resolved == ['python-dev'] + resolved = resolve_for_os('python3', view, installer, 'ubuntu', 'focal') + assert resolved == ['python3-dev'] except ValidationFailed: # tests fail on the server because 'rosdep init' has not been run pass From 267cf6afe0cdb4a3fc331cae3f46bcc2708a2969 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Tue, 22 Aug 2023 10:53:17 +0200 Subject: [PATCH 6/7] Add info about sudo --preserve-env option. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Steven! Ragnarök --- doc/overview.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/overview.rst b/doc/overview.rst index f0afa05b6..541462e51 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -39,6 +39,11 @@ for initializing the database, call:: sudo mkdir -p /usr/rosdep.sources sudo env ROSDEP_SOURCE_PATH=/usr/rosdep.sources rosdep init + +Alternatively you can use `--preserve-env` to pass through current environment variable values. + export ROSDEP_SOURCE_PATH=/tmp/example-rosdep-sources + mkdir $ROSDEP_SOURCE_PATH + sudo --preserve-env=ROSDEP_SOURCE_PATH rosdep init Updating rosdep --------------- From df842ecb61584ba4d40110db61b997b5eb2b7468 Mon Sep 17 00:00:00 2001 From: Martin Pecka Date: Tue, 22 Aug 2023 12:28:39 +0200 Subject: [PATCH 7/7] Fixed issues from review, added a test for conflicting ROSDEP_CACHE_PATH and CLI arguments. --- doc/overview.rst | 5 +++-- src/rosdep2/main.py | 3 ++- test/empty_cache/index | 1 + test/test_rosdep_main.py | 25 +++++++++++++++++++++++++ 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 test/empty_cache/index diff --git a/doc/overview.rst b/doc/overview.rst index 541462e51..d9e983a8c 100644 --- a/doc/overview.rst +++ b/doc/overview.rst @@ -58,6 +58,9 @@ also for this command. Default location of the local rosdep database is in ``$HOME/.ros/rosdep``. To change it, set environment variable ``ROSDEP_CACHE_PATH``, or pass command-line arguments ``--sources-cache-dir`` and ``--meta-cache-dir``. +If both the environment variable and the command-line options are specified, +the command-line options are used. Please note that usage of the command-line +options is deprecated and usage of the environment variable is preferred. Installating rosdeps -------------------- @@ -115,8 +118,6 @@ dependencies:: $ rosdep resolve eigen libeigen3-dev - - If you specified a custom ``ROSDEP_CACHE_PATH`` or used command-line arguments ``--sources-cache-dir`` and ``--meta-cache-dir``, you have to pass these to all rosdep commands used afterwards, including ``rosdep install``. diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 0f3255e0f..e1206fde6 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -443,7 +443,8 @@ def _rosdep_main(args): options.as_root = dict((k, str_to_bool(v)) for k, v in key_list_to_dict(options.as_root).items()) if command not in ['init', 'update', 'fix-permissions']: - check_for_sources_list_init(options.sources_cache_dir) + sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir() + check_for_sources_list_init(sources_cache_dir) # _package_args_handler uses `ROS_DISTRO`, so environment variables must be set before setup_environment_variables(options.ros_distro, options.meta_cache_dir) elif command not in ['fix-permissions']: diff --git a/test/empty_cache/index b/test/empty_cache/index new file mode 100644 index 000000000..ba2fda4a6 --- /dev/null +++ b/test/empty_cache/index @@ -0,0 +1 @@ +#autogenerated by rosdep, do not edit. use 'rosdep update' instead \ No newline at end of file diff --git a/test/test_rosdep_main.py b/test/test_rosdep_main.py index 532dee5d2..6636243ef 100644 --- a/test/test_rosdep_main.py +++ b/test/test_rosdep_main.py @@ -72,6 +72,12 @@ def get_cache_dir(): return p +def get_empty_cache_dir(): + p = os.path.join(get_test_dir(), 'empty_cache') + assert os.path.isdir(p) + return p + + @contextmanager def fakeout(): realstdout = sys.stdout @@ -308,3 +314,22 @@ def test_invalid_package_message(self): assert len(output) >= 2 assert test_package_dir in output[-2] assert 'Package version ":{version}" does not follow version conventions' in output[-1] + + def test_sources_cache_dir_cli(self): + # Try resolving a dependency from a env-var-defined cache + try: + with fakeout() as b: + rosdep_main(['resolve', 'testtinyxml']) + stdout, stderr = b + assert len(stdout.getvalue().strip()) > 0, stdout.getvalue() + assert len(stderr.getvalue().strip()) == 0, stderr.getvalue() + except SystemExit: + assert False, 'system exit occurred' + + # Try resolving a dependency from a CLI-defined cache (should have priority over env var) + try: + with fakeout() as b: + rosdep_main(['resolve', 'testtinyxml', '--sources-cache-dir', get_empty_cache_dir()]) + assert False, 'system exit should have occurred' + except SystemExit: + pass