Allow full configuration of the cache directories used by rosdep via environment variable, deprecate --sources-cache-dir

doc/overview.rst

@@ -29,13 +29,38 @@ rosdep gets its data from.
 ``rosdep update`` reads through this sources list to initialize your
 local database.
 
+You can override the location by setting environment variable
+``ROSDEP_SOURCE_PATH``. The custom path has to exist prior to calling
+``rosdep init``, otherwise the default one will be used.
+
+Please note that when using ``sudo``, environment
+variables from the user are not passed to the command. To specify the variable
+for initializing the database, call::
+
+    sudo mkdir -p /usr/rosdep.sources
+    sudo env ROSDEP_SOURCE_PATH=/usr/rosdep.sources rosdep init
+
+Alternatively you can use `--preserve-env` to pass through current environment variable values.
+    export ROSDEP_SOURCE_PATH=/tmp/example-rosdep-sources
+    mkdir $ROSDEP_SOURCE_PATH
+    sudo --preserve-env=ROSDEP_SOURCE_PATH rosdep init
+
 Updating rosdep
 ---------------
 
 You can update your rosdep database by running::
 
     rosdep update
 
+If you have specified a custom ``ROSDEP_SOURCE_PATH``, do not forget to set it
+also for this command.
+
+Default location of the local rosdep database is in ``$HOME/.ros/rosdep``.
+To change it, set environment variable ``ROSDEP_CACHE_PATH``, or pass
+command-line arguments ``--sources-cache-dir`` and ``--meta-cache-dir``.
+If both the environment variable and the command-line options are specified,
+the command-line options are used. Please note that usage of the command-line
+options is deprecated and usage of the environment variable is preferred.
 
 Installating rosdeps
 --------------------
@@ -93,7 +118,9 @@ dependencies::
     $ rosdep resolve eigen
     libeigen3-dev
 
-
+If you specified a custom ``ROSDEP_CACHE_PATH`` or used command-line arguments
+``--sources-cache-dir`` and ``--meta-cache-dir``, you have to pass these to
+all rosdep commands used afterwards, including ``rosdep install``.
 
 For more information, please see the :ref:`command reference <rosdep_usage>`.
 

doc/sources_list.rst

@@ -30,6 +30,26 @@ Sources list file format
 ``/etc/ros/rosdep/sources.list.d``.  ``sudo rosdep init`` will create
 a default configuration for you.
 
+You can override the location by setting environment variable
+``ROSDEP_SOURCE_PATH``. The custom path has to exist prior to calling
+``rosdep init``, otherwise the default one will be used.
+
+Please note that when using ``sudo``, environment
+variables from the user are not passed to the command. To specify the variable
+for initializing the database, call::
+
+    sudo mkdir -p /usr/rosdep.sources
+    sudo env ROSDEP_SOURCE_PATH=/usr/rosdep.sources rosdep init
+
+If you point ``ROSDEP_SOURCE_PATH`` to a user-writable directory, you can avoid
+obtaining root privileges, so you can just call::
+
+    mkdir -p $HOME/my_rosdep_sources
+    ROSDEP_SOURCE_PATH=$HOME/my_rosdep_sources rosdep init
+
+Be aware that ``~`` is not expanded in ``ROSDEP_SOURCE_PATH``, so you should
+specify an absolute path.
+
 rosdep processes the files in this directory, sorted by filename in
 ascending order.  Precedence is assigned to the files in the order
 they are processed, with the first file having the highest

src/rosdep2/cache_tools.py

@@ -37,6 +37,7 @@
     import pickle
 
 PICKLE_CACHE_EXT = '.pickle'
+CACHE_PATH_ENV = 'ROSDEP_CACHE_PATH'
 
 
 def compute_filename_hash(key_filenames):

src/rosdep2/main.py

@@ -53,7 +53,7 @@
     from urllib2 import URLError
 import warnings
 
-from optparse import OptionParser
+from optparse import OptionParser, SUPPRESS_HELP
 
 import rospkg
 
@@ -122,6 +122,16 @@ class UsageError(Exception):
   Recursively change the permissions of the user's ros home directory.
   May require sudo.  Can be useful to fix permissions after calling
   "rosdep update" with sudo accidentally.
+
+Environment variables:
+
+ROSDEP_SOURCE_PATH
+  Overrides path to the sources list directory (by default /etc/ros/rosdep/sources.list.d).
+  Applies to init and update commands.
+
+ROSDEP_CACHE_PATH
+  Overrides path to the cache directory (by default $HOME/.ros/rosdep).
+  Applies to all commands except init.
 """
 
 
@@ -270,11 +280,12 @@ def setup_proxy_opener():
             install_opener(opener)
 
 
-def setup_environment_variables(ros_distro):
+def setup_environment_variables(ros_distro, meta_cache_dir=None):
     """
     Set environment variables needed to find ROS packages and evaluate conditional dependencies.
 
     :param ros_distro: The requested ROS distro passed on the CLI, or None
+    :param meta_cache_dir: Path to the cache directory of meta information
     """
     if ros_distro is not None:
         if 'ROS_DISTRO' in os.environ and os.environ['ROS_DISTRO'] != ros_distro:
@@ -288,7 +299,7 @@ def setup_environment_variables(ros_distro):
 
     if 'ROS_PYTHON_VERSION' not in os.environ and 'ROS_DISTRO' in os.environ:
         # Set python version to version used by ROS distro
-        python_versions = MetaDatabase().get('ROS_PYTHON_VERSION', default=[])
+        python_versions = MetaDatabase(meta_cache_dir).get('ROS_PYTHON_VERSION', default=[])
         if os.environ['ROS_DISTRO'] in python_versions:
             os.environ['ROS_PYTHON_VERSION'] = str(python_versions[os.environ['ROS_DISTRO']])
 
@@ -300,13 +311,13 @@ def setup_environment_variables(ros_distro):
 
 def _rosdep_main(args):
     # sources cache dir is our local database.
-    default_sources_cache = get_sources_cache_dir()
-
     parser = OptionParser(usage=_usage, prog='rosdep')
     parser.add_option('--os', dest='os_override', default=None,
                       metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid')
-    parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache,
-                      metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache))
+    parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=None,
+                      metavar='SOURCES_CACHE_DIR', help=SUPPRESS_HELP)  # deprecated
+    parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=None,
+                      metavar='META_CACHE_DIR', help=SUPPRESS_HELP)  # deprecated
     parser.add_option('--verbose', '-v', dest='verbose', default=False,
                       action='store_true', help='verbose display')
     parser.add_option('--version', dest='print_version', default=False,
@@ -432,9 +443,10 @@ def _rosdep_main(args):
     options.as_root = dict((k, str_to_bool(v)) for k, v in key_list_to_dict(options.as_root).items())
 
     if command not in ['init', 'update', 'fix-permissions']:
-        check_for_sources_list_init(options.sources_cache_dir)
+        sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir()
+        check_for_sources_list_init(sources_cache_dir)
         # _package_args_handler uses `ROS_DISTRO`, so environment variables must be set before
-        setup_environment_variables(options.ros_distro)
+        setup_environment_variables(options.ros_distro, options.meta_cache_dir)
     elif command not in ['fix-permissions']:
         setup_proxy_opener()
 
@@ -654,20 +666,22 @@ def update_error_handler(data_source, exc):
     try:
         if not options.quiet:
             print('reading in sources list data from %s' % (sources_list_dir))
-        sources_cache_dir = get_sources_cache_dir()
         try:
             if os.geteuid() == 0:
                 print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr)
                 print("  You should run 'sudo rosdep fix-permissions' and invoke 'rosdep update' again without sudo.", file=sys.stderr)
         except AttributeError:
             # nothing we wanna do under Windows
             pass
-        update_sources_list(success_handler=update_success_handler,
+        update_sources_list(sources_cache_dir=options.sources_cache_dir,
+                            success_handler=update_success_handler,
                             error_handler=update_error_handler,
                             skip_eol_distros=not options.include_eol_distros,
                             ros_distro=options.ros_distro,
-                            quiet=options.quiet)
+                            quiet=options.quiet,
+                            meta_cache_dir=options.meta_cache_dir)
         if not options.quiet:
+            sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir()
             print('updated cache in %s' % (sources_cache_dir))
     except InvalidData as e:
         print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr)

src/rosdep2/meta.py

@@ -45,6 +45,7 @@
 from .cache_tools import compute_filename_hash
 from .cache_tools import write_cache_file
 from .cache_tools import PICKLE_CACHE_EXT
+from .cache_tools import CACHE_PATH_ENV
 
 """
 Rosdep needs to store data that isn't used to resolve rosdep keys, but needs to be cached during
@@ -56,6 +57,8 @@
 
 def get_meta_cache_dir():
     """Return storage location for cached meta data."""
+    if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]:
+        return os.path.join(os.environ[CACHE_PATH_ENV], META_CACHE_DIR)
     ros_home = rospkg.get_ros_home()
     return os.path.join(ros_home, 'rosdep', META_CACHE_DIR)
 

src/rosdep2/sources_list.py

@@ -37,7 +37,7 @@
 except ImportError:
     import pickle
 
-from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file
+from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file, CACHE_PATH_ENV
 from .core import InvalidData, DownloadFailure, CachePermissionError
 from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data
 from .meta import MetaDatabase
@@ -111,6 +111,8 @@ def get_default_sources_list_file():
 
 
 def get_sources_cache_dir():
+    if CACHE_PATH_ENV in os.environ and os.environ[CACHE_PATH_ENV]:
+        return os.path.join(os.environ[CACHE_PATH_ENV], SOURCES_CACHE_DIR)
     ros_home = rospkg.get_ros_home()
     return os.path.join(ros_home, 'rosdep', SOURCES_CACHE_DIR)
 
@@ -437,7 +439,8 @@ def _generate_key_from_urls(urls):
 def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
                         success_handler=None, error_handler=None,
                         skip_eol_distros=False, ros_distro=None,
-                        quiet=False):
+                        quiet=False,
+                        meta_cache_dir=None):
     """
     Re-downloaded data from remote sources and store in cache.  Also
     update the cache index based on current sources.
@@ -451,6 +454,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         if a particular source fails.  This hook is mainly for
         printing errors to console.
     :param skip_eol_distros: skip downloading sources for EOL distros
+    :param meta_cache_dir: override meta cache directory
 
     :returns: list of (`DataSource`, cache_file_path) pairs for cache
         files that were updated, ``[str]``
@@ -518,7 +522,7 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         sources.append(rds)
 
     # cache metadata that isn't a source list
-    MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)
+    MetaDatabase(meta_cache_dir).set('ROS_PYTHON_VERSION', python_versions)
 
     # Create a combined index of *all* the sources.  We do all the
     # sources regardless of failures because a cache from a previous

test/empty_cache/index

@@ -0,0 +1 @@
+#autogenerated by rosdep, do not edit. use 'rosdep update' instead

test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle

@@ -0,0 +1,4 @@
+�crosdep2.meta
+CacheWrapper
+q
)�q}q(Urosdep_versionqU0.22.1qU_CacheWrapper__dataq}q(UfoxyqKUmelodicq	KUhumbleq
+KUrollingqKUnoeticqKuUcategory_nameqUROS_PYTHON_VERSIONqub.

test/sources.list.d.good/20-default.list

@@ -0,0 +1,4 @@
+#autogenerated by rosdep, do not edit. use 'rosdep update' instead
+yaml https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml
+yaml https://github.com/ros/rosdistro/raw/master/rosdep/python.yaml
+gbpdistro https://github.com/ros/rosdistro/raw/master/releases/fuerte.yaml fuerte

test/test_rosdep_catkin_support.py

@@ -1,18 +1,50 @@
 from rosdep2.catkin_support import get_installer, get_catkin_view, ValidationFailed, resolve_for_os
 
 from rosdep2.platforms.debian import APT_INSTALLER
+from rosdep2.cache_tools import CACHE_PATH_ENV
+from rosdep2.sources_list import SOURCE_PATH_ENV
 import pytest
+import os
+from tempfile import mkdtemp
+
+
+def get_test_dir():
+    return os.path.abspath(os.path.dirname(__file__))
+
+
+def get_cache_dir():
+    # get_catkin_view calls update(), so we need a writable location
+    return mkdtemp()
+
+
+def get_source_list_dir():
+    p = os.path.join(get_test_dir(), "sources.list.d.good")
+    assert os.path.isdir(p)
+    return p
 
 
 @pytest.mark.online
 def test_workflow():
+    old_cpe = os.getenv(CACHE_PATH_ENV, None)
+    old_spe = os.getenv(SOURCE_PATH_ENV, None)
     try:
+        os.environ[CACHE_PATH_ENV] = get_cache_dir()
+        os.environ[SOURCE_PATH_ENV] = get_source_list_dir()
         installer = get_installer(APT_INSTALLER)
-        view = get_catkin_view('fuerte', 'ubuntu', 'lucid')
-        resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'lucid')
+        view = get_catkin_view('noetic', 'ubuntu', 'focal')
+        resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'focal')
         assert ['cmake'] == resolved
-        resolved = resolve_for_os('python', view, installer, 'ubuntu', 'lucid')
-        assert resolved == ['python-dev']
+        resolved = resolve_for_os('python3', view, installer, 'ubuntu', 'focal')
+        assert resolved == ['python3-dev']
     except ValidationFailed:
         # tests fail on the server because 'rosdep init' has not been run
         pass
+    finally:
+        if old_cpe is None:
+            del os.environ[CACHE_PATH_ENV]
+        else:
+            os.environ[CACHE_PATH_ENV] = old_cpe
+        if old_spe is None:
+            del os.environ[SOURCE_PATH_ENV]
+        else:
+            os.environ[SOURCE_PATH_ENV] = old_spe

test/test_rosdep_installers.py

@@ -47,7 +47,7 @@ def get_test_dir():
 
 
 def get_cache_dir():
-    p = os.path.join(get_test_dir(), 'sources_cache')
+    p = os.path.join(get_test_dir(), 'sources.cache')
     assert os.path.isdir(p)
     return p
 

test/test_rosdep_lookup.py

@@ -45,7 +45,7 @@ def get_test_tree_dir():
 
 
 def get_cache_dir():
-    p = os.path.join(get_test_dir(), 'sources_cache')
+    p = os.path.join(get_test_dir(), 'sources.cache')
     assert os.path.isdir(p)
     return p