Deprecate -c and -m command line options and use `ROSDEP_CACHE_PA…

…TH` instead.

Signed-off-by: Martin Pecka <[email protected]>

src/rosdep2/main.py

@@ -53,7 +53,7 @@
     from urllib2 import URLError
 import warnings
 
-from optparse import OptionParser
+from optparse import OptionParser, SUPPRESS_HELP
 
 import rospkg
 
@@ -122,6 +122,16 @@ class UsageError(Exception):
   Recursively change the permissions of the user's ros home directory.
   May require sudo.  Can be useful to fix permissions after calling
   "rosdep update" with sudo accidentally.
+
+Environment variables:
+
+ROSDEP_SOURCE_PATH
+  Overrides path to the sources list directory (by default /etc/ros/rosdep/sources.list.d).
+  Applies to init and update commands.
+
+ROSDEP_CACHE_PATH
+  Overrides path to the cache directory (by default $HOME/.ros/rosdep).
+  Applies to all commands except init.
 """
 
 
@@ -307,10 +317,10 @@ def _rosdep_main(args):
     parser = OptionParser(usage=_usage, prog='rosdep')
     parser.add_option('--os', dest='os_override', default=None,
                       metavar='OS_NAME:OS_VERSION', help='Override OS name and version (colon-separated), e.g. ubuntu:lucid')
-    parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=default_sources_cache,
-                      metavar='SOURCES_CACHE_DIR', help='Override %s' % (default_sources_cache))
-    parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=default_meta_cache,
-                      metavar='META_CACHE_DIR', help='Override %s' % (default_meta_cache))
+    parser.add_option('-c', '--sources-cache-dir', dest='sources_cache_dir', default=None,
+                      metavar='SOURCES_CACHE_DIR', help=SUPPRESS_HELP)  # deprecated
+    parser.add_option('-m', '--meta-cache-dir', dest='meta_cache_dir', default=None,
+                      metavar='META_CACHE_DIR', help=SUPPRESS_HELP)  # deprecated
     parser.add_option('--verbose', '-v', dest='verbose', default=False,
                       action='store_true', help='verbose display')
     parser.add_option('--version', dest='print_version', default=False,
@@ -657,22 +667,22 @@ def update_error_handler(data_source, exc):
     try:
         if not options.quiet:
             print('reading in sources list data from %s' % (sources_list_dir))
-        sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir()
         try:
             if os.geteuid() == 0:
                 print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr)
                 print("  You should run 'sudo rosdep fix-permissions' and invoke 'rosdep update' again without sudo.", file=sys.stderr)
         except AttributeError:
             # nothing we wanna do under Windows
             pass
-        update_sources_list(sources_cache_dir=sources_cache_dir,
+        update_sources_list(sources_cache_dir=options.sources_cache_dir,
                             success_handler=update_success_handler,
                             error_handler=update_error_handler,
                             skip_eol_distros=not options.include_eol_distros,
                             ros_distro=options.ros_distro,
                             quiet=options.quiet,
                             meta_cache_dir=options.meta_cache_dir)
         if not options.quiet:
+            sources_cache_dir = options.sources_cache_dir if options.sources_cache_dir else get_sources_cache_dir()
             print('updated cache in %s' % (sources_cache_dir))
     except InvalidData as e:
         print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr)

test/meta.cache/5c57306d8ea9d8eec9db440c6e9937664e0159b5.pickle

@@ -0,0 +1,4 @@
+�crosdep2.meta
+CacheWrapper
+q
)�q}q(Urosdep_versionqU0.22.1qU_CacheWrapper__dataq}q(UfoxyqKUmelodicq	KUhumbleq
+KUrollingqKUnoeticqKuUcategory_nameqUROS_PYTHON_VERSIONqub.

test/sources.list.d.good/20-default.list

@@ -0,0 +1,4 @@
+#autogenerated by rosdep, do not edit. use 'rosdep update' instead
+yaml https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml
+yaml https://github.com/ros/rosdistro/raw/master/rosdep/python.yaml
+gbpdistro https://github.com/ros/rosdistro/raw/master/releases/fuerte.yaml fuerte

test/test_rosdep_catkin_support.py

@@ -1,12 +1,35 @@
 from rosdep2.catkin_support import get_installer, get_catkin_view, ValidationFailed, resolve_for_os
 
 from rosdep2.platforms.debian import APT_INSTALLER
+from rosdep2.cache_tools import CACHE_PATH_ENV
+from rosdep2.sources_list import SOURCE_PATH_ENV
 import pytest
+import os
+from tempfile import mkdtemp
+
+
+def get_test_dir():
+    return os.path.abspath(os.path.dirname(__file__))
+
+
+def get_cache_dir():
+    # get_catkin_view calls update(), so we need a writable location
+    return mkdtemp()
+
+
+def get_source_list_dir():
+    p = os.path.join(get_test_dir(), "sources.list.d.good")
+    assert os.path.isdir(p)
+    return p
 
 
 @pytest.mark.online
 def test_workflow():
+    old_cpe = os.getenv(CACHE_PATH_ENV, None)
+    old_spe = os.getenv(SOURCE_PATH_ENV, None)
     try:
+        os.environ[CACHE_PATH_ENV] = get_cache_dir()
+        os.environ[SOURCE_PATH_ENV] = get_source_list_dir()
         installer = get_installer(APT_INSTALLER)
         view = get_catkin_view('fuerte', 'ubuntu', 'lucid')
         resolved = resolve_for_os('cmake', view, installer, 'ubuntu', 'lucid')
@@ -16,3 +39,12 @@ def test_workflow():
     except ValidationFailed:
         # tests fail on the server because 'rosdep init' has not been run
         pass
+    finally:
+        if old_cpe is None:
+            del os.environ[CACHE_PATH_ENV]
+        else:
+            os.environ[CACHE_PATH_ENV] = old_cpe
+        if old_spe is None:
+            del os.environ[SOURCE_PATH_ENV]
+        else:
+            os.environ[SOURCE_PATH_ENV] = old_spe

test/test_rosdep_installers.py

@@ -47,7 +47,7 @@ def get_test_dir():
 
 
 def get_cache_dir():
-    p = os.path.join(get_test_dir(), 'sources_cache')
+    p = os.path.join(get_test_dir(), 'sources.cache')
     assert os.path.isdir(p)
     return p
 

test/test_rosdep_lookup.py

@@ -45,7 +45,7 @@ def get_test_tree_dir():
 
 
 def get_cache_dir():
-    p = os.path.join(get_test_dir(), 'sources_cache')
+    p = os.path.join(get_test_dir(), 'sources.cache')
     assert os.path.isdir(p)
     return p
 

test/test_rosdep_main.py

@@ -47,6 +47,7 @@
 from rosdep2.ament_packages import AMENT_PREFIX_PATH_ENV_VAR
 from rosdep2.main import rosdep_main
 from rosdep2.main import setup_proxy_opener
+from rosdep2.cache_tools import CACHE_PATH_ENV
 
 
 GITHUB_BASE_URL = 'https://github.com/ros/rosdistro/raw/master/rosdep/base.yaml'
@@ -66,7 +67,7 @@ def get_test_catkin_tree_dir():
 
 
 def get_cache_dir():
-    p = os.path.join(get_test_dir(), 'sources_cache')
+    p = get_test_dir()
     assert os.path.isdir(p)
     return p
 
@@ -95,13 +96,15 @@ def setUp(self):
         self.old_rr = rospkg.get_ros_root()
         self.old_rpp = rospkg.get_ros_package_path()
         self.old_app = os.getenv(AMENT_PREFIX_PATH_ENV_VAR, None)
+        self.old_cpe = os.getenv(CACHE_PATH_ENV, None)
         if 'ROS_ROOT' in os.environ:
             del os.environ['ROS_ROOT']
         os.environ['ROS_PACKAGE_PATH'] = os.path.join(get_test_tree_dir())
         os.environ[AMENT_PREFIX_PATH_ENV_VAR] = os.path.join(get_test_tree_dir(), 'ament')
         if 'ROS_PYTHON_VERSION' not in os.environ:
             # avoid `test_check` failure due to warning on stderr
             os.environ['ROS_PYTHON_VERSION'] = sys.version[0]
+        os.environ[CACHE_PATH_ENV] = get_cache_dir()
 
     def tearDown(self):
         if self.old_rr is not None:
@@ -110,10 +113,13 @@ def tearDown(self):
             os.environ['ROS_PACKAGE_PATH'] = self.old_rpp
         if self.old_app is not None:
             os.environ[AMENT_PREFIX_PATH_ENV_VAR] = self.old_app
+        if self.old_cpe is None:
+            del os.environ[CACHE_PATH_ENV]
+        else:
+            os.environ[CACHE_PATH_ENV] = self.old_cpe
 
     def test_bad_commands(self):
-        sources_cache = get_cache_dir()
-        cmd_extras = ['-c', sources_cache]
+        cmd_extras = []
         for commands in [[], ['fake', 'something'], ['check'], ['install', '-a', 'rospack_fake'],
                          ['check', 'rospack_fake', '--os', 'ubuntulucid'],
                          ]:
@@ -124,8 +130,7 @@ def test_bad_commands(self):
                 pass
 
     def test_check(self):
-        sources_cache = get_cache_dir()
-        cmd_extras = ['-c', sources_cache]
+        cmd_extras = []
 
         with fakeout() as b:
             try:
@@ -163,8 +168,7 @@ def test_check(self):
     @patch('rosdep2.platforms.debian.read_stdout')
     @patch('rosdep2.installers.os.geteuid', return_value=1)
     def test_install(self, mock_geteuid, mock_read_stdout):
-        sources_cache = get_cache_dir()
-        cmd_extras = ['-c', sources_cache]
+        cmd_extras = []
         catkin_tree = get_test_catkin_tree_dir()
 
         def read_stdout(cmd, capture_stderr=False):
@@ -221,12 +225,11 @@ def read_stdout(cmd, capture_stderr=False):
 
     def test_where_defined(self):
         try:
-            sources_cache = get_cache_dir()
             expected = GITHUB_PYTHON_URL
             for command in (['where_defined', 'testpython'], ['where_defined', 'testpython']):
                 with fakeout() as b:
                     # set os to ubuntu so this test works on different platforms
-                    rosdep_main(command + ['-c', sources_cache, '--os=ubuntu:lucid'])
+                    rosdep_main(command + ['--os=ubuntu:lucid'])
                     stdout, stderr = b
                     output = stdout.getvalue().strip()
                     assert output == expected, output
@@ -235,8 +238,7 @@ def test_where_defined(self):
 
     def test_what_needs(self):
         try:
-            sources_cache = get_cache_dir()
-            cmd_extras = ['-c', sources_cache]
+            cmd_extras = []
             expected = ['python_dep']
             with fakeout() as b:
                 rosdep_main(['what-needs', 'testpython'] + cmd_extras)
@@ -253,8 +255,7 @@ def test_what_needs(self):
             assert False, 'system exit occurred'
 
     def test_keys(self):
-        sources_cache = get_cache_dir()
-        cmd_extras = ['-c', sources_cache]
+        cmd_extras = []
 
         try:
             with fakeout() as b:

test/test_rosdep_sources_list.py

@@ -38,6 +38,9 @@
 import pytest
 import rospkg.distro
 import rosdep2.sources_list
+from rosdep2.sources_list import SOURCES_CACHE_DIR
+from rosdep2.cache_tools import CACHE_PATH_ENV
+from rosdep2.meta import META_CACHE_DIR
 
 GITHUB_BASE_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/base.yaml'
 
@@ -499,6 +502,83 @@ def test_SourcesListLoader_create_default():
     loader.load_view(GITHUB_URL, rosdep_db)
 
 
+@pytest.mark.online
+def test_SourcesListLoader_cache_dir_from_env_var():
+    from rosdep2.sources_list import update_sources_list, SourcesListLoader, DataSourceMatcher
+    # create temp dir for holding sources cache
+    tempdir = tempfile.mkdtemp()
+    old_cpe = os.getenv(CACHE_PATH_ENV, None)
+    os.environ[CACHE_PATH_ENV] = tempdir
+
+    try:
+        # pull in cache data
+        sources_list_dir = get_test_dir()
+        retval = update_sources_list(sources_list_dir=sources_list_dir, error_handler=None)
+        assert retval
+        sources_cache_dir = os.path.join(tempdir, SOURCES_CACHE_DIR)
+        assert os.path.exists(sources_cache_dir)
+        assert len(os.listdir(sources_cache_dir)) > 0
+        meta_cache_dir = os.path.join(tempdir, META_CACHE_DIR)
+        assert os.path.exists(meta_cache_dir)
+        assert len(os.listdir(meta_cache_dir)) > 0
+
+        # now test with cached data
+        matcher = rosdep2.sources_list.DataSourceMatcher(['ubuntu', 'lucid'])
+        loader = SourcesListLoader.create_default(matcher)
+        assert loader.sources
+        assert not any([s for s in loader.sources if not matcher.matches(s)])
+
+        # test API
+
+        # very simple, always raises RNF
+        try:
+            loader.get_rosdeps('foo')
+        except rospkg.ResourceNotFound:
+            pass
+        try:
+            loader.get_view_key('foo')
+        except rospkg.ResourceNotFound:
+            pass
+
+        assert [] == loader.get_loadable_resources()
+        all_sources = [x.url for x in loader.sources]
+        assert all_sources == loader.get_loadable_views()
+
+        # test get_source early to make sure model matches expected
+        try:
+            loader.get_source('foo')
+            assert False, 'should have raised'
+        except rospkg.ResourceNotFound:
+            pass
+        s = loader.get_source(GITHUB_URL)
+        assert s.url == GITHUB_URL
+
+        # get_view_dependencies
+        # - loader doesn't new view name, so assume everything
+        assert all_sources == loader.get_view_dependencies('foo')
+        # - actual views don't depend on anything
+        assert [] == loader.get_view_dependencies(GITHUB_URL)
+
+        # load_view
+        from rosdep2.model import RosdepDatabase
+        for verbose in [True, False]:
+            rosdep_db = RosdepDatabase()
+            loader.load_view(GITHUB_URL, rosdep_db, verbose=verbose)
+            assert rosdep_db.is_loaded(GITHUB_URL)
+            assert [] == rosdep_db.get_view_dependencies(GITHUB_URL)
+            entry = rosdep_db.get_view_data(GITHUB_URL)
+            assert 'cmake' in entry.rosdep_data
+            assert GITHUB_URL == entry.origin
+
+        #  - coverage, repeat loader, should noop
+        loader.load_view(GITHUB_URL, rosdep_db)
+    finally:
+        if old_cpe is None:
+            del os.environ[CACHE_PATH_ENV]
+        else:
+            os.environ[CACHE_PATH_ENV] = old_cpe
+
+
 def test_unpickle_same_results():
     try:
         import cPickle as pickle