Added rosdep init-mirror and rosdep update-mirror instructions.

src/rosdep2/main.py

@@ -66,9 +66,9 @@
 from .meta import MetaDatabase
 from .rospkg_loader import DEFAULT_VIEW_KEY
 from .sources_list import update_sources_list, get_sources_cache_dir,\
-    download_default_sources_list, SourcesListLoader, CACHE_INDEX,\
-    get_sources_list_dir, get_default_sources_list_file,\
-    DEFAULT_SOURCES_LIST_URL
+    download_default_sources_list,download_mirror_sources_list, SourcesListLoader, CACHE_INDEX,\
+    get_sources_list_dir, get_default_sources_list_file, get_mirror_sources_list_file,\
+    DEFAULT_SOURCES_LIST_URL, MIRROR_SOURCES_LIST_URL
 from .rosdistrohelper import PreRep137Warning
 
 from .ament_packages import AMENT_PREFIX_PATH_ENV_VAR
@@ -625,6 +625,39 @@ def command_init(options):
     finally:
         os.umask(old_umask)
 
+def command_init_mirror(options):
+    try:
+        data = download_mirror_sources_list()
+    except URLError as e:
+        print('ERROR: cannot download mirror sources list from:\n%s\nWebsite may be down.' % (MIRROR_SOURCES_LIST_URL))
+        return 4
+    except DownloadFailure as e:
+        print('ERROR: cannot download mirror sources list from:\n%s\nWebsite may be down.' % (MIRROR_SOURCES_LIST_URL))
+        print(e)
+        return 4
+    # reuse path variable for error message
+    path = get_sources_list_dir()
+    old_umask = os.umask(0o022)
+    try:
+        if not os.path.exists(path):
+            os.makedirs(path)
+        path = get_mirror_sources_list_file()
+        if os.path.exists(path):
+            print('ERROR: mirror sources list file already exists:\n\t%s\nPlease delete if you wish to re-initialize' % (path))
+            return 1
+        with open(path, 'w') as f:
+            f.write(data)
+        print('Wrote %s' % (path))
+        print('Recommended: please run\n\n\trosdep update\n')
+    except IOError as e:
+        print('ERROR: cannot create %s:\n\t%s' % (path, e), file=sys.stderr)
+        return 2
+    except OSError as e:
+        print("ERROR: cannot create %s:\n\t%s\nPerhaps you need to run 'sudo rosdep init' instead" % (path, e), file=sys.stderr)
+        return 3
+    finally:
+        os.umask(old_umask)
+
 
 def command_update(options):
     error_occured = []
@@ -680,6 +713,60 @@ def update_error_handler(data_source, exc):
         print(']]]')
         return 1
 
+def command_update_mirror(options):
+    error_occured = []
+
+    def update_success_handler(data_source):
+        print('Hit %s' % (data_source.url))
+
+    def update_error_handler(data_source, exc):
+        error_string = 'ERROR: unable to process source [%s]:\n\t%s' % (data_source.url, exc)
+        print(error_string, file=sys.stderr)
+        error_occured.append(error_string)
+    sources_list_dir = get_sources_list_dir()
+
+    # disable deprecation warnings when using the command-line tool
+    warnings.filterwarnings('ignore', category=PreRep137Warning)
+
+    if not os.path.exists(sources_list_dir):
+        print('ERROR: no sources directory exists on the system meaning rosdep has not yet been initialized.\n\nPlease initialize your rosdep with\n\n\tsudo rosdep init\n')
+        return 1
+
+    filelist = [f for f in os.listdir(sources_list_dir) if f.endswith('.list')]
+    if not filelist:
+        print('ERROR: no data sources in %s\n\nPlease initialize your rosdep with\n\n\tsudo rosdep init\n' % sources_list_dir, file=sys.stderr)
+        return 1
+    try:
+        print('reading in sources list data from %s' % (sources_list_dir))
+        sources_cache_dir = get_sources_cache_dir()
+        try:
+            if os.geteuid() == 0:
+                print("Warning: running 'rosdep update' as root is not recommended.", file=sys.stderr)
+                print("  You should run 'sudo rosdep fix-permissions' and invoke 'rosdep update' again without sudo.", file=sys.stderr)
+        except AttributeError:
+            # nothing we wanna do under Windows
+            pass
+        update_sources_list(success_handler=update_success_handler,
+                            error_handler=update_error_handler,
+                            skip_eol_distros=not options.include_eol_distros,
+                            ros_distro=options.ros_distro)
+        print('updated cache in %s' % (sources_cache_dir))
+    except InvalidData as e:
+        print('ERROR: invalid sources list file:\n\t%s' % (e), file=sys.stderr)
+        return 1
+    except IOError as e:
+        print('ERROR: error loading sources list:\n\t%s' % (e), file=sys.stderr)
+        return 1
+    except ValueError as e:
+        print('ERROR: invalid argument value provided:\n\t%s' % (e), file=sys.stderr)
+        return 1
+    if error_occured:
+        print('ERROR: Not all sources were able to be updated.\n[[[')
+        for e in error_occured:
+            print(e)
+        print(']]]')
+        return 1
+
 
 def command_keys(lookup, packages, options):
     lookup = _get_default_RosdepLookup(options)
@@ -954,7 +1041,9 @@ def command_fix_permissions(options):
     'where-defined': command_where_defined,
     'resolve': command_resolve,
     'init': command_init,
+    'init-mirror': command_init_mirror,
     'update': command_update,
+    'update-mirror':command_update_mirror,
     'fix-permissions': command_fix_permissions,
 
     # backwards compat

src/rosdep2/sources_list.py

@@ -70,6 +70,7 @@
 # default file to download with 'init' command in order to bootstrap
 # rosdep
 DEFAULT_SOURCES_LIST_URL = 'https://raw.githubusercontent.com/ros/rosdistro/master/rosdep/sources.list.d/20-default.list'
+MIRROR_SOURCES_LIST_URL  = 'https://mirrors.tuna.tsinghua.edu.cn/github-raw/ros/rosdistro/master/rosdep/sources.list.d/20-default.list'
 
 # seconds to wait before aborting download of rosdep data
 DOWNLOAD_TIMEOUT = 15.0
@@ -115,6 +116,9 @@ def get_sources_list_dir():
 def get_default_sources_list_file():
     return os.path.join(get_sources_list_dir(), '20-default.list')
 
+def get_mirror_sources_list_file():
+    return os.path.join(get_sources_list_dir(), '20-default.list')
+
 
 def get_sources_cache_dir():
     ros_home = rospkg.get_ros_home()
@@ -354,6 +358,34 @@ def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL):
             ' The contents were:{{{%s}}} The error raised was: %s' % (url, data, e))
     return data
 
+def download_mirror_sources_list(url=MIRROR_SOURCES_LIST_URL):
+    """
+    Download (and validate) contents of mirror sources list.
+
+    :param url: override URL of mirror sources list file
+    :return: raw sources list data, ``str``
+    :raises: :exc:`DownloadFailure` If data cannot be
+            retrieved (e.g. 404, bad YAML format, server down).
+    :raises: :exc:`urllib2.URLError` If data cannot be
+        retrieved (e.g. 404, server down).
+    """
+    try:
+        f = urlopen(url, timeout=DOWNLOAD_TIMEOUT)
+    except (URLError, httplib.HTTPException) as e:
+        raise URLError(str(e) + ' (%s)' % url)
+    data = f.read().decode()
+    f.close()
+    if not data:
+        raise DownloadFailure('cannot download mirror file from %s : empty contents' % url)
+    # parse just for validation
+    try:
+        parse_sources_data(data)
+    except InvalidData as e:
+        raise DownloadFailure(
+            'The content downloaded from %s failed to pass validation.'
+            ' It is likely that the source is invalid unless the data was corrupted during the download.'
+            ' The contents were:{{{%s}}} The error raised was: %s' % (url, data, e))
+    return data
 
 def parse_sources_data(data, origin='<string>', model=None):
     """