Resolve the pesky "website may be down" errors during rosdep {init, update}

.travis.yml

@@ -9,7 +9,7 @@ install:
   - source .travis/install.sh
   - python --version
   - if [ $TRAVIS_PYTHON_VERSION == "3.4" ]; then python -m pip install PyYAML==5.2; fi # Forcing PyYAML 5.2 while we retain Python 3.4 support PyYAML 5.3 and higher does not support Python 3.4
-  - python -m pip install $PIP_USER_FLAG PyYAML argparse rospkg vcstools catkin_pkg python-dateutil rosdistro
+  - python -m pip install $PIP_USER_FLAG PyYAML argparse certifi rospkg vcstools catkin_pkg python-dateutil rosdistro
   - python -m pip install $PIP_USER_FLAG -e .
   - python -m pip install $PIP_USER_FLAG nose coverage flake8 mock codecov
 # command to run tests

requirements.txt

@@ -0,0 +1,16 @@
+catkin-pkg==0.4.23
+certifi==2020.12.5
+distro==1.5.0
+docutils==0.17.1
+flake8==3.9.1
+mccabe==0.6.1
+mock==4.0.3
+nose==1.3.7
+pycodestyle==2.7.0
+pyflakes==2.3.1
+pyparsing==2.4.7
+python-dateutil==2.8.1
+PyYAML==5.4.1
+rosdistro==0.8.3
+rospkg==1.3.0
+six==1.16.0

src/rosdep2/gbpdistro_support.py

@@ -7,6 +7,9 @@
     import urlparse
 except ImportError:
     import urllib.parse as urlparse  # py3k
+import ssl
+import certifi
+
 import os
 
 from rospkg.os_detect import OS_DEBIAN
@@ -201,7 +204,8 @@ def download_gbpdistro_as_rosdep_data(gbpdistro_url, targets_url=None):
     # will output a warning
     targets_data = download_targets_data(targets_url=targets_url)
     try:
-        f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT)
+        f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT,
+                    context=ssl.create_default_context(cafile=certifi.where()))
         text = f.read()
         f.close()
         gbpdistro_data = yaml.safe_load(text)

src/rosdep2/main.py

@@ -580,7 +580,7 @@ def command_init(options):
     try:
         data = download_default_sources_list()
     except URLError as e:
-        print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL))
+        print('ERROR: cannot download default sources list from:\n%s\nReason: %s' % (DEFAULT_SOURCES_LIST_URL, e.reason))
         return 4
     except DownloadFailure as e:
         print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL))

src/rosdep2/rep3.py

@@ -29,6 +29,11 @@
     from urllib.request import urlopen
 except ImportError:
     from urllib2 import urlopen
+
+# to fix errors with SSL certificate verification
+import ssl
+import certifi
+
 import yaml
 import warnings
 
@@ -59,7 +64,8 @@ def download_targets_data(targets_url=None):
     if targets_url is None:
         targets_url = REP3_TARGETS_URL
     try:
-        f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT)
+        f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT,
+                    context=ssl.create_default_context(cafile=certifi.where()))
         text = f.read()
         f.close()
         targets_data = yaml.safe_load(text)

src/rosdep2/sources_list.py

@@ -45,6 +45,10 @@
 except ImportError:
     import pickle
 
+# to fix errors with SSL certificate verification
+import ssl
+import certifi
+
 from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file
 from .core import InvalidData, DownloadFailure, CachePermissionError
 from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data
@@ -312,7 +316,8 @@ def download_rosdep_data(url):
             url_request = request.Request(url, headers={'User-Agent': 'rosdep/{version}'.format(version=__version__)})
         else:
             url_request = url
-        f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT)
+        f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT,
+                    context=ssl.create_default_context(cafile=certifi.where()))
         text = f.read()
         f.close()
         data = yaml.safe_load(text)
@@ -337,7 +342,8 @@ def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL):
         retrieved (e.g. 404, server down).
     """
     try:
-        f = urlopen(url, timeout=DOWNLOAD_TIMEOUT)
+        f = urlopen(url, timeout=DOWNLOAD_TIMEOUT,
+                    context=ssl.create_default_context(cafile=certifi.where()))
     except (URLError, httplib.HTTPException) as e:
         raise URLError(str(e) + ' (%s)' % url)
     data = f.read().decode()
@@ -474,21 +480,25 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
 
     sources = parse_sources_list(sources_list_dir=sources_list_dir)
     retval = []
+
+    def get_source(source):
+        if source.type == TYPE_YAML:
+            rosdep_data = download_rosdep_data(source.url)
+        elif source.type == TYPE_GBPDISTRO:  # DEPRECATED, do not use this file. See REP137
+            if not source.tags[0] in ['electric', 'fuerte']:
+                print('Ignore legacy gbpdistro "%s"' % source.tags[0])
+                sources.remove(source)
+                return  # do not store this entry in the cache
+            rosdep_data = download_gbpdistro_as_rosdep_data(source.url)
+        retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data)))
+        if success_handler is not None:
+            success_handler(source)
+
     for source in list(sources):
         try:
-            if source.type == TYPE_YAML:
-                rosdep_data = download_rosdep_data(source.url)
-            elif source.type == TYPE_GBPDISTRO:  # DEPRECATED, do not use this file. See REP137
-                if not source.tags[0] in ['electric', 'fuerte']:
-                    print('Ignore legacy gbpdistro "%s"' % source.tags[0])
-                    sources.remove(source)
-                    continue  # do not store this entry in the cache
-                rosdep_data = download_gbpdistro_as_rosdep_data(source.url)
-            retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data)))
-            if success_handler is not None:
-                success_handler(source)
-        except DownloadFailure as e:
-            if error_handler is not None:
+            get_source(source)
+        except (DownloadFailure, URLError) as e:
+            if error_handler:
                 error_handler(source, e)
 
     # Additional sources for ros distros
@@ -501,17 +511,16 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         raise ValueError(
             'Requested distribution "%s" is not in the index.' % ros_distro)
 
-    for dist_name in sorted(distribution_names):
+    def get_additional_sources(dist_name):
         distribution = get_index().distributions[dist_name]
         if dist_name != ros_distro:
             if ros_distro is not None:
                 print('Skip distro "%s" different from requested "%s"' % (dist_name, ros_distro))
-                continue
+                return
             if skip_eol_distros:
                 if distribution.get('distribution_status') == 'end-of-life':
                     print('Skip end-of-life distro "%s"' % dist_name)
-                    continue
-        print('Add distro "%s"' % dist_name)
+                    return
         rds = RosDistroSource(dist_name)
         rosdep_data = get_gbprepo_as_rosdep_data(dist_name)
         # Store Python version from REP153
@@ -523,6 +532,10 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None,
         retval.append((rds, write_cache_file(sources_cache_dir, key, rosdep_data)))
         sources.append(rds)
 
+    for dist_name in sorted(distribution_names):
+        print('Add distro "%s"' % dist_name)
+        get_additional_sources(dist_name)
+
     # cache metadata that isn't a source list
     MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)