From a298553593dfc4339fb64b593e6fd8287fa12a4d Mon Sep 17 00:00:00 2001 From: Kunal Tyagi Date: Thu, 6 May 2021 17:10:11 +0900 Subject: [PATCH] Add context to urlopen for SSL verification errors --- requirements.txt | 4 +++ src/rosdep2/gbpdistro_support.py | 6 +++- src/rosdep2/main.py | 2 +- src/rosdep2/rep3.py | 8 ++++- src/rosdep2/sources_list.py | 55 ++++++++++++++++++++------------ 5 files changed, 51 insertions(+), 24 deletions(-) diff --git a/requirements.txt b/requirements.txt index 363581769..1286ae7eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,12 @@ catkin-pkg==0.4.23 certifi==2020.12.5 distro==1.5.0 docutils==0.17.1 +flake8==3.9.1 +mccabe==0.6.1 mock==4.0.3 nose==1.3.7 +pycodestyle==2.7.0 +pyflakes==2.3.1 pyparsing==2.4.7 python-dateutil==2.8.1 PyYAML==5.4.1 diff --git a/src/rosdep2/gbpdistro_support.py b/src/rosdep2/gbpdistro_support.py index fd90ae096..769186d14 100644 --- a/src/rosdep2/gbpdistro_support.py +++ b/src/rosdep2/gbpdistro_support.py @@ -7,6 +7,9 @@ import urlparse except ImportError: import urllib.parse as urlparse # py3k +import ssl +import certifi + import os from rospkg.os_detect import OS_DEBIAN @@ -201,7 +204,8 @@ def download_gbpdistro_as_rosdep_data(gbpdistro_url, targets_url=None): # will output a warning targets_data = download_targets_data(targets_url=targets_url) try: - f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() gbpdistro_data = yaml.safe_load(text) diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 268fcbdc1..996ed1e50 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -580,7 +580,7 @@ def command_init(options): try: data = download_default_sources_list() except URLError as e: - print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL)) + print('ERROR: cannot download default sources list from:\n%s\nReason: %s' % (DEFAULT_SOURCES_LIST_URL, e.reason)) return 4 except DownloadFailure as e: print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL)) diff --git a/src/rosdep2/rep3.py b/src/rosdep2/rep3.py index 71b2f56f4..f8aad924e 100644 --- a/src/rosdep2/rep3.py +++ b/src/rosdep2/rep3.py @@ -29,6 +29,11 @@ from urllib.request import urlopen except ImportError: from urllib2 import urlopen + +# to fix errors with SSL certificate verification +import ssl +import certifi + import yaml import warnings @@ -59,7 +64,8 @@ def download_targets_data(targets_url=None): if targets_url is None: targets_url = REP3_TARGETS_URL try: - f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() targets_data = yaml.safe_load(text) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index bfec3a1d0..bf1fbcf00 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -45,6 +45,10 @@ except ImportError: import pickle +# to fix errors with SSL certificate verification +import ssl +import certifi + from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file from .core import InvalidData, DownloadFailure, CachePermissionError from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data @@ -312,7 +316,8 @@ def download_rosdep_data(url): url_request = request.Request(url, headers={'User-Agent': 'rosdep/{version}'.format(version=__version__)}) else: url_request = url - f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() data = yaml.safe_load(text) @@ -324,7 +329,6 @@ def download_rosdep_data(url): except yaml.YAMLError as e: raise DownloadFailure(str(e)) - def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): """ Download (and validate) contents of default sources list. @@ -337,7 +341,8 @@ def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): retrieved (e.g. 404, server down). """ try: - f = urlopen(url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) except (URLError, httplib.HTTPException) as e: raise URLError(str(e) + ' (%s)' % url) data = f.read().decode() @@ -474,22 +479,27 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, sources = parse_sources_list(sources_list_dir=sources_list_dir) retval = [] + + def get_source(source): + if source.type == TYPE_YAML: + rosdep_data = download_rosdep_data(source.url) + elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 + if not source.tags[0] in ['electric', 'fuerte']: + print('Ignore legacy gbpdistro "%s"' % source.tags[0]) + sources.remove(source) + return # do not store this entry in the cache + rosdep_data = download_gbpdistro_as_rosdep_data(source.url) + retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) + if success_handler is not None: + success_handler(source) + for source in list(sources): try: - if source.type == TYPE_YAML: - rosdep_data = download_rosdep_data(source.url) - elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 - if not source.tags[0] in ['electric', 'fuerte']: - print('Ignore legacy gbpdistro "%s"' % source.tags[0]) - sources.remove(source) - continue # do not store this entry in the cache - rosdep_data = download_gbpdistro_as_rosdep_data(source.url) - retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) - if success_handler is not None: - success_handler(source) - except DownloadFailure as e: - if error_handler is not None: - error_handler(source, e) + get_source(source) + except (DownloadFailure, URLError) as e: + raise error_handler(source, e) + except IOError as e: + raise error_handler("IO: " + source, e) # Additional sources for ros distros # In compliance with REP137 and REP143 @@ -501,17 +511,16 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, raise ValueError( 'Requested distribution "%s" is not in the index.' % ros_distro) - for dist_name in sorted(distribution_names): + def get_additional_sources(dist_name): distribution = get_index().distributions[dist_name] if dist_name != ros_distro: if ros_distro is not None: print('Skip distro "%s" different from requested "%s"' % (dist_name, ros_distro)) - continue + return if skip_eol_distros: if distribution.get('distribution_status') == 'end-of-life': print('Skip end-of-life distro "%s"' % dist_name) - continue - print('Add distro "%s"' % dist_name) + return rds = RosDistroSource(dist_name) rosdep_data = get_gbprepo_as_rosdep_data(dist_name) # Store Python version from REP153 @@ -523,6 +532,10 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, retval.append((rds, write_cache_file(sources_cache_dir, key, rosdep_data))) sources.append(rds) + for dist_name in sorted(distribution_names): + print('Add distro "%s"' % dist_name) + get_additional_sources(dist_name) + # cache metadata that isn't a source list MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)