diff --git a/.travis.yml b/.travis.yml index 1f4c5722b..09a083cce 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ install: - source .travis/install.sh - python --version - if [ $TRAVIS_PYTHON_VERSION == "3.4" ]; then python -m pip install PyYAML==5.2; fi # Forcing PyYAML 5.2 while we retain Python 3.4 support PyYAML 5.3 and higher does not support Python 3.4 - - python -m pip install $PIP_USER_FLAG PyYAML argparse rospkg vcstools catkin_pkg python-dateutil rosdistro + - python -m pip install $PIP_USER_FLAG PyYAML argparse certifi rospkg vcstools catkin_pkg python-dateutil rosdistro - python -m pip install $PIP_USER_FLAG -e . - python -m pip install $PIP_USER_FLAG nose coverage flake8 mock codecov # command to run tests diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..1286ae7eb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +catkin-pkg==0.4.23 +certifi==2020.12.5 +distro==1.5.0 +docutils==0.17.1 +flake8==3.9.1 +mccabe==0.6.1 +mock==4.0.3 +nose==1.3.7 +pycodestyle==2.7.0 +pyflakes==2.3.1 +pyparsing==2.4.7 +python-dateutil==2.8.1 +PyYAML==5.4.1 +rosdistro==0.8.3 +rospkg==1.3.0 +six==1.16.0 diff --git a/src/rosdep2/gbpdistro_support.py b/src/rosdep2/gbpdistro_support.py index fd90ae096..769186d14 100644 --- a/src/rosdep2/gbpdistro_support.py +++ b/src/rosdep2/gbpdistro_support.py @@ -7,6 +7,9 @@ import urlparse except ImportError: import urllib.parse as urlparse # py3k +import ssl +import certifi + import os from rospkg.os_detect import OS_DEBIAN @@ -201,7 +204,8 @@ def download_gbpdistro_as_rosdep_data(gbpdistro_url, targets_url=None): # will output a warning targets_data = download_targets_data(targets_url=targets_url) try: - f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(gbpdistro_url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() gbpdistro_data = yaml.safe_load(text) diff --git a/src/rosdep2/main.py b/src/rosdep2/main.py index 268fcbdc1..996ed1e50 100644 --- a/src/rosdep2/main.py +++ b/src/rosdep2/main.py @@ -580,7 +580,7 @@ def command_init(options): try: data = download_default_sources_list() except URLError as e: - print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL)) + print('ERROR: cannot download default sources list from:\n%s\nReason: %s' % (DEFAULT_SOURCES_LIST_URL, e.reason)) return 4 except DownloadFailure as e: print('ERROR: cannot download default sources list from:\n%s\nWebsite may be down.' % (DEFAULT_SOURCES_LIST_URL)) diff --git a/src/rosdep2/rep3.py b/src/rosdep2/rep3.py index 71b2f56f4..f8aad924e 100644 --- a/src/rosdep2/rep3.py +++ b/src/rosdep2/rep3.py @@ -29,6 +29,11 @@ from urllib.request import urlopen except ImportError: from urllib2 import urlopen + +# to fix errors with SSL certificate verification +import ssl +import certifi + import yaml import warnings @@ -59,7 +64,8 @@ def download_targets_data(targets_url=None): if targets_url is None: targets_url = REP3_TARGETS_URL try: - f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(targets_url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() targets_data = yaml.safe_load(text) diff --git a/src/rosdep2/sources_list.py b/src/rosdep2/sources_list.py index bfec3a1d0..91959838c 100644 --- a/src/rosdep2/sources_list.py +++ b/src/rosdep2/sources_list.py @@ -45,6 +45,10 @@ except ImportError: import pickle +# to fix errors with SSL certificate verification +import ssl +import certifi + from .cache_tools import compute_filename_hash, PICKLE_CACHE_EXT, write_atomic, write_cache_file from .core import InvalidData, DownloadFailure, CachePermissionError from .gbpdistro_support import get_gbprepo_as_rosdep_data, download_gbpdistro_as_rosdep_data @@ -312,7 +316,8 @@ def download_rosdep_data(url): url_request = request.Request(url, headers={'User-Agent': 'rosdep/{version}'.format(version=__version__)}) else: url_request = url - f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(url_request, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) text = f.read() f.close() data = yaml.safe_load(text) @@ -337,7 +342,8 @@ def download_default_sources_list(url=DEFAULT_SOURCES_LIST_URL): retrieved (e.g. 404, server down). """ try: - f = urlopen(url, timeout=DOWNLOAD_TIMEOUT) + f = urlopen(url, timeout=DOWNLOAD_TIMEOUT, + context=ssl.create_default_context(cafile=certifi.where())) except (URLError, httplib.HTTPException) as e: raise URLError(str(e) + ' (%s)' % url) data = f.read().decode() @@ -474,21 +480,25 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, sources = parse_sources_list(sources_list_dir=sources_list_dir) retval = [] + + def get_source(source): + if source.type == TYPE_YAML: + rosdep_data = download_rosdep_data(source.url) + elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 + if not source.tags[0] in ['electric', 'fuerte']: + print('Ignore legacy gbpdistro "%s"' % source.tags[0]) + sources.remove(source) + return # do not store this entry in the cache + rosdep_data = download_gbpdistro_as_rosdep_data(source.url) + retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) + if success_handler is not None: + success_handler(source) + for source in list(sources): try: - if source.type == TYPE_YAML: - rosdep_data = download_rosdep_data(source.url) - elif source.type == TYPE_GBPDISTRO: # DEPRECATED, do not use this file. See REP137 - if not source.tags[0] in ['electric', 'fuerte']: - print('Ignore legacy gbpdistro "%s"' % source.tags[0]) - sources.remove(source) - continue # do not store this entry in the cache - rosdep_data = download_gbpdistro_as_rosdep_data(source.url) - retval.append((source, write_cache_file(sources_cache_dir, source.url, rosdep_data))) - if success_handler is not None: - success_handler(source) - except DownloadFailure as e: - if error_handler is not None: + get_source(source) + except (DownloadFailure, URLError) as e: + if error_handler: error_handler(source, e) # Additional sources for ros distros @@ -501,17 +511,16 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, raise ValueError( 'Requested distribution "%s" is not in the index.' % ros_distro) - for dist_name in sorted(distribution_names): + def get_additional_sources(dist_name): distribution = get_index().distributions[dist_name] if dist_name != ros_distro: if ros_distro is not None: print('Skip distro "%s" different from requested "%s"' % (dist_name, ros_distro)) - continue + return if skip_eol_distros: if distribution.get('distribution_status') == 'end-of-life': print('Skip end-of-life distro "%s"' % dist_name) - continue - print('Add distro "%s"' % dist_name) + return rds = RosDistroSource(dist_name) rosdep_data = get_gbprepo_as_rosdep_data(dist_name) # Store Python version from REP153 @@ -523,6 +532,10 @@ def update_sources_list(sources_list_dir=None, sources_cache_dir=None, retval.append((rds, write_cache_file(sources_cache_dir, key, rosdep_data))) sources.append(rds) + for dist_name in sorted(distribution_names): + print('Add distro "%s"' % dist_name) + get_additional_sources(dist_name) + # cache metadata that isn't a source list MetaDatabase().set('ROS_PYTHON_VERSION', python_versions)