[deploy-agent] Return type hints in download folder (#1403)

Add type hints

deploy-agent/deployd/download/download_helper.py

@@ -22,19 +22,19 @@
 
 class DownloadHelper(metaclass=abc.ABCMeta):
 
-    def __init__(self, url):
+    def __init__(self, url) -> None:
         self._url = url
 
     @staticmethod
-    def hash_file(file_path):
+    def hash_file(file_path) -> str:
         sha = hashlib.sha1()
         with open(file_path, "rb") as f:
             for chunk in iter(lambda: f.read(4096), b""):
                 sha.update(chunk)
         return sha.hexdigest()
 
     @staticmethod
-    def md5_file(file_path):
+    def md5_file(file_path) -> str:
         md5 = hashlib.md5()
         with open(file_path, "rb") as f:
             for chunk in iter(lambda: f.read(4096), b""):

deploy-agent/deployd/download/download_helper_factory.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import logging
+from typing import Optional
 
 from future.moves.urllib.parse import urlparse
 from boto.s3.connection import S3Connection
@@ -28,7 +29,7 @@
 class DownloadHelperFactory(object):
 
     @staticmethod
-    def gen_downloader(url, config):
+    def gen_downloader(url, config) -> Optional[S3DownloadHelper|LocalDownloadHelper|HTTPDownloadHelper]:
         url_parse = urlparse(url)
         if url_parse.scheme == 's3':
             aws_access_key_id = config.get_aws_access_key()

deploy-agent/deployd/download/downloader.py

@@ -31,23 +31,23 @@
 
 class Downloader(object):
 
-    def __init__(self, config, build, url, env_name):
+    def __init__(self, config, build, url, env_name) -> None:
         self._matcher = re.compile(r'^.*?[.](?P<ext>tar\.gz|tar\.bz2|\w+)$')
         self._base_dir = config.get_builds_directory()
         self._build_name = env_name
         self._build = build
         self._url = url
         self._config = config
 
-    def _get_inner_extension(self, url):
+    def _get_inner_extension(self, url) -> str:
         outerExtension = self._get_extension(url)
         inverseExtLen = (len(outerExtension) * -1) - 1
         return self._get_extension(url[:inverseExtLen])
 
-    def _get_extension(self, url):
+    def _get_extension(self, url) -> str:
         return self._matcher.match(url).group('ext')
 
-    def download(self):
+    def download(self) -> int:
         extension = self._get_extension(self._url.lower())
         local_fn = u'{}-{}.{}'.format(self._build_name, self._build, extension)
         local_full_fn = os.path.join(self._base_dir, local_fn)

deploy-agent/deployd/download/gpg_helper.py

@@ -22,7 +22,7 @@
 class gpgHelper(object):
 
     @staticmethod
-    def decryptFile(source, destination):
+    def decryptFile(source, destination) -> int:
         download_cmd = ['gpg', '--batch', '--yes', '--output', destination, '--decrypt', source]
         log.info('Running command: {}'.format(' '.join(download_cmd)))
         error_code = Status.SUCCEEDED

deploy-agent/deployd/download/http_download_helper.py

@@ -29,11 +29,11 @@
 
 class HTTPDownloadHelper(DownloadHelper):
 
-    def __init__(self, url=None, config=None):
+    def __init__(self, url=None, config=None) -> None:
         super().__init__(url)
         self._config = config if config else Config()
 
-    def _download_files(self, local_full_fn):
+    def _download_files(self, local_full_fn) -> int:
         download_cmd = ['curl', '-o', local_full_fn, '-fksS', self._url]
         log.info('Running command: {}'.format(' '.join(download_cmd)))
         output, error, process_return_code = Caller.call_and_log(download_cmd, cwd=os.getcwd())
@@ -45,7 +45,7 @@ def _download_files(self, local_full_fn):
         log.info('Finish downloading: {} to {}'.format(self._url, local_full_fn))
         return status_code
 
-    def download(self, local_full_fn):
+    def download(self, local_full_fn) -> int:
         log.info("Start to download from url {} to {}".format(
             self._url, local_full_fn))
         if not self.validate_source():
@@ -76,7 +76,7 @@ def download(self, local_full_fn):
             log.error('Could not connect to: {}'.format(self._url))
             return Status.FAILED
 
-    def validate_source(self):
+    def validate_source(self) -> bool:
         tags = {'type': 'http', 'url': self._url}
         create_sc_increment(DOWNLOAD_VALIDATE_METRICS, tags=tags)
 

deploy-agent/deployd/download/local_download_helper.py

@@ -23,7 +23,7 @@
 
 class LocalDownloadHelper(DownloadHelper):
 
-    def _download_files(self, local_full_fn):
+    def _download_files(self, local_full_fn) -> int:
         download_cmd = ['curl', '-o', local_full_fn, '-ks', self._url]
         log.info('Running command: {}'.format(' '.join(download_cmd)))
         error_code = Status.SUCCEEDED
@@ -37,7 +37,7 @@ def _download_files(self, local_full_fn):
         log.info('Finished downloading: {} to {}'.format(self._url, local_full_fn))
         return error_code
 
-    def download(self, local_full_fn):
+    def download(self, local_full_fn) -> int:
         log.info("Start to download from local path {} to {}".format(
             self._url, local_full_fn))
 
@@ -46,5 +46,5 @@ def download(self, local_full_fn):
             log.error('Failed to download the local tar ball for {}'.format(local_full_fn))
         return error
 
-    def validate_source(self):
-        return True
+    def validate_source(self) -> bool:
+        return True

deploy-agent/deployd/download/s3_download_helper.py

@@ -27,7 +27,7 @@
 
 class S3DownloadHelper(DownloadHelper):
 
-    def __init__(self, local_full_fn, aws_connection=None, url=None):
+    def __init__(self, local_full_fn, aws_connection=None, url=None) -> None:
         super(S3DownloadHelper, self).__init__(local_full_fn)
         self._s3_matcher = "^s3://(?P<BUCKET>[a-zA-Z0-9\-_]+)/(?P<KEY>[a-zA-Z0-9\-_/\.]+)/?"
         self._config = Config()
@@ -45,7 +45,7 @@ def __init__(self, local_full_fn, aws_connection=None, url=None):
             self._key = s3url_parse.group("KEY")
 
 
-    def download(self, local_full_fn):
+    def download(self, local_full_fn) -> int:
         log.info(f"Start to download file {self._key} from s3 bucket {self._bucket_name} to {local_full_fn}")
         if not self.validate_source():
             log.error(f'Invalid url: {self._url}. Skip downloading.')
@@ -76,9 +76,9 @@ def download(self, local_full_fn):
             log.error("Failed to get package from s3: {}".format(traceback.format_exc()))
             return Status.FAILED
 
-    def validate_source(self):
+    def validate_source(self) -> bool:
         allow_list = self._config.get_s3_download_allow_list()
         tags = {'type': 's3', 'url': self._url, 'bucket' : self._bucket_name}
         create_sc_increment(DOWNLOAD_VALIDATE_METRICS, tags=tags)
 
-        return self._bucket_name in allow_list if allow_list else True
+        return self._bucket_name in allow_list if allow_list else True