From 0349e3b0f0d68f4c0cf1582fa98267d4dc2b87c9 Mon Sep 17 00:00:00 2001 From: Ian Roquebert <72234714+gzpcho@users.noreply.github.com> Date: Thu, 1 Feb 2024 00:28:13 -0600 Subject: [PATCH] [deploy-agent] Return type hints in download folder (#1403) Add type hints --- deploy-agent/deployd/download/download_helper.py | 6 +++--- deploy-agent/deployd/download/download_helper_factory.py | 3 ++- deploy-agent/deployd/download/downloader.py | 8 ++++---- deploy-agent/deployd/download/gpg_helper.py | 2 +- deploy-agent/deployd/download/http_download_helper.py | 8 ++++---- deploy-agent/deployd/download/local_download_helper.py | 8 ++++---- deploy-agent/deployd/download/s3_download_helper.py | 8 ++++---- 7 files changed, 22 insertions(+), 21 deletions(-) diff --git a/deploy-agent/deployd/download/download_helper.py b/deploy-agent/deployd/download/download_helper.py index 5d760ab0ed..823286ac28 100644 --- a/deploy-agent/deployd/download/download_helper.py +++ b/deploy-agent/deployd/download/download_helper.py @@ -22,11 +22,11 @@ class DownloadHelper(metaclass=abc.ABCMeta): - def __init__(self, url): + def __init__(self, url) -> None: self._url = url @staticmethod - def hash_file(file_path): + def hash_file(file_path) -> str: sha = hashlib.sha1() with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): @@ -34,7 +34,7 @@ def hash_file(file_path): return sha.hexdigest() @staticmethod - def md5_file(file_path): + def md5_file(file_path) -> str: md5 = hashlib.md5() with open(file_path, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): diff --git a/deploy-agent/deployd/download/download_helper_factory.py b/deploy-agent/deployd/download/download_helper_factory.py index d9b2c4ffc8..d483e76b7e 100644 --- a/deploy-agent/deployd/download/download_helper_factory.py +++ b/deploy-agent/deployd/download/download_helper_factory.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +from typing import Optional from future.moves.urllib.parse import urlparse from boto.s3.connection import S3Connection @@ -28,7 +29,7 @@ class DownloadHelperFactory(object): @staticmethod - def gen_downloader(url, config): + def gen_downloader(url, config) -> Optional[S3DownloadHelper|LocalDownloadHelper|HTTPDownloadHelper]: url_parse = urlparse(url) if url_parse.scheme == 's3': aws_access_key_id = config.get_aws_access_key() diff --git a/deploy-agent/deployd/download/downloader.py b/deploy-agent/deployd/download/downloader.py index 8e2b51514c..baf0dfa7da 100644 --- a/deploy-agent/deployd/download/downloader.py +++ b/deploy-agent/deployd/download/downloader.py @@ -31,7 +31,7 @@ class Downloader(object): - def __init__(self, config, build, url, env_name): + def __init__(self, config, build, url, env_name) -> None: self._matcher = re.compile(r'^.*?[.](?Ptar\.gz|tar\.bz2|\w+)$') self._base_dir = config.get_builds_directory() self._build_name = env_name @@ -39,15 +39,15 @@ def __init__(self, config, build, url, env_name): self._url = url self._config = config - def _get_inner_extension(self, url): + def _get_inner_extension(self, url) -> str: outerExtension = self._get_extension(url) inverseExtLen = (len(outerExtension) * -1) - 1 return self._get_extension(url[:inverseExtLen]) - def _get_extension(self, url): + def _get_extension(self, url) -> str: return self._matcher.match(url).group('ext') - def download(self): + def download(self) -> int: extension = self._get_extension(self._url.lower()) local_fn = u'{}-{}.{}'.format(self._build_name, self._build, extension) local_full_fn = os.path.join(self._base_dir, local_fn) diff --git a/deploy-agent/deployd/download/gpg_helper.py b/deploy-agent/deployd/download/gpg_helper.py index d680352908..1a2d02faa3 100644 --- a/deploy-agent/deployd/download/gpg_helper.py +++ b/deploy-agent/deployd/download/gpg_helper.py @@ -22,7 +22,7 @@ class gpgHelper(object): @staticmethod - def decryptFile(source, destination): + def decryptFile(source, destination) -> int: download_cmd = ['gpg', '--batch', '--yes', '--output', destination, '--decrypt', source] log.info('Running command: {}'.format(' '.join(download_cmd))) error_code = Status.SUCCEEDED diff --git a/deploy-agent/deployd/download/http_download_helper.py b/deploy-agent/deployd/download/http_download_helper.py index 43148099ed..1bdfa3198d 100644 --- a/deploy-agent/deployd/download/http_download_helper.py +++ b/deploy-agent/deployd/download/http_download_helper.py @@ -29,11 +29,11 @@ class HTTPDownloadHelper(DownloadHelper): - def __init__(self, url=None, config=None): + def __init__(self, url=None, config=None) -> None: super().__init__(url) self._config = config if config else Config() - def _download_files(self, local_full_fn): + def _download_files(self, local_full_fn) -> int: download_cmd = ['curl', '-o', local_full_fn, '-fksS', self._url] log.info('Running command: {}'.format(' '.join(download_cmd))) output, error, process_return_code = Caller.call_and_log(download_cmd, cwd=os.getcwd()) @@ -45,7 +45,7 @@ def _download_files(self, local_full_fn): log.info('Finish downloading: {} to {}'.format(self._url, local_full_fn)) return status_code - def download(self, local_full_fn): + def download(self, local_full_fn) -> int: log.info("Start to download from url {} to {}".format( self._url, local_full_fn)) if not self.validate_source(): @@ -76,7 +76,7 @@ def download(self, local_full_fn): log.error('Could not connect to: {}'.format(self._url)) return Status.FAILED - def validate_source(self): + def validate_source(self) -> bool: tags = {'type': 'http', 'url': self._url} create_sc_increment(DOWNLOAD_VALIDATE_METRICS, tags=tags) diff --git a/deploy-agent/deployd/download/local_download_helper.py b/deploy-agent/deployd/download/local_download_helper.py index 47d1188d42..1ca6cbac12 100644 --- a/deploy-agent/deployd/download/local_download_helper.py +++ b/deploy-agent/deployd/download/local_download_helper.py @@ -23,7 +23,7 @@ class LocalDownloadHelper(DownloadHelper): - def _download_files(self, local_full_fn): + def _download_files(self, local_full_fn) -> int: download_cmd = ['curl', '-o', local_full_fn, '-ks', self._url] log.info('Running command: {}'.format(' '.join(download_cmd))) error_code = Status.SUCCEEDED @@ -37,7 +37,7 @@ def _download_files(self, local_full_fn): log.info('Finished downloading: {} to {}'.format(self._url, local_full_fn)) return error_code - def download(self, local_full_fn): + def download(self, local_full_fn) -> int: log.info("Start to download from local path {} to {}".format( self._url, local_full_fn)) @@ -46,5 +46,5 @@ def download(self, local_full_fn): log.error('Failed to download the local tar ball for {}'.format(local_full_fn)) return error - def validate_source(self): - return True \ No newline at end of file + def validate_source(self) -> bool: + return True diff --git a/deploy-agent/deployd/download/s3_download_helper.py b/deploy-agent/deployd/download/s3_download_helper.py index d1ff0d8458..479c6c09e6 100644 --- a/deploy-agent/deployd/download/s3_download_helper.py +++ b/deploy-agent/deployd/download/s3_download_helper.py @@ -27,7 +27,7 @@ class S3DownloadHelper(DownloadHelper): - def __init__(self, local_full_fn, aws_connection=None, url=None): + def __init__(self, local_full_fn, aws_connection=None, url=None) -> None: super(S3DownloadHelper, self).__init__(local_full_fn) self._s3_matcher = "^s3://(?P[a-zA-Z0-9\-_]+)/(?P[a-zA-Z0-9\-_/\.]+)/?" self._config = Config() @@ -45,7 +45,7 @@ def __init__(self, local_full_fn, aws_connection=None, url=None): self._key = s3url_parse.group("KEY") - def download(self, local_full_fn): + def download(self, local_full_fn) -> int: log.info(f"Start to download file {self._key} from s3 bucket {self._bucket_name} to {local_full_fn}") if not self.validate_source(): log.error(f'Invalid url: {self._url}. Skip downloading.') @@ -76,9 +76,9 @@ def download(self, local_full_fn): log.error("Failed to get package from s3: {}".format(traceback.format_exc())) return Status.FAILED - def validate_source(self): + def validate_source(self) -> bool: allow_list = self._config.get_s3_download_allow_list() tags = {'type': 's3', 'url': self._url, 'bucket' : self._bucket_name} create_sc_increment(DOWNLOAD_VALIDATE_METRICS, tags=tags) - return self._bucket_name in allow_list if allow_list else True \ No newline at end of file + return self._bucket_name in allow_list if allow_list else True