Migrate boto to boto3

The `boto` dependency is no longer supported on py312. Switch to boto3

deploy-agent/deployd/download/download_helper_factory.py

@@ -15,8 +15,8 @@
 from logging import Logger, getLogger
 from typing import Optional
 
+import boto3
 from future.moves.urllib.parse import urlparse
-from boto.s3.connection import S3Connection
 
 from deployd.download.download_helper import DownloadHelper
 from deployd.download.s3_download_helper import S3DownloadHelper
@@ -38,8 +38,8 @@ def gen_downloader(url, config) -> Optional[DownloadHelper]:
             if aws_access_key_id is None or aws_secret_access_key is None:
                 log.error("aws access key id and secret access key not found")
                 return None
-            aws_conn = S3Connection(aws_access_key_id, aws_secret_access_key, True)
-            return S3DownloadHelper(local_full_fn=url, aws_connection=aws_conn, url=None, config=config)
+            s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+            return S3DownloadHelper(local_full_fn=url, s3_client=s3_client, url=None, config=config)
         elif url_parse.scheme == 'file':
             return LocalDownloadHelper(url=url)
         else:

deploy-agent/deployd/download/s3_download_helper.py

@@ -16,7 +16,9 @@
 import logging
 import re
 
-from boto.s3.connection import S3Connection
+import boto3
+import botocore
+
 from deployd.common.config import Config
 from deployd.common.status_code import Status
 from deployd.download.download_helper import DownloadHelper, DOWNLOAD_VALIDATE_METRICS
@@ -27,16 +29,16 @@
 
 class S3DownloadHelper(DownloadHelper):
 
-    def __init__(self, local_full_fn, aws_connection=None, url=None, config=None) -> None:
+    def __init__(self, local_full_fn, s3_client=None, url=None, config=None) -> None:
         super(S3DownloadHelper, self).__init__(local_full_fn)
         self._s3_matcher = "^s3://(?P<BUCKET>[a-zA-Z0-9\-_]+)/(?P<KEY>[a-zA-Z0-9\-_/\.]+)/?"
         self._config = config if config else Config()
-        if aws_connection:
-            self._aws_connection = aws_connection
+        if s3_client:
+            self._s3_client = s3_client
         else:
             aws_access_key_id = self._config.get_aws_access_key()
             aws_secret_access_key = self._config.get_aws_access_secret()
-            self._aws_connection = S3Connection(aws_access_key_id, aws_secret_access_key, True)
+            self._s3_client = boto3.client('s3', aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
 
         if url:
             self._url = url
@@ -52,13 +54,13 @@ def download(self, local_full_fn) -> int:
             return Status.FAILED
 
         try:
-            filekey = self._aws_connection.get_bucket(self._bucket_name).get_key(self._key)
-            if filekey is None:
+            object_metadata = self._get_object_metadata()
+            if object_metadata is None:
                 log.error("s3 key {} not found".format(self._key))
                 return Status.FAILED
 
-            filekey.get_contents_to_filename(local_full_fn)
-            etag = filekey.etag
+            self._s3_client.download_file(self._bucket_name, self._key, local_full_fn)
+            etag = object_metadata["ETag"]
             if "-" not in etag:
                 if etag.startswith('"') and etag.endswith('"'):
                     etag = etag[1:-1]
@@ -76,6 +78,20 @@ def download(self, local_full_fn) -> int:
             log.error("Failed to get package from s3: {}".format(traceback.format_exc()))
             return Status.FAILED
 
+    def _get_object_metadata(self):
+        """
+        Return the object metadata at the specified key, or None if not found
+        """
+        try:
+            return self._s3_client.head_object(Bucket=self._bucket_name, Key=self._key)
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "404":
+                # The key does not exist.
+                return None
+            else:
+              # Something else has gone wrong.
+              raise
+
     def validate_source(self) -> bool:
         allow_list = self._config.get_s3_download_allow_list()
         tags = {'type': 's3', 'url': self._url, 'bucket' : self._bucket_name}

deploy-agent/setup.py

@@ -29,7 +29,7 @@
     "gevent==20.12.0; python_version < '3.12'",
     "gevent==24.2.1; python_version >= '3.12'",
     "lockfile==0.10.2",
-    "boto>=2.39.0",
+    "boto3==1.34.134",
     "python-daemon==2.0.6",
     "future==0.18.2"
 ]

deploy-agent/tests/unit/deploy/download/test_download_helper.py

@@ -35,23 +35,23 @@ def setUpClass(cls):
 
         target = os.path.join(builds_dir, 'mock.txt')
         cls.target = target
-        cls.aws_conn = mock.Mock()
-        aws_filekey = cls.aws_conn.get_bucket.return_value.get_key.return_value
+        cls.s3_client = mock.MagicMock()
+        cls.s3_client.head_object.return_value.__getitem__.return_value = "f7673f4693aab49e3f8e643bc54cb70a"
 
-        def get_contents_to_filename(fn):
+        def download_file(bucket, key, fn):
             with open(fn, 'w') as file:
                 file.write("hello mock\n")
-        aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename)
-        aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a"
+        cls.s3_client.download_file = mock.Mock(side_effect=download_file)
+
 
     def test_download_s3(self):
-        downloader = S3DownloadHelper(self.target, self.aws_conn, self.url)
+        downloader = S3DownloadHelper(self.target, self.s3_client, self.url)
         downloader.download(self.target)
-        self.aws_conn.get_bucket.assert_called_once_with("pinterest-builds")
-        self.aws_conn.get_bucket.return_value.get_key.assert_called_once_with("teletraan/mock.txt")
-        self.aws_conn.get_bucket.return_value.get_key.return_value\
-            .get_contents_to_filename\
-            .assert_called_once_with(self.target)
+        self.s3_client.head_object.assert_called_once_with(Bucket="pinterest-builds", Key="teletraan/mock.txt")
+        self.s3_client.download_file\
+            .assert_called_once_with("pinterest-builds", "teletraan/mock.txt", self.target)
+        self.s3_client.head_object.return_value.__getitem__\
+            .assert_called_once_with("ETag")
 
     @classmethod
     def tearDownClass(cls):

deploy-agent/tests/unit/deploy/download/test_s3_download_helper.py

@@ -28,7 +28,7 @@ def setUp(self, mock_aws_key, mock_aws_secret):
         mock_aws_key.return_value = "test_key"
         mock_aws_secret.return_value= "test_secret"
         self.config = Config()
-        self.downloader = S3DownloadHelper(local_full_fn='', aws_connection=None, url="s3://bucket1/key1", config=self.config)
+        self.downloader = S3DownloadHelper(local_full_fn='', s3_client=None, url="s3://bucket1/key1", config=self.config)
 
     @mock.patch('deployd.common.config.Config.get_s3_download_allow_list')
     def test_validate_url_with_allow_list(self, mock_get_s3_download_allow_list):