U24 support for deploy-agent

Add support for Ubuntu 24 to deploy-agent. This includes adding
support for Python 3.12.

This requires updating some packages (e.g. requests, gevent, boto). For
boto, an `s3_client` class was added to support new (boto3) and old
(boto) packages.

Testing done:
- Unit tests passed
- Installed the deploy-agent package on an Ubuntu 24 host
  - Deploy agent ran, but failed due to missing `facter` command
- Installed the deploy-agent package on an Ubuntu 20 host
  - The deploy agent worked as expected. Was able to install new
    Teletraan builds and restart a Teletraan stage

.github/workflows/python.yml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.7, 3.8, 3.12]
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

deploy-agent/deployd/download/download_helper_factory.py

@@ -16,9 +16,9 @@
 from typing import Optional
 
 from future.moves.urllib.parse import urlparse
-from boto.s3.connection import S3Connection
 
 from deployd.download.download_helper import DownloadHelper
+from deployd.download.s3_client import S3Client
 from deployd.download.s3_download_helper import S3DownloadHelper
 from deployd.download.http_download_helper import HTTPDownloadHelper
 from deployd.download.local_download_helper import LocalDownloadHelper
@@ -38,7 +38,7 @@ def gen_downloader(url, config) -> Optional[DownloadHelper]:
             if aws_access_key_id is None or aws_secret_access_key is None:
                 log.error("aws access key id and secret access key not found")
                 return None
-            aws_conn = S3Connection(aws_access_key_id, aws_secret_access_key, True)
+            aws_conn = S3Client(aws_access_key_id, aws_secret_access_key)
             return S3DownloadHelper(local_full_fn=url, aws_connection=aws_conn, url=None, config=config)
         elif url_parse.scheme == 'file':
             return LocalDownloadHelper(url=url)

deploy-agent/deployd/download/s3_client.py

@@ -0,0 +1,87 @@
+USE_BOTO3 = False
+try:
+    from boto.s3.connection import S3Connection
+except ImportError:
+    import botocore
+    import boto3
+
+    USE_BOTO3 = True
+
+
+class Boto2Client:
+    """
+    Client to handle boto2 operations. This can be removed
+    once boto2 is no longer used
+    """
+    def __init__(self, aws_access_key_id, aws_secret_access_key):
+        self.client = S3Connection(aws_access_key_id, aws_secret_access_key, True)
+
+    def get_key(self, bucket_name, key_name):
+        """
+        Return the object at the specified key
+        """
+        return self.client.get_bucket(bucket_name).get_key(key_name)
+
+    def download_object_to_file(self, obj, file_name):
+        """
+        Download the object to the specified file name
+
+        :param obj: the object returned from `self.get_key`
+        :param file_name str: the file_name to download to
+        """
+        obj.get_contents_to_filename(file_name)
+
+    def get_etag(self, obj):
+        """
+        Get the etag of the specified object
+
+        :param obj: the object returned from `self.get_key`
+        """
+        return obj.etag
+
+
+class Boto3Client:
+    """
+    Client to handle boto3 operations. This can be renamed to
+    `S3Client` once boto2 is no longer used.
+    """
+    def __init__(self, aws_access_key_id, aws_secret_access_key):
+        session = boto3.Session(
+            aws_access_key_id=aws_access_key_id,
+            aws_secret_access_key=aws_secret_access_key,
+        )
+        self.client = session.resource('s3')
+
+    def get_key(self, bucket_name, key_name):
+        """
+        Return the object at the specified key
+        """
+        obj = self.client.Bucket(bucket_name).Object(key_name)
+        try:
+            # To be compatible with boto2, return None if key does not exist
+            obj.load()
+            return obj
+        except botocore.exceptions.ClientError as e:
+            if e.response['Error']['Code'] == "404":
+                return None
+            else:
+                raise
+
+    def download_object_to_file(self, obj, file_name):
+        """
+        Download the object to the specified file name
+
+        :param obj: the object returned from `self.get_key`
+        :param file_name str: the file_name to download to
+        """
+        obj.download_file(file_name)
+
+    def get_etag(self, obj):
+        """
+        Get the etag of the specified object
+
+        :param obj: the object returned from `self.get_key`
+        """
+        return obj.e_tag
+
+S3Client = Boto3Client if USE_BOTO3 else Boto2Client

deploy-agent/deployd/download/s3_download_helper.py

@@ -16,10 +16,10 @@
 import logging
 import re
 
-from boto.s3.connection import S3Connection
 from deployd.common.config import Config
 from deployd.common.status_code import Status
 from deployd.download.download_helper import DownloadHelper, DOWNLOAD_VALIDATE_METRICS
+from deployd.download.s3_client import S3Client
 from deployd.common.stats import create_sc_increment
 
 log = logging.getLogger(__name__)
@@ -36,7 +36,7 @@ def __init__(self, local_full_fn, aws_connection=None, url=None, config=None) ->
         else:
             aws_access_key_id = self._config.get_aws_access_key()
             aws_secret_access_key = self._config.get_aws_access_secret()
-            self._aws_connection = S3Connection(aws_access_key_id, aws_secret_access_key, True)
+            self._aws_connection = S3Client(aws_access_key_id, aws_secret_access_key)
 
         if url:
             self._url = url
@@ -52,13 +52,13 @@ def download(self, local_full_fn) -> int:
             return Status.FAILED
 
         try:
-            filekey = self._aws_connection.get_bucket(self._bucket_name).get_key(self._key)
+            filekey = self._aws_connection.get_key(self._bucket_name, self._key)
             if filekey is None:
                 log.error("s3 key {} not found".format(self._key))
                 return Status.FAILED
 
-            filekey.get_contents_to_filename(local_full_fn)
-            etag = filekey.etag
+            self._aws_connection.download_object_to_file(filekey, local_full_fn)
+            etag = self._aws_connection.get_etag(filekey)
             if "-" not in etag:
                 if etag.startswith('"') and etag.endswith('"'):
                     etag = etag[1:-1]

deploy-agent/requirements.txt

@@ -1,8 +1,8 @@
 -e .
 
-PyYAML==5.3.1
+PyYAML==5.3.1; python_version < '3.12'
+PyYAML==6.0.1; python_version >= '3.12'
 zipp==1.2.0
 configparser==4.0.2
 python-daemon==2.0.6
-setuptools==44.1.1; python_version < '3'
-setuptools==54.2.0; python_version >= '3'
+setuptools==54.2.0

deploy-agent/requirements_test.txt

@@ -3,8 +3,8 @@
 # tests
 tox==1.6.1
 pytest-cov
-pytest==6.2.2; python_version >= '3'
-pytest==4.6.11; python_version < '3'
+pytest==6.2.2; python_version < '3.12'
+pytest==8.2.1; python_version >= '3.12'
 mock==1.0.1
 flake8==2.5.4
 pep8

deploy-agent/setup.py

@@ -24,12 +24,14 @@
                    'deploy-stager = deployd.staging.stager:main']
 
 install_requires = [
-    "requests==2.20.0",
-    "gevent>=1.0.2,<=1.2.2; python_version < '3'",
+    "requests==2.20.0; python_version < '3.12'",
+    "requests==2.32.3; python_version >= '3.12'",
     "gevent>=1.0.2,<=1.5.0; python_version < '3.8'",
-    "gevent==20.12.0; python_version >= '3.8'",
+    "gevent==20.12.0; python_version >= '3.8' and python_version < '3.12'",
+    "gevent==24.2.1; python_version >= '3.12'",
     "lockfile==0.10.2",
-    "boto>=2.39.0",
+    "boto>=2.39.0; python_version < '3.12'",
+    "boto3==1.34.114; python_version >= '3.12'",
     "python-daemon==2.0.6",
     "future==0.18.2"
 ]

deploy-agent/tests/unit/deploy/download/test_download_helper.py

@@ -12,7 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+USE_BOTO3 = False
+try:
+    from boto.s3.connection import S3Connection
+except ImportError:
+    import boto3
+    USE_BOTO3 = True
+
 from deployd.download.s3_download_helper import S3DownloadHelper
+from deployd.download.s3_client import S3Client
 import os
 import mock
 import shutil
@@ -35,23 +43,33 @@ def setUpClass(cls):
 
         target = os.path.join(builds_dir, 'mock.txt')
         cls.target = target
-        cls.aws_conn = mock.Mock()
-        aws_filekey = cls.aws_conn.get_bucket.return_value.get_key.return_value
+        cls.aws_conn = mock.Mock(wraps=S3Client('test_access_key_id', 'test_secret_access_key'))
+        aws_filekey = cls.aws_conn.get_key.return_value
 
         def get_contents_to_filename(fn):
             with open(fn, 'w') as file:
                 file.write("hello mock\n")
-        aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename)
-        aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a"
+
+        if USE_BOTO3:
+            aws_filekey.download_file = mock.Mock(side_effect=get_contents_to_filename)
+            aws_filekey.e_tag = "f7673f4693aab49e3f8e643bc54cb70a"
+        else:
+            aws_filekey.get_contents_to_filename = mock.Mock(side_effect=get_contents_to_filename)
+            aws_filekey.etag = "f7673f4693aab49e3f8e643bc54cb70a"
 
     def test_download_s3(self):
         downloader = S3DownloadHelper(self.target, self.aws_conn, self.url)
         downloader.download(self.target)
-        self.aws_conn.get_bucket.assert_called_once_with("pinterest-builds")
-        self.aws_conn.get_bucket.return_value.get_key.assert_called_once_with("teletraan/mock.txt")
-        self.aws_conn.get_bucket.return_value.get_key.return_value\
-            .get_contents_to_filename\
-            .assert_called_once_with(self.target)
+        self.aws_conn.get_key.assert_called_once_with("pinterest-builds", "teletraan/mock.txt")
+
+        if USE_BOTO3:
+            self.aws_conn.get_key.return_value\
+                .download_file\
+                .assert_called_once_with(self.target)
+        else:
+            self.aws_conn.get_key.return_value\
+                .get_contents_to_filename\
+                .assert_called_once_with(self.target)
 
     @classmethod
     def tearDownClass(cls):

deploy-agent/tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist=py36,py38
+envlist=py36,py38,py312
 recreate=True
 
 [testenv]
@@ -10,3 +10,4 @@ commands=py.test --cov=deployd {posargs}
 python =
     3.6: py36
     3.8: py38
+    3.12: py312