Allow deploy agent run serverless on existing build (#1350)

* Add account id validator

* Enable serverless mode run with local build

* Enable serverless mode run with local build

deploy-agent/deployd/agent.py

@@ -26,19 +26,19 @@
 from deployd.common.config import Config
 from deployd.common.exceptions import AgentException
 from deployd.common.helper import Helper
-from deployd.common.single_instance import SingleInstance
 from deployd.common.env_status import EnvStatus
+from deployd.common.single_instance import SingleInstance
 from deployd.common.stats import TimeElapsed, create_sc_timing, create_sc_increment
 from deployd.common.utils import get_telefig_version,get_container_health_info, check_prereqs
 from deployd.common.utils import uptime as utils_uptime, listen as utils_listen
 from deployd.common.executor import Executor
 from deployd.common.types import DeployReport, PingStatus, DeployStatus, OpCode, DeployStage, AgentStatus
 from deployd import __version__, IS_PINTEREST, MAIN_LOGGER
 
-log = logging.getLogger(MAIN_LOGGER)
+log: logging.Logger = logging.getLogger(name=MAIN_LOGGER)
 
 class PingServer(object):
-    def __init__(self, ag):
+    def __init__(self, ag) -> None:
         self._agent = ag
 
     def __call__(self, deploy_report):
@@ -235,8 +235,8 @@ def serve_forever(self):
     def serve_once(self):
         log.info("Running deploy agent in non daemon mode")
         try:
-            if len(self._envs) > 0:
-                # randomly sleep some time before pinging server
+            if len(self._envs) > 0 and not isinstance(self._client, ServerlessClient):
+                # randomly sleep some time before pinging server. Skip sleeping if in serverless mode.
                 # TODO: consider pause stat_time_elapsed_internal here
                 sleep_secs = randrange(self._config.get_init_sleep_time())
                 log.info("Randomly sleep {} seconds before starting.".format(sleep_secs))
@@ -509,19 +509,21 @@ def main():
                              "json format.")
     parser.add_argument('--env-name', dest='env_name', default=None,
                         help="Optional. In 'serverless' mode, env_name needs to be passed in.")
+    parser.add_argument('--deploy-stage', dest='deploy_stage', default=None,
+                        help="Optional. In 'serverless' mode, initial deploy_stage to start with.")
     parser.add_argument('--script-variables', dest='script_variables', default='{}',
                         help="Optional. In 'serverless' mode,  script_variables is needed in "
                              "json format.")
     parser.add_argument('-v', '--version', action='version',
                         version=__version__, help='Deploy agent version.')
 
-    args = parser.parse_args()
+    args: argparse.Namespace = parser.parse_args()
 
     is_serverless_mode = AgentRunMode.is_serverless(args.mode)
     if args.daemon and is_serverless_mode:
         raise ValueError("daemon and serverless mode is mutually exclusive.")
 
-    config = Config(args.config_file)
+    config = Config(filenames=args.config_file)
 
     if IS_PINTEREST:
         import pinlogger
@@ -546,7 +548,7 @@ def main():
     if is_serverless_mode:
         log.info("Running agent with severless client")
         client = ServerlessClient(env_name=args.env_name, stage=args.stage, build=args.build,
-                                  script_variables=args.script_variables)
+                                  script_variables=args.script_variables, deploy_stage=args.deploy_stage)
 
     uptime = utils_uptime()
     agent = DeployAgent(client=client, conf=config)

deploy-agent/deployd/client/serverless_client.py

@@ -14,6 +14,7 @@
 
 import logging
 import json
+from typing import Optional
 import uuid
 
 from deployd.client.base_client import BaseClient
@@ -23,7 +24,7 @@
 from deployd.types.opcode import OperationCode
 from deployd.types.ping_response import PingResponse
 
-log = logging.getLogger(__name__)
+log: logging.Logger = logging.getLogger(__name__)
 
 _DEPLOY_STAGE_TRANSITIONS = dict([(i, i+1) for i in range(DeployStage.PRE_DOWNLOAD, DeployStage.SERVING_BUILD)])
 
@@ -37,35 +38,36 @@ class ServerlessClient(BaseClient):
       PRE_DOWNLOAD->DOWNLOADING->POST_DOWNLOAD->STAGING->PRE_RESTART
           ->RESTARTING->POST_RESTART->SERVING_BUILD
     """
-    def __init__(self, env_name, stage, build, script_variables):
+    def __init__(self, env_name, stage, build, script_variables, deploy_stage: Optional[DeployStage] = None) -> None:
         """build contains build information in json format. It contains information defined in types/build.py.
         """
-        self._env_name = utils.check_not_none(env_name, 'env_name can not be None')
-        self._stage = utils.check_not_none(stage, 'stage name can not be None')
-        self._build = json.loads(utils.check_not_none(build, 'build can not be None'))
-        self._script_variables = json.loads(utils.check_not_none(script_variables, 'script_variables can not be None'))
-        self._deploy_id = uuid.uuid4().hex
+        self._env_name: str = utils.check_not_none(env_name, 'env_name can not be None')
+        self._stage: str = utils.check_not_none(stage, 'stage name can not be None')
+        self._build: dict[str, str] = json.loads(utils.check_not_none(build, 'build can not be None'))
+
+        self._script_variables: dict[str, str] = json.loads(utils.check_not_none(script_variables, 'script_variables can not be None'))
+        self._deploy_id: str = uuid.uuid4().hex
+        self._deploy_stage: DeployStage = deploy_stage if deploy_stage is not None else DeployStage.PRE_DOWNLOAD
 
-    def send_reports(self, env_reports=None):
-        reports = [status.report for status in env_reports.values()]
+    def send_reports(self, env_reports=None) -> Optional[PingResponse]:
+        reports: list = [status.report for status in env_reports.values()]
         for report in reports:
             if report.envName != self._env_name:
                 continue
             self._env_id = report.envId 
-            ping_response = self._create_response(report)
-            log.info('%s -> %s' % (reports, ping_response))
-
+            ping_response: Optional[PingResponse] = self._create_response(report)
+            log.info(f"{str(report)} {str(ping_response)}")
             return ping_response
 
         # env_status file might be corrupted or first deploy.
         self._env_id = uuid.uuid4().hex
         return self._create_response(None)
 
-    def _create_response(self, report):
+    def _create_response(self, report) -> Optional[PingResponse]:
         # check if this is the first step
         if report is None or report.deployId is None or report.deployId != self._deploy_id:
             # first report from agent, start first deploy stage.
-            return self._new_response_value(DeployStage.PRE_DOWNLOAD)
+            return self._new_response_value(numeric_deploy_stage=self._deploy_stage)
         if report.errorCode != 0:
             # terminate the deployment.
             return None
@@ -86,15 +88,14 @@ def _create_response(self, report):
         # terminate deployment
         return None
 
-    def _new_response_value(self, numeric_deploy_stage):
+    def _new_response_value(self, numeric_deploy_stage) -> PingResponse:
         value= {'opCode': OperationCode.DEPLOY,
                 'deployGoal': {'deployId': self._deploy_id,
                                'envId': self._env_id,
                                'envName': self._env_name,
                                'stageName': self._stage,
+                               'build': self._build,
                                'deployStage': numeric_deploy_stage}}
-        if numeric_deploy_stage == DeployStage.DOWNLOADING:
-            value['deployGoal']['build'] = self._build
         if numeric_deploy_stage == DeployStage.PRE_DOWNLOAD:
             value['deployGoal']['scriptVariables'] = self._script_variables
-        return PingResponse(value)
+        return PingResponse(jsonValue=value)

deploy-agent/tests/unit/deploy/client/test_serverless_client.py

@@ -12,17 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Optional
 import unittest
 from tests import TestCase
 
 from deployd.client.serverless_client import ServerlessClient
-from deployd.common.types import DeployStatus, AgentStatus
+from deployd.common.types import DeployStage, DeployStatus, AgentStatus
 from deployd.types.ping_report import PingReport
+from deployd.types.ping_response import PingResponse
 
 
 class TestServerlessClient(TestCase):
 
-    def setUp(self):
+    def setUp(self) -> None:
         self.env_name = "test"
         self.stage = "prod"
         self.env_id = "12343434"
@@ -31,7 +33,7 @@ def setUp(self):
         self.client = ServerlessClient(env_name=self.env_name, stage=self.stage, build=self.build,
                                        script_variables=self.script_variables) 
 
-    def _new_report(self):
+    def _new_report(self) -> PingReport:
         report = PingReport()
         report.envName = self.env_name
         report.stageName = self.stage
@@ -41,16 +43,16 @@ def _new_report(self):
         report.status = AgentStatus.SUCCEEDED
         return report
 
-    def test_deploy_stage_trnasition(self):
-        report = self._new_report()
+    def test_deploy_stage_transition(self) -> None:
+        report: PingReport = self._new_report()
         deploy_status = DeployStatus()
         deploy_status.report = report
-        env_status = {self.env_name : deploy_status}
+        env_status: dict[str, DeployStatus] = {self.env_name : deploy_status}
 
-        deployStages = ['PRE_DOWNLOAD', 'DOWNLOADING', 'POST_DOWNLOAD', 'STAGING', 'PRE_RESTART', 'RESTARTING', 'POST_RESTART', 'SERVING_BUILD']
+        deployStages: list[str] = ['PRE_DOWNLOAD', 'DOWNLOADING', 'POST_DOWNLOAD', 'STAGING', 'PRE_RESTART', 'RESTARTING', 'POST_RESTART', 'SERVING_BUILD']
 
         for i in range(0, len(deployStages)):
-            response = self.client.send_reports(env_status) 
+            response: Optional[PingReport] = self.client.send_reports(env_status) 
             self.assertEqual(response.opCode, "DEPLOY")
             self.assertEqual(response.deployGoal.deployStage, deployStages[i])
             report.deployStage = response.deployGoal.deployStage
@@ -60,29 +62,42 @@ def test_deploy_stage_trnasition(self):
         response = self.client.send_reports(env_status)
         self.assertEqual(response.deployGoal, None)
 
+    def test_run_with_defined_deploy_stage(self) -> None:
+        self.client = ServerlessClient(env_name=self.env_name, stage=self.stage, build=self.build,
+                                       script_variables=self.script_variables, deploy_stage=DeployStage.PRE_RESTART)
+        report: PingReport = self._new_report()
+        report.deployId = None
+        deploy_status = DeployStatus()
+        deploy_status.report = report
+        env_status: dict[str, DeployStatus] = {self.env_name : deploy_status}
+
+        response: Optional[PingResponse] = self.client.send_reports(env_status)
+        self.assertEqual(response.opCode, "DEPLOY")
+        self.assertEqual(response.deployGoal.deployStage, 'PRE_RESTART')
+
     def test_errorcode_stop_deployment(self):
-        report = self._new_report()
+        report: PingReport = self._new_report()
         deploy_status = DeployStatus()
         deploy_status.report = report
-        env_status = {self.env_name : deploy_status}
+        env_status: dict[str, DeployStatus] = {self.env_name : deploy_status}
 
         # first try is allowed.
         report.errorCode = 123 
-        response = self.client.send_reports(env_status)
+        response: Optional[PingResponse] = self.client.send_reports(env_status)
         report.deployStage = response.deployGoal.deployStage
         report.deployId = response.deployGoal.deployId
 
         response = self.client.send_reports(env_status)
         self.assertEqual(response, None)
 
     def test_unknow_status_cause_retry(self):
-        report = self._new_report()
+        report: PingReport = self._new_report()
         deploy_status = DeployStatus()
         deploy_status.report = report
-        env_status = {self.env_name : deploy_status}
+        env_status: dict[str, DeployStatus] = {self.env_name : deploy_status}
 
         report.status = AgentStatus.UNKNOWN 
-        response = self.client.send_reports(env_status)
+        response: Optional[PingResponse] = self.client.send_reports(env_status)
         report.deployStage = response.deployGoal.deployStage
         report.deployId = response.deployGoal.deployId
 

deploy-service/teletraanservice/src/main/java/com/pinterest/teletraan/TeletraanServiceConfiguration.java

@@ -15,7 +15,14 @@
  */
 package com.pinterest.teletraan;
 
+import java.util.Collections;
+import java.util.List;
+
+import javax.validation.Valid;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.pinterest.teletraan.config.AnonymousAuthenticationFactory;
+import com.pinterest.teletraan.config.AppEventFactory;
 import com.pinterest.teletraan.config.AuthenticationFactory;
 import com.pinterest.teletraan.config.AuthorizationFactory;
 import com.pinterest.teletraan.config.AwsFactory;
@@ -27,7 +34,6 @@
 import com.pinterest.teletraan.config.DefaultHostGroupFactory;
 import com.pinterest.teletraan.config.EmailFactory;
 import com.pinterest.teletraan.config.EmbeddedDataSourceFactory;
-import com.pinterest.teletraan.config.AppEventFactory;
 import com.pinterest.teletraan.config.ExternalAlertsConfigFactory;
 import com.pinterest.teletraan.config.HostGroupFactory;
 import com.pinterest.teletraan.config.JenkinsFactory;
@@ -38,14 +44,9 @@
 import com.pinterest.teletraan.config.SystemFactory;
 import com.pinterest.teletraan.config.WorkerConfig;
 
-import com.fasterxml.jackson.annotation.JsonProperty;
 import io.dropwizard.Configuration;
 import io.dropwizard.health.conf.HealthConfiguration;
 
-import java.util.Collections;
-import java.util.List;
-import javax.validation.Valid;
-
 public class TeletraanServiceConfiguration extends Configuration {
     @Valid
     @JsonProperty("db")