Show Container Health Status On Teletraan UI (#1265)

* store container health in the database * show the container status on ui * add debug info * Show Container Health Status On Teletraan UI
pinterest · Sep 27, 2023 · d079d30 · d079d30
1 parent 161945e
commit d079d30
Show file tree

Hide file tree

Showing 10 changed files with 57 additions and 10 deletions.
diff --git a/deploy-agent/deployd/__init__.py b/deploy-agent/deployd/__init__.py
@@ -26,4 +26,4 @@
 # 2: puppet applied successfully with changes
 PUPPET_SUCCESS_EXIT_CODES = [0, 2]
 
-__version__ = '1.2.47'
+__version__ = '1.2.48'
diff --git a/deploy-agent/deployd/agent.py b/deploy-agent/deployd/agent.py
@@ -116,20 +116,25 @@ def _send_deploy_status_stats(self, deploy_report):
     def serve_build(self):
         """This is the main function of the ``DeployAgent``.
         """
+
         log.info('The deploy agent is starting.')
         if not self._executor:
             self._executor = Executor(callback=PingServer(self), config=self._config)
-        # start to ping server to get the latest deploy goal
+        # include healthStatus info for each container
         if len(self._envs) > 0:
             for status in self._envs.values():
                 # for each container, we check the health status
                 try:
                     healthStatus = utils.get_container_health_info(status.report.envName)
-                    status.report.extraInfo = {'serviceHealth': healthStatus}
+                    if healthStatus:
+                        status.report.containerHealthStatus = healthStatus
+                    else:
+                        status.report.containerHealthStatus = None
                 except Exception:
-                    status.report.extraInfo = None
+                    status.report.containerHealthStatus = None
                     log.exception('get exception while trying to check container health: {}'.format(traceback.format_exc()))
                     continue
+        # start to ping server to get the latest deploy goal
         self._response = self._client.send_reports(self._envs)
 
         if self._response:

diff --git a/deploy-agent/deployd/client/client.py b/deploy-agent/deployd/client/client.py
@@ -139,7 +139,7 @@ def _read_host_info(self):
             except Exception:
                 log.warn('Host ip information does not exist.')
                 pass
-        
+
         if IS_PINTEREST and self._use_host_info is False:
             # Read new keys from facter always
             az_key = self._config.get_facter_az_key()

diff --git a/deploy-agent/deployd/types/ping_report.py b/deploy-agent/deployd/types/ping_report.py
@@ -30,6 +30,7 @@ def __init__(self, jsonValue=None):
         self.failCount = 0
         self.extraInfo = None
         self.deployAlias = None
+        self.containerHealthStatus = None
 
         if jsonValue:
             self.deployId = jsonValue.get('deployId')
@@ -51,10 +52,11 @@ def __init__(self, jsonValue=None):
             self.failCount = jsonValue.get('failCount')
             self.extraInfo = jsonValue.get('extraInfo')
             self.deployAlias = jsonValue.get('deployAlias')
+            self.containerHealthStatus = jsonValue.get('containerHealthStatus')
 
     def __str__(self):
         return "PingReport(deployId={}, envId={}, deployStage={}, status={}, " \
                "errorCode={}, errorMessage={}, failCount={}, extraInfo={}, " \
-               "deployAlias={})".format(self.deployId, self.envId, self.deployStage,
+               "deployAlias={}, containerHealthStatus={})".format(self.deployId, self.envId, self.deployStage,
                                         self.status, self.errorCode, self.errorMessage,
-                                        self.failCount, self.extraInfo, self.deployAlias,)
+                                        self.failCount, self.extraInfo, self.deployAlias, self.containerHealthStatus)
diff --git a/deploy-agent/deployd/types/ping_request.py b/deploy-agent/deployd/types/ping_request.py
@@ -71,11 +71,12 @@ def to_json(self):
             ping_report["errorMessage"] = report.errorMessage
             ping_report["failCount"] = report.failCount
             ping_report["deployAlias"] = report.deployAlias
-            """
+            ping_report["containerHealthStatus"] = report.containerHealthStatus
+
             if report.extraInfo:
                 ping_report["extraInfo"] = \
                     json.dumps(report.extraInfo, ensure_ascii=False).encode('utf8')
-            """
+
             ping_requests["reports"].append(ping_report)
         return ping_requests
 

diff --git a/deploy-board/deploy_board/templates/hosts/host_details.tmpl b/deploy-board/deploy_board/templates/hosts/host_details.tmpl
@@ -18,6 +18,16 @@
                 <td>Status</td>
                 <td>{{ agent.status }}</td>
             </tr>
+            <tr>
+                <td>Container Health Status</td>
+                <td>
+                    {% if agent.containerHealthStatus == "" %}
+                        N/A (Not Applicable)
+                    {% else %}
+                        {{ agent.containerHealthStatus }}
+                    {% endif %}
+                </td>
+            </tr>
             <tr>
                 <td>StartDate(-08:00)</td>
                 <td>{{ agent.startDate|convertTimestamp }}</td>

diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/AgentBean.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/AgentBean.java
@@ -37,6 +37,7 @@
  * first_deploy   TINYINT(1)          NOT NULL DEFAULT 0,
  * first_deploy_time     BIGINT              NOT NULL,
  * stage_start_date     BIGINT              NOT NULL,
+ * container_health_status VARCHAR(32) NOT NULL DEFAULT "",
  * PRIMARY KEY    (host_id, env_id)
  * );
  */
@@ -87,6 +88,9 @@ public class AgentBean implements Updatable {
     @JsonProperty("stageStartDate")
     private Long stage_start_date;
 
+    @JsonProperty("containerHealthStatus")
+    private String container_health_status;
+
     public String getHost_id() {
         return host_id;
     }
@@ -207,6 +211,14 @@ public void setStage_start_date(Long stage_start_date) {
         this.stage_start_date = stage_start_date;
     }
 
+    public String getContainer_Health_Status() {
+        return container_health_status;
+    }
+
+    public void setContainer_Health_Status(String container_health_status) {
+        this.container_health_status = container_health_status;
+    }
+
     @Override
     public SetClause genSetClause() {
         SetClause clause = new SetClause();
@@ -225,6 +237,7 @@ public SetClause genSetClause() {
         clause.addColumn("first_deploy", first_deploy);
         clause.addColumn("first_deploy_time", first_deploy_time);
         clause.addColumn("stage_start_date", stage_start_date);
+        clause.addColumn("container_health_status", container_health_status);
         return clause;
     }
 
@@ -242,7 +255,8 @@ public SetClause genSetClause() {
             "last_err_no=VALUES(last_err_no)," +
             "first_deploy=VALUES(first_deploy)," +
             "first_deploy_time=CASE WHEN first_deploy_time IS NULL THEN VALUES(first_deploy_time) ELSE first_deploy_time END," +
-            "stage_start_date=VALUES(stage_start_date)";
+            "stage_start_date=VALUES(stage_start_date)," +
+            "container_health_status=VALUES(container_health_status)";
 
     @Override
     public String toString() {

diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingReportBean.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/bean/PingReportBean.java
@@ -29,6 +29,7 @@ public class PingReportBean {
     private Integer failCount;
     private Map<String, String> extraInfo;
     private String deployAlias;
+    private String containerHealthStatus;
 
     public String getDeployId() {
         return deployId;
@@ -102,6 +103,14 @@ public void setDeployAlias(String deployAlias) {
         this.deployAlias = deployAlias;
     }
 
+    public String getContainerHealthStatus() {
+        return containerHealthStatus;
+    }
+
+    public void setContainerHealthStatus(String containerHealthStatus) {
+        this.containerHealthStatus = containerHealthStatus;
+    }
+
     @Override
     public String toString() {
         return ReflectionToStringBuilder.toString(this);

diff --git a/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java b/deploy-service/common/src/main/java/com/pinterest/deployservice/handler/GoalAnalyst.java
@@ -319,6 +319,11 @@ AgentBean genUpdateBeanByReport(PingReportBean report, AgentBean agent) {
         updateBean.setState(proposeNewAgentState(report, agent));
         updateBean.setStage_start_date(System.currentTimeMillis());
         updateBean.setDeploy_stage(report.getDeployStage());
+        if (report.getContainerHealthStatus() == null) {
+            updateBean.setContainer_Health_Status("");
+        } else {
+            updateBean.setContainer_Health_Status(report.getContainerHealthStatus());
+        }
 
         if (agent == null) {
             // if agent is missing in agent table, treat it as not first_deploy.

diff --git a/deploy-service/common/src/main/resources/sql/deploy.sql b/deploy-service/common/src/main/resources/sql/deploy.sql
@@ -171,6 +171,7 @@ CREATE TABLE IF NOT EXISTS agents (
     first_deploy   TINYINT(1)          NOT NULL DEFAULT 0,
     first_deploy_time     BIGINT,
     stage_start_date     BIGINT,
+    container_health_status VARCHAR(32) NOT NULL DEFAULT "",
     PRIMARY KEY    (host_id, env_id)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 CREATE INDEX agent_env_idx ON agents (env_id, host_name);