only update the container health status each time at the very beginni…

…ng of deploy agent starts (#1260)

* only update the container health status each time at the very beginning of deploy agent starts

* import needed packages

* fix test

* add exception check and also check if it is the first time to run deploy-agent

* send map for extraInfo and make sure we get 200 from agent server

* move the external command call into utils.py

* revert test_agent.py

* bump deploy-agent version

deploy-agent/deployd/__init__.py

@@ -26,4 +26,4 @@
 # 2: puppet applied successfully with changes
 PUPPET_SUCCESS_EXIT_CODES = [0, 2]
 
-__version__ = '1.2.46'
+__version__ = '1.2.47'

deploy-agent/deployd/agent.py

@@ -120,6 +120,16 @@ def serve_build(self):
         if not self._executor:
             self._executor = Executor(callback=PingServer(self), config=self._config)
         # start to ping server to get the latest deploy goal
+        if len(self._envs) > 0:
+            for status in self._envs.values():
+                # for each container, we check the health status
+                try:
+                    healthStatus = utils.get_container_health_info(status.report.envName)
+                    status.report.extraInfo = {'serviceHealth': healthStatus}
+                except Exception:
+                    status.report.extraInfo = None
+                    log.exception('get exception while trying to check container health: {}'.format(traceback.format_exc()))
+                    continue
         self._response = self._client.send_reports(self._envs)
 
         if self._response:

deploy-agent/deployd/common/utils.py

@@ -220,6 +220,20 @@ def get_info_from_facter(keys):
         return None
 
 
+def get_container_health_info(key):
+    try:
+        log.info(f"Get health info for container {key}")
+        cmd = ['docker', 'inspect', '-f', '{{.State.Health.Status}}', key]
+        output = subprocess.run(cmd, check=True, stdout=subprocess.PIPE).stdout
+        if output:
+            return output.decode().strip()
+        else:
+            return None
+    except:
+        log.error("Failed to get container health info for {}".format(key))
+        return None
+
+
 def check_not_none(arg, msg=None):
     if arg is None:
         raise ValueError(msg)

deploy-agent/deployd/types/ping_request.py

@@ -71,9 +71,11 @@ def to_json(self):
             ping_report["errorMessage"] = report.errorMessage
             ping_report["failCount"] = report.failCount
             ping_report["deployAlias"] = report.deployAlias
+            """
             if report.extraInfo:
                 ping_report["extraInfo"] = \
                     json.dumps(report.extraInfo, ensure_ascii=False).encode('utf8')
+            """
             ping_requests["reports"].append(ping_report)
         return ping_requests
 

deploy-agent/tests/unit/deploy/server/test_agent.py

@@ -520,4 +520,4 @@ def test_send_deploy_status(self, mock_create_sc):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()