Fix hosts spinning indefinitely

deploy-board/deploy_board/templates/hosts/host_details.html

@@ -58,15 +58,15 @@
 
 <div id="hostDetailId" class="collapse in panel-body table-responsive">
 <table class="table table-striped table-bordered table-condensed table-hover">
-    <tr>  
+    <tr>
         {% if account_id and account_id != "null" %}
         <th class="col-lg-1">Acccount Id</th>
         {% endif %}
         <th class="col-lg-1">Host Name</th>
         <th class="col-lg-2">Host Id</th>
         <th class="col-lg-2">Host Group</th>
         <th class="col-lg-2">Ip Address</th>
-        <th class="col-lg-1">Status</th>
+        <th class="col-lg-1">State</th>
         <th class="col-lg-2">Create Date</th>
         <th class="col-lg-2">Last Update</th>
     </tr>
@@ -85,8 +85,12 @@
     </tr>
     {% endfor %}
 </table>
+    <div class="row">
     {% if pinterest %}
-    <a href="javascript:void(0)" id="seeMoreLink">See more host details</a>
+    <div class="row">
+        <a href="javascript:void(0)" id="seeMoreLink">See more host details</a>
+        <a class="pull-right" href="#" data-toggle="modal" data-target="#rawHostInfoModal">Raw Host Info</a>
+    </div>
     <div id="moreHostDetails" hidden="true">
     {% for key, value in host_details.items %}
     <div class="row">
@@ -117,9 +121,25 @@
         <a href="{{ host_information_url }}/instance/{{ host_id }}">click here</a>
     {% endif %}
     </div>
-    {% endif %}
-</div>
 
+    <div id="rawHostInfoModal" class="modal fade" tabindex="-1" role="dialog" aria-hidden="true">
+      <div class="modal-dialog">
+        <div class="modal-content">
+            <div class="modal-header">
+                <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span><span class="sr-only">Close</span></button>
+                <h4 class="modal-title">Raw Host Info</h4>
+            </div>
+            <div class="modal-body">
+                {% for host in hosts %}
+                <code>{{ host }}</code>
+                {% endfor %}
+            </div>
+        </div>
+      </div>
+      </div>
+    </div>
+    {% endif %}
+    </div>
 </div>
 
 <script>

deploy-board/deploy_board/webapp/agent_report.py

@@ -15,13 +15,15 @@
 # -*- coding: utf-8 -*-
 """Helper functions to help generate agents views
 """
+from deploy_board.webapp.helpers import agents_helper
 from .common import is_agent_failed
 from .helpers import builds_helper, deploys_helper, environs_helper, environ_hosts_helper
 from deploy_board.settings import IS_PINTEREST
 import time
 from collections import OrderedDict
 from functools import cmp_to_key
 
+
 # Constants used to distinguish the action to generate the host report
 TOTAL_ALIVE_HOST_REPORT = "TOTAL_ALIVE_HOST_REPORT"
 UNKNOWN_HOST_REPORT = "UNKNOWN_HOST_REPORT"
@@ -38,10 +40,11 @@
 
 
 class AgentStatistics(object):
-    def __init__(self, agent=None, isCurrent=False, isStale=False):
+    def __init__(self, agent=None, isCurrent=False, isStale=False, isHostFailed=False):
         self.agent = agent
         self.isCurrent = isCurrent
         self.isStale = isStale
+        self.isHostFailed = isHostFailed
 
 
 class DeployStatistics(object):
@@ -108,7 +111,12 @@ def addToEnvReport(request, deployStats, agent, env):
     if duration >= DEFAULT_STALE_THRESHOLD:
         isStale = True
 
-    return AgentStatistics(agent=agent, isCurrent=isCurrent, isStale=isStale)
+    isHostFailed = False
+    agent_ec2_tags = agents_helper.get_agent_ec2_tags(request, env['envName'], env['stageName'])
+    if agent_ec2_tags and agent_ec2_tags.get("service_mapping") == "shame":
+        isHostFailed = True
+
+    return AgentStatistics(agent, isCurrent, isStale, isHostFailed)
 
 
 def _compare_agent_status(agentStats1, agentStats2):

deploy-board/deploy_board/webapp/helpers/agents_helper.py

@@ -3,9 +3,9 @@
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-#  
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-#    
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -42,8 +42,8 @@ def retry_deploy(request, env_name, stage_name, host_id):
 
 def reset_all_environments(request, host_id):
     return deployclient.put(
-        "/agents/id/%s" % (host_id,), 
-        request.teletraan_user_id.token, 
+        "/agents/id/%s" % (host_id,),
+        request.teletraan_user_id.token,
         data={"state": "RESET"},
     )
 
@@ -59,8 +59,11 @@ def resume_deploy(request, env_name, stage_name, host_id):
 
 def get_agents_by_host(request, host_name):
     return deployclient.get("/agents/%s" % host_name, request.teletraan_user_id.token)
-    
+
 
 def get_agents_total_by_env(request, env_id):
     return deployclient.get("/agents/env/%s/total" % env_id, request.teletraan_user_id.token)
-
+
+def get_agent_ec2_tags(request, env_name, stage_name):
+    return deployclient.get(f"/env/{env_name}/{stage_name}/host_ec2_tags", request.teletraan_user_id.token)
+

deploy-board/deploy_board/webapp/templatetags/utils.py

@@ -25,7 +25,7 @@
 import pytz
 import logging
 from deploy_board.webapp.service_add_ons import ServiceAddOn, LogHealthReport
-from deploy_board.webapp.agent_report import UNKNOWN_HOSTS_CODE, PROVISION_HOST_CODE
+from deploy_board.webapp.agent_report import DEFAULT_STALE_THRESHOLD, UNKNOWN_HOSTS_CODE, PROVISION_HOST_CODE
 from deploy_board.webapp.common import is_agent_failed, BUILD_STAGE
 from deploy_board.webapp.helpers import environs_helper
 from deploy_board.webapp.helpers.tags_helper import TagValue
@@ -468,11 +468,11 @@ def successRatePercentage(deploy):
             return trunc(deploy["subAcctSucHostNum"] * 100 / deploy["subAcctTotalHostNum"])
     elif deploy.get("account") == AWS_PRIMARY_ACCOUNT:
         if deploy["primaryAcctTotalHostNum"] != 0:
-            return trunc(deploy["primaryAcctSucHostNum"] * 100 / deploy["primaryAcctTotalHostNum"])   
+            return trunc(deploy["primaryAcctSucHostNum"] * 100 / deploy["primaryAcctTotalHostNum"])
     elif deploy.get("account") == "others":
         if deploy["otherAcctTotalHostNum"] != 0:
-            return trunc(deploy["otherAcctSucHostNum"] * 100 / deploy["otherAcctTotalHostNum"])   
-    else:     
+            return trunc(deploy["otherAcctSucHostNum"] * 100 / deploy["otherAcctTotalHostNum"])
+    else:
         if deploy["total"] != 0:
             return trunc(deploy["successTotal"] * 100 / deploy["total"])
     return 0
@@ -634,10 +634,14 @@ def agentIcon(agentStats):
         return 'fa-recycle fa-spin'
 
     # normal state
-    if agent['deployStage'] == "SERVING_BUILD" and agentStats.isCurrent:
-        return 'fa-check'
-
     if agentStats.isCurrent:
+        if agent['deployStage'] == "SERVING_BUILD":
+            return 'fa-check'
+        if agentStats.isStale or agentStats.isHostFailed or (
+                agent['state']== "PROVISIONED" and
+                agent["ip"] is None
+                ):
+            return 'fa-exclamation-triangle'
         return 'fa-spinner fa-spin'
 
     return 'fa-clock-o'
@@ -653,8 +657,12 @@ def hostButton(host):
 
 @register.filter("hostIcon")
 def hostIcon(host):
-    if host['state'] == 'PROVISIONED':
-        return 'fa-refresh fa-spin'
+    if host["state"] == "PROVISIONED":
+        duration = (time.time() * 1000 - host["lastUpdateDate"]) / 1000
+        isStale = duration > DEFAULT_STALE_THRESHOLD * 3
+        if isStale and host["ip"] is None:
+            return "fa-exclamation-triangle"
+        return "fa-refresh fa-spin"
 
     if host['state'] == 'ACTIVE':
         return 'fa-check-square-o'