Add LongTaskTimer to track launching hosts

commit-id:9e058a5f

deploy-service/common/pom.xml

@@ -19,6 +19,26 @@
         <dropwizard.metrics.version>3.1.0</dropwizard.metrics.version>
     </properties>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-jar-plugin</artifactId>
+                <version>3.3.0</version>
+                <configuration>
+                    <skip>false</skip>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>test-jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
     <dependencies>
         <dependency>
             <groupId>com.pinterest.teletraan</groupId>

deploy-service/common/src/main/java/com/pinterest/deployservice/dao/HostDAO.java

@@ -58,6 +58,8 @@ public interface HostDAO {
 
     List<String> getStaleAgentlessHostIds(long lastUpdateBefore, int limit) throws SQLException;
 
+    List<HostBean> getAgentlessHosts(long lastUpdateAfter, int limit) throws SQLException;
+
     Collection<HostBean> getHostsByEnvId(String envId) throws Exception;
 
     HostBean getByEnvIdAndHostId(String envId, String hostId) throws Exception;

deploy-service/common/src/main/java/com/pinterest/deployservice/db/DBHostDAOImpl.java

@@ -53,6 +53,7 @@ public class DBHostDAOImpl implements HostDAO {
     private static final String GET_HOSTS_BY_STATES = "SELECT * FROM hosts WHERE state in (?, ?, ?) GROUP BY host_id ORDER BY last_update";
     private static final String GET_GROUP_NAMES_BY_HOST = "SELECT group_name FROM hosts WHERE host_name=?";
     private static final String GET_STALE_AGENTLESS_HOST_IDS = "SELECT DISTINCT hosts.host_id FROM hosts LEFT JOIN hosts_and_agents ON hosts.host_id = hosts_and_agents.host_id WHERE hosts.last_update < ? AND hosts_and_agents.host_id IS NULL ORDER BY hosts.last_update DESC LIMIT ?";
+    private static final String GET_AGENTLESS_HOSTS = "SELECT hosts.* FROM hosts LEFT JOIN hosts_and_agents ON hosts.host_id = hosts_and_agents.host_id WHERE hosts.last_update > ? AND hosts_and_agents.host_id IS NULL ORDER BY hosts.last_update DESC LIMIT ?";
     private static final String GET_HOST_NAMES_BY_GROUP = "SELECT host_name FROM hosts WHERE group_name=?";
     private static final String GET_HOST_IDS_BY_GROUP = "SELECT DISTINCT host_id FROM hosts WHERE group_name=?";
     private static final String GET_HOSTS_BY_ENVID = "SELECT h.* FROM hosts h INNER JOIN groups_and_envs ge ON ge.group_name = h.group_name WHERE ge.env_id=? UNION DISTINCT SELECT hs.* FROM hosts hs INNER JOIN hosts_and_envs he ON he.host_name = hs.host_name WHERE he.env_id=?";
@@ -203,6 +204,13 @@ public List<String> getStaleAgentlessHostIds(long lastUpdateBefore, int limit) t
                 SingleResultSetHandlerFactory.<String>newListObjectHandler(), lastUpdateBefore, limit);
     }
 
+
+    @Override
+    public List<HostBean> getAgentlessHosts(long lastUpdateAfter, int limit) throws SQLException {
+        ResultSetHandler<List<HostBean>> h = new BeanListHandler<>(HostBean.class);
+        return new QueryRunner(dataSource).query(GET_AGENTLESS_HOSTS, h, lastUpdateAfter, limit);
+    }
+
     @Override
     public List<HostBean> getAllActiveHostsByGroup(String groupName) throws Exception {
         ResultSetHandler<List<HostBean>> h = new BeanListHandler<>(HostBean.class);

deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/DefaultHostClassifier.java

@@ -0,0 +1,103 @@
+package com.pinterest.deployservice.metrics;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Map;
+
+import javax.annotation.Nonnull;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.pinterest.deployservice.bean.HostBean;
+
+public class DefaultHostClassifier implements HostClassifier {
+    private static final Logger LOG = LoggerFactory.getLogger(DefaultHostClassifier.class);
+
+    private List<HostBean> carryoverHosts = new ArrayList<>();
+    private List<HostBean> newHosts = new ArrayList<>();
+    private List<HostBean> removedHosts = new ArrayList<>();
+    private List<HostBean> timeoutHosts = new ArrayList<>();
+    private @Nonnull List<HostBean> initializingHosts = new ArrayList<>();
+
+    @Override
+    public List<HostBean> getTimeoutHosts() {
+        return timeoutHosts;
+    }
+
+    @Override
+    public List<HostBean> getRemovedHosts() {
+        return removedHosts;
+    }
+
+    @Override
+    public List<HostBean> getNewHosts() {
+        return newHosts;
+    }
+
+    @Override
+    public List<HostBean> getCarryoverHosts() {
+        return carryoverHosts;
+    }
+
+    @Override
+    public List<HostBean> getInitializingHosts() {
+        return initializingHosts;
+    }
+
+    private Map<String, HostBean> getInitializingHostMap() {
+        return Maps.uniqueIndex(initializingHosts, HostBean::getHost_id);
+    }
+
+    @Override
+    public void updateClassification(Collection<HostBean> agentlessHosts, Instant timeoutInstant) {
+        Map<String, HostBean> uniqueAgentlessHostMap = deduplicateHosts(agentlessHosts);
+        Map<String, HostBean> previousInitializingHosts = getInitializingHostMap();
+        Map<String, HostBean> removedHostMap = new HashMap<>(previousInitializingHosts);
+
+        List<HostBean> newTimeoutHosts = new ArrayList<>();
+        List<HostBean> newlyLaunchedHosts = new ArrayList<>();
+        List<HostBean> newCarryoverHosts = new ArrayList<>();
+
+        initializingHosts = new ArrayList<>(uniqueAgentlessHostMap.values());
+        for (HostBean host : initializingHosts) {
+            removedHostMap.remove(host.getHost_id());
+            Instant hostCreationInstant = Instant.ofEpochMilli(host.getCreate_date());
+            if (hostCreationInstant.isBefore(timeoutInstant)) {
+                newTimeoutHosts.add(host);
+            }
+            if (previousInitializingHosts.containsKey(host.getHost_id())) {
+                newCarryoverHosts.add(host);
+            } else {
+                newlyLaunchedHosts.add(host);
+            }
+        }
+
+        removedHosts = new ArrayList<>(removedHostMap.values());
+        carryoverHosts = newCarryoverHosts;
+        timeoutHosts = newTimeoutHosts;
+        newHosts = newlyLaunchedHosts;
+
+        LOG.info(
+                "Host classification of {} agentless hosts based on {} previous initializing hosts: {} new, {} carryover, {} removed, {} timeout, {} initializing",
+                agentlessHosts.size(), previousInitializingHosts.size(), newHosts.size(), carryoverHosts.size(),
+                removedHosts.size(), timeoutHosts.size(), initializingHosts.size());
+    }
+
+    private Map<String, HostBean> deduplicateHosts(Collection<HostBean> agentlessHosts) {
+        Map<String, HostBean> uniqueHosts = new HashMap<>();
+        for (HostBean host : agentlessHosts) {
+            if (!uniqueHosts.containsKey(host.getHost_id()) ||
+                    host.getCreate_date() < uniqueHosts.get(host.getHost_id()).getCreate_date()) {
+                uniqueHosts.put(host.getHost_id(), host);
+            }
+        }
+        return uniqueHosts;
+    }
+
+}

deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/HostClassifier.java

@@ -0,0 +1,68 @@
+package com.pinterest.deployservice.metrics;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.List;
+
+import com.pinterest.deployservice.bean.HostBean;
+
+public interface HostClassifier {
+
+    /**
+     * Retrieves hosts that are newly added.
+     *
+     * Note this is a subset of hosts that are initializing.
+     *
+     * @return a list of newly added hosts
+     */
+    List<HostBean> getNewHosts();
+
+    /**
+     * Retrieves hosts that are carried over from last update.
+     *
+     * Note this is a subset of hosts that are initializing.
+     *
+     * @return a list of carried over hosts
+     */
+    List<HostBean> getCarryoverHosts();
+
+    /**
+     * Retrieves hosts that have timed out.
+     *
+     * Note this is a subset of hosts that are initializing.
+     *
+     * @return a list of hosts that have timed out
+     */
+    List<HostBean> getTimeoutHosts();
+
+    /**
+     * Retrieves hosts that have been removed.
+     *
+     * Specifically, a previously initializing host that is no longer in the
+     * provided host list.
+     * A host can be absent from the provided host list for 2 reasons:
+     * 1. It has been initialized
+     * 2. It has been taking too long to initialize
+     *
+     * @return a list of hosts that have been removed
+     */
+    List<HostBean> getRemovedHosts();
+
+    /**
+     * Retrieves hosts that are currently initializing.
+     *
+     * Note this is the union of newly added and carryover hosts.
+     *
+     * @return a list of hosts that are currently initializing
+     */
+    List<HostBean> getInitializingHosts();
+
+    /**
+     * Updates the classification of hosts.
+     *
+     * @param hosts          the collection of hosts to update the classification
+     *                       with
+     * @param timeoutInstant the instant used to determine the timeout hosts
+     */
+    void updateClassification(Collection<HostBean> hosts, Instant timeoutInstant);
+}

deploy-service/common/src/test/java/com/pinterest/deployservice/bean/BeanUtils.java

@@ -0,0 +1,17 @@
+package com.pinterest.deployservice.bean;
+
+import java.time.Instant;
+import java.util.UUID;
+
+public class BeanUtils {
+    public static HostBean createHostBean(Instant createDate) {
+        HostBean bean = new HostBean();
+        bean.setHost_id("i-" + UUID.randomUUID().toString().substring(0, 8));
+        bean.setGroup_name("testEnv-testStage");
+        bean.setCreate_date(createDate.toEpochMilli());
+        bean.setLast_update(createDate.plusSeconds(1).toEpochMilli());
+        bean.setCan_retire(0);
+        bean.setState(HostState.PROVISIONED);
+        return bean;
+    }
+}