-
Notifications
You must be signed in to change notification settings - Fork 250
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into touyang/autopromote
- Loading branch information
Showing
25 changed files
with
866 additions
and
130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
103 changes: 103 additions & 0 deletions
103
...rvice/common/src/main/java/com/pinterest/deployservice/metrics/DefaultHostClassifier.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| package com.pinterest.deployservice.metrics; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.Collection; | ||
| import java.util.HashMap; | ||
| import java.util.ArrayList; | ||
| import java.util.Map; | ||
|
|
||
| import javax.annotation.Nonnull; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import com.google.common.collect.Maps; | ||
| import com.pinterest.deployservice.bean.HostBean; | ||
|
|
||
| public class DefaultHostClassifier implements HostClassifier { | ||
| private static final Logger LOG = LoggerFactory.getLogger(DefaultHostClassifier.class); | ||
|
|
||
| private List<HostBean> carryoverHosts = new ArrayList<>(); | ||
| private List<HostBean> newHosts = new ArrayList<>(); | ||
| private List<HostBean> removedHosts = new ArrayList<>(); | ||
| private List<HostBean> timeoutHosts = new ArrayList<>(); | ||
| private @Nonnull List<HostBean> initializingHosts = new ArrayList<>(); | ||
|
|
||
| @Override | ||
| public List<HostBean> getTimeoutHosts() { | ||
| return timeoutHosts; | ||
| } | ||
|
|
||
| @Override | ||
| public List<HostBean> getRemovedHosts() { | ||
| return removedHosts; | ||
| } | ||
|
|
||
| @Override | ||
| public List<HostBean> getNewHosts() { | ||
| return newHosts; | ||
| } | ||
|
|
||
| @Override | ||
| public List<HostBean> getCarryoverHosts() { | ||
| return carryoverHosts; | ||
| } | ||
|
|
||
| @Override | ||
| public List<HostBean> getInitializingHosts() { | ||
| return initializingHosts; | ||
| } | ||
|
|
||
| private Map<String, HostBean> getInitializingHostMap() { | ||
| return Maps.uniqueIndex(initializingHosts, HostBean::getHost_id); | ||
| } | ||
|
|
||
| @Override | ||
| public void updateClassification(Collection<HostBean> agentlessHosts, Instant timeoutInstant) { | ||
| Map<String, HostBean> uniqueAgentlessHostMap = deduplicateHosts(agentlessHosts); | ||
| Map<String, HostBean> previousInitializingHosts = getInitializingHostMap(); | ||
| Map<String, HostBean> removedHostMap = new HashMap<>(previousInitializingHosts); | ||
|
|
||
| List<HostBean> newTimeoutHosts = new ArrayList<>(); | ||
| List<HostBean> newlyLaunchedHosts = new ArrayList<>(); | ||
| List<HostBean> newCarryoverHosts = new ArrayList<>(); | ||
|
|
||
| initializingHosts = new ArrayList<>(uniqueAgentlessHostMap.values()); | ||
| for (HostBean host : initializingHosts) { | ||
| removedHostMap.remove(host.getHost_id()); | ||
| Instant hostCreationInstant = Instant.ofEpochMilli(host.getCreate_date()); | ||
| if (hostCreationInstant.isBefore(timeoutInstant)) { | ||
| newTimeoutHosts.add(host); | ||
| } | ||
| if (previousInitializingHosts.containsKey(host.getHost_id())) { | ||
| newCarryoverHosts.add(host); | ||
| } else { | ||
| newlyLaunchedHosts.add(host); | ||
| } | ||
| } | ||
|
|
||
| removedHosts = new ArrayList<>(removedHostMap.values()); | ||
| carryoverHosts = newCarryoverHosts; | ||
| timeoutHosts = newTimeoutHosts; | ||
| newHosts = newlyLaunchedHosts; | ||
|
|
||
| LOG.info( | ||
| "Host classification of {} agentless hosts based on {} previous initializing hosts: {} new, {} carryover, {} removed, {} timeout, {} initializing", | ||
| agentlessHosts.size(), previousInitializingHosts.size(), newHosts.size(), carryoverHosts.size(), | ||
| removedHosts.size(), timeoutHosts.size(), initializingHosts.size()); | ||
| } | ||
|
|
||
| private Map<String, HostBean> deduplicateHosts(Collection<HostBean> agentlessHosts) { | ||
| Map<String, HostBean> uniqueHosts = new HashMap<>(); | ||
| for (HostBean host : agentlessHosts) { | ||
| if (!uniqueHosts.containsKey(host.getHost_id()) || | ||
| host.getCreate_date() < uniqueHosts.get(host.getHost_id()).getCreate_date()) { | ||
| uniqueHosts.put(host.getHost_id(), host); | ||
| } | ||
| } | ||
| return uniqueHosts; | ||
| } | ||
|
|
||
| } |
68 changes: 68 additions & 0 deletions
68
deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/HostClassifier.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package com.pinterest.deployservice.metrics; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.Collection; | ||
| import java.util.List; | ||
|
|
||
| import com.pinterest.deployservice.bean.HostBean; | ||
|
|
||
| public interface HostClassifier { | ||
|
|
||
| /** | ||
| * Retrieves hosts that are newly added. | ||
| * | ||
| * Note this is a subset of hosts that are initializing. | ||
| * | ||
| * @return a list of newly added hosts | ||
| */ | ||
| List<HostBean> getNewHosts(); | ||
|
|
||
| /** | ||
| * Retrieves hosts that are carried over from last update. | ||
| * | ||
| * Note this is a subset of hosts that are initializing. | ||
| * | ||
| * @return a list of carried over hosts | ||
| */ | ||
| List<HostBean> getCarryoverHosts(); | ||
|
|
||
| /** | ||
| * Retrieves hosts that have timed out. | ||
| * | ||
| * Note this is a subset of hosts that are initializing. | ||
| * | ||
| * @return a list of hosts that have timed out | ||
| */ | ||
| List<HostBean> getTimeoutHosts(); | ||
|
|
||
| /** | ||
| * Retrieves hosts that have been removed. | ||
| * | ||
| * Specifically, a previously initializing host that is no longer in the | ||
| * provided host list. | ||
| * A host can be absent from the provided host list for 2 reasons: | ||
| * 1. It has been initialized | ||
| * 2. It has been taking too long to initialize | ||
| * | ||
| * @return a list of hosts that have been removed | ||
| */ | ||
| List<HostBean> getRemovedHosts(); | ||
|
|
||
| /** | ||
| * Retrieves hosts that are currently initializing. | ||
| * | ||
| * Note this is the union of newly added and carryover hosts. | ||
| * | ||
| * @return a list of hosts that are currently initializing | ||
| */ | ||
| List<HostBean> getInitializingHosts(); | ||
|
|
||
| /** | ||
| * Updates the classification of hosts. | ||
| * | ||
| * @param hosts the collection of hosts to update the classification | ||
| * with | ||
| * @param timeoutInstant the instant used to determine the timeout hosts | ||
| */ | ||
| void updateClassification(Collection<HostBean> hosts, Instant timeoutInstant); | ||
| } |
11 changes: 0 additions & 11 deletions
11
deploy-service/common/src/main/java/com/pinterest/deployservice/metrics/MeterConstants.java
This file was deleted.
Oops, something went wrong.
17 changes: 17 additions & 0 deletions
17
deploy-service/common/src/test/java/com/pinterest/deployservice/bean/BeanUtils.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package com.pinterest.deployservice.bean; | ||
|
|
||
| import java.time.Instant; | ||
| import java.util.UUID; | ||
|
|
||
| public class BeanUtils { | ||
| public static HostBean createHostBean(Instant createDate) { | ||
| HostBean bean = new HostBean(); | ||
| bean.setHost_id("i-" + UUID.randomUUID().toString().substring(0, 8)); | ||
| bean.setGroup_name("testEnv-testStage"); | ||
| bean.setCreate_date(createDate.toEpochMilli()); | ||
| bean.setLast_update(createDate.plusSeconds(1).toEpochMilli()); | ||
| bean.setCan_retire(0); | ||
| bean.setState(HostState.PROVISIONED); | ||
| return bean; | ||
| } | ||
| } |
Oops, something went wrong.