diff --git a/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java b/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java index acbb242..89e84dc 100644 --- a/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java +++ b/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java @@ -24,6 +24,7 @@ import com.pinterest.orion.core.automation.operator.OperatorFactory; import com.pinterest.orion.core.automation.sensor.SensorFactory; import com.pinterest.orion.core.metrics.MetricsStore; +import com.pinterest.orion.server.config.OrionConf; import com.pinterest.orion.server.config.OrionPluginConfig; public class ClusterManager { @@ -37,6 +38,7 @@ public class ClusterManager { private ClusterStateSink stateSink; private MetricsStore metricsStore; private CostCalculator costCalculator; + private OrionConf configuration; public ClusterManager(SensorFactory sensorFactory, OperatorFactory operatorFactory, @@ -116,5 +118,20 @@ public CostCalculator getCostCalculator() { public MetricsStore getMetricsStore() { return metricsStore; } + + /** + * @return the configuration + */ + public OrionConf getOrionConf() { + return configuration; + } + + /** + * @param configuration the Orion server configuration + */ + public void setOrionConf(OrionConf configuration) { + this.configuration = configuration; + return; + } } diff --git a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java index 9345a0a..88ad1e9 100644 --- a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java +++ b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.function.Function; import java.util.function.UnaryOperator; import com.pinterest.orion.server.api.Ami; @@ -34,6 +35,7 @@ import software.amazon.awssdk.services.ec2.model.CreateTagsRequest; import software.amazon.awssdk.services.ec2.model.CreateTagsResponse; import software.amazon.awssdk.services.ec2.model.Filter; +import software.amazon.awssdk.services.ec2.model.Image; import software.amazon.awssdk.services.ec2.model.Tag; /** @@ -52,6 +54,7 @@ public class AmiTagManager { public static final String KEY_APPLICATION_ENVIRONMENT = "application_environment"; public static final String VALUE_KAFKA = "kafka"; public static UnaryOperator tag = key -> "tag:" + key; + public static final String ENV_TYPES_KEY = "envTypes"; public AmiTagManager() { ec2Client = Ec2Client.create(); @@ -66,26 +69,38 @@ public AmiTagManager() { public List getAmiList(Map filter) { List amiList = new ArrayList<>(); DescribeImagesRequest.Builder builder = DescribeImagesRequest.builder(); - builder = builder.filters( - Filter.builder().name(tag.apply(KEY_APPLICATION)).values(VALUE_KAFKA).build() + Filter.Builder filterBuilder = Filter.builder(); + List filterList = new ArrayList<>(); + filterList.add( + filterBuilder.name(tag.apply(KEY_APPLICATION)) + .values(VALUE_KAFKA) + .build() ); if (filter.containsKey(KEY_RELEASE)) - builder = builder.filters( - Filter.builder().name(tag.apply(KEY_RELEASE)).values(filter.get(KEY_RELEASE)).build() + filterList.add( + filterBuilder.name(tag.apply(KEY_RELEASE)) + .values(filter.get(KEY_RELEASE)) + .build() ); if (filter.containsKey(KEY_CPU_ARCHITECTURE)) - builder = builder.filters( - Filter.builder().name(tag.apply(KEY_CPU_ARCHITECTURE)).values(filter.get(KEY_CPU_ARCHITECTURE)).build() + filterList.add( + filterBuilder.name(tag.apply(KEY_CPU_ARCHITECTURE)) + .values(filter.get(KEY_CPU_ARCHITECTURE)) + .build() ); - builder = builder.filters( - Filter.builder().name(tag.apply(KEY_APPLICATION_ENVIRONMENT)).values("*").build() + filterList.add( + filterBuilder.name(tag.apply(KEY_APPLICATION_ENVIRONMENT)) + .values("*") + .build() ); + builder = builder.filters(filterList); try { DescribeImagesResponse resp = ec2Client.describeImages(builder.build()); if (resp.hasImages() && !resp.images().isEmpty()) { - ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180); + // The limitation of images newer than 180 days is temporarily suspended + //ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180); resp.images().forEach(image -> { - if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) { + /*if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) {*/ Iterator i = image.tags().iterator(); Tag t; String appEnvTag = null; @@ -101,10 +116,10 @@ public List getAmiList(Map filter) { appEnvTag, image.creationDate() )); - } + // } }); - amiList.sort((a, b) -> - ZonedDateTime.parse(a.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME) - .compareTo(ZonedDateTime.parse(b.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME))); + Function parse = i -> ZonedDateTime.parse(i.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME); + amiList.sort((a, b) -> - parse.apply(a).compareTo(parse.apply(b))); } } catch (Exception e) { logger.log(Level.SEVERE, "AmiTagManager: could not retrieve AMI list", e); diff --git a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java index 67ad989..aa47b28 100644 --- a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java +++ b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java @@ -532,7 +532,7 @@ public AttributeSchema generateSchema(Map config) { .addOption("m6id.4xlarge", "m6id.4xlarge") .addOption("m6id.8xlarge", "m6id.8xlarge") ) - .addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will inherit current AMI if not provided)", false)) + .addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will use cluster filter criteria if not provided)", false)) .addSchema(super.generateSchema(config)); } } diff --git a/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java b/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java index 9e0f5cc..bafa2ac 100644 --- a/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java +++ b/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java @@ -16,7 +16,6 @@ package com.pinterest.orion.server; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; diff --git a/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java b/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java index c80643f..8316b7f 100644 --- a/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java +++ b/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java @@ -142,6 +142,18 @@ public void updateImageTag( amiTagManager.updateAmiTag(amiId, applicationEnvironment); } + + @Path("/getEnvTypes") + @GET + public List getEnvTypes() { + List envTypes = null; + Map additionalConfigs = mgr.getOrionConf().getAdditionalConfigs(); + if(additionalConfigs != null && additionalConfigs.containsKey(AmiTagManager.ENV_TYPES_KEY)) { + envTypes = (List) additionalConfigs.get(AmiTagManager.ENV_TYPES_KEY); + } + return envTypes; + } + @RolesAllowed({ OrionConf.ADMIN_ROLE, OrionConf.MGMT_ROLE }) @Path("/costByCluster") @GET diff --git a/orion-server/src/main/resources/webapp/src/actions/cluster.js b/orion-server/src/main/resources/webapp/src/actions/cluster.js index d06b64b..adf9f15 100644 --- a/orion-server/src/main/resources/webapp/src/actions/cluster.js +++ b/orion-server/src/main/resources/webapp/src/actions/cluster.js @@ -33,6 +33,8 @@ export const COST_RECEIVED = "COST_RECEIVED"; export const AMI_LIST_REQUESTED = "AMI_LIST_REQUESTED"; export const AMI_LIST_RECEIVED = "AMI_LIST_RECEIVED"; export const AMI_TAG_UPDATE = "AMI_TAG_UPDATE"; +export const ENV_TYPES_REQUESTED = "ENV_TYPES_REQUESTED"; +export const ENV_TYPES_RECEIVED = "ENV_TYPES_RECEIVED"; export function requestCluster(clusterId) { return { type: CLUSTER_REQUESTED, payload: { clusterId } }; @@ -134,3 +136,17 @@ export function updateAmiTag(amiId, applicationEnvironment) { payload: { amiId, applicationEnvironment }, }; } + +export function requestEnvTypes() { + return { + type: ENV_TYPES_REQUESTED, + payload: {}, + }; +} + +export function receiveEnvTypes(envTypeList) { + return { + type: ENV_TYPES_RECEIVED, + payload: { envTypeList }, + }; +} diff --git a/orion-server/src/main/resources/webapp/src/basic-components/Ami.js b/orion-server/src/main/resources/webapp/src/basic-components/Ami.js index d92bcc3..b4880c9 100644 --- a/orion-server/src/main/resources/webapp/src/basic-components/Ami.js +++ b/orion-server/src/main/resources/webapp/src/basic-components/Ami.js @@ -19,19 +19,21 @@ import { Button, FormControl, Grid, InputLabel, MenuItem, Select, TextField, FormGroup, FormControlLabel, Checkbox } from '@material-ui/core'; import { makeStyles } from "@material-ui/core/styles"; import { connect } from "react-redux"; -import { requestAmiList, updateAmiTag } from "../actions/cluster"; +import { requestAmiList, updateAmiTag, requestEnvTypes } from "../actions/cluster"; const mapState = (state, ownProps) => { - const { amiList } = state.app; + const { amiList, envTypes } = state.app; return { ...ownProps, amiList, + envTypes }; }; const mapDispatch = { requestAmiList, - updateAmiTag + requestEnvTypes, + updateAmiTag, }; const useStyles = makeStyles(theme => ({ @@ -41,7 +43,7 @@ const useStyles = makeStyles(theme => ({ }, })); -function Ami({ amiList, requestAmiList, updateAmiTag }) { +function Ami({ amiList, requestAmiList, envTypes, requestEnvTypes, updateAmiTag }) { const classes = useStyles(); const [os, setOS] = React.useState(); const handleOSChange = event => { @@ -65,23 +67,14 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) { const handleAppEnvChange = event => { setAppEnv(event.target.value); }; - const [env] = React.useState({ - dev: false, - test: false, - staging: false, - prod: false, - }); + const envMap = {}; + if (envTypes !== undefined) + envTypes.forEach(value => { envMap[value] = false; }); + const [env] = React.useState(envMap); const handleCheckboxChange = (event) => { env[event.target.name] = event.target.checked; const newAppEnv = []; - if (env.dev) - newAppEnv.push("dev"); - if (env.test) - newAppEnv.push("test"); - if (env.staging) - newAppEnv.push("staging"); - if (env.prod) - newAppEnv.push("prod"); + envTypes.forEach(envType => { if (env[envType]) newAppEnv.push(envType); }); setAppEnv(newAppEnv.join(',')); }; const applyFilter = () => { @@ -91,10 +84,13 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) { if (cpuArch) parms.push("cpu_architecture=" + cpuArch); requestAmiList(parms.join('&')); + requestEnvTypes(); } if (!amiList) amiList = []; + if (!envTypes) + envTypes = []; return (
@@ -207,46 +203,18 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
- } - label="dev" - /> - } - label="test" - /> - } - label="staging" - /> - } - label="prod" - /> + { envTypes.map((envType) => ( + } + label={envType} + /> + ))}
diff --git a/orion-server/src/main/resources/webapp/src/reducers/app.js b/orion-server/src/main/resources/webapp/src/reducers/app.js index 8ca7362..542e7d6 100644 --- a/orion-server/src/main/resources/webapp/src/reducers/app.js +++ b/orion-server/src/main/resources/webapp/src/reducers/app.js @@ -22,7 +22,12 @@ import { AUTO_REFRESH_ENABLED, AUTO_REFRESH_DISABLED, } from "../actions/app"; -import { UTILIZATION_RECEIVED, COST_RECEIVED, AMI_LIST_RECEIVED } from "../actions/cluster"; +import { + UTILIZATION_RECEIVED, + COST_RECEIVED, + AMI_LIST_RECEIVED, + ENV_TYPES_RECEIVED, +} from "../actions/cluster"; export default function showError( state = { @@ -54,6 +59,8 @@ export default function showError( return { ...state, cost: action.payload.cost }; case AMI_LIST_RECEIVED: return { ...state, amiList: action.payload.amiList }; + case ENV_TYPES_RECEIVED: + return { ...state, envTypes: action.payload.envTypeList }; default: return state; } diff --git a/orion-server/src/main/resources/webapp/src/sagas/index.js b/orion-server/src/main/resources/webapp/src/sagas/index.js index e1b8445..53b0a96 100644 --- a/orion-server/src/main/resources/webapp/src/sagas/index.js +++ b/orion-server/src/main/resources/webapp/src/sagas/index.js @@ -40,7 +40,9 @@ import { receiveCost, AMI_LIST_REQUESTED, receiveAmiList, - AMI_TAG_UPDATE + AMI_TAG_UPDATE, + ENV_TYPES_REQUESTED, + receiveEnvTypes } from "../actions/cluster"; import { CLUSTERS_SUMMARY_REQUESTED, @@ -71,6 +73,7 @@ export default function* rootSaga() { yield fork(globalSensorWatcher); yield fork(amiListWatcher); yield fork(amiTagUpdateWatcher); + yield fork(envTypesWatcher); } function* clusterSummaryWatcher() { @@ -105,6 +108,10 @@ function* amiTagUpdateWatcher() { yield takeEvery(AMI_TAG_UPDATE, fetchAmiTagUpdate); } +function* envTypesWatcher() { + yield takeEvery(ENV_TYPES_REQUESTED, fetchEnvTypes); +} + function* fetchCost() { try { const resp = yield fetch("/api/costByCluster"); @@ -268,3 +275,16 @@ function* fetchAmiTagUpdate(action) { yield put(hideLoading()); } } + +function* fetchEnvTypes() { + try { + yield put(showLoading()); + const resp = yield call(fetch, "/api/getEnvTypes"); + const data = yield resp.json(); + yield put(receiveEnvTypes(data)); + } catch (e) { + yield put(showAppError(e)); + } finally { + yield put(hideLoading()); + } +}