Merge pull request #259 from brunoasr/configurable_envtypes

Configurable environment types for Ami tag management
pinterest · Feb 26, 2024 · 0ad29fc · 0ad29fc
2 parents b8cd232 + e8801d3
commit 0ad29fc
Show file tree

Hide file tree

Showing 9 changed files with 129 additions and 75 deletions.
diff --git a/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java b/orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java
@@ -24,6 +24,7 @@
 import com.pinterest.orion.core.automation.operator.OperatorFactory;
 import com.pinterest.orion.core.automation.sensor.SensorFactory;
 import com.pinterest.orion.core.metrics.MetricsStore;
+import com.pinterest.orion.server.config.OrionConf;
 import com.pinterest.orion.server.config.OrionPluginConfig;
 
 public class ClusterManager {
@@ -37,6 +38,7 @@ public class ClusterManager {
   private ClusterStateSink stateSink;
   private MetricsStore metricsStore;
   private CostCalculator costCalculator;
+  private OrionConf configuration;
 
   public ClusterManager(SensorFactory sensorFactory,
                         OperatorFactory operatorFactory,
@@ -116,5 +118,20 @@ public CostCalculator getCostCalculator() {
   public MetricsStore getMetricsStore() {
     return metricsStore;
   }
+
+  /**
+   * @return the configuration
+   */
+  public OrionConf getOrionConf() {
+    return configuration;
+  }
+
+  /**
+   * @param configuration the Orion server configuration
+   */
+  public void setOrionConf(OrionConf configuration) {
+    this.configuration = configuration;
+    return;
+  }
 
 }
diff --git a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java
@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
+import java.util.function.Function;
 import java.util.function.UnaryOperator;
 
 import com.pinterest.orion.server.api.Ami;
@@ -34,6 +35,7 @@
 import software.amazon.awssdk.services.ec2.model.CreateTagsRequest;
 import software.amazon.awssdk.services.ec2.model.CreateTagsResponse;
 import software.amazon.awssdk.services.ec2.model.Filter;
+import software.amazon.awssdk.services.ec2.model.Image;
 import software.amazon.awssdk.services.ec2.model.Tag;
 
 /**
@@ -52,6 +54,7 @@ public class AmiTagManager {
   public static final String KEY_APPLICATION_ENVIRONMENT = "application_environment";
   public static final String VALUE_KAFKA = "kafka";
   public static UnaryOperator<String> tag = key -> "tag:" + key;
+  public static final String ENV_TYPES_KEY = "envTypes";
 
   public AmiTagManager() {
     ec2Client = Ec2Client.create();
@@ -66,26 +69,38 @@ public AmiTagManager() {
   public List<Ami> getAmiList(Map<String, String> filter) {
     List<Ami> amiList = new ArrayList<>();
     DescribeImagesRequest.Builder builder = DescribeImagesRequest.builder();
-    builder = builder.filters(
-      Filter.builder().name(tag.apply(KEY_APPLICATION)).values(VALUE_KAFKA).build()
+    Filter.Builder filterBuilder = Filter.builder();
+    List<Filter> filterList = new ArrayList<>();
+    filterList.add(
+      filterBuilder.name(tag.apply(KEY_APPLICATION))
+        .values(VALUE_KAFKA)
+        .build()
     );
     if (filter.containsKey(KEY_RELEASE))
-      builder = builder.filters(
-        Filter.builder().name(tag.apply(KEY_RELEASE)).values(filter.get(KEY_RELEASE)).build()
+      filterList.add(
+        filterBuilder.name(tag.apply(KEY_RELEASE))
+          .values(filter.get(KEY_RELEASE))
+          .build()
       );
     if (filter.containsKey(KEY_CPU_ARCHITECTURE))
-      builder = builder.filters(
-        Filter.builder().name(tag.apply(KEY_CPU_ARCHITECTURE)).values(filter.get(KEY_CPU_ARCHITECTURE)).build()
+      filterList.add(
+        filterBuilder.name(tag.apply(KEY_CPU_ARCHITECTURE))
+          .values(filter.get(KEY_CPU_ARCHITECTURE))
+          .build()
       );
-    builder = builder.filters(
-      Filter.builder().name(tag.apply(KEY_APPLICATION_ENVIRONMENT)).values("*").build()
+    filterList.add(
+      filterBuilder.name(tag.apply(KEY_APPLICATION_ENVIRONMENT))
+        .values("*")
+        .build()
     );
+    builder = builder.filters(filterList);
     try {
       DescribeImagesResponse resp = ec2Client.describeImages(builder.build());
       if (resp.hasImages() && !resp.images().isEmpty()) {
-        ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180);
+        // The limitation of images newer than 180 days is temporarily suspended
+        //ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180);
         resp.images().forEach(image -> {
-          if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) {
+          /*if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) {*/
             Iterator<Tag> i = image.tags().iterator();
             Tag t;
             String appEnvTag = null;
@@ -101,10 +116,10 @@ public List<Ami> getAmiList(Map<String, String> filter) {
               appEnvTag,
               image.creationDate()
             ));
-          }
+          // }
         });
-        amiList.sort((a, b) -> - ZonedDateTime.parse(a.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME)
-        .compareTo(ZonedDateTime.parse(b.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME)));
+        Function<Ami, ZonedDateTime> parse = i -> ZonedDateTime.parse(i.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME);
+        amiList.sort((a, b) -> - parse.apply(a).compareTo(parse.apply(b)));
       }
     } catch (Exception e) {
       logger.log(Level.SEVERE, "AmiTagManager: could not retrieve AMI list", e);

diff --git a/...n-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java b/...n-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java
@@ -532,7 +532,7 @@ public AttributeSchema generateSchema(Map<String, Object> config) {
                     .addOption("m6id.4xlarge", "m6id.4xlarge")
                     .addOption("m6id.8xlarge", "m6id.8xlarge")
         )
-        .addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will inherit current AMI if not provided)", false))
+        .addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will use cluster filter criteria if not provided)", false))
         .addSchema(super.generateSchema(config));
   }
 }
diff --git a/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java b/orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java
@@ -16,7 +16,6 @@
 package com.pinterest.orion.server;
 
 import java.io.IOException;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;

diff --git a/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java b/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java
@@ -142,6 +142,18 @@ public void updateImageTag(
     amiTagManager.updateAmiTag(amiId, applicationEnvironment);
   }
 
+
+  @Path("/getEnvTypes")
+  @GET
+  public List<String> getEnvTypes() {
+    List<String> envTypes = null;
+    Map<String, Object> additionalConfigs = mgr.getOrionConf().getAdditionalConfigs();
+    if(additionalConfigs != null && additionalConfigs.containsKey(AmiTagManager.ENV_TYPES_KEY)) {
+      envTypes = (List<String>) additionalConfigs.get(AmiTagManager.ENV_TYPES_KEY);
+    }
+    return envTypes;
+  }
+
   @RolesAllowed({ OrionConf.ADMIN_ROLE, OrionConf.MGMT_ROLE })
   @Path("/costByCluster")
   @GET

diff --git a/orion-server/src/main/resources/webapp/src/actions/cluster.js b/orion-server/src/main/resources/webapp/src/actions/cluster.js
@@ -33,6 +33,8 @@ export const COST_RECEIVED = "COST_RECEIVED";
 export const AMI_LIST_REQUESTED = "AMI_LIST_REQUESTED";
 export const AMI_LIST_RECEIVED = "AMI_LIST_RECEIVED";
 export const AMI_TAG_UPDATE = "AMI_TAG_UPDATE";
+export const ENV_TYPES_REQUESTED = "ENV_TYPES_REQUESTED";
+export const ENV_TYPES_RECEIVED = "ENV_TYPES_RECEIVED";
 
 export function requestCluster(clusterId) {
   return { type: CLUSTER_REQUESTED, payload: { clusterId } };
@@ -134,3 +136,17 @@ export function updateAmiTag(amiId, applicationEnvironment) {
     payload: { amiId, applicationEnvironment },
   };
 }
+
+export function requestEnvTypes() {
+  return {
+    type: ENV_TYPES_REQUESTED,
+    payload: {},
+  };
+}
+
+export function receiveEnvTypes(envTypeList) {
+  return {
+    type: ENV_TYPES_RECEIVED,
+    payload: { envTypeList },
+  };
+}
diff --git a/orion-server/src/main/resources/webapp/src/basic-components/Ami.js b/orion-server/src/main/resources/webapp/src/basic-components/Ami.js
@@ -19,19 +19,21 @@ import { Button, FormControl, Grid, InputLabel, MenuItem, Select, TextField,
   FormGroup, FormControlLabel, Checkbox } from '@material-ui/core';
 import { makeStyles } from "@material-ui/core/styles";
 import { connect } from "react-redux";
-import { requestAmiList, updateAmiTag } from "../actions/cluster";
+import { requestAmiList, updateAmiTag, requestEnvTypes } from "../actions/cluster";
 
 const mapState = (state, ownProps) => {
-  const { amiList } = state.app;
+  const { amiList, envTypes } = state.app;
   return {
     ...ownProps,
     amiList,
+    envTypes
   };
 };
 
 const mapDispatch = {
   requestAmiList,
-  updateAmiTag
+  requestEnvTypes,
+  updateAmiTag,
 };
 
 const useStyles = makeStyles(theme => ({
@@ -41,7 +43,7 @@ const useStyles = makeStyles(theme => ({
   },
 }));
 
-function Ami({ amiList, requestAmiList, updateAmiTag }) {
+function Ami({ amiList, requestAmiList, envTypes, requestEnvTypes, updateAmiTag }) {
   const classes = useStyles();
   const [os, setOS] = React.useState();
   const handleOSChange = event => {
@@ -65,23 +67,14 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
   const handleAppEnvChange = event => {
     setAppEnv(event.target.value);
   };
-  const [env] = React.useState({
-    dev: false,
-    test: false,
-    staging: false,
-    prod: false,
-  });
+  const envMap = {};
+  if (envTypes !== undefined)
+    envTypes.forEach(value => { envMap[value] = false; });
+  const [env] = React.useState(envMap);
   const handleCheckboxChange = (event) => {
     env[event.target.name] = event.target.checked;
     const newAppEnv = [];
-    if (env.dev)
-      newAppEnv.push("dev");
-    if (env.test)
-      newAppEnv.push("test");
-    if (env.staging)
-      newAppEnv.push("staging");
-    if (env.prod)
-      newAppEnv.push("prod");
+    envTypes.forEach(envType => { if (env[envType]) newAppEnv.push(envType); });
     setAppEnv(newAppEnv.join(','));
   };
   const applyFilter = () => {
@@ -91,10 +84,13 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
     if (cpuArch)
       parms.push("cpu_architecture=" + cpuArch);
     requestAmiList(parms.join('&'));
+    requestEnvTypes();
   }
 
   if (!amiList)
     amiList = [];
+  if (!envTypes)
+    envTypes = [];
   return (
     <div>
       <Grid container spacing={3}>
@@ -207,46 +203,18 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
           </div>
           <div>
             <FormGroup column>
-              <FormControlLabel
-                control={
-                  <Checkbox
-                    checked={env.dev}
-                    onChange={handleCheckboxChange}
-                    name="dev"
-                    color="primary"
-                  />}
-                label="dev"
-              />
-              <FormControlLabel
-                control={
-                  <Checkbox
-                    checked={env.test}
-                    onChange={handleCheckboxChange}
-                    name="test"
-                    color="primary"
-                  />}
-                label="test"
-              />
-              <FormControlLabel
-                control={
-                  <Checkbox
-                    checked={env.staging}
-                    onChange={handleCheckboxChange}
-                    name="staging"
-                    color="primary"
-                  />}
-                label="staging"
-              />
-              <FormControlLabel
-                control={
-                  <Checkbox
-                    checked={env.prod}
-                    onChange={handleCheckboxChange}
-                    name="prod"
-                    color="primary"
-                  />}
-                label="prod"
-              />
+              { envTypes.map((envType) => (
+                <FormControlLabel
+                  control={
+                    <Checkbox
+                      checked={env[envType]}
+                      onChange={handleCheckboxChange}
+                      name={envType}
+                      color="primary"
+                    />}
+                  label={envType}
+                />
+              ))}
             </FormGroup>
           </div>
           <div>

diff --git a/orion-server/src/main/resources/webapp/src/reducers/app.js b/orion-server/src/main/resources/webapp/src/reducers/app.js
@@ -22,7 +22,12 @@ import {
   AUTO_REFRESH_ENABLED,
   AUTO_REFRESH_DISABLED,
 } from "../actions/app";
-import { UTILIZATION_RECEIVED, COST_RECEIVED, AMI_LIST_RECEIVED } from "../actions/cluster";
+import {
+  UTILIZATION_RECEIVED,
+  COST_RECEIVED,
+  AMI_LIST_RECEIVED,
+  ENV_TYPES_RECEIVED,
+} from "../actions/cluster";
 
 export default function showError(
   state = {
@@ -54,6 +59,8 @@ export default function showError(
       return { ...state, cost: action.payload.cost };
     case AMI_LIST_RECEIVED:
       return { ...state, amiList: action.payload.amiList };
+    case ENV_TYPES_RECEIVED:
+      return { ...state, envTypes: action.payload.envTypeList };
     default:
       return state;
   }

diff --git a/orion-server/src/main/resources/webapp/src/sagas/index.js b/orion-server/src/main/resources/webapp/src/sagas/index.js
@@ -40,7 +40,9 @@ import {
   receiveCost,
   AMI_LIST_REQUESTED,
   receiveAmiList,
-  AMI_TAG_UPDATE
+  AMI_TAG_UPDATE,
+  ENV_TYPES_REQUESTED,
+  receiveEnvTypes
 } from "../actions/cluster";
 import {
   CLUSTERS_SUMMARY_REQUESTED,
@@ -71,6 +73,7 @@ export default function* rootSaga() {
   yield fork(globalSensorWatcher);
   yield fork(amiListWatcher);
   yield fork(amiTagUpdateWatcher);
+  yield fork(envTypesWatcher);
 }
 
 function* clusterSummaryWatcher() {
@@ -105,6 +108,10 @@ function* amiTagUpdateWatcher() {
   yield takeEvery(AMI_TAG_UPDATE, fetchAmiTagUpdate);
 }
 
+function* envTypesWatcher() {
+  yield takeEvery(ENV_TYPES_REQUESTED, fetchEnvTypes);
+}
+
 function* fetchCost() {
   try {
     const resp = yield fetch("/api/costByCluster");
@@ -268,3 +275,16 @@ function* fetchAmiTagUpdate(action) {
     yield put(hideLoading());
   }
 }
+
+function* fetchEnvTypes() {
+  try {
+    yield put(showLoading());
+    const resp = yield call(fetch, "/api/getEnvTypes");
+    const data = yield resp.json();
+    yield put(receiveEnvTypes(data));
+  } catch (e) {
+    yield put(showAppError(e));
+  } finally {
+    yield put(hideLoading());
+  }
+}