diff --git a/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java new file mode 100644 index 00000000..9345a0a8 --- /dev/null +++ b/orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java @@ -0,0 +1,141 @@ +/******************************************************************************* + * Copyright 2024 Pinterest, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.pinterest.orion.core.actions.aws; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.function.UnaryOperator; + +import com.pinterest.orion.server.api.Ami; + +import software.amazon.awssdk.awscore.exception.AwsServiceException; +import software.amazon.awssdk.services.ec2.Ec2Client; +import software.amazon.awssdk.services.ec2.model.DescribeImagesRequest; +import software.amazon.awssdk.services.ec2.model.DescribeImagesResponse; +import software.amazon.awssdk.services.ec2.model.CreateTagsRequest; +import software.amazon.awssdk.services.ec2.model.CreateTagsResponse; +import software.amazon.awssdk.services.ec2.model.Filter; +import software.amazon.awssdk.services.ec2.model.Tag; + +/** + * AmiTagManager interfaces APIs in + * {@link com.pinterest.orion.server.api.ClusterManagerApi ClusterManagerApi} + * with AWS for AMI tag management. + */ +public class AmiTagManager { + private static final Logger logger = Logger.getLogger(AmiTagManager.class.getCanonicalName()); + private Ec2Client ec2Client; + public static final String KEY_IS_PUBLIC = "is-public"; + public static final String KEY_AMI_ID = "ami_id"; + public static final String KEY_APPLICATION = "application"; + public static final String KEY_RELEASE = "release"; + public static final String KEY_CPU_ARCHITECTURE = "cpu_architecture"; + public static final String KEY_APPLICATION_ENVIRONMENT = "application_environment"; + public static final String VALUE_KAFKA = "kafka"; + public static UnaryOperator tag = key -> "tag:" + key; + + public AmiTagManager() { + ec2Client = Ec2Client.create(); + } + + /** + * retrieve AMI list from cloud provider + * + * @param filter - map of criteria fields + * @return list of Ami objects + */ + public List getAmiList(Map filter) { + List amiList = new ArrayList<>(); + DescribeImagesRequest.Builder builder = DescribeImagesRequest.builder(); + builder = builder.filters( + Filter.builder().name(tag.apply(KEY_APPLICATION)).values(VALUE_KAFKA).build() + ); + if (filter.containsKey(KEY_RELEASE)) + builder = builder.filters( + Filter.builder().name(tag.apply(KEY_RELEASE)).values(filter.get(KEY_RELEASE)).build() + ); + if (filter.containsKey(KEY_CPU_ARCHITECTURE)) + builder = builder.filters( + Filter.builder().name(tag.apply(KEY_CPU_ARCHITECTURE)).values(filter.get(KEY_CPU_ARCHITECTURE)).build() + ); + builder = builder.filters( + Filter.builder().name(tag.apply(KEY_APPLICATION_ENVIRONMENT)).values("*").build() + ); + try { + DescribeImagesResponse resp = ec2Client.describeImages(builder.build()); + if (resp.hasImages() && !resp.images().isEmpty()) { + ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180); + resp.images().forEach(image -> { + if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) { + Iterator i = image.tags().iterator(); + Tag t; + String appEnvTag = null; + while (i.hasNext()) { + t = i.next(); + if (t.key().equals(KEY_APPLICATION_ENVIRONMENT)) { + appEnvTag = t.value(); + break; + } + } + amiList.add(new Ami( + image.imageId(), + appEnvTag, + image.creationDate() + )); + } + }); + amiList.sort((a, b) -> - ZonedDateTime.parse(a.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME) + .compareTo(ZonedDateTime.parse(b.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME))); + } + } catch (Exception e) { + logger.log(Level.SEVERE, "AmiTagManager: could not retrieve AMI list", e); + throw e; + } + return amiList; + } + + /** + * update AMI tag 'application_environment' + * + * @param amiId - target AMI id + * @param applicationEnvironment - new tag value + */ + public void updateAmiTag(String amiId, String applicationEnvironment) { + CreateTagsRequest.Builder builder = CreateTagsRequest.builder(); + Tag newAppEnv = Tag.builder() + .key(KEY_APPLICATION_ENVIRONMENT) + .value(applicationEnvironment).build(); + CreateTagsRequest request = builder + .resources(amiId) + .tags(newAppEnv) + .build(); + CreateTagsResponse resp; + try { + resp = ec2Client.createTags(request); + if (!resp.sdkHttpResponse().isSuccessful()) + throw AwsServiceException.builder().message("Http code \" + resp.sdkHttpResponse().statusCode() + \" received").build(); + } catch (Exception e) { + logger.severe("AmiTagManager: tag update failed for " + amiId + " and application_environment tag = " + applicationEnvironment + ", " + e); + throw e; + } + } +} diff --git a/orion-server/src/main/java/com/pinterest/orion/server/api/Ami.java b/orion-server/src/main/java/com/pinterest/orion/server/api/Ami.java new file mode 100644 index 00000000..5e539029 --- /dev/null +++ b/orion-server/src/main/java/com/pinterest/orion/server/api/Ami.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright 2024 Pinterest, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +package com.pinterest.orion.server.api; + +/** + * Specialized object to store AMI information. + * Used to transmit a list of AMIs to the frontend. + */ +public class Ami { + private final String amiId; + private final String applicationEnvironment; + private final String creationDate; + + /** + * creates a new Ami instance + * + * @param amiId - for AWS, the string "ami-" and a sequence of 17 characters + * @param applicationEnvironment - comma-separated list of environments (dev, test, + * staging, prod) supported by this ami + * @param creationDate - ami creation date, UTC + * @return the new Ami instance + */ + public Ami(String amiId, String applicationEnvironment, String creationDate) { + this.amiId = amiId; + this.applicationEnvironment = applicationEnvironment; + this.creationDate = creationDate; + } + + public String getAmiId() { + return amiId; + } + + public String getApplicationEnvironment() { + return applicationEnvironment; + } + + public String getCreationDate() { + return creationDate; + } +} diff --git a/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java b/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java index 4157b5ec..c80643f2 100644 --- a/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java +++ b/orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java @@ -25,9 +25,11 @@ import javax.annotation.security.RolesAllowed; import javax.ws.rs.GET; +import javax.ws.rs.PUT; import javax.ws.rs.NotFoundException; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.SecurityContext; @@ -41,11 +43,13 @@ import com.pinterest.orion.core.Utilization; import com.pinterest.orion.core.global.sensor.GlobalPluginManager; import com.pinterest.orion.core.global.sensor.GlobalSensor; +import com.pinterest.orion.core.actions.aws.AmiTagManager; import com.pinterest.orion.server.config.OrionConf; @Path("/") @Produces({ MediaType.APPLICATION_JSON }) public class ClusterManagerApi extends BaseClustersApi { + private AmiTagManager amiTagManager; public ClusterManagerApi(ClusterManager mgr) { super(mgr); @@ -111,6 +115,33 @@ public Map> getUtilizationDetailsByCluster() { return utilizationMap; } + @Path("/describeImages") + @GET + public List describeImages( + @QueryParam(AmiTagManager.KEY_RELEASE) String os, + @QueryParam(AmiTagManager.KEY_CPU_ARCHITECTURE) String arch + ) { + Map filter = new HashMap<>(); + if (os != null) + filter.put(AmiTagManager.KEY_RELEASE, os); + if (arch != null) + filter.put(AmiTagManager.KEY_CPU_ARCHITECTURE, arch); + if (amiTagManager == null) + amiTagManager = new AmiTagManager(); + return amiTagManager.getAmiList(filter); + } + + @Path("/updateImageTag") + @PUT + public void updateImageTag( + @QueryParam(AmiTagManager.KEY_AMI_ID) String amiId, + @QueryParam(AmiTagManager.KEY_APPLICATION_ENVIRONMENT) String applicationEnvironment + ) { + if (amiTagManager == null) + amiTagManager = new AmiTagManager(); + amiTagManager.updateAmiTag(amiId, applicationEnvironment); + } + @RolesAllowed({ OrionConf.ADMIN_ROLE, OrionConf.MGMT_ROLE }) @Path("/costByCluster") @GET diff --git a/orion-server/src/main/resources/webapp/src/actions/cluster.js b/orion-server/src/main/resources/webapp/src/actions/cluster.js index d42caa55..d06b64bf 100644 --- a/orion-server/src/main/resources/webapp/src/actions/cluster.js +++ b/orion-server/src/main/resources/webapp/src/actions/cluster.js @@ -30,6 +30,9 @@ export const UTILIZATION_REQUESTED = "UTILIZATION_REQUESTED"; export const UTILIZATION_RECEIVED = "UTILIZATION_RECEIVED"; export const COST_REQUESTED = "COST_REQUESTED"; export const COST_RECEIVED = "COST_RECEIVED"; +export const AMI_LIST_REQUESTED = "AMI_LIST_REQUESTED"; +export const AMI_LIST_RECEIVED = "AMI_LIST_RECEIVED"; +export const AMI_TAG_UPDATE = "AMI_TAG_UPDATE"; export function requestCluster(clusterId) { return { type: CLUSTER_REQUESTED, payload: { clusterId } }; @@ -110,3 +113,24 @@ export function receiveClusterEndpoint(clusterId, field, data) { payload: { clusterId, field, data }, }; } + +export function requestAmiList(filter) { + return { + type: AMI_LIST_REQUESTED, + payload: { filter }, + }; +} + +export function receiveAmiList(amiList) { + return { + type: AMI_LIST_RECEIVED, + payload: { amiList }, + }; +} + +export function updateAmiTag(amiId, applicationEnvironment) { + return { + type: AMI_TAG_UPDATE, + payload: { amiId, applicationEnvironment }, + }; +} diff --git a/orion-server/src/main/resources/webapp/src/basic-components/Ami.js b/orion-server/src/main/resources/webapp/src/basic-components/Ami.js new file mode 100644 index 00000000..d92bcc35 --- /dev/null +++ b/orion-server/src/main/resources/webapp/src/basic-components/Ami.js @@ -0,0 +1,269 @@ +/******************************************************************************* + * Copyright 2024 Pinterest, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *******************************************************************************/ +import React from "react"; +import { Button, FormControl, Grid, InputLabel, MenuItem, Select, TextField, + Table, TableBody, TableCell, TableContainer, TableHead, TableRow, + FormGroup, FormControlLabel, Checkbox } from '@material-ui/core'; +import { makeStyles } from "@material-ui/core/styles"; +import { connect } from "react-redux"; +import { requestAmiList, updateAmiTag } from "../actions/cluster"; + +const mapState = (state, ownProps) => { + const { amiList } = state.app; + return { + ...ownProps, + amiList, + }; +}; + +const mapDispatch = { + requestAmiList, + updateAmiTag +}; + +const useStyles = makeStyles(theme => ({ + formControl: { + margin: theme.spacing(1), + minWidth: 120 + }, +})); + +function Ami({ amiList, requestAmiList, updateAmiTag }) { + const classes = useStyles(); + const [os, setOS] = React.useState(); + const handleOSChange = event => { + setOS(event.target.value); + }; + const [cpuArch, setCPUArch] = React.useState(); + const handleCPUArchChange = event => { + setCPUArch(event.target.value); + }; + const [selected, setSelected] = React.useState([]); + const handleTableRowSelect = (id, row) => { + setSelected(id); + setAppEnv(row.applicationEnvironment); + const envs_str = row.applicationEnvironment; + const envs = envs_str.split(','); + env.dev = env.test = env.staging = env.prod = false; + for (const env_str of envs) + env[env_str] = true; + }; + const [appEnv, setAppEnv] = React.useState(); + const handleAppEnvChange = event => { + setAppEnv(event.target.value); + }; + const [env] = React.useState({ + dev: false, + test: false, + staging: false, + prod: false, + }); + const handleCheckboxChange = (event) => { + env[event.target.name] = event.target.checked; + const newAppEnv = []; + if (env.dev) + newAppEnv.push("dev"); + if (env.test) + newAppEnv.push("test"); + if (env.staging) + newAppEnv.push("staging"); + if (env.prod) + newAppEnv.push("prod"); + setAppEnv(newAppEnv.join(',')); + }; + const applyFilter = () => { + const parms = []; + if (os) + parms.push("release=" + os); + if (cpuArch) + parms.push("cpu_architecture=" + cpuArch); + requestAmiList(parms.join('&')); + } + + if (!amiList) + amiList = []; + return ( +
+ + +

Tag filters

+
+ + + +
+
+ + OS + + +
+
+ + CPU Architecture + + +
+
+ + + +
+
+ + + +
+
+ + + + + + AMI Id + application_environment + Creation Date + + + + { amiList.map((row) => ( + handleTableRowSelect(row.amiId, row)} + selected={selected === row.amiId} + > + {row.amiId} + {row.applicationEnvironment} + {row.creationDate} + + ))} + +
+
+
+ +
+ +
+
+ + } + label="dev" + /> + } + label="test" + /> + } + label="staging" + /> + } + label="prod" + /> + +
+
+ + + +
+
+
+
+ ); +} + +export default connect(mapState, mapDispatch)(Ami); diff --git a/orion-server/src/main/resources/webapp/src/basic-components/Homepage.js b/orion-server/src/main/resources/webapp/src/basic-components/Homepage.js index 15ba874e..34f8c844 100644 --- a/orion-server/src/main/resources/webapp/src/basic-components/Homepage.js +++ b/orion-server/src/main/resources/webapp/src/basic-components/Homepage.js @@ -26,6 +26,7 @@ import { } from "@material-ui/core"; import Summary from "./Summary"; import Utilization from "./Utilization"; +import Ami from "./Ami"; const routes = [ { @@ -38,6 +39,11 @@ const routes = [ component: Utilization, label: "Utilization", }, + { + subpath: "ami", + component: Ami, + label: "AMI", + }, ]; export default function Homepage(props) { diff --git a/orion-server/src/main/resources/webapp/src/reducers/app.js b/orion-server/src/main/resources/webapp/src/reducers/app.js index 1d77a4bd..8ca73629 100644 --- a/orion-server/src/main/resources/webapp/src/reducers/app.js +++ b/orion-server/src/main/resources/webapp/src/reducers/app.js @@ -22,7 +22,7 @@ import { AUTO_REFRESH_ENABLED, AUTO_REFRESH_DISABLED, } from "../actions/app"; -import { UTILIZATION_RECEIVED, COST_RECEIVED } from "../actions/cluster"; +import { UTILIZATION_RECEIVED, COST_RECEIVED, AMI_LIST_RECEIVED } from "../actions/cluster"; export default function showError( state = { @@ -52,6 +52,8 @@ export default function showError( return { ...state, utilization: action.payload.utilization }; case COST_RECEIVED: return { ...state, cost: action.payload.cost }; + case AMI_LIST_RECEIVED: + return { ...state, amiList: action.payload.amiList }; default: return state; } diff --git a/orion-server/src/main/resources/webapp/src/sagas/index.js b/orion-server/src/main/resources/webapp/src/sagas/index.js index 22071218..e1b8445d 100644 --- a/orion-server/src/main/resources/webapp/src/sagas/index.js +++ b/orion-server/src/main/resources/webapp/src/sagas/index.js @@ -38,6 +38,9 @@ import { receiveUtilization, COST_REQUESTED, receiveCost, + AMI_LIST_REQUESTED, + receiveAmiList, + AMI_TAG_UPDATE } from "../actions/cluster"; import { CLUSTERS_SUMMARY_REQUESTED, @@ -66,6 +69,8 @@ export default function* rootSaga() { yield fork(utilizationWatcher); yield fork(costWatcher); yield fork(globalSensorWatcher); + yield fork(amiListWatcher); + yield fork(amiTagUpdateWatcher); } function* clusterSummaryWatcher() { @@ -92,6 +97,14 @@ function* globalSensorWatcher() { yield takeEvery(GLOBAL_SENSOR_REQUESTED, fetchGlobalSensors); } +function* amiListWatcher() { + yield takeEvery(AMI_LIST_REQUESTED, fetchAmiList); +} + +function* amiTagUpdateWatcher() { + yield takeEvery(AMI_TAG_UPDATE, fetchAmiTagUpdate); +} + function* fetchCost() { try { const resp = yield fetch("/api/costByCluster"); @@ -227,3 +240,31 @@ function* fetchClusterCustomEndpoint(action) { yield put(hideLoading()); } } + +function* fetchAmiList(action) { + const filter = action.payload.filter; + try { + yield put(showLoading()); + const resp = yield call(fetch, "/api/describeImages?" + filter); + const data = yield resp.json(); + yield put(receiveAmiList(data)); + } catch (e) { + yield put(showAppError(e)); + } finally { + yield put(hideLoading()); + } +} + +function* fetchAmiTagUpdate(action) { + const amiId = action.payload.amiId; + const applicationEnvironment = action.payload.applicationEnvironment; + try { + yield put(showLoading()); + yield call(fetch, "/api/updateImageTag?ami_id=" + amiId + + "&application_environment=" + applicationEnvironment, { method: 'PUT' }); + } catch (e) { + yield put(showAppError(e)); + } finally { + yield put(hideLoading()); + } +}