Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Configurable environment types for Ami tag management #259

Merged
merged 2 commits into from
Feb 26, 2024
Merged

Configurable environment types for Ami tag management #259

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f68a5d8
Initial work
brunoasr Feb 24, 2024
e8801d3
Fixup
brunoasr Feb 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions orion-server/src/main/java/com/pinterest/orion/core/ClusterManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.pinterest.orion.core.automation.operator.OperatorFactory;
import com.pinterest.orion.core.automation.sensor.SensorFactory;
import com.pinterest.orion.core.metrics.MetricsStore;
import com.pinterest.orion.server.config.OrionConf;
import com.pinterest.orion.server.config.OrionPluginConfig;

public class ClusterManager {
Expand All @@ -37,6 +38,7 @@ public class ClusterManager {
private ClusterStateSink stateSink;
private MetricsStore metricsStore;
private CostCalculator costCalculator;
private OrionConf configuration;

public ClusterManager(SensorFactory sensorFactory,
OperatorFactory operatorFactory,
Expand Down Expand Up @@ -116,5 +118,20 @@ public CostCalculator getCostCalculator() {
public MetricsStore getMetricsStore() {
return metricsStore;
}

/**
* @return the configuration
*/
public OrionConf getOrionConf() {
return configuration;
}

/**
* @param configuration the Orion server configuration
*/
public void setOrionConf(OrionConf configuration) {
this.configuration = configuration;
return;
}

}
41 changes: 28 additions & 13 deletions orion-server/src/main/java/com/pinterest/orion/core/actions/aws/AmiTagManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.function.Function;
import java.util.function.UnaryOperator;

import com.pinterest.orion.server.api.Ami;
Expand All @@ -34,6 +35,7 @@
import software.amazon.awssdk.services.ec2.model.CreateTagsRequest;
import software.amazon.awssdk.services.ec2.model.CreateTagsResponse;
import software.amazon.awssdk.services.ec2.model.Filter;
import software.amazon.awssdk.services.ec2.model.Image;
import software.amazon.awssdk.services.ec2.model.Tag;

/**
Expand All @@ -52,6 +54,7 @@ public class AmiTagManager {
public static final String KEY_APPLICATION_ENVIRONMENT = "application_environment";
public static final String VALUE_KAFKA = "kafka";
public static UnaryOperator<String> tag = key -> "tag:" + key;
public static final String ENV_TYPES_KEY = "envTypes";

public AmiTagManager() {
ec2Client = Ec2Client.create();
Expand All @@ -66,26 +69,38 @@ public AmiTagManager() {
public List<Ami> getAmiList(Map<String, String> filter) {
List<Ami> amiList = new ArrayList<>();
DescribeImagesRequest.Builder builder = DescribeImagesRequest.builder();
builder = builder.filters(
Filter.builder().name(tag.apply(KEY_APPLICATION)).values(VALUE_KAFKA).build()
Filter.Builder filterBuilder = Filter.builder();
List<Filter> filterList = new ArrayList<>();
filterList.add(
filterBuilder.name(tag.apply(KEY_APPLICATION))
.values(VALUE_KAFKA)
.build()
);
if (filter.containsKey(KEY_RELEASE))
builder = builder.filters(
Filter.builder().name(tag.apply(KEY_RELEASE)).values(filter.get(KEY_RELEASE)).build()
filterList.add(
filterBuilder.name(tag.apply(KEY_RELEASE))
.values(filter.get(KEY_RELEASE))
.build()
);
if (filter.containsKey(KEY_CPU_ARCHITECTURE))
builder = builder.filters(
Filter.builder().name(tag.apply(KEY_CPU_ARCHITECTURE)).values(filter.get(KEY_CPU_ARCHITECTURE)).build()
filterList.add(
filterBuilder.name(tag.apply(KEY_CPU_ARCHITECTURE))
.values(filter.get(KEY_CPU_ARCHITECTURE))
.build()
);
builder = builder.filters(
Filter.builder().name(tag.apply(KEY_APPLICATION_ENVIRONMENT)).values("*").build()
filterList.add(
filterBuilder.name(tag.apply(KEY_APPLICATION_ENVIRONMENT))
.values("*")
.build()
);
builder = builder.filters(filterList);
try {
DescribeImagesResponse resp = ec2Client.describeImages(builder.build());
if (resp.hasImages() && !resp.images().isEmpty()) {
ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180);
// The limitation of images newer than 180 days is temporarily suspended
//ZonedDateTime cutDate = ZonedDateTime.now().minusDays(180);
resp.images().forEach(image -> {
if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) {
/*if (ZonedDateTime.parse(image.creationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME).isAfter(cutDate)) {*/
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need to keep commented code?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. It was an initial requirement that only images newer than 180 days were used in the AMI promotion process. But during this early stage, Eric asked that this constraint be temporarily suspended.

Iterator<Tag> i = image.tags().iterator();
Tag t;
String appEnvTag = null;
Expand All @@ -101,10 +116,10 @@ public List<Ami> getAmiList(Map<String, String> filter) {
appEnvTag,
image.creationDate()
));
}
// }
});
amiList.sort((a, b) -> - ZonedDateTime.parse(a.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME)
.compareTo(ZonedDateTime.parse(b.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME)));
Function<Ami, ZonedDateTime> parse = i -> ZonedDateTime.parse(i.getCreationDate(), DateTimeFormatter.ISO_ZONED_DATE_TIME);
amiList.sort((a, b) -> - parse.apply(a).compareTo(parse.apply(b)));
}
} catch (Exception e) {
logger.log(Level.SEVERE, "AmiTagManager: could not retrieve AMI list", e);
Expand Down
2 changes: 1 addition & 1 deletion ...n-server/src/main/java/com/pinterest/orion/core/actions/aws/ReplaceEC2InstanceAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ public AttributeSchema generateSchema(Map<String, Object> config) {
.addOption("m6id.4xlarge", "m6id.4xlarge")
.addOption("m6id.8xlarge", "m6id.8xlarge")
)
.addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will inherit current AMI if not provided)", false))
.addValue(new TextValue(ATTR_AMI_KEY, "AMI id (optional, will use cluster filter criteria if not provided)", false))
.addSchema(super.generateSchema(config));
}
}
1 change: 0 additions & 1 deletion orion-server/src/main/java/com/pinterest/orion/server/OrionServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package com.pinterest.orion.server;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
Expand Down
12 changes: 12 additions & 0 deletions orion-server/src/main/java/com/pinterest/orion/server/api/ClusterManagerApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,18 @@ public void updateImageTag(
amiTagManager.updateAmiTag(amiId, applicationEnvironment);
}


@Path("/getEnvTypes")
@GET
public List<String> getEnvTypes() {
List<String> envTypes = null;
Map<String, Object> additionalConfigs = mgr.getOrionConf().getAdditionalConfigs();
if(additionalConfigs != null && additionalConfigs.containsKey(AmiTagManager.ENV_TYPES_KEY)) {
envTypes = (List<String>) additionalConfigs.get(AmiTagManager.ENV_TYPES_KEY);
}
return envTypes;
}

@RolesAllowed({ OrionConf.ADMIN_ROLE, OrionConf.MGMT_ROLE })
@Path("/costByCluster")
@GET
Expand Down
16 changes: 16 additions & 0 deletions orion-server/src/main/resources/webapp/src/actions/cluster.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ export const COST_RECEIVED = "COST_RECEIVED";
export const AMI_LIST_REQUESTED = "AMI_LIST_REQUESTED";
export const AMI_LIST_RECEIVED = "AMI_LIST_RECEIVED";
export const AMI_TAG_UPDATE = "AMI_TAG_UPDATE";
export const ENV_TYPES_REQUESTED = "ENV_TYPES_REQUESTED";
export const ENV_TYPES_RECEIVED = "ENV_TYPES_RECEIVED";

export function requestCluster(clusterId) {
return { type: CLUSTER_REQUESTED, payload: { clusterId } };
Expand Down Expand Up @@ -134,3 +136,17 @@ export function updateAmiTag(amiId, applicationEnvironment) {
payload: { amiId, applicationEnvironment },
};
}

export function requestEnvTypes() {
return {
type: ENV_TYPES_REQUESTED,
payload: {},
};
}

export function receiveEnvTypes(envTypeList) {
return {
type: ENV_TYPES_RECEIVED,
payload: { envTypeList },
};
}
84 changes: 26 additions & 58 deletions orion-server/src/main/resources/webapp/src/basic-components/Ami.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,21 @@ import { Button, FormControl, Grid, InputLabel, MenuItem, Select, TextField,
FormGroup, FormControlLabel, Checkbox } from '@material-ui/core';
import { makeStyles } from "@material-ui/core/styles";
import { connect } from "react-redux";
import { requestAmiList, updateAmiTag } from "../actions/cluster";
import { requestAmiList, updateAmiTag, requestEnvTypes } from "../actions/cluster";

const mapState = (state, ownProps) => {
const { amiList } = state.app;
const { amiList, envTypes } = state.app;
return {
...ownProps,
amiList,
envTypes
};
};

const mapDispatch = {
requestAmiList,
updateAmiTag
requestEnvTypes,
updateAmiTag,
};

const useStyles = makeStyles(theme => ({
Expand All @@ -41,7 +43,7 @@ const useStyles = makeStyles(theme => ({
},
}));

function Ami({ amiList, requestAmiList, updateAmiTag }) {
function Ami({ amiList, requestAmiList, envTypes, requestEnvTypes, updateAmiTag }) {
const classes = useStyles();
const [os, setOS] = React.useState();
const handleOSChange = event => {
Expand All @@ -65,23 +67,14 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
const handleAppEnvChange = event => {
setAppEnv(event.target.value);
};
const [env] = React.useState({
dev: false,
test: false,
staging: false,
prod: false,
});
const envMap = {};
if (envTypes !== undefined)
envTypes.forEach(value => { envMap[value] = false; });
const [env] = React.useState(envMap);
const handleCheckboxChange = (event) => {
env[event.target.name] = event.target.checked;
const newAppEnv = [];
if (env.dev)
newAppEnv.push("dev");
if (env.test)
newAppEnv.push("test");
if (env.staging)
newAppEnv.push("staging");
if (env.prod)
newAppEnv.push("prod");
envTypes.forEach(envType => { if (env[envType]) newAppEnv.push(envType); });
setAppEnv(newAppEnv.join(','));
};
const applyFilter = () => {
Expand All @@ -91,10 +84,13 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
if (cpuArch)
parms.push("cpu_architecture=" + cpuArch);
requestAmiList(parms.join('&'));
requestEnvTypes();
}

if (!amiList)
amiList = [];
if (!envTypes)
envTypes = [];
return (
<div>
<Grid container spacing={3}>
Expand Down Expand Up @@ -207,46 +203,18 @@ function Ami({ amiList, requestAmiList, updateAmiTag }) {
</div>
<div>
<FormGroup column>
<FormControlLabel
control={
<Checkbox
checked={env.dev}
onChange={handleCheckboxChange}
name="dev"
color="primary"
/>}
label="dev"
/>
<FormControlLabel
control={
<Checkbox
checked={env.test}
onChange={handleCheckboxChange}
name="test"
color="primary"
/>}
label="test"
/>
<FormControlLabel
control={
<Checkbox
checked={env.staging}
onChange={handleCheckboxChange}
name="staging"
color="primary"
/>}
label="staging"
/>
<FormControlLabel
control={
<Checkbox
checked={env.prod}
onChange={handleCheckboxChange}
name="prod"
color="primary"
/>}
label="prod"
/>
{ envTypes.map((envType) => (
<FormControlLabel
control={
<Checkbox
checked={env[envType]}
onChange={handleCheckboxChange}
name={envType}
color="primary"
/>}
label={envType}
/>
))}
</FormGroup>
</div>
<div>
Expand Down
9 changes: 8 additions & 1 deletion orion-server/src/main/resources/webapp/src/reducers/app.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@ import {
AUTO_REFRESH_ENABLED,
AUTO_REFRESH_DISABLED,
} from "../actions/app";
import { UTILIZATION_RECEIVED, COST_RECEIVED, AMI_LIST_RECEIVED } from "../actions/cluster";
import {
UTILIZATION_RECEIVED,
COST_RECEIVED,
AMI_LIST_RECEIVED,
ENV_TYPES_RECEIVED,
} from "../actions/cluster";

export default function showError(
state = {
Expand Down Expand Up @@ -54,6 +59,8 @@ export default function showError(
return { ...state, cost: action.payload.cost };
case AMI_LIST_RECEIVED:
return { ...state, amiList: action.payload.amiList };
case ENV_TYPES_RECEIVED:
return { ...state, envTypes: action.payload.envTypeList };
default:
return state;
}
Expand Down
22 changes: 21 additions & 1 deletion orion-server/src/main/resources/webapp/src/sagas/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ import {
receiveCost,
AMI_LIST_REQUESTED,
receiveAmiList,
AMI_TAG_UPDATE
AMI_TAG_UPDATE,
ENV_TYPES_REQUESTED,
receiveEnvTypes
} from "../actions/cluster";
import {
CLUSTERS_SUMMARY_REQUESTED,
Expand Down Expand Up @@ -71,6 +73,7 @@ export default function* rootSaga() {
yield fork(globalSensorWatcher);
yield fork(amiListWatcher);
yield fork(amiTagUpdateWatcher);
yield fork(envTypesWatcher);
}

function* clusterSummaryWatcher() {
Expand Down Expand Up @@ -105,6 +108,10 @@ function* amiTagUpdateWatcher() {
yield takeEvery(AMI_TAG_UPDATE, fetchAmiTagUpdate);
}

function* envTypesWatcher() {
yield takeEvery(ENV_TYPES_REQUESTED, fetchEnvTypes);
}

function* fetchCost() {
try {
const resp = yield fetch("/api/costByCluster");
Expand Down Expand Up @@ -268,3 +275,16 @@ function* fetchAmiTagUpdate(action) {
yield put(hideLoading());
}
}

function* fetchEnvTypes() {
try {
yield put(showLoading());
const resp = yield call(fetch, "/api/getEnvTypes");
const data = yield resp.json();
yield put(receiveEnvTypes(data));
} catch (e) {
yield put(showAppError(e));
} finally {
yield put(hideLoading());
}
}
Loading