Merge pull request #272 from ebukoski/master

Changes for Netflix environment compatibility
Netflix · Sep 20, 2016 · 8624383 · 8624383
2 parents a5fe562 + 164ae81
commit 8624383
Show file tree

Hide file tree

Showing 17 changed files with 263 additions and 145 deletions.
diff --git a/build.gradle b/build.gradle
@@ -38,7 +38,7 @@ dependencies {
     compile 'org.codehaus.jackson:jackson-core-asl:1.9.2'
     compile 'org.codehaus.jackson:jackson-mapper-asl:1.9.2'
     compile 'com.netflix.eureka:eureka-client:1.4.1'
-    compile 'com.amazonaws:aws-java-sdk:1.10.5.1'
+    compile 'com.amazonaws:aws-java-sdk:1.11.9'
     compile 'commons-lang:commons-lang:2.6'
     compile 'com.google.guava:guava:11.0.2'
     compile 'org.apache.httpcomponents:httpclient:4.3'

diff --git a/src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java b/src/main/java/com/netflix/simianarmy/aws/conformity/RDSConformityClusterTracker.java
@@ -107,7 +107,7 @@ public Object emailValue(String email) {
     @Override
     public void addOrUpdate(Cluster cluster) {
     	Cluster orig = getCluster(cluster.getName(), cluster.getRegion());    	
-        LOGGER.debug(String.format("Saving cluster %s to RDB table %s", cluster.getName(), table));    	
+        LOGGER.debug(String.format("Saving cluster %s to RDB table %s in region %s", cluster.getName(), cluster.getRegion(), table));
 		Map<String, String> map = cluster.getFieldToValueMap();
 
     	String conformityJson;

diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/aws/janitor/RDSJanitorResourceTracker.java
@@ -104,13 +104,13 @@ public Object emailValue(String email) {
 	/** {@inheritDoc} */
     @Override
     public void addOrUpdate(Resource resource) {
-    	Resource orig = getResource(resource.getId());    	
-        LOGGER.debug(String.format("Saving resource %s to RDB table %s", resource.getId(), table));    	
+    	Resource orig = getResource(resource.getId(), resource.getRegion());
+        LOGGER.debug(String.format("Saving resource %s to RDB table %s in region %s", resource.getId(), table, resource.getRegion()));
     	String json;
 		try {
 			json = new ObjectMapper().writeValueAsString(additionalFieldsAsMap(resource));
 		} catch (JsonProcessingException e) {
-			LOGGER.error("ERROR generating additonal field JSON when saving resource " + resource.getId(), e);
+			LOGGER.error("ERROR generating additional field JSON when saving resource " + resource.getId(), e);
 			return;
 		}
 
@@ -166,7 +166,8 @@ public void addOrUpdate(Resource resource) {
     		sb.append(AWSResource.FIELD_MARK_TIME).append("=?,");
 			sb.append(AWSResource.FIELD_OPT_OUT_OF_JANITOR).append("=?,");
     		sb.append("additionalFields").append("=? where ");
-    		sb.append(AWSResource.FIELD_RESOURCE_ID).append("=?");
+    		sb.append(AWSResource.FIELD_RESOURCE_ID).append("=? and ");
+			sb.append(AWSResource.FIELD_REGION).append("=?");
 
             LOGGER.debug(String.format("Update statement is '%s'", sb));
     		int updated = this.jdbcTemplate.update(sb.toString(),
@@ -183,7 +184,8 @@ public void addOrUpdate(Resource resource) {
     								 value(resource.getMarkTime()),
 					                 value(resource.isOptOutOfJanitor()),
     								 json,
-    								 resource.getId());
+    								 resource.getId(),
+						  			 resource.getRegion());
             LOGGER.debug(String.format("%d rows updated", updated));
     	}
     	LOGGER.debug("Successfully saved.");
@@ -309,9 +311,33 @@ public Resource mapRow(ResultSet rs, int rowNum) throws SQLException {
         	resource = resources.get(0);
         }
         return resource;
-    }            
-
-    /**
+    }
+
+	@Override
+	public Resource getResource(String resourceId, String region) {
+		Validate.notEmpty(resourceId);
+		Validate.notEmpty(region);
+		StringBuilder query = new StringBuilder();
+		query.append(String.format("select * from %s where resourceId=? and region=?", table));
+
+		LOGGER.debug(String.format("Query is '%s'", query));
+		List<Resource> resources = jdbcTemplate.query(query.toString(), new String[]{resourceId,region}, new RowMapper<Resource>() {
+			public Resource mapRow(ResultSet rs, int rowNum) throws SQLException {
+				return mapResource(rs);
+			}
+		});
+
+		Resource resource = null;
+		Validate.isTrue(resources.size() <= 1);
+		if (resources.size() == 0) {
+			LOGGER.info(String.format("Not found resource with id %s", resourceId));
+		} else {
+			resource = resources.get(0);
+		}
+		return resource;
+	}
+
+	/**
      * Creates the RDS table, if it does not already exist.
      */
     public void init() {

diff --git a/src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/aws/janitor/SimpleDBJanitorResourceTracker.java
@@ -17,28 +17,22 @@
  */
 package com.netflix.simianarmy.aws.janitor;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.Validate;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.amazonaws.services.simpledb.AmazonSimpleDB;
-import com.amazonaws.services.simpledb.model.Attribute;
-import com.amazonaws.services.simpledb.model.Item;
-import com.amazonaws.services.simpledb.model.PutAttributesRequest;
-import com.amazonaws.services.simpledb.model.ReplaceableAttribute;
-import com.amazonaws.services.simpledb.model.SelectRequest;
-import com.amazonaws.services.simpledb.model.SelectResult;
+import com.amazonaws.services.simpledb.model.*;
 import com.netflix.simianarmy.Resource;
 import com.netflix.simianarmy.Resource.CleanupState;
 import com.netflix.simianarmy.ResourceType;
 import com.netflix.simianarmy.aws.AWSResource;
 import com.netflix.simianarmy.client.aws.AWSClient;
 import com.netflix.simianarmy.janitor.JanitorResourceTracker;
+import org.apache.commons.lang.Validate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * The JanitorResourceTracker implementation in SimpleDB.
@@ -151,6 +145,32 @@ public Resource getResource(String resourceId) {
         }
     }
 
+    @Override
+    public Resource getResource(String resourceId, String region) {
+        Validate.notEmpty(resourceId);
+        Validate.notEmpty(region);
+        StringBuilder query = new StringBuilder();
+        query.append(String.format("select * from `%s` where resourceId = '%s' and region = '%s'", domain, resourceId, region));
+
+        LOGGER.debug(String.format("Query is '%s'", query));
+
+        List<Item> items = querySimpleDBItems(query.toString());
+        Validate.isTrue(items.size() <= 1);
+        if (items.size() == 0) {
+            LOGGER.info(String.format("Not found resource with id %s and region %s", resourceId, region));
+            return null;
+        } else {
+            Resource resource = null;
+            try {
+                resource = parseResource(items.get(0));
+            } catch (Exception e) {
+                // Ignore the item that cannot be parsed.
+                LOGGER.error(String.format("SimpleDB item %s cannot be parsed into a resource.", items.get(0)));
+            }
+            return resource;
+        }
+    }
+
     /**
      * Parses a SimpleDB item into an AWS resource.
      * @param item the item from SimpleDB

diff --git a/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java b/src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java
@@ -151,23 +151,38 @@ public void doMonkeyBusiness() {
 
     @Override
     public Event optInResource(String resourceId) {
-        return optInOrOutResource(resourceId, true);
+        return optInOrOutResource(resourceId, true, region);
     }
 
     @Override
     public Event optOutResource(String resourceId) {
-        return optInOrOutResource(resourceId, false);
+        return optInOrOutResource(resourceId, false, region);
     }
 
-    private Event optInOrOutResource(String resourceId, boolean optIn) {
-        Resource resource = resourceTracker.getResource(resourceId);
+    @Override
+    public Event optInResource(String resourceId, String resourceRegion) {
+        return optInOrOutResource(resourceId, true, resourceRegion);
+    }
+
+    @Override
+    public Event optOutResource(String resourceId, String resourceRegion) {
+        return optInOrOutResource(resourceId, false, resourceRegion);
+    }
+
+    private Event optInOrOutResource(String resourceId, boolean optIn, String resourceRegion) {
+        if (resourceRegion == null) {
+            resourceRegion = region;
+        }
+
+        Resource resource = resourceTracker.getResource(resourceId, resourceRegion);
         if (resource == null) {
             return null;
         }
+
         EventTypes eventType = optIn ? EventTypes.OPT_IN_RESOURCE : EventTypes.OPT_OUT_RESOURCE;
         long timestamp = calendar.now().getTimeInMillis();
         // The same resource can have multiple events, so we add the timestamp to the id.
-        Event evt = recorder.newEvent(Type.JANITOR, eventType, region, resourceId + "@" + timestamp);
+        Event evt = recorder.newEvent(Type.JANITOR, eventType, resourceRegion, resourceId + "@" + timestamp);
         recorder.recordEvent(evt);
         resource.setOptOutOfJanitor(!optIn);
         resourceTracker.addOrUpdate(resource);

diff --git a/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java b/src/main/java/com/netflix/simianarmy/client/aws/AWSClient.java
@@ -21,54 +21,16 @@
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.autoscaling.AmazonAutoScalingClient;
-import com.amazonaws.services.autoscaling.model.AutoScalingGroup;
-import com.amazonaws.services.autoscaling.model.AutoScalingInstanceDetails;
-import com.amazonaws.services.autoscaling.model.DeleteAutoScalingGroupRequest;
-import com.amazonaws.services.autoscaling.model.DeleteLaunchConfigurationRequest;
-import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest;
-import com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult;
-import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesRequest;
-import com.amazonaws.services.autoscaling.model.DescribeAutoScalingInstancesResult;
-import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsRequest;
-import com.amazonaws.services.autoscaling.model.DescribeLaunchConfigurationsResult;
-import com.amazonaws.services.autoscaling.model.LaunchConfiguration;
+import com.amazonaws.services.autoscaling.model.*;
 import com.amazonaws.services.ec2.AmazonEC2;
 import com.amazonaws.services.ec2.AmazonEC2Client;
-import com.amazonaws.services.ec2.model.CreateSecurityGroupRequest;
-import com.amazonaws.services.ec2.model.CreateSecurityGroupResult;
-import com.amazonaws.services.ec2.model.CreateTagsRequest;
-import com.amazonaws.services.ec2.model.DeleteSnapshotRequest;
-import com.amazonaws.services.ec2.model.DeleteVolumeRequest;
-import com.amazonaws.services.ec2.model.DeregisterImageRequest;
-import com.amazonaws.services.ec2.model.DescribeImagesRequest;
-import com.amazonaws.services.ec2.model.DescribeImagesResult;
-import com.amazonaws.services.ec2.model.DescribeInstancesRequest;
-import com.amazonaws.services.ec2.model.DescribeInstancesResult;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupsRequest;
-import com.amazonaws.services.ec2.model.DescribeSecurityGroupsResult;
-import com.amazonaws.services.ec2.model.DescribeSnapshotsRequest;
-import com.amazonaws.services.ec2.model.DescribeSnapshotsResult;
-import com.amazonaws.services.ec2.model.DescribeVolumesRequest;
-import com.amazonaws.services.ec2.model.DescribeVolumesResult;
-import com.amazonaws.services.ec2.model.DetachVolumeRequest;
-import com.amazonaws.services.ec2.model.EbsInstanceBlockDevice;
-import com.amazonaws.services.ec2.model.Image;
+import com.amazonaws.services.ec2.model.*;
 import com.amazonaws.services.ec2.model.Instance;
-import com.amazonaws.services.ec2.model.InstanceBlockDeviceMapping;
-import com.amazonaws.services.ec2.model.ModifyInstanceAttributeRequest;
-import com.amazonaws.services.ec2.model.Reservation;
-import com.amazonaws.services.ec2.model.SecurityGroup;
-import com.amazonaws.services.ec2.model.Snapshot;
 import com.amazonaws.services.ec2.model.Tag;
-import com.amazonaws.services.ec2.model.TerminateInstancesRequest;
-import com.amazonaws.services.ec2.model.Volume;
 import com.amazonaws.services.elasticloadbalancing.AmazonElasticLoadBalancingClient;
-import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancerAttributesRequest;
-import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancerAttributesResult;
+import com.amazonaws.services.elasticloadbalancing.model.*;
 import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersRequest;
 import com.amazonaws.services.elasticloadbalancing.model.DescribeLoadBalancersResult;
-import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerAttributes;
-import com.amazonaws.services.elasticloadbalancing.model.LoadBalancerDescription;
 import com.amazonaws.services.simpledb.AmazonSimpleDB;
 import com.amazonaws.services.simpledb.AmazonSimpleDBClient;
 import com.google.common.base.Objects;
@@ -79,7 +41,6 @@
 import com.google.inject.Module;
 import com.netflix.simianarmy.CloudClient;
 import com.netflix.simianarmy.NotFoundException;
-
 import org.apache.commons.lang.Validate;
 import org.jclouds.ContextBuilder;
 import org.jclouds.compute.ComputeService;
@@ -89,20 +50,13 @@
 import org.jclouds.compute.domain.NodeMetadata;
 import org.jclouds.compute.domain.NodeMetadataBuilder;
 import org.jclouds.domain.LoginCredentials;
-import org.jclouds.ec2.EC2ApiMetadata;
 import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
 import org.jclouds.ssh.SshClient;
 import org.jclouds.ssh.jsch.config.JschSshClientModule;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
 
 
 /**
@@ -515,8 +469,13 @@ public void deleteAutoScalingGroup(String asgName) {
         LOGGER.info(String.format("Deleting auto-scaling group with name %s in region %s.", asgName, region));
         AmazonAutoScalingClient asgClient = asgClient();
         DeleteAutoScalingGroupRequest request = new DeleteAutoScalingGroupRequest()
-        .withAutoScalingGroupName(asgName);
-        asgClient.deleteAutoScalingGroup(request);
+        .withAutoScalingGroupName(asgName).withForceDelete(true);
+        try {
+            asgClient.deleteAutoScalingGroup(request);
+            LOGGER.info(String.format("Deleted auto-scaling group with name %s in region %s.", asgName, region));
+        }catch(Exception e) {
+            LOGGER.error("Got an exception deleting ASG " + asgName, e);
+        }
     }
 
     /** {@inheritDoc} */

diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorMonkey.java
@@ -17,14 +17,14 @@
  */
 package com.netflix.simianarmy.janitor;
 
-import java.util.List;
-
 import com.netflix.simianarmy.EventType;
 import com.netflix.simianarmy.Monkey;
 import com.netflix.simianarmy.MonkeyConfiguration;
 import com.netflix.simianarmy.MonkeyRecorder.Event;
 import com.netflix.simianarmy.MonkeyType;
 
+import java.util.List;
+
 /**
  * The abstract class for a Janitor Monkey.
  */
@@ -150,4 +150,20 @@ public Context context() {
      */
     public abstract Event optOutResource(String resourceId);
 
+    /**
+     * Opt in a resource for Janitor Monkey.
+     * @param resourceId the resource id
+     * @param region the region of the resource
+     * @return the opt-in event
+     */
+    public abstract Event optInResource(String resourceId, String region);
+
+    /**
+     * Opt out a resource for Janitor Monkey.
+     * @param resourceId the resource id
+     * @param region the region of the resource
+     * @return the opt-out event
+     */
+    public abstract Event optOutResource(String resourceId, String region);
+
 }
diff --git a/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java b/src/main/java/com/netflix/simianarmy/janitor/JanitorResourceTracker.java
@@ -17,11 +17,11 @@
  */
 package com.netflix.simianarmy.janitor;
 
-import java.util.List;
-
 import com.netflix.simianarmy.Resource;
 import com.netflix.simianarmy.ResourceType;
 
+import java.util.List;
+
 /**
  * The interface to track the resources marked/cleaned by the Janitor Monkey.
  *
@@ -52,4 +52,12 @@ public interface JanitorResourceTracker {
      */
     Resource getResource(String resourceId);
 
+    /** Gets the resource of a specific id.
+     *
+     * @param resourceId the resource id
+     * @param regionId the region id
+     * @return the resource that matches the resource id and region
+     */
+    Resource getResource(String resourceId, String regionId);
+
 }