aws_delete_unused_clb.pt

name "AWS Delete Unused Classic Load Balancers"
rs_pt_ver 20180301
type "policy"
short_description "Report and remediate any Classic Load Balancers (CLB) that are not currently in use. \n See the [README](https://github.com/flexera-public/policy_templates/tree/master/cost/aws/elb/clb_unused) and [docs.flexera.com/flexera/EN/Automation](https://docs.flexera.com/flexera/EN/Automation/AutomationGS.htm) to learn more."
long_description ""
category "Cost"
severity "low"
default_frequency "daily"
info(
  version: "3.0",
  provider: "AWS",
  service: "ELB",
  policy_set: ""
)

###############################################################################
# Parameters
###############################################################################

parameter "param_allowed_regions" do
  type "list"
  label "Allowed Regions"
  allowed_pattern /^([a-zA-Z-_]+-[a-zA-Z0-9-_]+-[0-9-_]+,*|)+$/
  description "A list of allowed regions. See the README for more details"
end

parameter "param_email" do
  type "list"
  label "Email addresses"
  description "A list of email addresses to notify"
end

parameter "param_aws_account_number" do
  type "string"
  label "Account Number"
  description "The account number for AWS STS Cross Account Roles."
  default ""
end

parameter "param_exclude_tags" do
  type "list"
  label "Ignore tags"
  description "List of one or more Tags that will exclude Classic Load Balancer from actions being taken. Format: Key=Value"
  allowed_pattern /([\w]?)+\=([\w]?)+/
end

parameter "param_automatic_action" do
  type "list"
  label "Automatic Actions"
  description "When this value is set, this policy will automatically take the selected action(s)"
  allowed_values ["Delete Load balancers"]
end

parameter "param_log_to_cm_audit_entries" do
  type "string"
  label "Log to CM Audit Entries"
  description "Boolean for whether or not to log any debugging information from actions to CM Audit Entries, this should be left set to No on Flexera EU"
  default "No"
  allowed_values "Yes", "No"
end

###############################################################################
# Authentication
###############################################################################

#authenticate with AWS
credentials "auth_aws" do
  schemes "aws","aws_sts"
  label "AWS"
  description "Select the AWS Credential from the list"
  tags "provider=aws"
  aws_account_number $param_aws_account_number
end

credentials "auth_flexera" do
  schemes "oauth2"
  label "flexera"
  description "Select Flexera One OAuth2 credentials"
  tags "provider=flexera"
end

###############################################################################
# Datasources
###############################################################################

# ds_region_list is a list of regions that are opted-in or opt-in-not-required
datasource "ds_regions_list" do
  # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeRegions.html
  request do
    auth $auth_aws
    verb "GET"
    host "ec2.amazonaws.com"
    path "/"
    query "Action", "DescribeRegions"
    query "Version", "2016-11-15"
    query "Filter.1.Name", "opt-in-status"
    query "Filter.1.Value.1", "opt-in-not-required"
    query "Filter.1.Value.2", "opted-in"
  end
  result do
    encoding "xml"
    collect xpath(response, "//DescribeRegionsResponse/regionInfo/item", "array") do
      field "region", xpath(col_item, "regionName")
    end
  end
end

# Get only SCP enabled regions
datasource "ds_regions" do
  run_script $js_regions, $param_allowed_regions, $ds_regions_list
end

#To get list of All LoadBalancers.
#https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_DescribeLoadBalancers.html
datasource "ds_clb_list" do
  iterate $ds_regions
  request do
    auth $auth_aws
    host join(["elasticloadbalancing.",val(iter_item,"region"),".amazonaws.com"])
    path "/"
    verb "GET"
    query "Action", "DescribeLoadBalancers"
    query "Version", "2012-06-01"
    header "Accept", "application/json"
  end
  result do
    encoding "json"
    collect jmes_path(response, "DescribeLoadBalancersResponse.DescribeLoadBalancersResult.LoadBalancerDescriptions[*]") do
      field "loadBalancerName", jmes_path(col_item, "LoadBalancerName")
      field "availabilityZones", jmes_path(col_item, "AvailabilityZones")
      field "instancesList", jmes_path(col_item, "Instances[*].InstanceId")
      field "region", val(iter_item,"region")
    end
  end
end

#To get list of All LoadBalancers with Tags.
#https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_DescribeTags.html
datasource "ds_clb_list_tags" do
  iterate $ds_clb_list
  request do
    auth $auth_aws
    host join(["elasticloadbalancing.",val(iter_item,"region"),".amazonaws.com"])
    path "/"
    verb "GET"
    query "Action", "DescribeTags"
    query "Version", "2012-06-01"
    query "LoadBalancerNames.member.1",val(iter_item, "loadBalancerName")
    header "Accept", "application/json"
  end
  result do
    encoding "json"
    collect jmes_path(response, "DescribeTagsResponse.DescribeTagsResult.TagDescriptions[*]") do
      field "tags" do
        collect jmes_path(col_item,"Tags") do
          field "tagKey", jmes_path(col_item,"Key")
          field "tagValue", jmes_path(col_item,"Value")
        end
      end
      field "loadBalancerName", val(iter_item, "loadBalancerName")
      field "availabilityZones", val(iter_item, "availabilityZones")
      field "instancesList", val(iter_item, "instancesList")
      field "region", val(iter_item,"region")
    end
  end
end

#https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_DescribeInstanceHealth.html
datasource "ds_instances_health" do
  iterate $ds_clb_list
  request do
    auth $auth_aws
    host join(["elasticloadbalancing.",val(iter_item,"region"),".amazonaws.com"])
    path "/"
    verb "GET"
    query "Action", "DescribeInstanceHealth"
    query "Version", "2012-06-01"
    query "LoadBalancerName",val(iter_item, "loadBalancerName")
    header "Accept", "application/json"
  end
  result do
    encoding "json"
    collect jmes_path(response, "DescribeInstanceHealthResponse.DescribeInstanceHealthResult.InstanceStates[*]") do
      field "description", jmes_path(col_item,"Description")
      field "state", jmes_path(col_item,"State")
      field "instanceId", jmes_path(col_item,"InstanceId")
      field "loadBalancerName", val(iter_item, "loadBalancerName")
    end
  end
end

datasource "ds_unused_clb_map" do
  run_script $js_aws_clb_filter_map, $ds_clb_list_tags,$ds_instances_health,$param_exclude_tags
end

datasource "ds_currency_reference" do
  request do
    host "raw.githubusercontent.com"
    path "/rightscale/policy_templates/master/cost/scheduled_reports/currency_reference.json"
    header "User-Agent", "RS Policies"
  end
end

datasource "ds_currency_code" do
  request do
    auth $auth_flexera
    host rs_optima_host
    path join(["/bill-analysis/orgs/",rs_org_id,"/settings/currency_code"])
    header "Api-Version", "0.1"
    header "User-Agent", "RS Policies"
    ignore_status [403]
  end
  result do
    encoding "json"
    field "id", jmes_path(response,"id")
    field "value", jmes_path(response,"value")
  end
end

datasource "ds_billing_centers" do
  request do
    auth $auth_flexera
    host rs_optima_host
    path join(["/analytics/orgs/",rs_org_id,"/billing_centers"])
    header "Api-Version", "1.0"
    header "User-Agent", "RS Policies"
    query "view", "allocation_table"
    ignore_status [403]
  end
  result do
    encoding "json"
    collect jmes_path(response,"[*]") do
      field "href", jmes_path(col_item,"href")
      field "id", jmes_path(col_item,"id")
      field "name", jmes_path(col_item,"name")
      field "parent_id", jmes_path(col_item,"parent_id")
    end
  end
end

datasource "ds_top_level_billing_centers" do
  run_script $js_top_level_bc, $ds_billing_centers
end

datasource "ds_get_caller_identity" do
  request do
    auth $auth_aws
    verb "GET"
    host "sts.amazonaws.com"
    path "/"
    header "User-Agent", "RS Policies"
    query "Action", "GetCallerIdentity"
    query "Version", "2011-06-15"
  end
  result do
    encoding "xml"
    collect xpath(response, "//GetCallerIdentityResponse/GetCallerIdentityResult") do
      field "account",xpath(col_item, "Account")
    end
  end
end

datasource "ds_clbs_costs" do
  request do
    run_script $js_get_costs, $ds_get_caller_identity, $ds_top_level_billing_centers, rs_org_id, rs_optima_host
  end
  result do
    encoding "json"
    collect jmes_path(response,"rows[*]") do
      field "resource_id", jmes_path(col_item,"dimensions.resource_id")
      field "cost_nonamortized_unblended_adj", jmes_path(col_item,"metrics.cost_nonamortized_unblended_adj")
      field "resource_type", jmes_path(col_item,"dimensions.resource_type")
    end
  end
end

datasource "ds_clbs_cost_mapping" do
  run_script $js_clbs_cost_mapping, $ds_unused_clb_map, $ds_clbs_costs, $ds_currency_code, $ds_currency_reference, $ds_billing_centers, $ds_get_caller_identity
end

###############################################################################
# Script
###############################################################################

script "js_regions", type:"javascript" do
  parameters "user_entered_regions", "all_regions"
  result "regions"
  code <<-EOS
    if(_.isEmpty(user_entered_regions)){
      regions = all_regions;
    }else{
      //Filter unique regions
      var uniqueRegions = _.uniq(user_entered_regions);
      var all_regions_list = [];
      all_regions.forEach(function(all_region){
        all_regions_list.push(all_region.region)
      })

      //Filter valid regions
      var valid_regions = [];
      _.map(uniqueRegions, function(uniqueRegion){
        if(all_regions_list.indexOf(uniqueRegion) > -1){
          valid_regions.push({"region": uniqueRegion})
        }
      })

      //Throw an error if no valid regions found
      if (_.isEmpty(valid_regions)) {
        regions = all_regions;
      }else{
        regions = valid_regions
      }
    }
  EOS
end

script "js_aws_clb_filter_map", type: "javascript" do
  parameters "ds_clb_list_tags","ds_instances_health","param_exclude_tags"
  result "content"
  code <<-EOS
    var param_exclude_tags_lower=[];
    for(var j=0; j < param_exclude_tags.length; j++){
      param_exclude_tags_lower[j]=param_exclude_tags[j].toString().toLowerCase();
    }

    var content=[]
    for(var i=0; i<ds_clb_list_tags.length; i++){
      clb=ds_clb_list_tags[i]

      // Constructing availabilityZones with comma separated to display in detail_template
      var availabilityZones=clb['availabilityZones']
      var s_availabilityZones=""
      for(var j=0; j < availabilityZones.length; j++){
        availabilityZone = availabilityZones[j]
        s_availabilityZones = s_availabilityZones + ', '+ availabilityZone
      }

      // Constructing InstanceIds with comma separated to display in detail_template
      var instancesList = clb['instancesList']
      var s_instanceIds=""
      for(var j=0; j < instancesList.length; j++){
        instanceId = instancesList[j]
        s_instanceIds = s_instanceIds + ', '+ instanceId
      }

      //Constructing Tags of individual CLB into key=value format. Check, if the tag present in entered param_exclude_tags, ignore the CLB if the tag matches/present.
      var tags = clb['tags']
      var isTagMatched=false
      var tagKeyValue=""
      for(var j=0; j < tags.length; j++){
        tag = tags[j]
        if(param_exclude_tags_lower.indexOf((tag['tagKey']+'='+tag['tagValue']).toLowerCase()) !== -1){
          isTagMatched = true;
        }
        // Constructing tags with comma separated to display in detail_template
        tagKeyValue = tagKeyValue + ', '+ tag['tagKey']+'='+tag['tagValue']
      }

      //If the CLB tag does not match with entered param_exclude_tags, then check if any instance exists in the CLB and check for instances health.
      if(!(isTagMatched)){
        clb_details={
          id: clb['loadBalancerName'],
          instanceId:(s_instanceIds.slice(2)),
          availabilityZones:(s_availabilityZones.slice(2)),
          tagKeyValue:(tagKeyValue.slice(2)),
          region: clb['region'],
          savings:"N/A"
        }
        if(instancesList.length > 0){
          var isInstanceNotHealthy=false
          // Check if the CLB has no healthy instance with state and description matches for the values mentioned below
          var state= "OutOfService"
          var description= "Instance has failed at least the UnhealthyThreshold number of health checks consecutively."
          for(var k=0; k < ds_instances_health.length; k++){
            instance= ds_instances_health[k]
            if(((instance['loadBalancerName']) == (clb['loadBalancerName'])) && (instance['state'] == state && instance['description'] == description)){
              isInstanceNotHealthy=true
            }
          }
          if(isInstanceNotHealthy){
            content.push(clb_details)
          }
        }else{
          content.push(clb_details)
        }
      }
    }
  EOS
end

script "js_top_level_bc", type: "javascript" do
  parameters "billing_centers"
  result "filtered_billing_centers"
  code <<-EOS
    var filtered_billing_centers = _.reject(billing_centers, function(bc){ return bc.parent_id != null });
  EOS
end

script "js_get_costs", type:"javascript" do
  parameters  "account_id","billing_centers","org", "optima_host"
  result "request"
  code <<-EOS
    // returns date formatted as string: YYYY-mm-dd
    function getFormattedDailyDate(date) {
      var year = date.getFullYear();
      var month = (1 + date.getMonth()).toString();
      month = month.length > 1 ? month : '0' + month;
      var day = date.getDate().toString();
      day = day.length > 1 ? day : '0' + day;
      return year + '-' + month + '-' + day;
    }
    var start_date = getFormattedDailyDate(new Date(new Date().setDate(new Date().getDate() - 3)));
    var end_date = getFormattedDailyDate(new Date(new Date().setDate(new Date().getDate() - 2)));
    var request = {
      auth: "auth_flexera",
      host: optima_host,
      verb: "POST",
      path: "/bill-analysis/orgs/" + org + "/costs/select",
      body_fields: {
        "dimensions": ["resource_id","resource_type"],
        "granularity": "day",
        "start_at": start_date,
        "end_at": end_date,
        "metrics": ["cost_nonamortized_unblended_adj"],
        "billing_center_ids": _.compact(_.map(billing_centers, function(value){ return value.id})),
        "limit": 10000,
        "filter": {
          "expressions": [
            {
              "dimension": "service",
              "type": "equal",
              "value": "AWSELB"
            },
            {
              "dimension": "resource_type",
              "type": "equal",
              "value": "Load Balancer"
            },
            {
              "dimension": "vendor_account",
              "type": "equal",
              "value": account_id[0]["account"]
            }
          ],
          "type": "and"
        }
      },
      headers: {
        "User-Agent": "RS Policies",
        "Api-Version": "1.0"
      },
      ignore_status: [400]
    }
  EOS
end

script "js_clbs_cost_mapping", type:"javascript" do
  parameters  "clbs","clbs_cost","ds_currency_code","ds_currency_reference", "ds_billing_centers", "ds_get_caller_identity"
  result "result"
  code <<-EOS
  var total=0;
  var result = {};
  var loadbalancers = [];
  var message='';
  var count=0;
  function formatNumber(number, separator){
    var numString =number.toString();
    var values=numString.split(".");
    var result = ''
    while (values[0].length > 3){
      var chunk = values[0].substr(-3)
      values[0] = values[0].substr(0, values[0].length - 3)
      result = separator + chunk + result
    }
    if (values[0].length > 0){
      result = values[0] + result
    }
    if(values[1]==undefined){
      return result;
    }
    return result+"."+values[1];
  }
  if(ds_billing_centers.length!=0){
    // Put costs into a map by resource ID and only include them for resource IDs we actually need
    var costs_by_resource_id = {};
    _.each(clbs, function(clb) {
      costs_by_resource_id[clb.id] = [];
    });
    _.each(clbs_cost, function(cost) {
      var costs = costs_by_resource_id[cost.resource_id.replace(/^loadbalancer\\//, '')];
      if (costs != null) {
        costs.push(cost);
      }
    });
    // Format costs with currency symbol and thousands separator
    if( ds_currency_code['value'] !== undefined ) {
      if (ds_currency_reference[ds_currency_code['value']] !== undefined ) {
        var cur = ds_currency_reference[ds_currency_code['value']]['symbol']
        if( ds_currency_reference[ds_currency_code['value']]['t_separator'] !== undefined ) {
          var separator = ds_currency_reference[ds_currency_code['value']]['t_separator']
        } else {
          var separator = ""
        }
      } else {
        var cur = ""
        var separator = ""
      }
    } else {
      var cur = "$"
      var separator = ","
    }
    _.each(clbs, function(clb){
      var cost_objects = costs_by_resource_id[clb.id];
      if (_.size(cost_objects) > 0){
        count++;
        var sum = _.reduce(_.compact(_.map(cost_objects, function(value){ return value.cost_nonamortized_unblended_adj})), function(memo, num){ return memo + num; }, 0);
        var monthly_savings = sum*30;
        total=total+monthly_savings;
        loadbalancers.push({
          accountId: ds_get_caller_identity[0]['account'],
          id : clb['id'],
          region : clb['region'],
          availabilityZones : clb['availabilityZones'],
          instanceId : clb['instanceId'],
          tagKeyValue : clb['tagKeyValue'],
          savings: cur + ' '+formatNumber((Math.round(monthly_savings * 100) / 100), separator),
        })
      } else {
        loadbalancers.push({
          accountId: ds_get_caller_identity[0]['account'],
          id : clb['id'],
          region : clb['region'],
          availabilityZones : clb['availabilityZones'],
          instanceId : clb['instanceId'],
          tagKeyValue : clb['tagKeyValue'],
          savings: "N/A",
        })
      }
    })
    if(count){
      var total_savings=cur + ' '+formatNumber((Math.round(total * 100) / 100), separator);
      message=message="The total estimated monthly savings are "+total_savings;
    }
    else{
      message="The Flexera Optima system does not have any data to calculate savings for these resources";
    }
  }
  else{
    loadbalancers=clbs;
    message="You do not have the minimum required role of billing_center_viewer to view the savings";
  }
  result={
    "clbs": loadbalancers,
    "message": message
  };
  EOS
end

###############################################################################
# Escalations
###############################################################################

escalation "esc_email" do
  automatic true
  label "Send Email"
  description "Send incident email"
  email $param_email
end

escalation "esc_delete_clb" do
  automatic contains($param_automatic_action, "Delete Load balancers")
  label "Delete CLB"
  description "Delete selected unused CLB"
  run "delete_CLB", data, $param_log_to_cm_audit_entries, rs_optima_host
end

###############################################################################
# Policy
###############################################################################

policy "policy_delete_unused_CLB" do
  validate $ds_clbs_cost_mapping do
    check eq(size(val(data, "clbs")), 0)
    escalate $esc_email
    escalate $esc_delete_clb
    summary_template "AWS Account ID: {{with index data.clbs 0}}{{ .accountId }}{{end}} - {{ len data.clbs }} Unused Classic Load Balancers Found in AWS"
    detail_template <<-EOS
{{data.message}}
EOS
    export "clbs" do
      resource_level true
      field "accountId" do
        label "Account Id"
      end
      field "id" do
        label "LoadBalancer Name"
      end
      field "region" do
        label "Region"
      end
      field "instanceId" do
        label "Instance ID"
      end
      field "availabilityZones" do
        label "Availability Zones"
      end
      field "tagKeyValue" do
        label "Tag Value"
      end
      field "savings" do
        label "Estimated Monthly Savings"
      end
    end
  end
end

###############################################################################
# Cloud Workflow
###############################################################################

#https://docs.aws.amazon.com/elasticloadbalancing/2012-06-01/APIReference/API_DeleteLoadBalancer.html
define delete_CLB($data,$param_log_to_cm_audit_entries, $$rs_optima_host) return $all_responses do
  $$debug = $param_log_to_cm_audit_entries == "Yes"
  $all_responses = []
  foreach $item in $data do
    sub on_error: skip do
      $response = http_request(
        verb: "get",
        host: join(["elasticloadbalancing.",$item["region"],".amazonaws.com"]),
        auth: $$auth_aws,
        href: join(["/", "?Action=DeleteLoadBalancer", "&LoadBalancerName=", $item["id"], "&Version=2012-06-01"]),
        https: true,
        headers:{
          "cache-control": "no-cache",
          "content-type": "application/json"
        }
      )
      $all_responses << $response
      call sys_log('CLB delete response',to_s($response))
    end
  end
end

define sys_log($subject, $detail) do
  # Create empty errors array if doesn't already exist
  if !$$errors
    $$errors = []
  end
  # Check if debug is enabled
  if $$debug
    # Append to global $$errors
    # This is the suggested way to capture errors
    $$errors << "Unexpected error for " + $subject + "\n  " + to_s($detail)
    # If Flexera NAM Zone, create audit_entries [to be deprecated]
    # This is the legacy method for capturing errors and only supported on Flexera NAM
    if $$rs_optima_host == "api.optima.flexeraeng.com"
      # skip_error_and_append is used to catch error if rs_cm.audit_entries.create fails unexpectedly
      $task_label = "Creating audit entry for " + $subject
      sub task_label: $task, on_error: skip_error_and_append($task) do
        rs_cm.audit_entries.create(
          notify: "None",
          audit_entry: {
            auditee_href: @@account,
            summary: $subject,
            detail: $detail
          }
        )
      end # End sub on_error
    end # End if rs_optima_host
  end # End if debug is enabled
end

define skip_error_and_append($subject) do
  $$errors << "Unexpected error for " + $subject + "\n  " + to_s($_error)
  $_error_behavior = "skip"
end