aws_rightsize_redshift.pt

name "AWS Rightsize Redshift"
rs_pt_ver 20180301
type "policy"
short_description "Reports any underutilized AWS Redshift nodes and resizes them after approval. See the [README](https://github.com/flexera-public/policy_templates/tree/master/cost/aws/rightsize_redshift/) and [docs.flexera.com/flexera/EN/Automation](https://docs.flexera.com/flexera/EN/Automation/AutomationGS.htm) to learn more."
long_description ""
category "Cost"
severity "low"
default_frequency "weekly"
info(
  version: "0.1.3",
  provider: "AWS",
  service: "Database",
  policy_set: "Rightsize Database Instances",
  recommendation_type: "Usage Reduction",
  hide_skip_approvals: "true"
)

###############################################################################
# Parameters
###############################################################################

parameter "param_email" do
  type "list"
  category "Policy Settings"
  label "Email Addresses"
  description "Email addresses of the recipients you wish to notify."
  default []
end

parameter "param_aws_account_number" do
  type "string"
  category "Policy Settings"
  label "Account Number"
  description "Leave blank; this is for automated use with Meta Policies. See README for more details."
  default ""
end

parameter "param_min_savings" do
  type "number"
  category "Policy Settings"
  label "Minimum Savings Threshold"
  description "Minimum potential savings required to generate a recommendation"
  min_value 0
  default 0
end

parameter "param_exclusion_tags" do
  type "list"
  category "Filters"
  label "Exclusion Tags"
  description "Cloud native tags to ignore clusters that you don't want to produce recommendations for. Enter the Key name to filter clusters with a specific Key, regardless of Value, and enter Key==Value to filter clusters with a specific Key:Value pair. Other operators and regex are supported; please see the README for more details."
  default []
end

parameter "param_exclusion_tags_boolean" do
  type "string"
  category "Filters"
  label "Exclusion Tags: Any / All"
  description "Whether to filter clusters containing any of the specified tags or only those that contain all of them. Only applicable if more than one value is entered in the 'Exclusion Tags' field."
  allowed_values "Any", "All"
  default "Any"
end

parameter "param_regions_allow_or_deny" do
  type "string"
  category "Filters"
  label "Allow/Deny Regions"
  description "Allow or Deny entered regions. See the README for more details"
  allowed_values "Allow", "Deny"
  default "Allow"
end

parameter "param_regions_list" do
  type "list"
  category "Filters"
  label "Allow/Deny Regions List"
  description "A list of allowed or denied regions. See the README for more details"
  allowed_pattern /^([a-zA-Z-_]+-[a-zA-Z0-9-_]+-[0-9-_]+,*|)+$/
  default []
end

parameter "param_stats_lookback" do
  type "number"
  category "Statistics"
  label "Statistic Lookback Period"
  description "How many days back to look at CPU data for clusters. This value cannot be set higher than 90 because AWS does not retain metrics for longer than 90 days."
  min_value 1
  max_value 90
  default 30
end

parameter "param_stats_threshold" do
  type "string"
  category "Statistics"
  label "Threshold Statistic"
  description "Statistic to use when determining if a cluster is underutilized"
  allowed_values "Average", "Maximum", "p99", "p95", "p90"
  default "Average"
end

parameter "param_stats_cpu_target" do
  type "number"
  category "Statistics"
  label "Maximum CPU Usage Target (%)"
  description "Maximum predicted CPU usage for recommended node types. For example, if set to '90', only recommendations where the new size would not result in > 90% CPU usage will be included in the results."
  min_value 1
  max_value 100
  default 90
end

parameter "param_automatic_action" do
  type "list"
  category "Actions"
  label "Automatic Actions"
  description "When this value is set, this policy will automatically take the selected action(s)"
  allowed_values ["Downsize Redshift Nodes"]
  default []
end

###############################################################################
# Authentication
###############################################################################

credentials "auth_aws" do
  schemes "aws", "aws_sts"
  label "AWS"
  description "Select the AWS Credential from the list"
  tags "provider=aws"
  aws_account_number $param_aws_account_number
end

credentials "auth_flexera" do
  schemes "oauth2"
  label "Flexera"
  description "Select Flexera One OAuth2 credentials"
  tags "provider=flexera"
end

###############################################################################
# Pagination
###############################################################################

pagination "pagination_aws_getmetricdata" do
  get_page_marker do
    body_path "NextToken"
  end
  set_page_marker do
    body_field "NextToken"
  end
end

###############################################################################
# Datasources & Scripts
###############################################################################

# Get applied policy metadata for use later
datasource "ds_applied_policy" do
  request do
    auth $auth_flexera
    host rs_governance_host
    path join(["/api/governance/projects/", rs_project_id, "/applied_policies/", policy_id])
    header "Api-Version", "1.0"
  end
end

# Get region-specific Flexera API endpoints
datasource "ds_flexera_api_hosts" do
  run_script $js_flexera_api_hosts, rs_optima_host
end

script "js_flexera_api_hosts", type: "javascript" do
  parameters "rs_optima_host"
  result "result"
  code <<-EOS
  host_table = {
    "api.optima.flexeraeng.com": {
      flexera: "api.flexera.com",
      fsm: "api.fsm.flexeraeng.com"
    },
    "api.optima-eu.flexeraeng.com": {
      flexera: "api.flexera.eu",
      fsm: "api.fsm-eu.flexeraeng.com"
    },
    "api.optima-apac.flexeraeng.com": {
      flexera: "api.flexera.au",
      fsm: "api.fsm-apac.flexeraeng.com"
    }
  }

  result = host_table[rs_optima_host]
EOS
end

# Get AWS account info
datasource "ds_cloud_vendor_accounts" do
  request do
    auth $auth_flexera
    host val($ds_flexera_api_hosts, 'flexera')
    path join(["/finops-analytics/v1/orgs/", rs_org_id, "/cloud-vendor-accounts"])
    header "Api-Version", "1.0"
  end
  result do
    encoding "json"
    collect jmes_path(response, "values[*]") do
      field "id", jmes_path(col_item, "aws.accountId")
      field "name", jmes_path(col_item, "name")
      field "tags", jmes_path(col_item, "tags")
    end
  end
end

datasource "ds_get_caller_identity" do
  request do
    auth $auth_aws
    host "sts.amazonaws.com"
    path "/"
    query "Action", "GetCallerIdentity"
    query "Version", "2011-06-15"
    header "User-Agent", "RS Policies"
  end
  result do
    encoding "xml"
    collect xpath(response, "//GetCallerIdentityResponse/GetCallerIdentityResult") do
      field "account", xpath(col_item, "Account")
    end
  end
end

datasource "ds_aws_account" do
  run_script $js_aws_account, $ds_cloud_vendor_accounts, $ds_get_caller_identity
end

script "js_aws_account", type:"javascript" do
  parameters "ds_cloud_vendor_accounts", "ds_get_caller_identity"
  result "result"
  code <<-EOS
  result = _.find(ds_cloud_vendor_accounts, function(account) {
    return account['id'] == ds_get_caller_identity[0]['account']
  })

  // This is in case the API does not return the relevant account info
  if (result == undefined) {
    result = {
      id: ds_get_caller_identity[0]['account'],
      name: "",
      tags: {}
    }
  }
EOS
end

datasource "ds_billing_centers" do
  request do
    auth $auth_flexera
    host rs_optima_host
    path join(["/analytics/orgs/", rs_org_id, "/billing_centers"])
    query "view", "allocation_table"
    header "Api-Version", "1.0"
    header "User-Agent", "RS Policies"
    ignore_status [403]
  end
  result do
    encoding "json"
    collect jmes_path(response, "[*]") do
      field "href", jmes_path(col_item, "href")
      field "id", jmes_path(col_item, "id")
      field "name", jmes_path(col_item, "name")
      field "parent_id", jmes_path(col_item, "parent_id")
    end
  end
end

# Gather top level billing center IDs for when we pull cost data
datasource "ds_top_level_bcs" do
  run_script $js_top_level_bcs, $ds_billing_centers
end

script "js_top_level_bcs", type: "javascript" do
  parameters "ds_billing_centers"
  result "result"
  code <<-EOS
  filtered_bcs = _.filter(ds_billing_centers, function(bc) {
    return bc['parent_id'] == null || bc['parent_id'] == undefined
  })

  result = _.compact(_.pluck(filtered_bcs, 'id'))
EOS
end

datasource "ds_currency_reference" do
  request do
    host "raw.githubusercontent.com"
    path "/flexera-public/policy_templates/master/data/currency/currency_reference.json"
    header "User-Agent", "RS Policies"
  end
end

datasource "ds_currency_code" do
  request do
    auth $auth_flexera
    host rs_optima_host
    path join(["/bill-analysis/orgs/", rs_org_id, "/settings/currency_code"])
    header "Api-Version", "0.1"
    header "User-Agent", "RS Policies"
    ignore_status [403]
  end
  result do
    encoding "json"
    field "id", jmes_path(response, "id")
    field "value", jmes_path(response, "value")
  end
end

datasource "ds_currency" do
  run_script $js_currency, $ds_currency_reference, $ds_currency_code
end

script "js_currency", type:"javascript" do
  parameters "ds_currency_reference", "ds_currency_code"
  result "result"
  code <<-EOS
  symbol = "$"
  separator = ","

  if (ds_currency_code['value'] != undefined) {
    if (ds_currency_reference[ds_currency_code['value']] != undefined) {
      symbol = ds_currency_reference[ds_currency_code['value']]['symbol']

      if (ds_currency_reference[ds_currency_code['value']]['t_separator'] != undefined) {
        separator = ds_currency_reference[ds_currency_code['value']]['t_separator']
      } else {
        separator = ""
      }
    }
  }

  result = {
    symbol: symbol,
    separator: separator
  }
EOS
end

datasource "ds_describe_regions" do
  request do
    auth $auth_aws
    host "ec2.amazonaws.com"
    path "/"
    query "Action", "DescribeRegions"
    query "Version", "2016-11-15"
    query "Filter.1.Name", "opt-in-status"
    query "Filter.1.Value.1", "opt-in-not-required"
    query "Filter.1.Value.2", "opted-in"
    # Header X-Meta-Flexera has no affect on datasource query, but is required for Meta Policies
    # Forces `ds_is_deleted` datasource to run first during policy execution
    header "Meta-Flexera", val($ds_is_deleted, "path")
  end
  result do
    encoding "xml"
    collect xpath(response, "//DescribeRegionsResponse/regionInfo/item", "array") do
      field "region", xpath(col_item, "regionName")
    end
  end
end

datasource "ds_regions" do
  run_script $js_regions, $ds_describe_regions, $param_regions_list, $param_regions_allow_or_deny
end

script "js_regions", type:"javascript" do
  parameters "ds_describe_regions", "param_regions_list", "param_regions_allow_or_deny"
  result "result"
  code <<-EOS
  allow_deny_test = { "Allow": true, "Deny": false }

  if (param_regions_list.length > 0) {
    result = _.filter(ds_describe_regions, function(item) {
      return _.contains(param_regions_list, item['region']) == allow_deny_test[param_regions_allow_or_deny]
    })
  } else {
    result = ds_describe_regions
  }
EOS
end

datasource "ds_redshift_clusters" do
  iterate $ds_regions
  request do
    auth $auth_aws
    host join(["redshift.", val(iter_item, "region"), ".amazonaws.com"])
    path "/"
    query "Action", "DescribeClusters"
    query "Version", "2012-12-01"
    header "User-Agent", "RS Policies"
    header "Content-Type", "text/xml"
  end
  result do
    encoding "xml"
    collect xpath(response, "//DescribeClustersResponse/DescribeClustersResult/Clusters/Cluster", "array") do
      field "id", xpath(col_item, "ClusterIdentifier")
      field "automatedSnapshotRetentionPeriod", xpath(col_item, "AutomatedSnapshotRetentionPeriod")
      field "dbName", xpath(col_item, "DBName")
      field "encrypted", xpath(col_item, "Encrypted")
      field "enhancedVpcRouting", xpath(col_item, "EnhancedVpcRouting")
      field "maintenanceTrackName", xpath(col_item, "MaintenanceTrackName")
      field "masterUsername", xpath(col_item, "MasterUsername")
      field "nodeType", xpath(col_item, "NodeType")
      field "numberOfNodes", xpath(col_item, "NumberOfNodes")
      field "publiclyAccessible", xpath(col_item, "PubliclyAccessible")
      field "revisionNumber", xpath(col_item, "ClusterRevisionNumber")
      field "subnetGroupName", xpath(col_item, "ClusterSubnetGroupName")
      field "version", xpath(col_item, "ClusterVersion")
      field "vpcId", xpath(col_item, "VpcId")
      field "tags" do
        collect xpath(col_item, "Tags/Tag", "array") do
          field "key", xpath(col_item, "Key")
          field "value", xpath(col_item, "Value")
        end
      end
      field "nodes" do
        collect xpath(col_item, "ClusterNodes/member") do
          field "privateIPAddress", xpath(col_item, "PrivateIPAddress")
          field "nodeRole", xpath(col_item, "NodeRole")
          field "publicIPAddress", xpath(col_item, "PublicIPAddress")
        end
      end
      field "region", val(iter_item, "region")
    end
  end
end

datasource "ds_aws_redshift_size_map" do
  request do
    host "raw.githubusercontent.com"
    path "/flexera-public/policy_templates/master/data/aws/redshift_types.json"
    header "User-Agent", "RS Policies"
  end
end

# Filter out resources that have no smaller size available to avoid wasting time getting metrics for them
datasource "ds_redshift_clusters_resize_filtered" do
  run_script $js_redshift_clusters_resize_filtered, $ds_redshift_clusters, $ds_aws_redshift_size_map
end

script "js_redshift_clusters_resize_filtered", type: "javascript" do
  parameters "ds_redshift_clusters", "ds_aws_redshift_size_map"
  result "result"
  code <<-'EOS'
  result = _.filter(ds_redshift_clusters, function(cluster) {
    return ds_aws_redshift_size_map[cluster['nodeType']] && ds_aws_redshift_size_map[cluster['nodeType']]['down']
  })
EOS
end

datasource "ds_redshift_clusters_tag_filtered" do
  run_script $js_redshift_clusters_tag_filtered, $ds_redshift_clusters_resize_filtered, $param_exclusion_tags, $param_exclusion_tags_boolean
end

script "js_redshift_clusters_tag_filtered", type: "javascript" do
  parameters "ds_redshift_clusters_resize_filtered", "param_exclusion_tags", "param_exclusion_tags_boolean"
  result "result"
  code <<-'EOS'
  comparators = _.map(param_exclusion_tags, function(item) {
    if (item.indexOf('==') != -1) {
      return { comparison: '==', key: item.split('==')[0], value: item.split('==')[1], string: item }
    }

    if (item.indexOf('!=') != -1) {
      return { comparison: '!=', key: item.split('!=')[0], value: item.split('!=')[1], string: item }
    }

    if (item.indexOf('=~') != -1) {
      value = item.split('=~')[1]
      regex = new RegExp(value.slice(1, value.length - 1))
      return { comparison: '=~', key: item.split('=~')[0], value: regex, string: item }
    }

    if (item.indexOf('!~') != -1) {
      value = item.split('!~')[1]
      regex = new RegExp(value.slice(1, value.length - 1))
      return { comparison: '!~', key: item.split('!~')[0], value: regex, string: item }
    }

    // If = is present but none of the above are, assume user error and that the user intended ==
    if (item.indexOf('=') != -1) {
      return { comparison: '==', key: item.split('=')[0], value: item.split('=')[1], string: item }
    }

    // Assume we're just testing for a key if none of the comparators are found
    return { comparison: 'key', key: item, value: null, string: item }
  })

  if (param_exclusion_tags.length > 0) {
    result = _.reject(ds_redshift_clusters_resize_filtered, function(resource) {
      resource_tags = {}

      if (typeof(resource['tags']) == 'object') {
        _.each(resource['tags'], function(tag) {
          resource_tags[tag['key']] = tag['value']
        })
      }

      // Store a list of found tags
      found_tags = []

      _.each(comparators, function(comparator) {
        comparison = comparator['comparison']
        value = comparator['value']
        string = comparator['string']
        resource_tag = resource_tags[comparator['key']]

        if (comparison == 'key' && resource_tag != undefined) { found_tags.push(string) }
        if (comparison == '==' && resource_tag == value) { found_tags.push(string) }
        if (comparison == '!=' && resource_tag != value) { found_tags.push(string) }

        if (comparison == '=~') {
          if (resource_tag != undefined && value.test(resource_tag)) { found_tags.push(string) }
        }

        if (comparison == '!~') {
          if (resource_tag == undefined) { found_tags.push(string) }
          if (resource_tag != undefined && value.test(resource_tag)) { found_tags.push(string) }
        }
      })

      all_tags_found = found_tags.length == comparators.length
      any_tags_found = found_tags.length > 0 && param_exclusion_tags_boolean == 'Any'

      return all_tags_found || any_tags_found
    })
  } else {
    result = ds_redshift_clusters_resize_filtered
  }
EOS
end

datasource "ds_cloudwatch_queries" do
  run_script $js_cloudwatch_queries, $ds_redshift_clusters_tag_filtered, $param_stats_lookback
end

script "js_cloudwatch_queries", type: "javascript" do
  parameters "ds_redshift_clusters_tag_filtered", "param_stats_lookback"
  result "result"
  code <<-EOS
  // Create the various queries we're going to send to CloudWatch for each instance
  result = {}

  _.each(ds_redshift_clusters_tag_filtered, function(cluster) {
    // Make sure the queries object has an array for the region to push items to
    if (result[cluster['region']] == undefined || result[cluster['region']] == null) {
      result[cluster['region']] = []
    }

    // We want to collect each of these list of statistics we care about
    stats = ["Average", "Minimum", "Maximum", "p99", "p95", "p90"]
    lookback = param_stats_lookback * 86400

    statId = cluster['id'].replace(/-/g, '_')

    _.each(stats, function(stat) {
      query = {
        "Id": statId + "___cpu" + stat,
        "MetricStat": {
          "Metric": {
            "Namespace": "AWS/Redshift",
            "MetricName": "CPUUtilization",
            "Dimensions": [
              { "Name": "ClusterIdentifier", "Value": cluster['id'] }
            ]
          },
          "Period": lookback,
          "Stat": stat
        },
        "ReturnData": true
      }

      result[cluster['region']].push(query)
    })
  })
EOS
end

# Combine queries into 500 item blocks so we can make bulk requests to Cloudwatch
datasource "ds_cloudwatch_requests" do
  run_script $js_cloudwatch_requests, $ds_cloudwatch_queries, $param_stats_lookback
end

script "js_cloudwatch_requests", type: "javascript" do
  parameters "ds_cloudwatch_queries", "param_stats_lookback"
  result "result"
  code <<-EOS
  // Organize the queries into discrete requests to send in.
  // Queries are first sorted by region and then split into 500 item blocks.
  result = []
  query_block_size = 500

  // Round down to beginning of the hour to avoid getting multiple values
  // from CloudWatch due to how the data is sliced
  end_date = new Date()
  end_date.setMinutes(0, 0, 0)
  end_date = parseInt(end_date.getTime() / 1000)

  start_date = new Date()
  start_date.setDate(start_date.getDate() - param_stats_lookback)
  start_date.setMinutes(0, 0, 0)
  start_date = parseInt(start_date.getTime() / 1000)

  _.each(_.keys(ds_cloudwatch_queries), function(region) {
    for (i = 0; i < ds_cloudwatch_queries[region].length; i += query_block_size) {
      chunk = ds_cloudwatch_queries[region].slice(i, i + query_block_size)

      result.push({
        body: {
          "StartTime": start_date,
          "EndTime": end_date,
          "MetricDataQueries": chunk
        },
        region: region
      })
    }
  })
EOS
end

datasource "ds_cloudwatch_data" do
  iterate $ds_cloudwatch_requests
  request do
    run_script $js_cloudwatch_data, val(iter_item, "region"), val(iter_item, "body")
  end
  result do
    encoding "json"
    collect jmes_path(response, "MetricDataResults[*]") do
      field "region", val(iter_item, "region")
      field "id", jmes_path(col_item, "Id")
      field "label", jmes_path(col_item, "Label")
      field "values", jmes_path(col_item, "Values")
    end
  end
end

script "js_cloudwatch_data", type: "javascript" do
  parameters "region", "body"
  result "request"
  code <<-EOS
  // Slow down rate of requests to prevent
  api_wait = 5
  var now = new Date().getTime()
  while(new Date().getTime() < now + (api_wait * 1000)) { /* Do nothing */ }

  var request = {
    auth: "auth_aws",
    host: 'monitoring.' + region + '.amazonaws.com',
    pagination: "pagination_aws_getmetricdata",
    verb: "POST",
    path: "/",
    headers: {
      "User-Agent": "RS Policies",
      "Content-Type": "application/json",
      "x-amz-target": "GraniteServiceVersion20100801.GetMetricData",
      "Accept": "application/json",
      "Content-Encoding": "amz-1.0"
    }
    query_params: {
      'Action': 'GetMetricData',
      'Version': '2010-08-01'
    },
    body: JSON.stringify(body)
  }
EOS
end

datasource "ds_cloudwatch_data_sorted" do
  run_script $js_cloudwatch_data_sorted, $ds_cloudwatch_data
end

script "js_cloudwatch_data_sorted", type: "javascript" do
  parameters "ds_cloudwatch_data"
  result "result"
  code <<-EOS
  // Sort the CloudWatch data into an object with keys for regions and instance names.
  // This eliminates the need to "double loop" later on to match it with our instances list.
  result = {}

  _.each(ds_cloudwatch_data, function(item) {
    region = item['region']
    id = item['id'].split('___')[0]
    metric = item['id'].split('___')[1]

    // Grabbing index 0 SHOULD be safe because we should only get one result.
    // Just in case AWS slices the data weirdly and returns 2 results, we make
    // sure we grab the last item every time, which contains the actual data we need.
    value = item['values'][item['values'].length - 1]

    if (result[region] == undefined) { result[region] = {} }
    if (result[region][id] == undefined) { result[region][id] = {} }

    result[region][id][metric] = value
  })
EOS
end

datasource "ds_redshift_clusters_with_metrics" do
  run_script $js_redshift_clusters_with_metrics, $ds_redshift_clusters_tag_filtered, $ds_cloudwatch_data_sorted
end

script "js_redshift_clusters_with_metrics", type: "javascript" do
  parameters "ds_redshift_clusters_tag_filtered", "ds_cloudwatch_data_sorted"
  result "result"
  code <<-EOS
  result = _.map(ds_redshift_clusters_tag_filtered, function(cluster) {
    region = cluster['region']
    statId = cluster['id'].replace(/-/g, '_')

    metrics = null
    cpuAverage = null
    cpuMinimum = null
    cpuMaximum = null
    cpuP90 = null
    cpuP95 = null
    cpuP99 = null

    if (ds_cloudwatch_data_sorted[region] && ds_cloudwatch_data_sorted[region][statId]) {
      metrics = ds_cloudwatch_data_sorted[region][statId]
    }

    if (metrics && metrics["cpuAverage"]) { cpuAverage = metrics["cpuAverage"] }
    if (metrics && metrics["cpuMinimum"]) { cpuMinimum = metrics["cpuMinimum"] }
    if (metrics && metrics["cpuMaximum"]) { cpuMaximum = metrics["cpuMaximum"] }
    if (metrics && metrics["cpup90"]) { cpuP90 = metrics["cpup90"] }
    if (metrics && metrics["cpup95"]) { cpuP95 = metrics["cpup95"] }
    if (metrics && metrics["cpup99"]) { cpuP99 = metrics["cpup99"] }

    tags = []

    if (typeof(cluster['tags']) == 'object') {
      tags = _.map(_.keys(cluster['tags']), function(key) { return [ key, cluster['tags'][key] ].join('=') })
    }

    return {
      id: cluster['id'],
      automatedSnapshotRetentionPeriod: cluster['automatedSnapshotRetentionPeriod'],
      dbName: cluster['dbName'],
      encrypted: cluster['encrypted'],
      enhancedVpcRouting: cluster['enhancedVpcRouting'],
      maintenanceTrackName: cluster['maintenanceTrackName'],
      masterUsername: cluster['masterUsername'],
      nodes: cluster['nodes'],
      nodeType: cluster['nodeType'],
      numberOfNodes: cluster['numberOfNodes'],
      publiclyAccessible: cluster['publiclyAccessible'],
      revisionNumber: cluster['revisionNumber'],
      subnetGroupName: cluster['subnetGroupName'],
      version: cluster['version'],
      vpcId: cluster['vpcId'],
      region: cluster['region'],
      tags: tags.join(', '),
      cpuAverage: cpuAverage,
      cpuMinimum: cpuMinimum,
      cpuMaximum: cpuMaximum,
      cpuP90: cpuP90,
      cpuP95: cpuP95,
      cpuP99: cpuP99
    }
  })
EOS
end

datasource "ds_cluster_costs" do
  request do
    run_script $js_cluster_costs, $ds_aws_account, $ds_top_level_bcs, rs_org_id, rs_optima_host
  end
  result do
    encoding "json"
    collect jmes_path(response, "rows[*]") do
      field "resourceId", jmes_path(col_item, "dimensions.resource_id")
      field "resourceType", jmes_path(col_item, "dimensions.resource_type")
      field "vendorAccountName", jmes_path(col_item, "dimensions.vendor_account_name")
      field "adjustmentName", jmes_path(col_item, "dimensions.adjustment_name")
      field "cost", jmes_path(col_item, "metrics.cost_amortized_unblended_adj")
    end
  end
end

script "js_cluster_costs", type: "javascript" do
  parameters "ds_aws_account", "ds_top_level_bcs", "rs_org_id", "rs_optima_host"
  result "request"
  code <<-EOS
  end_date = new Date()
  end_date.setDate(end_date.getDate() - 2)
  end_date = end_date.toISOString().split('T')[0]

  start_date = new Date()
  start_date.setDate(start_date.getDate() - 3)
  start_date = start_date.toISOString().split('T')[0]

  var request = {
    auth: "auth_flexera",
    host: rs_optima_host,
    verb: "POST",
    path: "/bill-analysis/orgs/" + rs_org_id + "/costs/select",
    body_fields: {
      dimensions: ["resource_id", "vendor_account_name", "resource_type", "adjustment_name"],
      granularity: "day",
      start_at: start_date,
      end_at: end_date,
      metrics: ["cost_amortized_unblended_adj"],
      billing_center_ids: ds_top_level_bcs,
      limit: 100000,
      filter: {
        type: "and",
        expressions: [
          {
            dimension: "service",
            type: "equal",
            value: "AmazonRedshift"
          },
          {
            dimension: "vendor_account",
            type: "equal",
            value: ds_aws_account['id']
          },
          {
            type: "not",
            expression: {
              dimension: "adjustment_name",
              type: "substring",
              substring: "Shared"
            }
          }
        ]
      }
    },
    headers: {
      'User-Agent': "RS Policies",
      'Api-Version': "1.0"
    },
    ignore_status: [400]
  }
EOS
end

datasource "ds_cluster_costs_grouped" do
  run_script $js_cluster_costs_grouped, $ds_cluster_costs
end

script "js_cluster_costs_grouped", type: "javascript" do
  parameters "ds_cluster_costs"
  result "result"
  code <<-EOS
  // Multiple a single day's cost by the average number of days in a month.
  // The 0.25 is to account for leap years for extra precision.
  cost_multiplier = 365.25 / 12

  // Group cost data by resourceId for later use
  result = {}

  _.each(ds_cluster_costs, function(item) {
    if (typeof(item['resourceId']) == 'string' && item['resourceId'].indexOf("cluster:") == 0) {
      id = item['resourceId'].split("cluster:")[1].toLowerCase()

      if (result[id] == undefined) { result[id] = 0.0 }
      result[id] += item['cost'] * cost_multiplier
    }
  })
EOS
end

datasource "ds_underutilized_redshift_clusters" do
  run_script $js_underutilized_redshift_clusters, $ds_redshift_clusters_with_metrics, $ds_aws_redshift_size_map, $ds_cluster_costs_grouped, $ds_aws_account, $ds_currency, $ds_applied_policy, $param_stats_lookback, $param_stats_threshold, $param_min_savings, $param_stats_cpu_target
end

script "js_underutilized_redshift_clusters", type: "javascript" do
  parameters "ds_redshift_clusters_with_metrics", "ds_aws_redshift_size_map", "ds_cluster_costs_grouped", "ds_aws_account", "ds_currency", "ds_applied_policy", "param_stats_lookback", "param_stats_threshold", "param_min_savings", "param_stats_cpu_target"
  result "result"
  code <<-'EOS'
  // Used for formatting numbers to look pretty
  function formatNumber(number, separator) {
    formatted_number = "0"

    if (number) {
      formatted_number = (Math.round(number * 100) / 100).toString().split(".")[0]

      if (separator) {
        withSeparator = ""

        for (var i = 0; i < formatted_number.length; i++) {
          if (i > 0 && (formatted_number.length - i) % 3 == 0) { withSeparator += separator }
          withSeparator += formatted_number[i]
        }

        formatted_number = withSeparator
      }

      decimal = (Math.round(number * 100) / 100).toString().split(".")[1]
      if (decimal) { formatted_number += "." + decimal }
    }

    return formatted_number
  }

  result = []
  total_savings = 0.0

  // Find specs for current and potential downsize in the size map
  _.each(ds_redshift_clusters_with_metrics, function(cluster) {
    currentType = null
    potentialType = null
    newResourceType = null

    if (ds_aws_redshift_size_map[cluster['nodeType']] && ds_aws_redshift_size_map[cluster['nodeType']]['down']) {
      currentType = ds_aws_redshift_size_map[cluster['nodeType']]
      newResourceType = ds_aws_redshift_size_map[cluster['nodeType']]['down']
      potentialType = ds_aws_redshift_size_map[newResourceType]
    }

    // Calculate threshold for CPU usage to recommend a downgrade based on difference
    rawThreshold = null
    cpuThreshold = null

    if (potentialType) {
      percentage = (100 - param_stats_cpu_target) / 100
      rawThreshold = potentialType["vcpu"] / currentType["vcpu"]
      cpuThreshold = rawThreshold - (percentage * rawThreshold)
    }

    // Store the appropriate comparison stat
    stats = {
      "Average": cluster["cpuAverage"],
      "Maximum": cluster["cpuMaximum"],
      "p99": cluster["cpuP90"],
      "p95": cluster["cpuP95"],
      "p90": cluster["cpuP99"]
    }

    // Calculate estimated savings
    cost = ds_cluster_costs_grouped[cluster['id'].toLowerCase()]
    savings = 0.0

    if (cost && rawThreshold) {
      newCost = cost * rawThreshold
      savings = cost - newCost
    }

    // Check if CPU usage is below the calculated threshold and if savings is above threshold
    if (cpuThreshold && stats[param_stats_threshold] < cpuThreshold && savings >= param_min_savings) {
      total_savings += savings

      resourceARN = [
        "arn:aws:redshift:", cluster['region'], ":",
        ds_aws_account['id'], ":cluster/", cluster['id']
      ].join('')

      recommendationDetails = [
        "Change node type of Redshift cluster ", cluster['id'], " ",
        "in AWS Account ", ds_aws_account['name'], " ",
        "(", ds_aws_account['id'], ") ",
        "from ", cluster['nodeType'], " ",
        "to ", newResourceType
      ].join('')

      result.push({
        accountID: ds_aws_account['id'],
        accountName: ds_aws_account['name'],
        id: cluster['id'],
        resourceID: cluster['id'],
        resourceName: cluster['id'],
        automatedSnapshotRetentionPeriod: cluster['automatedSnapshotRetentionPeriod'],
        dbName: cluster['dbName'],
        encrypted: cluster['encrypted'],
        enhancedVpcRouting: cluster['enhancedVpcRouting'],
        maintenanceTrackName: cluster['maintenanceTrackName'],
        masterUsername: cluster['masterUsername'],
        nodes: cluster['nodes'],
        numberOfNodes: cluster['numberOfNodes'],
        publiclyAccessible: cluster['publiclyAccessible'],
        revisionNumber: cluster['revisionNumber'],
        subnetGroupName: cluster['subnetGroupName'],
        version: cluster['version'],
        vpcId: cluster['vpcId'],
        region: cluster['region'],
        tags: cluster['tags'],
        cpuAverage: Math.round(cluster['cpuAverage'] * 100) / 100,
        cpuMinimum: Math.round(cluster['cpuMinimum'] * 100) / 100,
        cpuMaximum: Math.round(cluster['cpuMaximum'] * 100) / 100,
        cpuP90: Math.round(cluster['cpuP90'] * 100) / 100,
        cpuP95: Math.round(cluster['cpuP95'] * 100) / 100,
        cpuP99: Math.round(cluster['cpuP99'] * 100) / 100,
        resourceType: cluster['nodeType'],
        newResourceType: newResourceType,
        resourceARN: resourceARN,
        service: "AmazonRedshift",
        recommendationDetails: recommendationDetails,
        savings: Math.round(savings * 1000) / 1000,
        savingsCurrency: ds_currency['symbol'],
        policy_name: ds_applied_policy['name'],
        // These are to avoid errors when we hash_exclude these fields
        message: "",
        total_savings: ""
      })
    }
  })

  // Message for incident output
  total_clusters = ds_redshift_clusters_with_metrics.length.toString()
  total_oversized = result.length.toString()
  oversized_percentage = (total_oversized / total_clusters * 100).toFixed(2).toString() + '%'

  cluster_noun = "cluster"
  if (total_clusters > 1) { cluster_noun += "s" }

  cluster_verb = "is"
  if (total_oversized > 1) { cluster_verb = "are" }

  findings = [
    "Out of ", total_clusters, " AWS Redshift ", cluster_noun, " analyzed, ",
    total_oversized, " (", oversized_percentage,
    ") ", cluster_verb, " underutilized and recommended for downsizing. "
  ].join('')

  settings = [
    "A cluster is consider underutilized if there exists a smaller size whose estimated ",
    param_stats_threshold.toLowerCase(), " CPU usage would be below ", param_stats_cpu_target, "%.\n\n"
  ].join('')

  disclaimer = "The above settings can be modified by editing the applied policy and changing the appropriate parameters."

  total_savings = ds_currency['symbol'] + ' ' + formatNumber(Math.round(total_savings * 100) / 100, ds_currency['separator'])

  // Dummy item to ensure the policy's check statement always executes at least once
  result.push({
    accountID: "",
    accountName: "",
    id: "",
    resourceID: "",
    resourceName: "",
    automatedSnapshotRetentionPeriod: "",
    dbName: "",
    encrypted: "",
    enhancedVpcRouting: "",
    maintenanceTrackName: "",
    masterUsername: "",
    nodes: "",
    numberOfNodes: "",
    publiclyAccessible: "",
    revisionNumber: "",
    subnetGroupName: "",
    version: "",
    vpcId: "",
    region: "",
    tags: "",
    cpuAverage: "",
    cpuMinimum: "",
    cpuMaximum: "",
    cpuP90: "",
    cpuP95: "",
    cpuP99: "",
    resourceType: "",
    newResourceType: "",
    resourceARN: "",
    service: "",
    recommendationDetails: "",
    savings: "",
    savingsCurrency: "",
    policy_name: "",
    message: "",
    total_savings: ""
  })

  result[0]['message'] = findings + settings + disclaimer
  result[0]['total_savings'] = total_savings
EOS
end

###############################################################################
# Policy
###############################################################################

policy "pol_underutilized_redshift_clusters" do
  validate_each $ds_underutilized_redshift_clusters do
    summary_template "{{ with index data 0 }}{{ .policy_name }}{{ end }}: {{ len data }} AWS Underutilized Redshift Clusters Found"
    detail_template <<-'EOS'
    **Potential Monthly Savings:** {{ with index data 0 }}{{ .total_savings }}{{ end }}

    {{ with index data 0 }}{{ .message }}{{ end }}
    EOS
    check logic_or($ds_parent_policy_terminated, eq(val(item, "resourceID"), ""))
    escalate $esc_email
    escalate $esc_resize_clusters
    hash_exclude "message", "total_savings", "tags", "savings", "savingsCurrency"
    export do
      resource_level true
      field "accountID" do
        label "Account ID"
      end
      field "accountName" do
        label "Account Name"
      end
      field "resourceID" do
        label "Resource ID"
      end
      field "region" do
        label "Region"
      end
      field "tags" do
        label "Resource Tags"
      end
      field "numberOfNodes" do
        label "Nodes (#)"
      end
      field "resourceType" do
        label "Node Type"
      end
      field "newResourceType" do
        label "Recommended Node Type"
      end
      field "recommendationDetails" do
        label "Recommendation"
      end
      field "savings" do
        label "Estimated Monthly Savings"
      end
      field "savingsCurrency" do
        label "Savings Currency"
      end
      field "cpuMaximum" do
        label "CPU Maximum %"
      end
      field "cpuMinimum" do
        label "CPU Minimum %"
      end
      field "cpuAverage" do
        label "CPU Average %"
      end
      field "cpuP99" do
        label "CPU p99"
      end
      field "cpuP95" do
        label "CPU p95"
      end
      field "cpuP90" do
        label "CPU p90"
      end
      field "vpcId" do
        label "VPC ID"
      end
      field "version" do
        label "Version"
      end
      field "service" do
        label "Service"
      end
      field "resourceARN" do
        label "Resource ARN"
      end
      field "resourceName" do
        label "Resource Name"
      end
      field "id" do
        label "ID"
      end
    end
  end
end

###############################################################################
# Escalations
###############################################################################

escalation "esc_email" do
  automatic true
  label "Send Email"
  description "Send incident email"
  email $param_email
end

escalation "esc_resize_clusters" do
  automatic contains($param_automatic_action, "Downsize Redshift Nodes")
  label "Downsize Redshift Nodes"
  description "Approval to downsize nodes in all selected Redshift clusters"
  run "resize_clusters", data
end

###############################################################################
# Cloud Workflow
###############################################################################

define resize_clusters($data) return $all_responses do
  $$all_responses = []

  foreach $clb in $data do
    sub on_error: handle_error() do
      call resize_cluster($cluster) retrieve $response
    end
  end

  if inspect($$errors) != "null"
    raise join($$errors, "\n")
  end
end

define resize_cluster($cluster) return $response do
  $host = "redshift." + $cluster["region"] + ".amazonaws.com"
  $href = "/"
  $params = "?Action=ModifyCluster&Version=2012-12-01&ClusterIdentifier=" + $cluster["resourceID"] + "&NodeType=" + $cluster["newResourceType"] + "&NumberOfNodes=" + $cluster["numberOfNodes"]
  $url = $host + $href + $params
  task_label("GET " + $url)

  $response = http_request(
    auth: $$auth_aws,
    https: true,
    verb: "get",
    host: $host,
    href: $href,
    query_strings: {
      "Action": "ModifyCluster",
      "Version": "2012-12-01",
      "ClusterIdentifier": $cluster["resourceID"],
      "NodeType": $cluster["newResourceType"],
      "NumberOfNodes": $cluster["numberOfNodes"]
    }
  )

  task_label("GET AWS Redshift cluster response: " + $cluster["resourceID"] + " " + to_json($response))
  $$all_responses << to_json({"req": "GET " + $url, "resp": $response})

  if $response["code"] != 204 && $response["code"] != 202 && $response["code"] != 200
    raise "Unexpected response from GET AWS Redshift cluster: "+ $cluster["resourceID"] + " " + to_json($response)
  else
    task_label("GET AWS Redshift cluster successful: " + $cluster["resourceID"])
  end
end

define handle_error() do
  if !$$errors
    $$errors = []
  end
  $$errors << $_error["type"] + ": " + $_error["message"]
  # We check for errors at the end, and raise them all together
  # Skip errors handled by this definition
  $_error_behavior = "skip"
end

###############################################################################
# Meta Policy [alpha]
# Not intended to be modified or used by policy developers
###############################################################################

# If the meta_parent_policy_id is not set it will evaluate to an empty string and we will look for the policy itself,
# if it is set we will look for the parent policy.
datasource "ds_get_policy" do
  request do
    auth $auth_flexera
    host rs_governance_host
    ignore_status [404]
    path join(["/api/governance/projects/", rs_project_id, "/applied_policies/", switch(ne(meta_parent_policy_id, ""), meta_parent_policy_id, policy_id)])
    header "Api-Version", "1.0"
  end
  result do
    encoding "json"
    field "id", jmes_path(response, "id")
  end
end

datasource "ds_parent_policy_terminated" do
  run_script $js_decide_if_self_terminate, $ds_get_policy, policy_id, meta_parent_policy_id
end

# If the policy was applied by a meta_parent_policy we confirm it exists if it doesn't we confirm we are deleting
# This information is used in two places:
# - determining whether or not we make a delete call
# - determining if we should create an incident (we don't want to create an incident on the run where we terminate)
script "js_decide_if_self_terminate", type: "javascript" do
  parameters "found", "self_policy_id", "meta_parent_policy_id"
  result "result"
  code <<-EOS
  var result
  if (meta_parent_policy_id != "" && found.id == undefined) {
    result = true
  } else {
    result = false
  }
EOS
end

# Two potentials ways to set this up:
# - this way and make a unneeded 'get' request when not deleting
# - make the delete request an interate and have it iterate over an empty array when not deleting and an array with one item when deleting
script "js_make_terminate_request", type: "javascript" do
  parameters "should_delete", "policy_id", "rs_project_id", "rs_governance_host"
  result "request"
  code <<-EOS

  var request = {
    auth:  'auth_flexera',
    host: rs_governance_host,
    path: "/api/governance/projects/" + rs_project_id + "/applied_policies/" + policy_id,
    headers: {
      "API-Version": "1.0",
      "Content-Type":"application/json"
    },
  }

  if (should_delete) {
    request.verb = 'DELETE'
  }
  EOS
end

datasource "ds_terminate_self" do
  request do
    run_script $js_make_terminate_request, $ds_parent_policy_terminated, policy_id, rs_project_id, rs_governance_host
  end
end

datasource "ds_is_deleted" do
  run_script $js_check_deleted, $ds_terminate_self
end

# This is just a way to have the check delete request connect to the farthest leaf from policy.
# We want the delete check to the first thing the policy does to avoid the policy erroring before it can decide whether or not it needs to self terminate
# Example a customer deletes a credential and then terminates the parent policy. We still want the children to self terminate
# The only way I could see this not happening is if the user who applied the parent_meta_policy was offboarded or lost policy access, the policies who are impersonating the user
# would not have access to self-terminate
# It may be useful for the backend to enable a mass terminate at some point for all meta_child_policies associated with an id.
script "js_check_deleted", type: "javascript" do
  parameters "response"
  result "result"
  code <<-EOS
  result = {"path":"/"}
  EOS
end