diff --git a/cost/flexera/cco/scheduled_report_unallocated/CHANGELOG.md b/cost/flexera/cco/scheduled_report_unallocated/CHANGELOG.md index 51568d4667..f735edddfb 100644 --- a/cost/flexera/cco/scheduled_report_unallocated/CHANGELOG.md +++ b/cost/flexera/cco/scheduled_report_unallocated/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## v0.3.0 + +- Fixed issue that would cause policy template to fail when "Last 7 Days" was selected for the "Date Range" parameter. +- "Dimensions List" parameter now accepts both dimension names and dimension IDs as valid inputs. +- Markdown tables in incident now uses pretty names for various fields to improve readability. + ## v0.2.2 - Added `hide_skip_approvals` field to the info section. It dynamically controls "Skip Action Approvals" visibility. diff --git a/cost/flexera/cco/scheduled_report_unallocated/README.md b/cost/flexera/cco/scheduled_report_unallocated/README.md index 64b26b9bcc..0205ea9b56 100644 --- a/cost/flexera/cco/scheduled_report_unallocated/README.md +++ b/cost/flexera/cco/scheduled_report_unallocated/README.md @@ -7,11 +7,11 @@ This policy allows you to set up scheduled reports that will provide summaries o ## Input Parameters - *Email List* - Email addresses of the recipients you wish to notify -- *Dimensions List* - List of Dimensions you want to report on (i.e. billing_center_id). Must provide at least one dimension +- *Dimensions List* - List of Dimension names/IDs you want to report on. Must provide at least one dimension. Examples: Billing Centers, Services, vendor_account - *Cost Filters* - JSON object (as a string) of filters to apply to the report. Example: `{\"dimension\": \"vendor\",\"type\": \"equal\",\"value\": \"aws\"}` - *Cost Metric* - The cost metric for the report - *Date Range" - Date Range for the Report -- *Filter Report Percent Threshold* - Filter out rows where the cost metric is less than this percentage of the total spend in the report. Enter 0 to show all rows +- *Filter Report Percent Threshold* - Filter out rows where the cost metric is less than this percentage of the total spend in the report. Enter 0 to show all rows ## Policy Actions diff --git a/cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt b/cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt index 0ac16b2cc3..f3e3f67024 100644 --- a/cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt +++ b/cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt @@ -6,12 +6,11 @@ long_description "" severity "low" category "Cost" default_frequency "monthly" -tenancy "single" info( - version: "0.2.2", - provider: "Flexera Optima", - service: "", - policy_set: "", + version: "0.3.0", + provider: "Flexera", + service: "Cloud Cost Optimization", + policy_set: "Cloud Cost Optimization", hide_skip_approvals: "true" ) @@ -31,15 +30,15 @@ parameter "param_cost_dimensions" do type "list" category "Policy Settings" label "Dimensions List" - description "List of Dimensions you want to report on (i.e. billing_center_id). Must provide at least one dimension" - default ["billing_center_id"] + description "List of Dimension names/IDs you want to report on. Must provide at least one dimension. Examples: Billing Centers, Services, vendor_account" + default ["Billing Centers"] end parameter "param_cost_filters" do type "string" category "Policy Settings" label "Cost Filters" - description "JSON object of filters to apply to the report. Example: `{\"dimension\": \"vendor\", \"type\": \"equal\", \"value\": \"aws\"}`" + description "JSON object of filters to apply to the report. Example: `{\"dimension\": \"vendor\", \"type\": \"equal\", \"value\": \"aws\"}`" default "" end @@ -86,6 +85,51 @@ end # Datasources & Scripts ############################################################################### +datasource "ds_get_dimensions" do + request do + auth $auth_flexera + host rs_optima_host + path join(["/bill-analysis/orgs/", rs_org_id, "/costs/dimensions"]) + header "Api-Version", "0.1" + header "User-Agent", "RS Policies" + end + result do + encoding "json" + collect jmes_path(response, "dimensions[].{id: id, name: name, type:type}") do + field "id", jmes_path(col_item, "id") + field "name", jmes_path(col_item, "name") + field "type", jmes_path(col_item, "type") + end + end +end + +datasource "ds_cost_dimensions" do + run_script $js_cost_dimensions, $ds_get_dimensions, $param_cost_dimensions +end + +script "js_cost_dimensions", type:"javascript" do + parameters "ds_get_dimensions", "param_cost_dimensions" + result "result" + code <<-EOS + dimension_ids = _.pluck(ds_get_dimensions, 'id') + dimension_names = _.pluck(ds_get_dimensions, 'name') + + dimensions = _.map(param_cost_dimensions, function(item) { + if (item.toLowerCase().trim() == "billing centers" || item.toLowerCase().trim() == "billing_center_id") { + return { id: "billing_center_id", name: "Billing Centers" } + } else if (_.contains(dimension_ids, item)) { + return _.find(ds_get_dimensions, function(dimension) { return dimension['id'] == item }) + } else if (_.contains(dimension_names, item)) { + return _.find(ds_get_dimensions, function(dimension) { return dimension['name'] == item }) + } else { + return null + } + }) + + result = _.compact(dimensions) +EOS +end + datasource "ds_currency_reference" do request do host "raw.githubusercontent.com" @@ -139,7 +183,7 @@ script "js_currency", type:"javascript" do EOS end -datasource "ds_toplevel_billing_centers" do +datasource "ds_billing_centers" do request do auth $auth_flexera host rs_optima_host @@ -147,21 +191,34 @@ datasource "ds_toplevel_billing_centers" do query "view", "allocation_table" header "Api-Version", "1.0" header "User-Agent", "RS Policies" + ignore_status [403] end result do encoding "json" - # Select the Billing Centers that have "parent_id" undefined or "" (i.e. top-level Billing Centers) - collect jq(response, '.[] | select(.parent_id == null)' ) do - field "href", jq(col_item, ".href") - field "id", jq(col_item, ".id") - field "name", jq(col_item, ".name") - field "parent_id", jq(col_item, ".parent_id") - field "ancestor_ids", jq(col_item, ".ancestor_ids") - field "allocation_table", jq(col_item, ".allocation_table") + collect jmes_path(response, "[*]") do + field "href", jmes_path(col_item, "href") + field "id", jmes_path(col_item, "id") + field "name", jmes_path(col_item, "name") + field "parent_id", jmes_path(col_item, "parent_id") end end end +# Gather top level billing center IDs for when we pull cost data +datasource "ds_top_level_bcs" do + run_script $js_top_level_bcs, $ds_billing_centers +end + +script "js_top_level_bcs", type: "javascript" do + parameters "ds_billing_centers" + result "result" + code <<-EOS + result = _.filter(ds_billing_centers, function(bc) { + return bc['parent_id'] == null || bc['parent_id'] == undefined + }) +EOS +end + datasource "ds_cost_request_params" do run_script $js_cost_request_params, $param_cost_time_period, $param_cost_filters end @@ -170,58 +227,49 @@ script "js_cost_request_params", type:"javascript" do parameters "param_cost_time_period", "param_cost_filters" result "result" code <<-EOS - filter = null - if (param_cost_filters.length > 0) { - filter = JSON.parse(param_cost_filters) - } + now = new Date() + result = { filter: null } + if (param_cost_filters.length > 0) { result['filter'] = JSON.parse(param_cost_filters) } - var now = new Date() - var start_at - var end_at switch(param_cost_time_period) { case "Last 7 Days": - start_at = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7); - end_at = now; - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7).toISOString().substring(0, 10) + result['end_at'] = now.toISOString().substring(0, 10) + result['granularity'] = "day" + break case "Last 30 Days": - start_at = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 30); - end_at = now; - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 30).toISOString().substring(0, 7) + result['end_at'] = now.toISOString().substring(0, 7) + result['granularity'] = "day" + break case "Last 45 Days": - start_at = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 45); - end_at = now; - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 45).toISOString().substring(0, 7) + result['end_at'] = now.toISOString().substring(0, 7) + result['granularity'] = "month" + break case "Last 90 Days": - start_at = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 90); - end_at = now; - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 90).toISOString().substring(0, 7) + result['end_at'] = now.toISOString().substring(0, 7) + result['granularity'] = "month" + break case "Previous Month": - start_at = new Date(now.getFullYear(), now.getMonth() - 1, 1); - end_at = new Date(now.getFullYear(), now.getMonth(), 0); - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth() - 1, 1).toISOString().substring(0, 7) + result['end_at'] = new Date(now.getFullYear(), now.getMonth(), 0).toISOString().substring(0, 7) + result['granularity'] = "month" + break case "Previous 3 Months": - start_at = new Date(now.getFullYear(), now.getMonth() - 3, 1); - end_at = new Date(now.getFullYear(), now.getMonth(), 0); - break; + result['start_at'] = new Date(now.getFullYear(), now.getMonth() - 3, 1).toISOString().substring(0, 7) + result['end_at'] = new Date(now.getFullYear(), now.getMonth(), 0).toISOString().substring(0, 7) + result['granularity'] = "month" + break } - - // Formet start_at and end_at into strings with format YYYY-MM - start_at_string = start_at.getFullYear() + "-" + ("0" + (start_at.getMonth() + 1)).slice(-2) - end_at_string = end_at.getFullYear() + "-" + ("0" + (end_at.getMonth() + 1)).slice(-2) - - result = { - "filter": filter, - "start_at": start_at_string, - "end_at": end_at_string - } - EOS end datasource "ds_costs_aggregated" do - iterate $ds_toplevel_billing_centers + iterate $ds_top_level_bcs request do - run_script $js_costs_aggregated, val(iter_item, "id"), $ds_cost_request_params, $param_cost_dimensions, $param_cost_metric, $param_cost_time_period, rs_optima_host, rs_org_id + run_script $js_costs_aggregated, val(iter_item, "id"), $ds_cost_request_params, $ds_cost_dimensions, $param_cost_metric, $param_cost_time_period, rs_optima_host, rs_org_id end result do encoding "json" @@ -235,10 +283,10 @@ datasource "ds_costs_aggregated" do end script "js_costs_aggregated", type: "javascript" do - parameters "bc_id", "ds_cost_request_params", "param_cost_dimensions", "param_cost_metric", "param_cost_time_period", "rs_optima_host", "rs_org_id" + parameters "bc_id", "ds_cost_request_params", "ds_cost_dimensions", "param_cost_metric", "param_cost_time_period", "rs_optima_host", "rs_org_id" result "request" code <<-EOS - var cost_metric = { + cost_metric = { "Unamortized Unblended": "cost_nonamortized_unblended_adj", "Amortized Unblended": "cost_amortized_unblended_adj", "Unamortized Blended": "cost_nonamortized_blended_adj", @@ -252,8 +300,8 @@ script "js_costs_aggregated", type: "javascript" do path: "/bill-analysis/orgs/" + rs_org_id + "/costs/aggregated", body_fields: { "filter": ds_cost_request_params["filter"], - "dimensions": param_cost_dimensions, - "granularity": "month", + "dimensions": _.pluck(ds_cost_dimensions, 'id'), + "granularity": ds_cost_request_params["granularity"], "start_at": ds_cost_request_params["start_at"], "end_at": ds_cost_request_params["end_at"], "metrics": [cost_metric[param_cost_metric]], @@ -268,149 +316,148 @@ EOS end datasource "ds_costs_flattened" do - run_script $js_costs_flattened, $ds_currency, $ds_costs_aggregated, $ds_cost_request_params, $param_cost_metric, $param_cost_dimensions + run_script $js_costs_flattened, $ds_currency, $ds_costs_aggregated, $ds_cost_request_params, $ds_cost_dimensions, $ds_billing_centers, $param_cost_metric end script "js_costs_flattened", type: "javascript" do - parameters "ds_currency", "ds_costs_aggregated", "ds_cost_request_params", "param_cost_metric", "param_cost_dimensions" + parameters "ds_currency", "ds_costs_aggregated", "ds_cost_request_params", "ds_cost_dimensions", "ds_billing_centers", "param_cost_metric" result "result" code <<-EOS - var cost_metric = { + cost_metric = { "Unamortized Unblended": "cost_nonamortized_unblended_adj", "Amortized Unblended": "cost_amortized_unblended_adj", "Unamortized Blended": "cost_nonamortized_blended_adj", "Amortized Blended": "cost_amortized_blended_adj" } - var total_spend = 0 - var unallocated_spend = 0 - rows = ds_costs_aggregated.map(function(row) { - var flattened = { - "unallocated": false, - "unallocated_details": "" - } + + metric_name = param_cost_metric + " Spend" + + bc_names = {} + _.each(ds_billing_centers, function(bc) { bc_names[bc['id']] = bc['name'] }) + + dimensions_table = {} + _.each(ds_cost_dimensions, function(dimension) { dimensions_table[dimension['id']] = dimension['name'] }) + + total_spend = 0 + unallocated_spend = 0 + + rows = _.map(ds_costs_aggregated, function(row) { + flattened = { "Unallocated": false } unallocatedDetailsList = [] - _.each(row.metrics, function(value, key) { - if (key == cost_metric[param_cost_metric]) { - total_spend = total_spend + value - } - // Round the metric values to 2 decimal places for report - if (typeof value == "number") { - flattened[key] = Math.round(value * 100) / 100 + + flattened[metric_name] = row['metrics'][cost_metric[param_cost_metric]] + total_spend += flattened[metric_name] + + _.each(row['dimensions'], function(value, key) { + if (dimensions_table[key]) { + flattened[dimensions_table[key]] = value } else { flattened[key] = value } - }) - _.each(row.dimensions, function(value, key) { - flattened[key] = value - // If billing_center_id dimension, add billing_center_name dimension to results - if (key == "billing_center_id") { - flattened["billing_center_name"] = row.bc_name - } + + // If billing_center_id dimension, add name dimension to results + if (key == "billing_center_id") { flattened["Billing Centers"] = bc_names[value] } + // Certain dimensions from bill dimensions we expect to be "None" or "" but that is a proper allocation // resource_group only exists on Azure, and certain usage types don't have a resource_group (Microsoft.Capacity, Support) // Check if the key is not one of these dimensions that have no value but are still considered properly allocated skip_dimensions = ["resource_group"] + if (!_.contains(skip_dimensions, key)) { // For billing_center_id dimension, unallocated costs are in "unallocated" billing_center_id // For all other dimensions (tag, rbd), unallocated costs are in "None" or "" value for that dimension - if ((key != "billing_center_id" && (value == "None" || value == "")) || (key == "billing_center_id" && value == "undefined")) { - flattened["unallocated"] = true - unallocatedDetailsList.push("`"+key + "=" + value+"`") + if ((key != "billing_center_id" && (value == "None" || value == "")) || (key == "billing_center_id" && value == "unallocated")) { + flattened["Unallocated"] = true + unallocatedDetailsList.push("`" + key + "=" + value + "`") } } }) - // // If any of the values are unallocated, add the cost to the unallocated_spend - if (flattened.hasOwnProperty("unallocated") && flattened["unallocated"] == true) { - unallocated_spend = unallocated_spend + flattened[cost_metric[param_cost_metric]] - } - if (unallocatedDetailsList.length > 0) { - flattened["unallocated_details"] = unallocatedDetailsList.join(" ") - } + + // If any of the values are unallocated, add the cost to the unallocated_spend + if (flattened["Unallocated"]) { unallocated_spend += flattened[metric_name] } + flattened["Unallocated Details"] = unallocatedDetailsList.join(" ") + return flattened }) + // Group by the dimensions grouped_rows = _.groupBy(rows, function(row) { - var group_by = [] + group_by = [] + _.each(row, function(value, key) { - if (key != cost_metric[param_cost_metric]) { - group_by.push(value) - } + if (key != metric_name) { group_by.push(value) } }) + return group_by.join("|") }) + // Sum the cost_* metrics for each group rows = [] + _.each(grouped_rows, function(group) { - var row = {} + row = {} + _.each(group, function(item) { _.each(item, function(value, key) { - if (key != cost_metric[param_cost_metric]) { - row[key] = value - } + if (key != metric_name) { row[key] = value } }) }) + _.each(group, function(item) { _.each(item, function(value, key) { - if (key == cost_metric[param_cost_metric]) { - if (!row.hasOwnProperty(key)) { - row[key] = 0.0 - } + if (key == metric_name) { + if (row[key] == undefined) { row[key] = 0.0 } row[key] += value } }) }) - rows.push(row) + + // Round any numeric values in the row + rounded_row = {} + + _.each(_.keys(row), function(key) { + rounded_row[key] = row[key] + if (typeof(row[key]) == 'number') { rounded_row[key] = Math.round(row[key] * 1000) / 1000 } + }) + + rows.push(rounded_row) }) + // Free up memory grouped_rows = null + // calculate the percent_of_total now that we have the total_spend rows_with_total = [] + _.each(rows, function(row) { - row[cost_metric[param_cost_metric]+"_percent_of_total"] = +(((row[cost_metric[param_cost_metric]] / total_spend) * 100).toFixed(2)) + row[param_cost_metric + " (% of Total)"] = +(((row[metric_name] / total_spend) * 100).toFixed(2)) rows_with_total.push(row) }) + // sort the rows by cost_*_percent_of_total metric - rows_with_total = _.sortBy(rows_with_total, function(row) { - return row[cost_metric[param_cost_metric]+"_percent_of_total"] - }).reverse() + rows_with_total = _.sortBy(rows_with_total, param_cost_metric + " (% of Total)").reverse() + // Get the column order from the first row // Put any cost_* columns at the end - columns = [] - columns_unsorted = Object.keys(rows[0]) // Reverse the order so that when we push/append them they are in their original order + columns_unsorted = _.keys(rows[0]) // Reverse the order so that when we push/append them they are in their original order + // Add the required dimension columns from user input first - _.each(param_cost_dimensions, function(key) { - columns.push(key) - // If billing_center_id dimension, also add billing_center_name dimension to columns output which we added as part of compiling results in previous step - if (key == "billing_center_id") { - columns.push("billing_center_name") - } - }) - // Put unallocated column after the required dimension columns - _.each(columns_unsorted, function(key) { - if (key.match(/^unallocated|unallocated_details/)) { - columns.push(key) - } - }) - // Add the cost_* columns, after all other dimensions - _.each(columns_unsorted, function(key) { - if (key.match(/^cost_.*/)) { - columns.push(key) - } - }) + columns = _.pluck(ds_cost_dimensions, 'name').concat([ "Unallocated", "Unallocated Details", metric_name ]) + result = { "columns": columns, - "percent_unallocated_column_name": cost_metric[param_cost_metric]+"_percent_of_total", - "currency_code": ds_currency.symbol, - "total_spend": +(total_spend.toFixed(2)), - "unallocated_spend": +(unallocated_spend.toFixed(2)), - "percent_allocated": +(((1 - (unallocated_spend / total_spend)) *100).toFixed(2)), - "percent_unallocated": +(((unallocated_spend / total_spend) *100).toFixed(2)), + "percent_unallocated_column_name": param_cost_metric + " (% of Total)", + "currency_code": ds_currency['symbol'], + "total_spend": Math.round(total_spend * 1000) / 1000, + "unallocated_spend": Math.round(unallocated_spend * 1000) / 1000, + "percent_allocated": Math.round((1 - (unallocated_spend / total_spend)) * 100 * 100) / 100, + "percent_unallocated": Math.round((unallocated_spend / total_spend) * 100 * 100) / 100, "rows": rows_with_total, "start_at": ds_cost_request_params["start_at"], "end_at": ds_cost_request_params["end_at"], "filter": ds_cost_request_params["filter"] } - EOS +EOS end ############################################################################### @@ -465,10 +512,9 @@ policy "pol_scheduled_report" do {{- end }} ###### Policy Applied in Account: {{ rs_project_name }} (Account ID: {{ rs_project_id }}) within Org: {{ rs_org_name }} (Org ID: {{ rs_org_id }}) - EOS check eq(1, 0) # Always trigger - escalate $esc_send_email + escalate $esc_email end end @@ -476,7 +522,7 @@ end # Escalations ############################################################################### -escalation "esc_send_email" do +escalation "esc_email" do automatic true label "Send Email" description "Send incident email" diff --git a/data/policy_permissions_list/master_policy_permissions_list.json b/data/policy_permissions_list/master_policy_permissions_list.json index 9a955e6a0f..26e68d4a22 100644 --- a/data/policy_permissions_list/master_policy_permissions_list.json +++ b/data/policy_permissions_list/master_policy_permissions_list.json @@ -5155,7 +5155,7 @@ { "id": "./cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt", "name": "Scheduled Report for Unallocated Costs", - "version": "0.2.2", + "version": "0.3.0", "providers": [ { "name": "flexera", diff --git a/data/policy_permissions_list/master_policy_permissions_list.yaml b/data/policy_permissions_list/master_policy_permissions_list.yaml index 8108d67618..2b6388727e 100644 --- a/data/policy_permissions_list/master_policy_permissions_list.yaml +++ b/data/policy_permissions_list/master_policy_permissions_list.yaml @@ -2970,7 +2970,7 @@ required: true - id: "./cost/flexera/cco/scheduled_report_unallocated/scheduled_report_unallocated.pt" name: Scheduled Report for Unallocated Costs - version: 0.2.2 + version: 0.3.0 :providers: - :name: flexera :permissions: