From 06478679474a6954f9d7206372362ce44a949863 Mon Sep 17 00:00:00 2001 From: Fernando Brito Date: Tue, 14 May 2024 18:43:12 +0200 Subject: [PATCH] Improve handling of overage spend (#154) * Improve handling of overage spend * Improve handling of overage spend * Prepare for release --- .changes/5.1.1.md | 9 +++++ CHANGELOG.md | 10 +++++ dbt_project.yml | 2 +- models/hourly_spend.sql | 88 ++++++++++++++++++++++++----------------- 4 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 .changes/5.1.1.md diff --git a/.changes/5.1.1.md b/.changes/5.1.1.md new file mode 100644 index 0000000..cc66428 --- /dev/null +++ b/.changes/5.1.1.md @@ -0,0 +1,9 @@ +## dbt-snowflake-monitoring 5.1.1 - May 14, 2024 + +### Fixes + +- Include overage spend on services recently introduced ([#154](https://github.com/get-select/dbt-snowflake-monitoring/pull/154)) + +### Contributors +- [@fernandobrito](https://github.com/fernandobrito) (Fixes) + diff --git a/CHANGELOG.md b/CHANGELOG.md index a008311..339eb96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-snowflake-monitoring 5.1.1 - May 14, 2024 + +### Fixes + +- Include overage spend on services recently introduced ([#154](https://github.com/get-select/dbt-snowflake-monitoring/pull/154)) + +### Contributors +- [@fernandobrito](https://github.com/fernandobrito) (Fixes) + + ## dbt-snowflake-monitoring 5.1.0 - May 06, 2024 ### Features diff --git a/dbt_project.yml b/dbt_project.yml index 0c8b09a..f051fad 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'dbt_snowflake_monitoring' -version: '5.1.0' +version: '5.1.1' config-version: 2 profile: dbt_snowflake_monitoring diff --git a/models/hourly_spend.sql b/models/hourly_spend.sql index 76fc1a8..e2a059c 100644 --- a/models/hourly_spend.sql +++ b/models/hourly_spend.sql @@ -31,6 +31,20 @@ hours as ( from hour_spine ), +-- GROUP BY to collapse possible overage and non-overage cost from the same service in the +-- same day into a single row so this model does not emit multiple rows for the same service +-- and hour +usage_in_currency_daily as ( + select + usage_date, + account_locator, + replace(usage_type, 'overage-', '') as usage_type, + currency, + sum(usage_in_currency) as usage_in_currency, + from {{ ref('stg_usage_in_currency_daily') }} + group by all +), + storage_terabytes_daily as ( select date, @@ -163,14 +177,14 @@ data_transfer_spend_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, spend as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} as stg_usage_in_currency_daily on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = 'data transfer' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = 'data transfer' + and hours.hour::date = usage_in_currency_daily.usage_date ), ai_services_spend_hourly as ( @@ -184,14 +198,14 @@ ai_services_spend_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, spend as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} as stg_usage_in_currency_daily on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = 'ai services' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = 'ai services' + and hours.hour::date = usage_in_currency_daily.usage_date ), logging_spend_hourly as ( @@ -204,14 +218,14 @@ logging_spend_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, spend as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} as stg_usage_in_currency_daily on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = 'logging' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = 'logging' + and hours.hour::date = usage_in_currency_daily.usage_date ), -- For now we just use the daily reported usage and evenly distribute it across the day @@ -228,14 +242,14 @@ logging_spend_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, spend as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} as stg_usage_in_currency_daily on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = '{{ reader_usage_type }}' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = '{{ reader_usage_type }}' + and hours.hour::date = usage_in_currency_daily.usage_date ), {% endfor %} @@ -246,14 +260,14 @@ reader_adj_for_incl_cloud_services_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, 0 as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} as stg_usage_in_currency_daily on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = 'reader adj for incl cloud services' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = 'reader adj for incl cloud services' + and hours.hour::date = usage_in_currency_daily.usage_date ), reader_cloud_services_hourly as ( @@ -263,14 +277,14 @@ reader_cloud_services_hourly as ( null as storage_type, null as warehouse_name, null as database_name, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, - coalesce(stg_usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) + reader_adj_for_incl_cloud_services_hourly.spend as spend_net_cloud_services, - stg_usage_in_currency_daily.currency as currency + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) as spend, + coalesce(usage_in_currency_daily.usage_in_currency / hours.hours_thus_far, 0) + reader_adj_for_incl_cloud_services_hourly.spend as spend_net_cloud_services, + usage_in_currency_daily.currency as currency from hours - left join {{ ref('stg_usage_in_currency_daily') }} on - stg_usage_in_currency_daily.account_locator = {{ account_locator() }} - and stg_usage_in_currency_daily.usage_type = 'reader cloud services' - and hours.hour::date = stg_usage_in_currency_daily.usage_date + left join usage_in_currency_daily on + usage_in_currency_daily.account_locator = {{ account_locator() }} + and usage_in_currency_daily.usage_type = 'reader cloud services' + and hours.hour::date = usage_in_currency_daily.usage_date left join reader_adj_for_incl_cloud_services_hourly on hours.hour = reader_adj_for_incl_cloud_services_hourly.hour ),