diff --git a/integration_test_project/macros/account_locator.sql b/integration_test_project/macros/account_locator.sql new file mode 100644 index 0000000..c570112 --- /dev/null +++ b/integration_test_project/macros/account_locator.sql @@ -0,0 +1,7 @@ +{% macro account_locator() %} +{%- if var('account_locator', none) -%} +'{{ var('account_locator') }}' +{%- else -%} +current_account() +{%- endif -%} +{%- endmacro %} diff --git a/models/daily_rates.sql b/models/daily_rates.sql index 257d8a6..0614c99 100644 --- a/models/daily_rates.sql +++ b/models/daily_rates.sql @@ -9,14 +9,6 @@ This model guarantees 1 row per day per usage type, by filling in missing values known day. */ -{%- set account_locator -%} -{%- if var('account_locator', none) -%} -'{{ var('account_locator') }}' -{%- else -%} -current_account() -{%- endif -%} -{%- endset -%} - with dates_base as ( select date_day as date from ( @@ -38,7 +30,7 @@ rate_sheet_daily_base as ( service_type from {{ ref('stg_rate_sheet_daily') }} where - account_locator = {{ account_locator }} + account_locator = {{ account_locator() }} ), stop_thresholds as ( diff --git a/models/hourly_spend.sql b/models/hourly_spend.sql index c02660f..17e87a2 100644 --- a/models/hourly_spend.sql +++ b/models/hourly_spend.sql @@ -26,6 +26,7 @@ hours as ( select date_hour as hour, hour::date as date, + count(hour) over (partition by date) as hours_thus_far, day(last_day(date)) as days_in_month from hour_spine ), @@ -83,6 +84,28 @@ storage_spend_hourly as ( group by 1, 2, 3, 4, 5 ), +data_transfer_spend_hourly as ( + -- Right now we don't have a way of getting this at an hourly grain + -- We can get source cloud + region, target cloud + region, and bytes transferred at an hourly grain + -- But Snowflake doesn't provide data transfer rates programmatically, so we can't get the cost + -- We could make a LUT from https://www.snowflake.com/legal-files/CreditConsumptionTable.pdf but it would be a lot of work to maintain and would frequently become out of date + -- So for now we just use the daily reported usage and evenly distribute it across the day + select + hours.hour, + 'Data Transfer' as service, + null as storage_type, + null as warehouse_name, + null as database_name, + stg_usage_in_currency_daily.usage_in_currency/hours.hours_thus_far as spend, + spend as spend_net_cloud_services, + stg_usage_in_currency_daily.currency as currency + from hours + left join {{ ref('stg_usage_in_currency_daily') }} on + stg_usage_in_currency_daily.account_locator = {{ account_locator() }} + and stg_usage_in_currency_daily.usage_type = 'data transfer' + and hours.hour::date = stg_usage_in_currency_daily.usage_date +), + compute_spend_hourly as ( select hours.hour, @@ -396,6 +419,8 @@ search_optimization_spend_hourly as ( unioned as ( select * from storage_spend_hourly union all + select * from data_transfer_spend_hourly + union all select * from compute_spend_hourly union all select * from adj_for_incl_cloud_services_hourly diff --git a/models/staging/sources.yml b/models/staging/sources.yml index ab11139..144d55e 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -5,15 +5,16 @@ sources: database: snowflake schema: account_usage tables: - - name: stage_storage_usage_history + - name: access_history + - name: data_transfer_history - name: database_storage_usage_history - - name: query_history - - name: warehouse_metering_history - - name: metering_history - name: metering_daily_history + - name: metering_history + - name: query_history - name: serverless_task_history + - name: stage_storage_usage_history - name: warehouse_events_history - - name: access_history + - name: warehouse_metering_history - name: snowflake_organization_usage database: snowflake @@ -21,4 +22,5 @@ sources: tables: - name: rate_sheet_daily - name: remaining_balance_daily + - name: usage_in_currency_daily diff --git a/models/staging/stg_data_transfer_history.sql b/models/staging/stg_data_transfer_history.sql new file mode 100644 index 0000000..4a17e31 --- /dev/null +++ b/models/staging/stg_data_transfer_history.sql @@ -0,0 +1,10 @@ +select + start_time, + end_time, + source_cloud, + source_region, + target_cloud, + target_region, + bytes_transferred, + transfer_type +from {{ source('snowflake_account_usage', 'data_transfer_history')}} diff --git a/models/staging/stg_usage_in_currency_daily.sql b/models/staging/stg_usage_in_currency_daily.sql new file mode 100644 index 0000000..3095fa9 --- /dev/null +++ b/models/staging/stg_usage_in_currency_daily.sql @@ -0,0 +1,14 @@ +select + organization_name, + contract_number, + account_name, + account_locator, + region, + service_level, + usage_date, + usage_type, + currency, + usage, + usage_in_currency, + balance_source +from {{ source('snowflake_organization_usage', 'usage_in_currency_daily') }}