diff --git a/models/hourly_spend.sql b/models/hourly_spend.sql index 27f490c..76e0ce8 100644 --- a/models/hourly_spend.sql +++ b/models/hourly_spend.sql @@ -48,14 +48,6 @@ storage_terabytes_daily as ( from {{ ref('stg_database_storage_usage_history') }} group by 1, 2, 3 union all - select - date, - 'Hybrid Tables' as storage_type, - database_name, - sum(average_hybrid_table_storage_bytes) / power(1024, 4) as storage_terabytes - from {{ ref('stg_database_storage_usage_history') }} - group by 1, 2, 3 - union all select date, 'Stage' as storage_type, @@ -92,6 +84,73 @@ storage_spend_hourly as ( group by 1, 2, 3, 4, 5 ), +-- Hybrid Table Storage has its own service type in `usage_in_currency_daily`, +-- so we also handle it separately, and not with "Storage". +_hybrid_table_terabytes_daily as ( + select + date, + null as storage_type, + database_name, + sum(average_hybrid_table_storage_bytes) / power(1024, 4) as storage_terabytes + from {{ ref('stg_database_storage_usage_history') }} + group by 1, 2, 3 +), + +hybrid_table_storage_spend_hourly as ( + select + hours.hour, + 'Hybrid Table Storage' as service, + null as storage_type, + null as warehouse_name, + _hybrid_table_terabytes_daily.database_name, + coalesce( + sum( + div0( + _hybrid_table_terabytes_daily.storage_terabytes, + hours.days_in_month * 24 + ) * daily_rates.effective_rate + ), + 0 + ) as spend, + spend as spend_net_cloud_services, + any_value(daily_rates.currency) as currency + from hours + left join _hybrid_table_terabytes_daily on hours.date = convert_timezone('UTC', _hybrid_table_terabytes_daily.date) + left join {{ ref('daily_rates') }} as daily_rates + on _hybrid_table_terabytes_daily.date = daily_rates.date + and daily_rates.service_type = 'STORAGE' + and daily_rates.usage_type = 'hybrid table storage' + group by 1, 2, 3, 4, 5 +), + +hybrid_table_requests_spend_hourly as ( + select + hours.hour, + 'Hybrid Table Requests' as service, + null as storage_type, + null as warehouse_name, + null as database_name, + coalesce( + sum( + stg_metering_history.credits_used * daily_rates.effective_rate + ), + 0 + ) as spend, + spend as spend_net_cloud_services, + any_value(daily_rates.currency) as currency + from hours + left join {{ ref('stg_metering_history') }} as stg_metering_history on + hours.hour = convert_timezone( + 'UTC', stg_metering_history.start_time + ) + and stg_metering_history.service_type = 'HYBRID_TABLE_REQUESTS' + left join {{ ref('daily_rates') }} as daily_rates + on hours.hour::date = daily_rates.date + and daily_rates.service_type = 'COMPUTE' + and daily_rates.usage_type = 'hybrid table requests' + group by 1, 2, 3, 4 +), + data_transfer_spend_hourly as ( -- Right now we don't have a way of getting this at an hourly grain -- We can get source cloud + region, target cloud + region, and bytes transferred at an hourly grain from DATA_TRANSFER_HISTORY @@ -547,34 +606,6 @@ snowpark_container_services_spend_hourly as ( group by 1, 2, 3, 4 ), -hybrid_table_requests_spend_hourly as ( - select - hours.hour, - 'Hybrid Table Requests' as service, - null as storage_type, - null as warehouse_name, - null as database_name, - coalesce( - sum( - stg_metering_history.credits_used * daily_rates.effective_rate - ), - 0 - ) as spend, - spend as spend_net_cloud_services, - any_value(daily_rates.currency) as currency - from hours - left join {{ ref('stg_metering_history') }} as stg_metering_history on - hours.hour = convert_timezone( - 'UTC', stg_metering_history.start_time - ) - and stg_metering_history.service_type = 'HYBRID_TABLE_REQUESTS' - left join {{ ref('daily_rates') }} as daily_rates - on hours.hour::date = daily_rates.date - and daily_rates.service_type = 'COMPUTE' - and daily_rates.usage_type = 'hybrid table requests' - group by 1, 2, 3, 4 -), - copy_files_spend_hourly as ( select hours.hour, @@ -608,6 +639,10 @@ copy_files_spend_hourly as ( unioned as ( select * from storage_spend_hourly union all + select * from hybrid_table_storage_spend_hourly + union all + select * from hybrid_table_requests_spend_hourly + union all select * from data_transfer_spend_hourly union all select * from ai_services_spend_hourly @@ -642,8 +677,6 @@ unioned as ( union all select * from snowpark_container_services_spend_hourly union all - select * from hybrid_table_requests_spend_hourly - union all select * from copy_files_spend_hourly ) diff --git a/models/staging/stg_rate_sheet_daily.sql b/models/staging/stg_rate_sheet_daily.sql index 460732d..44b9454 100644 --- a/models/staging/stg_rate_sheet_daily.sql +++ b/models/staging/stg_rate_sheet_daily.sql @@ -17,7 +17,7 @@ select -- Have recently seen new values introduced for one account: WAREHOUSE_METERING and CLOUD_SERVICES -- For now, we'll force these to either be COMPUTE or STORAGE since that's what the downstream models expect -- May adjust this in the future if Snowflake is permanently changing these fields for all accounts and starts offering different credit rates per usage_type - when service_type = 'STORAGE' then 'STORAGE' + when service_type in ('STORAGE', 'HYBRID_TABLE_STORAGE') then 'STORAGE' else 'COMPUTE' end as service_type from {{ source('snowflake_organization_usage', 'rate_sheet_daily') }}