Make Hybrid Table Storage its own service type

get-select · May 6, 2024 · 20cad4e · 20cad4e
1 parent 9a74d5a
commit 20cad4e
Show file tree

Hide file tree

Showing 2 changed files with 72 additions and 39 deletions.
diff --git a/models/hourly_spend.sql b/models/hourly_spend.sql
@@ -48,14 +48,6 @@ storage_terabytes_daily as (
     from {{ ref('stg_database_storage_usage_history') }}
     group by 1, 2, 3
     union all
-    select
-        date,
-        'Hybrid Tables' as storage_type,
-        database_name,
-        sum(average_hybrid_table_storage_bytes) / power(1024, 4) as storage_terabytes
-    from {{ ref('stg_database_storage_usage_history') }}
-    group by 1, 2, 3
-    union all
     select
         date,
         'Stage' as storage_type,
@@ -92,6 +84,73 @@ storage_spend_hourly as (
     group by 1, 2, 3, 4, 5
 ),
 
+-- Hybrid Table Storage has its own service type in `usage_in_currency_daily`,
+-- so we also handle it separately, and not with "Storage".
+_hybrid_table_terabytes_daily as (
+    select
+        date,
+        null as storage_type,
+        database_name,
+        sum(average_hybrid_table_storage_bytes) / power(1024, 4) as storage_terabytes
+    from {{ ref('stg_database_storage_usage_history') }}
+    group by 1, 2, 3
+),
+
+hybrid_table_storage_spend_hourly as (
+    select
+        hours.hour,
+        'Hybrid Table Storage' as service,
+        null as storage_type,
+        null as warehouse_name,
+        _hybrid_table_terabytes_daily.database_name,
+        coalesce(
+            sum(
+                div0(
+                    _hybrid_table_terabytes_daily.storage_terabytes,
+                    hours.days_in_month * 24
+                ) * daily_rates.effective_rate
+            ),
+            0
+        ) as spend,
+        spend as spend_net_cloud_services,
+        any_value(daily_rates.currency) as currency
+    from hours
+    left join _hybrid_table_terabytes_daily on hours.date = convert_timezone('UTC', _hybrid_table_terabytes_daily.date)
+    left join {{ ref('daily_rates') }} as daily_rates
+        on _hybrid_table_terabytes_daily.date = daily_rates.date
+            and daily_rates.service_type = 'STORAGE'
+            and daily_rates.usage_type = 'hybrid table storage'
+    group by 1, 2, 3, 4, 5
+),
+
+hybrid_table_requests_spend_hourly as (
+    select
+        hours.hour,
+        'Hybrid Table Requests' as service,
+        null as storage_type,
+        null as warehouse_name,
+        null as database_name,
+        coalesce(
+            sum(
+                stg_metering_history.credits_used * daily_rates.effective_rate
+            ),
+            0
+        ) as spend,
+        spend as spend_net_cloud_services,
+        any_value(daily_rates.currency) as currency
+    from hours
+    left join {{ ref('stg_metering_history') }} as stg_metering_history on
+        hours.hour = convert_timezone(
+            'UTC', stg_metering_history.start_time
+        )
+        and stg_metering_history.service_type = 'HYBRID_TABLE_REQUESTS'
+    left join {{ ref('daily_rates') }} as daily_rates
+        on hours.hour::date = daily_rates.date
+            and daily_rates.service_type = 'COMPUTE'
+            and daily_rates.usage_type = 'hybrid table requests'
+    group by 1, 2, 3, 4
+),
+
 data_transfer_spend_hourly as (
     -- Right now we don't have a way of getting this at an hourly grain
     -- We can get source cloud + region, target cloud + region, and bytes transferred at an hourly grain from DATA_TRANSFER_HISTORY
@@ -547,34 +606,6 @@ snowpark_container_services_spend_hourly as (
     group by 1, 2, 3, 4
 ),
 
-hybrid_table_requests_spend_hourly as (
-    select
-        hours.hour,
-        'Hybrid Table Requests' as service,
-        null as storage_type,
-        null as warehouse_name,
-        null as database_name,
-        coalesce(
-            sum(
-                stg_metering_history.credits_used * daily_rates.effective_rate
-            ),
-            0
-        ) as spend,
-        spend as spend_net_cloud_services,
-        any_value(daily_rates.currency) as currency
-    from hours
-    left join {{ ref('stg_metering_history') }} as stg_metering_history on
-        hours.hour = convert_timezone(
-            'UTC', stg_metering_history.start_time
-        )
-        and stg_metering_history.service_type = 'HYBRID_TABLE_REQUESTS'
-    left join {{ ref('daily_rates') }} as daily_rates
-        on hours.hour::date = daily_rates.date
-            and daily_rates.service_type = 'COMPUTE'
-            and daily_rates.usage_type = 'hybrid table requests'
-    group by 1, 2, 3, 4
-),
-
 copy_files_spend_hourly as (
     select
         hours.hour,
@@ -608,6 +639,10 @@ copy_files_spend_hourly as (
 unioned as (
     select * from storage_spend_hourly
     union all
+    select * from hybrid_table_storage_spend_hourly
+    union all
+    select * from hybrid_table_requests_spend_hourly
+    union all
     select * from data_transfer_spend_hourly
     union all
     select * from ai_services_spend_hourly
@@ -642,8 +677,6 @@ unioned as (
     union all
     select * from snowpark_container_services_spend_hourly
     union all
-    select * from hybrid_table_requests_spend_hourly
-    union all
     select * from copy_files_spend_hourly
 )
 

diff --git a/models/staging/stg_rate_sheet_daily.sql b/models/staging/stg_rate_sheet_daily.sql
@@ -17,7 +17,7 @@ select
         -- Have recently seen new values introduced for one account: WAREHOUSE_METERING and CLOUD_SERVICES
         -- For now, we'll force these to either be COMPUTE or STORAGE since that's what the downstream models expect
         -- May adjust this in the future if Snowflake is permanently changing these fields for all accounts and starts offering different credit rates per usage_type
-        when service_type = 'STORAGE' then 'STORAGE'
+        when service_type in ('STORAGE', 'HYBRID_TABLE_STORAGE') then 'STORAGE'
         else 'COMPUTE'
     end as service_type
 from {{ source('snowflake_organization_usage', 'rate_sheet_daily') }}