Merge pull request #123 from get-select/add_data_transfer_cost

Add data transfer costs

.changes/4.4.0.md

@@ -0,0 +1,7 @@
+## dbt-snowflake-monitoring 4.4.0 - July 26, 2023
+
+### Features
+
+- Add Data Transfer costs ([#123](https://github.com/get-select/dbt-snowflake-monitoring/pull/123))
+
+

CHANGELOG.md

@@ -5,6 +5,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
+## dbt-snowflake-monitoring 4.4.0 - July 26, 2023
+
+### Features
+
+- Add Data Transfer costs ([#123](https://github.com/get-select/dbt-snowflake-monitoring/pull/123))
+
+
+
 ## dbt-snowflake-monitoring 4.3.1 - June 23, 2023
 
 ### Features

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'dbt_snowflake_monitoring'
-version: '4.3.1'
+version: '4.4.0'
 config-version: 2
 
 profile: dbt_snowflake_monitoring

integration_test_project/seeds/monthly_spend_fixture.csv

@@ -10,3 +10,4 @@ MONTH,SERVICE,SPEND
 2022-04-01,Replication,0
 2022-04-01,Query Acceleration,0
 2022-04-01,Materialized Views,0
+2022-04-01,Data Transfer,0

macros/account_locator.sql

@@ -0,0 +1,7 @@
+{% macro account_locator() %}
+{%- if var('account_locator', none) -%}
+'{{ var('account_locator') }}'
+{%- else -%}
+current_account()
+{%- endif -%}
+{%- endmacro %}

models/daily_rates.sql

@@ -9,14 +9,6 @@ This model guarantees 1 row per day per usage type, by filling in missing values
 known day.
 */
 
-{%- set account_locator -%}
-{%- if var('account_locator', none) -%}
-'{{ var('account_locator') }}'
-{%- else -%}
-current_account()
-{%- endif -%}
-{%- endset -%}
-
 with
 dates_base as (
     select date_day as date from (
@@ -38,7 +30,7 @@ rate_sheet_daily_base as (
         service_type
     from {{ ref('stg_rate_sheet_daily') }}
     where
-        account_locator = {{ account_locator }}
+        account_locator = {{ account_locator() }}
 ),
 
 stop_thresholds as (

models/hourly_spend.sql

@@ -26,6 +26,7 @@ hours as (
     select
         date_hour as hour,
         hour::date as date,
+        count(hour) over (partition by date) as hours_thus_far,
         day(last_day(date)) as days_in_month
     from hour_spine
 ),
@@ -83,6 +84,28 @@ storage_spend_hourly as (
     group by 1, 2, 3, 4, 5
 ),
 
+data_transfer_spend_hourly as (
+    -- Right now we don't have a way of getting this at an hourly grain
+    -- We can get source cloud + region, target cloud + region, and bytes transferred at an hourly grain from DATA_TRANSFER_HISTORY
+    -- But Snowflake doesn't provide data transfer rates programmatically, so we can't get the cost
+    -- We could make a LUT from https://www.snowflake.com/legal-files/CreditConsumptionTable.pdf but it would be a lot of work to maintain and would frequently become out of date
+    -- So for now we just use the daily reported usage and evenly distribute it across the day
+    select
+        hours.hour,
+        'Data Transfer' as service,
+        null as storage_type,
+        null as warehouse_name,
+        null as database_name,
+        coalesce(stg_usage_in_currency_daily.usage_in_currency/hours.hours_thus_far, 0) as spend,
+        spend as spend_net_cloud_services,
+        stg_usage_in_currency_daily.currency as currency
+    from hours
+    left join {{ ref('stg_usage_in_currency_daily') }} on
+        stg_usage_in_currency_daily.account_locator = {{ account_locator() }}
+        and stg_usage_in_currency_daily.usage_type = 'data transfer'
+        and hours.hour::date = stg_usage_in_currency_daily.usage_date
+),
+
 compute_spend_hourly as (
     select
         hours.hour,
@@ -396,6 +419,8 @@ search_optimization_spend_hourly as (
 unioned as (
     select * from storage_spend_hourly
     union all
+    select * from data_transfer_spend_hourly
+    union all
     select * from compute_spend_hourly
     union all
     select * from adj_for_incl_cloud_services_hourly

models/staging/sources.yml

@@ -5,20 +5,22 @@ sources:
     database: snowflake
     schema: account_usage
     tables:
-      - name: stage_storage_usage_history
+      - name: access_history
+      - name: data_transfer_history
       - name: database_storage_usage_history
-      - name: query_history
-      - name: warehouse_metering_history
-      - name: metering_history
       - name: metering_daily_history
+      - name: metering_history
+      - name: query_history
       - name: serverless_task_history
+      - name: stage_storage_usage_history
       - name: warehouse_events_history
-      - name: access_history
+      - name: warehouse_metering_history
 
   - name: snowflake_organization_usage
     database: snowflake
     schema: organization_usage
     tables:
       - name: rate_sheet_daily
       - name: remaining_balance_daily
+      - name: usage_in_currency_daily
 

models/staging/stg_data_transfer_history.sql

@@ -0,0 +1,10 @@
+select
+    start_time,
+    end_time,
+    source_cloud,
+    source_region,
+    target_cloud,
+    target_region,
+    bytes_transferred,
+    transfer_type
+from {{ source('snowflake_account_usage', 'data_transfer_history') }}

models/staging/stg_usage_in_currency_daily.sql

@@ -0,0 +1,14 @@
+select
+    organization_name,
+    contract_number,
+    account_name,
+    account_locator,
+    region,
+    service_level,
+    usage_date,
+    usage_type,
+    currency,
+    usage,
+    usage_in_currency,
+    balance_source
+from {{ source('snowflake_organization_usage', 'usage_in_currency_daily') }}