Bugfix/backwards compatible conversion value (#41)

* add seer + discourse article

* contributors subsection

* fix

* testing

* changelog

* docs and tests

* update tests

CHANGELOG.md

@@ -1,3 +1,12 @@
+# dbt_linkedin v0.9.1
+[PR #41](https://github.com/fivetran/dbt_linkedin/pull/41) includes the following updates:
+
+## Bug Fix
+- Ensures that the recent addition of conversions (see v0.9.0 below) is indeed backwards compatible with users already including `conversion_value_by_local_currency` as a passthrough column.
+
+## Under the Hood
+- Updates the consistency data validation tests (maintainers only) to more accurately test the grain of each report.
+
 # dbt_linkedin v0.9.0
 [PR #36](https://github.com/fivetran/dbt_linkedin/pull/36) includes the following updates:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'linkedin'
-version: '0.9.0'
+version: '0.9.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'linkedin_integration_tests'
-version: '0.9.0'
+version: '0.9.1'
 profile: 'integration_tests'
 config-version: 2
 
@@ -13,8 +13,9 @@ vars:
     linkedin_ads_ad_analytics_by_campaign_identifier: "linkedin_ad_analytics_by_campaign_data"
 
     linkedin_ads__campaign_passthrough_metrics: []
-    linkedin_ads__creative_passthrough_metrics: []
-    linkedin_ads__conversion_fields: ['external_website_conversions', 'one_click_leads', 'external_website_post_click_conversions']
+    linkedin_ads__creative_passthrough_metrics:
+      - name: 'conversion_value_in_local_currency'
+    linkedin_ads__conversion_fields: ['external_website_conversions', 'one_click_leads']
 
 models:
   +schema: "linkedin_{{ var('directed_schema','dev') }}"

integration_tests/tests/consistency/consistency_account_report.sql

@@ -8,7 +8,9 @@ with prod as (
         account_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_prod.linkedin_ads__account_report
     group by 1
 ),
@@ -18,7 +20,9 @@ dev as (
         account_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_dev.linkedin_ads__account_report
     group by 1
 ),
@@ -31,7 +35,11 @@ final as (
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.cost as prod_cost,
-        dev.cost as dev_cost
+        dev.cost as dev_cost,
+        prod.total_conversions as prod_total_conversions,
+        dev.total_conversions as dev_total_conversions,
+        prod.conversion_value_in_local_currency as prod_conversion_value_in_local_currency,
+        dev.conversion_value_in_local_currency as dev_conversion_value_in_local_currency
     from prod
     full outer join dev 
         on dev.account_id = prod.account_id
@@ -42,4 +50,6 @@ from final
 where
     abs(prod_clicks - dev_clicks) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
-    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_value_in_local_currency - dev_conversion_value_in_local_currency) >= .01

integration_tests/tests/consistency/consistency_campaign_group_report.sql

@@ -5,41 +5,51 @@
 
 with prod as (
     select
-        account_id,
+        campaign_group_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_prod.linkedin_ads__campaign_group_report
     group by 1
 ),
 
 dev as (
     select
-        account_id,
+        campaign_group_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_dev.linkedin_ads__campaign_group_report
     group by 1
 ),
 
 final as (
     select 
-        prod.account_id,
+        prod.campaign_group_id,
         prod.clicks as prod_clicks,
         dev.clicks as dev_clicks,
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.cost as prod_cost,
-        dev.cost as dev_cost
+        dev.cost as dev_cost,
+        prod.total_conversions as prod_total_conversions,
+        dev.total_conversions as dev_total_conversions,
+        prod.conversion_value_in_local_currency as prod_conversion_value_in_local_currency,
+        dev.conversion_value_in_local_currency as dev_conversion_value_in_local_currency
     from prod
     full outer join dev 
-        on dev.account_id = prod.account_id
+        on dev.campaign_group_id = prod.campaign_group_id
 )
 
 select *
 from final
 where
     abs(prod_clicks - dev_clicks) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
-    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_value_in_local_currency - dev_conversion_value_in_local_currency) >= .01

integration_tests/tests/consistency/consistency_campaign_report.sql

@@ -5,41 +5,51 @@
 
 with prod as (
     select
-        account_id,
+        campaign_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_prod.linkedin_ads__campaign_report
     group by 1
 ),
 
 dev as (
     select
-        account_id,
+        campaign_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_dev.linkedin_ads__campaign_report
     group by 1
 ),
 
 final as (
     select 
-        prod.account_id,
+        prod.campaign_id,
         prod.clicks as prod_clicks,
         dev.clicks as dev_clicks,
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.cost as prod_cost,
-        dev.cost as dev_cost
+        dev.cost as dev_cost,
+        prod.total_conversions as prod_total_conversions,
+        dev.total_conversions as dev_total_conversions,
+        prod.conversion_value_in_local_currency as prod_conversion_value_in_local_currency,
+        dev.conversion_value_in_local_currency as dev_conversion_value_in_local_currency
     from prod
     full outer join dev 
-        on dev.account_id = prod.account_id
+        on dev.campaign_id = prod.campaign_id
 )
 
 select *
 from final
 where
     abs(prod_clicks - dev_clicks) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
-    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_value_in_local_currency - dev_conversion_value_in_local_currency) >= .01

integration_tests/tests/consistency/consistency_creative_report.sql

@@ -5,41 +5,51 @@
 
 with prod as (
     select
-        account_id,
+        creative_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_prod.linkedin_ads__creative_report
     group by 1
 ),
 
 dev as (
     select
-        account_id,
+        creative_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_dev.linkedin_ads__creative_report
     group by 1
 ),
 
 final as (
     select 
-        prod.account_id,
+        prod.creative_id,
         prod.clicks as prod_clicks,
         dev.clicks as dev_clicks,
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.cost as prod_cost,
-        dev.cost as dev_cost
+        dev.cost as dev_cost,
+        prod.total_conversions as prod_total_conversions,
+        dev.total_conversions as dev_total_conversions,
+        prod.conversion_value_in_local_currency as prod_conversion_value_in_local_currency,
+        dev.conversion_value_in_local_currency as dev_conversion_value_in_local_currency
     from prod
     full outer join dev 
-        on dev.account_id = prod.account_id
+        on dev.creative_id = prod.creative_id
 )
 
 select *
 from final
 where
     abs(prod_clicks - dev_clicks) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
-    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_value_in_local_currency - dev_conversion_value_in_local_currency) >= .01

integration_tests/tests/consistency/consistency_url_report.sql

@@ -5,41 +5,51 @@
 
 with prod as (
     select
-        account_id,
+        creative_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_prod.linkedin_ads__url_report
     group by 1
 ),
 
 dev as (
     select
-        account_id,
+        creative_id,
         sum(clicks) as clicks, 
         sum(impressions) as impressions,
-        sum(cost) as cost
+        sum(cost) as cost,
+        sum(total_conversions) as total_conversions,
+        sum(conversion_value_in_local_currency) as conversion_value_in_local_currency
     from {{ target.schema }}_linkedin_dev.linkedin_ads__url_report
     group by 1
 ),
 
 final as (
     select 
-        prod.account_id,
+        prod.creative_id,
         prod.clicks as prod_clicks,
         dev.clicks as dev_clicks,
         prod.impressions as prod_impressions,
         dev.impressions as dev_impressions,
         prod.cost as prod_cost,
-        dev.cost as dev_cost
+        dev.cost as dev_cost,
+        prod.total_conversions as prod_total_conversions,
+        dev.total_conversions as dev_total_conversions,
+        prod.conversion_value_in_local_currency as prod_conversion_value_in_local_currency,
+        dev.conversion_value_in_local_currency as dev_conversion_value_in_local_currency
     from prod
     full outer join dev 
-        on dev.account_id = prod.account_id
+        on dev.creative_id = prod.creative_id
 )
 
 select *
 from final
 where
     abs(prod_clicks - dev_clicks) >= .01
     or abs(prod_impressions - dev_impressions) >= .01
-    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_cost - dev_cost) >= .01
+    or abs(prod_total_conversions - dev_total_conversions) >= .01
+    or abs(prod_conversion_value_in_local_currency - dev_conversion_value_in_local_currency) >= .01

macros/linkedin_ads_persist_pass_through_columns.sql

@@ -4,6 +4,7 @@
 
 {% set except_fields = [] %}
 {% if except_variable is not none %}
+    {# Start creating list of fields to exclude #}
     {% for item in var(except_variable) %}
         {% do except_fields.append(item.name) %}
     {% endfor %}
@@ -15,9 +16,12 @@
 
 {% if var(pass_through_variable, none) %}
     {% for field in var(pass_through_variable) %}
-        {% if field not in except_fields %}
-        , {{ transform ~ '(' ~ ('coalesce(' if coalesce_with is not none else '') ~ (identifier ~ '.' if identifier else '') ~ field ~ ((', ' ~ coalesce_with ~ ')') if coalesce_with is not none else '') ~ ')' }} as {{ field }}
+    {% set field_name = field.alias|default(field.name)|lower if field is mapping else field|lower %}
+
+        {% if field_name not in except_fields %}
+        , {{ transform ~ '(' ~ ('coalesce(' if coalesce_with is not none else '') ~ (identifier ~ '.' if identifier else '') ~ field_name ~ ((', ' ~ coalesce_with ~ ')') if coalesce_with is not none else '') ~ ')' }} as {{ field_name }}
         {% endif %}
+
     {% endfor %}
 {% endif %}
 

models/linkedin_ads__account_report.sql

@@ -45,9 +45,8 @@ final as (
         sum(coalesce(report.conversion_value_in_local_currency, 0)) as conversion_value_in_local_currency
 
         {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__conversion_fields', transform='sum', coalesce_with=0, except_variable='linkedin_ads__campaign_passthrough_metrics', exclude_fields=['conversion_value_in_local_currency']) }}
-
-        {{ fivetran_utils.persist_pass_through_columns('linkedin_ads__campaign_passthrough_metrics', transform='sum') }}
-
+
+        {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__campaign_passthrough_metrics', transform='sum', exclude_fields=['conversion_value_in_local_currency']) }}
 
     from report 
     left join campaign 

models/linkedin_ads__campaign_group_report.sql

@@ -56,7 +56,7 @@ final as (
 
         {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__conversion_fields', transform='sum', coalesce_with=0, except_variable='linkedin_ads__campaign_passthrough_metrics', exclude_fields=['conversion_value_in_local_currency']) }}
 
-        {{ fivetran_utils.persist_pass_through_columns('linkedin_ads__campaign_passthrough_metrics', transform='sum') }}
+        {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__campaign_passthrough_metrics', transform='sum', exclude_fields=['conversion_value_in_local_currency']) }}
 
     from report 
     left join campaign 

models/linkedin_ads__campaign_report.sql

@@ -72,8 +72,8 @@ final as (
         sum(coalesce(report.conversion_value_in_local_currency, 0)) as conversion_value_in_local_currency
 
         {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__conversion_fields', transform='sum', coalesce_with=0, except_variable='linkedin_ads__campaign_passthrough_metrics', exclude_fields=['conversion_value_in_local_currency']) }}
-
-        {{ fivetran_utils.persist_pass_through_columns('linkedin_ads__campaign_passthrough_metrics', transform='sum') }}
+        
+        {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__campaign_passthrough_metrics', transform='sum', exclude_fields=['conversion_value_in_local_currency']) }}
 
     from report 
     left join campaign 

models/linkedin_ads__creative_report.sql

@@ -66,8 +66,8 @@ final as (
 
         {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__conversion_fields', transform='sum', coalesce_with=0, except_variable='linkedin_ads__creative_passthrough_metrics', exclude_fields=['conversion_value_in_local_currency']) }}
 
-        {{ fivetran_utils.persist_pass_through_columns('linkedin_ads__creative_passthrough_metrics', transform='sum') }}
-    
+        {{ linkedin_ads_persist_pass_through_columns(pass_through_variable='linkedin_ads__creative_passthrough_metrics', transform='sum', exclude_fields=['conversion_value_in_local_currency']) }}
+
     from report 
     left join creative 
         on report.creative_id = creative.creative_id