Persist deleted entities who still have _report records to end models

CHANGELOG.md

@@ -1,3 +1,12 @@
+# dbt_apple_search_ads v0.4.1
+[PR #26](https://github.com/fivetran/dbt_apple_search_ads/pull/26) includes the following updates:
+
+## Bug Fixes
+- Adjusts `inner joins` in each end model to be `left joins`. 
+  - When ads and other entities are deleted from Apple Search Ads, their records are hard-deleted from the `<entity>_history` source tables, though any associated `<entity>_report` records persist with new `<entity>_id` values of `-1`. This update ensures that the report records persist into the package's end models in these cases.
+>**Note**: This will likely increase the row count of your data models. To remove these newly included records, filter out rows where the `<entity>_id = -1`.
+- We have accordingly made sure to select fields, specifically entity IDs, from the left side of these joins (the report tables).
+
 # dbt_apple_search_ads v0.4.0
 [PR #24](https://github.com/fivetran/dbt_apple_search_ads/pull/24) includes the following **BREAKING CHANGE** updates:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'apple_search_ads'
-version: '0.4.0'
+version: '0.4.1'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 vars:

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'apple_search_ads_integration_tests'
-version: '0.4.0'
+version: '0.4.1'
 
 profile: 'integration_tests' 
 
@@ -26,6 +26,8 @@ vars:
     - name: conversions
       alias: conversions_alias_should_be_included
 
+  ad_consistency_inclusion_criteria: 'ad_name is not null' # Validating change persisting deleted ads (and other entities, but we only have deleted ads) to end models
+
 models:
   +schema: "apple_search_ads_{{ var('directed_schema','dev') }}"
 

integration_tests/tests/consistency/consistency_ad_report.sql

@@ -8,12 +8,20 @@ with prod as (
     select
         {{ dbt_utils.star(from=ref('apple_search_ads__ad_report'), except=var('consistency_test_exclude_metrics', '[]')) }}
     from {{ target.schema }}_apple_search_ads_prod.apple_search_ads__ad_report
+
+    {% if var('ad_consistency_inclusion_criteria') %}
+        where {{ var('ad_consistency_inclusion_criteria') }}
+    {% endif %}
 ),
 
 dev as (
     select
         {{ dbt_utils.star(from=ref('apple_search_ads__ad_report'), except=var('consistency_test_exclude_metrics', '[]')) }}
     from {{ target.schema }}_apple_search_ads_dev.apple_search_ads__ad_report
+
+    {% if var('ad_consistency_inclusion_criteria') %}
+        where {{ var('ad_consistency_inclusion_criteria') }}
+    {% endif %}
 ),
 
 prod_not_in_dev as (

integration_tests/tests/integrity/vertical_sum_conversions.sql

@@ -8,7 +8,6 @@ with ad_source as (
     select 
         sum(coalesce(conversions, 0)) as conversions
     from {{ target.schema }}_apple_search_ads_dev.stg_apple_search_ads__ad_report
-    where ad_id > 0
 ),
 
 ad_model as (
@@ -23,7 +22,6 @@ ad_group_source as (
     select 
         sum(coalesce(conversions, 0)) as conversions
     from {{ target.schema }}_apple_search_ads_dev.stg_apple_search_ads__ad_group_report
-    where ad_group_id > 0
 ),
 
 ad_group_model as (
@@ -38,7 +36,6 @@ campaign_source as (
     select 
         sum(coalesce(conversions, 0)) as conversions
     from {{ target.schema }}_apple_search_ads_dev.stg_apple_search_ads__campaign_report
-    where campaign_id > 0
 ),
 
 campaign_model as (
@@ -53,7 +50,6 @@ keyword_source as (
     select 
         sum(coalesce(conversions, 0)) as conversions
     from {{ target.schema }}_apple_search_ads_dev.stg_apple_search_ads__keyword_report
-    where keyword_id > 0
 ),
 
 keyword_model as (

models/apple_search_ads__ad_group_report.sql

@@ -35,7 +35,7 @@ joined as (
         organization.organization_name,
         campaign.campaign_id, 
         campaign.campaign_name, 
-        ad_group.ad_group_id,
+        report.ad_group_id,
         ad_group.ad_group_name,
         report.currency,
         ad_group.ad_group_status,
@@ -57,13 +57,13 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join ad_group 
+    left join ad_group 
         on report.ad_group_id = ad_group.ad_group_id
         and report.source_relation = ad_group.source_relation
-    join campaign 
+    left join campaign 
         on ad_group.campaign_id = campaign.campaign_id
         and ad_group.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on ad_group.organization_id = organization.organization_id
         and ad_group.source_relation = organization.source_relation
     {{ dbt_utils.group_by(12) }}

models/apple_search_ads__ad_report.sql

@@ -40,11 +40,11 @@ joined as (
         report.date_day,
         organization.organization_id,
         organization.organization_name,
-        campaign.campaign_id, 
+        report.campaign_id, 
         campaign.campaign_name, 
-        ad_group.ad_group_id,
+        report.ad_group_id,
         ad_group.ad_group_name,
-        ad.ad_id,
+        report.ad_id,
         ad.ad_name,
         report.currency,
         ad.ad_status,
@@ -64,16 +64,16 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join ad 
+    left join ad 
         on report.ad_id = ad.ad_id
         and report.source_relation = ad.source_relation
-    join ad_group 
+    left join ad_group 
         on report.ad_group_id = ad_group.ad_group_id
         and report.source_relation = ad_group.source_relation
-    join campaign 
+    left join campaign 
         on report.campaign_id = campaign.campaign_id
         and report.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on ad.organization_id = organization.organization_id
         and ad.source_relation = organization.source_relation
     {{ dbt_utils.group_by(12) }}

models/apple_search_ads__campaign_report.sql

@@ -28,7 +28,7 @@ joined as (
         report.date_day,
         campaign.organization_id,
         organization.organization_name,
-        campaign.campaign_id, 
+        report.campaign_id, 
         campaign.campaign_name, 
         report.currency,
         campaign.campaign_status,
@@ -50,10 +50,10 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join campaign 
+    left join campaign 
         on report.campaign_id = campaign.campaign_id
         and report.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on campaign.organization_id = organization.organization_id
         and campaign.source_relation = organization.source_relation
     {{ dbt_utils.group_by(10) }}

models/apple_search_ads__keyword_report.sql

@@ -44,7 +44,7 @@ joined as (
         campaign.campaign_name, 
         ad_group.ad_group_id,
         ad_group.ad_group_name,
-        keyword.keyword_id,
+        report.keyword_id,
         keyword.keyword_text,
         keyword.match_type,
         report.currency,
@@ -65,16 +65,16 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join keyword 
+    left join keyword 
         on report.keyword_id = keyword.keyword_id
         and report.source_relation = keyword.source_relation
-    join ad_group 
+    left join ad_group 
         on keyword.ad_group_id = ad_group.ad_group_id
         and keyword.source_relation = ad_group.source_relation
-    join campaign 
+    left join campaign 
         on ad_group.campaign_id = campaign.campaign_id
         and ad_group.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on ad_group.organization_id = organization.organization_id
         and ad_group.source_relation = organization.source_relation
     {{ dbt_utils.group_by(13) }}

models/apple_search_ads__organization_report.sql

@@ -43,10 +43,10 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join campaign 
+    left join campaign 
         on report.campaign_id = campaign.campaign_id
         and report.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on campaign.organization_id = organization.organization_id
         and campaign.source_relation = organization.source_relation
     {{ dbt_utils.group_by(5) }}

models/apple_search_ads__search_term_report.sql

@@ -26,7 +26,7 @@ joined as (
         report.date_day,
         organization.organization_id,
         organization.organization_name,
-        campaign.campaign_id, 
+        report.campaign_id, 
         campaign.campaign_name, 
         report.ad_group_id,
         report.ad_group_name,
@@ -51,10 +51,10 @@ joined as (
             exclude_fields = ['conversions']) }}
 
     from report
-    join campaign 
+    left join campaign 
         on report.campaign_id = campaign.campaign_id
         and report.source_relation = campaign.source_relation
-    join organization 
+    left join organization 
         on campaign.organization_id = organization.organization_id
         and campaign.source_relation = organization.source_relation
     where report.search_term_text is not null