Merge pull request #43 from fivetran/default_enable

switch disable default config to enable default

.buildkite/scripts/run_models.sh

@@ -20,7 +20,7 @@ dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
 ## UPDATE FOR VARS HERE, IF NO VARS, PLEASE REMOVE
-dbt run --vars '{lead_history_columns: ['first_name', 'lead_status', 'urgency', 'priority', 'relative_score', 'relative_urgency', 'demographic_score_marketing', 'behavior_score_marketing'], marketo__enable_campaigns: true, marketo__enable_programs: true, marketo__activity_delete_lead_enabled: false}' --target "$db" --full-refresh
-dbt test --target "$db" --vars '{lead_history_columns: ['first_name', 'lead_status', 'urgency', 'priority', 'relative_score', 'relative_urgency', 'demographic_score_marketing', 'behavior_score_marketing'], marketo__enable_campaigns: true, marketo__enable_programs: true, marketo__activity_delete_lead_enabled: false}' --target "$db"
+dbt run --vars '{lead_history_columns: ['first_name', 'lead_status', 'urgency', 'priority', 'relative_score', 'relative_urgency', 'demographic_score_marketing', 'behavior_score_marketing'], marketo__enable_campaigns: false, marketo__enable_programs: false, marketo__activity_delete_lead_enabled: false}' --target "$db" --full-refresh
+dbt test --target "$db" --vars '{lead_history_columns: ['first_name', 'lead_status', 'urgency', 'priority', 'relative_score', 'relative_urgency', 'demographic_score_marketing', 'behavior_score_marketing'], marketo__enable_campaigns: false, marketo__enable_programs: false, marketo__activity_delete_lead_enabled: false}' --target "$db"
 ### END VARS CHUNK, REMOVE IF NOT USING
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

CHANGELOG.md

@@ -1,3 +1,27 @@
+# dbt_marketo v0.13.0
+[PR #43](https://github.com/fivetran/dbt_marketo/pull/43) includes the following updates:
+
+## Breaking Change
+- Updated the default configuration for the `marketo__enable_campaigns` and `marketo__enable_programs` variables from disabled to enabled.
+
+- Each following variable will enable the listed models by default:
+  - `marketo__enable_campaigns`:
+    - the end models `marketo__campaigns` and `marketo__email_stats__by_campaign`
+    - the upstream staging model `stg_marketo__campaigns`
+  - `marketo__enable_campaigns` and `marketo__enable_programs`:
+    - the end models `marketo__programs` and `marketo__email_stats__by_program`
+    - the upstream staging model `stg_marketo__program`
+
+- Additionally, the [`marketo__email_sends`](https://fivetran.github.io/dbt_marketo/#!/model/model.marketo.marketo__email_sends) model will now output the following fields which were previously skipped by default now that `marketo__enable_campaigns` is enabled at onset:
+  - `campaign_type`
+  - `program_id`
+
+- Quickstart dynamically manages these settings, but non-Quickstart users or those not syncing `campaign` or `program` tables should adjust the variables accordingly. Refer to the [README](https://github.com/fivetran/dbt_marketo?tab=readme-ov-file#step-4-enablingdisabling-models) for details.
+
+## Under the Hood
+- Updated the `run_models.sh` script to now test for when `marketo__enable_campaigns` and `marketo__enable_programs` is disabled.
+- Added validation tests for `marketo__campaigns` (`consistency_campaigns`) and `marketo__programs` (`consistency_campaigns`) now that they are enabled by default.
+
 # dbt_marketo v0.12.1
 [PR #42](https://github.com/fivetran/dbt_marketo/pull/42) includes the following updates:
 

README.md

@@ -60,7 +60,7 @@ Include the following Marketo package version in your `packages.yml` file.
 ```yml
 packages:
   - package: fivetran/marketo
-    version: [">=0.12.0", "<0.13.0"]
+    version: [">=0.13.0", "<0.14.0"]
 ```
 Do **NOT** include the `marketo_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
 
@@ -76,19 +76,14 @@ vars:
 For additional configurations for the source models, please visit the [Marketo source package](https://github.com/fivetran/dbt_marketo_source).
 
 ### Step 4: Enabling/Disabling Models
-This package takes into consideration tables that may not be synced due to slowness caused by the Marketo API.  By default the `campaign` and `program` related-models are disabled.  If you sync these tables, enable the modeling done by adding the following to your `dbt_project.yml` file:
-
+This package takes into consideration tables that may not be synced due to slowness caused by the Marketo API. By default the `campaign`, `program`, and `activity_delete_lead` tables are enabled. If you do not sync these tables, disable the related models or fields by adding the following to your `dbt_project.yml` file:
 ```yml
 vars:
-    marketo__enable_campaigns:   true      # Enable if Fivetran is syncing the campaign table
-    marketo__enable_programs:    true      # Enable if Fivetran is syncing the program table
+    marketo__enable_campaigns:   False      # Disable if Fivetran is not syncing the campaign table. This will disable the marketo__campaigns, marketo__programs, marketo__email_stats__by_campaign, marketo__email_stats__by_program models.
+    marketo__enable_programs:    False      # Disable if Fivetran is not syncing the program table. This will disable the marketo__programs and marketo__email_stats__by_program models.
+    marketo__activity_delete_lead_enabled:  false     # Disable if Fivetran is not syncing the activity_delete_lead table
 ```
 
-Alternatively, you may need to disable certain models. The below models can be disabled by adding them to your `dbt_project.yml` file:
-```yml
-vars:
-    marketo__activity_delete_lead_enabled:  false     # Disable if you do not have the activity_delete_lead table 
-```
 ### (Optional) Step 5: Additional configurations
 <details open><summary>Expand/Collapse details</summary>
 <br>
@@ -156,7 +151,7 @@ This dbt package is dependent on the following dbt packages. These dependencies
 ```yml
 packages:
     - package: fivetran/marketo_source
-      version: [">=0.12.0", "<0.13.0"]
+      version: [">=0.13.0", "<0.14.0"]
 
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'marketo'
-version: '0.12.1'
+version: '0.13.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 on-run-start: "{{ lead_history_columns_warning() }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: marketo_integration_tests_6
+      schema: marketo_integration_tests_8
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: marketo_integration_tests_6
+      schema: marketo_integration_tests_8
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: marketo_integration_tests_6
+      schema: marketo_integration_tests_8
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: marketo_integration_tests_6
+      schema: marketo_integration_tests_8
       threads: 8
     databricks:
       catalog: null
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: marketo_integration_tests_6
+      schema: marketo_integration_tests_8
       threads: 2
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,17 +1,15 @@
 name: 'marketo_integration_tests'
-version: '0.12.1'
+version: '0.13.0'
 profile: 'integration_tests'
 config-version: 2
 
 models:  
     +schema: "marketo_{{ var('directed_schema','dev') }}"  
 
 vars:
-    # enable when regenerating docs
-    # marketo__enable_campaigns: true
-    # marketo__enable_programs: true
-
-    marketo_schema: marketo_integration_tests_6
+    marketo_schema: marketo_integration_tests_8
+    # marketo__enable_campaigns: false ## Uncomment for testing to disable campaigns
+    # marketo__enable_programs: false ## Uncomment for testing to disable programs
     marketo_source:
         marketo_lead_identifier: "marketo_lead_data"
         marketo_lead_describe_identifier: "marketo_lead_describe_data"
@@ -31,7 +29,7 @@ vars:
 dispatch:
     - macro_namespace: dbt_utils
       search_order: ['spark_utils', 'dbt_utils']
-    
+
 seeds:
     marketo_integration_tests:
         marketo_email_template_history_data:

integration_tests/tests/consistency/consistency_campaigns.sql

@@ -0,0 +1,25 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false) and var('marketo__enable_campaigns', true)
+) }}
+
+
+with prod as (
+    select count(*) as prod_row_count
+    from {{ target.schema }}_marketo_prod.marketo__campaigns
+),
+
+dev as (
+    select count(*) as dev_row_count
+    from {{ target.schema }}_marketo_dev.marketo__campaigns
+), 
+
+count_check as (
+    select *
+    from prod
+    join dev
+        on prod.prod_row_count != dev.dev_row_count
+)
+
+select *
+from count_check

integration_tests/tests/consistency/consistency_email_sends.sql

@@ -3,17 +3,25 @@
     enabled=var('fivetran_validation_tests_enabled', false)
 ) }}
 
+{% set columns_to_exclude = ['activity_id'] %}
+
+{% if not var('marketo__enable_campaigns', True) %}
+    {% do columns_to_exclude.extend(['campaign_type', 'program_id']) %}
+{% endif %}
+
+{% set columns_to_exclude = columns_to_exclude + var('consistency_test_exclude_metrics', []) %}
+
 with prod as (
     select {{ dbt_utils.star(
         from=ref('marketo__email_sends'), 
-        except=['activity_id'] + var('consistency_test_exclude_metrics', [])) }} -- marketo__email_sends_deduped does not create a consistent activity_id
+        except=columns_to_exclude) }}
     from {{ target.schema }}_marketo_prod.marketo__email_sends
 ),
 
 dev as (
     select {{ dbt_utils.star(
         from=ref('marketo__email_sends'),
-        except=['activity_id'] + var('consistency_test_exclude_metrics', [])) }}
+        except=columns_to_exclude) }}
     from {{ target.schema }}_marketo_dev.marketo__email_sends
 ), 
 

integration_tests/tests/consistency/consistency_programs.sql

@@ -0,0 +1,24 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false) and var('marketo__enable_campaigns', true) and var('marketo__enable_programs', true)
+) }}
+
+with prod as (
+    select count(*) as prod_row_count
+    from {{ target.schema }}_marketo_prod.marketo__programs
+),
+
+dev as (
+    select count(*) as dev_row_count
+    from {{ target.schema }}_marketo_dev.marketo__programs
+), 
+
+count_check as (
+    select *
+    from prod
+    join dev
+        on prod.prod_row_count != dev.dev_row_count
+)
+
+select *
+from count_check

models/intermediate/marketo__email_stats__by_campaign.sql

@@ -1,4 +1,4 @@
-{{ config(enabled=var('marketo__enable_campaigns', False)) }}
+{{ config(enabled=var('marketo__enable_campaigns', True)) }}
 
 with email_sends as (
 

models/intermediate/marketo__email_stats__by_program.sql

@@ -1,4 +1,4 @@
-{{ config(enabled=var('marketo__enable_campaigns', False) and var('marketo__enable_programs', False)) }}
+{{ config(enabled=var('marketo__enable_campaigns', True) and var('marketo__enable_programs', True)) }}
 
 with email_sends as (
 

models/marketo__campaigns.sql

@@ -1,4 +1,4 @@
-{{ config(enabled=var('marketo__enable_campaigns', False)) }}
+{{ config(enabled=var('marketo__enable_campaigns', True)) }}
 
 with campaigns as (
 

models/marketo__email_sends.sql

@@ -28,7 +28,7 @@ with sends as (
     select *
     from {{ ref('marketo__unsubscribes__by_sent_email') }}
 
-{% if var('marketo__enable_campaigns', False) %}
+{% if var('marketo__enable_campaigns', True) %}
 
 ), campaigns as (
 
@@ -73,13 +73,13 @@ with sends as (
 
     select 
         booleans.*,
-        {% if var('marketo__enable_campaigns', False) %}
+        {% if var('marketo__enable_campaigns', True) %}
         campaigns.campaign_type,
         campaigns.program_id,
         {% endif %}
         email_templates.is_operational
     from booleans
-    {% if var('marketo__enable_campaigns', False) %}
+    {% if var('marketo__enable_campaigns', True) %}
     left join campaigns using (campaign_id)
     {% endif %}
     left join email_templates

models/marketo__programs.sql

@@ -1,4 +1,4 @@
-{{ config(enabled=var('marketo__enable_campaigns', False) and var('marketo__enable_programs', False)) }}
+{{ config(enabled=var('marketo__enable_campaigns', True) and var('marketo__enable_programs', True)) }}
 
 with programs as (
 

packages.yml

@@ -1,3 +1,3 @@
 packages:
 - package: fivetran/marketo_source
-  version: [">=0.12.0", "<0.13.0"]
+  version: [">=0.13.0", "<0.14.0"]