[Feature] Brand and organization variable configs

.quickstart/quickstart.yml

@@ -20,6 +20,10 @@ table_variables:
     - audit_log
   using_holidays:
     - schedule_holiday
+  using_brands:
+    - brand
+  using_organizations:
+    - organization
 
 destination_configurations:
   databricks:

CHANGELOG.md

@@ -1,3 +1,24 @@
+# dbt_zendesk v0.19.2
+[PR #181](https://github.com/fivetran/dbt_zendesk/pull/181) includes the following updates:
+
+## New Features
+- Introduced new config variables `using_brands` and `using_organizations` to allow the customers to enable and disable `brand` and `organization` source data. 
+- End models that could be impacted by these variables include: 
+  - `zendesk__ticket_backlog`: `using_brands` and/or `using_organizations` can be set to `false` to disable `brand` and/or `organization` fields.
+  - `zendesk__ticket_enriched`: `using_organizations` can be set to `false` to remove `organization` fields from the final model.
+- Intermediate models that could be impacted by these variables include:
+  - `int_zendesk__organization_aggregates`: `using_organizations` can be utilized to completely disable the model.
+  - `int_zendesk__ticket_aggregates`: `using_brands` can be set to `false` to remove `brand` fields from the final model.
+  - `int_zendesk__updater_information`: `using_organizations` can be set to `false` to remove `organization` fields from the final model.
+
+## Under the Hood
+- Updated `table_variables` in the `quickstart.yml` with the new `brand` and `organization` tables.
+- Updated our Buildkite model run script to ensure we test for when `using_brands` and `using_organizations` is set to either true or false. 
+
+## Documentation Updates
+- [Updated README](https://github.com/fivetran/dbt_zendesk?tab=readme-ov-file#step-4-enabledisable-models-for-non-existent-sources) with instructions on how to disable `brand` and `organization` sources. 
+
+
 # dbt_zendesk v0.19.1
 [PR #180](https://github.com/fivetran/dbt_zendesk/pull/180) includes the following update:
 

README.md

@@ -138,7 +138,7 @@ vars:
 ### Step 4: Enable/Disable models for non-existent sources
 
 > _This step is optional if you are unioning multiple connectors together in the previous step. The `union_data` macro will create empty staging models for sources that are not found in any of your Zendesk schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._
-This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable:
+This package takes into consideration that not every Zendesk Support account utilizes the `schedule`, `schedule_holiday`, `ticket_schedule`, `daylight_time`, `time_zone`, `audit_log`, `domain_name`, `user_tag`, `brand`,`organization`, `organization_tag`, or `ticket_form_history` features, and allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`, except for `using_schedule_histories`. Add variables for only the tables you want to enable/disable:
 ```yml
 vars:
     using_schedule_histories:   True          #Enable if you are using audit_logs for schedule histories
@@ -147,6 +147,8 @@ vars:
     using_domain_names:         False         #Disable if you are not using domain names
     using_user_tags:            False         #Disable if you are not using user tags
     using_ticket_form_history:  False         #Disable if you are not using ticket form history
+    using_brands:               False         #Disable if you are not using brands
+    using_organizations:        False         #Disable if you are not using organizations
     using_organization_tags:    False         #Disable if you are not using organization tags
 ```
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'zendesk'
-version: '0.19.1'
+version: '0.19.2'
 
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: zendesk_integration_tests_60
+      schema: zendesk_integration_tests_61
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -6,7 +6,7 @@ version: '0.19.1'
 profile: 'integration_tests'
 
 vars:
-  zendesk_schema: zendesk_integration_tests_60
+  zendesk_schema: zendesk_integration_tests_61
 
   zendesk_source:
     zendesk_organization_identifier:           "organization_data"
@@ -32,16 +32,15 @@ vars:
   # zendesk__unstructured_enabled: True
   # using_schedule_histories: True
 
-  ## For validation testing. To be commented out before release.
-  # zendesk_schema: zendesk_test_env
+  # For validation testing. To be commented out before release.
   # using_domain_names: false
   # using_user_tags: false
   # using_organization_tags: false
   # using_holidays: false
   # fivetran_integrity_sla_metric_parity_exclusion_tickets: (56,80)
   # fivetran_integrity_sla_first_reply_time_exclusion_tickets: (56,80)
-  consistency_test_exclude_fields: ['source_relation', 'ticket_tags', 'ticket_day_id', 'assignee_ticket_last_update_at', 'assignee_last_login_at', 'requester_created_at', 'requester_updated_at', 'requester_ticket_last_update_at','requester_organization_created_at', 'requester_organization_updated_at', 'requester_last_login_at', 'created_at', 'updated_at'] 
-  # get rid of source_relation and ticket_day_id after v0.19.0
+  # fivetran_integrity_sla_count_match_tickets: (11109)
+  # consistency_test_exclude_fields: ['ticket_tags', 'assignee_ticket_last_update_at', 'assignee_last_login_at', 'requester_created_at', 'requester_updated_at', 'requester_ticket_last_update_at','requester_organization_created_at', 'requester_organization_updated_at', 'requester_last_login_at', 'created_at', 'updated_at'] 
 
 models:
   +schema: "zendesk_{{ var('directed_schema','dev') }}"

integration_tests/tests/integrity/sla_count_match.sql

@@ -8,7 +8,7 @@
 with source as (
     select
         *,
-        case when field_name = 'first_reply_time' then row_number() over (partition by ticket_id, field_name, source_relation order by valid_starting_at) else 1 end as latest_sla
+        case when field_name in ('first_reply_time') then row_number() over (partition by ticket_id, field_name, source_relation order by valid_starting_at) else 1 end as latest_sla
     from {{ ref('stg_zendesk__ticket_field_history') }}
 ),
 

models/intermediate/int_zendesk__organization_aggregates.sql

@@ -1,9 +1,11 @@
+{{ config(enabled=var('using_organizations', True)) }}
+
 with organizations as (
     select * 
     from {{ ref('stg_zendesk__organization') }}
 
---If you use organization tags this will be included, if not it will be ignored.
-{% if var('using_organization_tags', True) %}
+--If you use organization tags and organizations, this will be included, if not it will be ignored.
+{% if var('using_organization_tags', True) and var('using_organizations', True) %}
 ), organization_tags as (
     select * 
     from {{ ref('stg_zendesk__organization_tag') }}
@@ -19,11 +21,12 @@ with organizations as (
         on organizations.organization_id = organization_tags.organization_id 
         and organizations.source_relation = organization_tags.source_relation
 
+
     group by 1, 2
 {% endif %}
 
 --If you use using_domain_names tags this will be included, if not it will be ignored.
-{% if var('using_domain_names', True) %}
+{% if var('using_domain_names', True) and var('using_organizations', True) %}
 ), domain_names as (
 
     select *
@@ -49,26 +52,26 @@ with organizations as (
         organizations.*
 
         --If you use organization tags this will be included, if not it will be ignored.
-        {% if var('using_organization_tags', True) %}
+        {% if var('using_organization_tags', True) and var('using_organizations', True) %}
         ,tag_aggregates.organization_tags
         {% endif %}
 
         --If you use using_domain_names tags this will be included, if not it will be ignored.
-        {% if var('using_domain_names', True) %}
+        {% if var('using_domain_names', True) and var('using_organizations', True) %}
         ,domain_aggregates.domain_names
         {% endif %}
 
     from organizations
 
     --If you use using_domain_names tags this will be included, if not it will be ignored.
-    {% if var('using_domain_names', True) %}
+    {% if var('using_domain_names', True) and var('using_organizations', True) %}
     left join domain_aggregates
         on organizations.organization_id = domain_aggregates.organization_id 
         and organizations.source_relation = domain_aggregates.source_relation
     {% endif %}
 
     --If you use organization tags this will be included, if not it will be ignored.
-    {% if var('using_organization_tags', True) %}
+    {% if var('using_organization_tags', True) and var('using_organizations', True) %}
     left join tag_aggregates
         on organizations.organization_id = tag_aggregates.organization_id 
         and organizations.source_relation = tag_aggregates.source_relation

models/intermediate/int_zendesk__ticket_aggregates.sql

@@ -7,10 +7,13 @@ with tickets as (
   select *
   from {{ ref('stg_zendesk__ticket_tag') }}
 
+--If you use using_brands this will be included, if not it will be ignored.
+{% if var('using_brands', True) %}
 ), brands as (
 
   select *
   from {{ ref('stg_zendesk__brand') }}
+{% endif %}
 
 ), ticket_tag_aggregate as (
   select
@@ -27,17 +30,21 @@ with tickets as (
       then true
       else false
         end as is_incident,
+    {% if var('using_brands', True) %}
     brands.name as ticket_brand_name,
+    {% endif %}
     ticket_tag_aggregate.ticket_tags
   from tickets
 
   left join ticket_tag_aggregate
     on tickets.ticket_id = ticket_tag_aggregate.ticket_id 
     and tickets.source_relation = ticket_tag_aggregate.source_relation
 
+  {% if var('using_brands', True) %}
   left join brands
     on brands.brand_id = tickets.brand_id
     and brands.source_relation = tickets.source_relation
+  {% endif %}    
 )
 
 select *

models/ticket_history/int_zendesk__updater_information.sql

@@ -2,9 +2,12 @@ with users as (
     select *
     from {{ ref('int_zendesk__user_aggregates') }}
 
+--If using organizations, this will be included, if not it will be ignored.
+{% if var('using_organizations', True) %}
 ), organizations as (
     select *
     from {{ ref('int_zendesk__organization_aggregates') }}
+{% endif %}
 
 ), final as (
     select
@@ -24,22 +27,26 @@ with users as (
 
         ,users.last_login_at as updater_last_login_at
         ,users.time_zone as updater_time_zone
+        {% if var('using_organizations', True) %}
         ,organizations.organization_id as updater_organization_id
+        {% endif %}
 
         --If you use using_domain_names tags this will be included, if not it will be ignored.
-        {% if var('using_domain_names', True) %}
+        {% if var('using_domain_names', True) and var('using_organizations', True) %}
         ,organizations.domain_names as updater_organization_domain_names
         {% endif %}
 
-        --If you use organization tags this will be included, if not it will be ignored.
-        {% if var('using_organization_tags', True) %}
+        --If you use organization tags, this will be included, if not it will be ignored.
+        {% if var('using_organization_tags', True) and var('using_organizations', True) %}
         ,organizations.organization_tags as updater_organization_organization_tags
         {% endif %}
     from users
 
+    {% if var('using_organizations', True) %}
     left join organizations
         on users.source_relation = organizations.source_relation
         and users.organization_id = organizations.organization_id
+    {% endif %}
 )
 
 select * 

models/zendesk__ticket_backlog.sql

@@ -17,9 +17,11 @@ with ticket_field_history as (
     select *
     from {{ ref('stg_zendesk__user') }}
 
+{% if var('using_brands', True) %}
 ), brands as (
     select *
     from {{ ref('stg_zendesk__brand') }}
+{% endif %}
 
 --The below model is excluded if the user does not include ticket_form_id in the variable as a low percentage of accounts use ticket forms.
 {% if 'ticket_form_id' in var('ticket_field_history_columns') %}
@@ -28,9 +30,12 @@ with ticket_field_history as (
     from {{ ref('int_zendesk__latest_ticket_form') }}
 {% endif %}
 
+--If using organizations, this will be included, if not it will be ignored.
+{% if var('using_organizations', True) %}
 ), organizations as (
     select *
     from {{ ref('stg_zendesk__organization') }}
+{% endif %}
 
 ), backlog as (
     select
@@ -49,10 +54,10 @@ with ticket_field_history as (
             {% elif col in ['ticket_form_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,ticket_forms.name as ticket_form_name
 
-            {% elif col in ['organization_id'] %} --Standard ID field where the name can easily be joined from stg model.
+            {% elif var('using_organizations', True) and col in ['organization_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,organizations.name as organization_name
 
-            {% elif col in ['brand_id'] %} --Standard ID field where the name can easily be joined from stg model.
+            {% elif var('using_brands', True) and col in ['brand_id'] %} --Standard ID field where the name can easily be joined from stg model.
                 ,brands.name as brand_name
 
             {% elif col in ['group_id'] %} --Standard ID field where the name can easily be joined from stg model.
@@ -96,13 +101,13 @@ with ticket_field_history as (
         and requester.source_relation = ticket_field_history.source_relation
     {% endif %}
 
-    {% if 'brand_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
+    {% if var('using_brands', True) and 'brand_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
     left join brands
         on brands.brand_id = cast(ticket_field_history.brand_id as {{ dbt.type_bigint() }})
         and brands.source_relation = ticket_field_history.source_relation
     {% endif %}
 
-    {% if 'organization_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
+    {% if var('using_organizations', True) and 'organization_id' in var('ticket_field_history_columns') %} --Join not needed if field is not located in variable, otherwise it is included.
     left join organizations
         on organizations.organization_id = cast(ticket_field_history.organization_id as {{ dbt.type_bigint() }})
         and organizations.source_relation = ticket_field_history.source_relation