From eb9e0411f7df79a3ec73e10f16a4564db9f06361 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Tue, 14 Nov 2023 18:50:02 +0000 Subject: [PATCH 01/11] building responses clean staging table --- .../intermediate/int_surveys__forms_responses_clean.sql | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql new file mode 100644 index 0000000000..3270659a89 --- /dev/null +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql @@ -0,0 +1,5 @@ +SELECT * --clean this up +FROM {{ ref("base_google_forms__form_responses") }} as fr +INNER JOIN {{ ref("src_google_forms__form_items_extension")}} as fi +ON fr.form_id = fi.form_id +AND fr.item_id = fi.question_id \ No newline at end of file From 6e79dd8a3fba26a912ce8230a4cffb04d895a17f Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Tue, 14 Nov 2023 21:09:33 +0000 Subject: [PATCH 02/11] part of first of support survey view --- .../int_surveys__forms_responses_clean.sql | 5 -- .../int_surveys__support_survey_details.sql | 85 +++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) delete mode 100644 src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql create mode 100644 src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql deleted file mode 100644 index 3270659a89..0000000000 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__forms_responses_clean.sql +++ /dev/null @@ -1,5 +0,0 @@ -SELECT * --clean this up -FROM {{ ref("base_google_forms__form_responses") }} as fr -INNER JOIN {{ ref("src_google_forms__form_items_extension")}} as fi -ON fr.form_id = fi.form_id -AND fr.item_id = fi.question_id \ No newline at end of file diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql new file mode 100644 index 0000000000..bbe327b992 --- /dev/null +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -0,0 +1,85 @@ +with response_identifiers as ( + select fr.form_id as survey_id, + fr.info_title as survey_title, + fr.response_id as survey_response_id, + + rt.Academic_year as campaign_academic_year, + rt.name as campaign_name, + rt.code as campaign_reporting_term, + + TIMESTAMP(fr.create_time) as date_started, + TIMESTAMP(fr.last_submitted_time) as date_submitted, + + safe_cast(regexp_extract(fr.text_value, r'\((\d{6})\)') as integer) as respondent_df_employee_number, + fr.respondent_email + +from {{ ref("base_google_forms__form_responses") }} as fr +left join {{ ref("stg_reporting__terms") }} as rt + on DATE(fr.last_submitted_time) between rt.Start_Date and rt.End_Date + AND rt.type = 'SURVEY' + AND rt.code IN ('SUP1','SUP2') +WHERE fr.question_item__question__question_id = '55f7fb30' + AND fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' + +UNION DISTINCT + + select fr.form_id as survey_id, + fr.info_title as survey_title, + fr.response_id as survey_response_id, + + rt.Academic_year as campaign_academic_year, + rt.name as campaign_name, + rt.code as campaign_reporting_term, + + TIMESTAMP(fr.create_time) as date_started, + TIMESTAMP(fr.last_submitted_time) as date_submitted, + + up.employee_number as respondent_df_employee_number, + fr.respondent_email + +from {{ ref("base_google_forms__form_responses") }} as fr +left join {{ ref("stg_reporting__terms") }} as rt + on DATE(fr.last_submitted_time) between rt.Start_Date and rt.End_Date + AND rt.type = 'SURVEY' + AND rt.code IN ('SUP1','SUP2') +inner join {{ ref("stg_ldap__user_person") }} as up + on fr.respondent_email = up.google_email +WHERE fr.question_item__question__question_id = '55f7fb30' + AND fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' + ) + +select ri.survey_id, + ri.survey_title, + ri.survey_response_id, + ri.date_started, + ri.date_submitted, + ri.campaign_academic_year, + ri.campaign_name, + ri.campaign_reporting_term, + CASE WHEN safe_cast(fr.text_value as integer) is null then 1 else 0 end as is_open_ended, + + fi.abbreviation as question_shortname, + fi.title as question_title, + + fr.text_value as answer, + safe_cast(fr.text_value as integer) as answer_value, + + ri.respondent_df_employee_number, + eh.preferred_name_lastfirst, + ri.respondent_email, + + --fill out with remaining fields from: + --https://github.com/TEAMSchools/mssql-warehouse/blob/f32a07eedc29a622e90b1175ff3b5848a01eb91b/gabby/surveys/view/cmo_engagement_regional_survey_detail.sql#L2 + +from response_identifiers as ri +inner join {{ ref("base_google_forms__form_responses") }} as fr + on ri.survey_id = fr.form_id + and ri.survey_response_id = fr.response_id +inner join {{ source("google_forms","src_google_forms__form_items_extension") }} as fi + on fr.form_id = fi.form_id + and fr.question_item__question__question_id = fi.question_id +inner join {{ ref("base_people__staff_roster_history") }} as eh + on ri.respondent_df_employee_number = eh.employee_number + and ri.date_submitted between eh.work_assignment__fivetran_start and eh.work_assignment__fivetran_end + + --ADD HISTORIC DATE WITH UNION \ No newline at end of file From ddc3e7c9584d2984812f236ca33461bf0776c043 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 14:15:25 +0000 Subject: [PATCH 03/11] Support survey data model --- .../int_surveys__support_survey_details.sql | 173 ++++++++++-------- 1 file changed, 95 insertions(+), 78 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index bbe327b992..cd190ef382 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -1,85 +1,102 @@ -with response_identifiers as ( - select fr.form_id as survey_id, - fr.info_title as survey_title, - fr.response_id as survey_response_id, - - rt.Academic_year as campaign_academic_year, - rt.name as campaign_name, - rt.code as campaign_reporting_term, - - TIMESTAMP(fr.create_time) as date_started, - TIMESTAMP(fr.last_submitted_time) as date_submitted, - - safe_cast(regexp_extract(fr.text_value, r'\((\d{6})\)') as integer) as respondent_df_employee_number, - fr.respondent_email - -from {{ ref("base_google_forms__form_responses") }} as fr -left join {{ ref("stg_reporting__terms") }} as rt - on DATE(fr.last_submitted_time) between rt.Start_Date and rt.End_Date - AND rt.type = 'SURVEY' - AND rt.code IN ('SUP1','SUP2') -WHERE fr.question_item__question__question_id = '55f7fb30' - AND fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' - -UNION DISTINCT - - select fr.form_id as survey_id, - fr.info_title as survey_title, - fr.response_id as survey_response_id, - - rt.Academic_year as campaign_academic_year, - rt.name as campaign_name, - rt.code as campaign_reporting_term, - - TIMESTAMP(fr.create_time) as date_started, - TIMESTAMP(fr.last_submitted_time) as date_submitted, - - up.employee_number as respondent_df_employee_number, - fr.respondent_email - -from {{ ref("base_google_forms__form_responses") }} as fr -left join {{ ref("stg_reporting__terms") }} as rt - on DATE(fr.last_submitted_time) between rt.Start_Date and rt.End_Date - AND rt.type = 'SURVEY' - AND rt.code IN ('SUP1','SUP2') -inner join {{ ref("stg_ldap__user_person") }} as up - on fr.respondent_email = up.google_email -WHERE fr.question_item__question__question_id = '55f7fb30' - AND fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' - ) +with + response_identifiers as ( + select + fr.form_id as survey_id, + fr.info_title as survey_title, + fr.response_id as survey_response_id, + + rt.academic_year as campaign_academic_year, + rt.name as campaign_name, + rt.code as campaign_reporting_term, + + timestamp(fr.create_time) as date_started, + timestamp(fr.last_submitted_time) as date_submitted, + + safe_cast( + regexp_extract(fr.text_value, r'\((\d{6})\)') as integer + ) as respondent_df_employee_number, + fr.respondent_email + + from {{ ref("base_google_forms__form_responses") }} as fr + left join + {{ ref("stg_reporting__terms") }} as rt + on date(fr.last_submitted_time) between rt.start_date and rt.end_date + and rt.type = 'SURVEY' + and rt.code in ('SUP1', 'SUP2') + where + fr.question_item__question__question_id = '55f7fb30' + and fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' + + union distinct -select ri.survey_id, - ri.survey_title, - ri.survey_response_id, - ri.date_started, - ri.date_submitted, - ri.campaign_academic_year, - ri.campaign_name, - ri.campaign_reporting_term, - CASE WHEN safe_cast(fr.text_value as integer) is null then 1 else 0 end as is_open_ended, + select + fr.form_id as survey_id, + fr.info_title as survey_title, + fr.response_id as survey_response_id, - fi.abbreviation as question_shortname, - fi.title as question_title, + rt.academic_year as campaign_academic_year, + rt.name as campaign_name, + rt.code as campaign_reporting_term, - fr.text_value as answer, - safe_cast(fr.text_value as integer) as answer_value, + timestamp(fr.create_time) as date_started, + timestamp(fr.last_submitted_time) as date_submitted, - ri.respondent_df_employee_number, - eh.preferred_name_lastfirst, - ri.respondent_email, + up.employee_number as respondent_df_employee_number, + fr.respondent_email - --fill out with remaining fields from: - --https://github.com/TEAMSchools/mssql-warehouse/blob/f32a07eedc29a622e90b1175ff3b5848a01eb91b/gabby/surveys/view/cmo_engagement_regional_survey_detail.sql#L2 + from {{ ref("base_google_forms__form_responses") }} as fr + left join + {{ ref("stg_reporting__terms") }} as rt + on date(fr.last_submitted_time) between rt.start_date and rt.end_date + and rt.type = 'SURVEY' + and rt.code in ('SUP1', 'SUP2') + inner join + {{ ref("stg_ldap__user_person") }} as up + on fr.respondent_email = up.google_email + where + fr.question_item__question__question_id = '55f7fb30' + and fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' + ) + +select + ri.survey_id, + ri.survey_title, + ri.survey_response_id, + ri.date_started, + ri.date_submitted, + ri.campaign_academic_year, + ri.campaign_name, + ri.campaign_reporting_term, + case + when safe_cast(fr.text_value as integer) is null then 1 else 0 + end as is_open_ended, + + fi.abbreviation as question_shortname, + fi.title as question_title, + fr.text_value as answer, + safe_cast(fr.text_value as integer) as answer_value, + + ri.respondent_df_employee_number, + eh.preferred_name_lastfirst, + ri.respondent_email, + +-- fill out with remaining fields from: +-- https://github.com/TEAMSchools/mssql-warehouse/blob/f32a07eedc29a622e90b1175ff3b5848a01eb91b/gabby/surveys/view/cmo_engagement_regional_survey_detail.sql#L2 from response_identifiers as ri -inner join {{ ref("base_google_forms__form_responses") }} as fr - on ri.survey_id = fr.form_id - and ri.survey_response_id = fr.response_id -inner join {{ source("google_forms","src_google_forms__form_items_extension") }} as fi - on fr.form_id = fi.form_id - and fr.question_item__question__question_id = fi.question_id -inner join {{ ref("base_people__staff_roster_history") }} as eh - on ri.respondent_df_employee_number = eh.employee_number - and ri.date_submitted between eh.work_assignment__fivetran_start and eh.work_assignment__fivetran_end - - --ADD HISTORIC DATE WITH UNION \ No newline at end of file +inner join + {{ ref("base_google_forms__form_responses") }} as fr + on ri.survey_id = fr.form_id + and ri.survey_response_id = fr.response_id +inner join + {{ source("google_forms", "src_google_forms__form_items_extension") }} as fi + on fr.form_id = fi.form_id + and fr.question_item__question__question_id = fi.question_id +inner join + {{ ref("base_people__staff_roster_history") }} as eh + on ri.respondent_df_employee_number = eh.employee_number + and ri.date_submitted + between eh.work_assignment__fivetran_start and eh.work_assignment__fivetran_end + + -- ADD HISTORIC DATA WITH UNION + From 4217456e1e244beae9f932384d1c5d335a13af3b Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 14:16:13 +0000 Subject: [PATCH 04/11] finishing touches --- .../int_surveys__support_survey_details.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index cd190ef382..f604f326fd 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -80,9 +80,15 @@ select ri.respondent_df_employee_number, eh.preferred_name_lastfirst, ri.respondent_email, + eh.management_position_indicator as is_manager, + eh.department_home_name as respondent_department_name, + eh.business_unit_home_name as respondent_legal_entity_name, + eh.report_to_preferred_name_lastfirst as respondent_manager_name, + eh.job_title as respondent_primary_job, + eh.home_work_location_name as respondent_primary_site, + eh.race_ethnicity_reporting, + eh.gender_identity as gender, --- fill out with remaining fields from: --- https://github.com/TEAMSchools/mssql-warehouse/blob/f32a07eedc29a622e90b1175ff3b5848a01eb91b/gabby/surveys/view/cmo_engagement_regional_survey_detail.sql#L2 from response_identifiers as ri inner join {{ ref("base_google_forms__form_responses") }} as fr From 688b9fbd0ec74146e650f7a5208a391fe85d5524 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 14:33:48 +0000 Subject: [PATCH 05/11] adding historic --- .../int_surveys__support_survey_details.sql | 72 +++++++++++++++---- 1 file changed, 58 insertions(+), 14 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index f604f326fd..df0389feeb 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -4,18 +4,18 @@ with fr.form_id as survey_id, fr.info_title as survey_title, fr.response_id as survey_response_id, + fr.respondent_email, rt.academic_year as campaign_academic_year, rt.name as campaign_name, rt.code as campaign_reporting_term, - timestamp(fr.create_time) as date_started, - timestamp(fr.last_submitted_time) as date_submitted, - safe_cast( regexp_extract(fr.text_value, r'\((\d{6})\)') as integer ) as respondent_df_employee_number, - fr.respondent_email + + timestamp(fr.create_time) as date_started, + timestamp(fr.last_submitted_time) as date_submitted, from {{ ref("base_google_forms__form_responses") }} as fr left join @@ -33,17 +33,17 @@ with fr.form_id as survey_id, fr.info_title as survey_title, fr.response_id as survey_response_id, + fr.respondent_email, rt.academic_year as campaign_academic_year, rt.name as campaign_name, rt.code as campaign_reporting_term, + up.employee_number as respondent_df_employee_number, + timestamp(fr.create_time) as date_started, timestamp(fr.last_submitted_time) as date_submitted, - up.employee_number as respondent_df_employee_number, - fr.respondent_email - from {{ ref("base_google_forms__form_responses") }} as fr left join {{ ref("stg_reporting__terms") }} as rt @@ -67,19 +67,19 @@ select ri.campaign_academic_year, ri.campaign_name, ri.campaign_reporting_term, - case - when safe_cast(fr.text_value as integer) is null then 1 else 0 - end as is_open_ended, + ri.respondent_df_employee_number, + ri.respondent_email, fi.abbreviation as question_shortname, fi.title as question_title, fr.text_value as answer, safe_cast(fr.text_value as integer) as answer_value, + case + when safe_cast(fr.text_value as integer) is null then 1 else 0 + end as is_open_ended, - ri.respondent_df_employee_number, eh.preferred_name_lastfirst, - ri.respondent_email, eh.management_position_indicator as is_manager, eh.department_home_name as respondent_department_name, eh.business_unit_home_name as respondent_legal_entity_name, @@ -104,5 +104,49 @@ inner join and ri.date_submitted between eh.work_assignment__fivetran_start and eh.work_assignment__fivetran_end - -- ADD HISTORIC DATA WITH UNION - +union all + +select + 'historic_alchemer_cmo_support_survey' as survey_id, + 'CMO & Support Survey History' as survey_title, + null as survey_response_id, + null as date_started, + timestamp(sda.date_submitted) as date_submitted, + sda.campaign_academic_year, + null as campaign_name, + sda.campaign_reporting_term, + sda.respondent_df_employee_number, + null as respondent_email, + + sda.question_shortname as question_shortname, + coalesce(fi.title, sda.question_shortname) as question_title, + + sda.answer, + sda.answer_value, + case when sda.answer_value is null then 1 else 0 end as is_open_ended, + + eh.preferred_name_lastfirst, + eh.management_position_indicator as is_manager, + eh.department_home_name as respondent_department_name, + eh.business_unit_home_name as respondent_legal_entity_name, + eh.report_to_preferred_name_lastfirst as respondent_manager_name, + eh.job_title as respondent_primary_job, + eh.home_work_location_name as respondent_primary_site, + eh.race_ethnicity_reporting, + eh.gender_identity as gender, + +from + {{ + source( + "surveys", "src_surveys__cmo_engagement_regional_survey_detail_archive" + ) + }} as sda +inner join + {{ source("google_forms", "src_google_forms__form_items_extension") }} as fi + on fi.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' + and sda.question_shortname = fi.abbreviation +left join + {{ ref("base_people__staff_roster_history") }} as eh + on sda.respondent_df_employee_number = eh.employee_number + and timestamp(sda.date_submitted) + between eh.work_assignment__fivetran_start and eh.work_assignment__fivetran_end From f52045e0396d4ed4846ca6bf6ea24d49056800eb Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 15:03:08 +0000 Subject: [PATCH 06/11] converting int to float --- .../intermediate/int_surveys__support_survey_details.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index df0389feeb..12de45f80f 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -74,7 +74,7 @@ select fi.title as question_title, fr.text_value as answer, - safe_cast(fr.text_value as integer) as answer_value, + safe_cast(fr.text_value as numeric) as answer_value, case when safe_cast(fr.text_value as integer) is null then 1 else 0 end as is_open_ended, @@ -122,7 +122,7 @@ select coalesce(fi.title, sda.question_shortname) as question_title, sda.answer, - sda.answer_value, + safe_cast(sda.answer_value as numeric) as answer_value, case when sda.answer_value is null then 1 else 0 end as is_open_ended, eh.preferred_name_lastfirst, From 6f70db4b02021cc092938bd21c7b8611e062455a Mon Sep 17 00:00:00 2001 From: Charlie Bini Date: Wed, 15 Nov 2023 16:07:48 +0000 Subject: [PATCH 07/11] fix data types --- src/dbt/kipptaf/models/surveys/sources.yml | 25 ++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/sources.yml b/src/dbt/kipptaf/models/surveys/sources.yml index 7220ca72f6..cdb633f8a8 100644 --- a/src/dbt/kipptaf/models/surveys/sources.yml +++ b/src/dbt/kipptaf/models/surveys/sources.yml @@ -16,14 +16,31 @@ sources: skip_leading_rows: 1 - name: src_surveys__cmo_engagement_regional_survey_detail_archive external: - location: - "{{ var('couchdrop_cloud_storage_uri_base') + location: "{{ var('couchdrop_cloud_storage_uri_base') }}/surveys/cmo_engagement_regional_survey_detail_archive.csv" options: format: CSV + columns: + - name: respondent_df_employee_number + data_type: STRING + - name: respondent_legal_entity_name + data_type: STRING + - name: respondent_primary_site + data_type: STRING + - name: date_submitted + data_type: STRING + - name: campaign_academic_year + data_type: STRING + - name: campaign_reporting_term + data_type: STRING + - name: question_shortname + data_type: STRING + - name: answer + data_type: STRING + - name: answer_value + data_type: STRING - name: src_surveys__manager_survey_detail_archive external: - location: - "{{ var('couchdrop_cloud_storage_uri_base') }}/surveys/manager_survey_detail_archive.csv" + location: "{{ var('couchdrop_cloud_storage_uri_base') }}/surveys/manager_survey_detail_archive.csv" options: format: CSV From 6c5b764d68a939ca0c1c580e856e91be061f45b7 Mon Sep 17 00:00:00 2001 From: Charlie Bini Date: Wed, 15 Nov 2023 16:36:32 +0000 Subject: [PATCH 08/11] fix skip row --- src/dbt/kipptaf/models/surveys/sources.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/sources.yml b/src/dbt/kipptaf/models/surveys/sources.yml index cdb633f8a8..014c2bb92c 100644 --- a/src/dbt/kipptaf/models/surveys/sources.yml +++ b/src/dbt/kipptaf/models/surveys/sources.yml @@ -20,17 +20,18 @@ sources: }}/surveys/cmo_engagement_regional_survey_detail_archive.csv" options: format: CSV + skip_leading_rows: 1 columns: - name: respondent_df_employee_number - data_type: STRING + data_type: INTEGER - name: respondent_legal_entity_name data_type: STRING - name: respondent_primary_site data_type: STRING - name: date_submitted - data_type: STRING + data_type: DATE - name: campaign_academic_year - data_type: STRING + data_type: INTEGER - name: campaign_reporting_term data_type: STRING - name: question_shortname @@ -38,9 +39,10 @@ sources: - name: answer data_type: STRING - name: answer_value - data_type: STRING + data_type: NUMERIC - name: src_surveys__manager_survey_detail_archive external: location: "{{ var('couchdrop_cloud_storage_uri_base') }}/surveys/manager_survey_detail_archive.csv" options: format: CSV + skip_leading_rows: 1 From e870ce1f8bba658bdcc4dcc28ab14f9f39847786 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 16:37:14 +0000 Subject: [PATCH 09/11] data types + staging table --- .../int_surveys__support_survey_details.sql | 7 +------ ...gagement_regional_survey_detail_archive.sql | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index 12de45f80f..986bbb4340 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -135,12 +135,7 @@ select eh.race_ethnicity_reporting, eh.gender_identity as gender, -from - {{ - source( - "surveys", "src_surveys__cmo_engagement_regional_survey_detail_archive" - ) - }} as sda +from {{ ref("stg_surveys__cmo_engagement_regional_survey_detail_archive") }} as sda inner join {{ source("google_forms", "src_google_forms__form_items_extension") }} as fi on fi.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' diff --git a/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql b/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql new file mode 100644 index 0000000000..e2cee2bd69 --- /dev/null +++ b/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql @@ -0,0 +1,18 @@ +select + safe_cast( + sda.respondent_df_employee_number as integer + ) as respondent_df_employee_number, + sda.respondent_legal_entity_name, + sda.respondent_primary_site, + safe_cast(sda.date_submitted as timestamp) as date_submitted, + safe_cast(sda.campaign_academic_year as integer) as campaign_academic_year, + sda.campaign_reporting_term, + sda.question_shortname, + sda.answer, + safe_cast(sda.answer_value as numeric) as answer_value +from + {{ + source( + "surveys", "src_surveys__cmo_engagement_regional_survey_detail_archive" + ) + }} as sda From 6f2201630645837de91836f9bb3f8da868be2313 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 16:41:26 +0000 Subject: [PATCH 10/11] trunk shit --- ...cmo_engagement_regional_survey_detail_archive.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql b/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql index e2cee2bd69..7e3957723e 100644 --- a/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql +++ b/src/dbt/kipptaf/models/surveys/staging/stg_surveys__cmo_engagement_regional_survey_detail_archive.sql @@ -1,15 +1,15 @@ select - safe_cast( - sda.respondent_df_employee_number as integer - ) as respondent_df_employee_number, sda.respondent_legal_entity_name, sda.respondent_primary_site, - safe_cast(sda.date_submitted as timestamp) as date_submitted, - safe_cast(sda.campaign_academic_year as integer) as campaign_academic_year, sda.campaign_reporting_term, sda.question_shortname, sda.answer, - safe_cast(sda.answer_value as numeric) as answer_value + safe_cast(sda.date_submitted as timestamp) as date_submitted, + safe_cast(sda.campaign_academic_year as integer) as campaign_academic_year, + safe_cast( + sda.respondent_df_employee_number as integer + ) as respondent_df_employee_number, + safe_cast(sda.answer_value as numeric) as answer_value, from {{ source( From c208e67e5d01fa295b9ae93b4e7462b8ebf706f3 Mon Sep 17 00:00:00 2001 From: Kevin Verhoff Date: Wed, 15 Nov 2023 17:21:02 +0000 Subject: [PATCH 11/11] removing id matching thing --- .../int_surveys__support_survey_details.sql | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql index 986bbb4340..4d02b34cca 100644 --- a/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql +++ b/src/dbt/kipptaf/models/surveys/intermediate/int_surveys__support_survey_details.sql @@ -1,34 +1,5 @@ with response_identifiers as ( - select - fr.form_id as survey_id, - fr.info_title as survey_title, - fr.response_id as survey_response_id, - fr.respondent_email, - - rt.academic_year as campaign_academic_year, - rt.name as campaign_name, - rt.code as campaign_reporting_term, - - safe_cast( - regexp_extract(fr.text_value, r'\((\d{6})\)') as integer - ) as respondent_df_employee_number, - - timestamp(fr.create_time) as date_started, - timestamp(fr.last_submitted_time) as date_submitted, - - from {{ ref("base_google_forms__form_responses") }} as fr - left join - {{ ref("stg_reporting__terms") }} as rt - on date(fr.last_submitted_time) between rt.start_date and rt.end_date - and rt.type = 'SURVEY' - and rt.code in ('SUP1', 'SUP2') - where - fr.question_item__question__question_id = '55f7fb30' - and fr.form_id = '1YdgXFZE1yjJa-VfpclZrBtxvW0w4QvxNrvbDUBxIiWI' - - union distinct - select fr.form_id as survey_id, fr.info_title as survey_title,