From eab8d4224bafc3788a7e2a1ec616100f534dd0f1 Mon Sep 17 00:00:00 2001 From: Charlie Bini <5003326+cbini@users.noreply.github.com> Date: Wed, 4 Dec 2024 21:20:33 +0000 Subject: [PATCH] feat: add incidents_attachments, dbt dev schema --- .../staging/stg_deanslist__incidents.sql | 1 + .../stg_deanslist__incidents__attachments.sql | 32 +++++++++++++++++++ .../models/amplify/dds/sources-external.yml | 7 ++-- .../models/amplify/dibels/sources-drive.yml | 7 ++-- .../amplify/mclass/sources-external.yml | 7 ++-- .../models/deanslist/sources-kippcamden.yml | 12 +++++++ .../models/deanslist/sources-kippmiami.yml | 12 +++++++ .../models/deanslist/sources-kippnewark.yml | 12 +++++++ .../stg_deanslist__incidents__attachments.sql | 9 ++++++ .../models/iready/sources-kippmiami.yml | 7 ++-- .../kipptaf/models/iready/sources-kippnj.yml | 7 ++-- .../stg_iready__instruction_by_lesson_pro.sql | 4 +-- 12 files changed, 100 insertions(+), 17 deletions(-) create mode 100644 src/dbt/deanslist/models/staging/stg_deanslist__incidents__attachments.sql create mode 100644 src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents__attachments.sql diff --git a/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql b/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql index 9922c8692e..5213bcae4f 100644 --- a/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql +++ b/src/dbt/deanslist/models/staging/stg_deanslist__incidents.sql @@ -29,6 +29,7 @@ select actions, custom_fields, penalties, + attachments, /* transformations */ nullif(addlreqs, '') as addl_reqs, diff --git a/src/dbt/deanslist/models/staging/stg_deanslist__incidents__attachments.sql b/src/dbt/deanslist/models/staging/stg_deanslist__incidents__attachments.sql new file mode 100644 index 0000000000..9927eb18fa --- /dev/null +++ b/src/dbt/deanslist/models/staging/stg_deanslist__incidents__attachments.sql @@ -0,0 +1,32 @@ +select + i.incident_id, + + a.attachmenttype as attachment_type, + a.contenttype as content_type, + a.entityname as entity_name, + a.entitytype as entity_type, + a.internalfilename as internal_filename, + a.internalfolder as internal_folder, + a.minuserlevel as min_user_level, + a.minuserlevelgroupname as min_user_level_group_name, + a.publicfilename as public_filename, + a.reporttype as report_type, + a.sourcetype as source_type, + a.url, + + a.filepostedat.timezone as file_posted_at__timezone, + a.filepostedat.timezone_type as file_posted_at__timezone_type, + + cast(a.attachmentid as int) as attachment_id, + cast(a.bytes as int) as bytes, + cast(a.entityid as int) as entity_id, + cast(a.schoolid as int) as school_id, + cast(a.sourceid as int) as source_id, + cast(a.studentid as int) as student_id, + cast(a.termid as int) as term_id, + + cast(a.reportdate as date) as report_date, + + cast(a.filepostedat.`date` as datetime) as file_posted_at__date, +from {{ ref("stg_deanslist__incidents") }} as i +cross join unnest(i.attachments) as a diff --git a/src/dbt/kipptaf/models/amplify/dds/sources-external.yml b/src/dbt/kipptaf/models/amplify/dds/sources-external.yml index 19b75880c0..42e877118d 100644 --- a/src/dbt/kipptaf/models/amplify/dds/sources-external.yml +++ b/src/dbt/kipptaf/models/amplify/dds/sources-external.yml @@ -2,9 +2,10 @@ version: 2 sources: - name: amplify - schema: - "{% if env_var('DBT_DEV', '') == 'true' %}_dev_kipptaf_amplify{% else %}kipptaf_amplify{% - endif %}" + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kipptaf_amplify + {%- else -%}kipptaf_amplify + {%- endif %} tags: - stage_external_sources tables: diff --git a/src/dbt/kipptaf/models/amplify/dibels/sources-drive.yml b/src/dbt/kipptaf/models/amplify/dibels/sources-drive.yml index 05854169e4..75c6be0127 100644 --- a/src/dbt/kipptaf/models/amplify/dibels/sources-drive.yml +++ b/src/dbt/kipptaf/models/amplify/dibels/sources-drive.yml @@ -2,9 +2,10 @@ version: 2 sources: - name: amplify - schema: - "{% if env_var('DBT_DEV', '') == 'true' %}_dev_kipptaf_amplify{% else %}kipptaf_amplify{% - endif %}" + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kipptaf_amplify + {%- else -%}kipptaf_amplify + {%- endif %} tags: - stage_external_sources tables: diff --git a/src/dbt/kipptaf/models/amplify/mclass/sources-external.yml b/src/dbt/kipptaf/models/amplify/mclass/sources-external.yml index 29b55fd512..6075da7f1f 100644 --- a/src/dbt/kipptaf/models/amplify/mclass/sources-external.yml +++ b/src/dbt/kipptaf/models/amplify/mclass/sources-external.yml @@ -2,9 +2,10 @@ version: 2 sources: - name: amplify - schema: - "{% if env_var('DBT_DEV', '') == 'true' %}_dev_kipptaf_amplify{% else %}kipptaf_amplify{% - endif %}" + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kipptaf_amplify + {%- else -%}kipptaf_amplify + {%- endif %} tags: - stage_external_sources tables: diff --git a/src/dbt/kipptaf/models/deanslist/sources-kippcamden.yml b/src/dbt/kipptaf/models/deanslist/sources-kippcamden.yml index 54d3dfc14f..0cea91bbe9 100644 --- a/src/dbt/kipptaf/models/deanslist/sources-kippcamden.yml +++ b/src/dbt/kipptaf/models/deanslist/sources-kippcamden.yml @@ -2,6 +2,10 @@ version: 2 sources: - name: kippcamden_deanslist + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kippcamden_deanslist + {%- else -%}kippcamden_deanslist + {%- endif %} tables: - name: stg_deanslist__behavior meta: @@ -139,3 +143,11 @@ sources: - kippcamden - deanslist - stg_deanslist__dff_stats + - name: stg_deanslist__incidents__attachments + meta: + dagster: + group: deanslist + asset_key: + - kippcamden + - deanslist + - stg_deanslist__incidents__attachments diff --git a/src/dbt/kipptaf/models/deanslist/sources-kippmiami.yml b/src/dbt/kipptaf/models/deanslist/sources-kippmiami.yml index 1c747feba4..cd570b28d1 100644 --- a/src/dbt/kipptaf/models/deanslist/sources-kippmiami.yml +++ b/src/dbt/kipptaf/models/deanslist/sources-kippmiami.yml @@ -2,6 +2,10 @@ version: 2 sources: - name: kippmiami_deanslist + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kippmiami_deanslist + {%- else -%}kippmiami_deanslist + {%- endif %} tables: - name: stg_deanslist__behavior meta: @@ -139,3 +143,11 @@ sources: - kippmiami - deanslist - stg_deanslist__dff_stats + - name: stg_deanslist__incidents__attachments + meta: + dagster: + group: deanslist + asset_key: + - kippmiami + - deanslist + - stg_deanslist__incidents__attachments diff --git a/src/dbt/kipptaf/models/deanslist/sources-kippnewark.yml b/src/dbt/kipptaf/models/deanslist/sources-kippnewark.yml index 6860a2ca1b..1daa4a167c 100644 --- a/src/dbt/kipptaf/models/deanslist/sources-kippnewark.yml +++ b/src/dbt/kipptaf/models/deanslist/sources-kippnewark.yml @@ -2,6 +2,10 @@ version: 2 sources: - name: kippnewark_deanslist + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kippnewark_deanslist + {%- else -%}kippnewark_deanslist + {%- endif %} tables: - name: stg_deanslist__behavior meta: @@ -139,3 +143,11 @@ sources: - kippnewark - deanslist - stg_deanslist__dff_stats + - name: stg_deanslist__incidents__attachments + meta: + dagster: + group: deanslist + asset_key: + - kippnewark + - deanslist + - stg_deanslist__incidents__attachments diff --git a/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents__attachments.sql b/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents__attachments.sql new file mode 100644 index 0000000000..73af51d0be --- /dev/null +++ b/src/dbt/kipptaf/models/deanslist/staging/stg_deanslist__incidents__attachments.sql @@ -0,0 +1,9 @@ +{{ + dbt_utils.union_relations( + relations=[ + source("kippnewark_deanslist", "stg_deanslist__incidents__attachments"), + source("kippcamden_deanslist", "stg_deanslist__incidents__attachments"), + source("kippmiami_deanslist", "stg_deanslist__incidents__attachments"), + ] + ) +}} diff --git a/src/dbt/kipptaf/models/iready/sources-kippmiami.yml b/src/dbt/kipptaf/models/iready/sources-kippmiami.yml index 57e915974b..861ed611a1 100644 --- a/src/dbt/kipptaf/models/iready/sources-kippmiami.yml +++ b/src/dbt/kipptaf/models/iready/sources-kippmiami.yml @@ -2,9 +2,10 @@ version: 2 sources: - name: kippmiami_iready - schema: - "{% if env_var('DBT_DEV', '') == 'true' %}_dev_kippmiami_iready{% else %}kippmiami_iready{% - endif %}" + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kippmiami_iready + {%- else -%}kippmiami_iready + {%- endif %} tables: - name: stg_iready__diagnostic_results meta: diff --git a/src/dbt/kipptaf/models/iready/sources-kippnj.yml b/src/dbt/kipptaf/models/iready/sources-kippnj.yml index 2302d542c9..4815246f45 100644 --- a/src/dbt/kipptaf/models/iready/sources-kippnj.yml +++ b/src/dbt/kipptaf/models/iready/sources-kippnj.yml @@ -2,9 +2,10 @@ version: 2 sources: - name: kippnj_iready - schema: - "{% if env_var('DBT_DEV', '') == 'true' %}_dev_kippnewark_iready{% else %}kippnewark_iready{% - endif %}" + schema: | + {% if env_var('DBT_DEV', '') == 'true' -%}_dev_kippnewark_iready + {%- else -%}kippnewark_iready + {%- endif %} tables: - name: stg_iready__diagnostic_results meta: diff --git a/src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql b/src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql index e6a879643b..f9cc98bfaa 100644 --- a/src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql +++ b/src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql @@ -1,8 +1,8 @@ {{ dbt_utils.union_relations( relations=[ - source("kippnj_iready", model.name), - source("kippmiami_iready", model.name), + source("kippnj_iready", "stg_iready__instruction_by_lesson_pro"), + source("kippmiami_iready", "stg_iready__instruction_by_lesson_pro"), ] ) }}