pipeline fix + tags

.gitignore

@@ -3,6 +3,7 @@
 target/
 dbt_packages/
 logs/
+/profiles.yml
 
 # legacy -- renamed to dbt_packages in v1
 dbt_modules/

models/marts/education/education_scholarship_categories_agg.sql

@@ -1,6 +1,6 @@
 {{ config(
   materialized='table',
-  tags="education_expansion"
+  tags=["education_expansion", "education"]
 ) }}
 
 SELECT

models/marts/education/followup_attendance.sql

@@ -1,71 +1,66 @@
 {{ config(
     materialized='table',
-    tags="education_attendance"
+    tags=["education_attendance", "education"]
 ) }}
 
 
 WITH clean_data AS (
     SELECT
-        grade,
-        stream,
-        name_of_student,
-        absence_causes,
-        _airbyte_extracted_at,
-        school_type,
-        -- Directly convert "Date_" to a proper date
-        TO_DATE(date, 'DD/MM/YYYY') AS absence_date,
-        TO_DATE(estimated_reporting_date, 'DD/MM/YYYY') AS reporting_date
+        "grade",
+        "stream",
+        "name_of_student",
+        "absence_causes",
+        "_airbyte_extracted_at",
+        "school_type",
+        TO_DATE("date", 'DD/MM/YYYY') AS "absence_date",
+        TO_DATE("estimated_reporting_date", 'DD/MM/YYYY') AS "reporting_date"
     FROM {{ ref('staging_followup_attendance') }}
 )
 
 SELECT
-    absence_date,
-    EXTRACT(YEAR FROM absence_date) AS year,
+    "absence_date",
+    EXTRACT(YEAR FROM "absence_date") AS "year",
 
     -- Calculate term based on the valid date
     CASE 
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date)
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '3 months - 1 day' THEN 'Term 1'
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date) + INTERVAL '3 months'
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '7 months - 1 day' THEN 'Term 2'
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date) + INTERVAL '7 months'
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '12 months - 1 day' THEN 'Term 3'
-    END AS term,
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date")
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '3 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '7 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
+        ELSE NULL
+    END AS "term",
 
     -- Transform Grade values
     CASE 
-        WHEN grade = 'K' THEN 'Kindergarten'
-        ELSE CONCAT('Grade ', grade)
-    END AS grade,
+        WHEN "grade" = 'K' THEN 'Kindergarten'
+        ELSE CONCAT('Grade ', "grade")
+    END AS "grade",
 
     -- Retain other columns and transformations
-    stream,
-    absence_causes,
-    reporting_date,
-    LOWER(school_type) AS school_type,
-    COUNT(*) AS number_of_absences
+    "stream",
+    "absence_causes",
+    "reporting_date",
+    LOWER("school_type") AS "school_type",
+    COUNT(*) AS "number_of_absences"
 FROM clean_data
 GROUP BY
-    absence_date,
+    "absence_date",
     CASE 
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date)
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '3 months - 1 day' THEN 'Term 1'
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date) + INTERVAL '3 months'
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '7 months - 1 day' THEN 'Term 2'
-        WHEN
-            absence_date BETWEEN DATE_TRUNC('year', absence_date) + INTERVAL '7 months'
-            AND DATE_TRUNC('year', absence_date) + INTERVAL '12 months - 1 day' THEN 'Term 3'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date")
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '3 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '7 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
+        ELSE NULL
     END,
     CASE 
-        WHEN grade = 'K' THEN 'Kindergarten'
-        ELSE CONCAT('Grade ', grade)
+        WHEN "grade" = 'K' THEN 'Kindergarten'
+        ELSE CONCAT('Grade ', "grade")
     END,
-    stream,
-    absence_causes,
-    reporting_date,
-    LOWER(school_type)
+    "stream",
+    "absence_causes",
+    "reporting_date",
+    LOWER("school_type")

models/marts/education/nudges.sql

@@ -1,18 +1,18 @@
 {{ config(
     materialized='table',
-    tags="education_expansion"
+    tags=["education_expansion", "education"]
 ) }}
 
 SELECT
-    term,
-    CONCAT('Grade ', grade) AS grade,
-    cohort
-    AS year,
-    county,
-    subcounty,
-    gender,
-    nudge_type,
-    COUNT(*) AS nudge_count
+    "term",
+    CONCAT('Grade ', "grade") AS "grade",
+    "cohort",
+    "year",
+    "county",
+    "subcounty",
+    "gender",
+    "nudge_type",
+    COUNT(*) AS "nudge_count"
 FROM {{ ref('staging_nudges') }}
 GROUP BY
     term,

models/marts/education/parent_satisfaction.sql

@@ -1,19 +1,19 @@
 {{ config(
   materialized='table',
-    tags="education_satisfaction_surveys"
+    tags=["education_satisfaction_surveys", "education"]
 ) }}
 
 SELECT
     -- Transform term to "Term X"
-    INITCAP(REPLACE(term, '_', ' ')) AS term,
+    INITCAP(REPLACE("term", '_', ' ')) AS "term",
 
     -- Transform grade to "Kindergarten" or "Grade X"
     CASE
-        WHEN LOWER(grade) LIKE '%kindergarten%' THEN 'Kindergarten'
-        ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER(grade), '.*grade', ''))
-    END AS grade,
-    LOWER(school) AS school_type,
-    education_satisfaction,
-    class_year_of_child AS year
-FROM {{ ref("staging_parents_satisfaction_survey") }}
+        WHEN LOWER("grade") LIKE '%kindergarten%' THEN 'Kindergarten'
+        ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER("grade"), '.*grade', ''))
+    END AS "grade",
 
+    LOWER("school") AS "school_type",
+    "education_satisfaction",
+    "class_year_of_child" AS "year"
+FROM {{ ref("staging_parents_satisfaction_survey") }}

models/marts/education/parents_attendance.sql

@@ -1,17 +1,17 @@
 {{ config(
   materialized='table',
-    tags="education_attendance"
+    tags=["education_attendance", "education"]
 ) }}
 
 SELECT
-    "Term" AS term, 
-    "Year" AS year,
-    "Grade" AS grade,
-    TO_DATE("Meeting_Date", 'DD/MM/YYYY') AS date,
-    "Meeting_Type"  AS meeting_type, 
-    "Number_Present" AS number_present,
-    "Number_of_Parents" AS number_of_parents,
-    "Share_of_Parents_Engaged" AS share_of_parents_engaged,
-    "Attendance_Percentage_" AS attendance_percentage,
-    LOWER(school_type) AS school_type
+    "Term" as "term", 
+    "Year" as "year",
+    "Grade" as "grade",
+    TO_DATE("Meeting_Date", 'DD/MM/YYYY') AS "date",
+    "Meeting_Type" as "meeting_type", 
+    "Number_Present" as "number_present",
+    "Number_of_Parents" as "number_of_parents",
+    "Share_of_Parents_Engaged" as "share_of_parents_engaged",
+    "Attendance_Percentage_" as "attendance_percentage",
+    LOWER("school_type") as "school_type"
 FROM {{ ref("staging_parents_attendance") }}

models/marts/education/public_partnerships.sql

@@ -1,17 +1,17 @@
 {{ config(
   materialized='table',
-    tags="education_expansion"
+    tags=["education_expansion", "education"]
 ) }}
 
 SELECT
-    year,
-    LOWER(school_name) AS school_name,
-    county,
-    mean_kpce_score,
-    subcounty,
-    teachers_trained,
-    students_enrolled,
-    computers_provided,
-    toilet_stances_built,
-    high_touch_low_touch
+    "year",
+    LOWER("school_name"),
+    "county",
+    "mean_kpce_score",
+    "subcounty",
+    "teachers_trained",
+    "students_enrolled",
+    "computers_provided",
+    "toilet_stances_built",
+    "high_touch_low_touch"
 FROM {{ ref("staging_public_partnerships") }}

models/marts/education/student_satisfaction.sql

@@ -1,18 +1,19 @@
 {{ config(
   materialized='table',
-    tags="education_satisfaction_surveys"
+    tags=["education_satisfaction_surveys", "education"]
 ) }}
 
 SELECT
     -- Transform term to "Term X"
-    INITCAP(REPLACE(term, 'term', 'Term ')) AS term,
+    INITCAP(REPLACE("term", 'term', 'Term ')) AS "term",
 
     -- Transform grade to "Kindergarten" or "Grade X"
     CASE
-        WHEN LOWER(grade) LIKE '%kindergarten%' THEN 'Kindergarten'
-        ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER(grade), '.*grade', ''))
-    END AS grade,
-    LOWER(school) AS school_type,
-    school_satisfaction,
-    class_year AS year
+        WHEN LOWER("grade") LIKE '%kindergarten%' THEN 'Kindergarten'
+        ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER("grade"), '.*grade', ''))
+    END AS "grade",
+
+    LOWER("school") AS "school_type",
+    "school_satisfaction",
+    "class_year" AS "year"
 FROM {{ ref("staging_student_satisfaction_survey") }}

models/marts/education/students_attendance.sql

@@ -1,6 +1,6 @@
 {{ config(
   materialized='table',
-    tags="education_attendance"
+    tags=["education_attendance", "education"]
 ) }}
 
 SELECT
@@ -14,5 +14,4 @@ SELECT
     "All_Students___Total_Days_Absent" AS total_days_absent,
     "All_Students___Total_Days_Present" AS total_days_present,
     LOWER("School_Type") AS school_type
-FROM {{ ref("staging_students_attendance") }}
-
+FROM {{ ref("staging_students_attendance") }}

models/marts/education/teacher_satisfaction.sql

@@ -1,13 +1,11 @@
 {{ config(
   materialized='table',
-    tags="education_satisfaction_surveys"
+    tags=["education_satisfaction_surveys", "education"]
 ) }}
 
 SELECT
-    teaching_level,
-    school_name,
-    education_satisfaction,
-    LOWER(school_name) AS school_type,
-    year
-FROM {{ ref("staging_teacher_satisfaction_survey") }}
-
+    "teaching_level", 
+    "education_satisfaction",
+    LOWER("school_name") AS "school_type",
+    "year"
+FROM {{ ref("staging_teacher_satisfaction_survey") }}

models/marts/education/well_being_sessions.sql

@@ -1,15 +1,14 @@
 {{ config(
   materialized='table',
-    tags="education_well_being_sessions"
+    tags=["education_well_being_sessions", "education"]
 ) }}
 
 SELECT  
-    TO_DATE("Date", 'DD/MM/YYYY') AS date,           
-    "Grade" AS grade,          
-    "Topic" AS topic,               
-    LOWER("School") AS school_type,             
-    "Stream" AS stream,              
-    CAST("Number_of_stdents_trained" AS INTEGER) AS number_of_students_trained, 
-    "Session_Type" AS session_type   
-FROM {{ ref("staging_well_being_sessions") }}
-
+    TO_DATE("Date", 'DD/MM/YYYY') AS "date",           
+    "Grade" as "grade",          
+    "Topic" as "topic",               
+    LOWER("School") as "school_type",             
+    "Stream" as "stream",              
+    CAST("Number_of_stdents_trained" AS INTEGER) AS "number_of_students_trained", 
+    "Session_Type" as "session_type"    
+FROM {{ ref("staging_well_being_sessions") }}

models/marts/gender/case_occurence.sql

@@ -1,5 +1,6 @@
 {{ config(
-  materialized='table'
+  materialized='table',
+  tags= ["commcare_extraction", "gender_cases", "gender"]
 ) }}
 
 with

models/marts/gender/case_occurence_pii.sql

@@ -1,5 +1,6 @@
 {{ config(
   materialized='table',
+  tags= ["commcare_extraction", "gender_cases", "gender"]
 
 ) }}