Merge pull request #71 from DalgoT4D/pratiksha-dev

more tests
DalgoT4D · Jan 15, 2025 · a1d3919 · a1d3919
2 parents 42a71c9 + cc39523
commit a1d3919
Show file tree

Hide file tree

Showing 12 changed files with 435 additions and 51 deletions.
diff --git a/models/marts/education/education marts_tests.yml b/models/marts/education/education marts_tests.yml
@@ -0,0 +1,267 @@
+version: 2
+
+models:
+  - name: parent_satisfaction
+    description: "Aggregates parent satisfaction data for analysis."
+    columns:
+      - name: term
+        tests:
+          - not_null
+      - name: grade
+        tests:
+          - not_null
+      - name: school_type
+        tests:
+          - accepted_values:
+              values: ["ksg", "msg"]
+      - name: education_satisfaction
+        tests:
+          - accepted_values:
+              values: ["very_satisfied_5", "satisfied_4", "neutral_3", "dissatisfied_2", "very_dissatisfied_1"]
+      - name: year
+        tests:
+          - not_null
+
+  - name: student_satisfaction
+    description: "Analyzes student satisfaction with schools."
+    columns:
+      - name: term
+        tests:
+          - not_null
+      - name: grade
+        tests:
+          - not_null
+      - name: school_type
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["ksg", "msg"]
+      - name: school_satisfaction
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["very_satisfied", "neutral", "very_dissatisfied", "satisfied", "dissatisfied"]
+      - name: year
+        tests:
+          - not_null
+
+  - name: teacher_satisfaction
+    description: "Summarizes teacher satisfaction survey data."
+    columns:
+      - name: teaching_level
+        tests:
+          - not_null
+      - name: education_satisfaction
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["very_satisfied", "neutral", "very_dissatisfied", "satisfied", "dissatisfied"]
+      - name: school_type
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["ksg", "msg"]
+      - name: year
+        tests:
+          - not_null
+
+  - name: well_being_sessions
+    description: "Aggregates data on well-being sessions."
+    columns:
+      - name: "date"
+        tests:
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: date
+      - name: grade
+        tests:
+          - not_null
+      - name: school_type
+        tests:
+          - accepted_values:
+              values: ["ksg", "msg"]
+      - name: "number_of_students_trained"
+        tests:
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: integer
+      - name: "session_type"
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["Group", "Individual"]
+
+  - name: followup_attendance
+    description: "Tracks follow-ups for student attendance to address absenteeism."
+    columns:
+      - name: absence_date
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: date
+      - name: grade
+        tests:
+          - not_null
+      - name: year
+        tests:
+          - not_null
+      - name: term
+        tests:
+          - not_null
+      - name: reporting_date
+        tests:
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: date
+      - name: school_type
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["ksg", "msg"]
+      - name: number_of_absences
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: bigint
+
+  - name: parents_attendance
+    description: "Captures parent attendance data for school meetings and engagement levels."
+    columns:
+      - name: year
+        tests:
+          - not_null
+      - name: grade
+        tests:
+          - not_null
+      - name: date
+        tests:
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: date
+      - name: "meeting_type"
+        tests:
+          - not_null
+      - name: "number_present"
+        tests:
+          - not_null
+      - name: "attendance_percentage"
+        tests:
+          - not_null
+      - name: school_type
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["ksg", "msg"]
+
+  - name: students_attendance
+    description: "Tracks student attendance statistics."
+    columns:
+      - name: term
+        tests:
+          - not_null
+      - name: year
+        tests:
+          - not_null
+      - name: grade
+        tests:
+          - not_null
+      - name: "avg_days_attended"
+        tests:
+          - not_null
+      - name: "total_school_days"
+        tests:
+          - not_null
+      - name: "number_of_students"
+        tests:
+          - not_null
+      - name: "attendance_percentage"
+        tests:
+          - not_null
+      - name: "total_days_absent"
+        tests:
+          - not_null
+      - name: "total_days_present"
+        tests:
+          - not_null
+      - name: school_type
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["ksg", "msg"]
+
+  - name: education_scholarship_categories_agg
+    description: "Aggregates scholarship data to create categorized insights based on student demographics."
+    columns:
+      - name: year
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["2021", "2022", "2023", "2024", "2025"]
+      - name: cohort
+        tests:
+          - not_null
+      - name: categories
+        tests:
+          - not_null
+      - name: county
+        tests:
+          - not_null
+      - name: subcounty
+        tests:
+          - not_null
+      - name: term
+        tests:
+          - not_null
+      - name: boarding_or_day
+        tests:
+          - accepted_values:
+              values: ["Boarding", "Day"]
+
+  - name: nudges
+    description: "Analyzes distribution of education nudges based on demographics."
+    columns:
+      - name: term
+        tests:
+          - not_null
+      - name: year
+        tests:
+          - not_null
+      - name: county
+        tests:
+          - not_null
+      - name: grade
+        tests:
+          - not_null
+      - name: gender
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["male", "female"]
+      - name: nudge_type
+        tests:
+          - not_null
+      - name: nudge_count
+        tests:
+          - not_null
+          - dbt_expectations.expect_column_values_to_be_of_type:
+              column_type: bigint
+
+  - name: public_partnerships
+    description: "Summarizes public school partnership data, including KPCE scores and resource allocations."
+    columns:
+      - name: year
+        tests:
+          - not_null
+      - name: county
+        tests:
+          - not_null
+      - name: subcounty
+        tests:
+          - not_null
+      - name: mean_kpce_score
+        tests:
+          - not_null
+      - name: students_enrolled
+        tests:
+          - not_null
+      - name: high_touch_low_touch
+        tests:
+          - not_null
+          - accepted_values:
+              values: ["High", "Low"]
diff --git a/models/marts/education/followup_attendance.sql b/models/marts/education/followup_attendance.sql
@@ -11,24 +11,23 @@ WITH clean_data AS (
         "absence_causes",
         "_airbyte_extracted_at",
         "school_type",
-        -- Directly convert "Date_" to a proper date
-        TO_DATE("date", 'DD/MM/YYYY') AS "date",
+        TO_DATE("date", 'DD/MM/YYYY') AS "absence_date",
         TO_DATE("estimated_reporting_date", 'DD/MM/YYYY') AS "reporting_date"
     FROM {{ ref('staging_followup_attendance') }}
 )
 
 SELECT
-    "date" AS "absence_date",
-    EXTRACT(YEAR FROM "date") AS "year",
+    "absence_date",
+    EXTRACT(YEAR FROM "absence_date") AS "year",
 
     -- Calculate term based on the valid date
     CASE 
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date")
-                          AND DATE_TRUNC('year', "date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date") + INTERVAL '3 months'
-                          AND DATE_TRUNC('year', "date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date") + INTERVAL '7 months'
-                          AND DATE_TRUNC('year', "date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date")
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '3 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '7 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
         ELSE NULL
     END AS "term",
 
@@ -43,18 +42,17 @@ SELECT
     "absence_causes",
     "reporting_date",
     LOWER("school_type") AS "school_type",
-
     COUNT(*) AS "number_of_absences"
 FROM clean_data
 GROUP BY
-    "date",
+    "absence_date",
     CASE 
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date")
-                          AND DATE_TRUNC('year', "date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date") + INTERVAL '3 months'
-                          AND DATE_TRUNC('year', "date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
-        WHEN "date" BETWEEN DATE_TRUNC('year', "date") + INTERVAL '7 months'
-                          AND DATE_TRUNC('year', "date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date")
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '3 months - 1 day' THEN 'Term 1'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '3 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '7 months - 1 day' THEN 'Term 2'
+        WHEN "absence_date" BETWEEN DATE_TRUNC('year', "absence_date") + INTERVAL '7 months'
+                          AND DATE_TRUNC('year', "absence_date") + INTERVAL '12 months - 1 day' THEN 'Term 3'
         ELSE NULL
     END,
     CASE 

diff --git a/models/marts/education/parent_satisfaction.sql b/models/marts/education/parent_satisfaction.sql
@@ -13,7 +13,7 @@ SELECT
         ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER("grade"), '.*grade', ''))
     END AS "grade",
 
-    "school" AS "school_type",
+    LOWER("school") AS "school_type",
     "education_satisfaction",
     "class_year_of_child" AS "year"
 FROM {{ ref("staging_parents_satisfaction_survey") }}
diff --git a/models/marts/education/parents_attendance.sql b/models/marts/education/parents_attendance.sql
@@ -7,11 +7,11 @@ SELECT
     "Term" as "term", 
     "Year" as "year",
     "Grade" as "grade",
-    "Meeting_Date" as "date",
-    "Meeting_Type",
-    "Number_Present",
-    "Number_of_Parents",
-    "Share_of_Parents_Engaged",
-    "Attendance_Percentage_",
-    "school_type"
+    TO_DATE("Meeting_Date", 'DD/MM/YYYY') AS "date",
+    "Meeting_Type" as "meeting_type", 
+    "Number_Present" as "number_present",
+    "Number_of_Parents" as "number_of_parents",
+    "Share_of_Parents_Engaged" as "share_of_parents_engaged",
+    "Attendance_Percentage_" as "attendance_percentage",
+    LOWER("school_type") as "school_type"
 FROM {{ ref("staging_parents_attendance") }}
diff --git a/models/marts/education/public_partnerships.sql b/models/marts/education/public_partnerships.sql
@@ -5,7 +5,7 @@
 
 SELECT
     "year",
-    "school_name",
+    LOWER("school_name"),
     "county",
     "mean_kpce_score",
     "subcounty",

diff --git a/models/marts/education/student_satisfaction.sql b/models/marts/education/student_satisfaction.sql
@@ -13,7 +13,7 @@ SELECT
         ELSE CONCAT('Grade ', REGEXP_REPLACE(LOWER("grade"), '.*grade', ''))
     END AS "grade",
 
-    "school" AS "school_type",
+    LOWER("school") AS "school_type",
     "school_satisfaction",
     "class_year" AS "year"
 FROM {{ ref("staging_student_satisfaction_survey") }}
diff --git a/models/marts/education/students_attendance.sql b/models/marts/education/students_attendance.sql
@@ -7,11 +7,11 @@ SELECT
     "Term" as "term",
     "Grade" as "grade",
     "Year" as "year",
-    "Avg_Days_Attended",
-    "Total_School_Days",
-    "Number_of_Students",
-    "Attendance_Percentage",
-    "All_Students___Total_Days_Absent",
-    "All_Students___Total_Days_Present",
+    "Avg_Days_Attended" as "avg_days_attended",
+    "Total_School_Days" as "total_school_days",
+    "Number_of_Students" as "number_of_students",
+    "Attendance_Percentage" as "attendance_percentage",
+    "All_Students___Total_Days_Absent" as "total_days_absent",
+    "All_Students___Total_Days_Present" as "total_days_present",
     LOWER("School_Type") as "school_type"
 FROM {{ ref("staging_students_attendance") }}