Modify mart_ad_hoc.fct_scheduled_service_by_daypart -> `mart_gtfs.f…

…ct_monthly_route_service_by_timeofday` (#3298) * use inner join to filter for max_date * filter out nulls in results * remove hour and aggregate to time-of-day * move from ad_hoc to mart_gtfs * remove ad_hoc yml, now in mart_gtfs * run table * add primary key test * add year back
cal-itp · Mar 12, 2024 · 9c3ac72 · 9c3ac72
1 parent 1b436b6
commit 9c3ac72
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 38 deletions.
diff --git a/warehouse/models/mart/ad_hoc/_mart_ad_hoc.yml b/warehouse/models/mart/ad_hoc/_mart_ad_hoc.yml
diff --git a/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml b/warehouse/models/mart/gtfs/_mart_gtfs_fcts.yml
@@ -2119,3 +2119,36 @@ models:
       - name: pt_array
         description: |
           Array of points describing this shape, looked up from `dim_shapes_arrays` via `shape_array_key`.
+  - name: fct_monthly_route_service_by_timeofday
+    description: |
+      An aggregation of GTFS schedule service by day and time characteristics.
+    columns:
+      - name: key
+        description: |
+          Synthetic primary key constructed from `source_record_id`, `route_id`, `month`, `year`, and `time_of_day`.
+        tests: *primary_key_tests
+      - name: name
+      - name: source_record_id
+      - name: route_id
+      - name: route_short_name
+      - name: route_long_name
+      - name: time_of_day
+        description: |
+          Categorized based on the Pacific Time departure of the trip's first departure.
+      - name: month
+        description: |
+          Actual calendar month (Pacific Time dates) in which this service was scheduled to occur.
+      - name: year
+        description: |
+          Actual calendar year (Pacific Time dates) in which this service was scheduled to occur.
+      - name: day_type
+        description: |
+          Actual calendar day type (Pacific Time dates) in which this service was scheduled to occur.
+          This means that overnight service is associated with the calendar date on which it was scheduled,
+          even if it was associated with the prior `service_date` by the agency.
+      - name: n_trips
+        description: |
+          Total trips that occurred for the route for this month and `time_of_day`.
+      - name: ttl_service_hours
+        description: |
+          Total scheduled service hours that occurred for the route for this month and `time_of_day`.
diff --git a/..._hoc/fct_scheduled_service_by_daypart.sql → ...ct_monthly_route_service_by_timeofday.sql b/..._hoc/fct_scheduled_service_by_daypart.sql → ...ct_monthly_route_service_by_timeofday.sql
@@ -4,17 +4,30 @@ WITH dim_gtfs_datasets AS (
     SELECT * FROM {{ ref('dim_gtfs_datasets') }}
 ),
 
+-- keep the most recent of month_last_day
+feeds AS (
+    SELECT
+        gtfs_dataset_key as gtfs_key,
+        MAX(LAST_DAY(date, MONTH)) as max_date
+    FROM {{ ref('fct_daily_schedule_feeds') }}
+    WHERE date = LAST_DAY(date, MONTH)
+    GROUP BY 1
+),
+
 fct_scheduled_trips AS (
     SELECT *
     FROM {{ ref('fct_scheduled_trips') }}
+    INNER JOIN feeds
+        ON feeds.gtfs_key = fct_scheduled_trips.gtfs_dataset_key
+    WHERE fct_scheduled_trips.service_date <= feeds.max_date
 ),
 
 extract_trip_date_types AS (
 
     SELECT
 
         CASE
-            WHEN EXTRACT(hour FROM trip_first_departure_datetime_pacific) < 4 THEN "OWL"
+            WHEN EXTRACT(hour FROM trip_first_departure_datetime_pacific) < 4 THEN "Owl"
             WHEN EXTRACT(hour FROM trip_first_departure_datetime_pacific) < 7 THEN "Early AM"
             WHEN EXTRACT(hour FROM trip_first_departure_datetime_pacific) < 10 THEN "AM Peak"
             WHEN EXTRACT(hour FROM trip_first_departure_datetime_pacific) < 15 THEN "Midday"
@@ -68,7 +81,6 @@ daypart_aggregations AS (
         route_short_name,
         route_long_name,
         time_of_day,
-        hour,
         month,
         year,
         day_type,
@@ -77,26 +89,27 @@ daypart_aggregations AS (
         SUM(service_hours) AS ttl_service_hours
 
     FROM service_with_daypart
-    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
+    GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9
 ),
 
-fct_scheduled_service_by_daypart AS (
+fct_monthly_route_service_by_timeofday AS (
     SELECT
-
+       {{ dbt_utils.generate_surrogate_key(['source_record_id', 'route_id', 'month', 'year', 'time_of_day']) }} AS key,
         name,
         source_record_id,
         route_id,
         route_short_name,
         route_long_name,
         time_of_day,
-        hour,
         month,
         year,
         day_type,
         n_trips,
         ttl_service_hours
 
     FROM daypart_aggregations
+    WHERE ttl_service_hours IS NOT NULL
+
 )
 
-SELECT * FROM fct_scheduled_service_by_daypart
+SELECT * FROM fct_monthly_route_service_by_timeofday