start running rest of funnel on Hub

edasmalchi · edasmalchi · commit 1b1f3a988232 · 2025-10-20T04:02:32.000Z
diff --git a/gtfs_funnel/Makefile b/gtfs_funnel/Makefile
@@ -41,7 +41,7 @@ monthly_scheduled_data:
 	python concatenate_monthly_scheduled_service.py
     
 funnel_gtfs_single_day: 
-	make download_gtfs_data 
+#	make download_gtfs_data 
 	make preprocess_schedule_vp_dependency 
 	make preprocess_vp
 	make preprocess_schedule_only 
@@ -51,9 +51,9 @@ funnel_gtfs_single_day:
 all:
 	make funnel_gtfs_single_day
     # update open_data/update_vars.py        
-	cd ../open_data/ && make create_gtfs_schedule_geospatial_open_data -f Makefile
+#	cd ../open_data/ && make create_gtfs_schedule_geospatial_open_data -f Makefile
     # update high_quality_transit_areas/update_vars.py    
-	cd ../high_quality_transit_areas/ && make hqta_data -f Makefile    
+#	cd ../high_quality_transit_areas/ && make hqta_data -f Makefile    
     # update rt_segment_speeds/segment_speed_utils/project_vars.py
 	cd ../rt_segment_speeds/scripts/ && make all_speeds_pipeline -f Makefile && cd ../../
     # update rt_scheduled_v_ran/scripts/update_vars.py
diff --git a/gtfs_funnel/logs/preprocessing.log b/gtfs_funnel/logs/preprocessing.log
@@ -11,3 +11,5 @@
 2025-10-01 08:48:28.079 | INFO     | __main__:assemble_stop_times_with_direction:274 - scheduled stop times with direction 2025-09-24: 0:07:30.478709
 2025-10-01 09:06:37.933 | INFO     | __main__:assemble_stop_times_with_direction:274 - scheduled stop times with direction 2025-09-24: 0:07:26.845928
 2025-10-10 14:46:55.865 | INFO     | __main__:assemble_stop_times_with_direction:274 - scheduled stop times with direction 2025-08-20: 0:07:46.742330
+2025-10-19 16:42:08.177 | INFO     | __main__:assemble_stop_times_with_direction:274 - scheduled stop times with direction 2025-09-23: 0:07:52.345224
+2025-10-19 19:50:07.202 | INFO     | __main__:assemble_stop_times_with_direction:274 - scheduled stop times with direction 2025-09-25: 0:08:13.800359
diff --git a/gtfs_funnel/logs/vp_preprocessing.log b/gtfs_funnel/logs/vp_preprocessing.log
@@ -180,3 +180,23 @@
 2025-10-10 15:11:01.820 | INFO     | __main__:<module>:222 - unmerged vp with dwell time 2025-08-20: 0:05:07.421247
 2025-10-10 15:13:15.780 | INFO     | __main__:<module>:63 - merge with original and export: 0:01:46.218647
 2025-10-10 15:23:38.753 | INFO     | __main__:<module>:81 - 2025-08-20: condense vp for trip 0:09:55.368682
+2025-10-19 16:46:01.539 | INFO     | __main__:pare_down_to_valid_trips:92 - pare down vp: 0:02:16.171725
+2025-10-19 16:46:09.167 | INFO     | __main__:<module>:116 - 2025-09-23: pare down vp execution time: 0:02:23.800239
+2025-10-19 16:57:54.907 | INFO     | __main__:get_vp_direction_column:62 - export vp direction: 0:11:16.744822
+2025-10-19 16:58:02.652 | INFO     | __main__:merge_in_vp_direction:80 - read vp_direction
+2025-10-19 16:59:36.015 | INFO     | __main__:merge_in_vp_direction:108 - 2025-09-23: export usable vp with direction: 0:01:37.294967
+2025-10-19 16:59:36.394 | INFO     | __main__:<module>:131 - 2025-09-23: pare down vp, add direction execution time: 0:12:58.231986
+2025-10-19 17:04:56.876 | INFO     | __main__:<module>:215 - compute dwell df: 0:04:16.442982
+2025-10-19 17:05:12.991 | INFO     | __main__:<module>:222 - unmerged vp with dwell time 2025-09-23: 0:04:32.557844
+2025-10-19 17:07:20.234 | INFO     | __main__:<module>:63 - merge with original and export: 0:01:38.834154
+2025-10-19 17:17:12.253 | INFO     | __main__:<module>:81 - 2025-09-23: condense vp for trip 0:09:24.836720
+2025-10-19 19:53:40.064 | INFO     | __main__:pare_down_to_valid_trips:92 - pare down vp: 0:02:11.361348
+2025-10-19 19:53:47.653 | INFO     | __main__:<module>:116 - 2025-09-25: pare down vp execution time: 0:02:18.950647
+2025-10-19 20:05:44.783 | INFO     | __main__:get_vp_direction_column:62 - export vp direction: 0:11:28.582611
+2025-10-19 20:05:52.351 | INFO     | __main__:merge_in_vp_direction:80 - read vp_direction
+2025-10-19 20:07:39.925 | INFO     | __main__:merge_in_vp_direction:108 - 2025-09-25: export usable vp with direction: 0:01:51.765946
+2025-10-19 20:07:40.326 | INFO     | __main__:<module>:131 - 2025-09-25: pare down vp, add direction execution time: 0:13:24.125328
+2025-10-19 20:12:51.976 | INFO     | __main__:<module>:215 - compute dwell df: 0:04:08.660077
+2025-10-19 20:13:08.552 | INFO     | __main__:<module>:222 - unmerged vp with dwell time 2025-09-25: 0:04:25.236793
+2025-10-19 20:15:16.353 | INFO     | __main__:<module>:63 - merge with original and export: 0:01:40.013100
+2025-10-19 20:25:51.892 | INFO     | __main__:<module>:81 - 2025-09-25: condense vp for trip 0:10:08.297769
diff --git a/gtfs_funnel/published_operators.yml b/gtfs_funnel/published_operators.yml
@@ -18,18 +18,19 @@
   - Foothill Schedule
 2025-08-20:
   - Bear Schedule
-  - Victor Valley Schedule
   - eTrans Schedule
   - Roseville Transit GMV Schedule
-  - StanRTA Schedule
 2025-09-24:
+  - San Juan Capistrano Trolley Schedule
+  - Culver City Schedule
+2025-10-15:
+  - Yolobus Schedule
   - Santa Ynez Mecatran Schedule
   - SLO Peak Transit Schedule
   - GET Schedule
   - Flixbus Schedule
   - Tracy Schedule
   - San Joaquin Schedule
-  - San Juan Capistrano Trolley Schedule
   - Tahoe Transportation District GMV Schedule
   - Merced GMV Schedule
   - Torrance Schedule
@@ -46,7 +47,6 @@
   - Inglewood Schedule
   - Lompoc Schedule
   - Get Around Town Express Schedule
-  - Culver City Schedule
   - Alhambra Schedule
   - Antelope Valley Transit Authority Schedule
   - Big Blue Bus Schedule
@@ -64,7 +64,6 @@
   - Riverside Schedule
   - WeHo Schedule
   - Go West Schedule
-  - Yolobus Schedule
   - Yuma Schedule
   - San Diego International Airport Shuttles Schedule
   - Bay Area 511 Tri Delta Schedule
@@ -106,6 +105,7 @@
   - Bay Area 511 Vine Transit Schedule
   - Bay Area 511 WestCAT Schedule
   - Bay Area 511 Tri-Valley Wheels Schedule
+  - Victor Valley Schedule
   - SLO Schedule
   - Santa Clarita Schedule
   - Anaheim Resort Schedule
@@ -155,20 +155,22 @@
   - Plumas Schedule
   - Palos Verdes PTA Schedule
   - County Express Schedule
-  - Sage Stage Schedule
   - Siskiyou Schedule
   - Santa Cruz Schedule
   - DowneyLink Avail Schedule
   - Dana Point Trolley Schedule
   - Long Beach Schedule
   - Fresno Schedule
   - LA Metro Bus Schedule
+  - Sage Stage Schedule
   - LA Metro Rail Schedule
   - VCTC GMV Schedule
   - Glendale Schedule
   - Artesia Schedule
   - Maywood Schedule
+  - Yurok Schedule
   - Tuolumne Remix Schedule
+  - Bellflower Remix Schedule
   - eTrans Schedule Remix
   - SunLine Avail Schedule
   - Roseville Schedule
@@ -195,6 +197,7 @@
   - B-Line Schedule
   - Redding Schedule
   - Carson Circuit Schedule
+  - El Monte RTAP Schedule
   - Fresno County Schedule
   - Beach Cities GMV Schedule
   - Basin Transit GMV Schedule
@@ -208,7 +211,7 @@
   - SBMTD Schedule
   - SLORTA Schedule
   - Santa Maria Schedule
-  - Yurok Schedule
+  - StanRTA Schedule
   - Metrolink Schedule
 2024-02-14:
   - Rosemead Schedule
diff --git a/gtfs_funnel/update_vars.py b/gtfs_funnel/update_vars.py
@@ -11,7 +11,7 @@
     apr2024_week + oct2024_week + apr2025_week
 )
 
-analysis_date_list = [rt_dates.DATES['oct2025b']]
+analysis_date_list = [rt_dates.DATES['sep2025b']]
 
 
 GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")
diff --git a/rt_scheduled_v_ran/logs/rt_v_scheduled_operator_metrics.log b/rt_scheduled_v_ran/logs/rt_v_scheduled_operator_metrics.log
@@ -10,3 +10,4 @@
 2025-07-17 14:56:07.777 | INFO     | __main__:operator_metrics:46 - operator aggregation 2025-07-16: 0:00:01.650595
 2025-10-02 12:50:50.002 | INFO     | __main__:operator_metrics:46 - operator aggregation 2025-09-24: 0:00:01.654268
 2025-10-12 22:27:26.134 | INFO     | __main__:operator_metrics:46 - operator aggregation 2025-08-20: 0:00:01.665938
+2025-10-19 19:35:46.966 | INFO     | __main__:operator_metrics:46 - operator aggregation 2025-09-23: 0:00:01.475094
diff --git a/rt_scheduled_v_ran/logs/rt_v_scheduled_route_metrics.log b/rt_scheduled_v_ran/logs/rt_v_scheduled_route_metrics.log
@@ -9,3 +9,4 @@
 2025-07-17 14:55:40.887 | INFO     | __main__:route_metrics:86 - route aggregation 2025-07-16: 0:00:02.672922
 2025-10-02 12:50:09.859 | INFO     | __main__:route_metrics:86 - route aggregation 2025-09-24: 0:00:02.639320
 2025-10-12 22:26:47.004 | INFO     | __main__:route_metrics:86 - route aggregation 2025-08-20: 0:00:02.557894
+2025-10-19 19:35:11.024 | INFO     | __main__:route_metrics:86 - route aggregation 2025-09-23: 0:00:03.019200
diff --git a/rt_scheduled_v_ran/logs/rt_v_scheduled_trip_metrics.log b/rt_scheduled_v_ran/logs/rt_v_scheduled_trip_metrics.log
@@ -47,3 +47,6 @@
 2025-10-12 22:20:29.867 | INFO     | __main__:rt_schedule_trip_metrics:281 - tabular trip metrics 2025-08-20: 0:02:44.651993
 2025-10-12 22:25:00.433 | INFO     | __main__:rt_schedule_trip_metrics:286 - spatial trip metrics 2025-08-20: 0:04:30.566491
 2025-10-12 22:26:17.936 | INFO     | __main__:rt_schedule_trip_metrics:335 - Total run time for metrics on 2025-08-20: 0:08:32.721669
+2025-10-19 19:27:28.707 | INFO     | __main__:rt_schedule_trip_metrics:281 - tabular trip metrics 2025-09-23: 0:02:50.849612
+2025-10-19 19:33:21.379 | INFO     | __main__:rt_schedule_trip_metrics:286 - spatial trip metrics 2025-09-23: 0:05:52.671274
+2025-10-19 19:34:28.624 | INFO     | __main__:rt_schedule_trip_metrics:335 - Total run time for metrics on 2025-09-23: 0:09:50.766167
diff --git a/rt_scheduled_v_ran/scripts/update_vars.py b/rt_scheduled_v_ran/scripts/update_vars.py
@@ -7,7 +7,7 @@
 oct2024_week = rt_dates.get_week("oct2024", exclude_wed=True)
 
 
-analysis_date_list = [rt_dates.DATES["aug2025"]]
+analysis_date_list = [rt_dates.DATES["sep2025b"]]
 
 GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")
 
diff --git a/rt_segment_speeds/logs/avg_speeds.log b/rt_segment_speeds/logs/avg_speeds.log
@@ -45,3 +45,5 @@
 2025-10-07 13:51:43.959 | INFO     | __main__:summary_speeds_by_peak_offpeak:161 - rt_stop_times summary speed averaging by peak/offpeak for 2025-09-24 execution time: 0:00:15.879254
 2025-10-12 21:56:30.885 | INFO     | __main__:trip_summary_speeds_by_time_of_day:92 - rt_stop_times summary speed averaging by time-of-day 2025-08-20 execution time: 0:00:06.903237
 2025-10-12 21:56:45.216 | INFO     | __main__:summary_speeds_by_peak_offpeak:161 - rt_stop_times summary speed averaging by peak/offpeak for 2025-08-20 execution time: 0:00:14.309292
+2025-10-19 19:00:35.874 | INFO     | __main__:trip_summary_speeds_by_time_of_day:92 - rt_stop_times summary speed averaging by time-of-day 2025-09-23 execution time: 0:00:08.887894
+2025-10-19 19:00:51.797 | INFO     | __main__:summary_speeds_by_peak_offpeak:161 - rt_stop_times summary speed averaging by peak/offpeak for 2025-09-23 execution time: 0:00:15.906525
diff --git a/rt_segment_speeds/logs/cut_stop_segments.log b/rt_segment_speeds/logs/cut_stop_segments.log
@@ -21,3 +21,6 @@
 2025-10-02 11:24:35.280 | INFO     | __main__:<module>:246 - speedmap segments and proxy_stop_times 2025-09-24: 0:05:19.891479
 2025-10-12 20:57:02.822 | INFO     | __main__:<module>:178 - cut segments 2025-08-20: 0:24:05.921156
 2025-10-12 21:03:04.112 | INFO     | __main__:<module>:246 - speedmap segments and proxy_stop_times 2025-08-20: 0:05:03.856297
+2025-10-19 17:54:01.754 | INFO     | __main__:<module>:178 - cut segments 2025-09-23: 0:30:33.023322
+2025-10-19 18:00:53.768 | INFO     | __main__:<module>:246 - speedmap segments and proxy_stop_times 2025-09-23: 0:05:50.980020
+2025-10-19 21:02:08.879 | INFO     | __main__:<module>:178 - cut segments 2025-09-25: 0:30:04.329898
diff --git a/rt_segment_speeds/logs/interpolate_stop_arrival.log b/rt_segment_speeds/logs/interpolate_stop_arrival.log
@@ -41,3 +41,5 @@ interpolate arrivals for stop_segments 2024-12-11:  2024-12-11: 0:14:09.409333
 2025-10-02 12:32:19.587 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-09-24:  2025-09-24: 0:02:34.825501
 2025-10-12 21:50:00.697 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for rt_stop_times 2025-08-20:  2025-08-20: 0:10:00.190167
 2025-10-12 22:08:43.593 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-08-20:  2025-08-20: 0:02:43.838577
+2025-10-19 18:53:25.228 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for rt_stop_times 2025-09-23:  2025-09-23: 0:11:05.591224
+2025-10-19 19:14:07.047 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-09-23:  2025-09-23: 0:02:44.252794
diff --git a/rt_segment_speeds/logs/nearest_vp.log b/rt_segment_speeds/logs/nearest_vp.log
@@ -44,3 +44,5 @@
 2025-10-07 13:50:15.213 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - speedmap_segments: time-of-day averages for 2025-09-24 execution time: 0:07:31.813305
 2025-10-12 21:40:00.407 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:142 - nearest neighbor for rt_stop_times 2025-08-20: 0:36:27.464417
 2025-10-12 22:05:59.717 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:142 - nearest neighbor for speedmap_segments 2025-08-20: 0:08:48.612521
+2025-10-19 18:42:19.532 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:142 - nearest neighbor for rt_stop_times 2025-09-23: 0:40:54.155933
+2025-10-19 19:11:22.754 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:142 - nearest neighbor for speedmap_segments 2025-09-23: 0:10:01.486645
diff --git a/rt_segment_speeds/logs/speeds_by_segment_trip.log b/rt_segment_speeds/logs/speeds_by_segment_trip.log
@@ -53,3 +53,5 @@
 2025-10-02 12:34:19.136 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for speedmap_segments 2025-09-24: 0:01:51.479630
 2025-10-12 21:52:03.878 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for rt_stop_times 2025-08-20: 0:02:03.047938
 2025-10-12 22:10:48.517 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for speedmap_segments 2025-08-20: 0:01:57.125288
+2025-10-19 18:55:18.124 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for rt_stop_times 2025-09-23: 0:01:52.835733
+2025-10-19 19:16:30.360 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:154 - speeds by segment for speedmap_segments 2025-09-23: 0:02:14.713820
diff --git a/rt_segment_speeds/logs/speeds_timeofday.log b/rt_segment_speeds/logs/speeds_timeofday.log
@@ -23,3 +23,5 @@
 2025-10-02 12:39:49.268 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:148 - speedmap_segments: time-of-day averages for 2025-09-24 execution time: 0:05:30.075009
 2025-10-12 21:55:54.623 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - rt_stop_times: time-of-day averages for 2025-08-20 execution time: 0:03:50.717900
 2025-10-12 22:16:25.111 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - speedmap_segments: time-of-day averages for 2025-08-20 execution time: 0:05:36.578536
+2025-10-19 18:59:56.889 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - rt_stop_times: time-of-day averages for 2025-09-23 execution time: 0:04:38.719586
+2025-10-19 19:23:27.366 | INFO     | speeds_by_time_of_day:aggregate_by_time_of_day:151 - speedmap_segments: time-of-day averages for 2025-09-23 execution time: 0:06:56.959103
diff --git a/rt_segment_speeds/segment_speed_utils/project_vars.py b/rt_segment_speeds/segment_speed_utils/project_vars.py
@@ -11,7 +11,7 @@
 SHARED_GCS = GTFS_DATA_DICT.gcs_paths.SHARED_GCS
 PUBLIC_GCS = GTFS_DATA_DICT.gcs_paths.PUBLIC_GCS
 
-analysis_date = rt_dates.DATES["aug2025"]
+analysis_date = rt_dates.DATES["sep2025b"]
 analysis_date_list = [analysis_date]
 
 oct2023_week = rt_dates.get_week("oct2023", exclude_wed=True)

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`apr2024_week + oct2024_week + apr2025_week`
`12`	`12`	`)`
`13`	`13`
`14`		`-analysis_date_list = [rt_dates.DATES['oct2025b']]`
	`14`	`+analysis_date_list = [rt_dates.DATES['sep2025b']]`
`15`	`15`
`16`	`16`
`17`	`17`	`GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")`