diff --git a/pyproject.toml b/pyproject.toml index 4c223d88..40f673c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,7 +62,7 @@ requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.black] -line-length = 80 +line-length = 120 target-version = ['py310'] [tool.mypy] @@ -104,7 +104,7 @@ disable = [ "too-many-lines", ] good-names = ["e", "i", "s"] -max-line-length = 80 +max-line-length = 120 min-similarity-lines = 10 # ignore session maker as it gives pylint fits # https://github.com/PyCQA/pylint/issues/7090 diff --git a/src/lamp_py/aws/ecs.py b/src/lamp_py/aws/ecs.py index 7834a8ec..4afd8550 100644 --- a/src/lamp_py/aws/ecs.py +++ b/src/lamp_py/aws/ecs.py @@ -74,9 +74,7 @@ def check_for_parallel_tasks() -> None: # count matches the ecs task group. match_count = 0 if task_arns: - running_tasks = client.describe_tasks( - cluster=ecs_cluster, tasks=task_arns - )["tasks"] + running_tasks = client.describe_tasks(cluster=ecs_cluster, tasks=task_arns)["tasks"] for task in running_tasks: if ecs_task_group == task["group"]: @@ -84,9 +82,7 @@ def check_for_parallel_tasks() -> None: # if the group matches, raise an exception that will terminate the process if match_count > 1: - raise SystemError( - f"Multiple {ecs_task_group} ECS Tasks Running in {ecs_cluster}" - ) + raise SystemError(f"Multiple {ecs_task_group} ECS Tasks Running in {ecs_cluster}") except Exception as exception: process_logger.log_failure(exception) diff --git a/src/lamp_py/aws/kinesis.py b/src/lamp_py/aws/kinesis.py index c2b54511..575070c4 100644 --- a/src/lamp_py/aws/kinesis.py +++ b/src/lamp_py/aws/kinesis.py @@ -27,15 +27,11 @@ def update_shard_id(self) -> None: Get the stream description and the shard id for the first shard in the Kinesis Stream. Throws if the stream has more than one shard. """ - process_logger = ProcessLogger( - process_name="update_shard_id", stream_name=self.stream_name - ) + process_logger = ProcessLogger(process_name="update_shard_id", stream_name=self.stream_name) process_logger.log_start() # Describe the stream and pull out the shard IDs - stream_description = self.kinesis_client.describe_stream( - StreamName=self.stream_name - ) + stream_description = self.kinesis_client.describe_stream(StreamName=self.stream_name) shards = stream_description["StreamDescription"]["Shards"] # Per conversation with Glides, their Kinesis Stream only consists of a @@ -54,9 +50,7 @@ def update_shard_iterator(self) -> None: that case, get the Trim Horizon iterator which is the oldest one in the shard. Otherwise, get the next iterator after the last sequence number. """ - process_logger = ProcessLogger( - process_name="update_shard_iterator", stream_name=self.stream_name - ) + process_logger = ProcessLogger(process_name="update_shard_iterator", stream_name=self.stream_name) process_logger.log_start() if self.shard_id is None: @@ -90,9 +84,7 @@ def get_records(self) -> List[Dict]: records that can be processed and the next shard iterator to use for the next read. """ - process_logger = ProcessLogger( - process_name="kinesis.get_records", stream_name=self.stream_name - ) + process_logger = ProcessLogger(process_name="kinesis.get_records", stream_name=self.stream_name) process_logger.log_start() all_records = [] @@ -106,9 +98,7 @@ def get_records(self) -> List[Dict]: while True: try: - response = self.kinesis_client.get_records( - ShardIterator=self.shard_iterator - ) + response = self.kinesis_client.get_records(ShardIterator=self.shard_iterator) shard_count += 1 self.shard_iterator = response["NextShardIterator"] records = response["Records"] @@ -125,9 +115,7 @@ def get_records(self) -> List[Dict]: except self.kinesis_client.exceptions.ExpiredIteratorException: self.update_shard_iterator() - process_logger.add_metadata( - record_count=len(all_records), shard_count=shard_count - ) + process_logger.add_metadata(record_count=len(all_records), shard_count=shard_count) except Exception as e: process_logger.log_failure(e) diff --git a/src/lamp_py/aws/s3.py b/src/lamp_py/aws/s3.py index 5368ef26..7b16b564 100644 --- a/src/lamp_py/aws/s3.py +++ b/src/lamp_py/aws/s3.py @@ -36,9 +36,7 @@ def get_s3_client() -> boto3.client: return boto3.client("s3") -def upload_file( - file_name: str, object_path: str, extra_args: Optional[Dict] = None -) -> bool: +def upload_file(file_name: str, object_path: str, extra_args: Optional[Dict] = None) -> bool: """ Upload a local file to an S3 Bucket @@ -66,9 +64,7 @@ def upload_file( s3_client = get_s3_client() - s3_client.upload_file( - file_name, bucket, object_name, ExtraArgs=extra_args - ) + s3_client.upload_file(file_name, bucket, object_name, ExtraArgs=extra_args) upload_log.log_complete() @@ -270,9 +266,7 @@ def file_list_from_s3( object path as s3://bucket-name/object-key ] """ - process_logger = ProcessLogger( - "file_list_from_s3", bucket_name=bucket_name, file_prefix=file_prefix - ) + process_logger = ProcessLogger("file_list_from_s3", bucket_name=bucket_name, file_prefix=file_prefix) process_logger.log_start() try: @@ -288,9 +282,7 @@ def file_list_from_s3( if obj["Size"] == 0: continue if in_filter is None or in_filter in obj["Key"]: - filepaths.append( - os.path.join("s3://", bucket_name, obj["Key"]) - ) + filepaths.append(os.path.join("s3://", bucket_name, obj["Key"])) if len(filepaths) > max_list_size: break @@ -303,9 +295,7 @@ def file_list_from_s3( return [] -def file_list_from_s3_with_details( - bucket_name: str, file_prefix: str -) -> List[Dict]: +def file_list_from_s3_with_details(bucket_name: str, file_prefix: str) -> List[Dict]: """ get a list of s3 objects with additional details @@ -341,9 +331,7 @@ def file_list_from_s3_with_details( continue filepaths.append( { - "s3_obj_path": os.path.join( - "s3://", bucket_name, obj["Key"] - ), + "s3_obj_path": os.path.join("s3://", bucket_name, obj["Key"]), "size_bytes": obj["Size"], "last_modified": obj["LastModified"], } @@ -357,16 +345,12 @@ def file_list_from_s3_with_details( return [] -def get_last_modified_object( - bucket_name: str, file_prefix: str, version: Optional[str] = None -) -> Optional[Dict]: +def get_last_modified_object(bucket_name: str, file_prefix: str, version: Optional[str] = None) -> Optional[Dict]: """ For a given bucket, find the last modified object that matches a prefix. If a version is passed, only return the newest object matching this version. """ - files = file_list_from_s3_with_details( - bucket_name=bucket_name, file_prefix=file_prefix - ) + files = file_list_from_s3_with_details(bucket_name=bucket_name, file_prefix=file_prefix) # sort the objects by last modified files.sort(key=lambda o: o["last_modified"], reverse=True) @@ -453,9 +437,7 @@ def _init_process_session() -> None: """ process_data = current_thread() process_data.__dict__["boto_session"] = boto3.session.Session() - process_data.__dict__["boto_s3_resource"] = process_data.__dict__[ - "boto_session" - ].resource("s3") + process_data.__dict__["boto_s3_resource"] = process_data.__dict__["boto_session"].resource("s3") # pylint: disable=R0914 @@ -495,13 +477,9 @@ def move_s3_objects(files: List[str], to_bucket: str) -> List[str]: process_logger.add_metadata(pool_size=pool_size) results = [] try: - with ThreadPoolExecutor( - max_workers=pool_size, initializer=_init_process_session - ) as pool: + with ThreadPoolExecutor(max_workers=pool_size, initializer=_init_process_session) as pool: for filename in files_to_move: - results.append( - pool.submit(_move_s3_object, filename, to_bucket) - ) + results.append(pool.submit(_move_s3_object, filename, to_bucket)) for result in results: current_result = result.result() if isinstance(current_result, str): @@ -517,9 +495,7 @@ def move_s3_objects(files: List[str], to_bucket: str) -> List[str]: # wait for gremlins to disappear time.sleep(15) - process_logger.add_metadata( - failed_count=len(files_to_move), retry_attempts=retry_attempt - ) + process_logger.add_metadata(failed_count=len(files_to_move), retry_attempts=retry_attempt) if len(files_to_move) == 0: process_logger.log_complete() @@ -575,9 +551,7 @@ def write_parquet_file( @filename - if set, the filename that will be written to. if left empty, the basename template (or _its_ fallback) will be used. """ - process_logger = ProcessLogger( - "write_parquet", file_type=file_type, number_of_rows=table.num_rows - ) + process_logger = ProcessLogger("write_parquet", file_type=file_type, number_of_rows=table.num_rows) process_logger.log_start() # pull out the partition information into a list of strings. @@ -588,9 +562,7 @@ def write_parquet_file( for col in partition_cols: unique_list = pc.unique(table.column(col)).to_pylist() - assert ( - len(unique_list) == 1 - ), f"Table {s3_dir} column {col} had {len(unique_list)} unique elements" + assert len(unique_list) == 1, f"Table {s3_dir} column {col} had {len(unique_list)} unique elements" partition_strings.append(f"{col}={unique_list[0]}") @@ -609,9 +581,7 @@ def write_parquet_file( process_logger.add_metadata(write_path=write_path) # write teh parquet file to the partitioned path - with pq.ParquetWriter( - where=write_path, schema=table.schema, filesystem=fs.S3FileSystem() - ) as pq_writer: + with pq.ParquetWriter(where=write_path, schema=table.schema, filesystem=fs.S3FileSystem()) as pq_writer: pq_writer.write(table) # call the visitor function if it exists @@ -624,9 +594,16 @@ def write_parquet_file( # pylint: enable=R0913 -def get_datetime_from_partition_path(path: str) -> datetime: +def dt_from_obj_path(path: str) -> datetime: """ process and return datetime from partitioned s3 path + + handles the following formats: + - year=YYYY/month=MM/day=DD/hour=HH + - year=YYYY/month=MM/day=DD + - timestamp=DDDDDDDDDD + + :return datetime(tz=UTC): """ try: # handle gtfs-rt paths @@ -696,9 +673,7 @@ def read_parquet( read_columns = list(set(ds.schema.names) & set(columns)) table = ds.to_table(columns=read_columns) for null_column in set(columns).difference(ds.schema.names): - table = table.append_column( - null_column, pa.nulls(table.num_rows) - ) + table = table.append_column(null_column, pa.nulls(table.num_rows)) df = table.to_pandas(self_destruct=True) break diff --git a/src/lamp_py/bus_performance_manager/event_files.py b/src/lamp_py/bus_performance_manager/event_files.py index d0d3a70d..94561afd 100644 --- a/src/lamp_py/bus_performance_manager/event_files.py +++ b/src/lamp_py/bus_performance_manager/event_files.py @@ -1,57 +1,47 @@ -from datetime import timedelta, date -from typing import Optional, Dict, List import re +from typing import Optional +from typing import Dict +from typing import List +from datetime import timedelta +from datetime import date import polars as pl -from lamp_py.aws.s3 import ( - file_list_from_s3_with_details, - get_datetime_from_partition_path, - get_last_modified_object, -) - -from lamp_py.runtime_utils.remote_files import ( - rt_vehicle_positions, - tm_stop_crossing, - bus_events, -) - - -def get_new_event_files() -> List[Dict[str, date | List[str]]]: +from lamp_py.aws.s3 import file_list_from_s3_with_details +from lamp_py.aws.s3 import dt_from_obj_path +from lamp_py.aws.s3 import get_last_modified_object +from lamp_py.runtime_utils.remote_files import rt_vehicle_positions +from lamp_py.runtime_utils.remote_files import tm_stop_crossing +from lamp_py.runtime_utils.remote_files import bus_events + + +def service_date_from_filename(tm_filename: str) -> Optional[date]: + """pull the service date from a filename formatted '...YYYYMMDD.parquet'""" + try: + service_date_int = re.findall(r"(\d{8}).parquet", tm_filename)[0] + year = int(service_date_int[:4]) + month = int(service_date_int[4:6]) + day = int(service_date_int[6:]) + + return date(year=year, month=month, day=day) + except IndexError: + # the tm files may have a lamp version file that will throw when + # pulling out a match from the regular expression. ask for + # forgiveness and assert that it was this file that caused the + # error. + assert "lamp_version" in tm_filename + return None + + +def vehicle_position_files_as_frame() -> pl.DataFrame: """ - Generate a list of dictionaries that contains a record for every service date to be - processed. - * Collect all of the potential input filepaths, their last modified - timestamp, and potential service dates. - * Get the last modified timestamp for the output filepaths. - * Generate a list of all service dates where the input files have been - modified since the last output file write. - * For each service date, generate a list of input files associated with - that service date. - - @return pl.DataFrame - - 'service_date': datetime.date - 'gtfs_rt': list[str] - s3 filepaths for vehicle position files - 'transit_master': list[str] - s3 filepath for tm files + :return dataframe: + s3_obj_path -> String + size_bytes -> Int64 + last_modified -> Datetime + service_date -> Date + source -> String """ - - def get_service_date_from_filename(tm_filename: str) -> Optional[date]: - """pull the service date from a filename formatted '1YYYYMMDD.parquet'""" - try: - service_date_int = re.findall(r"(\d{8}).parquet", tm_filename)[0] - year = int(service_date_int[:4]) - month = int(service_date_int[4:6]) - day = int(service_date_int[6:]) - - return date(year=year, month=month, day=day) - except IndexError: - # the tm files may have a lamp version file that will throw when - # pulling out a match from the regular expression. ask for - # forgiveness and assert that it was this file that caused the - # error. - assert "lamp_version" in tm_filename - return None - # pull all of the vehicle position files from s3 along with their last # modified datetime. convert to a dataframe and generate a service date # from the partition paths. add a source column for later merging. @@ -60,9 +50,9 @@ def get_service_date_from_filename(tm_filename: str) -> Optional[date]: file_prefix=rt_vehicle_positions.prefix, ) vp_df = pl.DataFrame(vp_objects).with_columns( - pl.col("s3_obj_path") - .map_elements(lambda x: get_datetime_from_partition_path(x).date()) - .alias("service_date"), + (pl.col("s3_obj_path").map_elements(lambda x: dt_from_obj_path(x).date(), return_dtype=pl.Date)).alias( + "service_date" + ), pl.lit("gtfs_rt").alias("source"), ) @@ -85,30 +75,38 @@ def get_service_date_from_filename(tm_filename: str) -> Optional[date]: # contain data from the previous service date vp_shifted = vp_df.filter( - pl.col("s3_obj_path").str.contains("hour=0") - | pl.col("s3_obj_path").str.contains("hour=1") - | pl.col("s3_obj_path").str.contains("hour=2") - | pl.col("s3_obj_path").str.contains("hour=3") - | pl.col("s3_obj_path").str.contains("hour=4") - | pl.col("s3_obj_path").str.contains("hour=5") - | pl.col("s3_obj_path").str.contains("hour=6") - | pl.col("s3_obj_path").str.contains("hour=7") - | pl.col("s3_obj_path").str.contains("hour=8") + pl.col("s3_obj_path").str.contains("hour=0/") + | pl.col("s3_obj_path").str.contains("hour=1/") + | pl.col("s3_obj_path").str.contains("hour=2/") + | pl.col("s3_obj_path").str.contains("hour=3/") + | pl.col("s3_obj_path").str.contains("hour=4/") + | pl.col("s3_obj_path").str.contains("hour=5/") + | pl.col("s3_obj_path").str.contains("hour=6/") + | pl.col("s3_obj_path").str.contains("hour=7/") + | pl.col("s3_obj_path").str.contains("hour=8/") | ~pl.col("s3_obj_path").str.contains("hour") - ).with_columns( - (pl.col("service_date") - timedelta(days=1)).alias("service_date") - ) + ).with_columns((pl.col("service_date") - timedelta(days=1)).alias("service_date")) # these files contain data from the current service day vp_unshifted = vp_df.filter( - ~pl.col("s3_obj_path").str.contains("hour=0") - & ~pl.col("s3_obj_path").str.contains("hour=1") - & ~pl.col("s3_obj_path").str.contains("hour=2") + ~pl.col("s3_obj_path").str.contains("hour=0/") + & ~pl.col("s3_obj_path").str.contains("hour=1/") + & ~pl.col("s3_obj_path").str.contains("hour=2/") ) # merge the shifted and unshifted dataframes - vp_df = pl.concat([vp_unshifted, vp_shifted]) + return pl.concat([vp_unshifted, vp_shifted]) + +def transit_master_files_as_frame() -> pl.DataFrame: + """ + :return dataframe: + s3_obj_path -> String + size_bytes -> Int64 + last_modified -> Datetime + service_date -> Date + source -> String + """ # pull all of the transit master files from s3 along with their last # modified datetime. convert to a dataframe and generate a service date # from the filename. add a source column for later merging. @@ -116,13 +114,39 @@ def get_service_date_from_filename(tm_filename: str) -> Optional[date]: bucket_name=tm_stop_crossing.bucket, file_prefix=tm_stop_crossing.prefix, ) - tm_df = pl.DataFrame(tm_objects).with_columns( - pl.col("s3_obj_path") - .map_elements(get_service_date_from_filename) - .alias("service_date"), - pl.lit("transit_master").alias("source"), + return ( + pl.DataFrame(tm_objects) + .with_columns( + (pl.col("s3_obj_path").map_elements(service_date_from_filename, return_dtype=pl.Date)).alias( + "service_date" + ), + pl.lit("transit_master").alias("source"), + ) + .filter(pl.col("service_date").is_not_null()) ) + +def event_files_to_load() -> Dict[date, Dict[str, List[str]]]: + """ + Generate a dictionary containing a record for every service date to be processed. + * Collect all of the potential input filepaths, their last modified + timestamp, and potential service dates. + * Get the last modified timestamp for the output filepaths. + * Generate a list of all service dates where the input files have been + modified since the last output file write. + * For each service date, generate a list of input files associated with + that service date. + + :return { + datetime.date (service date): { + 'gtfs_rt': list[str] - s3 filepaths for vehicle position files, + 'transit_master': list[str] - s3 filepath for tm files, + } + } + """ + vp_df = vehicle_position_files_as_frame() + tm_df = transit_master_files_as_frame() + # a merged dataframe of all files to operate on all_files = pl.concat([vp_df, tm_df]) @@ -130,29 +154,26 @@ def get_service_date_from_filename(tm_filename: str) -> Optional[date]: latest_event_file = get_last_modified_object( bucket_name=bus_events.bucket, file_prefix=bus_events.prefix, - version="1.0", + version=bus_events.version, ) # if there is a event file, pull the service date from it and filter # all_files to only contain objects with service dates on or after this # date. if latest_event_file: - latest_service_date = get_service_date_from_filename( - latest_event_file["s3_obj_path"] - ) - all_files = all_files.filter( - pl.col("service_date") >= latest_service_date - ) - - # filter all files to only files associated with new service dates - # aggregate records by source and pivot on service date. - # - # each record of the new dataframe will have a list of gtfs_rt and tm input - # files. - grouped_files = ( - all_files.group_by(["service_date", "source"]) - .agg([pl.col("s3_obj_path").alias("file_list")]) - .pivot(values="file_list", index="service_date", on="source") - ) - - return grouped_files.to_dicts() + latest_service_date = service_date_from_filename(latest_event_file["s3_obj_path"]) + all_files = all_files.filter(pl.col("service_date") >= latest_service_date) + + all_files = all_files.group_by(["service_date", "source"]).agg([pl.col("s3_obj_path")]) + # all_files as dataframe: + # service_date -> Date + # source -> String{"gtfs_rt" or "transit_master"} + # s3_obj_path -> List[String] + + return_dict: Dict[date, Dict[str, List[str]]] = { + date: {"gtfs_rt": [], "transit_master": []} for date in all_files.get_column("service_date").unique() + } + for row in all_files.iter_rows(named=True): + return_dict[row["service_date"]].update({row["source"]: row["s3_obj_path"]}) + + return return_dict diff --git a/src/lamp_py/bus_performance_manager/events_gtfs_rt.py b/src/lamp_py/bus_performance_manager/events_gtfs_rt.py index 06e8ae52..f31d7510 100644 --- a/src/lamp_py/bus_performance_manager/events_gtfs_rt.py +++ b/src/lamp_py/bus_performance_manager/events_gtfs_rt.py @@ -2,54 +2,26 @@ from typing import List import polars as pl +from pyarrow.fs import S3FileSystem +import pyarrow.compute as pc -from lamp_py.bus_performance_manager.gtfs_utils import ( - bus_routes_for_service_date, -) +from lamp_py.bus_performance_manager.gtfs_utils import bus_routes_for_service_date +from lamp_py.performance_manager.gtfs_utils import start_time_to_seconds +from lamp_py.runtime_utils.process_logger import ProcessLogger -def read_vehicle_positions( - service_date: date, gtfs_rt_files: List[str] -) -> pl.DataFrame: +def _read_with_polars(service_date: date, gtfs_rt_files: List[str], bus_routes: List[str]) -> pl.DataFrame: """ - Read gtfs realtime vehicle position files and pull out unique bus vehicle - positions for a given service day. - - :param service_date: the service date to filter on - :param gtfs_rt_files: a list of gtfs realtime files, either s3 urls or a - local path + Read RT_VEHICLE_POSITIONS parquet files with polars engine - :return dataframe: - route_id -> String - trip_id -> String - stop_id -> String - stop_sequence -> String - direction_id -> Int8 - start_time -> String - service_date -> String - vehicle_id -> String - vehicle_label -> String - current_status -> String - vehicle_timestamp -> Datetime + Polars engine appears to be faster and use less memory than pyarrow enginer, but is not as + compatible with all parquet file formats as pyarrow engine """ - - bus_routes = bus_routes_for_service_date(service_date) - - # build a dataframe of every gtfs record - # * scan all of the gtfs realtime files - # * only pull out bus records and records for the service date with current status fields - # * rename / convert columns as appropriate - # * convert - # * sort by vehicle id and timestamp - # * keep only the first record for a given trip / stop / status vehicle_positions = ( pl.scan_parquet(gtfs_rt_files) .filter( (pl.col("vehicle.trip.route_id").is_in(bus_routes)) - & ( - pl.col("vehicle.trip.start_date") - == service_date.strftime("%Y%m%d") - ) + & (pl.col("vehicle.trip.start_date") == service_date.strftime("%Y%m%d")) & pl.col("vehicle.current_status").is_not_null() & pl.col("vehicle.stop_id").is_not_null() & pl.col("vehicle.trip.trip_id").is_not_null() @@ -61,25 +33,80 @@ def read_vehicle_positions( pl.col("vehicle.trip.route_id").cast(pl.String).alias("route_id"), pl.col("vehicle.trip.trip_id").cast(pl.String).alias("trip_id"), pl.col("vehicle.stop_id").cast(pl.String).alias("stop_id"), - pl.col("vehicle.current_stop_sequence") - .cast(pl.Int64) - .alias("stop_sequence"), - pl.col("vehicle.trip.direction_id") - .cast(pl.Int8) - .alias("direction_id"), - pl.col("vehicle.trip.start_time") - .cast(pl.String) - .alias("start_time"), - pl.col("vehicle.trip.start_date") - .cast(pl.String) - .alias("service_date"), + pl.col("vehicle.current_stop_sequence").cast(pl.Int64).alias("stop_sequence"), + pl.col("vehicle.trip.direction_id").cast(pl.Int8).alias("direction_id"), + pl.col("vehicle.trip.start_time").cast(pl.String).alias("start_time"), + pl.col("vehicle.trip.start_date").cast(pl.String).alias("service_date"), pl.col("vehicle.vehicle.id").cast(pl.String).alias("vehicle_id"), - pl.col("vehicle.vehicle.label") - .cast(pl.String) - .alias("vehicle_label"), - pl.col("vehicle.current_status") + pl.col("vehicle.vehicle.label").cast(pl.String).alias("vehicle_label"), + pl.col("vehicle.current_status").cast(pl.String).alias("current_status"), + pl.from_epoch("vehicle.timestamp").alias("vehicle_timestamp"), + ) + .with_columns( + pl.when(pl.col("current_status") == "INCOMING_AT") + .then(pl.lit("IN_TRANSIT_TO")) + .otherwise(pl.col("current_status")) .cast(pl.String) .alias("current_status"), + ) + .collect() + ) + + return vehicle_positions + + +def _read_with_pyarrow(service_date: date, gtfs_rt_files: List[str], bus_routes: List[str]) -> pl.DataFrame: + """ + Read RT_VEHICLE_POSITIONS parquet files with pyarrow engine, instead of polars engine + + the polars implmentation of parquet reader sometimes has issues with files in staging bucket + pyarrow engine is more forgiving in reading some parquet file formats at the cost of read speed + and memory usage, compared to polars native parquet reader/scanner + """ + gtfs_rt_files = [uri.replace("s3://", "") for uri in gtfs_rt_files] + columns = [ + "vehicle.trip.route_id", + "vehicle.trip.trip_id", + "vehicle.trip.direction_id", + "vehicle.trip.start_time", + "vehicle.trip.start_date", + "vehicle.vehicle.id", + "vehicle.vehicle.label", + "vehicle.stop_id", + "vehicle.current_stop_sequence", + "vehicle.current_status", + "vehicle.timestamp", + ] + # pyarrow_exp filter expression is used to limit memory usage during read operation + pyarrow_exp = pc.field("vehicle.trip.route_id").isin(bus_routes) + vehicle_positions = ( + pl.read_parquet( + gtfs_rt_files, + columns=columns, + use_pyarrow=True, + pyarrow_options={"filesystem": S3FileSystem(), "filters": pyarrow_exp}, + ) + .filter( + (pl.col("vehicle.trip.route_id").is_in(bus_routes)) + & (pl.col("vehicle.trip.start_date") == service_date.strftime("%Y%m%d")) + & pl.col("vehicle.current_status").is_not_null() + & pl.col("vehicle.stop_id").is_not_null() + & pl.col("vehicle.trip.trip_id").is_not_null() + & pl.col("vehicle.vehicle.id").is_not_null() + & pl.col("vehicle.timestamp").is_not_null() + & pl.col("vehicle.trip.start_time").is_not_null() + ) + .select( + pl.col("vehicle.trip.route_id").cast(pl.String).alias("route_id"), + pl.col("vehicle.trip.trip_id").cast(pl.String).alias("trip_id"), + pl.col("vehicle.stop_id").cast(pl.String).alias("stop_id"), + pl.col("vehicle.current_stop_sequence").cast(pl.Int64).alias("stop_sequence"), + pl.col("vehicle.trip.direction_id").cast(pl.Int8).alias("direction_id"), + pl.col("vehicle.trip.start_time").cast(pl.String).alias("start_time"), + pl.col("vehicle.trip.start_date").cast(pl.String).alias("service_date"), + pl.col("vehicle.vehicle.id").cast(pl.String).alias("vehicle_id"), + pl.col("vehicle.vehicle.label").cast(pl.String).alias("vehicle_label"), + pl.col("vehicle.current_status").cast(pl.String).alias("current_status"), pl.from_epoch("vehicle.timestamp").alias("vehicle_timestamp"), ) .with_columns( @@ -89,16 +116,55 @@ def read_vehicle_positions( .cast(pl.String) .alias("current_status"), ) - .sort(["vehicle_id", "vehicle_timestamp"]) - .collect() ) return vehicle_positions +def read_vehicle_positions(service_date: date, gtfs_rt_files: List[str]) -> pl.DataFrame: + """ + Read gtfs realtime vehicle position files and pull out unique bus vehicle + positions for a given service day. + + :param service_date: the service date to filter on + :param gtfs_rt_files: a list of gtfs realtime files, either s3 urls or a + local path + + :return dataframe: + route_id -> String + trip_id -> String + stop_id -> String + stop_sequence -> String + direction_id -> Int8 + start_time -> String + service_date -> String + vehicle_id -> String + vehicle_label -> String + current_status -> String + vehicle_timestamp -> Datetime + """ + logger = ProcessLogger( + "read_vehicle_positions", + service_date=service_date, + file_count=len(gtfs_rt_files), + reader_engine="polars", + ) + logger.log_start() + bus_routes = bus_routes_for_service_date(service_date) + + try: + vehicle_positions = _read_with_polars(service_date, gtfs_rt_files, bus_routes) + except Exception as _: + logger.add_metadata(reader_engine="pyarrow") + vehicle_positions = _read_with_pyarrow(service_date, gtfs_rt_files, bus_routes) + + logger.log_complete() + return vehicle_positions + + def positions_to_events(vehicle_positions: pl.DataFrame) -> pl.DataFrame: """ - using the vehicle positions dataframe, create a dataframe for each event by + using the vehicle positions dataframe, create a row for each event by pivoting and mapping the current status onto arrivals and departures. :param vehicle_positions: Dataframe of vehiclie positions @@ -108,6 +174,8 @@ def positions_to_events(vehicle_positions: pl.DataFrame) -> pl.DataFrame: route_id -> String trip_id -> String start_time -> String + start_dt -> Datetime + stop_count -> UInt32 direction_id -> Int8 stop_id -> String stop_sequence -> Int64 @@ -135,37 +203,51 @@ def positions_to_events(vehicle_positions: pl.DataFrame) -> pl.DataFrame: for column in ["STOPPED_AT", "IN_TRANSIT_TO"]: if column not in vehicle_events.columns: - vehicle_events = vehicle_events.with_columns( - pl.lit(None).cast(pl.Datetime).alias(column) - ) + vehicle_events = vehicle_events.with_columns(pl.lit(None).cast(pl.Datetime).alias(column)) - vehicle_events = vehicle_events.rename( - { - "STOPPED_AT": "gtfs_arrival_dt", - "IN_TRANSIT_TO": "gtfs_travel_to_dt", - } - ).select( - [ - "service_date", - "route_id", - "trip_id", - "start_time", - "direction_id", - "stop_id", - "stop_sequence", - "vehicle_id", - "vehicle_label", - "gtfs_travel_to_dt", - "gtfs_arrival_dt", - ] + stop_count = vehicle_events.group_by("trip_id").len("stop_count") + + vehicle_events = ( + vehicle_events.join( + stop_count, + on="trip_id", + how="left", + ) + .rename( + { + "STOPPED_AT": "gtfs_arrival_dt", + "IN_TRANSIT_TO": "gtfs_travel_to_dt", + } + ) + .with_columns(pl.col("start_time").map_elements(start_time_to_seconds, return_dtype=pl.Int64)) + .with_columns( + (pl.col("service_date").str.to_datetime("%Y%m%d") + pl.duration(seconds=pl.col("start_time"))).alias( + "start_dt" + ) + ) + .select( + [ + "service_date", + "route_id", + "trip_id", + "start_time", + "start_dt", + "stop_count", + "direction_id", + "stop_id", + "stop_sequence", + "vehicle_id", + "vehicle_label", + "gtfs_travel_to_dt", + "gtfs_arrival_dt", + ] + ) ) return vehicle_events -def generate_gtfs_rt_events( - service_date: date, gtfs_rt_files: List[str] -) -> pl.DataFrame: +def generate_gtfs_rt_events(service_date: date, gtfs_rt_files: List[str]) -> pl.DataFrame: """ generate a polars dataframe for bus vehicle events from gtfs realtime vehicle position files for a given service date @@ -175,23 +257,32 @@ def generate_gtfs_rt_events( local path :return dataframe: + service_date -> String route_id -> String trip_id -> String - stop_id -> String - stop_sequence -> String + start_time -> Int64 + start_dt -> Datetime + stop_count -> UInt32 direction_id -> Int8 - start_time -> String - service_date -> String + stop_id -> String + stop_sequence -> Int64 vehicle_id -> String vehicle_label -> String - current_status -> String - arrival_gtfs -> Datetime - travel_towards_gtfs -> Datetime + gtfs_travel_to_dt -> Datetime + gtfs_arrival_dt -> Datetime """ - vehicle_positions = read_vehicle_positions( - service_date=service_date, gtfs_rt_files=gtfs_rt_files - ) + logger = ProcessLogger("generate_gtfs_rt_events", service_date=service_date) + logger.log_start() + + # if RT_VEHICLE_POSITIONS exists for whole day, filter out hour files for that day + for year_file in [f for f in gtfs_rt_files if f.endswith("T00:00:00.parquet")]: + prefix, _ = year_file.rsplit("/", 1) + gtfs_rt_files = [f for f in gtfs_rt_files if f == year_file or not f.startswith(prefix)] + vehicle_positions = read_vehicle_positions(service_date=service_date, gtfs_rt_files=gtfs_rt_files) + logger.add_metadata(rows_from_parquet=vehicle_positions.shape[0]) vehicle_events = positions_to_events(vehicle_positions=vehicle_positions) + logger.add_metadata(events_for_day=vehicle_events.shape[0]) + logger.log_complete() return vehicle_events diff --git a/src/lamp_py/bus_performance_manager/events_gtfs_schedule.py b/src/lamp_py/bus_performance_manager/events_gtfs_schedule.py index a7c2579b..e04f6b57 100644 --- a/src/lamp_py/bus_performance_manager/events_gtfs_schedule.py +++ b/src/lamp_py/bus_performance_manager/events_gtfs_schedule.py @@ -1,100 +1,12 @@ -import os -import shutil from datetime import date import polars as pl -from lamp_py.runtime_utils.remote_files import compressed_gtfs -from lamp_py.aws.s3 import file_list_from_s3, download_file +from lamp_py.bus_performance_manager.gtfs_utils import gtfs_from_parquet from lamp_py.performance_manager.gtfs_utils import start_time_to_seconds from lamp_py.runtime_utils.process_logger import ProcessLogger -def sync_gtfs_files(service_date: date) -> None: - """ - sync local tmp folder with parquet schedule data for service_date from S3 - - resulting files will be located at /tmp/gtfs_archive/YYYYMMDD/... - - the /tmp/gtfs_archive/ folder will only contain one service date at a time - - :param service_date: service date of requested GTFS data - """ - gtfs_year = service_date.year - service_date_str = service_date.strftime("%Y%m%d") - gtfs_archive_folder = os.path.join("/tmp", "gtfs_archive") - gtfs_date_folder = os.path.join(gtfs_archive_folder, service_date_str) - - # local files already exist - if ( - os.path.exists(gtfs_date_folder) - and len(os.listdir(gtfs_date_folder)) > 0 - ): - return - - # clean gtfs_archive folder - shutil.rmtree(gtfs_archive_folder, ignore_errors=True) - os.makedirs(gtfs_date_folder, exist_ok=True) - - s3_objects = file_list_from_s3( - bucket_name=compressed_gtfs.bucket, - file_prefix=os.path.join(compressed_gtfs.prefix, str(gtfs_year)), - ) - - # check previous calendar year for s3_files, if none for current year - # for when year just turned over, but new year files are not yet available - # this may be fixed in the future with the compressed ingestion process - if len(s3_objects) == 0: - gtfs_year -= 1 - - s3_objects = file_list_from_s3( - bucket_name=compressed_gtfs.bucket, - file_prefix=os.path.join(compressed_gtfs.prefix, str(gtfs_year)), - ) - - if len(s3_objects) == 0: - raise FileNotFoundError( - f"No Compressed GTFS archive files available for {gtfs_year}" - ) - - for s3_object in s3_objects: - if s3_object.endswith(".parquet"): - parquet_file = s3_object.split("/")[-1] - download_file( - s3_object, os.path.join(gtfs_date_folder, parquet_file) - ) - - -def gtfs_from_parquet(file: str, service_date: date) -> pl.DataFrame: - """ - Get GTFS data from specified file and service date - - This will read from local gtfs_archive location "tmp/gtfs_archive/YYYYMMDD/..." - - :param file: gtfs file to acces (i.e. "feed_info") - :param service_date: service date of requested GTFS data - """ - gtfs_archive_folder = os.path.join("/tmp", "gtfs_archive") - service_date_str = service_date.strftime("%Y%m%d") - service_date_int = int(service_date_str) - - if not file.endswith(".parquet"): - file = f"{file}.parquet" - - gtfs_file = os.path.join(gtfs_archive_folder, service_date_str, file) - - gtfs_df = ( - pl.read_parquet(gtfs_file) - .filter( - (pl.col("gtfs_active_date") <= service_date_int) - & (pl.col("gtfs_end_date") >= service_date_int) - ) - .drop(["gtfs_active_date", "gtfs_end_date"]) - ) - - return gtfs_df - - def service_ids_for_date(service_date: date) -> pl.DataFrame: """ Retrieve service_id values applicable to service_date @@ -110,19 +22,17 @@ def service_ids_for_date(service_date: date) -> pl.DataFrame: calendar = gtfs_from_parquet("calendar", service_date) service_ids = calendar.filter( - pl.col(day_of_week) - == True - & (pl.col("start_date") <= service_date_int) - & (pl.col("end_date") >= service_date_int) + pl.col(day_of_week) == True & (pl.col("start_date") <= service_date_int), + (pl.col("end_date") >= service_date_int), ).select("service_id") calendar_dates = gtfs_from_parquet("calendar_dates", service_date) - exclude_ids = calendar_dates.filter( - (pl.col("date") == service_date_int) & (pl.col("exception_type") == 2) - ).select("service_id") - include_ids = calendar_dates.filter( - (pl.col("date") == service_date_int) & (pl.col("exception_type") == 1) - ).select("service_id") + exclude_ids = calendar_dates.filter((pl.col("date") == service_date_int) & (pl.col("exception_type") == 2)).select( + "service_id" + ) + include_ids = calendar_dates.filter((pl.col("date") == service_date_int) & (pl.col("exception_type") == 1)).select( + "service_id" + ) service_ids = service_ids.join( exclude_ids, @@ -146,7 +56,7 @@ def trips_for_date(service_date: date) -> pl.DataFrame: service_id -> String route_pattern_id -> String route_pattern_typicality -> Int64 - direction_id -> Int64 + direction_id -> Int8 direction -> String direction_destination -> String """ @@ -192,7 +102,7 @@ def trips_for_date(service_date: date) -> pl.DataFrame: "service_id", "route_pattern_id", "route_pattern_typicality", - "direction_id", + pl.col("direction_id").cast(pl.Int8), "direction", "direction_destination", ) @@ -214,10 +124,7 @@ def canonical_stop_sequence(service_date: date) -> pl.DataFrame: canonical_trip_ids = ( gtfs_from_parquet("route_patterns", service_date) - .filter( - (pl.col("route_pattern_typicality") == 1) - | (pl.col("route_pattern_typicality") == 5) - ) + .filter((pl.col("route_pattern_typicality") == 1) | (pl.col("route_pattern_typicality") == 5)) .sort(pl.col("route_pattern_typicality"), descending=True) .unique(["route_id", "direction_id"], keep="first") .select( @@ -239,10 +146,7 @@ def canonical_stop_sequence(service_date: date) -> pl.DataFrame: "route_id", "direction_id", "stop_id", - pl.col("stop_sequence") - .rank("ordinal") - .over("route_id", "direction_id") - .alias("canon_stop_sequence"), + pl.col("stop_sequence").rank("ordinal").over("route_id", "direction_id").alias("canon_stop_sequence"), ) ) @@ -257,8 +161,6 @@ def stop_events_for_date(service_date: date) -> pl.DataFrame: trip_id -> String stop_id -> String stop_sequence -> Int64 - timepoint -> Int64 - checkpoint_id -> String block_id -> String route_id -> String service_id -> String @@ -269,10 +171,12 @@ def stop_events_for_date(service_date: date) -> pl.DataFrame: direction_destination -> String stop_name -> String parent_station -> String - static_stop_count -> UInt32 + plan_stop_count -> UInt32 # canon_stop_sequence -> UInt32 arrival_seconds -> Int64 departure_seconds -> Int64 + plan_start_time -> Int64 + plan_start_dt -> Datetime """ trips = trips_for_date(service_date) @@ -282,11 +186,10 @@ def stop_events_for_date(service_date: date) -> pl.DataFrame: "departure_time", "stop_id", "stop_sequence", - "timepoint", - "checkpoint_id", ) - stop_count = stop_times.group_by("trip_id").len("static_stop_count") + stop_count = stop_times.group_by("trip_id").len("plan_stop_count") + trip_start = stop_times.group_by("trip_id").agg(pl.col("arrival_time").min().alias("plan_start_time")) stops = gtfs_from_parquet("stops", service_date).select( "stop_id", @@ -313,18 +216,31 @@ def stop_events_for_date(service_date: date) -> pl.DataFrame: on="trip_id", how="left", ) + .join( + trip_start, + on="trip_id", + how="left", + ) # .join( # canon_stop_sequences, # on=["route_pattern_id", "stop_id"], # how="left", # ) .with_columns( - pl.col("arrival_time") - .map_elements(start_time_to_seconds, return_dtype=pl.Int64) - .alias("arrival_seconds"), - pl.col("departure_time") - .map_elements(start_time_to_seconds, return_dtype=pl.Int64) - .alias("departure_seconds"), + (pl.col("arrival_time").map_elements(start_time_to_seconds, return_dtype=pl.Int64)).alias( + "arrival_seconds" + ), + (pl.col("departure_time").map_elements(start_time_to_seconds, return_dtype=pl.Int64)).alias( + "departure_seconds" + ), + pl.col("plan_start_time").map_elements(start_time_to_seconds, return_dtype=pl.Int64), + pl.col("direction_id").cast(pl.Int8), + ) + .with_columns( + ( + pl.datetime(service_date.year, service_date.month, service_date.day) + + pl.duration(seconds=pl.col("plan_start_time")) + ).alias("plan_start_dt"), ) .drop( "arrival_time", @@ -347,18 +263,14 @@ def stop_event_metrics(stop_events: pl.DataFrame) -> pl.DataFrame: # travel times stop_events = stop_events.with_columns( ( - pl.col("arrival_seconds") - - pl.col("departure_seconds") - .shift() - .over("trip_id", order_by="stop_sequence") + (pl.col("arrival_seconds") - pl.col("departure_seconds")).shift().over("trip_id", order_by="stop_sequence") ).alias("plan_travel_time_seconds") ) # direction_id headway stop_events = stop_events.with_columns( ( - pl.col("departure_seconds") - - pl.col("departure_seconds") + (pl.col("departure_seconds") - pl.col("departure_seconds")) .shift() .over( ["stop_id", "direction_id", "route_id"], @@ -370,8 +282,7 @@ def stop_event_metrics(stop_events: pl.DataFrame) -> pl.DataFrame: # direction_destination headway stop_events = stop_events.with_columns( ( - pl.col("departure_seconds") - - pl.col("departure_seconds") + (pl.col("departure_seconds") - pl.col("departure_seconds")) .shift() .over( ["stop_id", "direction_destination"], @@ -383,43 +294,44 @@ def stop_event_metrics(stop_events: pl.DataFrame) -> pl.DataFrame: return stop_events -def gtfs_events_for_date(service_date: date) -> pl.DataFrame: +def bus_gtfs_events_for_date(service_date: date) -> pl.DataFrame: """ Create data frame of all GTFS data needed by Bus PM app for a service_date :return dataframe: - trip_id -> String + plan_trip_id -> String stop_id -> String stop_sequence -> Int64 - timepoint -> Int64 - checkpoint_id -> String block_id -> String route_id -> String service_id -> String route_pattern_id -> String route_pattern_typicality -> Int64 - direction_id -> Int64 + direction_id -> Int8 direction -> String direction_destination -> String stop_name -> String - parent_station -> String - static_stop_count -> UInt32 - # canon_stop_sequence -> UInt32 - arrival_seconds -> Int64 - departure_seconds -> Int64 + plan_stop_count -> UInt32 + plan_start_time -> Int64 + plan_start_dt -> Datetime plan_travel_time_seconds -> Int64 plan_route_direction_headway_seconds -> Int64 plan_direction_destination_headway_seconds -> Int64 """ - logger = ProcessLogger("gtfs_events_for_date", service_date=service_date) + logger = ProcessLogger("bus_gtfs_events_for_date", service_date=service_date) logger.log_start() - sync_gtfs_files(service_date) - stop_events = stop_events_for_date(service_date) stop_events = stop_event_metrics(stop_events) - logger.log_complete() + drop_columns = [ + "arrival_seconds", + "departure_seconds", + "parent_station", + ] + stop_events = stop_events.drop(drop_columns).rename({"trip_id": "plan_trip_id"}) + logger.add_metadata(events_for_day=stop_events.shape[0]) + logger.log_complete() return stop_events diff --git a/src/lamp_py/bus_performance_manager/events_joined.py b/src/lamp_py/bus_performance_manager/events_joined.py index 9d03263c..39b7392c 100644 --- a/src/lamp_py/bus_performance_manager/events_joined.py +++ b/src/lamp_py/bus_performance_manager/events_joined.py @@ -1,60 +1,235 @@ +from datetime import datetime + import polars as pl +from lamp_py.bus_performance_manager.events_gtfs_schedule import bus_gtfs_events_for_date + -def join_gtfs_tm_events(gtfs: pl.DataFrame, tm: pl.DataFrame) -> pl.DataFrame: +def match_plan_trips(gtfs: pl.DataFrame, schedule: pl.DataFrame) -> pl.DataFrame: """ - Join gtfs-rt and transit master (tm) event dataframes + match all GTFS-RT trip_id's to a plan_trip_id from gtfs schedule + + 3 matching strategies are used + 1. exact trip_id match + 2. exact match on route_id, direction_id, and first_stop of trip and then closest start_dt + 3. exact match on route_id, direction_id and then closest start_dt with the most amount of stop_id's in common on trip :return dataframe: - service_date -> String - route_id -> String trip_id -> String - start_time -> String - direction_id -> Int8 - stop_id -> String - stop_sequence -> String - vehicle_id -> String - vehicle_label -> String - gtfs_travel_to_dt -> Datetime - gtfs_arrival_dt -> Datetime - tm_stop_sequence -> Int64 - tm_is_layover -> Bool - tm_arrival_dt -> Datetime - tm_departure_dt -> Datetime - gtfs_sort_dt -> Datetime - gtfs_depart_dt -> Datetime + plan_trip_id -> String """ + # list of scheduled trips, resulting frame should only have 1 row per plan_trip_id + # if multiple plan_trip_id's exist, trips with least number of stop_counts will be dropped + schedule_trips = ( + schedule.with_columns( + pl.col("stop_id").first().over("plan_trip_id", order_by="stop_sequence").alias("first_stop") + ) + .group_by(["route_id", "plan_trip_id", "direction_id", "plan_start_dt", "first_stop"]) + .agg( + pl.col("stop_id"), + pl.col("stop_id").len().alias("stop_count"), + ) + .sort("stop_count", descending=True) + .unique("plan_trip_id", keep="first") + .drop("stop_count") + ) - # join gtfs and tm datasets using "asof" strategy for stop_sequence columns - # asof strategy finds nearest value match between "asof" columns if exact match is not found - # will perform regular left join on "by" columns + # list of RT trips, resulting frame should only have 1 row per trip_id + # if multiple trip_id's exist, trips with least number of stop_counts will be dropped + rt_trips = ( + gtfs.with_columns(pl.col("stop_id").first().over("trip_id", order_by="stop_sequence").alias("first_stop")) + .group_by(["route_id", "trip_id", "direction_id", "start_dt", "first_stop"]) + .agg( + pl.col("stop_id"), + pl.col("stop_id").len().alias("stop_count"), + ) + .sort("stop_count", descending=True) + .unique("trip_id", keep="first") + .drop("stop_count") + ) + + # capture exact matches between actual and schedule trip_id's + exact_matches = rt_trips.join( + schedule_trips.select("plan_trip_id"), + how="left", + left_on="trip_id", + right_on="plan_trip_id", + coalesce=False, + validate="1:1", + ) - return ( - gtfs.sort(by="stop_sequence") + # asof join will match actual to schedule trips first by exact match on: + # - route_id + # - direction_id + # - first_stop of trip + # then will match to closest schedule start_dt within 1 hour of actual start_dt + asof_matches = ( + exact_matches.filter(pl.col("plan_trip_id").is_null()) + .drop("plan_trip_id") + .sort("start_dt") .join_asof( - tm.sort("tm_stop_sequence"), - left_on="stop_sequence", - right_on="tm_stop_sequence", - by=["trip_id", "route_id", "vehicle_label", "stop_id"], + schedule_trips.drop(["stop_id"]).sort("plan_start_dt"), + left_on="start_dt", + right_on="plan_start_dt", + by=["route_id", "direction_id", "first_stop"], strategy="nearest", - coalesce=True, + tolerance="1h", ) - .with_columns( - ( - pl.coalesce( - ["gtfs_travel_to_dt", "gtfs_arrival_dt"], - ).alias("gtfs_sort_dt") - ) - ) - .with_columns( - ( - pl.col("gtfs_travel_to_dt") - .shift(-1) - .over( - ["vehicle_label", "trip_id"], - order_by="gtfs_sort_dt", + ) + + # last match attempts to match trips that did not produce matches from exact or asof join + # find all scheduled trip within 1 hour of actual drop and most overlap some stop_id's + # sort by most number of stop_id's in common and then duration difference between start_dt + last_matches = [] + for row in asof_matches.filter(pl.col("plan_trip_id").is_null()).iter_rows(named=True): + plan_trip_id = None + try: + plan_trip_id = ( + schedule_trips.filter( + pl.col("route_id") == row["route_id"], + pl.col("direction_id") == row["direction_id"], + pl.Expr.abs(pl.col("plan_start_dt") - row["start_dt"]) < pl.duration(hours=1), + pl.col("stop_id").list.set_difference(row["stop_id"]).list.len() < pl.col("stop_id").list.len(), ) - .alias("gtfs_depart_dt") + .sort( + pl.col("stop_id").list.set_difference(row["stop_id"]).list.len(), + pl.Expr.abs(pl.col("plan_start_dt") - row["start_dt"]), + ) + .get_column("plan_trip_id")[0] ) - ) + except Exception as _: + pass + last_matches.append({"trip_id": row["trip_id"], "plan_trip_id": plan_trip_id}) + + # join all sets of matches into a single dataframe + # print("exact", exact_matches.filter(pl.col("plan_trip_id").is_not_null()).select("trip_id", "plan_trip_id")) + # print("asof", asof_matches.filter(pl.col("plan_trip_id").is_not_null()).select("trip_id", "plan_trip_id")) + # print("last", pl.DataFrame(last_matches)) + return_df = pl.concat( + [ + exact_matches.filter(pl.col("plan_trip_id").is_not_null()).select("trip_id", "plan_trip_id"), + asof_matches.filter(pl.col("plan_trip_id").is_not_null()).select("trip_id", "plan_trip_id"), + pl.DataFrame(last_matches, schema={"trip_id": str, "plan_trip_id": str}), + ], + how="vertical", + rechunk=True, + ) + + assert return_df.shape[0] == rt_trips.shape[0], "must produce trip match for every RT trip" + + return return_df + + +def join_schedule_to_rt(gtfs: pl.DataFrame) -> pl.DataFrame: + """ + Join gtfs-rt records to gtfs schedule data + + join steps: + 1. match all RT trips to a plan trip. + 2. match plan trip data to RT events + 3. match plan event data to RT events + + :return added-columns: + plan_trip_id -> String + exact_plan_trip_match -> Bool + block_id -> String + service_id -> String + route_pattern_id -> String + route_pattern_typicality -> Int64 + direction -> String + direction_destination -> String + plan_stop_count -> UInt32 + plan_start_time -> Int64 + plan_start_dt -> Datetime + stop_name -> String + plan_travel_time_seconds -> Int64 + plan_route_direction_headway_seconds -> Int64 + plan_direction_destination_headway_seconds -> Int64 + """ + service_dates = gtfs.get_column("service_date").unique() + assert len(service_dates) == 1, f"more than 1 service_date found: {service_dates}" + service_date = datetime.strptime(service_dates[0], "%Y%m%d") + + schedule = bus_gtfs_events_for_date(service_date) + + # get a plan_trip_id from the schedule for every rt trip_id + gtfs = gtfs.join( + match_plan_trips(gtfs, schedule), on="trip_id", how="left", coalesce=True, validate="m:1" + ).with_columns(pl.col("trip_id").eq(pl.col("plan_trip_id")).alias("exact_plan_trip_match")) + + # join plan scheudle trip data to rt gtfs + gtfs = gtfs.join( + ( + schedule.select( + "plan_trip_id", + "block_id", + "service_id", + "route_pattern_id", + "route_pattern_typicality", + "direction", + "direction_destination", + "plan_stop_count", + "plan_start_time", + "plan_start_dt", + ).unique() + ), + on="plan_trip_id", + how="left", + coalesce=True, + validate="m:1", + ).join( + ( + schedule.select( + "stop_id", + "stop_name", + ).unique() + ), + on="stop_id", + how="left", + coalesce=True, + validate="m:1", + ) + + # join plan schedule evenat data to rt gtfs + # asof join on stop_sequence after normal join on plan_trip_id and stop_id + # this is because the same stop_id can appear on a trip multiple times + gtfs = gtfs.join_asof( + schedule.select( + "plan_trip_id", + "stop_id", + "stop_sequence", + "plan_travel_time_seconds", + "plan_route_direction_headway_seconds", + "plan_direction_destination_headway_seconds", + ), + on="stop_sequence", + by=["plan_trip_id", "stop_id"], + strategy="nearest", + tolerance=5, + coalesce=True, + ) + + return gtfs + + +def join_tm_to_rt(gtfs: pl.DataFrame, tm: pl.DataFrame) -> pl.DataFrame: + """ + Join gtfs-rt and transit master (tm) event dataframes + + :return added-columns: + tm_arrival_dt -> Datetime + tm_departure_dt -> Datetime + """ + + # join gtfs and tm datasets using "asof" strategy for stop_sequence columns + # asof strategy finds nearest value match between "asof" columns if exact match is not found + # will perform regular left join on "by" columns + + return gtfs.sort(by="stop_sequence").join_asof( + tm.sort("tm_stop_sequence"), + left_on="stop_sequence", + right_on="tm_stop_sequence", + by=["trip_id", "route_id", "vehicle_label", "stop_id"], + strategy="nearest", + coalesce=True, ) diff --git a/src/lamp_py/bus_performance_manager/events_metrics.py b/src/lamp_py/bus_performance_manager/events_metrics.py new file mode 100644 index 00000000..d41df194 --- /dev/null +++ b/src/lamp_py/bus_performance_manager/events_metrics.py @@ -0,0 +1,140 @@ +from typing import List +from datetime import date + +import polars as pl + +from lamp_py.bus_performance_manager.events_gtfs_rt import generate_gtfs_rt_events +from lamp_py.bus_performance_manager.events_tm import generate_tm_events +from lamp_py.bus_performance_manager.events_joined import join_tm_to_rt +from lamp_py.bus_performance_manager.events_joined import join_schedule_to_rt + + +def bus_performance_metrics(service_date: date, gtfs_files: List[str], tm_files: List[str]) -> pl.DataFrame: + """ + create dataframe of Bus Performance metrics to write to S3 + + :param service_date: date of service being processed + :param gtfs_files: list of RT_VEHCILE_POSITION parquet file paths, from S3, that cover service date + :param tm_files: list of TM/STOP_CROSSING parquet file paths, from S3, that cover service date + + :return dataframe: + service_date -> String + route_id -> String + trip_id -> String + start_time -> Int64 + start_dt -> Datetime + stop_count -> UInt32 + direction_id -> Int8 + stop_id -> String + stop_sequence -> String + vehicle_id -> String + vehicle_label -> String + gtfs_travel_to_dt -> Datetime + gtfs_travel_to_seconds -> Int64 + stop_arrival_dt -> Datetime + stop_arrival_seconds -> Int64 + stop_departure_dt -> Datetime + stop_departure_seconds -> Int64 + plan_trip_id -> String + exact_plan_trip_match -> Bool + block_id -> String + service_id -> String + route_pattern_id -> String + route_pattern_typicality -> Int64 + direction -> String + direction_destination -> String + plan_stop_count -> UInt32 + plan_start_time -> Int64 + plan_start_dt -> Datetime + stop_name -> String + plan_travel_time_seconds -> Int64 + plan_route_direction_headway_seconds -> Int64 + plan_direction_destination_headway_seconds -> Int64 + travel_time_seconds -> Int64 + dwell_time_seconds -> Int64 + route_direction_headway_seconds -> Int64 + direction_destination_headway_seconds -> Int64 + """ + # gtfs-rt events from parquet + gtfs_df = generate_gtfs_rt_events(service_date, gtfs_files) + # transit master events from parquet + tm_df = generate_tm_events(tm_files) + # create events dataframe with static schedule data, gtfs-rt events and transit master events + bus_df = join_schedule_to_rt(join_tm_to_rt(gtfs_df, tm_df)) + + bus_df = ( + bus_df.with_columns(pl.coalesce(["gtfs_travel_to_dt", "gtfs_arrival_dt"]).alias("gtfs_sort_dt")) + .with_columns( + ( + pl.col("gtfs_travel_to_dt") + .shift(-1) + .over( + ["vehicle_label", "trip_id"], + order_by="gtfs_sort_dt", + ) + ).alias("gtfs_departure_dt"), + ( + pl.when(pl.col("tm_arrival_dt") > pl.col("gtfs_travel_to_dt")) + .then(pl.col("tm_arrival_dt")) + .otherwise(pl.col("gtfs_arrival_dt")) + ).alias("stop_arrival_dt"), + ) + .with_columns( + pl.when(pl.col("tm_departure_dt") >= pl.col("stop_arrival_dt")) + .then(pl.col("tm_departure_dt")) + .otherwise(pl.col("gtfs_departure_dt")) + .alias("stop_departure_dt") + ) + # convert dt columns to seconds after midnight + .with_columns( + (pl.col("gtfs_travel_to_dt") - pl.col("service_date").str.strptime(pl.Date, "%Y%m%d")) + .dt.total_seconds() + .alias("gtfs_travel_to_seconds"), + (pl.col("stop_arrival_dt") - pl.col("service_date").str.strptime(pl.Date, "%Y%m%d")) + .dt.total_seconds() + .alias("stop_arrival_seconds"), + (pl.col("stop_departure_dt") - pl.col("service_date").str.strptime(pl.Date, "%Y%m%d")) + .dt.total_seconds() + .alias("stop_departure_seconds"), + ) + # add metrics columns to events + .with_columns( + (pl.coalesce(["stop_arrival_seconds", "stop_departure_seconds"]) - pl.col("gtfs_travel_to_seconds")).alias( + "travel_time_seconds" + ), + (pl.col("stop_departure_seconds") - pl.col("stop_arrival_seconds")).alias("dwell_time_seconds"), + ( + pl.coalesce(["stop_departure_seconds", "stop_arrival_seconds"]) + - pl.coalesce(["stop_departure_seconds", "stop_arrival_seconds"]) + .shift() + .over( + ["stop_id", "direction_id", "route_id"], + order_by="gtfs_sort_dt", + ) + ).alias("route_direction_headway_seconds"), + ( + pl.coalesce(["stop_departure_seconds", "stop_arrival_seconds"]) + - ( + pl.coalesce(["stop_departure_seconds", "stop_arrival_seconds"]) + .shift() + .over( + ["stop_id", "direction_destination"], + order_by="gtfs_sort_dt", + ) + ) + ).alias("direction_destination_headway_seconds"), + ) + # sort to reduce parquet file size + .sort(["route_id", "vehicle_label", "gtfs_sort_dt"]) + .drop( + [ + "gtfs_departure_dt", + "gtfs_arrival_dt", + "tm_departure_dt", + "tm_arrival_dt", + "gtfs_sort_dt", + ] + ) + ) + + return bus_df diff --git a/src/lamp_py/bus_performance_manager/events_tm.py b/src/lamp_py/bus_performance_manager/events_tm.py index 74e6f6b3..ef3c69f5 100644 --- a/src/lamp_py/bus_performance_manager/events_tm.py +++ b/src/lamp_py/bus_performance_manager/events_tm.py @@ -12,6 +12,23 @@ tm_run_file, tm_operator_file, ) +from lamp_py.runtime_utils.process_logger import ProcessLogger + + +def _empty_stop_crossing() -> pl.DataFrame: + """ + create empty stop crossing dataframe with expected columns + """ + schema = { + "vehicle_label": pl.String, + "route_id": pl.String, + "trip_id": pl.String, + "stop_id": pl.String, + "tm_stop_sequence": pl.Int64, + "tm_arrival_dt": pl.Datetime, + "tm_departure_dt": pl.Datetime, + } + return pl.DataFrame(schema=schema) def generate_tm_events(tm_files: List[str]) -> pl.DataFrame: @@ -23,15 +40,16 @@ def generate_tm_events(tm_files: List[str]) -> pl.DataFrame: :param tm_files: transit master parquet files from the StopCrossings table. :return dataframe: - service_date -> Date - tm_vehicle_label -> String - tm_route_id -> String - tm_geo_node_id -> String - tm_stop_id -> String - tm_trip_id -> String + vehicle_label -> String + route_id -> String + trip_id -> String + stop_id -> String + tm_stop_sequence -> Int64 tm_arrival_dt -> Datetime(time_unit='us', time_zone=None) as UTC tm_departure_dt -> Datetime(time_unit='us', time_zone=None) as UTC """ + logger = ProcessLogger("generate_tm_events", tm_files=tm_files) + logger.log_start() # the geo node id is the transit master key and the geo node abbr is the # gtfs stop id tm_geo_nodes = ( @@ -88,103 +106,80 @@ def generate_tm_events(tm_files: List[str]) -> pl.DataFrame: # remove leading zeros from route ids where they exist # convert arrival and departure times to utc datetimes # cast everything else as a string - tm_stop_crossings = ( - pl.scan_parquet(tm_files) - .filter( - (pl.col("IsRevenue") == "R") - & pl.col("ROUTE_ID").is_not_null() - & pl.col("GEO_NODE_ID").is_not_null() - & pl.col("TRIP_ID").is_not_null() - & pl.col("VEHICLE_ID").is_not_null() - & ( - (pl.col("ACT_ARRIVAL_TIME").is_not_null()) - | (pl.col("ACT_DEPARTURE_TIME").is_not_null()) + tm_stop_crossings = _empty_stop_crossing() + if len(tm_files) > 0: + tm_stop_crossings = ( + pl.scan_parquet(tm_files) + .filter( + (pl.col("IsRevenue") == "R") + & pl.col("ROUTE_ID").is_not_null() + & pl.col("GEO_NODE_ID").is_not_null() + & pl.col("TRIP_ID").is_not_null() + & pl.col("VEHICLE_ID").is_not_null() + & ((pl.col("ACT_ARRIVAL_TIME").is_not_null()) | (pl.col("ACT_DEPARTURE_TIME").is_not_null())) ) - ) - .join( - tm_geo_nodes, - on="GEO_NODE_ID", - how="left", - coalesce=True, - ) - .join( - tm_routes, - on="ROUTE_ID", - how="left", - coalesce=True, - ) - .join( - tm_trips, - on="TRIP_ID", - how="left", - coalesce=True, - ) - .join( - tm_vehicles, - on="VEHICLE_ID", - how="left", - coalesce=True, - ) - .with_columns( - ( - pl.col("CALENDAR_ID") - .cast(pl.Utf8) - .str.slice(1) - .str.strptime(pl.Datetime, format="%Y%m%d") - .alias("service_date") - ), - ) - .select( - ( - pl.col("ROUTE_ABBR") - .cast(pl.String) - .str.strip_chars_start("0") - .alias("route_id") - ), - pl.col("TRIP_SERIAL_NUMBER").cast(pl.String).alias("trip_id"), - pl.col("GEO_NODE_ABBR").cast(pl.String).alias("stop_id"), - pl.col("PATTERN_GEO_NODE_SEQ") - .cast(pl.Int64) - .alias("tm_stop_sequence"), - pl.col("IS_LAYOVER").cast(pl.String).alias("tm_is_layover"), - pl.col("PROPERTY_TAG").cast(pl.String).alias("vehicle_label"), - ( + .join( + tm_geo_nodes, + on="GEO_NODE_ID", + how="left", + coalesce=True, + ) + .join( + tm_routes, + on="ROUTE_ID", + how="left", + coalesce=True, + ) + .join( + tm_trips, + on="TRIP_ID", + how="left", + coalesce=True, + ) + .join( + tm_vehicles, + on="VEHICLE_ID", + how="left", + coalesce=True, + ) + .with_columns( ( - pl.col("service_date") - + pl.duration(seconds="ACT_ARRIVAL_TIME") - ) - .dt.replace_time_zone("America/New_York") - .dt.convert_time_zone("UTC") - .dt.replace_time_zone(None) - .alias("tm_arrival_dt") - ), - ( + pl.col("CALENDAR_ID") + .cast(pl.Utf8) + .str.slice(1) + .str.strptime(pl.Datetime, format="%Y%m%d") + .alias("service_date") + ), + ) + .select( + (pl.col("ROUTE_ABBR").cast(pl.String).str.strip_chars_start("0").alias("route_id")), + pl.col("TRIP_SERIAL_NUMBER").cast(pl.String).alias("trip_id"), + pl.col("GEO_NODE_ABBR").cast(pl.String).alias("stop_id"), + pl.col("PATTERN_GEO_NODE_SEQ").cast(pl.Int64).alias("tm_stop_sequence"), + pl.col("PROPERTY_TAG").cast(pl.String).alias("vehicle_label"), ( - pl.col("service_date") - + pl.duration(seconds="ACT_DEPARTURE_TIME") - ) - .dt.replace_time_zone("America/New_York") - .dt.convert_time_zone("UTC") - .dt.replace_time_zone(None) - .alias("tm_departure_dt") - ), + (pl.col("service_date") + pl.duration(seconds="ACT_ARRIVAL_TIME")) + .dt.replace_time_zone("America/New_York", ambiguous="earliest") + .dt.convert_time_zone("UTC") + .dt.replace_time_zone(None) + .alias("tm_arrival_dt") + ), + ( + (pl.col("service_date") + pl.duration(seconds="ACT_DEPARTURE_TIME")) + .dt.replace_time_zone("America/New_York", ambiguous="earliest") + .dt.convert_time_zone("UTC") + .dt.replace_time_zone(None) + .alias("tm_departure_dt") + ), + ) + .collect() ) - .collect() - ) if tm_stop_crossings.shape[0] == 0: - schema = { - "route_id": pl.String, - "trip_id": pl.String, - "stop_id": pl.String, - "tm_stop_sequence": pl.Int64, - "tm_is_layover": pl.Boolean, - "vehicle_label": pl.String, - "tm_arrival_dt": pl.Datetime, - "tm_departure_dt": pl.Datetime, - } - tm_stop_crossings = pl.DataFrame(schema=schema) + tm_stop_crossings = _empty_stop_crossing() + logger.add_metadata(events_for_day=tm_stop_crossings.shape[0]) + logger.log_complete() return tm_stop_crossings @@ -307,10 +302,7 @@ def get_daily_work_pieces(daily_work_piece_files: List[str]) -> pl.DataFrame: on=["BLOCK_ID", "TIME_TABLE_VERSION_ID"], coalesce=True, ) - .filter( - (pl.col("BEGIN_TIME") < pl.col("TRIP_END_TIME")) - & (pl.col("END_TIME") >= pl.col("TRIP_END_TIME")) - ) + .filter((pl.col("BEGIN_TIME") < pl.col("TRIP_END_TIME")) & (pl.col("END_TIME") >= pl.col("TRIP_END_TIME"))) ) # Collect the Realtime Details of who operated what vehicle for which piece @@ -406,28 +398,18 @@ def get_daily_work_pieces(daily_work_piece_files: List[str]) -> pl.DataFrame: pl.col("BLOCK_ABBR").cast(pl.String).alias("tm_block_id"), pl.col("RUN_DESIGNATOR").cast(pl.String).alias("tm_run_id"), pl.col("TRIP_SERIAL_NUMBER").cast(pl.String).alias("tm_trip_id"), - ( - pl.col("ONBOARD_LOGON_ID") - .cast(pl.String) - .alias("operator_badge_number") - ), + (pl.col("ONBOARD_LOGON_ID").cast(pl.String).alias("operator_badge_number")), pl.col("PROPERTY_TAG").cast(pl.String).alias("tm_vehicle_label"), ( - ( - pl.col("service_date") - + pl.duration(seconds="ACTUAL_LOGON_TIME") - ) - .dt.replace_time_zone("America/New_York") + (pl.col("service_date") + pl.duration(seconds="ACTUAL_LOGON_TIME")) + .dt.replace_time_zone("America/New_York", ambiguous="earliest") .dt.convert_time_zone("UTC") .dt.replace_time_zone(None) .alias("logon_time") ), ( - ( - pl.col("service_date") - + pl.duration(seconds="ACTUAL_LOGOFF_TIME") - ) - .dt.replace_time_zone("America/New_York") + (pl.col("service_date") + pl.duration(seconds="ACTUAL_LOGOFF_TIME")) + .dt.replace_time_zone("America/New_York", ambiguous="earliest") .dt.convert_time_zone("UTC") .dt.replace_time_zone(None) .alias("logoff_time") diff --git a/src/lamp_py/bus_performance_manager/gtfs_utils.py b/src/lamp_py/bus_performance_manager/gtfs_utils.py index c976e4cb..078bbdb3 100644 --- a/src/lamp_py/bus_performance_manager/gtfs_utils.py +++ b/src/lamp_py/bus_performance_manager/gtfs_utils.py @@ -3,28 +3,57 @@ import polars as pl +from lamp_py.aws.s3 import object_exists +from lamp_py.runtime_utils.process_logger import ProcessLogger from lamp_py.runtime_utils.remote_files import compressed_gtfs -def bus_routes_for_service_date(service_date: date) -> List[str]: - """get a list of bus route ids for a given service date""" - routes_file = compressed_gtfs.parquet_path( - year=service_date.year, file="routes" - ).s3_uri +def gtfs_from_parquet(file: str, service_date: date) -> pl.DataFrame: + """ + Get GTFS data from specified file and service date - # generate an integer date for the service date - target_date = int(service_date.strftime("%Y%m%d")) + This will read from s3_uri of file - bus_routes = ( - pl.scan_parquet(routes_file) + :param file: gtfs file to acces (i.e. "feed_info") + :param service_date: service date of requested GTFS data + + :return dataframe: + data columns of parquet file for service_date + """ + logger = ProcessLogger("gtfs_from_parquet", file=file, service_date=service_date) + logger.log_start() + + gtfs_year = service_date.year + service_date_int = int(service_date.strftime("%Y%m%d")) + + gtfs_file = compressed_gtfs.parquet_path(gtfs_year, file).s3_uri + + if not object_exists(gtfs_file): + gtfs_file = compressed_gtfs.parquet_path(gtfs_year - 1, file).s3_uri + if not object_exists(gtfs_file): + exception = FileNotFoundError(f"No GTFS archive files available for {service_date}") + logger.log_failure(exception) + raise exception + + logger.add_metadata(gtfs_file=gtfs_file) + + gtfs_df = ( + pl.read_parquet(gtfs_file) .filter( - (pl.col("gtfs_active_date") <= target_date) - & (pl.col("gtfs_end_date") >= target_date) - & (pl.col("route_type") == 3) + (pl.col("gtfs_active_date") <= service_date_int), + (pl.col("gtfs_end_date") >= service_date_int), ) - .select("route_id") - .unique() - .collect() + .drop(["gtfs_active_date", "gtfs_end_date"]) + ) + logger.add_metadata(gtfs_row_count=gtfs_df.shape[0]) + logger.log_complete() + return gtfs_df + + +def bus_routes_for_service_date(service_date: date) -> List[str]: + """get a list of bus route ids for a given service date""" + bus_routes = ( + gtfs_from_parquet("routes", service_date).filter((pl.col("route_type") == 3)).get_column("route_id").unique() ) - return bus_routes.get_column("route_id").to_list() + return bus_routes.to_list() diff --git a/src/lamp_py/bus_performance_manager/pipeline.py b/src/lamp_py/bus_performance_manager/pipeline.py index 6a137501..79c01ebe 100644 --- a/src/lamp_py/bus_performance_manager/pipeline.py +++ b/src/lamp_py/bus_performance_manager/pipeline.py @@ -12,6 +12,7 @@ from lamp_py.aws.ecs import handle_ecs_sigterm, check_for_sigterm from lamp_py.runtime_utils.env_validation import validate_environment from lamp_py.runtime_utils.process_logger import ProcessLogger +from lamp_py.bus_performance_manager.write_events import write_bus_metrics logging.getLogger().setLevel("INFO") @@ -46,8 +47,8 @@ def iteration() -> None: check_for_sigterm() process_logger = ProcessLogger("event_loop") process_logger.log_start() - try: + write_bus_metrics() process_logger.log_complete() except Exception as exception: process_logger.log_failure(exception) @@ -76,7 +77,6 @@ def start() -> None: "PUBLIC_ARCHIVE_BUCKET", "SERVICE_NAME", ], - db_prefixes=["MD"], ) # run main method diff --git a/src/lamp_py/bus_performance_manager/write_events.py b/src/lamp_py/bus_performance_manager/write_events.py new file mode 100644 index 00000000..d63ba112 --- /dev/null +++ b/src/lamp_py/bus_performance_manager/write_events.py @@ -0,0 +1,57 @@ +import os +import tempfile + +from lamp_py.bus_performance_manager.event_files import event_files_to_load +from lamp_py.bus_performance_manager.events_metrics import bus_performance_metrics +from lamp_py.runtime_utils.remote_files import bus_events +from lamp_py.runtime_utils.remote_files import VERSION_KEY +from lamp_py.runtime_utils.process_logger import ProcessLogger +from lamp_py.aws.s3 import upload_file + + +def write_bus_metrics() -> None: + """ + Write bus-performance parquet files to S3 for service dates neeing to be processed + """ + logger = ProcessLogger("write_bus_metrics") + logger.log_start() + + event_files = event_files_to_load() + logger.add_metadata(service_date_count=len(event_files)) + + for service_date in event_files.keys(): + gtfs_files = event_files[service_date]["gtfs_rt"] + tm_files = event_files[service_date]["transit_master"] + + day_logger = ProcessLogger( + "write_bus_metrics_day", + service_date=service_date, + gtfs_file_count=len(gtfs_files), + tm_file_count=len(tm_files), + ) + day_logger.log_start() + + # need gtfs_rt files to run process + if len(gtfs_files) == 0: + day_logger.log_failure(FileNotFoundError(f"No RT_VEHICLE_POSITION files found for {service_date}")) + continue + + try: + events_df = bus_performance_metrics(service_date, gtfs_files, tm_files) + day_logger.add_metadata(bus_performance_rows=events_df.shape[0]) + + with tempfile.TemporaryDirectory() as tempdir: + write_file = f"{service_date.strftime('%Y%m%d')}.parquet" + events_df.write_parquet(os.path.join(tempdir, write_file), use_pyarrow=True) + + upload_file( + file_name=os.path.join(tempdir, write_file), + object_path=os.path.join(bus_events.s3_uri, write_file), + extra_args={"Metadata": {VERSION_KEY: bus_events.version}}, + ) + + day_logger.log_complete() + except Exception as exception: + day_logger.log_failure(exception) + + logger.log_complete() diff --git a/src/lamp_py/ingestion/compress_gtfs/gtfs_to_parquet.py b/src/lamp_py/ingestion/compress_gtfs/gtfs_to_parquet.py index 1bd829c4..0f93df4c 100644 --- a/src/lamp_py/ingestion/compress_gtfs/gtfs_to_parquet.py +++ b/src/lamp_py/ingestion/compress_gtfs/gtfs_to_parquet.py @@ -48,12 +48,8 @@ def frame_parquet_diffs( new_records: polars.DataFrame, ] """ - pq_filter = (pc.field("gtfs_active_date") <= filter_date) & ( - pc.field("gtfs_end_date") >= filter_date - ) - pq_frame = pl.read_parquet( - pq_path, use_pyarrow=True, pyarrow_options={"filters": pq_filter} - ) + pq_filter = (pc.field("gtfs_active_date") <= filter_date) & (pc.field("gtfs_end_date") >= filter_date) + pq_frame = pl.read_parquet(pq_path, use_pyarrow=True, pyarrow_options={"filters": pq_filter}) join_columns = tuple(gtfs_schema(gtfs_table_file).keys()) @@ -81,9 +77,7 @@ def frame_parquet_diffs( return old_records, same_records, new_records -def merge_frame_with_parquet( - merge_df: pl.DataFrame, export_path: str, filter_date: int -) -> None: +def merge_frame_with_parquet(merge_df: pl.DataFrame, export_path: str, filter_date: int) -> None: """ merge merge_df with existing parqut file (export_path) and over-write with results @@ -110,9 +104,7 @@ def merge_frame_with_parquet( tmp_path = os.path.join(temp_dir, "filter.parquet") # create filtered parquet file, excluding records from merge_frame - pq_filter = (pc.field("gtfs_active_date") > filter_date) | ( - pc.field("gtfs_end_date") < filter_date - ) + pq_filter = (pc.field("gtfs_active_date") > filter_date) | (pc.field("gtfs_end_date") < filter_date) filter_ds = pd.dataset(export_path).filter(pq_filter) with pq.ParquetWriter(tmp_path, schema=merge_df.schema) as writer: for batch in filter_ds.to_batches(batch_size=batch_size): @@ -125,9 +117,7 @@ def merge_frame_with_parquet( writer.write_batch(batch) -def compress_gtfs_file( - gtfs_table_file: str, schedule_details: ScheduleDetails -) -> None: +def compress_gtfs_file(gtfs_table_file: str, schedule_details: ScheduleDetails) -> None: """ compress an indivdual gtfs_table_file (ie. stop_times.txt) into yearly parquet partitioned parquet file(s) @@ -179,12 +169,8 @@ def compress_gtfs_file( # "gtfs_end_date": # (same or new records) set to schedule_details.active_to_int # (old records) set to schedule_details.published_int (day before active_to_int) - same_records = same_records.with_columns( - pl.lit(schedule_details.active_to_int).alias("gtfs_end_date") - ) - old_records = old_records.with_columns( - pl.lit(schedule_details.published_int).alias("gtfs_end_date") - ) + same_records = same_records.with_columns(pl.lit(schedule_details.active_to_int).alias("gtfs_end_date")) + old_records = old_records.with_columns(pl.lit(schedule_details.published_int).alias("gtfs_end_date")) merge_records = pl.concat( (old_records, same_records, new_records), @@ -238,9 +224,7 @@ def compress_gtfs_file( how="diagonal", ).filter( pl.col("gtfs_end_date") > pl.col("gtfs_active_date") - ).write_parquet( - export_path, use_pyarrow=True, statistics=True - ) + ).write_parquet(export_path, use_pyarrow=True, statistics=True) else: # # no partition file exists (current or last) @@ -249,9 +233,7 @@ def compress_gtfs_file( if new_frame.shape[0] == 0: return - new_frame.drop("from_zip").write_parquet( - export_path, use_pyarrow=True, statistics=True - ) + new_frame.drop("from_zip").write_parquet(export_path, use_pyarrow=True, statistics=True) def compress_gtfs_schedule(schedule_details: ScheduleDetails) -> None: @@ -295,9 +277,7 @@ def gtfs_to_parquet() -> None: while processing Feb-2018 to April-2024 """ gtfs_tmp_folder = os.path.join("/tmp", compressed_gtfs.prefix) - logger = ProcessLogger( - "compress_gtfs_schedules", gtfs_tmp_folder=gtfs_tmp_folder - ) + logger = ProcessLogger("compress_gtfs_schedules", gtfs_tmp_folder=gtfs_tmp_folder) logger.log_start() feed = schedules_to_compress(gtfs_tmp_folder) diff --git a/src/lamp_py/ingestion/compress_gtfs/pq_to_sqlite.py b/src/lamp_py/ingestion/compress_gtfs/pq_to_sqlite.py index 552f7321..bddf4ea6 100644 --- a/src/lamp_py/ingestion/compress_gtfs/pq_to_sqlite.py +++ b/src/lamp_py/ingestion/compress_gtfs/pq_to_sqlite.py @@ -29,9 +29,7 @@ def sqlite_table_query(table_name: str, schema: pyarrow.Schema) -> str: """ logger = ProcessLogger("sqlite_create_table") logger.log_start() - field_list = [ - f"{field.name} {sqlite_type(str(field.type))}" for field in schema - ] + field_list = [f"{field.name} {sqlite_type(str(field.type))}" for field in schema] query = f""" CREATE TABLE IF NOT EXISTS diff --git a/src/lamp_py/ingestion/compress_gtfs/schedule_details.py b/src/lamp_py/ingestion/compress_gtfs/schedule_details.py index a2cd8dc5..86947747 100644 --- a/src/lamp_py/ingestion/compress_gtfs/schedule_details.py +++ b/src/lamp_py/ingestion/compress_gtfs/schedule_details.py @@ -75,9 +75,7 @@ def headers_from_file(self, gtfs_table_file: str) -> List[str]: :return List[header_names] """ if gtfs_table_file not in self.file_list: - raise KeyError( - f"{gtfs_table_file} not found in {self.file_location} archive" - ) + raise KeyError(f"{gtfs_table_file} not found in {self.file_location} archive") with zipfile.ZipFile(self.gtfs_bytes) as zf: with zf.open(gtfs_table_file) as f_bytes: @@ -157,16 +155,11 @@ def gtfs_to_frame(self, gtfs_table_file: str) -> pl.DataFrame: # add missing columns as all NULL values for null_col in missing_columns: - frame = frame.with_columns( - pl.lit(None).cast(table_schema[null_col]).alias(null_col) - ) + frame = frame.with_columns(pl.lit(None).cast(table_schema[null_col]).alias(null_col)) # update String values containing only spaces to NULL frame = frame.with_columns( - pl.when( - pl.col(pl.Utf8).str.replace(r"\s*", "", n=1).str.len_chars() - == 0 - ) + pl.when(pl.col(pl.Utf8).str.replace(r"\s*", "", n=1).str.len_chars() == 0) .then(None) .otherwise(pl.col(pl.Utf8)) .name.keep() @@ -227,9 +220,7 @@ def schedules_to_compress(tmp_folder: str) -> pl.DataFrame: for obj_path in s3_files: if not obj_path.endswith(".parquet"): continue - local_path = obj_path.replace( - f"s3://{compressed_gtfs.bucket}", "/tmp" - ) + local_path = obj_path.replace(f"s3://{compressed_gtfs.bucket}", "/tmp") download_file(obj_path, local_path) else: continue @@ -242,16 +233,11 @@ def schedules_to_compress(tmp_folder: str) -> pl.DataFrame: # values between `feed_info` file in schedule and "archived_feeds.txt" file feed = feed.filter( (pl.col("published_date") > int(f"{year}0000")) - & ( - pl.col("feed_start_date") - > pq_fi_frame.get_column("feed_start_date").max() - ) + & (pl.col("feed_start_date") > pq_fi_frame.get_column("feed_start_date").max()) ) else: # anti join against records for 'year' to find records not already in feed_info.parquet - feed = feed.filter( - pl.col("published_date") > int(f"{year}0000") - ).join( + feed = feed.filter(pl.col("published_date") > int(f"{year}0000")).join( pq_fi_frame.select("feed_version"), on="feed_version", how="anti", diff --git a/src/lamp_py/ingestion/config_busloc_trip.py b/src/lamp_py/ingestion/config_busloc_trip.py index 28fb6e9f..6fd4eaa9 100644 --- a/src/lamp_py/ingestion/config_busloc_trip.py +++ b/src/lamp_py/ingestion/config_busloc_trip.py @@ -18,11 +18,7 @@ class RtBusTripDetail(GTFSRTDetail): def transform_for_write(self, table: pyarrow.table) -> pyarrow.table: """modify table schema before write to parquet""" - return flatten_schema( - explode_table_column( - flatten_schema(table), "trip_update.stop_time_update" - ) - ) + return flatten_schema(explode_table_column(flatten_schema(table), "trip_update.stop_time_update")) @property def partition_column(self) -> str: diff --git a/src/lamp_py/ingestion/config_rt_trip.py b/src/lamp_py/ingestion/config_rt_trip.py index da955ead..7c70a44e 100644 --- a/src/lamp_py/ingestion/config_rt_trip.py +++ b/src/lamp_py/ingestion/config_rt_trip.py @@ -18,11 +18,7 @@ class RtTripDetail(GTFSRTDetail): def transform_for_write(self, table: pyarrow.table) -> pyarrow.table: """modify table schema before write to parquet""" - return flatten_schema( - explode_table_column( - flatten_schema(table), "trip_update.stop_time_update" - ) - ) + return flatten_schema(explode_table_column(flatten_schema(table), "trip_update.stop_time_update")) @property def partition_column(self) -> str: diff --git a/src/lamp_py/ingestion/convert_gtfs.py b/src/lamp_py/ingestion/convert_gtfs.py index cd552593..85464cf6 100644 --- a/src/lamp_py/ingestion/convert_gtfs.py +++ b/src/lamp_py/ingestion/convert_gtfs.py @@ -46,15 +46,10 @@ def gtfs_files_to_convert() -> List[Tuple[str, int]]: # add version_key column mbta_schedule_feed = mbta_schedule_feed.with_columns( - pl.col("published_dt") - .dt.timestamp("ms") - .floordiv(1000) - .alias("version_key") + pl.col("published_dt").dt.timestamp("ms").floordiv(1000).alias("version_key") ) - return mbta_schedule_feed.select(["archive_url", "version_key"]).rows( - named=False - ) + return mbta_schedule_feed.select(["archive_url", "version_key"]).rows(named=False) class GtfsConverter(Converter): @@ -102,9 +97,7 @@ def process_schedule(self, url: str, version_key: int) -> None: self.create_table(gtfs_zip, "feed_info.txt", version_key) - def create_table( - self, gtfs_zip: zipfile.ZipFile, table_filename: str, version_key: int - ) -> None: + def create_table(self, gtfs_zip: zipfile.ZipFile, table_filename: str, version_key: int) -> None: """ read a csv table out of a gtfs static schedule file, add a timestamp column to each row, and write it as a parquet file on s3, partitioned diff --git a/src/lamp_py/ingestion/convert_gtfs_rt.py b/src/lamp_py/ingestion/convert_gtfs_rt.py index 70dce1f9..29424c64 100644 --- a/src/lamp_py/ingestion/convert_gtfs_rt.py +++ b/src/lamp_py/ingestion/convert_gtfs_rt.py @@ -84,9 +84,7 @@ class GtfsRtConverter(Converter): https_mbta_integration.mybluemix.net_vehicleCount.gz """ - def __init__( - self, config_type: ConfigType, metadata_queue: Queue[Optional[str]] - ) -> None: + def __init__(self, config_type: ConfigType, metadata_queue: Queue[Optional[str]]) -> None: Converter.__init__(self, config_type, metadata_queue) # Depending on filename, assign self.details to correct implementation @@ -173,12 +171,8 @@ def process_files(self) -> Iterable[pyarrow.table]: ) process_logger.log_start() - with ThreadPoolExecutor( - max_workers=max_workers, initializer=self.thread_init - ) as pool: - for result_dt, result_filename, rt_data in pool.map( - self.gz_to_pyarrow, self.files - ): + with ThreadPoolExecutor(max_workers=max_workers, initializer=self.thread_init) as pool: + for result_dt, result_filename, rt_data in pool.map(self.gz_to_pyarrow, self.files): # errors in gtfs_rt conversions are handled in the gz_to_pyarrow # function. if one is encountered, the datetime will be none. log # the error and move on to the next file. @@ -200,9 +194,7 @@ def process_files(self) -> Iterable[pyarrow.table]: # create new self.table_groups entry for key if it doesn't exist if dt_part not in self.data_parts: self.data_parts[dt_part] = TableData() - self.data_parts[dt_part].table = ( - self.detail.transform_for_write(rt_data) - ) + self.data_parts[dt_part].table = self.detail.transform_for_write(rt_data) else: self.data_parts[dt_part].table = pyarrow.concat_tables( [ @@ -221,9 +213,7 @@ def process_files(self) -> Iterable[pyarrow.table]: process_logger.add_metadata(file_count=0, number_of_rows=0) process_logger.log_complete() - def yield_check( - self, process_logger: ProcessLogger, min_rows: int = 2_000_000 - ) -> Iterable[pyarrow.table]: + def yield_check(self, process_logger: ProcessLogger, min_rows: int = 2_000_000) -> Iterable[pyarrow.table]: """ yield all tables in the data_parts map that have been sufficiently processed. @@ -245,17 +235,13 @@ def yield_check( ) process_logger.log_complete() # reset process logger - process_logger.add_metadata( - file_count=0, number_of_rows=0, print_log=False - ) + process_logger.add_metadata(file_count=0, number_of_rows=0, print_log=False) process_logger.log_start() yield table del self.data_parts[iter_ts] - def gz_to_pyarrow( - self, filename: str - ) -> Tuple[Optional[datetime], str, Optional[pyarrow.table]]: + def gz_to_pyarrow(self, filename: str) -> Tuple[Optional[datetime], str, Optional[pyarrow.table]]: """ Convert a gzipped json of gtfs realtime data into a pyarrow table. This function is executed inside of a thread, so all exceptions must be @@ -284,42 +270,30 @@ def gz_to_pyarrow( with file_system.open_input_stream(filename) as file: json_data = json.load(file) except UnicodeDecodeError as _: - with file_system.open_input_stream( - filename, compression="gzip" - ) as file: + with file_system.open_input_stream(filename, compression="gzip") as file: json_data = json.load(file) # parse timestamp info out of the header feed_timestamp = json_data["header"]["timestamp"] timestamp = datetime.fromtimestamp(feed_timestamp, timezone.utc) - table = pyarrow.Table.from_pylist( - json_data["entity"], schema=self.detail.import_schema - ) + table = pyarrow.Table.from_pylist(json_data["entity"], schema=self.detail.import_schema) table = table.append_column( "year", - pyarrow.array( - [timestamp.year] * table.num_rows, pyarrow.uint16() - ), + pyarrow.array([timestamp.year] * table.num_rows, pyarrow.uint16()), ) table = table.append_column( "month", - pyarrow.array( - [timestamp.month] * table.num_rows, pyarrow.uint8() - ), + pyarrow.array([timestamp.month] * table.num_rows, pyarrow.uint8()), ) table = table.append_column( "day", - pyarrow.array( - [timestamp.day] * table.num_rows, pyarrow.uint8() - ), + pyarrow.array([timestamp.day] * table.num_rows, pyarrow.uint8()), ) table = table.append_column( "feed_timestamp", - pyarrow.array( - [feed_timestamp] * table.num_rows, pyarrow.uint64() - ), + pyarrow.array([feed_timestamp] * table.num_rows, pyarrow.uint64()), ) except FileNotFoundError as _: @@ -386,9 +360,7 @@ def sync_with_s3(self, local_path: str) -> bool: return False - def make_hash_dataset( - self, table: pyarrow.Table, local_path: str - ) -> pyarrow.dataset: + def make_hash_dataset(self, table: pyarrow.Table, local_path: str) -> pyarrow.dataset: """ create dataset, with hash column, that will be written to parquet file @@ -418,20 +390,14 @@ def write_local_pq(self, table: pyarrow.Table, local_path: str) -> None: :param table: pyarrow Table :param local_path: path to local parquet file """ - logger = ProcessLogger( - "write_local_pq", local_path=local_path, table_rows=table.num_rows - ) + logger = ProcessLogger("write_local_pq", local_path=local_path, table_rows=table.num_rows) logger.log_start() out_ds = self.make_hash_dataset(table, local_path) - unique_ts_min = pc.min(table.column("feed_timestamp")).as_py() - ( - 60 * 45 - ) + unique_ts_min = pc.min(table.column("feed_timestamp")).as_py() - (60 * 45) - no_hash_schema = out_ds.schema.remove( - out_ds.schema.get_field_index(GTFS_RT_HASH_COL) - ) + no_hash_schema = out_ds.schema.remove(out_ds.schema.get_field_index(GTFS_RT_HASH_COL)) with tempfile.TemporaryDirectory() as temp_dir: hash_pq_path = os.path.join(temp_dir, "hash.parquet") @@ -440,9 +406,7 @@ def write_local_pq(self, table: pyarrow.Table, local_path: str) -> None: upload_writer = pq.ParquetWriter(upload_path, schema=no_hash_schema) partitions = pc.unique( - out_ds.to_table(columns=[self.detail.partition_column]).column( - self.detail.partition_column - ) + out_ds.to_table(columns=[self.detail.partition_column]).column(self.detail.partition_column) ) for part in partitions: unique_table = ( @@ -461,20 +425,15 @@ def write_local_pq(self, table: pyarrow.Table, local_path: str) -> None: ) ds_table = out_ds.to_table( filter=( - (pc.field(self.detail.partition_column) == part) - & (pc.field("feed_timestamp") < unique_ts_min) + (pc.field(self.detail.partition_column) == part) & (pc.field("feed_timestamp") < unique_ts_min) ) ) - write_table = pyarrow.concat_tables( - [unique_table, ds_table] - ).sort_by(self.detail.table_sort_order) + write_table = pyarrow.concat_tables([unique_table, ds_table]).sort_by(self.detail.table_sort_order) hash_writer.write_table(write_table) # drop GTFS_RT_HASH_COL column for S3 upload - upload_writer.write_table( - write_table.drop_columns(GTFS_RT_HASH_COL) - ) + upload_writer.write_table(write_table.drop_columns(GTFS_RT_HASH_COL)) hash_writer.close() upload_writer.close() @@ -513,9 +472,7 @@ def continuous_pq_update(self, table: pyarrow.Table) -> None: log.add_metadata(local_path=local_path) self.write_local_pq(table, local_path) - self.send_metadata( - local_path.replace(self.tmp_folder, S3_SPRINGBOARD) - ) + self.send_metadata(local_path.replace(self.tmp_folder, S3_SPRINGBOARD)) log.log_complete() @@ -546,11 +503,7 @@ def clean_local_folders(self) -> None: for w_dir, _, files in os.walk(root_folder): if len(files) == 0: continue - paths[ - datetime.strptime( - w_dir, f"{root_folder}/year=%Y/month=%m/day=%d" - ) - ] = w_dir + paths[datetime.strptime(w_dir, f"{root_folder}/year=%Y/month=%m/day=%d")] = w_dir # remove all local day folders except two most recent for key in sorted(paths.keys())[:-days_to_keep]: diff --git a/src/lamp_py/ingestion/converter.py b/src/lamp_py/ingestion/converter.py index fa998bca..14003ca8 100644 --- a/src/lamp_py/ingestion/converter.py +++ b/src/lamp_py/ingestion/converter.py @@ -99,9 +99,7 @@ class Converter(ABC): into pyarrow tables. """ - def __init__( - self, config_type: ConfigType, metadata_queue: Queue[Optional[str]] - ) -> None: + def __init__(self, config_type: ConfigType, metadata_queue: Queue[Optional[str]]) -> None: self.config_type = config_type self.files: List[str] = [] self.metadata_queue: Queue[Optional[str]] = metadata_queue diff --git a/src/lamp_py/ingestion/glides.py b/src/lamp_py/ingestion/glides.py index ea49a903..75897f74 100644 --- a/src/lamp_py/ingestion/glides.py +++ b/src/lamp_py/ingestion/glides.py @@ -55,9 +55,7 @@ def __init__(self, base_filename: str) -> None: self.base_filename = base_filename self.type = self.base_filename.replace(".parquet", "") self.local_path = os.path.join(self.tmp_dir, self.base_filename) - self.remote_path = ( - f"s3://{S3_SPRINGBOARD}/{LAMP}/GLIDES/{base_filename}" - ) + self.remote_path = f"s3://{S3_SPRINGBOARD}/{LAMP}/GLIDES/{base_filename}" self.records: List[Dict] = [] @@ -86,9 +84,7 @@ def convert_records(self) -> pd.Dataset: def append_records(self) -> None: """Add incoming records to a local parquet file""" - process_logger = ProcessLogger( - process_name="append_glides_records", type=self.type - ) + process_logger = ProcessLogger(process_name="append_glides_records", type=self.type) process_logger.log_start() new_dataset = self.convert_records() @@ -114,25 +110,15 @@ def append_records(self) -> None: end = start + relativedelta(months=1) if end < now: row_group = pl.DataFrame( - joined_ds.filter( - (pc.field("time") >= start) - & (pc.field("time") < end) - ).to_table() + joined_ds.filter((pc.field("time") >= start) & (pc.field("time") < end)).to_table() ) else: - row_group = pl.DataFrame( - joined_ds.filter( - (pc.field("time") >= start) - ).to_table() - ) + row_group = pl.DataFrame(joined_ds.filter((pc.field("time") >= start)).to_table()) if not row_group.is_empty(): unique_table = ( - row_group.unique(keep="first") - .sort(by=["time"]) - .to_arrow() - .cast(new_dataset.schema) + row_group.unique(keep="first").sort(by=["time"]).to_arrow().cast(new_dataset.schema) ) row_group_count += 1 @@ -198,14 +184,10 @@ def unique_key(self) -> str: return "changes" def convert_records(self) -> pd.Dataset: - process_logger = ProcessLogger( - process_name="convert_records", type=self.type - ) + process_logger = ProcessLogger(process_name="convert_records", type=self.type) process_logger.log_start() - editors_table = pyarrow.Table.from_pylist( - self.records, schema=self.event_schema - ) + editors_table = pyarrow.Table.from_pylist(self.records, schema=self.event_schema) editors_table = flatten_schema(editors_table) editors_table = explode_table_column(editors_table, "data.changes") editors_table = flatten_schema(editors_table) @@ -222,9 +204,7 @@ class OperatorSignIns(GlidesConverter): """ def __init__(self) -> None: - GlidesConverter.__init__( - self, base_filename="operator_sign_ins.parquet" - ) + GlidesConverter.__init__(self, base_filename="operator_sign_ins.parquet") @property def event_schema(self) -> pyarrow.schema: @@ -239,9 +219,7 @@ def event_schema(self) -> pyarrow.schema: ("metadata", self.glides_metadata), ( "operator", - pyarrow.struct( - [("badgeNumber", pyarrow.string())] - ), + pyarrow.struct([("badgeNumber", pyarrow.string())]), ), # a timestamp but it needs reformatting for pyarrow ("signedInAt", pyarrow.string()), @@ -271,13 +249,9 @@ def unique_key(self) -> str: return "operator" def convert_records(self) -> pd.Dataset: - process_logger = ProcessLogger( - process_name="convert_records", type=self.type - ) + process_logger = ProcessLogger(process_name="convert_records", type=self.type) process_logger.log_start() - osi_table = pyarrow.Table.from_pylist( - self.records, schema=self.event_schema - ) + osi_table = pyarrow.Table.from_pylist(self.records, schema=self.event_schema) osi_table = flatten_schema(osi_table) osi_dataset = pd.dataset(osi_table) @@ -373,15 +347,11 @@ def flatten_multitypes(record: Dict) -> Dict: return record - process_logger = ProcessLogger( - process_name="convert_records", type=self.type - ) + process_logger = ProcessLogger(process_name="convert_records", type=self.type) process_logger.log_start() modified_records = [flatten_multitypes(r) for r in self.records] - tu_table = pyarrow.Table.from_pylist( - modified_records, schema=self.event_schema - ) + tu_table = pyarrow.Table.from_pylist(modified_records, schema=self.event_schema) tu_table = flatten_schema(tu_table) tu_table = explode_table_column(tu_table, "data.tripUpdates") tu_table = flatten_schema(tu_table) @@ -391,9 +361,7 @@ def flatten_multitypes(record: Dict) -> Dict: return tu_dataset -def ingest_glides_events( - kinesis_reader: KinesisReader, metadata_queue: Queue[Optional[str]] -) -> None: +def ingest_glides_events(kinesis_reader: KinesisReader, metadata_queue: Queue[Optional[str]]) -> None: """ ingest glides records from the kinesis stream and add them to parquet files """ @@ -410,9 +378,7 @@ def ingest_glides_events( for record in kinesis_reader.get_records(): try: # format this so it can be used to partition parquet files - record["time"] = datetime.fromisoformat( - record["time"].replace("Z", "+00:00") - ) + record["time"] = datetime.fromisoformat(record["time"].replace("Z", "+00:00")) data_keys = record["data"].keys() diff --git a/src/lamp_py/ingestion/ingest_gtfs.py b/src/lamp_py/ingestion/ingest_gtfs.py index 5803dcc2..2eee11f1 100644 --- a/src/lamp_py/ingestion/ingest_gtfs.py +++ b/src/lamp_py/ingestion/ingest_gtfs.py @@ -95,18 +95,14 @@ def ingest_s3_files(metadata_queue: Queue[Optional[str]]) -> None: try: config_type = ConfigType.from_filename(file_group[0]) if config_type not in converters: - converters[config_type] = GtfsRtConverter( - config_type, metadata_queue - ) + converters[config_type] = GtfsRtConverter(config_type, metadata_queue) converters[config_type].add_files(file_group) except IgnoreIngestion: continue except (ConfigTypeFromFilenameException, NoImplException): error_files += file_group - converters[ConfigType.ERROR] = NoImplConverter( - ConfigType.ERROR, metadata_queue - ) + converters[ConfigType.ERROR] = NoImplConverter(ConfigType.ERROR, metadata_queue) converters[ConfigType.ERROR].add_files(error_files) except Exception as exception: diff --git a/src/lamp_py/ingestion/light_rail_gps.py b/src/lamp_py/ingestion/light_rail_gps.py index b9b157e9..d97bb44a 100644 --- a/src/lamp_py/ingestion/light_rail_gps.py +++ b/src/lamp_py/ingestion/light_rail_gps.py @@ -69,13 +69,7 @@ def thread_gps_to_frame(path: str) -> Tuple[Optional[pl.DataFrame], str]: .select( pl.col("serial_number").cast(pl.String), pl.col("data").struct.field("speed").cast(pl.Float64), - ( - pl.col("data") - .struct.field("updated_at") - .str.slice(0, length=10) - .str.to_date() - .alias("date") - ), + (pl.col("data").struct.field("updated_at").str.slice(0, length=10).str.to_date().alias("date")), pl.col("data").struct.field("updated_at").cast(pl.String), pl.col("data").struct.field("bearing").cast(pl.Float64), pl.col("data").struct.field("latitude").cast(pl.String), @@ -115,9 +109,7 @@ def dataframe_from_gz( archive_files: correctly loaded gzip paths error_files: gzip paths that failed to load """ - logger = ProcessLogger( - process_name="light_rail_df_from_gz", num_files=len(files) - ) + logger = ProcessLogger(process_name="light_rail_df_from_gz", num_files=len(files)) logger.log_start() init_file = files[0] @@ -126,9 +118,7 @@ def dataframe_from_gz( error_files = [] try: - with ThreadPoolExecutor( - max_workers=16, initializer=thread_init, initargs=(init_file,) - ) as pool: + with ThreadPoolExecutor(max_workers=16, initializer=thread_init, initargs=(init_file,)) as pool: for df, path in pool.map(thread_gps_to_frame, files): if df is not None: archive_files.append(path) @@ -160,9 +150,7 @@ def write_parquet(dataframe: pl.DataFrame) -> None: will merge any existing parquet files in memory """ for date in dataframe.get_column("date").unique(): - logger = ProcessLogger( - process_name="light_rail_write_parquet", date=date - ) + logger = ProcessLogger(process_name="light_rail_write_parquet", date=date) logger.log_start() remote_obj = os.path.join( @@ -175,9 +163,7 @@ def write_parquet(dataframe: pl.DataFrame) -> None: day_frame = dataframe.filter(pl.col("date") == date) if object_exists(remote_obj): - day_frame = pl.concat( - [day_frame, pl.read_parquet(f"s3://{remote_obj}")] - ) + day_frame = pl.concat([day_frame, pl.read_parquet(f"s3://{remote_obj}")]) day_frame = day_frame.unique().sort(by=["serial_number", "updated_at"]) diff --git a/src/lamp_py/ingestion/utils.py b/src/lamp_py/ingestion/utils.py index 6026265e..371cf285 100644 --- a/src/lamp_py/ingestion/utils.py +++ b/src/lamp_py/ingestion/utils.py @@ -98,9 +98,7 @@ def date_from_feed_version(feed_version: str) -> datetime.datetime: if pattern_1_result is not None: date_str = pattern_1_result.group(0) - date_dt = datetime.datetime.fromisoformat(date_str).replace( - tzinfo=utc_tz - ) + date_dt = datetime.datetime.fromisoformat(date_str).replace(tzinfo=utc_tz) date_dt = date_dt.astimezone(local_tz).replace(tzinfo=None) elif pattern_2_result is not None: date_str = pattern_2_result.group(0) @@ -143,21 +141,14 @@ def ordered_schedule_frame() -> pl.DataFrame: # Accept-Encoding header required to avoid cloudfront cache-hit req = request.Request(archive_url, headers={"Accept-Encoding": "gzip"}) with request.urlopen(req) as res: - feed = pl.read_csv( - res.read(), columns=feed_columns, schema_overrides=feed_dtypes - ) + feed = pl.read_csv(res.read(), columns=feed_columns, schema_overrides=feed_dtypes) feed = ( feed.with_columns( - pl.col("feed_version") - .map_elements(date_from_feed_version, pl.Datetime) - .alias("published_dt"), + pl.col("feed_version").map_elements(date_from_feed_version, pl.Datetime).alias("published_dt"), ) .with_columns( - pl.col("published_dt") - .dt.strftime("%Y%m%d") - .cast(pl.Int32) - .alias("published_date"), + pl.col("published_dt").dt.strftime("%Y%m%d").cast(pl.Int32).alias("published_date"), ) .sort( by=["feed_start_date", "published_dt"], @@ -206,14 +197,10 @@ def explode_table_column(table: pyarrow.table, column: str) -> pyarrow.table: table.select(other_columns) .take(indices) .append_column( - pyarrow.field( - column, table.schema.field(column).type.value_type - ), + pyarrow.field(column, table.schema.field(column).type.value_type), pc.list_flatten(table[column]), ), - table.filter(pc.list_value_length(table[column]).is_null()).select( - other_columns - ), + table.filter(pc.list_value_length(table[column]).is_null()).select(other_columns), ], promote_options="default", ) @@ -235,9 +222,7 @@ def hash_gtfs_rt_table(table: pyarrow.Table) -> pyarrow.Table: hash_columns.remove("feed_timestamp") hash_columns = sorted(hash_columns) - hash_schema = table.schema.append( - pyarrow.field(GTFS_RT_HASH_COL, pyarrow.large_binary()) - ) + hash_schema = table.schema.append(pyarrow.field(GTFS_RT_HASH_COL, pyarrow.large_binary())) table = pl.from_arrow(table) @@ -265,9 +250,7 @@ def hash_gtfs_rt_parquet(path: str) -> None: hash_columns.remove("feed_timestamp") hash_columns = sorted(hash_columns) - hash_schema = ds.schema.append( - pyarrow.field(GTFS_RT_HASH_COL, pyarrow.large_binary()) - ) + hash_schema = ds.schema.append(pyarrow.field(GTFS_RT_HASH_COL, pyarrow.large_binary())) with tempfile.TemporaryDirectory() as temp_dir: tmp_pq = os.path.join(temp_dir, "temp.parquet") @@ -296,9 +279,7 @@ def gzip_file(path: str, keep_original: bool = False) -> None: :param path: local file path :param keep_original: keep original non-gzip file = False """ - logger = ProcessLogger( - "gzip_file", path=path, remove_original=keep_original - ) + logger = ProcessLogger("gzip_file", path=path, remove_original=keep_original) logger.log_start() with open(path, "rb") as f_in: with gzip.open(f"{path}.gz", "wb") as f_out: diff --git a/src/lamp_py/ingestion_tm/jobs/parition_table.py b/src/lamp_py/ingestion_tm/jobs/parition_table.py index 3276b8aa..9ec9f562 100644 --- a/src/lamp_py/ingestion_tm/jobs/parition_table.py +++ b/src/lamp_py/ingestion_tm/jobs/parition_table.py @@ -65,9 +65,7 @@ def dates_from_tm(self, tm_db: MSSQLManager) -> List[int]: :return [120240101, 120240102] """ - tm_dates_query = sa.text( - f"SELECT DISTINCT CALENDAR_ID FROM {self.tm_table};" - ) + tm_dates_query = sa.text(f"SELECT DISTINCT CALENDAR_ID FROM {self.tm_table};") tm_dates = tm_db.select_as_dataframe(tm_dates_query) return sorted(tm_dates["CALENDAR_ID"].astype(int).to_list()) @@ -80,9 +78,7 @@ def dates_from_s3(self) -> List[int]: :return [120240101, 120240102] """ - s3_files = file_list_from_s3( - self.s3_location.bucket, self.s3_location.prefix - ) + s3_files = file_list_from_s3(self.s3_location.bucket, self.s3_location.prefix) def date_match(s: str) -> Optional[int]: match = re.search(r"\d{9}", s) @@ -110,9 +106,7 @@ def dates_to_export(self, tm_db: MSSQLManager) -> List[int]: os.path.join(self.s3_location.prefix, self.version_key), ) if len(version_file) == 1: - s3_version = object_metadata(self.s3_version_path).get( - self.version_key - ) + s3_version = object_metadata(self.s3_version_path).get(self.version_key) if s3_version != self.lamp_version: return tm_dates @@ -129,9 +123,7 @@ def run_export(self, tm_db: MSSQLManager) -> None: for date in self.dates_to_export(tm_db): try: - logger = ProcessLogger( - "tm_daily_log_export", tm_table=self.tm_table, date=date - ) + logger = ProcessLogger("tm_daily_log_export", tm_table=self.tm_table, date=date) logger.log_start() query = sa.text( diff --git a/src/lamp_py/ingestion_tm/jobs/whole_table.py b/src/lamp_py/ingestion_tm/jobs/whole_table.py index 8528cd33..ec52003f 100644 --- a/src/lamp_py/ingestion_tm/jobs/whole_table.py +++ b/src/lamp_py/ingestion_tm/jobs/whole_table.py @@ -45,12 +45,8 @@ def run_export(self, tm_db: MSSQLManager) -> None: try: with tempfile.TemporaryDirectory() as temp_dir: local_export_path = os.path.join(temp_dir, "out.parquet") - tm_db.write_to_parquet( - query, local_export_path, self.export_schema - ) - logger.add_metadata( - pq_export_bytes=os.stat(local_export_path).st_size - ) + tm_db.write_to_parquet(query, local_export_path, self.export_schema) + logger.add_metadata(pq_export_bytes=os.stat(local_export_path).st_size) upload_file(local_export_path, self.s3_location.s3_uri) logger.log_complete() diff --git a/src/lamp_py/migrations/versions/metadata_dev/002_26db393ea854_update_glides_location_column_names.py b/src/lamp_py/migrations/versions/metadata_dev/002_26db393ea854_update_glides_location_column_names.py index f958b17b..66da95dd 100644 --- a/src/lamp_py/migrations/versions/metadata_dev/002_26db393ea854_update_glides_location_column_names.py +++ b/src/lamp_py/migrations/versions/metadata_dev/002_26db393ea854_update_glides_location_column_names.py @@ -67,11 +67,7 @@ def update_glides_archive(temp_dir: str, base_filename: str) -> None: # unique the records # cast to new schema (polars converts things) new_table = ( - pl.DataFrame(old_table.rename_columns(schema.names)) - .unique() - .sort(by=["time"]) - .to_arrow() - .cast(schema) + pl.DataFrame(old_table.rename_columns(schema.names)).unique().sort(by=["time"]).to_arrow().cast(schema) ) pq.write_table(new_table, new_local_path) diff --git a/src/lamp_py/migrations/versions/metadata_prod/001_07903947aabe_initial_changes.py b/src/lamp_py/migrations/versions/metadata_prod/001_07903947aabe_initial_changes.py index 05c345b5..cb1fc622 100644 --- a/src/lamp_py/migrations/versions/metadata_prod/001_07903947aabe_initial_changes.py +++ b/src/lamp_py/migrations/versions/metadata_prod/001_07903947aabe_initial_changes.py @@ -51,17 +51,13 @@ def upgrade() -> None: # metadata database. the table may or may not exist, so wrap this in a try # except try: - rpm_db_manager = DatabaseManager( - db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER - ) + rpm_db_manager = DatabaseManager(db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER) insert_data = [] # pull metadata from the rail performance manager database via direct # sql query. the metadata_log table may or may not exist. with rpm_db_manager.session.begin() as session: - result = session.execute( - text("SELECT path, processed, process_fail FROM metadata_log") - ) + result = session.execute(text("SELECT path, processed, process_fail FROM metadata_log")) for row in result: (path, processed, process_fail) = row insert_data.append( @@ -79,11 +75,7 @@ def upgrade() -> None: # Raise all other sql errors insert_data = [] original_error = error.orig - if ( - original_error is not None - and hasattr(original_error, "pgcode") - and original_error.pgcode == "42P01" - ): + if original_error is not None and hasattr(original_error, "pgcode") and original_error.pgcode == "42P01": logging.info("No Metadata Table in Rail Performance Manager") else: raise diff --git a/src/lamp_py/migrations/versions/metadata_prod/003_26db393ea854_update_glides_location_column_names.py b/src/lamp_py/migrations/versions/metadata_prod/003_26db393ea854_update_glides_location_column_names.py index 6ce3de44..5da16d24 100644 --- a/src/lamp_py/migrations/versions/metadata_prod/003_26db393ea854_update_glides_location_column_names.py +++ b/src/lamp_py/migrations/versions/metadata_prod/003_26db393ea854_update_glides_location_column_names.py @@ -67,11 +67,7 @@ def update_glides_archive(temp_dir: str, base_filename: str) -> None: # unique the records # cast to new schema (polars converts things) new_table = ( - pl.DataFrame(old_table.rename_columns(schema.names)) - .unique() - .sort(by=["time"]) - .to_arrow() - .cast(schema) + pl.DataFrame(old_table.rename_columns(schema.names)).unique().sort(by=["time"]).to_arrow().cast(schema) ) pq.write_table(new_table, new_local_path) diff --git a/src/lamp_py/migrations/versions/metadata_staging/001_07903947aabe_initial_changes.py b/src/lamp_py/migrations/versions/metadata_staging/001_07903947aabe_initial_changes.py index 05c345b5..cb1fc622 100644 --- a/src/lamp_py/migrations/versions/metadata_staging/001_07903947aabe_initial_changes.py +++ b/src/lamp_py/migrations/versions/metadata_staging/001_07903947aabe_initial_changes.py @@ -51,17 +51,13 @@ def upgrade() -> None: # metadata database. the table may or may not exist, so wrap this in a try # except try: - rpm_db_manager = DatabaseManager( - db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER - ) + rpm_db_manager = DatabaseManager(db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER) insert_data = [] # pull metadata from the rail performance manager database via direct # sql query. the metadata_log table may or may not exist. with rpm_db_manager.session.begin() as session: - result = session.execute( - text("SELECT path, processed, process_fail FROM metadata_log") - ) + result = session.execute(text("SELECT path, processed, process_fail FROM metadata_log")) for row in result: (path, processed, process_fail) = row insert_data.append( @@ -79,11 +75,7 @@ def upgrade() -> None: # Raise all other sql errors insert_data = [] original_error = error.orig - if ( - original_error is not None - and hasattr(original_error, "pgcode") - and original_error.pgcode == "42P01" - ): + if original_error is not None and hasattr(original_error, "pgcode") and original_error.pgcode == "42P01": logging.info("No Metadata Table in Rail Performance Manager") else: raise diff --git a/src/lamp_py/migrations/versions/metadata_staging/002_26db393ea854_update_glides_location_column_names.py b/src/lamp_py/migrations/versions/metadata_staging/002_26db393ea854_update_glides_location_column_names.py index f958b17b..66da95dd 100644 --- a/src/lamp_py/migrations/versions/metadata_staging/002_26db393ea854_update_glides_location_column_names.py +++ b/src/lamp_py/migrations/versions/metadata_staging/002_26db393ea854_update_glides_location_column_names.py @@ -67,11 +67,7 @@ def update_glides_archive(temp_dir: str, base_filename: str) -> None: # unique the records # cast to new schema (polars converts things) new_table = ( - pl.DataFrame(old_table.rename_columns(schema.names)) - .unique() - .sort(by=["time"]) - .to_arrow() - .cast(schema) + pl.DataFrame(old_table.rename_columns(schema.names)).unique().sort(by=["time"]).to_arrow().cast(schema) ) pq.write_table(new_table, new_local_path) diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/001_5d9a7ee21ae5_initial_prod_schema.py b/src/lamp_py/migrations/versions/performance_manager_dev/001_5d9a7ee21ae5_initial_prod_schema.py index 1cce5f58..7ef6ba67 100644 --- a/src/lamp_py/migrations/versions/performance_manager_dev/001_5d9a7ee21ae5_initial_prod_schema.py +++ b/src/lamp_py/migrations/versions/performance_manager_dev/001_5d9a7ee21ae5_initial_prod_schema.py @@ -101,9 +101,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("direction", sa.String(length=30), nullable=False), - sa.Column( - "direction_destination", sa.String(length=60), nullable=False - ), + sa.Column("direction_destination", sa.String(length=60), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -162,12 +160,8 @@ def upgrade() -> None: sa.Column("arrival_time", sa.Integer(), nullable=False), sa.Column("departure_time", sa.Integer(), nullable=False), sa.Column("schedule_travel_time_seconds", sa.Integer(), nullable=True), - sa.Column( - "schedule_headway_trunk_seconds", sa.Integer(), nullable=True - ), - sa.Column( - "schedule_headway_branch_seconds", sa.Integer(), nullable=True - ), + sa.Column("schedule_headway_trunk_seconds", sa.Integer(), nullable=True), + sa.Column("schedule_headway_branch_seconds", sa.Integer(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("stop_sequence", sa.SmallInteger(), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), @@ -249,9 +243,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("route_pattern_typicality", sa.SmallInteger(), nullable=True), - sa.Column( - "representative_trip_id", sa.String(length=128), nullable=False - ), + sa.Column("representative_trip_id", sa.String(length=128), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -297,9 +289,7 @@ def upgrade() -> None: sa.Column("sync_stop_sequence", sa.SmallInteger(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("parent_station", sa.String(length=60), nullable=False), - sa.Column( - "previous_trip_stop_pm_event_id", sa.Integer(), nullable=True - ), + sa.Column("previous_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("next_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("vp_move_timestamp", sa.Integer(), nullable=True), sa.Column("vp_stop_timestamp", sa.Integer(), nullable=True), @@ -333,9 +323,7 @@ def upgrade() -> None: "vehicle_events", ["pm_event_id"], unique=False, - postgresql_where=sa.text( - "vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL" - ), + postgresql_where=sa.text("vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL"), ) op.create_table( @@ -448,37 +436,25 @@ def upgrade() -> None: def downgrade() -> None: - drop_opmi_all_rt_fields_joined = ( - "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" - ) + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" op.execute(drop_opmi_all_rt_fields_joined) - drop_static_service_id_lookup = ( - "DROP VIEW IF EXISTS static_service_id_lookup;" - ) + drop_static_service_id_lookup = "DROP VIEW IF EXISTS static_service_id_lookup;" op.execute(drop_static_service_id_lookup) - drop_service_id_by_date_and_route = ( - "DROP VIEW IF EXISTS service_id_by_date_and_route;" - ) + drop_service_id_by_date_and_route = "DROP VIEW IF EXISTS service_id_by_date_and_route;" op.execute(drop_service_id_by_date_and_route) - drop_trigger = ( - "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" op.execute(drop_trigger) - drop_function = ( - "DROP function if EXISTS public.update_rt_branch_trunk_id();" - ) + drop_function = "DROP function if EXISTS public.update_rt_branch_trunk_id();" op.execute(drop_function) drop_function = "DROP function if EXISTS public.rt_red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function if EXISTS public.rt_red_is_braintree_branch();" - ) + drop_function = "DROP function if EXISTS public.rt_red_is_braintree_branch();" op.execute(drop_function) drop_trigger = "DROP TRIGGER IF EXISTS static_trips_create_branch_trunk ON static_trips;" @@ -490,14 +466,10 @@ def downgrade() -> None: drop_function = "DROP function IF EXISTS public.red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" - ) + drop_function = "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" op.execute(drop_function) - drop_trigger = ( - "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" op.execute(drop_trigger) drop_function = "DROP function IF EXISTS public.insert_feed_info();" op.execute(drop_function) @@ -510,9 +482,7 @@ def downgrade() -> None: drop_trigger = f"DROP TRIGGER IF EXISTS {trigger_name} on {table};" op.execute(drop_trigger) - drop_update_on_and_triggers = ( - "DROP function if EXISTS public.update_modified_columns();" - ) + drop_update_on_and_triggers = "DROP function if EXISTS public.update_modified_columns();" op.execute(drop_update_on_and_triggers) op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") @@ -540,12 +510,8 @@ def downgrade() -> None: op.drop_index("ix_static_stops_composite_1", table_name="static_stops") op.drop_table("static_stops") - op.drop_index( - "ix_static_stop_times_composite_2", table_name="static_stop_times" - ) - op.drop_index( - "ix_static_stop_times_composite_1", table_name="static_stop_times" - ) + op.drop_index("ix_static_stop_times_composite_2", table_name="static_stop_times") + op.drop_index("ix_static_stop_times_composite_1", table_name="static_stop_times") op.drop_table("static_stop_times") op.drop_index("ix_static_routes_composite_1", table_name="static_routes") @@ -553,9 +519,7 @@ def downgrade() -> None: op.drop_table("static_feed_info") - op.drop_index( - "ix_static_directions_composite_1", table_name="static_directions" - ) + op.drop_index("ix_static_directions_composite_1", table_name="static_directions") op.drop_table("static_directions") op.drop_index( @@ -564,9 +528,7 @@ def downgrade() -> None: ) op.drop_table("static_calendar_dates") - op.drop_index( - "ix_static_calendar_composite_1", table_name="static_calendar" - ) + op.drop_index("ix_static_calendar_composite_1", table_name="static_calendar") op.drop_table("static_calendar") op.drop_index("ix_metadata_log_not_processed", table_name="metadata_log") diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/005_96187da84955_remove_metadata.py b/src/lamp_py/migrations/versions/performance_manager_dev/005_96187da84955_remove_metadata.py index 84587e1d..f9f89fea 100644 --- a/src/lamp_py/migrations/versions/performance_manager_dev/005_96187da84955_remove_metadata.py +++ b/src/lamp_py/migrations/versions/performance_manager_dev/005_96187da84955_remove_metadata.py @@ -39,15 +39,9 @@ def downgrade() -> None: op.create_table( "metadata_log", sa.Column("pk_id", sa.INTEGER(), autoincrement=True, nullable=False), - sa.Column( - "processed", sa.BOOLEAN(), autoincrement=False, nullable=True - ), - sa.Column( - "process_fail", sa.BOOLEAN(), autoincrement=False, nullable=True - ), - sa.Column( - "path", sa.VARCHAR(length=256), autoincrement=False, nullable=False - ), + sa.Column("processed", sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column("process_fail", sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column("path", sa.VARCHAR(length=256), autoincrement=False, nullable=False), sa.Column( "created_on", postgresql.TIMESTAMP(timezone=True), diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/006_2dfbde5ec151_sync_stop_trunk.py b/src/lamp_py/migrations/versions/performance_manager_dev/006_2dfbde5ec151_sync_stop_trunk.py index 1a3f9d43..cbd6ae36 100644 --- a/src/lamp_py/migrations/versions/performance_manager_dev/006_2dfbde5ec151_sync_stop_trunk.py +++ b/src/lamp_py/migrations/versions/performance_manager_dev/006_2dfbde5ec151_sync_stop_trunk.py @@ -28,9 +28,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/007_896dedd8a4db_dwell_time_update.py b/src/lamp_py/migrations/versions/performance_manager_dev/007_896dedd8a4db_dwell_time_update.py index a13a266b..9e3975a8 100644 --- a/src/lamp_py/migrations/versions/performance_manager_dev/007_896dedd8a4db_dwell_time_update.py +++ b/src/lamp_py/migrations/versions/performance_manager_dev/007_896dedd8a4db_dwell_time_update.py @@ -27,9 +27,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_dev/008_32ba735d080c_add_revenue_columns.py b/src/lamp_py/migrations/versions/performance_manager_dev/008_32ba735d080c_add_revenue_columns.py index d6d7ebb1..3bf161c0 100644 --- a/src/lamp_py/migrations/versions/performance_manager_dev/008_32ba735d080c_add_revenue_columns.py +++ b/src/lamp_py/migrations/versions/performance_manager_dev/008_32ba735d080c_add_revenue_columns.py @@ -30,21 +30,13 @@ def upgrade() -> None: - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;") op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") op.drop_constraint("vehicle_trips_unique_trip", table_name="vehicle_trips") - op.add_column( - "temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True) - ) - op.add_column( - "vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True) - ) + op.add_column("temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True)) + op.add_column("vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True)) op.execute(sa.update(TempEventCompare).values(revenue=True)) op.execute(sa.update(VehicleTrips).values(revenue=True)) op.alter_column("temp_event_compare", "revenue", nullable=False) @@ -61,12 +53,8 @@ def upgrade() -> None: ["route_id", "direction_id", "vehicle_id"], unique=False, ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;") def downgrade() -> None: diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/001_5d9a7ee21ae5_initial_prod_schema.py b/src/lamp_py/migrations/versions/performance_manager_prod/001_5d9a7ee21ae5_initial_prod_schema.py index d1c5fbe2..b1738e26 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/001_5d9a7ee21ae5_initial_prod_schema.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/001_5d9a7ee21ae5_initial_prod_schema.py @@ -78,9 +78,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("direction", sa.String(length=30), nullable=False), - sa.Column( - "direction_destination", sa.String(length=60), nullable=False - ), + sa.Column("direction_destination", sa.String(length=60), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -139,12 +137,8 @@ def upgrade() -> None: sa.Column("arrival_time", sa.Integer(), nullable=False), sa.Column("departure_time", sa.Integer(), nullable=False), sa.Column("schedule_travel_time_seconds", sa.Integer(), nullable=True), - sa.Column( - "schedule_headway_trunk_seconds", sa.Integer(), nullable=True - ), - sa.Column( - "schedule_headway_branch_seconds", sa.Integer(), nullable=True - ), + sa.Column("schedule_headway_trunk_seconds", sa.Integer(), nullable=True), + sa.Column("schedule_headway_branch_seconds", sa.Integer(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("stop_sequence", sa.SmallInteger(), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), @@ -232,9 +226,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("route_pattern_typicality", sa.SmallInteger(), nullable=True), - sa.Column( - "representative_trip_id", sa.String(length=512), nullable=False - ), + sa.Column("representative_trip_id", sa.String(length=512), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -280,9 +272,7 @@ def upgrade() -> None: sa.Column("sync_stop_sequence", sa.SmallInteger(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("parent_station", sa.String(length=60), nullable=False), - sa.Column( - "previous_trip_stop_pm_event_id", sa.Integer(), nullable=True - ), + sa.Column("previous_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("next_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("vp_move_timestamp", sa.Integer(), nullable=True), sa.Column("vp_stop_timestamp", sa.Integer(), nullable=True), @@ -316,9 +306,7 @@ def upgrade() -> None: "vehicle_events", ["pm_event_id"], unique=False, - postgresql_where=sa.text( - "vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL" - ), + postgresql_where=sa.text("vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL"), ) op.create_table( @@ -431,37 +419,25 @@ def upgrade() -> None: def downgrade() -> None: - drop_opmi_all_rt_fields_joined = ( - "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" - ) + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" op.execute(drop_opmi_all_rt_fields_joined) - drop_static_service_id_lookup = ( - "DROP VIEW IF EXISTS static_service_id_lookup;" - ) + drop_static_service_id_lookup = "DROP VIEW IF EXISTS static_service_id_lookup;" op.execute(drop_static_service_id_lookup) - drop_service_id_by_date_and_route = ( - "DROP VIEW IF EXISTS service_id_by_date_and_route;" - ) + drop_service_id_by_date_and_route = "DROP VIEW IF EXISTS service_id_by_date_and_route;" op.execute(drop_service_id_by_date_and_route) - drop_trigger = ( - "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" op.execute(drop_trigger) - drop_function = ( - "DROP function if EXISTS public.update_rt_branch_trunk_id();" - ) + drop_function = "DROP function if EXISTS public.update_rt_branch_trunk_id();" op.execute(drop_function) drop_function = "DROP function if EXISTS public.rt_red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function if EXISTS public.rt_red_is_braintree_branch();" - ) + drop_function = "DROP function if EXISTS public.rt_red_is_braintree_branch();" op.execute(drop_function) drop_trigger = "DROP TRIGGER IF EXISTS static_trips_create_branch_trunk ON static_trips;" @@ -473,14 +449,10 @@ def downgrade() -> None: drop_function = "DROP function IF EXISTS public.red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" - ) + drop_function = "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" op.execute(drop_function) - drop_trigger = ( - "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" op.execute(drop_trigger) drop_function = "DROP function IF EXISTS public.insert_feed_info();" op.execute(drop_function) @@ -493,9 +465,7 @@ def downgrade() -> None: drop_trigger = f"DROP TRIGGER IF EXISTS {trigger_name} on {table};" op.execute(drop_trigger) - drop_update_on_and_triggers = ( - "DROP function if EXISTS public.update_modified_columns();" - ) + drop_update_on_and_triggers = "DROP function if EXISTS public.update_modified_columns();" op.execute(drop_update_on_and_triggers) op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") @@ -524,12 +494,8 @@ def downgrade() -> None: op.drop_index("ix_static_stops_composite_1", table_name="static_stops") op.drop_table("static_stops") - op.drop_index( - "ix_static_stop_times_composite_2", table_name="static_stop_times" - ) - op.drop_index( - "ix_static_stop_times_composite_1", table_name="static_stop_times" - ) + op.drop_index("ix_static_stop_times_composite_2", table_name="static_stop_times") + op.drop_index("ix_static_stop_times_composite_1", table_name="static_stop_times") op.drop_table("static_stop_times") op.drop_index("ix_static_routes_composite_1", table_name="static_routes") @@ -537,9 +503,7 @@ def downgrade() -> None: op.drop_table("static_feed_info") - op.drop_index( - "ix_static_directions_composite_1", table_name="static_directions" - ) + op.drop_index("ix_static_directions_composite_1", table_name="static_directions") op.drop_table("static_directions") op.drop_index( @@ -548,8 +512,6 @@ def downgrade() -> None: ) op.drop_table("static_calendar_dates") - op.drop_index( - "ix_static_calendar_composite_1", table_name="static_calendar" - ) + op.drop_index("ix_static_calendar_composite_1", table_name="static_calendar") op.drop_table("static_calendar") # ### end Alembic commands ### diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/002_f09e853d5672_update_prod_stop_sync.py b/src/lamp_py/migrations/versions/performance_manager_prod/002_f09e853d5672_update_prod_stop_sync.py index e408f623..858bab98 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/002_f09e853d5672_update_prod_stop_sync.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/002_f09e853d5672_update_prod_stop_sync.py @@ -28,9 +28,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/003_2dfbde5ec151_sync_stop_trunk.py b/src/lamp_py/migrations/versions/performance_manager_prod/003_2dfbde5ec151_sync_stop_trunk.py index fd457ae8..72634fd8 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/003_2dfbde5ec151_sync_stop_trunk.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/003_2dfbde5ec151_sync_stop_trunk.py @@ -28,9 +28,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/004_896dedd8a4db_dwell_time_update.py b/src/lamp_py/migrations/versions/performance_manager_prod/004_896dedd8a4db_dwell_time_update.py index a13a266b..9e3975a8 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/004_896dedd8a4db_dwell_time_update.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/004_896dedd8a4db_dwell_time_update.py @@ -27,9 +27,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/005_32ba735d080c_add_revenue_columns.py b/src/lamp_py/migrations/versions/performance_manager_prod/005_32ba735d080c_add_revenue_columns.py index d6d7ebb1..3bf161c0 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/005_32ba735d080c_add_revenue_columns.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/005_32ba735d080c_add_revenue_columns.py @@ -30,21 +30,13 @@ def upgrade() -> None: - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;") op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") op.drop_constraint("vehicle_trips_unique_trip", table_name="vehicle_trips") - op.add_column( - "temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True) - ) - op.add_column( - "vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True) - ) + op.add_column("temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True)) + op.add_column("vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True)) op.execute(sa.update(TempEventCompare).values(revenue=True)) op.execute(sa.update(VehicleTrips).values(revenue=True)) op.alter_column("temp_event_compare", "revenue", nullable=False) @@ -61,12 +53,8 @@ def upgrade() -> None: ["route_id", "direction_id", "vehicle_id"], unique=False, ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;") def downgrade() -> None: diff --git a/src/lamp_py/migrations/versions/performance_manager_prod/sql_strings/strings_001.py b/src/lamp_py/migrations/versions/performance_manager_prod/sql_strings/strings_001.py index 50bb725b..3acb676d 100644 --- a/src/lamp_py/migrations/versions/performance_manager_prod/sql_strings/strings_001.py +++ b/src/lamp_py/migrations/versions/performance_manager_prod/sql_strings/strings_001.py @@ -15,7 +15,9 @@ ashmond_stop_ids = "('70087', '70088', '70089', '70090', '70091', '70092', '70093', '70094', '70085', '70086')" -braintree_stop_ids = "('70097', '70098', '70099', '70100', '70101', '70102', '70103', '70104', '70105', '70095', '70096')" +braintree_stop_ids = ( + "('70097', '70098', '70099', '70100', '70101', '70102', '70103', '70104', '70105', '70095', '70096')" +) func_red_is_ashmont_branch = f""" diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/001_5d9a7ee21ae5_initial_prod_schema.py b/src/lamp_py/migrations/versions/performance_manager_staging/001_5d9a7ee21ae5_initial_prod_schema.py index 1cce5f58..7ef6ba67 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/001_5d9a7ee21ae5_initial_prod_schema.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/001_5d9a7ee21ae5_initial_prod_schema.py @@ -101,9 +101,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("direction", sa.String(length=30), nullable=False), - sa.Column( - "direction_destination", sa.String(length=60), nullable=False - ), + sa.Column("direction_destination", sa.String(length=60), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -162,12 +160,8 @@ def upgrade() -> None: sa.Column("arrival_time", sa.Integer(), nullable=False), sa.Column("departure_time", sa.Integer(), nullable=False), sa.Column("schedule_travel_time_seconds", sa.Integer(), nullable=True), - sa.Column( - "schedule_headway_trunk_seconds", sa.Integer(), nullable=True - ), - sa.Column( - "schedule_headway_branch_seconds", sa.Integer(), nullable=True - ), + sa.Column("schedule_headway_trunk_seconds", sa.Integer(), nullable=True), + sa.Column("schedule_headway_branch_seconds", sa.Integer(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("stop_sequence", sa.SmallInteger(), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), @@ -249,9 +243,7 @@ def upgrade() -> None: sa.Column("route_id", sa.String(length=60), nullable=False), sa.Column("direction_id", sa.Boolean(), nullable=True), sa.Column("route_pattern_typicality", sa.SmallInteger(), nullable=True), - sa.Column( - "representative_trip_id", sa.String(length=128), nullable=False - ), + sa.Column("representative_trip_id", sa.String(length=128), nullable=False), sa.Column("static_version_key", sa.Integer(), nullable=False), sa.PrimaryKeyConstraint("pk_id"), ) @@ -297,9 +289,7 @@ def upgrade() -> None: sa.Column("sync_stop_sequence", sa.SmallInteger(), nullable=True), sa.Column("stop_id", sa.String(length=60), nullable=False), sa.Column("parent_station", sa.String(length=60), nullable=False), - sa.Column( - "previous_trip_stop_pm_event_id", sa.Integer(), nullable=True - ), + sa.Column("previous_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("next_trip_stop_pm_event_id", sa.Integer(), nullable=True), sa.Column("vp_move_timestamp", sa.Integer(), nullable=True), sa.Column("vp_stop_timestamp", sa.Integer(), nullable=True), @@ -333,9 +323,7 @@ def upgrade() -> None: "vehicle_events", ["pm_event_id"], unique=False, - postgresql_where=sa.text( - "vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL" - ), + postgresql_where=sa.text("vp_move_timestamp IS NOT NULL OR vp_stop_timestamp IS NOT NULL"), ) op.create_table( @@ -448,37 +436,25 @@ def upgrade() -> None: def downgrade() -> None: - drop_opmi_all_rt_fields_joined = ( - "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" - ) + drop_opmi_all_rt_fields_joined = "DROP VIEW IF EXISTS opmi_all_rt_fields_joined;" op.execute(drop_opmi_all_rt_fields_joined) - drop_static_service_id_lookup = ( - "DROP VIEW IF EXISTS static_service_id_lookup;" - ) + drop_static_service_id_lookup = "DROP VIEW IF EXISTS static_service_id_lookup;" op.execute(drop_static_service_id_lookup) - drop_service_id_by_date_and_route = ( - "DROP VIEW IF EXISTS service_id_by_date_and_route;" - ) + drop_service_id_by_date_and_route = "DROP VIEW IF EXISTS service_id_by_date_and_route;" op.execute(drop_service_id_by_date_and_route) - drop_trigger = ( - "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS rt_trips_update_branch_trunk ON vehicle_trips;" op.execute(drop_trigger) - drop_function = ( - "DROP function if EXISTS public.update_rt_branch_trunk_id();" - ) + drop_function = "DROP function if EXISTS public.update_rt_branch_trunk_id();" op.execute(drop_function) drop_function = "DROP function if EXISTS public.rt_red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function if EXISTS public.rt_red_is_braintree_branch();" - ) + drop_function = "DROP function if EXISTS public.rt_red_is_braintree_branch();" op.execute(drop_function) drop_trigger = "DROP TRIGGER IF EXISTS static_trips_create_branch_trunk ON static_trips;" @@ -490,14 +466,10 @@ def downgrade() -> None: drop_function = "DROP function IF EXISTS public.red_is_ashmont_branch();" op.execute(drop_function) - drop_function = ( - "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" - ) + drop_function = "DROP function IF EXISTS public.insert_static_trips_branch_trunk();" op.execute(drop_function) - drop_trigger = ( - "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" - ) + drop_trigger = "DROP TRIGGER IF EXISTS insert_into_feed_info ON static_feed_info;" op.execute(drop_trigger) drop_function = "DROP function IF EXISTS public.insert_feed_info();" op.execute(drop_function) @@ -510,9 +482,7 @@ def downgrade() -> None: drop_trigger = f"DROP TRIGGER IF EXISTS {trigger_name} on {table};" op.execute(drop_trigger) - drop_update_on_and_triggers = ( - "DROP function if EXISTS public.update_modified_columns();" - ) + drop_update_on_and_triggers = "DROP function if EXISTS public.update_modified_columns();" op.execute(drop_update_on_and_triggers) op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") @@ -540,12 +510,8 @@ def downgrade() -> None: op.drop_index("ix_static_stops_composite_1", table_name="static_stops") op.drop_table("static_stops") - op.drop_index( - "ix_static_stop_times_composite_2", table_name="static_stop_times" - ) - op.drop_index( - "ix_static_stop_times_composite_1", table_name="static_stop_times" - ) + op.drop_index("ix_static_stop_times_composite_2", table_name="static_stop_times") + op.drop_index("ix_static_stop_times_composite_1", table_name="static_stop_times") op.drop_table("static_stop_times") op.drop_index("ix_static_routes_composite_1", table_name="static_routes") @@ -553,9 +519,7 @@ def downgrade() -> None: op.drop_table("static_feed_info") - op.drop_index( - "ix_static_directions_composite_1", table_name="static_directions" - ) + op.drop_index("ix_static_directions_composite_1", table_name="static_directions") op.drop_table("static_directions") op.drop_index( @@ -564,9 +528,7 @@ def downgrade() -> None: ) op.drop_table("static_calendar_dates") - op.drop_index( - "ix_static_calendar_composite_1", table_name="static_calendar" - ) + op.drop_index("ix_static_calendar_composite_1", table_name="static_calendar") op.drop_table("static_calendar") op.drop_index("ix_metadata_log_not_processed", table_name="metadata_log") diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/005_96187da84955_remove_metadata.py b/src/lamp_py/migrations/versions/performance_manager_staging/005_96187da84955_remove_metadata.py index 7840e2bc..6a239a26 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/005_96187da84955_remove_metadata.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/005_96187da84955_remove_metadata.py @@ -31,22 +31,14 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### while True: try: - rpm_db_manager = DatabaseManager( - db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER - ) + rpm_db_manager = DatabaseManager(db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER) md_db_manager = DatabaseManager(db_index=DatabaseIndex.METADATA) with rpm_db_manager.session.begin() as session: - legacy_result = session.execute( - text("SELECT path FROM metadata_log") - ) - legacy_paths = set( - [record[0] for record in legacy_result.fetchall()] - ) + legacy_result = session.execute(text("SELECT path FROM metadata_log")) + legacy_paths = set([record[0] for record in legacy_result.fetchall()]) - modern_result = md_db_manager.select_as_list( - sa.select(MetadataLog.path) - ) + modern_result = md_db_manager.select_as_list(sa.select(MetadataLog.path)) modern_paths = set([record["path"] for record in modern_result]) missing_paths = legacy_paths - modern_paths @@ -63,17 +55,11 @@ def upgrade() -> None: # # Raise all other sql errors original_error = error.orig - if ( - original_error is not None - and hasattr(original_error, "pgcode") - and original_error.pgcode == "42P01" - ): + if original_error is not None and hasattr(original_error, "pgcode") and original_error.pgcode == "42P01": logging.info("No Metadata Table in Rail Performance Manager") legacy_paths = set() else: - logging.exception( - "Programming Error when checking Metadata Log" - ) + logging.exception("Programming Error when checking Metadata Log") time.sleep(15) continue @@ -92,15 +78,9 @@ def downgrade() -> None: op.create_table( "metadata_log", sa.Column("pk_id", sa.INTEGER(), autoincrement=True, nullable=False), - sa.Column( - "processed", sa.BOOLEAN(), autoincrement=False, nullable=True - ), - sa.Column( - "process_fail", sa.BOOLEAN(), autoincrement=False, nullable=True - ), - sa.Column( - "path", sa.VARCHAR(length=256), autoincrement=False, nullable=False - ), + sa.Column("processed", sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column("process_fail", sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.Column("path", sa.VARCHAR(length=256), autoincrement=False, nullable=False), sa.Column( "created_on", postgresql.TIMESTAMP(timezone=True), diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/007_2dfbde5ec151_sync_stop_trunk.py b/src/lamp_py/migrations/versions/performance_manager_staging/007_2dfbde5ec151_sync_stop_trunk.py index c02d5cba..9bc1a592 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/007_2dfbde5ec151_sync_stop_trunk.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/007_2dfbde5ec151_sync_stop_trunk.py @@ -28,9 +28,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/008_896dedd8a4db_dwell_time_update.py b/src/lamp_py/migrations/versions/performance_manager_staging/008_896dedd8a4db_dwell_time_update.py index a13a266b..9e3975a8 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/008_896dedd8a4db_dwell_time_update.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/008_896dedd8a4db_dwell_time_update.py @@ -27,9 +27,7 @@ def upgrade() -> None: - date_query = sa.text( - "SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date" - ) + date_query = sa.text("SELECT DISTINCT service_date, static_version_key FROM vehicle_trips ORDER BY service_date") conn = op.get_bind() result = conn.execute(date_query) diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/009_32ba735d080c_add_revenue_columns.py b/src/lamp_py/migrations/versions/performance_manager_staging/009_32ba735d080c_add_revenue_columns.py index d6d7ebb1..3bf161c0 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/009_32ba735d080c_add_revenue_columns.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/009_32ba735d080c_add_revenue_columns.py @@ -30,21 +30,13 @@ def upgrade() -> None: - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips DISABLE TRIGGER update_vehicle_trips_modified;") op.drop_index("ix_vehicle_trips_composite_1", table_name="vehicle_trips") op.drop_constraint("vehicle_trips_unique_trip", table_name="vehicle_trips") - op.add_column( - "temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True) - ) - op.add_column( - "vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True) - ) + op.add_column("temp_event_compare", sa.Column("revenue", sa.Boolean(), nullable=True)) + op.add_column("vehicle_trips", sa.Column("revenue", sa.Boolean(), nullable=True)) op.execute(sa.update(TempEventCompare).values(revenue=True)) op.execute(sa.update(VehicleTrips).values(revenue=True)) op.alter_column("temp_event_compare", "revenue", nullable=False) @@ -61,12 +53,8 @@ def upgrade() -> None: ["route_id", "direction_id", "vehicle_id"], unique=False, ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;" - ) - op.execute( - f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;" - ) + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER rt_trips_update_branch_trunk;") + op.execute(f"ALTER TABLE public.vehicle_trips ENABLE TRIGGER update_vehicle_trips_modified;") def downgrade() -> None: diff --git a/src/lamp_py/migrations/versions/performance_manager_staging/sql_strings/strings_001.py b/src/lamp_py/migrations/versions/performance_manager_staging/sql_strings/strings_001.py index 9f9be969..03871144 100644 --- a/src/lamp_py/migrations/versions/performance_manager_staging/sql_strings/strings_001.py +++ b/src/lamp_py/migrations/versions/performance_manager_staging/sql_strings/strings_001.py @@ -15,7 +15,9 @@ ashmond_stop_ids = "('70087', '70088', '70089', '70090', '70091', '70092', '70093', '70094', '70085', '70086')" -braintree_stop_ids = "('70097', '70098', '70099', '70100', '70101', '70102', '70103', '70104', '70105', '70095', '70096')" +braintree_stop_ids = ( + "('70097', '70098', '70099', '70100', '70101', '70102', '70103', '70104', '70105', '70095', '70096')" +) func_red_is_ashmont_branch = f""" diff --git a/src/lamp_py/mssql/mssql_utils.py b/src/lamp_py/mssql/mssql_utils.py index 3e42742a..41a5c19c 100644 --- a/src/lamp_py/mssql/mssql_utils.py +++ b/src/lamp_py/mssql/mssql_utils.py @@ -34,9 +34,7 @@ def get_local_engine(echo: bool = False) -> sa.future.engine.Engine: assert db_password is not None assert db_port != 0 - process_logger.add_metadata( - host=db_host, database_name=db_name, user=db_user, port=db_port - ) + process_logger.add_metadata(host=db_host, database_name=db_name, user=db_user, port=db_port) connection = sa.URL.create( "mssql+pyodbc", @@ -101,20 +99,14 @@ def execute( return result - def select_as_dataframe( - self, select_query: sa.sql.selectable.Select - ) -> pandas.DataFrame: + def select_as_dataframe(self, select_query: sa.sql.selectable.Select) -> pandas.DataFrame: """ select data from db table and return pandas dataframe """ with self.session.begin() as cursor: - return pandas.DataFrame( - [row._asdict() for row in cursor.execute(select_query)] - ) + return pandas.DataFrame([row._asdict() for row in cursor.execute(select_query)]) - def select_as_list( - self, select_query: sa.sql.selectable.Select - ) -> Union[List[Any], List[Dict[str, Any]]]: + def select_as_list(self, select_query: sa.sql.selectable.Select) -> Union[List[Any], List[Dict[str, Any]]]: """ select data from db table and return list """ @@ -165,12 +157,8 @@ def write_to_parquet( process_logger.add_metadata(retry_attempts=retry_attempts) try: with self.session.begin() as cursor: - with pq.ParquetWriter( - write_path, schema=schema - ) as pq_writer: - for part in cursor.execute(part_stmt).partitions( - batch_size - ): + with pq.ParquetWriter(write_path, schema=schema) as pq_writer: + for part in cursor.execute(part_stmt).partitions(batch_size): pq_writer.write_batch( pyarrow.RecordBatch.from_pylist( [row._asdict() for row in part], diff --git a/src/lamp_py/mssql/test_connect.py b/src/lamp_py/mssql/test_connect.py index 6e95c86b..2501421c 100644 --- a/src/lamp_py/mssql/test_connect.py +++ b/src/lamp_py/mssql/test_connect.py @@ -7,9 +7,7 @@ def start() -> None: Test MSSQL DB Connection """ db = MSSQLManager(verbose=True) - select_query = sa.text( - "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE';" - ) + select_query = sa.text("SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE';") for record in db.select_as_list(select_query): print(record) diff --git a/src/lamp_py/performance_manager/alerts.py b/src/lamp_py/performance_manager/alerts.py index 194552b1..170bed45 100644 --- a/src/lamp_py/performance_manager/alerts.py +++ b/src/lamp_py/performance_manager/alerts.py @@ -97,9 +97,7 @@ def existing_id_timestamp_pairs(self) -> pandas.DataFrame: columns = ["id", "last_modified_timestamp"] if os.path.exists(self.local_path): - existing_alerts = pq.read_table( - self.local_path, columns=columns - ).to_pandas() + existing_alerts = pq.read_table(self.local_path, columns=columns).to_pandas() existing_alerts = existing_alerts.drop_duplicates() return existing_alerts @@ -115,9 +113,7 @@ def append_new_records(self, alerts: pandas.DataFrame) -> None: process_logger.log_start() alerts = alerts.reset_index(drop=True) - alerts_table = pyarrow.Table.from_pandas( - alerts, schema=self.parquet_schema - ) + alerts_table = pyarrow.Table.from_pandas(alerts, schema=self.parquet_schema) if alerts_table.num_rows == 0: process_logger.log_complete() @@ -141,9 +137,7 @@ def append_new_records(self, alerts: pandas.DataFrame) -> None: row_group_count = 0 partition_key = "active_period.start_timestamp" - partition_key_arr = joined_ds.to_table(columns=[partition_key]).column( - partition_key - ) + partition_key_arr = joined_ds.to_table(columns=[partition_key]).column(partition_key) # the start is the start of the month containing the minimum timestamp start = pc.min(partition_key_arr).as_py() @@ -163,9 +157,7 @@ def append_new_records(self, alerts: pandas.DataFrame) -> None: & (pc.field(partition_key) < int(end.timestamp())) ).to_table() else: - table = joined_ds.filter( - (pc.field(partition_key) >= int(start.timestamp())) - ).to_table() + table = joined_ds.filter((pc.field(partition_key) >= int(start.timestamp()))).to_table() if table.num_rows > 0: row_group_count += 1 @@ -173,9 +165,7 @@ def append_new_records(self, alerts: pandas.DataFrame) -> None: start = end - table = joined_ds.filter( - (pc.field(partition_key).is_null()) - ).to_table() + table = joined_ds.filter((pc.field(partition_key).is_null())).to_table() if table.num_rows > 0: row_group_count += 1 @@ -195,17 +185,11 @@ def upload_data(self) -> None: upload_file( file_name=self.local_path, object_path=self.s3_path, - extra_args={ - "Metadata": { - AlertsS3Info.version_key: AlertsS3Info.file_version - } - }, + extra_args={"Metadata": {AlertsS3Info.version_key: AlertsS3Info.file_version}}, ) -def extract_alerts( - alert_files: List[str], existing_id_timestamp_pairs: pandas.DataFrame -) -> pandas.DataFrame: +def extract_alerts(alert_files: List[str], existing_id_timestamp_pairs: pandas.DataFrame) -> pandas.DataFrame: """Read alerts data from unprocessed files, remove duplicates, and set types""" columns = [ "id", @@ -268,12 +252,8 @@ def extract_alerts( alerts["alert_lifecycle"] = alerts["alert_lifecycle"].astype("string") alerts["duration_certainty"] = alerts["duration_certainty"].astype("string") alerts["created_timestamp"] = alerts["created_timestamp"].astype("Int64") - alerts["last_modified_timestamp"] = alerts[ - "last_modified_timestamp" - ].astype("Int64") - alerts["last_push_notification_timestamp"] = alerts[ - "last_push_notification_timestamp" - ].astype("Int64") + alerts["last_modified_timestamp"] = alerts["last_modified_timestamp"].astype("Int64") + alerts["last_push_notification_timestamp"] = alerts["last_push_notification_timestamp"].astype("Int64") alerts["closed_timestamp"] = alerts["closed_timestamp"].astype("Int64") # perform an anti-join against existing alerts. merge with existing pairs @@ -294,9 +274,7 @@ def extract_alerts( def transform_translations(alerts: pandas.DataFrame) -> pandas.DataFrame: """For each string field with translations, pull out the English string""" - def process_translation( - translations: Optional[List[Dict[str, str]]] - ) -> Optional[str]: + def process_translation(translations: Optional[List[Dict[str, str]]]) -> Optional[str]: """small lambda for processing the translation""" if translations is None: return None @@ -316,9 +294,7 @@ def process_translation( drop_columns = [] for key in translation_columns: translation_key = f"{key}.translation" - alerts[f"{translation_key}.text"] = alerts[translation_key].apply( - process_translation - ) + alerts[f"{translation_key}.text"] = alerts[translation_key].apply(process_translation) drop_columns.append(translation_key) alerts = alerts.drop(columns=drop_columns) @@ -382,9 +358,7 @@ def explode_active_periods(alerts: pandas.DataFrame) -> pandas.DataFrame: # pull out the active period timestamps from the dict in that column alerts = alerts.explode("active_period") - def extract_start_end( - period: Dict[str, int] | float | None - ) -> Tuple[int | None, int | None]: + def extract_start_end(period: Dict[str, int] | float | None) -> Tuple[int | None, int | None]: """ small lambda for extracting start and end timestamps @@ -406,12 +380,8 @@ def extract_start_end( ) # convert these timestamps to Int64 to avoid floating point errors - alerts["active_period.start_timestamp"] = alerts[ - "active_period.start_timestamp" - ].astype("Int64") - alerts["active_period.end_timestamp"] = alerts[ - "active_period.end_timestamp" - ].astype("Int64") + alerts["active_period.start_timestamp"] = alerts["active_period.start_timestamp"].astype("Int64") + alerts["active_period.end_timestamp"] = alerts["active_period.end_timestamp"].astype("Int64") # drop the active period list column alerts = alerts.drop(columns=["active_period"]) @@ -446,21 +416,13 @@ def explode_informed_entity(alerts: pandas.DataFrame) -> pandas.DataFrame: # extract information from the informed entity for key in informed_entity_keys: full_key = f"informed_entity.{key}" - alerts[full_key] = alerts["informed_entity"].apply( - lambda x, k=key: None if x is None else x.get(k) - ) + alerts[full_key] = alerts["informed_entity"].apply(lambda x, k=key: None if x is None else x.get(k)) alerts = alerts.drop(columns=["informed_entity"]) # transform the activities field from a list to a pipe delimitated string - alerts["informed_entity.activities"] = alerts[ - "informed_entity.activities" - ].apply( - lambda x: ( - None - if x is None - else "|".join(str(item) for item in x if item is not None) - ) + alerts["informed_entity.activities"] = alerts["informed_entity.activities"].apply( + lambda x: (None if x is None else "|".join(str(item) for item in x if item is not None)) ) # the commuter rail informed entity contains extra details that aren't @@ -471,9 +433,7 @@ def explode_informed_entity(alerts: pandas.DataFrame) -> pandas.DataFrame: return alerts -def get_alert_files( - md_db_manager: DatabaseManager, unprocessed_only: bool -) -> List[Dict[str, str]]: +def get_alert_files(md_db_manager: DatabaseManager, unprocessed_only: bool) -> List[Dict[str, str]]: """ Get unprocessed RT Alert Files from the MetadataLog table. @@ -485,13 +445,10 @@ def get_alert_files( """ if unprocessed_only: read_md = sa.select(MetadataLog.pk_id, MetadataLog.path).where( - (MetadataLog.rail_pm_processed == sa.false()) - & (MetadataLog.path.contains("RT_ALERTS")) + (MetadataLog.rail_pm_processed == sa.false()) & (MetadataLog.path.contains("RT_ALERTS")) ) else: - read_md = sa.select(MetadataLog.pk_id, MetadataLog.path).where( - (MetadataLog.path.contains("RT_ALERTS")) - ) + read_md = sa.select(MetadataLog.pk_id, MetadataLog.path).where((MetadataLog.path.contains("RT_ALERTS"))) return md_db_manager.select_as_list(read_md) @@ -503,9 +460,7 @@ def process_alerts(md_db_manager: DatabaseManager) -> None: process_logger = ProcessLogger("process_alerts") process_logger.log_start() - version_match = version_check( - obj=AlertsS3Info.s3_path, version=AlertsS3Info.file_version - ) + version_match = version_check(obj=AlertsS3Info.s3_path, version=AlertsS3Info.file_version) metadata_records = get_alert_files( md_db_manager=md_db_manager, @@ -535,9 +490,7 @@ def process_alerts(md_db_manager: DatabaseManager) -> None: pk_ids = [record["pk_id"] for record in chunk] alert_files = [record["path"] for record in chunk] - subprocess_logger = ProcessLogger( - "process_alerts_chunk", alert_files=alert_files - ) + subprocess_logger = ProcessLogger("process_alerts_chunk", alert_files=alert_files) subprocess_logger.log_start() try: @@ -560,15 +513,11 @@ def process_alerts(md_db_manager: DatabaseManager) -> None: # add new id timestamp pairs to existing for next pass new_id_timestamp_pairs = alerts[["id", "last_modified_timestamp"]] - existing_id_timestamp_pairs = pandas.concat( - [existing_id_timestamp_pairs, new_id_timestamp_pairs] - ) + existing_id_timestamp_pairs = pandas.concat([existing_id_timestamp_pairs, new_id_timestamp_pairs]) # update metadata for the files that were processed md_db_manager.execute( - sa.update(MetadataLog.__table__) - .where(MetadataLog.pk_id.in_(pk_ids)) - .values(rail_pm_processed=True) + sa.update(MetadataLog.__table__).where(MetadataLog.pk_id.in_(pk_ids)).values(rail_pm_processed=True) ) subprocess_logger.log_complete() diff --git a/src/lamp_py/performance_manager/flat_file.py b/src/lamp_py/performance_manager/flat_file.py index 80bf122b..fd493794 100644 --- a/src/lamp_py/performance_manager/flat_file.py +++ b/src/lamp_py/performance_manager/flat_file.py @@ -40,9 +40,7 @@ class S3Archive: """ BUCKET_NAME = S3_PUBLIC - RAIL_PERFORMANCE_PREFIX = os.path.join( - LAMP, "subway-on-time-performance-v1" - ) + RAIL_PERFORMANCE_PREFIX = os.path.join(LAMP, "subway-on-time-performance-v1") INDEX_FILENAME = "index.csv" VERSION_KEY = "rpm_version" RPM_VERSION = "1.1.0" @@ -64,10 +62,7 @@ def db_service_dates_to_datetimes(df: pandas.DataFrame) -> Set[datetime]: if df.size == 0: return set() - return set( - datetime.strptime(str(service_date), "%Y%m%d") - for service_date in df["service_date"] - ) + return set(datetime.strptime(str(service_date), "%Y%m%d") for service_date in df["service_date"]) def filepaths_to_datetimes(filepaths: List[str]) -> Set[datetime]: """ @@ -83,9 +78,7 @@ def filepaths_to_datetimes(filepaths: List[str]) -> Set[datetime]: for filepath in filepaths: match = re.search(r"(?P\d{4}-\d{1,2}-\d{1,2})", filepath) if match is not None: - datetimes.add( - datetime.strptime(match.group("date"), "%Y-%m-%d") - ) + datetimes.add(datetime.strptime(match.group("date"), "%Y-%m-%d")) return datetimes @@ -97,17 +90,11 @@ def filepaths_to_datetimes(filepaths: List[str]) -> Set[datetime]: archived_datetimes = filepaths_to_datetimes(archive_filepaths) # get the processed service dates as a set - vehicle_events_service_dates = db_manager.select_as_dataframe( - sa.select(VehicleTrips.service_date).distinct() - ) - processed_datetimes = db_service_dates_to_datetimes( - vehicle_events_service_dates - ) + vehicle_events_service_dates = db_manager.select_as_dataframe(sa.select(VehicleTrips.service_date).distinct()) + processed_datetimes = db_service_dates_to_datetimes(vehicle_events_service_dates) # get service dates with new data from the last event loop - new_data_service_dates = db_manager.select_as_dataframe( - sa.select(TempEventCompare.service_date).distinct() - ) + new_data_service_dates = db_manager.select_as_dataframe(sa.select(TempEventCompare.service_date).distinct()) new_data_datetimes = db_service_dates_to_datetimes(new_data_service_dates) # return the processed dates that have yet to be archived plus dates with new data @@ -150,9 +137,7 @@ def write_flat_files(db_manager: DatabaseManager) -> None: sub_process_logger.log_start() try: - write_daily_table( - db_manager=db_manager, service_date=service_date - ) + write_daily_table(db_manager=db_manager, service_date=service_date) except Exception as e: sub_process_logger.log_failure(e) else: @@ -198,9 +183,7 @@ def check_version() -> None: for file in files_to_remove: success = delete_object(file) if not success: - raise RuntimeError( - f"Failed to delete {file} when updating flat files" - ) + raise RuntimeError(f"Failed to delete {file} when updating flat files") def write_csv_index() -> None: @@ -218,14 +201,10 @@ def write_csv_index() -> None: # drop details for the index cvs and add in service date column df = df[~df["s3_obj_path"].str.endswith(S3Archive.INDEX_FILENAME)] - df["service_date"] = df["s3_obj_path"].apply( - lambda x: x.split("/")[-1][:10] - ) + df["service_date"] = df["s3_obj_path"].apply(lambda x: x.split("/")[-1][:10]) # replace "s3://[S3Archive.BUCKET_NAME]" with "https://performancedata.mbta.com" - df["file_url"] = df["s3_obj_path"].str.replace( - f"s3://{S3Archive.BUCKET_NAME}", "https://performancedata.mbta.com" - ) + df["file_url"] = df["s3_obj_path"].str.replace(f"s3://{S3Archive.BUCKET_NAME}", "https://performancedata.mbta.com") df = df.drop(columns=["s3_obj_path"]) # write to local csv and upload file to s3 @@ -245,31 +224,21 @@ def write_csv_index() -> None: os.remove(csv_path) -def write_daily_table( - db_manager: DatabaseManager, service_date: datetime -) -> pyarrow.Table: +def write_daily_table(db_manager: DatabaseManager, service_date: datetime) -> pyarrow.Table: """ Generate a dataframe of all events and metrics for a single service date """ service_date_int = int(service_date.strftime("%Y%m%d")) service_date_str = service_date.strftime("%Y-%m-%d") - static_version_key = static_version_key_from_service_date( - service_date=service_date_int, db_manager=db_manager - ) + static_version_key = static_version_key_from_service_date(service_date=service_date_int, db_manager=db_manager) static_subquery = ( sa.select( StaticStopTimes.arrival_time.label("scheduled_arrival_time"), StaticStopTimes.departure_time.label("scheduled_departure_time"), - StaticStopTimes.schedule_travel_time_seconds.label( - "scheduled_travel_time" - ), - StaticStopTimes.schedule_headway_branch_seconds.label( - "scheduled_headway_branch" - ), - StaticStopTimes.schedule_headway_trunk_seconds.label( - "scheduled_headway_trunk" - ), + StaticStopTimes.schedule_travel_time_seconds.label("scheduled_travel_time"), + StaticStopTimes.schedule_headway_branch_seconds.label("scheduled_headway_branch"), + StaticStopTimes.schedule_headway_trunk_seconds.label("scheduled_headway_trunk"), StaticStopTimes.trip_id, sa.func.coalesce( StaticStops.parent_station, @@ -280,8 +249,7 @@ def write_daily_table( .join( StaticStops, sa.and_( - StaticStopTimes.static_version_key - == StaticStops.static_version_key, + StaticStopTimes.static_version_key == StaticStops.static_version_key, StaticStopTimes.stop_id == StaticStops.stop_id, ), ) @@ -331,8 +299,7 @@ def write_daily_table( static_subquery, sa.and_( static_subquery.c.trip_id == VehicleTrips.static_trip_id_guess, - static_subquery.c.parent_station - == VehicleEvents.parent_station, + static_subquery.c.parent_station == VehicleEvents.parent_station, ), isouter=True, ) @@ -340,8 +307,7 @@ def write_daily_table( StaticRoutes, sa.and_( VehicleTrips.route_id == StaticRoutes.route_id, - VehicleTrips.static_version_key - == StaticRoutes.static_version_key, + VehicleTrips.static_version_key == StaticRoutes.static_version_key, ), ) .where( @@ -398,9 +364,7 @@ def write_daily_table( # generate temp local and s3 paths from the service date filename = f"{service_date_str}-subway-on-time-performance-v1.parquet" temp_local_path = f"/tmp/{filename}" - s3_path = os.path.join( - S3Archive.BUCKET_NAME, S3Archive.RAIL_PERFORMANCE_PREFIX, filename - ) + s3_path = os.path.join(S3Archive.BUCKET_NAME, S3Archive.RAIL_PERFORMANCE_PREFIX, filename) # the local path shouldn't exist, but make sure if os.path.exists(temp_local_path): diff --git a/src/lamp_py/performance_manager/gtfs_utils.py b/src/lamp_py/performance_manager/gtfs_utils.py index 3d76f393..67137579 100644 --- a/src/lamp_py/performance_manager/gtfs_utils.py +++ b/src/lamp_py/performance_manager/gtfs_utils.py @@ -14,7 +14,7 @@ StaticStops, ) from lamp_py.runtime_utils.process_logger import ProcessLogger -from lamp_py.aws.s3 import get_datetime_from_partition_path +from lamp_py.aws.s3 import dt_from_obj_path # boston tzinfo to be used with datetimes that require DST considerations. @@ -51,11 +51,7 @@ def start_timestamp_to_seconds(start_timestamp: int) -> int: year = int(service_date_string[:4]) month = int(service_date_string[4:6]) day = int(service_date_string[6:8]) - start_of_service_day = int( - BOSTON_TZ.localize( - datetime.datetime(year=year, month=month, day=day) - ).timestamp() - ) + start_of_service_day = int(BOSTON_TZ.localize(datetime.datetime(year=year, month=month, day=day)).timestamp()) return start_timestamp - start_of_service_day @@ -74,14 +70,10 @@ def service_date_from_timestamp(timestamp: int) -> int: else: service_date = date_and_time.date() - return int( - f"{service_date.year:04}{service_date.month:02}{service_date.day:02}" - ) + return int(f"{service_date.year:04}{service_date.month:02}{service_date.day:02}") -def add_missing_service_dates( - events_dataframe: pandas.DataFrame, timestamp_key: str -) -> pandas.DataFrame: +def add_missing_service_dates(events_dataframe: pandas.DataFrame, timestamp_key: str) -> pandas.DataFrame: """ # generate the service date from the vehicle timestamp if null """ @@ -105,9 +97,7 @@ def unique_trip_stop_columns() -> List[str]: ] -def static_version_key_from_service_date( - service_date: int, db_manager: DatabaseManager -) -> int: +def static_version_key_from_service_date(service_date: int, db_manager: DatabaseManager) -> int: """ for a given service date, determine the correct static schedule to use """ @@ -158,9 +148,7 @@ def static_version_key_from_service_date( # exists for this trip update data, so the data # should not be processed until valid static schedule data exists if len(result) == 0: - raise IndexError( - f"StaticFeedInfo table has no matching schedule for service_date={service_date}" - ) + raise IndexError(f"StaticFeedInfo table has no matching schedule for service_date={service_date}") return int(result[0]["static_version_key"]) @@ -195,14 +183,10 @@ def add_static_version_key_column( for date in events_dataframe["service_date"].unique(): service_date = int(date) - static_version_key = static_version_key_from_service_date( - service_date=service_date, db_manager=db_manager - ) + static_version_key = static_version_key_from_service_date(service_date=service_date, db_manager=db_manager) service_date_mask = events_dataframe["service_date"] == service_date - events_dataframe.loc[service_date_mask, "static_version_key"] = ( - static_version_key - ) + events_dataframe.loc[service_date_mask, "static_version_key"] = static_version_key process_logger.log_complete() @@ -234,10 +218,7 @@ def add_parent_station_column( return events_dataframe # unique list of "static_version_key" values for pulling parent stations - lookup_v_keys = [ - int(s_v_key) - for s_v_key in events_dataframe["static_version_key"].unique() - ] + lookup_v_keys = [int(s_v_key) for s_v_key in events_dataframe["static_version_key"].unique()] # pull parent station data for joining to events dataframe parent_station_query = sa.select( @@ -248,9 +229,7 @@ def add_parent_station_column( parent_stations = db_manager.select_as_dataframe(parent_station_query) # join parent stations to events on "stop_id" and "static_version_key" foreign key - events_dataframe = events_dataframe.merge( - parent_stations, how="left", on=["static_version_key", "stop_id"] - ) + events_dataframe = events_dataframe.merge(parent_stations, how="left", on=["static_version_key", "stop_id"]) # is parent station is not provided, transfer "stop_id" value to # "parent_station" column events_dataframe["parent_station"] = numpy.where( @@ -264,9 +243,7 @@ def add_parent_station_column( return events_dataframe -def rail_routes_from_filepath( - filepath: Union[List[str], str], db_manager: DatabaseManager -) -> List[str]: +def rail_routes_from_filepath(filepath: Union[List[str], str], db_manager: DatabaseManager) -> List[str]: """ get a list of rail route_ids that were in effect on a given service date described by a timestamp. the schedule version is derived from the service @@ -276,12 +253,10 @@ def rail_routes_from_filepath( if isinstance(filepath, list): filepath = filepath[0] - date = get_datetime_from_partition_path(filepath) + date = dt_from_obj_path(filepath) service_date = int(f"{date.year:04}{date.month:02}{date.day:02}") - static_version_key = static_version_key_from_service_date( - service_date=service_date, db_manager=db_manager - ) + static_version_key = static_version_key_from_service_date(service_date=service_date, db_manager=db_manager) result = db_manager.execute( sa.select(StaticRoutes.route_id).where( diff --git a/src/lamp_py/performance_manager/l0_gtfs_rt_events.py b/src/lamp_py/performance_manager/l0_gtfs_rt_events.py index 3689fb06..70c8755f 100644 --- a/src/lamp_py/performance_manager/l0_gtfs_rt_events.py +++ b/src/lamp_py/performance_manager/l0_gtfs_rt_events.py @@ -6,7 +6,7 @@ from sqlalchemy.dialects import postgresql from sqlalchemy.sql.functions import count -from lamp_py.aws.s3 import get_datetime_from_partition_path +from lamp_py.aws.s3 import dt_from_obj_path from lamp_py.postgres.metadata_schema import MetadataLog from lamp_py.postgres.rail_performance_manager_schema import ( TempEventCompare, @@ -26,9 +26,7 @@ from .l1_rt_metrics import update_metrics_from_temp_events -def get_gtfs_rt_paths( - md_db_manager: DatabaseManager, path_count: int = 12 -) -> Dict[str, List]: +def get_gtfs_rt_paths(md_db_manager: DatabaseManager, path_count: int = 12) -> Dict[str, List]: """ get lists of GTFS-RT paths, grouped by type, and DB primary keys of each path limit number of paths in each list to a max of `path_count` @@ -48,9 +46,7 @@ def get_gtfs_rt_paths( vp_files = get_unprocessed_files("RT_VEHICLE_POSITIONS", md_db_manager) for record in vp_files: - timestamp = get_datetime_from_partition_path( - record["paths"][0] - ).timestamp() + timestamp = dt_from_obj_path(record["paths"][0]).timestamp() grouped_files[timestamp] = { "ids": record["ids"], @@ -60,9 +56,7 @@ def get_gtfs_rt_paths( tu_files = get_unprocessed_files("RT_TRIP_UPDATES", md_db_manager) for record in tu_files: - timestamp = get_datetime_from_partition_path( - record["paths"][0] - ).timestamp() + timestamp = dt_from_obj_path(record["paths"][0]).timestamp() if timestamp in grouped_files: grouped_files[timestamp]["ids"] += record["ids"] grouped_files[timestamp]["tu_paths"] += record["paths"] @@ -94,9 +88,7 @@ def get_gtfs_rt_paths( return return_dict -def combine_events( - vp_events: pandas.DataFrame, tu_events: pandas.DataFrame -) -> pandas.DataFrame: +def combine_events(vp_events: pandas.DataFrame, tu_events: pandas.DataFrame) -> pandas.DataFrame: """ collapse the vp events and tu events into a single vehicle events df @@ -118,9 +110,7 @@ def combine_events( # merge together the trip_stop_columns and the timestamps events = pandas.merge( - vp_events[ - trip_stop_columns + ["vp_stop_timestamp", "vp_move_timestamp"] - ], + vp_events[trip_stop_columns + ["vp_stop_timestamp", "vp_move_timestamp"]], tu_events[ trip_stop_columns + [ @@ -161,18 +151,12 @@ def combine_events( "static_version_key", ] - event_details = pandas.concat( - [vp_events[details_columns], tu_events[details_columns]] - ) + event_details = pandas.concat([vp_events[details_columns], tu_events[details_columns]]) # create sort column to indicate which records have null values for select columns # we want to drop these null value records whenever possible # to prioritize records from vehicle_positions - event_details["na_sort"] = ( - event_details[["stop_sequence", "vehicle_label", "vehicle_consist"]] - .isna() - .sum(axis=1) - ) + event_details["na_sort"] = event_details[["stop_sequence", "vehicle_label", "vehicle_consist"]].isna().sum(axis=1) event_details = ( event_details.sort_values( @@ -187,9 +171,7 @@ def combine_events( ) # join `details_columns` to df with timestamps - events = events.merge( - event_details, how="left", on=trip_stop_columns, validate="one_to_one" - ) + events = events.merge(event_details, how="left", on=trip_stop_columns, validate="one_to_one") process_logger.add_metadata(total_event_count=events.shape[0]) process_logger.log_complete() @@ -225,16 +207,14 @@ def flag_insert_update_events(db_manager: DatabaseManager) -> Tuple[int, int]: TempEventCompare.vp_move_timestamp.is_not(None), sa.or_( VehicleEvents.vp_move_timestamp.is_(None), - VehicleEvents.vp_move_timestamp - > TempEventCompare.vp_move_timestamp, + VehicleEvents.vp_move_timestamp > TempEventCompare.vp_move_timestamp, ), ), sa.and_( TempEventCompare.vp_stop_timestamp.is_not(None), sa.or_( VehicleEvents.vp_stop_timestamp.is_(None), - VehicleEvents.vp_stop_timestamp - > TempEventCompare.vp_move_timestamp, + VehicleEvents.vp_stop_timestamp > TempEventCompare.vp_move_timestamp, ), ), TempEventCompare.tu_stop_timestamp.is_not(None), @@ -244,12 +224,8 @@ def flag_insert_update_events(db_manager: DatabaseManager) -> Tuple[int, int]: db_manager.execute(update_do_update) # get count of do_update records - update_count_query = sa.select(count(TempEventCompare.do_update)).where( - TempEventCompare.do_update == sa.true() - ) - update_count = int( - db_manager.select_as_list(update_count_query)[0]["count"] - ) + update_count_query = sa.select(count(TempEventCompare.do_update)).where(TempEventCompare.do_update == sa.true()) + update_count = int(db_manager.select_as_list(update_count_query)[0]["count"]) # populate do_insert column of temp_event_compare do_insert_pre_select = ( @@ -271,12 +247,8 @@ def flag_insert_update_events(db_manager: DatabaseManager) -> Tuple[int, int]: db_manager.execute(update_do_insert) # get count of do_insert records - insert_count_query = sa.select(count(TempEventCompare.do_insert)).where( - TempEventCompare.do_insert == sa.true() - ) - insert_count = int( - db_manager.select_as_list(insert_count_query)[0]["count"] - ) + insert_count_query = sa.select(count(TempEventCompare.do_insert)).where(TempEventCompare.do_insert == sa.true()) + insert_count = int(db_manager.select_as_list(insert_count_query)[0]["count"]) # remove records from temp_event_compare that are not related to updates or inserts delete_temp = sa.delete(TempEventCompare.__table__).where( @@ -288,9 +260,7 @@ def flag_insert_update_events(db_manager: DatabaseManager) -> Tuple[int, int]: return (update_count, insert_count) -def build_temp_events( - events: pandas.DataFrame, db_manager: DatabaseManager -) -> pandas.DataFrame: +def build_temp_events(events: pandas.DataFrame, db_manager: DatabaseManager) -> pandas.DataFrame: """ add vehicle event data to the database @@ -395,8 +365,7 @@ def update_events_from_temp(db_manager: DatabaseManager) -> None: sa.text("excluded.vp_move_timestamp IS NOT NULL"), sa.or_( VehicleEvents.vp_move_timestamp.is_(None), - VehicleEvents.vp_move_timestamp - > sa.text("excluded.vp_move_timestamp"), + VehicleEvents.vp_move_timestamp > sa.text("excluded.vp_move_timestamp"), ), ), ) @@ -418,8 +387,7 @@ def update_events_from_temp(db_manager: DatabaseManager) -> None: TempEventCompare.vp_stop_timestamp.is_not(None), sa.or_( VehicleEvents.vp_stop_timestamp.is_(None), - VehicleEvents.vp_stop_timestamp - > TempEventCompare.vp_stop_timestamp, + VehicleEvents.vp_stop_timestamp > TempEventCompare.vp_stop_timestamp, ), ) ) @@ -539,9 +507,7 @@ def process_gtfs_rt_files( update_metrics_from_temp_events(rpm_db_manager) md_db_manager.execute( - sa.update(MetadataLog.__table__) - .where(MetadataLog.pk_id.in_(files["ids"])) - .values(rail_pm_processed=True) + sa.update(MetadataLog.__table__).where(MetadataLog.pk_id.in_(files["ids"])).values(rail_pm_processed=True) ) process_logger.add_metadata(event_count=events.shape[0]) process_logger.log_complete() diff --git a/src/lamp_py/performance_manager/l0_gtfs_static_load.py b/src/lamp_py/performance_manager/l0_gtfs_static_load.py index 438a3479..06aec50a 100644 --- a/src/lamp_py/performance_manager/l0_gtfs_static_load.py +++ b/src/lamp_py/performance_manager/l0_gtfs_static_load.py @@ -296,29 +296,21 @@ def get_static_parquet_paths(table_type: str, feed_info_path: str) -> List[str]: return file_list_from_s3(S3_SPRINGBOARD, static_prefix) -def load_parquet_files( - static_tables: Dict[str, StaticTableDetails], feed_info_path: str -) -> None: +def load_parquet_files(static_tables: Dict[str, StaticTableDetails], feed_info_path: str) -> None: """ get parquet paths to load from feed_info_path and load parquet files as dataframe into StaticTableDetails objects """ for table in static_tables.values(): - paths_to_load = get_static_parquet_paths( - table.table_name, feed_info_path - ) + paths_to_load = get_static_parquet_paths(table.table_name, feed_info_path) try: - table.data_table = read_parquet( - paths_to_load[:1], columns=table.column_info.columns_to_pull - ) + table.data_table = read_parquet(paths_to_load[:1], columns=table.column_info.columns_to_pull) assert table.data_table.shape[0] > 0 except (pyarrow.ArrowInvalid, AssertionError) as exception: if table.allow_empty_dataframe is False: raise exception - table.data_table = pandas.DataFrame( - columns=table.column_info.columns_to_pull - ) + table.data_table = pandas.DataFrame(columns=table.column_info.columns_to_pull) def transform_data_tables(static_tables: Dict[str, StaticTableDetails]) -> None: @@ -330,27 +322,17 @@ def transform_data_tables(static_tables: Dict[str, StaticTableDetails]) -> None: if table.column_info.int64_cols is not None: for col in table.column_info.int64_cols: - table.data_table[col] = pandas.to_numeric( - table.data_table[col] - ).astype("Int64") + table.data_table[col] = pandas.to_numeric(table.data_table[col]).astype("Int64") if table.column_info.bool_cols is not None: for col in table.column_info.bool_cols: - table.data_table[col] = numpy.where( - table.data_table[col] == 1, True, False - ).astype(numpy.bool_) + table.data_table[col] = numpy.where(table.data_table[col] == 1, True, False).astype(numpy.bool_) if table.column_info.time_to_seconds_cols is not None: for col in table.column_info.time_to_seconds_cols: - table.data_table[col] = ( - table.data_table[col] - .apply(start_time_to_seconds) - .astype("Int64") - ) - - table.data_table = table.data_table.fillna(numpy.nan).replace( - [numpy.nan], [None] - ) + table.data_table[col] = table.data_table[col].apply(start_time_to_seconds).astype("Int64") + + table.data_table = table.data_table.fillna(numpy.nan).replace([numpy.nan], [None]) table.data_table = table.data_table.replace([""], [None]) table.data_table = table.data_table.rename( @@ -366,9 +348,7 @@ def drop_bus_records(static_tables: Dict[str, StaticTableDetails]) -> None: """ process_logger = ProcessLogger( "gtfs.remove_bus_records", - stop_times_start_row_count=static_tables["stop_times"].data_table.shape[ - 0 - ], + stop_times_start_row_count=static_tables["stop_times"].data_table.shape[0], ) process_logger.log_start() @@ -397,9 +377,9 @@ def drop_bus_records(static_tables: Dict[str, StaticTableDetails]) -> None: # save new stop_times dataframe for RDS insertion static_tables["stop_times"].data_table = stop_times - static_tables["route_patterns"].data_table = static_tables[ - "route_patterns" - ].data_table.merge(no_bus_route_ids, how="inner", on="route_id") + static_tables["route_patterns"].data_table = static_tables["route_patterns"].data_table.merge( + no_bus_route_ids, how="inner", on="route_id" + ) process_logger.add_metadata( stop_times_after_row_count=stop_times.shape[0], @@ -418,15 +398,11 @@ def insert_data_tables( """ try: for table in static_tables.values(): - process_logger = ProcessLogger( - "gtfs_insert", table_name=table.table_name - ) + process_logger = ProcessLogger("gtfs_insert", table_name=table.table_name) process_logger.log_start() if table.data_table.shape[0] > 0: - db_manager.insert_dataframe( - table.data_table, table.insert_table - ) + db_manager.insert_dataframe(table.data_table, table.insert_table) db_manager.vacuum_analyze(table.insert_table) process_logger.log_complete() except Exception as error: @@ -434,13 +410,9 @@ def insert_data_tables( # from all tables matching the same static key. re-raise the error so # it can be properly logged. for table in static_tables.values(): - process_logger = ProcessLogger( - "gtfs_clean", table_name=table.table_name - ) + process_logger = ProcessLogger("gtfs_clean", table_name=table.table_name) process_logger.log_start() - delete_static = sa.delete(table.insert_table).where( - table.static_version_key_column == static_version_key - ) + delete_static = sa.delete(table.insert_table).where(table.static_version_key_column == static_version_key) db_manager.execute(delete_static) process_logger.log_complete() raise error @@ -453,9 +425,7 @@ def process_static_tables( """ process gtfs static table files from metadataLog table """ - process_logger = ProcessLogger( - "l0_tables_loader", table_type="static_schedule" - ) + process_logger = ProcessLogger("l0_tables_loader", table_type="static_schedule") process_logger.log_start() # pull list of objects that need processing from metadata table @@ -465,9 +435,7 @@ def process_static_tables( for folder_data in paths_to_load: check_for_sigterm() folder = str(pathlib.Path(folder_data["paths"][0]).parent) - individual_logger = ProcessLogger( - "l0_load_table", table_type="static_schedule", s3_path=folder - ) + individual_logger = ProcessLogger("l0_load_table", table_type="static_schedule", s3_path=folder) individual_logger.log_start() ids = folder_data["ids"] @@ -479,21 +447,13 @@ def process_static_tables( transform_data_tables(static_tables) drop_bus_records(static_tables) - static_version_key = int( - static_tables["feed_info"].data_table.loc[ - 0, "static_version_key" - ] - ) + static_version_key = int(static_tables["feed_info"].data_table.loc[0, "static_version_key"]) - insert_data_tables( - static_tables, static_version_key, rpm_db_manager - ) + insert_data_tables(static_tables, static_version_key, rpm_db_manager) modify_static_tables(static_version_key, rpm_db_manager) update_md_log = ( - sa.update(MetadataLog.__table__) - .where(MetadataLog.pk_id.in_(ids)) - .values(rail_pm_processed=True) + sa.update(MetadataLog.__table__).where(MetadataLog.pk_id.in_(ids)).values(rail_pm_processed=True) ) md_db_manager.execute(update_md_log) individual_logger.log_complete() diff --git a/src/lamp_py/performance_manager/l0_gtfs_static_mod.py b/src/lamp_py/performance_manager/l0_gtfs_static_mod.py index af3a1605..f06e67e2 100644 --- a/src/lamp_py/performance_manager/l0_gtfs_static_mod.py +++ b/src/lamp_py/performance_manager/l0_gtfs_static_mod.py @@ -10,9 +10,7 @@ from lamp_py.runtime_utils.process_logger import ProcessLogger -def generate_scheduled_travel_times( - static_version_key: int, db_manager: DatabaseManager -) -> None: +def generate_scheduled_travel_times(static_version_key: int, db_manager: DatabaseManager) -> None: """ generate scheduled travel_times and insert into static_stop_times table """ @@ -50,9 +48,7 @@ def generate_scheduled_travel_times( process_logger.log_complete() -def generate_scheduled_branch_headways( - static_version_key: int, db_manager: DatabaseManager -) -> None: +def generate_scheduled_branch_headways(static_version_key: int, db_manager: DatabaseManager) -> None: """ generate scheduled branch headways and insert into static_stop_times table """ @@ -97,9 +93,7 @@ def generate_scheduled_branch_headways( process_logger.log_complete() -def generate_scheduled_trunk_headways( - static_version_key: int, db_manager: DatabaseManager -) -> None: +def generate_scheduled_trunk_headways(static_version_key: int, db_manager: DatabaseManager) -> None: """ generate scheduled trunk headways and insert into static_stop_times table """ @@ -155,9 +149,7 @@ def static_headways_subquery( sa.select( StaticStopTimes.pk_id, StaticStopTimes.departure_time, - sa.func.coalesce( - StaticStops.parent_station, StaticStops.stop_id - ).label("parent_station"), + sa.func.coalesce(StaticStops.parent_station, StaticStops.stop_id).label("parent_station"), StaticTrips.service_id, StaticTrips.direction_id, StaticTrips.trunk_route_id, @@ -168,16 +160,14 @@ def static_headways_subquery( StaticStops, sa.and_( StaticStops.stop_id == StaticStopTimes.stop_id, - StaticStops.static_version_key - == StaticStopTimes.static_version_key, + StaticStops.static_version_key == StaticStopTimes.static_version_key, ), ) .join( StaticTrips, sa.and_( StaticTrips.trip_id == StaticStopTimes.trip_id, - StaticTrips.static_version_key - == StaticStopTimes.static_version_key, + StaticTrips.static_version_key == StaticStopTimes.static_version_key, ), ) .where( @@ -188,9 +178,7 @@ def static_headways_subquery( ).subquery("temp_static_headways") -def modify_static_tables( - static_version_key: int, db_manager: DatabaseManager -) -> None: +def modify_static_tables(static_version_key: int, db_manager: DatabaseManager) -> None: """ This function is responsible for modifying any GTFS static schedule tables after a new schedule as been loaded diff --git a/src/lamp_py/performance_manager/l0_rt_trip_updates.py b/src/lamp_py/performance_manager/l0_rt_trip_updates.py index 5df7d1e7..629c96f2 100644 --- a/src/lamp_py/performance_manager/l0_rt_trip_updates.py +++ b/src/lamp_py/performance_manager/l0_rt_trip_updates.py @@ -18,9 +18,7 @@ ) -def get_tu_dataframe_chunks( - to_load: Union[str, List[str]], route_ids: List[str] -) -> Iterator[pandas.DataFrame]: +def get_tu_dataframe_chunks(to_load: Union[str, List[str]], route_ids: List[str]) -> Iterator[pandas.DataFrame]: """ return interator of dataframe chunks from a trip updates parquet file (or list of files) @@ -55,9 +53,7 @@ def get_tu_dataframe_chunks( ) -def get_and_unwrap_tu_dataframe( - paths: Union[str, List[str]], route_ids: List[str] -) -> pandas.DataFrame: +def get_and_unwrap_tu_dataframe(paths: Union[str, List[str]], route_ids: List[str]) -> pandas.DataFrame: """ get trip updates records from parquet files to create predicted trip update stop events @@ -94,43 +90,23 @@ def get_and_unwrap_tu_dataframe( batch_events = batch_events.drop(columns=["feed_timestamp"]) # store start_date as int64 and rename to service_date - batch_events.rename( - columns={"start_date": "service_date"}, inplace=True - ) - batch_events["service_date"] = pandas.to_numeric( - batch_events["service_date"] - ).astype("Int64") + batch_events.rename(columns={"start_date": "service_date"}, inplace=True) + batch_events["service_date"] = pandas.to_numeric(batch_events["service_date"]).astype("Int64") # store direction_id as bool - batch_events["direction_id"] = pandas.to_numeric( - batch_events["direction_id"] - ).astype(numpy.bool_) + batch_events["direction_id"] = pandas.to_numeric(batch_events["direction_id"]).astype(numpy.bool_) # store start_time as seconds from start of day int64 - batch_events["start_time"] = ( - batch_events["start_time"] - .apply(start_time_to_seconds) - .astype("Int64") - ) + batch_events["start_time"] = batch_events["start_time"].apply(start_time_to_seconds).astype("Int64") - batch_events["tu_stop_timestamp"] = pandas.to_numeric( - batch_events["tu_stop_timestamp"] - ).astype("Int64") + batch_events["tu_stop_timestamp"] = pandas.to_numeric(batch_events["tu_stop_timestamp"]).astype("Int64") # filter out stop event predictions that are too far into the future # and are unlikely to be used as a final stop event prediction # (2 minutes) or predictions that go into the past (negative values) batch_events = batch_events[ - ( - batch_events["tu_stop_timestamp"] - - batch_events["timestamp"] - >= 0 - ) - & ( - batch_events["tu_stop_timestamp"] - - batch_events["timestamp"] - < 120 - ) + (batch_events["tu_stop_timestamp"] - batch_events["timestamp"] >= 0) + & (batch_events["tu_stop_timestamp"] - batch_events["timestamp"] < 120) ] trip_updates = pandas.concat([trip_updates, batch_events]) @@ -151,9 +127,7 @@ def reduce_trip_updates(trip_updates: pandas.DataFrame) -> pandas.DataFrame: """ reduce the data frame to a single record per trip / stop. """ - process_logger = ProcessLogger( - "tu.reduce", start_row_count=trip_updates.shape[0] - ) + process_logger = ProcessLogger("tu.reduce", start_row_count=trip_updates.shape[0]) process_logger.log_start() trip_stop_columns = unique_trip_stop_columns() @@ -162,16 +136,12 @@ def reduce_trip_updates(trip_updates: pandas.DataFrame) -> pandas.DataFrame: # for the same trip and same station but the first one. the first update will # be the most recent arrival time prediction trip_updates = trip_updates.sort_values(by=["timestamp"], ascending=False) - trip_updates = trip_updates.drop_duplicates( - subset=trip_stop_columns, keep="first" - ) + trip_updates = trip_updates.drop_duplicates(subset=trip_stop_columns, keep="first") # after group and sort, "timestamp" longer needed trip_updates = trip_updates.drop(columns=["timestamp"]) - trip_updates["tu_stop_timestamp"] = trip_updates[ - "tu_stop_timestamp" - ].astype("Int64") + trip_updates["tu_stop_timestamp"] = trip_updates["tu_stop_timestamp"].astype("Int64") # add selected columns # trip_updates and vehicle_positions dataframes must all have the same columns @@ -194,17 +164,13 @@ def process_tu_files( """ Generate a dataframe of Vehicle Events from gtfs_rt trip updates parquet files. """ - process_logger = ProcessLogger( - "process_trip_updates", file_count=len(paths), paths=paths - ) + process_logger = ProcessLogger("process_trip_updates", file_count=len(paths), paths=paths) process_logger.log_start() route_ids = rail_routes_from_filepath(paths, db_manager) trip_updates = get_and_unwrap_tu_dataframe(paths, route_ids) if trip_updates.shape[0] > 0: - trip_updates = add_missing_service_dates( - events_dataframe=trip_updates, timestamp_key="timestamp" - ) + trip_updates = add_missing_service_dates(events_dataframe=trip_updates, timestamp_key="timestamp") trip_updates = add_static_version_key_column(trip_updates, db_manager) trip_updates = add_parent_station_column(trip_updates, db_manager) trip_updates = reduce_trip_updates(trip_updates) diff --git a/src/lamp_py/performance_manager/l0_rt_vehicle_positions.py b/src/lamp_py/performance_manager/l0_rt_vehicle_positions.py index 40ab0606..4fea94b0 100644 --- a/src/lamp_py/performance_manager/l0_rt_vehicle_positions.py +++ b/src/lamp_py/performance_manager/l0_rt_vehicle_positions.py @@ -17,9 +17,7 @@ ) -def get_vp_dataframe( - to_load: Union[str, List[str]], route_ids: List[str] -) -> pandas.DataFrame: +def get_vp_dataframe(to_load: Union[str, List[str]], route_ids: List[str]) -> pandas.DataFrame: """ return a dataframe from a vehicle position parquet file (or list of files) with expected columns without null data. @@ -94,9 +92,7 @@ def transform_vp_datatypes( ingest dataframe of vehicle position data from parquet file and transform column datatypes """ - process_logger = ProcessLogger( - "vp.transform_datatypes", row_count=vehicle_positions.shape[0] - ) + process_logger = ProcessLogger("vp.transform_datatypes", row_count=vehicle_positions.shape[0]) process_logger.log_start() # current_status: 1 = MOVING, 0 = STOPPED_AT @@ -106,38 +102,22 @@ def transform_vp_datatypes( vehicle_positions = vehicle_positions.drop(columns=["current_status"]) # rename start_date to service date and store as int64 instead of string - vehicle_positions.rename( - columns={"start_date": "service_date"}, inplace=True - ) - vehicle_positions["service_date"] = pandas.to_numeric( - vehicle_positions["service_date"] - ).astype("Int64") + vehicle_positions.rename(columns={"start_date": "service_date"}, inplace=True) + vehicle_positions["service_date"] = pandas.to_numeric(vehicle_positions["service_date"]).astype("Int64") # rename current_stop_sequence to stop_sequence # and convert to int64 - vehicle_positions.rename( - columns={"current_stop_sequence": "stop_sequence"}, inplace=True - ) - vehicle_positions["stop_sequence"] = pandas.to_numeric( - vehicle_positions["stop_sequence"] - ).astype("int64") + vehicle_positions.rename(columns={"current_stop_sequence": "stop_sequence"}, inplace=True) + vehicle_positions["stop_sequence"] = pandas.to_numeric(vehicle_positions["stop_sequence"]).astype("int64") # store direction_id as bool - vehicle_positions["direction_id"] = pandas.to_numeric( - vehicle_positions["direction_id"] - ).astype(numpy.bool_) + vehicle_positions["direction_id"] = pandas.to_numeric(vehicle_positions["direction_id"]).astype(numpy.bool_) # fix revenue field, NULL is True - vehicle_positions["revenue"] = numpy.where( - vehicle_positions["revenue"].eq(False), False, True - ).astype(numpy.bool_) + vehicle_positions["revenue"] = numpy.where(vehicle_positions["revenue"].eq(False), False, True).astype(numpy.bool_) # store start_time as seconds from start of day as int64 - vehicle_positions["start_time"] = ( - vehicle_positions["start_time"] - .apply(start_time_to_seconds) - .astype("Int64") - ) + vehicle_positions["start_time"] = vehicle_positions["start_time"].apply(start_time_to_seconds).astype("Int64") process_logger.log_complete() return vehicle_positions @@ -155,9 +135,7 @@ def transform_vp_timestamps( this method will remove "is_moving" and "vehicle_timestamp" """ - process_logger = ProcessLogger( - "vp.transform_timestamps", start_row_count=vehicle_positions.shape[0] - ) + process_logger = ProcessLogger("vp.transform_timestamps", start_row_count=vehicle_positions.shape[0]) process_logger.log_start() trip_stop_columns = unique_trip_stop_columns() @@ -173,9 +151,7 @@ def transform_vp_timestamps( aggfunc={"vehicle_timestamp": "min"}, ).reset_index(drop=False) - rename_mapper: Dict[Tuple[str, Union[str, bool]], str] = { - (column, ""): column for column in trip_stop_columns - } + rename_mapper: Dict[Tuple[str, Union[str, bool]], str] = {(column, ""): column for column in trip_stop_columns} rename_mapper.update({("vehicle_timestamp", True): "vp_move_timestamp"}) rename_mapper.update({("vehicle_timestamp", False): "vp_stop_timestamp"}) @@ -188,9 +164,9 @@ def transform_vp_timestamps( # we no longer need is moving or vehicle timestamp as those are all # stored in the vp_timestamps dataframe. drop duplicated trip-stop events - vehicle_positions = vehicle_positions.drop( - columns=["is_moving", "vehicle_timestamp"] - ).drop_duplicates(subset=trip_stop_columns) + vehicle_positions = vehicle_positions.drop(columns=["is_moving", "vehicle_timestamp"]).drop_duplicates( + subset=trip_stop_columns + ) # join the timestamps to trip-stop details, leaving us with vp move and # stop times @@ -202,25 +178,17 @@ def transform_vp_timestamps( validate="one_to_one", ) - vehicle_positions["vp_move_timestamp"] = vehicle_positions[ - "vp_move_timestamp" - ].astype("Int64") - vehicle_positions["vp_stop_timestamp"] = vehicle_positions[ - "vp_stop_timestamp" - ].astype("Int64") + vehicle_positions["vp_move_timestamp"] = vehicle_positions["vp_move_timestamp"].astype("Int64") + vehicle_positions["vp_stop_timestamp"] = vehicle_positions["vp_stop_timestamp"].astype("Int64") # change vehicle_consist to pipe delimited string - vehicle_positions["vehicle_consist"] = vehicle_positions[ - "vehicle_consist" - ].map( + vehicle_positions["vehicle_consist"] = vehicle_positions["vehicle_consist"].map( lambda vc: "|".join(str(vc_val["label"]) for vc_val in vc), na_action="ignore", ) # change multi_carriage_details to pipe delimited string - vehicle_positions["multi_carriage_details"] = vehicle_positions[ - "multi_carriage_details" - ].map( + vehicle_positions["multi_carriage_details"] = vehicle_positions["multi_carriage_details"].map( lambda vc: "|".join(str(vc_val["label"]) for vc_val in vc), na_action="ignore", ) @@ -232,9 +200,7 @@ def transform_vp_timestamps( vehicle_positions["multi_carriage_details"], vehicle_positions["vehicle_consist"], ) - vehicle_positions = vehicle_positions.drop( - columns=["multi_carriage_details"] - ) + vehicle_positions = vehicle_positions.drop(columns=["multi_carriage_details"]) process_logger.add_metadata(after_row_count=vehicle_positions.shape[0]) process_logger.log_complete() @@ -248,24 +214,16 @@ def process_vp_files( """ Generate a dataframe of Vehicle Events from gtfs_rt vehicle position parquet files. """ - process_logger = ProcessLogger( - "process_vehicle_positions", file_count=len(paths), paths=paths - ) + process_logger = ProcessLogger("process_vehicle_positions", file_count=len(paths), paths=paths) process_logger.log_start() route_ids = rail_routes_from_filepath(paths, db_manager) vehicle_positions = get_vp_dataframe(paths, route_ids) if vehicle_positions.shape[0] > 0: vehicle_positions = transform_vp_datatypes(vehicle_positions) - vehicle_positions = add_missing_service_dates( - vehicle_positions, timestamp_key="vehicle_timestamp" - ) - vehicle_positions = add_static_version_key_column( - vehicle_positions, db_manager - ) - vehicle_positions = add_parent_station_column( - vehicle_positions, db_manager - ) + vehicle_positions = add_missing_service_dates(vehicle_positions, timestamp_key="vehicle_timestamp") + vehicle_positions = add_static_version_key_column(vehicle_positions, db_manager) + vehicle_positions = add_parent_station_column(vehicle_positions, db_manager) vehicle_positions = transform_vp_timestamps(vehicle_positions) process_logger.add_metadata(vehicle_events_count=vehicle_positions.shape[0]) diff --git a/src/lamp_py/performance_manager/l1_cte_statements.py b/src/lamp_py/performance_manager/l1_cte_statements.py index 491e9ef1..3c5d236a 100644 --- a/src/lamp_py/performance_manager/l1_cte_statements.py +++ b/src/lamp_py/performance_manager/l1_cte_statements.py @@ -11,9 +11,7 @@ ) -def static_trips_subquery( - static_version_key: int, service_date: int -) -> sa.sql.selectable.Subquery: +def static_trips_subquery(static_version_key: int, service_date: int) -> sa.sql.selectable.Subquery: """ return Selectable representing all static trips on given service_date and static_version_key value combo @@ -72,24 +70,21 @@ def static_trips_subquery( .join( StaticTrips, sa.and_( - StaticStopTimes.static_version_key - == StaticTrips.static_version_key, + StaticStopTimes.static_version_key == StaticTrips.static_version_key, StaticStopTimes.trip_id == StaticTrips.trip_id, ), ) .join( StaticStops, sa.and_( - StaticStopTimes.static_version_key - == StaticStops.static_version_key, + StaticStopTimes.static_version_key == StaticStops.static_version_key, StaticStopTimes.stop_id == StaticStops.stop_id, ), ) .join( ServiceIdDates, sa.and_( - StaticStopTimes.static_version_key - == ServiceIdDates.static_version_key, + StaticStopTimes.static_version_key == ServiceIdDates.static_version_key, StaticTrips.service_id == ServiceIdDates.service_id, StaticTrips.route_id == ServiceIdDates.route_id, ), @@ -97,8 +92,7 @@ def static_trips_subquery( .join( StaticRoutes, sa.and_( - StaticStopTimes.static_version_key - == StaticRoutes.static_version_key, + StaticStopTimes.static_version_key == StaticRoutes.static_version_key, StaticTrips.route_id == StaticRoutes.route_id, ), ) @@ -183,9 +177,7 @@ def rt_trips_subquery(service_date: int) -> sa.sql.selectable.Subquery: ).subquery(name="rt_trips_sub") -def trips_for_metrics_subquery( - static_version_key: int, service_date: int -) -> sa.sql.selectable.Subquery: +def trips_for_metrics_subquery(static_version_key: int, service_date: int) -> sa.sql.selectable.Subquery: """ return Selectable named "trips_for_metrics" with fields needed to develop metrics tables @@ -242,14 +234,10 @@ def trips_for_metrics_subquery( .join( static_trips_sub, sa.and_( - rt_trips_sub.c.static_trip_id_guess - == static_trips_sub.c.static_trip_id, - rt_trips_sub.c.static_version_key - == static_trips_sub.c.static_version_key, - rt_trips_sub.c.parent_station - == static_trips_sub.c.parent_station, - rt_trips_sub.c.rt_trip_stop_rank - >= static_trips_sub.c.static_trip_stop_rank, + rt_trips_sub.c.static_trip_id_guess == static_trips_sub.c.static_trip_id, + rt_trips_sub.c.static_version_key == static_trips_sub.c.static_version_key, + rt_trips_sub.c.parent_station == static_trips_sub.c.parent_station, + rt_trips_sub.c.rt_trip_stop_rank >= static_trips_sub.c.static_trip_stop_rank, ), isouter=True, ) diff --git a/src/lamp_py/performance_manager/l1_rt_metrics.py b/src/lamp_py/performance_manager/l1_rt_metrics.py index 99f9e8be..4bae1a4e 100644 --- a/src/lamp_py/performance_manager/l1_rt_metrics.py +++ b/src/lamp_py/performance_manager/l1_rt_metrics.py @@ -27,9 +27,7 @@ def update_metrics_columns( process_logger = ProcessLogger("l1_rt_metrics_table_loader") process_logger.log_start() - trips_for_metrics = trips_for_metrics_subquery( - static_version_key, seed_service_date - ) + trips_for_metrics = trips_for_metrics_subquery(static_version_key, seed_service_date) trips_for_headways = trips_for_headways_subquery( service_date=seed_service_date, ) @@ -40,18 +38,14 @@ def update_metrics_columns( # negative travel times are error records, should flag??? update_travel_times = ( sa.update(VehicleEvents.__table__) - .values( - travel_time_seconds=trips_for_metrics.c.stop_timestamp - - trips_for_metrics.c.move_timestamp - ) + .values(travel_time_seconds=trips_for_metrics.c.stop_timestamp - trips_for_metrics.c.move_timestamp) .where( VehicleEvents.pm_trip_id == trips_for_metrics.c.pm_trip_id, VehicleEvents.service_date == trips_for_metrics.c.service_date, VehicleEvents.parent_station == trips_for_metrics.c.parent_station, trips_for_metrics.c.stop_timestamp.is_not(None), trips_for_metrics.c.move_timestamp.is_not(None), - trips_for_metrics.c.stop_timestamp - > trips_for_metrics.c.move_timestamp, + trips_for_metrics.c.stop_timestamp > trips_for_metrics.c.move_timestamp, ) ) @@ -146,14 +140,11 @@ def update_metrics_columns( # get it to work with sqlalchemy update_branch_headways = ( sa.update(VehicleEvents.__table__) - .values( - headway_branch_seconds=t_headways_branch_sub.c.headway_branch_seconds - ) + .values(headway_branch_seconds=t_headways_branch_sub.c.headway_branch_seconds) .where( VehicleEvents.pm_trip_id == t_headways_branch_sub.c.pm_trip_id, VehicleEvents.service_date == t_headways_branch_sub.c.service_date, - VehicleEvents.parent_station - == t_headways_branch_sub.c.parent_station, + VehicleEvents.parent_station == t_headways_branch_sub.c.parent_station, t_headways_branch_sub.c.headway_branch_seconds.is_not(None), t_headways_branch_sub.c.headway_branch_seconds > 0, ) @@ -190,14 +181,11 @@ def update_metrics_columns( # get it to work with sqlalchemy update_trunk_headways = ( sa.update(VehicleEvents.__table__) - .values( - headway_trunk_seconds=t_headways_trunk_sub.c.headway_trunk_seconds - ) + .values(headway_trunk_seconds=t_headways_trunk_sub.c.headway_trunk_seconds) .where( VehicleEvents.pm_trip_id == t_headways_trunk_sub.c.pm_trip_id, VehicleEvents.service_date == t_headways_trunk_sub.c.service_date, - VehicleEvents.parent_station - == t_headways_trunk_sub.c.parent_station, + VehicleEvents.parent_station == t_headways_trunk_sub.c.parent_station, t_headways_trunk_sub.c.headway_trunk_seconds.is_not(None), t_headways_trunk_sub.c.headway_trunk_seconds > 0, ) diff --git a/src/lamp_py/performance_manager/l1_rt_trips.py b/src/lamp_py/performance_manager/l1_rt_trips.py index 14e1fb96..6cbeba83 100644 --- a/src/lamp_py/performance_manager/l1_rt_trips.py +++ b/src/lamp_py/performance_manager/l1_rt_trips.py @@ -202,9 +202,7 @@ def update_static_version_key(db_manager: DatabaseManager) -> None: version_key_sub = ( sa.select( TempEventCompare.service_date, - sa.func.max(TempEventCompare.static_version_key).label( - "max_version_key" - ), + sa.func.max(TempEventCompare.static_version_key).label("max_version_key"), ) .group_by( TempEventCompare.service_date, @@ -240,9 +238,7 @@ def update_start_times(db_manager: DatabaseManager) -> None: missing_start_times = ( sa.select(TempEventCompare.pm_trip_id) .distinct() - .join( - VehicleTrips, VehicleTrips.pm_trip_id == TempEventCompare.pm_trip_id - ) + .join(VehicleTrips, VehicleTrips.pm_trip_id == TempEventCompare.pm_trip_id) .where( TempEventCompare.start_time.is_(None), VehicleTrips.start_time.is_(None), @@ -267,8 +263,7 @@ def update_start_times(db_manager: DatabaseManager) -> None: StaticStopTimes, sa.and_( VehicleTrips.trip_id == StaticStopTimes.trip_id, - VehicleTrips.static_version_key - == StaticStopTimes.static_version_key, + VehicleTrips.static_version_key == StaticStopTimes.static_version_key, ), ) .group_by( @@ -315,9 +310,7 @@ def update_start_times(db_manager: DatabaseManager) -> None: if unscheduled_start_times.shape[0] > 0: unscheduled_start_times["b_start_time"] = ( - unscheduled_start_times["b_start_time"] - .apply(start_timestamp_to_seconds) - .astype("int64") + unscheduled_start_times["b_start_time"].apply(start_timestamp_to_seconds).astype("int64") ) start_times_update_query = ( @@ -338,9 +331,7 @@ def update_branch_trunk_route_id(db_manager: DatabaseManager) -> None: update `branch_route_id` and `trunk_route_id` fields in trips table """ distinct_t_trips = ( - sa.select(TempEventCompare.service_date, TempEventCompare.pm_trip_id) - .distinct() - .subquery("distinct_trips") + sa.select(TempEventCompare.service_date, TempEventCompare.pm_trip_id).distinct().subquery("distinct_trips") ) distinct_trips = ( @@ -414,9 +405,7 @@ def get_red_branch(pm_trip_id: int) -> Optional[str]: "70095", "70096", } - trip_stop_ids = set( - red_events_df[red_events_df["pm_trip_id"] == pm_trip_id]["stop_id"] - ) + trip_stop_ids = set(red_events_df[red_events_df["pm_trip_id"] == pm_trip_id]["stop_id"]) if trip_stop_ids & ashmont_stop_ids: return "Red-A" if trip_stop_ids & braintree_stop_ids: @@ -474,9 +463,7 @@ def update_trip_stop_counts(db_manager: DatabaseManager) -> None: Update "stop_count" field for trips with new events """ distinct_trips = ( - sa.select(TempEventCompare.service_date, TempEventCompare.pm_trip_id) - .distinct() - .subquery("distinct_trips") + sa.select(TempEventCompare.service_date, TempEventCompare.pm_trip_id).distinct().subquery("distinct_trips") ) new_stop_counts_cte = ( @@ -535,13 +522,7 @@ def update_static_trip_id_guess_exact(db_manager: DatabaseManager) -> None: ) .where( StaticStopTimes.static_version_key - == sa.func.any( - sa.func.array( - sa.select(TempEventCompare.static_version_key) - .distinct() - .scalar_subquery() - ) - ) + == sa.func.any(sa.func.array(sa.select(TempEventCompare.static_version_key).distinct().scalar_subquery())) ) .group_by( StaticStopTimes.static_version_key, @@ -563,16 +544,14 @@ def update_static_trip_id_guess_exact(db_manager: DatabaseManager) -> None: .join( StaticTrips, sa.and_( - StaticTrips.static_version_key - == TempEventCompare.static_version_key, + StaticTrips.static_version_key == TempEventCompare.static_version_key, StaticTrips.trip_id == TempEventCompare.trip_id, ), ) .join( static_stop_sub, sa.and_( - static_stop_sub.c.static_version_key - == TempEventCompare.static_version_key, + static_stop_sub.c.static_version_key == TempEventCompare.static_version_key, static_stop_sub.c.trip_id == TempEventCompare.trip_id, ), ) @@ -624,8 +603,7 @@ def update_directions(db_manager: DatabaseManager) -> None: .join( StaticDirections, sa.and_( - temp_trips.c.static_version_key - == StaticDirections.static_version_key, + temp_trips.c.static_version_key == StaticDirections.static_version_key, temp_trips.c.direction_id == StaticDirections.direction_id, temp_trips.c.route_id == StaticDirections.route_id, ), @@ -671,15 +649,11 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: StaticRoutePatterns.direction_id, StaticRoutePatterns.representative_trip_id, StaticTrips.trunk_route_id, - sa.func.coalesce( - StaticTrips.branch_route_id, StaticTrips.trunk_route_id - ).label("route_id"), + sa.func.coalesce(StaticTrips.branch_route_id, StaticTrips.trunk_route_id).label("route_id"), StaticRoutePatterns.static_version_key, ) .distinct( - sa.func.coalesce( - StaticTrips.branch_route_id, StaticTrips.trunk_route_id - ), + sa.func.coalesce(StaticTrips.branch_route_id, StaticTrips.trunk_route_id), StaticRoutePatterns.direction_id, StaticRoutePatterns.static_version_key, ) @@ -687,24 +661,19 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: .join( StaticTrips, sa.and_( - StaticRoutePatterns.representative_trip_id - == StaticTrips.trip_id, - StaticRoutePatterns.static_version_key - == StaticTrips.static_version_key, + StaticRoutePatterns.representative_trip_id == StaticTrips.trip_id, + StaticRoutePatterns.static_version_key == StaticTrips.static_version_key, ), ) .where( - StaticRoutePatterns.static_version_key - == record["static_version_key"], + StaticRoutePatterns.static_version_key == record["static_version_key"], sa.or_( StaticRoutePatterns.route_pattern_typicality == 1, StaticRoutePatterns.route_pattern_typicality == 5, ), ) .order_by( - sa.func.coalesce( - StaticTrips.branch_route_id, StaticTrips.trunk_route_id - ), + sa.func.coalesce(StaticTrips.branch_route_id, StaticTrips.trunk_route_id), StaticRoutePatterns.direction_id, StaticRoutePatterns.static_version_key, StaticRoutePatterns.route_pattern_typicality.desc(), @@ -737,18 +706,15 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: .join( StaticStopTimes, sa.and_( - canon_trips.c.representative_trip_id - == StaticStopTimes.trip_id, - canon_trips.c.static_version_key - == StaticStopTimes.static_version_key, + canon_trips.c.representative_trip_id == StaticStopTimes.trip_id, + canon_trips.c.static_version_key == StaticStopTimes.static_version_key, ), ) .join( StaticStops, sa.and_( StaticStopTimes.stop_id == StaticStops.stop_id, - StaticStopTimes.static_version_key - == StaticStops.static_version_key, + StaticStopTimes.static_version_key == StaticStops.static_version_key, ), ) .subquery("static_canon") @@ -774,10 +740,8 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: VehicleTrips.trunk_route_id, ) == static_canon.c.route_id, - VehicleTrips.static_version_key - == static_canon.c.static_version_key, - VehicleEvents.parent_station - == static_canon.c.parent_station, + VehicleTrips.static_version_key == static_canon.c.static_version_key, + VehicleEvents.parent_station == static_canon.c.parent_station, ), ) .where( @@ -826,10 +790,7 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: count( static_canon.c.stop_sequence, ).desc(), - ( - sa.func.max(static_canon.c.stop_sequence) - - sa.func.min(static_canon.c.stop_sequence) - ).desc(), + (sa.func.max(static_canon.c.stop_sequence) - sa.func.min(static_canon.c.stop_sequence)).desc(), ) .subquery("zero_points") ) @@ -846,10 +807,8 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: .join( zero_point_stop, sa.and_( - zero_point_stop.c.trunk_route_id - == static_canon.c.trunk_route_id, - zero_point_stop.c.parent_station - == static_canon.c.parent_station, + zero_point_stop.c.trunk_route_id == static_canon.c.trunk_route_id, + zero_point_stop.c.parent_station == static_canon.c.parent_station, ), ) .subquery("zero_seq_vals") @@ -863,9 +822,7 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: static_canon.c.direction_id, static_canon.c.trunk_route_id, sa.func.min(static_canon.c.stop_sequence).label("min_seq"), - sa.func.min( - static_canon.c.stop_sequence - zero_seq_vals.c.seq_adjust - ).label("min_sync"), + sa.func.min(static_canon.c.stop_sequence - zero_seq_vals.c.seq_adjust).label("min_sync"), ) .select_from(static_canon) .join( @@ -910,10 +867,8 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: .join( sync_adjust_vals, sa.and_( - sync_adjust_vals.c.direction_id - == static_canon.c.direction_id, - sync_adjust_vals.c.trunk_route_id - == static_canon.c.trunk_route_id, + sync_adjust_vals.c.direction_id == static_canon.c.direction_id, + sync_adjust_vals.c.trunk_route_id == static_canon.c.trunk_route_id, ), ) .subquery(("sync_values")) @@ -934,10 +889,8 @@ def update_stop_sequence(db_manager: DatabaseManager) -> None: sa.and_( VehicleTrips.direction_id == sync_values.c.direction_id, VehicleTrips.trunk_route_id == sync_values.c.trunk_route_id, - VehicleTrips.static_version_key - == sync_values.c.static_version_key, - VehicleEvents.parent_station - == sync_values.c.parent_station, + VehicleTrips.static_version_key == sync_values.c.static_version_key, + VehicleEvents.parent_station == sync_values.c.parent_station, ), ) .where( @@ -979,9 +932,7 @@ def backup_rt_static_trip_match( this matches an RT trip to a static trip with the same branch_route_id or trunk_route_id if branch is null and direction with the closest start_time """ - static_trips_sub = static_trips_subquery( - static_version_key, seed_service_date - ) + static_trips_sub = static_trips_subquery(static_version_key, seed_service_date) # to build a 'summary' trips table only the first and last records for each # static trip are needed. @@ -1010,8 +961,7 @@ def backup_rt_static_trip_match( .select_from(static_trips_sub) .join( first_stop_static_sub, - static_trips_sub.c.static_trip_id - == first_stop_static_sub.c.static_trip_id, + static_trips_sub.c.static_trip_id == first_stop_static_sub.c.static_trip_id, ) .where(static_trips_sub.c.static_trip_last_stop == sa.true()) .subquery(name="static_trips_summary_sub") @@ -1030,9 +980,7 @@ def backup_rt_static_trip_match( sa.select( VehicleTrips.pm_trip_id, VehicleTrips.direction_id, - sa.func.coalesce( - VehicleTrips.branch_route_id, VehicleTrips.trunk_route_id - ).label("route_id"), + sa.func.coalesce(VehicleTrips.branch_route_id, VehicleTrips.trunk_route_id).label("route_id"), VehicleTrips.start_time, ) .select_from(VehicleTrips) @@ -1065,18 +1013,13 @@ def backup_rt_static_trip_match( .join( static_trips_summary_sub, sa.and_( - rt_trips_summary_sub.c.direction_id - == static_trips_summary_sub.c.direction_id, - rt_trips_summary_sub.c.route_id - == static_trips_summary_sub.c.route_id, + rt_trips_summary_sub.c.direction_id == static_trips_summary_sub.c.direction_id, + rt_trips_summary_sub.c.route_id == static_trips_summary_sub.c.route_id, ), ) .order_by( rt_trips_summary_sub.c.pm_trip_id, - sa.func.abs( - rt_trips_summary_sub.c.start_time - - static_trips_summary_sub.c.static_start_time - ), + sa.func.abs(rt_trips_summary_sub.c.start_time - static_trips_summary_sub.c.static_start_time), ) ).subquery(name="backup_trips_match") diff --git a/src/lamp_py/performance_manager/pipeline.py b/src/lamp_py/performance_manager/pipeline.py index 45360ce8..192f00e7 100755 --- a/src/lamp_py/performance_manager/pipeline.py +++ b/src/lamp_py/performance_manager/pipeline.py @@ -65,12 +65,8 @@ def main(args: argparse.Namespace) -> None: main_process_logger.log_start() # get the engine that manages sessions that read and write to the db - rpm_db_manager = DatabaseManager( - db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER, verbose=args.verbose - ) - md_db_manager = DatabaseManager( - db_index=DatabaseIndex.METADATA, verbose=args.verbose - ) + rpm_db_manager = DatabaseManager(db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER, verbose=args.verbose) + md_db_manager = DatabaseManager(db_index=DatabaseIndex.METADATA, verbose=args.verbose) # schedule object that will control the "event loop" scheduler = sched.scheduler(time.monotonic, time.sleep) diff --git a/src/lamp_py/postgres/postgres_utils.py b/src/lamp_py/postgres/postgres_utils.py index 13f42c41..cb57b737 100644 --- a/src/lamp_py/postgres/postgres_utils.py +++ b/src/lamp_py/postgres/postgres_utils.py @@ -15,7 +15,7 @@ import pyarrow import pyarrow.parquet as pq -from lamp_py.aws.s3 import get_datetime_from_partition_path +from lamp_py.aws.s3 import dt_from_obj_path from lamp_py.runtime_utils.process_logger import ProcessLogger from .metadata_schema import MetadataLog @@ -135,18 +135,12 @@ def get_local_engine( # set the ssl cert path to the file that should be added to the # lambda function at deploy time - db_ssl_cert = os.path.abspath( - os.path.join( - "/", "usr", "local", "share", "amazon-certs.pem" - ) - ) + db_ssl_cert = os.path.abspath(os.path.join("/", "usr", "local", "share", "amazon-certs.pem")) assert os.path.isfile(db_ssl_cert) # update the ssl options string to add to the database url - db_ssl_options = ( - f"?sslmode=verify-full&sslrootcert={db_ssl_cert}" - ) + db_ssl_options = f"?sslmode=verify-full&sslrootcert={db_ssl_cert}" database_url = ( f"postgresql+psycopg2://{self.user}:" @@ -345,9 +339,7 @@ def execute_with_data( return result # type: ignore - def insert_dataframe( - self, dataframe: pandas.DataFrame, insert_table: Any - ) -> None: + def insert_dataframe(self, dataframe: pandas.DataFrame, insert_table: Any) -> None: """ insert data into db table from pandas dataframe """ @@ -359,20 +351,14 @@ def insert_dataframe( dataframe.to_dict(orient="records"), ) - def select_as_dataframe( - self, select_query: sa.sql.selectable.Select - ) -> pandas.DataFrame: + def select_as_dataframe(self, select_query: sa.sql.selectable.Select) -> pandas.DataFrame: """ select data from db table and return pandas dataframe """ with self.session.begin() as cursor: - return pandas.DataFrame( - [row._asdict() for row in cursor.execute(select_query)] - ) + return pandas.DataFrame([row._asdict() for row in cursor.execute(select_query)]) - def select_as_list( - self, select_query: sa.sql.selectable.Select - ) -> Union[List[Any], List[Dict[str, Any]]]: + def select_as_list(self, select_query: sa.sql.selectable.Select) -> Union[List[Any], List[Dict[str, Any]]]: """ select data from db table and return list """ @@ -418,9 +404,7 @@ def write_to_parquet( with pq.ParquetWriter(write_path, schema=schema) as pq_writer: for part in cursor.execute(part_stmt).partitions(batch_size): pq_writer.write_batch( - pyarrow.RecordBatch.from_pylist( - [row._asdict() for row in part], schema=schema - ) + pyarrow.RecordBatch.from_pylist([row._asdict() for row in part], schema=schema) ) process_logger.log_complete() @@ -471,9 +455,7 @@ def _disable_trip_trigger(self) -> None: simple queries. """ table = self._get_schema_table(VehicleTrips) - disable_trigger = ( - f"ALTER TABLE {table} DISABLE TRIGGER rt_trips_update_branch_trunk;" - ) + disable_trigger = f"ALTER TABLE {table} DISABLE TRIGGER rt_trips_update_branch_trunk;" with self.session.begin() as cursor: cursor.execute(sa.text(disable_trigger)) @@ -483,9 +465,7 @@ def _enable_trip_trigger(self) -> None: ENABLE rt_trips_update_branch_trunk TRIGGER on vehicle_trips table """ table = self._get_schema_table(VehicleTrips) - enable_trigger = ( - f"ALTER TABLE {table} ENABLE TRIGGER rt_trips_update_branch_trunk;" - ) + enable_trigger = f"ALTER TABLE {table} ENABLE TRIGGER rt_trips_update_branch_trunk;" with self.session.begin() as cursor: cursor.execute(sa.text(enable_trigger)) @@ -518,21 +498,18 @@ def get_unprocessed_files( "paths": [s3 paths of parquet files that share path] } """ - process_logger = ProcessLogger( - "get_unprocessed_files", seed_string=path_contains - ) + process_logger = ProcessLogger("get_unprocessed_files", seed_string=path_contains) process_logger.log_start() paths_to_load: Dict[float, Dict[str, List]] = {} try: read_md_log = sa.select(MetadataLog.pk_id, MetadataLog.path).where( - (MetadataLog.rail_pm_processed == sa.false()) - & (MetadataLog.path.contains(path_contains)) + (MetadataLog.rail_pm_processed == sa.false()) & (MetadataLog.path.contains(path_contains)) ) for path_record in db_manager.select_as_list(read_md_log): path_id = path_record.get("pk_id") path = str(path_record.get("path")) - path_timestamp = get_datetime_from_partition_path(path).timestamp() + path_timestamp = dt_from_obj_path(path).timestamp() if path_timestamp not in paths_to_load: paths_to_load[path_timestamp] = {"ids": [], "paths": []} @@ -544,18 +521,14 @@ def get_unprocessed_files( if file_limit is not None: paths_returned = file_limit - process_logger.add_metadata( - paths_found=paths_found, paths_returned=paths_returned - ) + process_logger.add_metadata(paths_found=paths_found, paths_returned=paths_returned) process_logger.log_complete() except Exception as exception: process_logger.log_failure(exception) - return [ - paths_to_load[timestamp] for timestamp in sorted(paths_to_load.keys()) - ][:file_limit] + return [paths_to_load[timestamp] for timestamp in sorted(paths_to_load.keys())][:file_limit] def _rds_writer_process(metadata_queue: Queue[Optional[str]]) -> None: diff --git a/src/lamp_py/postgres/rail_performance_manager_schema.py b/src/lamp_py/postgres/rail_performance_manager_schema.py index 71e27caa..26937dfe 100644 --- a/src/lamp_py/postgres/rail_performance_manager_schema.py +++ b/src/lamp_py/postgres/rail_performance_manager_schema.py @@ -108,9 +108,7 @@ class VehicleTrips(RpmSqlBase): # pylint: disable=too-few-public-methods static_trip_id_guess = sa.Column(sa.String(512), nullable=True) static_start_time = sa.Column(sa.Integer, nullable=True) static_stop_count = sa.Column(sa.SmallInteger, nullable=True) - first_last_station_match = sa.Column( - sa.Boolean, nullable=False, default=sa.false() - ) + first_last_station_match = sa.Column(sa.Boolean, nullable=False, default=sa.false()) # forign key to static schedule expected values static_version_key = sa.Column( diff --git a/src/lamp_py/postgres/seed_metadata.py b/src/lamp_py/postgres/seed_metadata.py index 7f17a5dd..a493c943 100644 --- a/src/lamp_py/postgres/seed_metadata.py +++ b/src/lamp_py/postgres/seed_metadata.py @@ -80,9 +80,7 @@ def reset_rpm(parsed_args: argparse.Namespace) -> None: rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) except Exception as exception: - logging.exception( - "Unable to Reset Rail Performance Manager DB\n%s", exception - ) + logging.exception("Unable to Reset Rail Performance Manager DB\n%s", exception) def run() -> None: diff --git a/src/lamp_py/runtime_utils/alembic_migration.py b/src/lamp_py/runtime_utils/alembic_migration.py index 4927c995..add31104 100644 --- a/src/lamp_py/runtime_utils/alembic_migration.py +++ b/src/lamp_py/runtime_utils/alembic_migration.py @@ -14,9 +14,7 @@ def get_alembic_config(db_name: str) -> Config: here = os.path.dirname(os.path.abspath(__file__)) alembic_cfg_file = os.path.join(here, "..", "..", "..", "alembic.ini") alembic_cfg_file = os.path.abspath(alembic_cfg_file) - logging.info( - "getting alembic config for %s from %s", db_name, alembic_cfg_file - ) + logging.info("getting alembic config for %s from %s", db_name, alembic_cfg_file) db_names = ( "performance_manager_dev", diff --git a/src/lamp_py/runtime_utils/env_validation.py b/src/lamp_py/runtime_utils/env_validation.py index 357472ac..adbebe94 100644 --- a/src/lamp_py/runtime_utils/env_validation.py +++ b/src/lamp_py/runtime_utils/env_validation.py @@ -65,9 +65,7 @@ def validate_environment( # if required variables are missing, log a failure and throw. if missing_required: - exception = EnvironmentError( - f"Missing required environment variables {missing_required}" - ) + exception = EnvironmentError(f"Missing required environment variables {missing_required}") process_logger.log_failure(exception) raise exception diff --git a/src/lamp_py/runtime_utils/remote_files.py b/src/lamp_py/runtime_utils/remote_files.py index 1ebbf4f2..beab681f 100644 --- a/src/lamp_py/runtime_utils/remote_files.py +++ b/src/lamp_py/runtime_utils/remote_files.py @@ -14,6 +14,8 @@ TM = os.path.join(LAMP, "TM") TABLEAU = os.path.join(LAMP, "tableau") +VERSION_KEY = "lamp_version" + @dataclass class S3Location: @@ -23,6 +25,7 @@ class S3Location: bucket: str prefix: str + version: str = "1.0" @property def s3_uri(self) -> str: @@ -112,7 +115,8 @@ def s3_uri(self) -> str: prefix=os.path.join(TABLEAU, "alerts", "LAMP_RT_ALERTS.parquet"), ) tableau_rail = S3Location( - bucket=S3_PUBLIC, prefix=os.path.join(TABLEAU, "rail") + bucket=S3_PUBLIC, + prefix=os.path.join(TABLEAU, "rail"), ) diff --git a/src/lamp_py/tableau/__init__.py b/src/lamp_py/tableau/__init__.py index 0c3a9350..3849797d 100644 --- a/src/lamp_py/tableau/__init__.py +++ b/src/lamp_py/tableau/__init__.py @@ -26,9 +26,7 @@ def start_parquet_updates(db_manager: DatabaseManager) -> None: an error and do nothing. else, run the function. """ if pipeline is None: - logging.error( - "Unable to run parquet files on this machine due to Module Not Found error" - ) + logging.error("Unable to run parquet files on this machine due to Module Not Found error") else: pipeline.start_parquet_updates(db_manager=db_manager) @@ -40,8 +38,6 @@ def clean_parquet_paths() -> None: an error and do nothing. else, run the function. """ if pipeline is None: - logging.error( - "Unable to run parquet files on this machine due to Module Not Found error" - ) + logging.error("Unable to run parquet files on this machine due to Module Not Found error") else: pipeline.clean_parquet_paths() diff --git a/src/lamp_py/tableau/hyper.py b/src/lamp_py/tableau/hyper.py index 6e7740a8..17614213 100644 --- a/src/lamp_py/tableau/hyper.py +++ b/src/lamp_py/tableau/hyper.py @@ -46,9 +46,7 @@ def __init__( ) -> None: environment = os.getenv("ECS_TASK_GROUP", "-").split("-")[-1] if environment != "prod": - hyper_file_name = ( - f"{hyper_file_name.replace('.hyper','')}_{environment}.hyper" - ) + hyper_file_name = f"{hyper_file_name.replace('.hyper','')}_{environment}.hyper" self.hyper_file_name = hyper_file_name self.hyper_table_name = hyper_file_name.replace(".hyper", "") @@ -61,9 +59,7 @@ def __init__( self.remote_fs = fs.LocalFileSystem() if remote_parquet_path.startswith("s3://"): self.remote_fs = fs.S3FileSystem() - self.remote_parquet_path = self.remote_parquet_path.replace( - "s3://", "" - ) + self.remote_parquet_path = self.remote_parquet_path.replace("s3://", "") @property @abstractmethod @@ -129,21 +125,13 @@ def max_stats_of_parquet(self) -> Dict[str, str]: :return Dict[column_name: max_column_value] """ # get row_groups from parquet metadata (list of dicts) - row_groups = pq.read_metadata(self.local_parquet_path).to_dict()[ - "row_groups" - ] + row_groups = pq.read_metadata(self.local_parquet_path).to_dict()["row_groups"] # explode columns element from all row groups into flat list - parquet_column_stats = list( - chain.from_iterable( - [row_group["columns"] for row_group in row_groups] - ) - ) + parquet_column_stats = list(chain.from_iterable([row_group["columns"] for row_group in row_groups])) return { - col["path_in_schema"]: col["statistics"].get("max") - for col in parquet_column_stats - if col["statistics"] + col["path_in_schema"]: col["statistics"].get("max") for col in parquet_column_stats if col["statistics"] } def remote_version_match(self) -> bool: @@ -152,9 +140,7 @@ def remote_version_match(self) -> bool: :return True if remote and expected version match, else False """ - lamp_version = object_metadata(self.remote_parquet_path).get( - "lamp_version", "" - ) + lamp_version = object_metadata(self.remote_parquet_path).get("lamp_version", "") return lamp_version == self.lamp_version @@ -169,10 +155,7 @@ def create_local_hyper(self) -> int: hyper_table_schema = TableDefinition( table_name=self.hyper_table_name, columns=[ - TableDefinition.Column( - col.name, self.convert_parquet_dtype(col.type) - ) - for col in self.parquet_schema + TableDefinition.Column(col.name, self.convert_parquet_dtype(col.type)) for col in self.parquet_schema ], ) @@ -192,9 +175,7 @@ def create_local_hyper(self) -> int: database=self.local_hyper_path, create_mode=CreateMode.CREATE_AND_REPLACE, ) as connect: - connect.catalog.create_table( - table_definition=hyper_table_schema - ) + connect.catalog.create_table(table_definition=hyper_table_schema) copy_command = ( f"COPY {hyper_table_schema.table_name} " f"FROM {escape_string_literal(self.local_parquet_path)} " @@ -224,20 +205,14 @@ def run_hyper(self) -> None: try: process_log.add_metadata(retry_count=retry_count) # get datasource from Tableau to check "updated_at" datetime - datasource = datasource_from_name( - self.hyper_table_name, self.project_name - ) + datasource = datasource_from_name(self.hyper_table_name, self.project_name) # get file_info on remote parquet file to check "mtime" datetime - pq_file_info = self.remote_fs.get_file_info( - self.remote_parquet_path - ) + pq_file_info = self.remote_fs.get_file_info(self.remote_parquet_path) # Parquet file does not exist, can not run upload if pq_file_info.type == fs.FileType.NotFound: - raise FileNotFoundError( - f"{self.remote_parquet_path} does not exist" - ) + raise FileNotFoundError(f"{self.remote_parquet_path} does not exist") process_log.add_metadata( parquet_last_mod=pq_file_info.mtime.isoformat(), @@ -249,18 +224,13 @@ def run_hyper(self) -> None: ) # if datasource exists and parquet file was not modified, skip HyperFile update - if ( - datasource is not None - and pq_file_info.mtime < datasource.updated_at - ): + if datasource is not None and pq_file_info.mtime < datasource.updated_at: process_log.add_metadata(update_hyper_file=False) process_log.log_complete() break hyper_row_count = self.create_local_hyper() - hyper_file_size = os.path.getsize(self.local_hyper_path) / ( - 1024 * 1024 - ) + hyper_file_size = os.path.getsize(self.local_hyper_path) / (1024 * 1024) process_log.add_metadata( hyper_row_count=hyper_row_count, hyper_file_siz_mb=f"{hyper_file_size:.2f}", @@ -321,9 +291,7 @@ def run_parquet(self, db_manager: DatabaseManager) -> None: run_action = "update" upload_parquet = self.update_parquet(db_manager) - parquet_file_size_mb = os.path.getsize(self.local_parquet_path) / ( - 1024 * 1024 - ) + parquet_file_size_mb = os.path.getsize(self.local_parquet_path) / (1024 * 1024) process_log.add_metadata( remote_schema_match=remote_schema_match, @@ -336,9 +304,7 @@ def run_parquet(self, db_manager: DatabaseManager) -> None: upload_file( file_name=self.local_parquet_path, object_path=self.remote_parquet_path, - extra_args={ - "Metadata": {"lamp_version": self.lamp_version} - }, + extra_args={"Metadata": {"lamp_version": self.lamp_version}}, ) os.remove(self.local_parquet_path) diff --git a/src/lamp_py/tableau/jobs/gtfs_rail.py b/src/lamp_py/tableau/jobs/gtfs_rail.py index 9ef617b4..f5ed41cc 100644 --- a/src/lamp_py/tableau/jobs/gtfs_rail.py +++ b/src/lamp_py/tableau/jobs/gtfs_rail.py @@ -68,9 +68,7 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: max_parquet_key = max_stats["static_version_key"] - max_key_query = ( - f"SELECT MAX(static_version_key) FROM {self.gtfs_table_name};" - ) + max_key_query = f"SELECT MAX(static_version_key) FROM {self.gtfs_table_name};" max_db_key = db_manager.select_as_list(sa.text(max_key_query))[0]["max"] @@ -83,9 +81,7 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: return True # add WHERE clause to UPDATE query - update_query = self.update_query % ( - f" WHERE static_version_key > {max_parquet_key} ", - ) + update_query = self.update_query % (f" WHERE static_version_key > {max_parquet_key} ",) db_parquet_path = "/tmp/db_local.parquet" db_manager.write_to_parquet( @@ -104,9 +100,7 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: schema=self.parquet_schema, ).to_batches(batch_size=self.ds_batch_size) - with pq.ParquetWriter( - combine_parquet_path, schema=self.parquet_schema - ) as writer: + with pq.ParquetWriter(combine_parquet_path, schema=self.parquet_schema) as writer: for batch in combine_batches: writer.write_batch(batch) diff --git a/src/lamp_py/tableau/jobs/rt_alerts.py b/src/lamp_py/tableau/jobs/rt_alerts.py index a7c5a06f..2c926f96 100644 --- a/src/lamp_py/tableau/jobs/rt_alerts.py +++ b/src/lamp_py/tableau/jobs/rt_alerts.py @@ -22,9 +22,7 @@ def parquet_schema(self) -> pyarrow.schema: return AlertsS3Info.parquet_schema def create_parquet(self, _: DatabaseManager) -> None: - raise NotImplementedError( - "Alerts Hyper Job does not create parquet file" - ) + raise NotImplementedError("Alerts Hyper Job does not create parquet file") def update_parquet(self, _: DatabaseManager) -> bool: download_file( diff --git a/src/lamp_py/tableau/jobs/rt_rail.py b/src/lamp_py/tableau/jobs/rt_rail.py index 3d933070..64cc4cc9 100644 --- a/src/lamp_py/tableau/jobs/rt_rail.py +++ b/src/lamp_py/tableau/jobs/rt_rail.py @@ -203,9 +203,7 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: # subtract additional day incase of early spurious service_date record max_start_date -= datetime.timedelta(days=1) - update_query = self.table_query % ( - f" AND vt.service_date >= {max_start_date.strftime('%Y%m%d')} ", - ) + update_query = self.table_query % (f" AND vt.service_date >= {max_start_date.strftime('%Y%m%d')} ",) db_manager.write_to_parquet( select_query=sa.text(update_query), @@ -215,9 +213,9 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: ) check_filter = pc.field("service_date") >= max_start_date - if pd.dataset(self.db_parquet_path).count_rows() == pd.dataset( - self.local_parquet_path - ).count_rows(filter=check_filter): + if pd.dataset(self.db_parquet_path).count_rows() == pd.dataset(self.local_parquet_path).count_rows( + filter=check_filter + ): process_logger.add_metadata(new_data=False) process_logger.log_complete() # No new records from database, no upload required @@ -227,18 +225,14 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: # update downloaded parquet file with filtered service_date old_filter = pc.field("service_date") < max_start_date - old_batches = pd.dataset(self.local_parquet_path).to_batches( - filter=old_filter, batch_size=self.ds_batch_size - ) + old_batches = pd.dataset(self.local_parquet_path).to_batches(filter=old_filter, batch_size=self.ds_batch_size) filter_path = "/tmp/filter_local.parquet" old_batch_count = 0 old_batch_rows = 0 old_batch_bytes = 0 - with pq.ParquetWriter( - filter_path, schema=self.parquet_schema - ) as writer: + with pq.ParquetWriter(filter_path, schema=self.parquet_schema) as writer: for batch in old_batches: old_batch_count += 1 old_batch_rows += batch.num_rows @@ -267,9 +261,7 @@ def update_parquet(self, db_manager: DatabaseManager) -> bool: combine_batch_rows = 0 combine_batch_bytes = 0 - with pq.ParquetWriter( - combine_parquet_path, schema=self.parquet_schema - ) as writer: + with pq.ParquetWriter(combine_parquet_path, schema=self.parquet_schema) as writer: for batch in combine_batches: combine_batch_count += 1 combine_batch_rows += batch.num_rows diff --git a/src/lamp_py/tableau/server.py b/src/lamp_py/tableau/server.py index 0280a99f..24f23468 100644 --- a/src/lamp_py/tableau/server.py +++ b/src/lamp_py/tableau/server.py @@ -113,10 +113,7 @@ def datasource_from_name( auth = tableau_authentication() for datasource in datasource_list(server, auth): - if ( - datasource.name == datasource_name - and datasource.project_name == project_name - ): + if datasource.name == datasource_name and datasource.project_name == project_name: return datasource return None diff --git a/tests/aws/test_s3_utils.py b/tests/aws/test_s3_utils.py index f9938cfb..ecb08bdd 100644 --- a/tests/aws/test_s3_utils.py +++ b/tests/aws/test_s3_utils.py @@ -84,9 +84,7 @@ def test_file_list_s3(s3_stub): # type: ignore # Process large page_obj_response from json file # 'test_files/large_page_obj_response.json' # large json file contains 1,000 Contents records. - large_response_file = os.path.join( - incoming_dir, "large_page_obj_response.json" - ) + large_response_file = os.path.join(incoming_dir, "large_page_obj_response.json") page_obj_params = { "Bucket": "mbta-gtfs-s3", "Prefix": ANY, diff --git a/tests/bus_performance_manager/bus_test_gtfs.csv b/tests/bus_performance_manager/bus_test_gtfs.csv index bdab1a43..1b098f22 100644 --- a/tests/bus_performance_manager/bus_test_gtfs.csv +++ b/tests/bus_performance_manager/bus_test_gtfs.csv @@ -1,489 +1,489 @@ -trip_id,stop_id,stop_sequence,timepoint,checkpoint_id,block_id,route_id,service_id,route_pattern_id,route_pattern_typicality,direction_id,direction,direction_destination,stop_name,parent_station,static_stop_count,arrival_seconds,departure_seconds,plan_travel_time_seconds,plan_route_direction_headway_seconds,plan_direction_destination_headway_seconds -63070693,334,1,1,ashmt,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Ashmont,place-asmnl,29,55800,55800,,540,480 -63070693,367,2,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Dorchester Ave,,29,55860,55860,60,540,420 -63070693,369,3,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Welles Ave,,29,55920,55920,60,540,420 -63070693,371,4,0,codmn,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Centre St,,29,55980,55980,60,540,360 -63070693,372,5,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Southern Ave,,29,56040,56040,60,540,540 -63070693,373,6,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Spencer St,,29,56100,56100,60,540,540 -63070693,374,7,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Talbot Station,,29,56160,56160,60,540,540 -63070693,375,8,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Bernard St,,29,56220,56220,60,540,540 -63070693,376,9,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Kerwin St,,29,56280,56280,60,540,540 -63070693,378,10,0,bltal,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Nightingale St,,29,56340,56340,60,540,540 -63070693,1737,11,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Harvard St,,29,56400,56400,60,540,360 -63070693,381,12,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Wales St,,29,56460,56460,60,540,300 -63070693,382,13,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Charlotte St,,29,56520,56520,60,540,300 -63070693,383,14,0,frnpk,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Ellington St,,29,56580,56580,60,540,240 -63070693,9448,15,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Blue Hill Ave,,29,56640,56640,60,540,540 -63070693,1739,16,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Maple St,,29,56700,56700,60,540,540 -63070693,1740,17,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Elm Hill Ave,,29,56760,56760,60,540,540 -63070693,1741,18,0,svrhm,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Humboldt Ave,,29,56820,56820,60,540,540 -63070693,1742,19,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Harold St,,29,56880,56880,60,540,540 -63070693,1743,20,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Walnut Ave,,29,56940,56940,60,540,540 -63070693,1188,21,0,egles,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave opp Weld Ave - Egleston Sq,,29,57000,57000,60,540,540 -63070693,1215,22,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Bray St,,29,57060,57060,60,540,540 -63070693,1216,23,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Dimock St,,29,57120,57120,60,540,540 -63070693,11531,24,0,jasst,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Jackson Square,place-jaksn,29,57360,57360,240,540,540 -63070693,1219,25,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Centre St,,29,57480,57480,120,540,540 -63070693,1221,26,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Cedar St,,29,57540,57540,60,540,540 -63070693,1222,27,0,roxbs,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Malcolm X Blvd,,29,57660,57660,120,540,540 -63070693,1224,28,0,,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Tremont St opp Prentiss St,,29,57840,57840,180,540,60 -63070693,17861,29,0,rugg,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Ruggles,place-rugg,29,58200,58200,360,540,180 -63070975,33,1,1,ctypt,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,City Point Bus Terminal,,14,26880,26880,,480,480 -63070975,10033,2,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,E 1st St @ O St,,14,26940,26940,60,480,480 -63070975,34,3,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,P St @ E 2nd St,,14,27000,27000,60,480,480 -63070975,35,4,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,P St @ E Broadway,,14,27000,27000,0,480,480 -63070975,895,5,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,E Broadway @ N St,,14,27120,27120,120,480,480 -63070975,886,6,0,lbrwy,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,L St @ Broadway,,14,27180,27180,60,480,480 -63070975,884,7,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ E 1st St,,14,27240,27240,60,480,480 -63070975,210,8,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ Drydock Ave,,14,27360,27360,120,480,480 -63070975,212,9,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ D St,,14,27420,27420,60,480,480 -63070975,890,10,0,bcec,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ WTC Ave,,14,27480,27480,60,480,480 -63070975,891,11,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St opp Melcher St,,14,27600,27600,120,480,480 -63070975,892,12,0,sosta,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ Atlantic Ave,,14,27720,27720,120,480,480 -63070975,6535,13,0,,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Federal St @ Matthews St,,14,27960,27960,240,480,480 -63070975,16535,14,0,otsum,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Otis St @ Summer St,,14,28140,28140,180,480,480 -63071767,1475,1,1,kane,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Bowdoin St,,21,57000,57000,,600,600 -63071767,1478,2,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Rowell St,,21,57060,57060,60,600,600 -63071767,1479,3,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Jerome St,,21,57060,57060,0,600,600 -63071767,1480,4,0,upham,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Columbia Rd @ Hancock St,,21,57120,57120,60,600,600 -63071767,1481,5,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Belden St,,21,57180,57180,60,600,600 -63071767,11482,6,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Uphams Corner Station,,21,57240,57240,60,600,600 -63071767,14831,7,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St opp Howard Ave,,21,57300,57300,60,600,600 -63071767,1484,8,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ E Cottage St,,21,57360,57360,60,600,600 -63071767,1485,9,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Langdon St,,21,57420,57420,60,600,600 -63071767,1486,10,0,bhdud,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Magazine St,,21,57480,57480,60,600,600 -63071767,1487,11,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Hampden St,,21,57540,57540,60,600,600 -63071767,1488,12,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Adams St,,21,57660,57660,120,600,600 -63071767,1489,13,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Dearborn St,,21,57780,57780,120,660,660 -63071767,1491,14,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Harrison Ave,,21,57960,57960,180,720,0 -63071767,64000,15,0,nubn,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Nubian,place-nubn,21,58020,58020,60,720,0 -63071767,1148,16,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Shawmut Ave,,21,58140,58140,120,720,60 -63071767,11149,17,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ O'Bryant HS,,21,58200,58200,60,720,120 -63071767,11148,18,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Madison Park HS,,21,58260,58260,60,720,120 -63071767,21148,19,0,roxbs,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Tremont St,,21,58320,58320,60,720,180 -63071767,1224,20,0,,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Tremont St opp Prentiss St,,21,58440,58440,120,720,120 -63071767,17861,21,0,rugg,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Ruggles,place-rugg,21,58740,58740,300,720,60 -63072828,6551,1,1,fedfr,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Federal St @ Franklin St,,12,29220,29220,,960,960 -63072828,16535,2,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Otis St @ Summer St,,12,29340,29340,120,960,960 -63072828,6555,3,0,kneel,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Chinatown Gate,,12,29520,29520,180,960,960 -63072828,9983,4,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Stuart St @ Charles St S,,12,29700,29700,180,960,960 -63072828,177,5,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Arlington St,,12,29820,29820,120,960,960 -63072828,173,6,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Berkeley St,,12,29940,29940,120,960,960 -63072828,178,7,0,stjim,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Dartmouth St,,12,30000,30000,60,960,960 -63072828,903,8,0,nwcor,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Washington St @ Bacon St,,12,30540,30540,540,960,960 -63072828,19031,9,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,400 Centre St - West,,12,30720,30720,180,960,960 -63072828,988,10,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Centre St @ Jefferson St,,12,30900,30900,180,960,480 -63072828,989,11,0,,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Galen St @ Maple St,,12,30960,30960,60,960,420 -63072828,999,12,0,wtryd,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Galen St @ Water St,,12,31140,31140,180,960,420 -63075088,64000,1,1,nubn,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Nubian,place-nubn,33,20700,20700,,720,720 -63075088,1148,2,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Shawmut Ave,,33,20760,20760,60,720,720 -63075088,11149,3,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ O'Bryant HS,,33,20820,20820,60,720,720 -63075088,11148,4,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Madison Park HS,,33,20880,20880,60,720,720 -63075088,21148,5,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Tremont St,,33,20940,20940,60,720,720 -63075088,1357,6,0,roxbs,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St opp Roxbury Crossing Sta,,33,21000,21000,60,720,720 -63075088,13590,7,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Tobin Community Center,,33,21060,21060,60,720,720 -63075088,1360,8,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Saint Alphonsus St,,33,21060,21060,0,720,720 -63075088,1362,9,0,brghm,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Huntington Ave,,33,21120,21120,60,720,720 -63075088,1363,10,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Huntington Ave @ Fenwood Rd,,33,21180,21180,60,720,720 -63075088,1365,11,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,835 Huntington Ave opp Parker Hill Ave,,33,21240,21240,60,720,720 -63075088,1366,12,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Huntington Ave @ Riverway,,33,21360,21360,120,720,720 -63075088,1526,13,0,brkvl,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Washington St @ Pearl St,,33,21420,21420,60,720,720 -63075088,1367,14,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Kent St,,33,21480,21480,60,720,720 -63075088,1369,15,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Aspinwall Ave,,33,21480,21480,0,720,720 -63075088,1371,16,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St opp Vernon St,,33,21540,21540,60,720,720 -63075088,1372,17,0,cool,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Beacon St,,33,21600,21600,60,720,720 -63075088,1373,18,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Stedman St,,33,21660,21660,60,720,720 -63075088,1375,19,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Coolidge St,,33,21720,21720,60,720,720 -63075088,1376,20,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St opp Verndale St,,33,21780,21780,60,720,720 -63075088,1378,21,0,hvdcm,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Ave @ Commonwealth Ave,,33,21840,21840,60,720,720 -63075088,1379,22,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Ave @ Brighton Ave,,33,21900,21900,60,720,720 -63075088,964,23,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Brighton Ave opp Quint Ave,,33,21960,21960,60,780,780 -63075088,1111,24,0,union,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Cambridge St opp Hano St,,33,22020,22020,60,780,780 -63075088,1112,25,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Cambridge St @ Harvard Ave,,33,22080,22080,60,780,780 -63075088,2558,26,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Empire St,,33,22260,22260,180,840,840 -63075088,2559,27,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Oxford St,,33,22260,22260,0,780,780 -63075088,2560,28,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Kingsley St,,33,22320,22320,60,840,840 -63075088,2561,29,0,nhvws,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Western Ave,,33,22380,22380,60,840,840 -63075088,2564,30,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St opp Harvard Stadium Gate 2,,33,22560,22560,180,840,840 -63075088,25641,31,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,JFK St @ Eliot St,,33,22680,22680,120,840,840 -63075088,2168,32,0,,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Massachusetts Ave @ Johnston Gate,,33,22860,22860,180,840,840 -63075088,22549,33,0,harsq,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Sq @ Garden St - Dawes Island,,33,22920,22920,60,840,840 -63089353,32002,1,1,qnctr,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Quincy Center,place-qnctr,53,72900,72900,,900,900 -63089353,3237,2,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Washington St,,53,72960,72960,60,900,900 -63089353,3239,3,0,codwd,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Quincy YMCA,,53,73020,73020,60,900,900 -63089353,3425,4,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Southern Artery,,53,73080,73080,60,900,900 -63089353,3240,5,0,sosea,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,55 Sea St,,53,73140,73140,60,900,900 -63089353,3241,6,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Overlook Rd,,53,73200,73200,60,900,900 -63089353,3242,7,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Samoset Ave,,53,73200,73200,0,900,900 -63089353,3243,8,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,227 Sea St,,53,73200,73200,0,900,900 -63089353,3244,9,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Barbour Terr,,53,73260,73260,60,900,900 -63089353,13245,10,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Moffat Rd,,53,73260,73260,0,900,900 -63089353,3245,11,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,405 Sea St,,53,73260,73260,0,900,900 -63089353,3246,12,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,455 Sea St,,53,73260,73260,0,900,900 -63089353,3247,13,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Braintree Ave,,53,73320,73320,60,900,900 -63089353,3292,14,0,seast,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Sea St,,53,73320,73320,0,900,900 -63089353,3295,15,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Oakwood Rd,,53,73380,73380,60,900,900 -63089353,3298,16,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Forbush Ave,,53,73440,73440,60,900,900 -63089353,3299,17,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Brockton Ave,,53,73440,73440,0,900,900 -63089353,3300,18,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Roach St,,53,73440,73440,0,900,900 -63089353,3301,19,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Bowes Ave,,53,73440,73440,0,900,900 -63089353,3303,20,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,375 Palmer St,,53,73500,73500,60,900,900 -63089353,3304,21,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Taffrail Rd,,53,73560,73560,60,900,900 -63089353,3305,22,0,grmnt,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,26 Taffrail Rd,,53,73560,73560,0,900,900 -63089353,3306,23,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Taffrail Rd @ Bicknell St,,53,73560,73560,0,900,900 -63089353,3307,24,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ O'Brien Towers Driveway,,53,73560,73560,0,900,900 -63089353,3308,25,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,O'Brien Towers,,53,73620,73620,60,900,900 -63089353,3309,26,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,O'Brien Towers Driveway @ Bicknell St,,53,73620,73620,0,900,900 -63089353,3310,27,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ Binnacle Lane,,53,73620,73620,0,900,900 -63089353,3311,28,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ Yardarm Ln,,53,73680,73680,60,900,900 -63089353,3312,29,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Shed St,,53,73680,73680,0,900,900 -63089353,3314,30,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Bowes Ave,,53,73740,73740,60,900,900 -63089353,3315,31,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Roach St,,53,73740,73740,0,900,900 -63089353,3316,32,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Brockton Ave,,53,73740,73740,0,900,900 -63089353,3317,33,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Empire St,,53,73800,73800,60,900,900 -63089353,3320,34,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Oakwood Rd,,53,73800,73800,0,900,900 -63089353,3321,35,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Wilgus Rd,,53,73800,73800,0,900,900 -63089353,3322,36,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Utica St,,53,73860,73860,60,900,900 -63089353,3323,37,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Sea St,,53,73860,73860,0,900,900 -63089353,3249,38,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,565 Sea St,,53,73860,73860,0,900,900 -63089353,3250,39,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Lee St,,53,73920,73920,60,900,900 -63089353,3251,40,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Opp 700 Sea St,,53,73980,73980,60,900,900 -63089353,3252,41,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,749 Sea St,,53,73980,73980,0,900,900 -63089353,3253,42,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Babcock St,,53,74040,74040,60,900,900 -63089353,3254,43,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Manet Ave,,53,74040,74040,0,900,900 -63089353,3255,44,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Newton St,,53,74040,74040,0,900,900 -63089353,3256,45,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Malvern St,,53,74100,74100,60,900,900 -63089353,3257,46,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Rockland St,,53,74100,74100,0,900,900 -63089353,3258,47,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Darrow St,,53,74100,74100,0,900,900 -63089353,3259,48,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Manet Ave,,53,74160,74160,60,900,900 -63089353,3260,49,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Wall St,,53,74220,74220,60,900,900 -63089353,3261,50,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Bell St,,53,74220,74220,0,900,900 -63089353,3262,51,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Thomas St,,53,74220,74220,0,900,900 -63089353,3263,52,0,,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,1263 Sea St,,53,74220,74220,0,900,900 -63089353,3265,53,0,hsnck,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Fensmere Ave,,53,74280,74280,60,900,900 -63101043,5072,1,1,malst,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Malden Center,place-mlmnl,39,43200,43200,,1200,1200 -63101043,5328,2,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Elm St,,39,43260,43260,60,1200,1200 -63101043,9028,3,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St opp Russell St,,39,43320,43320,60,1200,1200 -63101043,5330,4,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Prospect St,,39,43320,43320,0,1200,1200 -63101043,5331,5,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St opp West St,,39,43380,43380,60,1200,1200 -63101043,45332,6,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Vista St,,39,43380,43380,0,1200,1200 -63101043,5332,7,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Fellsway E,,39,43380,43380,0,1200,1200 -63101043,8308,8,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Fellsway W,,39,43440,43440,60,1200,1200 -63101043,5282,9,0,fells,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Grant Ave,,39,43500,43500,60,1200,1200 -63101043,5283,10,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,390 Salem St,,39,43500,43500,0,1200,1200 -63101043,5284,11,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Almont St,,39,43560,43560,60,1200,1200 -63101043,5285,12,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Paris St,,39,43560,43560,0,1200,1200 -63101043,5286,13,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Court St,,39,43620,43620,60,1200,1200 -63101043,5287,14,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Allen Ct,,39,43680,43680,60,1200,1200 -63101043,5002,15,0,medfd,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St opp River St,,39,43800,43800,120,1200,1200 -63101043,5031,16,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ High St,,39,43800,43800,0,1200,1200 -63101043,5032,17,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Emerson St,,39,43860,43860,60,1200,1200 -63101043,5290,18,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Summer St,,39,43860,43860,0,1200,1200 -63101043,5291,19,0,mngrg,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ George St,,39,43860,43860,0,1200,1200 -63101043,5292,20,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Stearns Ave,,39,43920,43920,60,1200,1200 -63101043,5293,21,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Windsor Rd,,39,43920,43920,0,1200,1200 -63101043,5294,22,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Wellesley St,,39,43980,43980,60,1200,1200 -63101043,5295,23,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Princeton St,,39,43980,43980,0,1200,1200 -63101043,5296,24,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Harvard St,,39,44040,44040,60,1200,1200 -63101043,5297,25,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Marion St,,39,44040,44040,0,1200,1200 -63101043,5298,26,0,tufts,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Medford St,,39,44100,44100,60,1200,1200 -63101043,5299,27,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Albion St,,39,44160,44160,60,1200,1200 -63101043,5300,28,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Dexter St,,39,44220,44220,60,1200,1200 -63101043,5301,29,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Bow St,,39,44280,44280,60,1200,1200 -63101043,5302,30,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St opp Moreland St,,39,44280,44280,0,1200,1200 -63101043,5303,31,0,whill,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Broadway,,39,44340,44340,60,1200,1200 -63101043,2704,32,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Thurston St,,39,44400,44400,60,1200,600 -63101043,2706,33,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Marshall St,,39,44460,44460,60,1200,600 -63101043,2707,34,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Montgomery Ave,,39,44520,44520,60,1200,600 -63101043,2710,35,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Cross St,,39,44640,44640,120,1200,600 -63101043,2711,36,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Glen St,,39,44700,44700,60,1200,600 -63101043,2713,37,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Lincoln St,,39,44760,44760,60,1200,600 -63101043,2714,38,0,,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Mt Vernon St,,39,44820,44820,60,1200,60 -63101043,29001,39,0,sull,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Sullivan Square,place-sull,39,44880,44880,60,1200,0 -63156984,141,1,1,alewf,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Alewife,place-alfcl,21,40200,40200,,3000,3000 -63156984,2482,2,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,West Service Rd @ Lake St,,21,40380,40380,180,3000,3000 -63156984,23530,3,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,244 Pleasant St opp Brunswick Rd,,21,40500,40500,120,3000,3000 -63156984,23531,4,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Gould Rd,,21,40500,40500,0,3000,3000 -63156984,23532,5,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Spring Valley St,,21,40560,40560,60,3000,3000 -63156984,23533,6,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Wellington St,,21,40560,40560,0,3000,3000 -63156984,2283,7,0,arlct,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Massachusetts Ave @ Mystic St,,21,40620,40620,60,3000,3000 -63156984,2284,8,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Massachusetts Ave @ Central St,,21,40680,40680,60,3000,3000 -63156984,7976,9,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Mill St @ Bacon St,,21,40680,40680,0,3000,3000 -63156984,7977,10,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Winthrop Rd,,21,40740,40740,60,3000,3000 -63156984,7978,11,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Oak Hill Dr,,21,40800,40800,60,3000,3000 -63156984,7979,12,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Symmes Rd,,21,40860,40860,60,3000,3000 -63156984,7981,13,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Symmes Circle @ Symmes Rd,,21,40980,40980,120,3000,3000 -63156984,7953,14,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St opp Washington St,,21,41160,41160,180,3000,3000 -63156984,7954,15,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Candia St,,21,41160,41160,0,3000,3000 -63156984,7955,16,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Ronald Rd,,21,41220,41220,60,3000,3000 -63156984,7956,17,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Mountain Ave,,21,41280,41280,60,3000,3000 -63156984,7959,18,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Clyde Terr @ Lawrence Ln,,21,41340,41340,60,3000,3000 -63156984,7960,19,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Thomas St,,21,41400,41400,60,3000,3000 -63156984,7962,20,0,,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Hancock St,,21,41460,41460,60,3000,3000 -63156984,7961,21,0,trkhl,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Heard Rd,,21,41460,41460,0,3000,3000 -63157369,20761,1,1,bally,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Harvard,place-harsq,22,40620,40620,,960,960 -63157369,2020,2,0,mtsty,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Story St,,22,40740,40740,120,960,960 -63157369,2021,3,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Ash St,,22,40800,40800,60,960,960 -63157369,2023,4,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Sparks St,,22,40860,40860,60,960,960 -63157369,2025,5,0,mthsp,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Longfellow Rd,,22,40920,40920,60,960,960 -63157369,2026,6,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Traill St,,22,40980,40980,60,960,960 -63157369,2027,7,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St opp Coolidge Ave,,22,41040,41040,60,960,960 -63157369,2028,8,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Brattle St,,22,41100,41100,60,960,960 -63157369,2030,9,0,mtaub,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Homer Ave,,22,41160,41160,60,960,960 -63157369,2032,10,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Saint Marys St,,22,41220,41220,60,960,960 -63157369,2033,11,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Keenan St,,22,41280,41280,60,960,960 -63157369,2034,12,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Kimball Rd,,22,41280,41280,0,960,960 -63157369,2036,13,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Upland Rd,,22,41340,41340,60,960,960 -63157369,2037,14,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Winsor Ave,,22,41400,41400,60,960,960 -63157369,2038,15,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Adams Ave,,22,41460,41460,60,960,960 -63157369,2040,16,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Amherst Rd,,22,41520,41520,60,960,960 -63157369,2047,17,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Bates Rd E,,22,41580,41580,60,960,960 -63157369,2042,18,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Russell Ave,,22,41580,41580,0,960,960 -63157369,2043,19,0,wtrhs,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Marshall St,,22,41640,41640,60,960,960 -63157369,2044,20,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Summer St,,22,41760,41760,120,960,960 -63157369,2046,21,0,,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Main St,,22,41880,41880,120,960,960 -63157369,8178,22,0,wtrsq,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Watertown Sq Terminal,,22,41940,41940,60,960,480 -63157836,2362,1,1,arlct,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Massachusetts Ave,,37,29400,29400,,2700,2700 -63157836,2363,2,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Lewis Ave,,37,29460,29460,60,2700,2700 -63157836,2365,3,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Hamlet St,,37,29460,29460,0,2700,2700 -63157836,2366,4,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St opp Hayes St - Medford Line,,37,29520,29520,60,2700,2700 -63157836,2367,5,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,High St @ Jerome St,,37,29520,29520,0,2700,2700 -63157836,2368,6,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,High St @ Monument St,,37,29580,29580,60,2700,2700 -63157836,3513,7,0,boshi,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ High St,,37,29580,29580,0,2700,2700 -63157836,2369,8,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Harvard Ave,,37,29640,29640,60,2700,2700 -63157836,2370,9,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Holton St,,37,29640,29640,0,2700,2700 -63157836,2371,10,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Arlington St,,37,29700,29700,60,2700,2700 -63157836,2372,11,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Mystic Valley Pkwy,,37,29760,29760,60,2700,2700 -63157836,2373,12,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Harris Rd,,37,29760,29760,0,2700,2700 -63157836,2374,13,0,nostr,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ North St,,37,29820,29820,60,2700,2700 -63157836,2375,14,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Hillsdale Rd,,37,29880,29880,60,2700,2700 -63157836,2376,15,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Winthrop St,,37,29880,29880,0,2700,2700 -63157836,2378,16,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Tufts Garage,,37,29940,29940,60,2700,2700 -63157836,2379,17,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Boston Ave,,37,30000,30000,60,2700,2700 -63157836,2380,18,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Professors Row,,37,30060,30060,60,2700,2700 -63157836,2381,19,0,powdr,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Powder House Sq,,37,30120,30120,60,2700,2700 -63157836,2695,20,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway opp Warner St - Powder House Sq,,37,30180,30180,60,2700,2700 -63157836,2696,21,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Bay State Ave,,37,30180,30180,0,2700,2700 -63157836,2697,22,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Josephine Ave - Ball Sq,,37,30240,30240,60,2700,2700 -63157836,2698,23,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Cedar St,,37,30360,30360,120,2700,2700 -63157836,2699,24,0,mgnsq,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Hinckley St - Magoun Sq,,37,30420,30420,60,2700,2700 -63157836,12699,25,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Broadway - Magoun Sq,,37,30420,30420,0,2700,2700 -63157836,23841,26,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Glenwood Rd,,37,30540,30540,120,2700,2700 -63157836,2385,27,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Central St,,37,30600,30600,60,2700,2700 -63157836,2386,28,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Sycamore St,,37,30660,30660,60,2700,2700 -63157836,2388,29,0,mdscl,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ School St,,37,30720,30720,60,2700,2700 -63157836,2389,30,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ Skilton Ave,,37,30780,30780,60,2700,2700 -63157836,2390,31,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ Walnut St,,37,30840,30840,60,2700,2700 -63157836,2391,32,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ McGrath Hwy,,37,30900,30900,60,2700,2700 -63157836,2689,33,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,422 McGrath Hwy,,37,31080,31080,180,2700,300 -63157836,2690,34,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Washington St,,37,31140,31140,60,2700,300 -63157836,2600,35,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,McGrath Hwy @ Medford St,,37,31320,31320,180,2700,300 -63157836,2601,36,0,,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,McGrath Hwy @ Twin City Plaza,,37,31560,31560,240,2700,300 -63157836,70500,37,0,lchmr,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Lechmere,place-lech,37,31680,31680,120,2700,300 -63256578,5327,1,1,malst,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Malden Center,place-mlmnl,22,65100,65100,,2100,60 -63256578,45327,2,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Commercial St @ Charles St,,22,65160,65160,60,2100,2100 -63256578,45326,3,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Commercial St @ Medford St,,22,65160,65160,0,2100,2100 -63256578,45330,4,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Green St,,22,65220,65220,60,2100,2100 -63256578,44330,5,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Brackenbury St,,22,65280,65280,60,2100,2100 -63256578,45331,6,0,mfdmn,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Main St,,22,65280,65280,0,2100,2100 -63256578,5395,7,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Main St @ Converse Ave,,22,65340,65340,60,2100,900 -63256578,5047,8,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Belmont St @ Main St,,22,65400,65400,60,2100,2100 -63256578,5048,9,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Belmont St @ Kinsman St,,22,65400,65400,0,2100,2100 -63256578,5049,10,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Belmont St,,22,65460,65460,60,2100,2100 -63256578,5050,11,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Tappan St,,22,65520,65520,60,2100,2100 -63256578,5051,12,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Dean St,,22,65580,65580,60,2100,2100 -63256578,15492,13,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Maple Ave,,22,65700,65700,120,2100,2100 -63256578,5495,14,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Church St,,22,65760,65760,60,2100,2100 -63256578,5496,15,0,evrtc,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Norwood St,,22,65820,65820,60,2100,540 -63256578,5559,16,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway opp Second St,,22,65820,65820,0,2100,540 -63256578,5560,17,0,sweet,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Gladstone St,,22,65880,65880,60,2100,540 -63256578,5561,18,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Santilli Circle,,22,66060,66060,180,2100,180 -63256578,55631,19,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Gateway Center opp Texas Roadhouse,,22,66180,66180,120,2100,2100 -63256578,55632,20,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Gateway Center @ Target,,22,66240,66240,60,2100,2100 -63256578,6569,21,0,,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Corporation Way,,22,66600,66600,360,2100,360 -63256578,5271,22,0,welst,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Wellington Station Busway,place-welln,22,66660,66660,60,2100,240 -63256596,53270,1,1,malst,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Malden Center,place-mlmnl,29,74880,74880,,1080,1080 -63256596,5289,2,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Centre St @ Stop & Shop,,29,74940,74940,60,1080,1080 -63256596,5342,3,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Main St opp Pleasant St,,29,75060,75060,120,1080,1080 -63256596,5343,4,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Irving St,,29,75060,75060,0,1080,1080 -63256596,5344,5,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Eastern Ave,,29,75120,75120,60,1080,1080 -63256596,5345,6,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St opp Clayton St,,29,75180,75180,60,1080,1080 -63256596,5347,7,0,fercr,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Cross St,,29,75240,75240,60,1080,1080 -63256596,5348,8,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Winthrop St,,29,75300,75300,60,1080,1080 -63256596,5349,9,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Belmont St,,29,75360,75360,60,1080,1080 -63256596,5350,10,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Bennett St,,29,75360,75360,0,1080,1080 -63256596,5351,11,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Central Ave,,29,75420,75420,60,1080,1080 -63256596,5352,12,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Glendale St,,29,75480,75480,60,1080,1080 -63256596,5353,13,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Walnut St - Glendale Towers,,29,75480,75480,0,1080,1080 -63256596,5354,14,0,glndl,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Broadway,,29,75540,75540,60,1080,1080 -63256596,5490,15,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Raymond St,,29,75600,75600,60,1080,120 -63256596,15492,16,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Maple Ave,,29,75660,75660,60,1080,120 -63256596,5495,17,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Church St,,29,75720,75720,60,1080,120 -63256596,5496,18,0,evrtc,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Norwood St,,29,75720,75720,0,1080,120 -63256596,5559,19,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Second St,,29,75780,75780,60,1080,120 -63256596,5560,20,0,sweet,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Gladstone St,,29,75900,75900,120,1080,120 -63256596,5497,21,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Bowdoin St,,29,76020,76020,120,1080,120 -63256596,5498,22,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Beacham St,,29,76020,76020,0,1080,120 -63256596,5499,23,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Thorndike St,,29,76020,76020,0,1080,120 -63256596,5500,24,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Horizon Way,,29,76080,76080,60,1080,120 -63256596,5501,25,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Opp 173 Alford St,,29,76140,76140,60,1080,120 -63256596,55011,26,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ MWRA Pump Station,,29,76140,76140,0,1080,120 -63256596,55012,27,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ MBTA Charlestown Garage,,29,76200,76200,60,1080,120 -63256596,5502,28,0,,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ West St,,29,76260,76260,60,1080,120 -63256596,29001,29,0,sull,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Sullivan Square,place-sull,29,76320,76320,60,1080,120 -63256882,29011,1,1,sull,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Sullivan Square,place-sull,21,64620,64620,,420,420 -63256882,2718,2,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Austin St,,21,64800,64800,180,360,360 -63256882,2719,3,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Illinois Ave,,21,64860,64860,60,420,420 -63256882,2721,4,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Cross St,,21,64920,64920,60,420,420 -63256882,2723,5,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Fellsway W,,21,64980,64980,60,360,360 -63256882,2725,6,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Temple St,,21,65040,65040,60,360,360 -63256882,2726,7,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Winter Hill Plaza,,21,65100,65100,60,360,360 -63256882,2729,8,0,whill,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Main St,,21,65160,65160,60,360,360 -63256882,2731,9,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway opp Glenwood Rd,,21,65220,65220,60,360,360 -63256882,2733,10,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Medford St - Magoun Sq,,21,65340,65340,120,360,360 -63256882,2734,11,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ William St,,21,65340,65340,0,360,360 -63256882,2735,12,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Alfred St,,21,65400,65400,60,360,360 -63256882,2736,13,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Boston Ave - Ball Sq,,21,65460,65460,60,360,360 -63256882,2737,14,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Pearson Rd,,21,65520,65520,60,360,360 -63256882,2738,15,0,powdr,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Warner St - Powder House Sq,,21,65580,65580,60,360,360 -63256882,5012,16,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ Broadway,,21,65640,65640,60,1080,1080 -63256882,5014,17,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ William St,,21,65700,65700,60,1080,1080 -63256882,5015,18,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ Highland Ave,,21,65760,65760,60,1080,1080 -63256882,2582,19,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Elm St @ Chester St,,21,65820,65820,60,1080,1080 -63256882,2628,20,0,,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Grove St @ Highland Ave,,21,65940,65940,120,1140,1140 -63256882,5104,21,0,davis,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Davis,place-davis,21,66000,66000,60,1140,1140 -63361157,1618,1,1,gtown,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,84 Georgetowne Pl,,32,20160,20160,,, -63361157,11618,2,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Georgetowne Dr @ Georgetowne Pl,,32,20220,20220,60,, -63361157,6523,3,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Dedham Pkwy,,32,20220,20220,0,, -63361157,6524,4,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Leighton Rd,,32,20280,20280,60,, -63361157,6526,5,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Turtle Pond Pkwy,,32,20280,20280,0,, -63361157,6522,6,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Dedham Pkwy @ Georgetowne Dr,,32,20400,20400,120,, -63361157,1616,7,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Margaretta Dr @ Crown Point Dr,,32,20460,20460,60,, -63361157,1617,8,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Opp 67 Crown Point Dr,,32,20580,20580,120,, -63361157,1620,9,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Georgetowne Dr opp Margaretta Dr,,32,20580,20580,0,, -63361157,1621,10,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,W Boundary Rd @ Georgetowne Dr,,32,20580,20580,0,, -63361157,1623,11,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,W Boundary Rd @ Cedarcrest Rd,,32,20640,20640,60,, -63361157,625,12,0,wawbw,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ W Boundary Rd,,32,20760,20760,120,,600 -63361157,10625,13,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Heron St,,32,20760,20760,0,,600 -63361157,626,14,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Maplewood St,,32,20760,20760,0,,540 -63361157,627,15,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Cowing St,,32,20820,20820,60,,540 -63361157,628,16,0,walag,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Enneking Pkwy,,32,20820,20820,0,,540 -63361157,629,17,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Schubert St,,32,20880,20880,60,,480 -63361157,630,18,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Blue Ledge Dr,,32,20940,20940,60,,540 -63361157,631,19,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Beech St,,32,21000,21000,60,,540 -63361157,632,20,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Cornell St,,32,21000,21000,0,,480 -63361157,633,21,0,wamet,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Metropolitan Ave,,32,21060,21060,60,,480 -63361157,10633,22,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Rosecliff St,,32,21060,21060,0,,480 -63361157,634,23,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Albano St,,32,21120,21120,60,,480 -63361157,635,24,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Poplar St,,32,21180,21180,60,,480 -63361157,636,25,0,rossq,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Cummins Hwy,,32,21240,21240,60,,180 -63361157,637,26,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Firth Rd,,32,21300,21300,60,,180 -63361157,638,27,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Granfield Ave,,32,21360,21360,60,,180 -63361157,639,28,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Whipple Ave,,32,21420,21420,60,,180 -63361157,640,29,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Archdale Rd,,32,21420,21420,0,,180 -63361157,641,30,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Aldwin Rd,,32,21480,21480,60,,180 -63361157,642,31,0,,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Tollgate Way,,32,21540,21540,60,,180 -63361157,10642,32,0,fhill,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Forest Hills,place-forhl,32,21660,21660,120,,180 -63361458,18511,1,1,matpn,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Mattapan,place-matt,28,83100,83100,,3120,3120 -63361458,1722,2,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,1624 Blue Hill Ave @ Mattapan Sq,,28,83100,83100,0,3120,3120 -63361458,1723,3,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Babson St,,28,83160,83160,60,3120,3120 -63361458,1724,4,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave opp Woodhaven St,,28,83160,83160,0,3120,3120 -63361458,1725,5,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,1458 Blue Hill Ave opp Almont St,,28,83160,83160,0,3120,3120 -63361458,1726,6,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Mattapan Library,,28,83220,83220,60,3120,3120 -63361458,1728,7,0,wellh,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Fessenden St,,28,83220,83220,0,3120,3120 -63361458,1730,8,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Woolson St,,28,83280,83280,60,3120,3120 -63361458,1731,9,0,mortn,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Morton St,,28,83340,83340,60,3120,3120 -63361458,1732,10,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Woodrow Ave,,28,83340,83340,0,3120,3120 -63361458,1733,11,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Arbutus St,,28,83400,83400,60,3120,3120 -63361458,1734,12,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Callender St,,28,83400,83400,0,3120,3120 -63361458,1735,13,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Westview St,,28,83460,83460,60,3120,3120 -63361458,1736,14,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave opp Health Ctr,,28,83520,83520,60,3120,3120 -63361458,1737,15,0,bltal,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Harvard St,,28,83520,83520,0,3120,3120 -63361458,381,16,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Wales St,,28,83580,83580,60,3120,3120 -63361458,382,17,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Charlotte St,,28,83640,83640,60,3120,3120 -63361458,383,18,0,frnpk,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Ellington St,,28,83700,83700,60,3120,3120 -63361458,9448,19,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Blue Hill Ave,,28,83760,83760,60,3120,3120 -63361458,1739,20,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Maple St,,28,83820,83820,60,3120,3120 -63361458,1740,21,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Elm Hill Ave,,28,83820,83820,0,3120,3120 -63361458,1741,22,0,svrhm,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Humboldt Ave,,28,83880,83880,60,3120,3120 -63361458,1742,23,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Harold St,,28,83940,83940,60,3120,1200 -63361458,1743,24,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Walnut Ave,,28,84000,84000,60,3120,1200 -63361458,1188,25,0,egles,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave opp Weld Ave - Egleston Sq,,28,84060,84060,60,3120,1200 -63361458,1215,26,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Bray St,,28,84120,84120,60,3120,1140 -63361458,1216,27,0,,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Dimock St,,28,84180,84180,60,3120,1140 -63361458,11531,28,0,jasst,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Jackson Square,place-jaksn,28,84300,84300,120,3120,960 -63362022,900,1,1,wtryd,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Water St @ Watertown Yard,,55,42480,42480,,3600,3600 -63362022,902,2,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Galen St @ Boyd St,,55,42600,42600,120,3600,3600 -63362022,1900,3,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Pearl St,,55,42600,42600,0,3600,3600 -63362022,903,4,0,nwcor,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Washington St @ Bacon St,,55,42660,42660,60,3600,3600 -63362022,8517,5,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Church St,,55,42720,42720,60,3600,3600 -63362022,8518,6,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Newtonville Ave,,55,42780,42780,60,3600,3600 -63362022,8519,7,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Bellevue St,,55,42780,42780,0,3600,3600 -63362022,8520,8,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Lombard St,,55,42780,42780,0,3600,3600 -63362022,8521,9,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Cabot St,,55,42840,42840,60,3600,3600 -63362022,8522,10,0,mtalv,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,785 Centre St,,55,42840,42840,0,3600,3600 -63362022,8523,11,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Cotton St,,55,42900,42900,60,3600,3600 -63362022,8524,12,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Mill St,,55,42900,42900,0,3600,3600 -63362022,8525,13,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Ward St,,55,42960,42960,60,3600,3600 -63362022,8526,14,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Commonwealth Ave,,55,42960,42960,0,3600,3600 -63362022,8527,15,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Bowen St,,55,43020,43020,60,3600,3600 -63362022,8528,16,0,newct,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Beacon St,,55,43080,43080,60,3600,3600 -63362022,8529,17,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Cypress St @ Braeland Ave,,55,43080,43080,0,3600,3600 -63362022,8530,18,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Cypress St,,55,43140,43140,60,3600,3600 -63362022,8531,19,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Athelstane Rd,,55,43200,43200,60,3600,3600 -63362022,8532,20,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Stearns St,,55,43200,43200,0,3600,3600 -63362022,8533,21,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Boylston St,,55,43200,43200,0,3600,3600 -63362022,8534,22,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Parker Ave,,55,43260,43260,60,3600,3600 -63362022,8535,23,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Hagen Rd,,55,43260,43260,0,3600,3600 -63362022,8536,24,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Parker Rd,,55,43320,43320,60,3600,3600 -63362022,8537,25,0,prkwl,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St opp Wheeler Rd,,55,43320,43320,0,3600,3600 -63362022,85371,26,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wheeler Rd @ Meadowbrook Rd,,55,43380,43380,60,3600,3600 -63362022,8540,27,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Meadowbrook Rd,,55,43440,43440,60,3600,3600 -63362022,8541,28,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Greenwood St,,55,43440,43440,0,3600,3600 -63362022,8542,29,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Arnold Rd,,55,43500,43500,60,3600,3600 -63362022,18542,30,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Rosalie Rd,,55,43500,43500,0,3600,3600 -63362022,8544,31,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St @ Carlson Ave,,55,43560,43560,60,3600,3600 -63362022,8545,32,0,oak,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St @ Wiswall Rd,,55,43560,43560,0,3600,3600 -63362022,8546,33,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ Indian Ridge Rd,,55,43560,43560,0,3600,3600 -63362022,8547,34,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd opp McCarthy Rd,,55,43560,43560,0,3600,3600 -63362022,8548,35,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ O Roadway,,55,43620,43620,60,3600,3600 -63362022,8549,36,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ M Roadway,,55,43620,43620,0,3600,3600 -63362022,8550,37,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ Walsh Rd,,55,43680,43680,60,3600,3600 -63362022,8552,38,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Walsh Rd,,55,43680,43680,0,3600,3600 -63362022,85511,39,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Spiers Rd,,55,43680,43680,0,3600,3600 -63362022,8553,40,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Keller Path,,55,43740,43740,60,3600,3600 -63362022,8554,41,0,sbpfd,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ McCarthy Rd,,55,43740,43740,0,3600,3600 -63362022,85551,42,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Fredette Rd @ Spiers Rd,,55,43800,43800,60,3600,3600 -63362022,8556,43,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spiers Rd @ Dedham St,,55,43800,43800,0,3600,3600 -63362022,85562,44,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ VFW Pkwy,,55,43980,43980,180,3600,3600 -63362022,85563,45,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ VFW Pkwy,,55,43980,43980,0,3600,3600 -63362022,85564,46,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Capital St,,55,43980,43980,0,3600,3600 -63362022,85565,47,0,baker,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St opp Varick Rd,,55,44040,44040,60,3600,3600 -63362022,85566,48,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Amesbury St,,55,44100,44100,60,3600,3600 -63362022,85567,49,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Cutter Rd,,55,44100,44100,0,3600,3600 -63362022,85568,50,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Dunwell St,,55,44160,44160,60,3600,3600 -63362022,85569,51,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Wycliff Ave,,55,44220,44220,60,3600,3600 -63362022,816,52,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spring St @ Moville St,,55,44220,44220,0,3600,3600 -63362022,817,53,0,cloop,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spring St @ VA Hospital,,55,44280,44280,60,3600,3600 -63362022,10835,54,0,,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham Mall @ South Side,,55,44700,44700,420,3600,3600 -63362022,10833,55,0,dmall,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham Mall @ Stop & Shop,,55,44820,44820,120,3600,3600 -Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,5271,1,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,place-welln,5,56160,56160,,60,60 -Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,28743,2,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,,5,56700,56700,540,120,120 -Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,29001,3,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,place-sull,5,57660,57660,960,60,60 -Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,9070028,4,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,,5,58260,58260,600,120,120 -Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,9070026,5,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,,5,58560,58560,300,180,180 -Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,9070026,1,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,North Station,,5,75420,75420,,180,180 -Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,9070029,2,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Community College - Rutherford Ave @ Austin St,,5,75600,75600,180,120,120 -Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,29001,3,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Sullivan Square,place-sull,5,76200,76200,600,60,60 -Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,28742,4,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Grand Union Blvd @ Foley St,,5,76440,76440,240,180,180 -Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,5271,5,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Wellington Station Busway,place-welln,5,76860,76860,420,120,120 -Orange-NorthStationWellington-Saturday-df498-0-22:01:30,5271,1,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,place-welln,5,79260,79260,,0,0 -Orange-NorthStationWellington-Saturday-df498-0-22:01:30,28743,2,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,,5,79680,79680,420,0,0 -Orange-NorthStationWellington-Saturday-df498-0-22:01:30,29001,3,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,place-sull,5,80340,80340,660,0,0 -Orange-NorthStationWellington-Saturday-df498-0-22:01:30,9070028,4,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,,5,80760,80760,420,0,0 -Orange-NorthStationWellington-Saturday-df498-0-22:01:30,9070026,5,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,,5,80940,80940,180,0,0 -Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,5271,1,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,place-welln,5,24060,24060,,120,120 -Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,28743,2,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,,5,24420,24420,360,180,180 -Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,29001,3,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,place-sull,5,24960,24960,540,120,120 -Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,9070028,4,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,,5,25320,25320,360,120,120 -Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,9070026,5,,,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,,5,25500,25500,180,120,120 +plan_trip_id,stop_id,stop_sequence,block_id,route_id,service_id,route_pattern_id,route_pattern_typicality,direction_id,direction,direction_destination,stop_name,plan_stop_count,plan_start_time,plan_start_dt,plan_travel_time_seconds,plan_route_direction_headway_seconds,plan_direction_destination_headway_seconds +63070693,334,1,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Ashmont,29,55800,2024-08-01T15:30:00.000000,,0,0 +63070693,367,2,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Dorchester Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,369,3,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Welles Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,371,4,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Centre St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,372,5,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Southern Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,373,6,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Spencer St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,374,7,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Talbot Station,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,375,8,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Bernard St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,376,9,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Kerwin St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,378,10,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Talbot Ave @ Nightingale St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1737,11,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Harvard St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,381,12,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Wales St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,382,13,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Charlotte St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,383,14,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Blue Hill Ave @ Ellington St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,9448,15,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Blue Hill Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1739,16,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Maple St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1740,17,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Elm Hill Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1741,18,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Humboldt Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1742,19,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Seaver St @ Harold St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1743,20,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Walnut Ave,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1188,21,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave opp Weld Ave - Egleston Sq,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1215,22,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Bray St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1216,23,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Dimock St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,11531,24,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Jackson Square,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1219,25,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Centre St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1221,26,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Cedar St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1222,27,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Columbus Ave @ Malcolm X Blvd,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,1224,28,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Tremont St opp Prentiss St,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070693,17861,29,C22-146,22,BUS32024-hbc34ns1-Weekday-02,22-_-1,1,1,Inbound,Ruggles Station,Ruggles,29,55800,2024-08-01T15:30:00.000000,0,0,0 +63070975,33,1,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,City Point Bus Terminal,14,26880,2024-08-01T07:28:00.000000,,0,0 +63070975,10033,2,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,E 1st St @ O St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,34,3,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,P St @ E 2nd St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,35,4,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,P St @ E Broadway,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,895,5,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,E Broadway @ N St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,886,6,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,L St @ Broadway,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,884,7,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ E 1st St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,210,8,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ Drydock Ave,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,212,9,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ D St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,890,10,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ WTC Ave,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,891,11,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St opp Melcher St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,892,12,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Summer St @ Atlantic Ave,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,6535,13,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Federal St @ Matthews St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63070975,16535,14,C07-23,7,BUS32024-hbc34ns1-Weekday-02,7-1-1,1,1,Inbound,Otis Street & Summer Street,Otis St @ Summer St,14,26880,2024-08-01T07:28:00.000000,0,0,0 +63071767,1475,1,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Bowdoin St,21,57000,2024-08-01T15:50:00.000000,,0,0 +63071767,1478,2,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Rowell St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1479,3,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Hancock St @ Jerome St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1480,4,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Columbia Rd @ Hancock St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1481,5,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Belden St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,11482,6,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Uphams Corner Station,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,14831,7,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St opp Howard Ave,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1484,8,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ E Cottage St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1485,9,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Langdon St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1486,10,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Magazine St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1487,11,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Hampden St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1488,12,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Adams St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1489,13,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Dearborn St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1491,14,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Dudley St @ Harrison Ave,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,64000,15,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Nubian,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1148,16,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Shawmut Ave,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,11149,17,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ O'Bryant HS,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,11148,18,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Madison Park HS,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,21148,19,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Malcolm X Blvd @ Tremont St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,1224,20,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Tremont St opp Prentiss St,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63071767,17861,21,C15-106,15,BUS32024-hbc34ns1-Weekday-02,15-_-1,1,1,Inbound,Ruggles Station,Ruggles,21,57000,2024-08-01T15:50:00.000000,0,0,0 +63072828,6551,1,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Federal St @ Franklin St,12,29220,2024-08-01T08:07:00.000000,,0,0 +63072828,16535,2,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Otis St @ Summer St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,6555,3,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Chinatown Gate,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,9983,4,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Stuart St @ Charles St S,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,177,5,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Arlington St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,173,6,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Berkeley St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,178,7,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Saint James Ave @ Dartmouth St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,903,8,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Washington St @ Bacon St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,19031,9,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,400 Centre St - West,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,988,10,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Centre St @ Jefferson St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,989,11,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Galen St @ Maple St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63072828,999,12,C18-134,504,BUS32024-hbc34ns1-Weekday-02,504-2-0,1,0,Outbound,Watertown Yard,Galen St @ Water St,12,29220,2024-08-01T08:07:00.000000,0,0,0 +63075088,64000,1,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Nubian,33,20700,2024-08-01T05:45:00.000000,,0,0 +63075088,1148,2,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Shawmut Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,11149,3,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ O'Bryant HS,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,11148,4,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Madison Park HS,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,21148,5,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Malcolm X Blvd @ Tremont St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1357,6,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St opp Roxbury Crossing Sta,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,13590,7,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Tobin Community Center,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1360,8,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Saint Alphonsus St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1362,9,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Tremont St @ Huntington Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1363,10,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Huntington Ave @ Fenwood Rd,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1365,11,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,835 Huntington Ave opp Parker Hill Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1366,12,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Huntington Ave @ Riverway,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1526,13,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Washington St @ Pearl St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1367,14,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Kent St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1369,15,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Aspinwall Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1371,16,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St opp Vernon St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1372,17,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Beacon St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1373,18,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Stedman St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1375,19,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St @ Coolidge St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1376,20,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard St opp Verndale St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1378,21,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Ave @ Commonwealth Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1379,22,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Ave @ Brighton Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,964,23,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Brighton Ave opp Quint Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1111,24,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Cambridge St opp Hano St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,1112,25,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Cambridge St @ Harvard Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2558,26,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Empire St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2559,27,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Oxford St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2560,28,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Kingsley St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2561,29,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St @ Western Ave,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2564,30,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,N Harvard St opp Harvard Stadium Gate 2,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,25641,31,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,JFK St @ Eliot St,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,2168,32,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Massachusetts Ave @ Johnston Gate,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63075088,22549,33,A66-325,66,BUS32024-hba34ns1-Weekday-02,66-6-0,1,0,Outbound,Harvard Square,Harvard Sq @ Garden St - Dawes Island,33,20700,2024-08-01T05:45:00.000000,0,0,0 +63089353,32002,1,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Quincy Center,53,72900,2024-08-01T20:15:00.000000,,0,0 +63089353,3237,2,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Washington St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3239,3,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Quincy YMCA,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3425,4,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Coddington St @ Southern Artery,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3240,5,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,55 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3241,6,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Overlook Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3242,7,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Samoset Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3243,8,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,227 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3244,9,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Barbour Terr,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,13245,10,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Moffat Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3245,11,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,405 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3246,12,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,455 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3247,13,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Braintree Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3292,14,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3295,15,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Oakwood Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3298,16,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Forbush Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3299,17,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Brockton Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3300,18,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Roach St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3301,19,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Bowes Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3303,20,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,375 Palmer St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3304,21,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Taffrail Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3305,22,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,26 Taffrail Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3306,23,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Taffrail Rd @ Bicknell St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3307,24,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ O'Brien Towers Driveway,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3308,25,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,O'Brien Towers,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3309,26,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,O'Brien Towers Driveway @ Bicknell St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3310,27,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ Binnacle Lane,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3311,28,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Bicknell St @ Yardarm Ln,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3312,29,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Shed St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3314,30,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Bowes Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3315,31,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Roach St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3316,32,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Brockton Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3317,33,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Empire St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3320,34,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Oakwood Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3321,35,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Wilgus Rd,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3322,36,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St opp Utica St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3323,37,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Palmer St @ Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3249,38,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,565 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3250,39,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Lee St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3251,40,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Opp 700 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3252,41,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,749 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3253,42,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Babcock St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3254,43,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Manet Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3255,44,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Newton St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3256,45,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St opp Malvern St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3257,46,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Rockland St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3258,47,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Darrow St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3259,48,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Manet Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3260,49,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Wall St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3261,50,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Bell St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3262,51,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Thomas St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3263,52,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,1263 Sea St,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63089353,3265,53,Q222-59,216,BUS32024-hbq34ns1-Weekday-02,216-2-0,1,0,Outbound,Houghs Neck,Sea St @ Fensmere Ave,53,72900,2024-08-01T20:15:00.000000,0,0,0 +63101043,5072,1,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Malden Center,39,43200,2024-08-01T12:00:00.000000,,0,0 +63101043,5328,2,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Elm St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,9028,3,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St opp Russell St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5330,4,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Prospect St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5331,5,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St opp West St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,45332,6,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Vista St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5332,7,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Pleasant St @ Fellsway E,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,8308,8,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Fellsway W,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5282,9,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Grant Ave,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5283,10,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,390 Salem St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5284,11,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Almont St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5285,12,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Paris St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5286,13,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Court St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5287,14,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St @ Allen Ct,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5002,15,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Salem St opp River St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5031,16,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ High St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5032,17,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Emerson St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5290,18,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Summer St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5291,19,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ George St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5292,20,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Stearns Ave,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5293,21,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Windsor Rd,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5294,22,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Wellesley St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5295,23,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Princeton St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5296,24,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Harvard St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5297,25,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Marion St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5298,26,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Medford St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5299,27,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Albion St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5300,28,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Dexter St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5301,29,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Bow St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5302,30,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St opp Moreland St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,5303,31,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Main St @ Broadway,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2704,32,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Thurston St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2706,33,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Marshall St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2707,34,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Montgomery Ave,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2710,35,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Cross St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2711,36,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Glen St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2713,37,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Lincoln St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,2714,38,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Broadway @ Mt Vernon St,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63101043,29001,39,F101-47,101,BUS32024-hbf34ns1-Weekday-02,101-3-1,1,1,Inbound,Sullivan Square Station,Sullivan Square,39,43200,2024-08-01T12:00:00.000000,0,0,0 +63156984,141,1,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Alewife,21,40200,2024-08-01T11:10:00.000000,,0,0 +63156984,2482,2,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,West Service Rd @ Lake St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,23530,3,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,244 Pleasant St opp Brunswick Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,23531,4,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Gould Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,23532,5,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Spring Valley St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,23533,6,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Pleasant St @ Wellington St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,2283,7,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Massachusetts Ave @ Mystic St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,2284,8,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Massachusetts Ave @ Central St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7976,9,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Mill St @ Bacon St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7977,10,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Winthrop Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7978,11,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Oak Hill Dr,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7979,12,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St @ Symmes Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7981,13,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Symmes Circle @ Symmes Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7953,14,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Summer St opp Washington St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7954,15,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Candia St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7955,16,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Ronald Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7956,17,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Washington St @ Mountain Ave,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7959,18,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Clyde Terr @ Lawrence Ln,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7960,19,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Thomas St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7962,20,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Hancock St,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63156984,7961,21,T67-22,67,BUS32024-hbt34ns1-Weekday-02,67-4-0,1,0,Outbound,Turkey Hill,Forest St @ Heard Rd,21,40200,2024-08-01T11:10:00.000000,0,0,0 +63157369,20761,1,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Harvard,22,40620,2024-08-01T11:17:00.000000,,0,0 +63157369,2020,2,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Story St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2021,3,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Ash St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2023,4,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Sparks St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2025,5,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Longfellow Rd,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2026,6,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Traill St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2027,7,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St opp Coolidge Ave,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2028,8,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Brattle St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2030,9,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Homer Ave,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2032,10,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Saint Marys St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2033,11,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Keenan St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2034,12,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Kimball Rd,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2036,13,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Upland Rd,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2037,14,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Winsor Ave,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2038,15,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Adams Ave,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2040,16,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Amherst Rd,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2047,17,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Bates Rd E,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2042,18,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Russell Ave,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2043,19,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Marshall St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2044,20,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Summer St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,2046,21,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Mt Auburn St @ Main St,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157369,8178,22,T71-68,71,BUS32024-hbt34ns1-Weekday-02,71-2-0,1,0,Outbound,Watertown Square,Watertown Sq Terminal,22,40620,2024-08-01T11:17:00.000000,0,0,0 +63157836,2362,1,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Massachusetts Ave,37,29400,2024-08-01T08:10:00.000000,,0,0 +63157836,2363,2,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Lewis Ave,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2365,3,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Hamlet St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2366,4,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St opp Hayes St - Medford Line,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2367,5,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,High St @ Jerome St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2368,6,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,High St @ Monument St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,3513,7,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ High St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2369,8,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Harvard Ave,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2370,9,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Holton St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2371,10,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Arlington St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2372,11,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Mystic Valley Pkwy,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2373,12,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Harris Rd,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2374,13,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ North St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2375,14,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Hillsdale Rd,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2376,15,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Winthrop St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2378,16,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Boston Ave @ Tufts Garage,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2379,17,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Boston Ave,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2380,18,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Professors Row,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2381,19,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,College Ave @ Powder House Sq,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2695,20,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway opp Warner St - Powder House Sq,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2696,21,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Bay State Ave,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2697,22,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Josephine Ave - Ball Sq,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2698,23,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Cedar St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2699,24,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Broadway @ Hinckley St - Magoun Sq,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,12699,25,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Broadway - Magoun Sq,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,23841,26,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Glenwood Rd,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2385,27,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Central St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2386,28,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Sycamore St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2388,29,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ School St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2389,30,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ Skilton Ave,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2390,31,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ Walnut St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2391,32,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Pearl St @ McGrath Hwy,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2689,33,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,422 McGrath Hwy,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2690,34,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Medford St @ Washington St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2600,35,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,McGrath Hwy @ Medford St,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,2601,36,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,McGrath Hwy @ Twin City Plaza,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63157836,70500,37,T80-134,80,BUS32024-hbt34ns1-Weekday-02,80-_-1,1,1,Inbound,Lechmere Station,Lechmere,37,29400,2024-08-01T08:10:00.000000,0,0,0 +63256578,5327,1,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Malden Center,22,65100,2024-08-01T18:05:00.000000,,0,0 +63256578,45327,2,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Commercial St @ Charles St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,45326,3,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Commercial St @ Medford St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,45330,4,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Green St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,44330,5,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Brackenbury St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,45331,6,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Medford St @ Main St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5395,7,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Main St @ Converse Ave,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5047,8,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Belmont St @ Main St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5048,9,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Belmont St @ Kinsman St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5049,10,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Belmont St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5050,11,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Tappan St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5051,12,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Hancock St @ Dean St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,15492,13,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Maple Ave,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5495,14,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Church St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5496,15,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Norwood St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5559,16,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway opp Second St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5560,17,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Broadway @ Gladstone St,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5561,18,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Santilli Circle,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,55631,19,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Gateway Center opp Texas Roadhouse,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,55632,20,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Gateway Center @ Target,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,6569,21,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Corporation Way,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256578,5271,22,G101-66,97,BUS32024-hbg34ns1-Weekday-02,97-5-1,1,1,Inbound,Wellington Station,Wellington Station Busway,22,65100,2024-08-01T18:05:00.000000,0,0,0 +63256596,53270,1,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Malden Center,29,74880,2024-08-01T20:48:00.000000,,0,0 +63256596,5289,2,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Centre St @ Stop & Shop,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5342,3,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Main St opp Pleasant St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5343,4,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Irving St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5344,5,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Eastern Ave,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5345,6,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St opp Clayton St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5347,7,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Cross St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5348,8,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Winthrop St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5349,9,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Belmont St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5350,10,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Bennett St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5351,11,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Central Ave,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5352,12,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Glendale St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5353,13,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Walnut St - Glendale Towers,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5354,14,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Ferry St @ Broadway,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5490,15,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Raymond St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,15492,16,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Maple Ave,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5495,17,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Church St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5496,18,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Norwood St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5559,19,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Second St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5560,20,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Gladstone St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5497,21,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Bowdoin St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5498,22,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Beacham St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5499,23,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway opp Thorndike St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5500,24,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Broadway @ Horizon Way,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5501,25,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Opp 173 Alford St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,55011,26,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ MWRA Pump Station,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,55012,27,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ MBTA Charlestown Garage,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,5502,28,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Alford St @ West St,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256596,29001,29,G109-113,104,BUS32024-hbg34ns1-Weekday-02,104-_-1,1,1,Inbound,Sullivan Square Station,Sullivan Square,29,74880,2024-08-01T20:48:00.000000,0,0,0 +63256882,29011,1,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Sullivan Square,21,64620,2024-08-01T17:57:00.000000,,0,0 +63256882,2718,2,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Austin St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2719,3,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Illinois Ave,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2721,4,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Cross St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2723,5,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Fellsway W,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2725,6,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Temple St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2726,7,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Winter Hill Plaza,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2729,8,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Main St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2731,9,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway opp Glenwood Rd,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2733,10,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Medford St - Magoun Sq,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2734,11,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ William St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2735,12,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Alfred St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2736,13,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Boston Ave - Ball Sq,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2737,14,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Pearson Rd,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2738,15,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Broadway @ Warner St - Powder House Sq,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,5012,16,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ Broadway,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,5014,17,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ William St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,5015,18,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,College Ave @ Highland Ave,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2582,19,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Elm St @ Chester St,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,2628,20,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Grove St @ Highland Ave,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63256882,5104,21,G89-5,89,BUS32024-hbg34ns1-Weekday-02,89-2-0,1,0,Outbound,Clarendon Hill or Davis Station,Davis,21,64620,2024-08-01T17:57:00.000000,0,0,0 +63361157,1618,1,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,84 Georgetowne Pl,32,20160,2024-08-01T05:36:00.000000,,, +63361157,11618,2,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Georgetowne Dr @ Georgetowne Pl,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,6523,3,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Dedham Pkwy,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,6524,4,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Leighton Rd,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,6526,5,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Alwin St @ Turtle Pond Pkwy,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,6522,6,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Dedham Pkwy @ Georgetowne Dr,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,1616,7,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Margaretta Dr @ Crown Point Dr,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,1617,8,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Opp 67 Crown Point Dr,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,1620,9,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Georgetowne Dr opp Margaretta Dr,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,1621,10,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,W Boundary Rd @ Georgetowne Dr,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,1623,11,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,W Boundary Rd @ Cedarcrest Rd,32,20160,2024-08-01T05:36:00.000000,0,, +63361157,625,12,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ W Boundary Rd,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,10625,13,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Heron St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,626,14,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Maplewood St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,627,15,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Cowing St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,628,16,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Enneking Pkwy,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,629,17,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St opp Schubert St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,630,18,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Blue Ledge Dr,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,631,19,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Beech St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,632,20,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Cornell St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,633,21,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Metropolitan Ave,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,10633,22,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Rosecliff St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,634,23,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Albano St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,635,24,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Poplar St,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,636,25,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Cummins Hwy,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,637,26,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Firth Rd,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,638,27,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Granfield Ave,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,639,28,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Whipple Ave,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,640,29,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Archdale Rd,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,641,30,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Aldwin Rd,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,642,31,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Washington St @ Tollgate Way,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361157,10642,32,B40-149,40,BUS32024-hbb34ns1-Weekday-02,40-2-1,1,1,Inbound,Forest Hills Station,Forest Hills,32,20160,2024-08-01T05:36:00.000000,0,,0 +63361458,18511,1,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Mattapan,28,83100,2024-08-01T23:05:00.000000,,0,0 +63361458,1722,2,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,1624 Blue Hill Ave @ Mattapan Sq,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1723,3,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Babson St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1724,4,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave opp Woodhaven St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1725,5,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,1458 Blue Hill Ave opp Almont St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1726,6,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Mattapan Library,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1728,7,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Fessenden St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1730,8,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Woolson St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1731,9,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Morton St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1732,10,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Woodrow Ave,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1733,11,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Arbutus St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1734,12,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Callender St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1735,13,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Westview St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1736,14,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave opp Health Ctr,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1737,15,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Harvard St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,381,16,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Wales St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,382,17,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Charlotte St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,383,18,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Blue Hill Ave @ Ellington St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,9448,19,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Blue Hill Ave,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1739,20,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Maple St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1740,21,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Elm Hill Ave,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1741,22,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Humboldt Ave,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1742,23,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Seaver St @ Harold St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1743,24,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Walnut Ave,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1188,25,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave opp Weld Ave - Egleston Sq,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1215,26,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Bray St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,1216,27,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Columbus Ave @ Dimock St,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63361458,11531,28,B29-40,29,BUS32024-hbb34ns1-Weekday-02,29-5-1,1,1,Inbound,Jackson Square Station,Jackson Square,28,83100,2024-08-01T23:05:00.000000,0,0,0 +63362022,900,1,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Water St @ Watertown Yard,55,42480,2024-08-01T11:48:00.000000,,0,0 +63362022,902,2,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Galen St @ Boyd St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,1900,3,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Pearl St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,903,4,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Washington St @ Bacon St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8517,5,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Church St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8518,6,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Newtonville Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8519,7,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Bellevue St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8520,8,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Lombard St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8521,9,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Cabot St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8522,10,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,785 Centre St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8523,11,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Cotton St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8524,12,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Mill St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8525,13,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Ward St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8526,14,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Commonwealth Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8527,15,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Bowen St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8528,16,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Centre St @ Beacon St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8529,17,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Cypress St @ Braeland Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8530,18,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Cypress St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8531,19,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Athelstane Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8532,20,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Stearns St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8533,21,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Boylston St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8534,22,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Parker Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8535,23,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Hagen Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8536,24,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St @ Parker Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8537,25,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Parker St opp Wheeler Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85371,26,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wheeler Rd @ Meadowbrook Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8540,27,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Meadowbrook Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8541,28,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Greenwood St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8542,29,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Arnold Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,18542,30,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St opp Rosalie Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8544,31,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St @ Carlson Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8545,32,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham St @ Wiswall Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8546,33,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ Indian Ridge Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8547,34,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd opp McCarthy Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8548,35,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ O Roadway,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8549,36,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ M Roadway,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8550,37,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Wiswall Rd @ Walsh Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8552,38,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Walsh Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85511,39,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Spiers Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8553,40,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ Keller Path,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8554,41,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Sawmill Brook Pkwy @ McCarthy Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85551,42,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Fredette Rd @ Spiers Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,8556,43,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spiers Rd @ Dedham St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85562,44,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ VFW Pkwy,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85563,45,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ VFW Pkwy,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85564,46,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Capital St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85565,47,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St opp Varick Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85566,48,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Amesbury St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85567,49,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Cutter Rd,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85568,50,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Dunwell St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,85569,51,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Baker St @ Wycliff Ave,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,816,52,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spring St @ Moville St,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,817,53,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Spring St @ VA Hospital,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,10835,54,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham Mall @ South Side,55,42480,2024-08-01T11:48:00.000000,0,0,0 +63362022,10833,55,B52-191,52,BUS32024-hbb34ns1-Weekday-02,52-5-0,1,0,Outbound,Dedham Mall,Dedham Mall @ Stop & Shop,55,42480,2024-08-01T11:48:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,5271,1,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,5,56160,2024-08-01T15:36:00.000000,,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,28743,2,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,5,56160,2024-08-01T15:36:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,29001,3,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,5,56160,2024-08-01T15:36:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,9070028,4,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,5,56160,2024-08-01T15:36:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-0-15:36:06,9070026,5,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,5,56160,2024-08-01T15:36:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,9070026,1,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,North Station,5,75420,2024-08-01T20:57:00.000000,,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,9070029,2,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Community College - Rutherford Ave @ Austin St,5,75420,2024-08-01T20:57:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,29001,3,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Sullivan Square,5,75420,2024-08-01T20:57:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,28742,4,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Grand Union Blvd @ Foley St,5,75420,2024-08-01T20:57:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-8b4a8-1-20:57:18,5271,5,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-8b8,Shuttle-NorthStationWellington-0-1,5,1,North,Oak Grove,Wellington Station Busway,5,75420,2024-08-01T20:57:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-df498-0-22:01:30,5271,1,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,5,79260,2024-08-01T22:01:00.000000,,0,0 +Orange-NorthStationWellington-Saturday-df498-0-22:01:30,28743,2,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,5,79260,2024-08-01T22:01:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-df498-0-22:01:30,29001,3,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,5,79260,2024-08-01T22:01:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-df498-0-22:01:30,9070028,4,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,5,79260,2024-08-01T22:01:00.000000,0,0,0 +Orange-NorthStationWellington-Saturday-df498-0-22:01:30,9070026,5,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Sa-df8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,5,79260,2024-08-01T22:01:00.000000,0,0,0 +Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,5271,1,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Wellington Station Busway,5,24060,2024-08-01T06:41:00.000000,,0,0 +Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,28743,2,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Grand Union Blvd @ Foley St,5,24060,2024-08-01T06:41:00.000000,0,0,0 +Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,29001,3,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Sullivan Square,5,24060,2024-08-01T06:41:00.000000,0,0,0 +Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,9070028,4,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,Community College - Rutherford Ave @ Austin St,5,24060,2024-08-01T06:41:00.000000,0,0,0 +Orange-NorthStationWellington-Sunday-8b4a8-0-06:41:48,9070026,5,,Shuttle-NorthStationWellington,Orange-NorthStationWellington-Su-8b8,Shuttle-NorthStationWellington-0-0,5,0,South,Forest Hills,North Station,5,24060,2024-08-01T06:41:00.000000,0,0,0 diff --git a/tests/bus_performance_manager/test_gtfs.py b/tests/bus_performance_manager/test_gtfs.py index 46e27661..c733cf51 100644 --- a/tests/bus_performance_manager/test_gtfs.py +++ b/tests/bus_performance_manager/test_gtfs.py @@ -1,67 +1,62 @@ import os import logging from unittest import mock -from urllib import request +from dataclasses import dataclass from datetime import date import polars as pl import polars.testing as pl_test -from lamp_py.bus_performance_manager.events_gtfs_schedule import ( - gtfs_events_for_date, -) +from lamp_py.bus_performance_manager.events_gtfs_schedule import bus_gtfs_events_for_date +from lamp_py.runtime_utils.remote_files import GTFSArchive current_dir = os.path.join(os.path.dirname(__file__)) SERVICE_DATE = date(2024, 8, 1) -mock_file_list = [ - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/calendar.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/calendar_dates.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/directions.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/feed_info.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/route_patterns.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/routes.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/stop_times.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/stops.parquet", - "https://performancedata.mbta.com/lamp/gtfs_archive/2024/trips.parquet", -] - - -def mock_file_download(object_path: str, file_name: str) -> bool: - """dummy download function for https files""" - request.urlretrieve(object_path, file_name) - return True - - -@mock.patch( - "lamp_py.bus_performance_manager.events_gtfs_schedule.file_list_from_s3" -) -@mock.patch( - "lamp_py.bus_performance_manager.events_gtfs_schedule.download_file", - new=mock_file_download, -) -def test_gtfs_events_for_date(s3_patch: mock.MagicMock) -> None: +gtfs = GTFSArchive(bucket="https://performancedata.mbta.com", prefix="lamp/gtfs_archive") + + +@dataclass +class S3Location: + """ + wrapper for a bucket name and prefix pair used to define an s3 location + """ + + bucket: str + prefix: str + version: str = "1.0" + + @property + def s3_uri(self) -> str: + """generate the full s3 uri for the location""" + return f"{self.bucket}/{self.prefix}" + + +@mock.patch("lamp_py.bus_performance_manager.gtfs_utils.object_exists") +@mock.patch("lamp_py.bus_performance_manager.gtfs_utils.compressed_gtfs", gtfs) +@mock.patch("lamp_py.runtime_utils.remote_files.S3Location", S3Location) +def test_gtfs_events_for_date(exists_patch: mock.MagicMock) -> None: """ test gtfs_events_for_date pipeline """ # mock files from S3 with https://performancedata paths - s3_patch.return_value = mock_file_list + exists_patch.return_value = True - bus_events = gtfs_events_for_date(SERVICE_DATE) + bus_events = bus_gtfs_events_for_date(SERVICE_DATE) # CSV Bus events expected_bus_events = pl.read_csv( os.path.join(current_dir, "bus_test_gtfs.csv"), schema=bus_events.schema, - ).sort(by=["trip_id", "stop_sequence"]) + ).sort(by=["plan_trip_id", "stop_sequence"]) # CSV trips - expected_trips = expected_bus_events.select("trip_id").unique() + expected_trips = expected_bus_events.select("plan_trip_id").unique() # Filter and sort pipeline events for CSV trips - bus_events = bus_events.join( - expected_trips, on="trip_id", how="right" - ).sort(by=["trip_id", "stop_sequence"]) + bus_events = bus_events.join(expected_trips, on="plan_trip_id", how="right").sort( + by=["plan_trip_id", "stop_sequence"] + ) # Compare pipeline values to CSV values by column column_exceptions = [] @@ -76,19 +71,15 @@ def test_gtfs_events_for_date(s3_patch: mock.MagicMock) -> None: for column in expected_bus_events.columns: if column in skip_columns: continue - for trip_id in expected_bus_events.get_column("trip_id").unique(): + for trip_id in expected_bus_events.get_column("plan_trip_id").unique(): try: pl_test.assert_series_equal( - bus_events.filter( - (pl.col("trip_id") == trip_id) - ).get_column(column), - expected_bus_events.filter( - (pl.col("trip_id") == trip_id) - ).get_column(column), + bus_events.filter((pl.col("plan_trip_id") == trip_id)).get_column(column), + expected_bus_events.filter((pl.col("plan_trip_id") == trip_id)).get_column(column), ) except Exception as exception: logging.error( - "Process values (column=%s - trip_id=%s) do not match bus_test_gtfs.csv", + "Process values (column=%s - plan_trip_id=%s) do not match bus_test_gtfs.csv", column, trip_id, ) diff --git a/tests/bus_performance_manager/test_gtfs_rt_ingestion.py b/tests/bus_performance_manager/test_gtfs_rt_ingestion.py index b6c24160..f60c0dc5 100644 --- a/tests/bus_performance_manager/test_gtfs_rt_ingestion.py +++ b/tests/bus_performance_manager/test_gtfs_rt_ingestion.py @@ -1,10 +1,11 @@ import os +from unittest import mock from datetime import datetime, timedelta, date from typing import Tuple, List import polars as pl -from lamp_py.aws.s3 import get_datetime_from_partition_path +from lamp_py.aws.s3 import dt_from_obj_path from lamp_py.bus_performance_manager.events_gtfs_rt import ( read_vehicle_positions, positions_to_events, @@ -36,7 +37,7 @@ def get_service_date_and_files() -> Tuple[date, List[str]]: # get the service date for all the files in the springboard service_date = None for vp_file in vp_files: - new_service_date = get_datetime_from_partition_path(vp_file).date() + new_service_date = dt_from_obj_path(vp_file).date() if service_date is None: service_date = new_service_date @@ -68,7 +69,9 @@ def get_service_date_and_files() -> Tuple[date, List[str]]: "service_date": pl.String, "route_id": pl.String, "trip_id": pl.String, - "start_time": pl.String, + "start_time": pl.Int64, + "start_dt": pl.Datetime, + "stop_count": pl.UInt32, "direction_id": pl.Int8, "stop_id": pl.String, "stop_sequence": pl.Int64, @@ -80,7 +83,8 @@ def get_service_date_and_files() -> Tuple[date, List[str]]: } -def test_gtfs_rt_to_bus_events() -> None: +@mock.patch("lamp_py.bus_performance_manager.gtfs_utils.object_exists") +def test_gtfs_rt_to_bus_events(exists_patch: mock.MagicMock) -> None: """ generate vehicle event dataframes from gtfs realtime vehicle position files. inspect them to ensure they @@ -89,12 +93,12 @@ def test_gtfs_rt_to_bus_events() -> None: generate an empty vechicle event frame and ensure it has the correct columns. """ + exists_patch.return_value = True + service_date, vp_files = get_service_date_and_files() # get the bus vehicle events for these files / service date - bus_vehicle_events = generate_gtfs_rt_events( - service_date=service_date, gtfs_rt_files=vp_files - ) + bus_vehicle_events = generate_gtfs_rt_events(service_date=service_date, gtfs_rt_files=vp_files) assert not bus_vehicle_events.is_empty() @@ -102,9 +106,7 @@ def test_gtfs_rt_to_bus_events() -> None: assert VE_SCHEMA[col] == data_type # the following properties are known to be in the consumed dataset - y1808_events = bus_vehicle_events.filter( - (pl.col("vehicle_id") == "y1808") & (pl.col("trip_id") == "61348621") - ) + y1808_events = bus_vehicle_events.filter((pl.col("vehicle_id") == "y1808") & (pl.col("trip_id") == "61348621")) assert not y1808_events.is_empty() @@ -113,34 +115,19 @@ def test_gtfs_rt_to_bus_events() -> None: assert event["direction_id"] == 0 if event["stop_id"] == "173": - assert event["gtfs_travel_to_dt"] == datetime( - year=2024, month=6, day=1, hour=13, minute=1, second=19 - ) - assert event["gtfs_arrival_dt"] == datetime( - year=2024, month=6, day=1, hour=13, minute=2, second=34 - ) + assert event["gtfs_travel_to_dt"] == datetime(year=2024, month=6, day=1, hour=13, minute=1, second=19) + assert event["gtfs_arrival_dt"] == datetime(year=2024, month=6, day=1, hour=13, minute=2, second=34) if event["stop_id"] == "655": - assert event["gtfs_travel_to_dt"] == datetime( - year=2024, month=6, day=1, hour=12, minute=50, second=31 - ) - assert event["gtfs_arrival_dt"] == datetime( - year=2024, month=6, day=1, hour=12, minute=53, second=29 - ) + assert event["gtfs_travel_to_dt"] == datetime(year=2024, month=6, day=1, hour=12, minute=50, second=31) + assert event["gtfs_arrival_dt"] == datetime(year=2024, month=6, day=1, hour=12, minute=53, second=29) if event["stop_id"] == "903": - assert event["gtfs_travel_to_dt"] == datetime( - year=2024, month=6, day=1, hour=13, minute=3, second=39 - ) - assert event["gtfs_arrival_dt"] == datetime( - year=2024, month=6, day=1, hour=13, minute=11, second=3 - ) + assert event["gtfs_travel_to_dt"] == datetime(year=2024, month=6, day=1, hour=13, minute=3, second=39) + assert event["gtfs_arrival_dt"] == datetime(year=2024, month=6, day=1, hour=13, minute=11, second=3) # the following properties are known to be in the consumed dataset - y1329_events = bus_vehicle_events.filter( - (pl.col("vehicle_id") == "y1329") - & (pl.col("trip_id") == "61884885-OL1") - ) + y1329_events = bus_vehicle_events.filter((pl.col("vehicle_id") == "y1329") & (pl.col("trip_id") == "61884885-OL1")) assert not y1329_events.is_empty() @@ -153,22 +140,16 @@ def test_gtfs_rt_to_bus_events() -> None: # no arrival time at this stop if event["stop_id"] == "12005": - assert event["gtfs_travel_to_dt"] == datetime( - year=2024, month=6, day=1, hour=12, minute=47, second=23 - ) + assert event["gtfs_travel_to_dt"] == datetime(year=2024, month=6, day=1, hour=12, minute=47, second=23) assert event["gtfs_arrival_dt"] is None if event["stop_id"] == "17091": - assert event["gtfs_travel_to_dt"] == datetime( - year=2024, month=6, day=1, hour=12, minute=52, second=41 - ) + assert event["gtfs_travel_to_dt"] == datetime(year=2024, month=6, day=1, hour=12, minute=52, second=41) assert event["gtfs_arrival_dt"] is None # get an empty dataframe by reading the same files but for events the day prior. previous_service_date = service_date - timedelta(days=1) - empty_bus_vehicle_events = generate_gtfs_rt_events( - service_date=previous_service_date, gtfs_rt_files=vp_files - ) + empty_bus_vehicle_events = generate_gtfs_rt_events(service_date=previous_service_date, gtfs_rt_files=vp_files) assert empty_bus_vehicle_events.is_empty() @@ -178,16 +159,16 @@ def test_gtfs_rt_to_bus_events() -> None: pl.concat([bus_vehicle_events, empty_bus_vehicle_events]) -def test_read_vehicle_positions() -> None: +@mock.patch("lamp_py.bus_performance_manager.gtfs_utils.object_exists") +def test_read_vehicle_positions(exists_patch: mock.MagicMock) -> None: """ test that vehicle positions can be read from files and return a df with the correct schema. """ + exists_patch.return_value = True service_date, vp_files = get_service_date_and_files() - vehicle_positions = read_vehicle_positions( - service_date=service_date, gtfs_rt_files=vp_files - ) + vehicle_positions = read_vehicle_positions(service_date=service_date, gtfs_rt_files=vp_files) for col, data_type in vehicle_positions.schema.items(): assert VP_SCHEMA[col] == data_type @@ -377,27 +358,19 @@ def test_positions_to_events() -> None: correctly for different edge cases. """ route_one_positions = route_one() - route_one_events = positions_to_events( - vehicle_positions=route_one_positions - ) + route_one_events = positions_to_events(vehicle_positions=route_one_positions) assert len(route_one_events) == 2 route_two_positions = route_two() - route_two_events = positions_to_events( - vehicle_positions=route_two_positions - ) + route_two_events = positions_to_events(vehicle_positions=route_two_positions) assert len(route_two_events) == 2 route_three_positions = route_three() - route_three_events = positions_to_events( - vehicle_positions=route_three_positions - ) + route_three_events = positions_to_events(vehicle_positions=route_three_positions) assert len(route_three_events) == 2 route_four_positions = route_four() - route_four_events = positions_to_events( - vehicle_positions=route_four_positions - ) + route_four_events = positions_to_events(vehicle_positions=route_four_positions) assert len(route_four_events) == 2 empty_positions = pl.DataFrame(schema=VP_SCHEMA) diff --git a/tests/bus_performance_manager/test_gtfs_utils.py b/tests/bus_performance_manager/test_gtfs_utils.py index 75baafd2..55412a4e 100644 --- a/tests/bus_performance_manager/test_gtfs_utils.py +++ b/tests/bus_performance_manager/test_gtfs_utils.py @@ -1,11 +1,13 @@ from datetime import date +from unittest import mock from lamp_py.bus_performance_manager.gtfs_utils import ( bus_routes_for_service_date, ) -def test_bus_routes_for_service_date() -> None: +@mock.patch("lamp_py.bus_performance_manager.gtfs_utils.object_exists") +def test_bus_routes_for_service_date(exists_patch: mock.MagicMock) -> None: """ Test that bus routes be generated for a given service date. For the generated list ensure @@ -13,7 +15,7 @@ def test_bus_routes_for_service_date() -> None: * don't have a leading zero * contain a subset of known routes """ - assert True + exists_patch.return_value = True service_date = date(year=2023, month=2, day=1) bus_routes = bus_routes_for_service_date(service_date) diff --git a/tests/bus_performance_manager/test_tm_ingestion.py b/tests/bus_performance_manager/test_tm_ingestion.py index d43b838b..8056e1a0 100644 --- a/tests/bus_performance_manager/test_tm_ingestion.py +++ b/tests/bus_performance_manager/test_tm_ingestion.py @@ -57,10 +57,7 @@ def check_stop_crossings(stop_crossings_filepath: str) -> None: # this is the df of all useful records from the stop crossings files raw_stop_crossings = ( pl.scan_parquet(stop_crossings_filepath) - .filter( - pl.col("ACT_ARRIVAL_TIME").is_not_null() - | pl.col("ACT_DEPARTURE_TIME").is_not_null() - ) + .filter(pl.col("ACT_ARRIVAL_TIME").is_not_null() | pl.col("ACT_DEPARTURE_TIME").is_not_null()) .collect() ) @@ -71,9 +68,7 @@ def check_stop_crossings(stop_crossings_filepath: str) -> None: assert not bus_events.is_empty() # ensure we didn't lose any Revenue data from the raw dataset when joining - assert len(bus_events) == len( - raw_stop_crossings.filter((pl.col("IsRevenue") == "R")) - ) + assert len(bus_events) == len(raw_stop_crossings.filter((pl.col("IsRevenue") == "R"))) # check that crossings without trips are garage pullouts bus_garages = { @@ -91,14 +86,11 @@ def check_stop_crossings(stop_crossings_filepath: str) -> None: # check that all arrival and departure timestamps happen after the start of the service date assert bus_events.filter( - (pl.col("tm_arrival_dt") < service_date) - | (pl.col("tm_departure_dt") < service_date) + (pl.col("tm_arrival_dt") < service_date) | (pl.col("tm_departure_dt") < service_date) ).is_empty() # check that all departure times are after the arrival times - assert bus_events.filter( - pl.col("tm_arrival_dt") > pl.col("tm_departure_dt") - ).is_empty() + assert bus_events.filter(pl.col("tm_arrival_dt") > pl.col("tm_departure_dt")).is_empty() # check that there are no leading zeros on route ids assert bus_events.filter( diff --git a/tests/conftest.py b/tests/conftest.py index eb93e2f0..0e533084 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,9 +49,7 @@ def mock__get_pyarrow_dataset( return ds - monkeypatch.setattr( - "lamp_py.aws.s3._get_pyarrow_dataset", mock__get_pyarrow_dataset - ) + monkeypatch.setattr("lamp_py.aws.s3._get_pyarrow_dataset", mock__get_pyarrow_dataset) yield @@ -66,8 +64,6 @@ def fixture_remote_file_locations_patch( don't have access to s3, so tests need to be run against local files. Use monkeypatch to redefine how these utilities work. """ - monkeypatch.setattr( - "lamp_py.runtime_utils.remote_files.S3Location", LocalS3Location - ) + monkeypatch.setattr("lamp_py.runtime_utils.remote_files.S3Location", LocalS3Location) yield diff --git a/tests/ingestion/test_configuration.py b/tests/ingestion/test_configuration.py index 5e6f3306..df83908a 100644 --- a/tests/ingestion/test_configuration.py +++ b/tests/ingestion/test_configuration.py @@ -7,9 +7,7 @@ VEHICLE_POSITIONS_FILENAME = "2022-01-01T00:00:03Z_https_cdn.mbta.com_realtime_VehiclePositions_enhanced.json.gz" -ALERTS_FILENAME = ( - "2022-01-01T00:00:38Z_https_cdn.mbta.com_realtime_Alerts_enhanced.json.gz" -) +ALERTS_FILENAME = "2022-01-01T00:00:38Z_https_cdn.mbta.com_realtime_Alerts_enhanced.json.gz" def test_filname_parsing() -> None: @@ -20,9 +18,7 @@ def test_filname_parsing() -> None: trip_updates_type = ConfigType.from_filename(UPDATE_FILENAME) assert trip_updates_type == ConfigType.RT_TRIP_UPDATES - vehicle_positions_type = ConfigType.from_filename( - VEHICLE_POSITIONS_FILENAME - ) + vehicle_positions_type = ConfigType.from_filename(VEHICLE_POSITIONS_FILENAME) assert vehicle_positions_type == ConfigType.RT_VEHICLE_POSITIONS alerts_type = ConfigType.from_filename(ALERTS_FILENAME) diff --git a/tests/ingestion/test_gtfs_compress.py b/tests/ingestion/test_gtfs_compress.py index 0cd32227..f363d4fc 100644 --- a/tests/ingestion/test_gtfs_compress.py +++ b/tests/ingestion/test_gtfs_compress.py @@ -26,9 +26,7 @@ def test_gtfs_to_parquet_compression() -> None: will test compression of 3 randomly selected schedules from the past year """ with tempfile.TemporaryDirectory() as temp_dir: - with mock.patch( - "lamp_py.ingestion.compress_gtfs.schedule_details.file_list_from_s3" - ) as patch_s3: + with mock.patch("lamp_py.ingestion.compress_gtfs.schedule_details.file_list_from_s3") as patch_s3: patch_s3.return_value = [] feed = schedules_to_compress(temp_dir) patch_s3.assert_called() @@ -84,12 +82,10 @@ def test_gtfs_to_parquet_compression() -> None: ) pq_start_count = 0 if os.path.exists(start_path): - pq_filter = ( - pc.field("gtfs_active_date") <= active_start_date - ) & (pc.field("gtfs_end_date") >= active_start_date) - pq_start_count = ( - pd.dataset(start_path).filter(pq_filter).count_rows() + pq_filter = (pc.field("gtfs_active_date") <= active_start_date) & ( + pc.field("gtfs_end_date") >= active_start_date ) + pq_start_count = pd.dataset(start_path).filter(pq_filter).count_rows() assert ( pq_start_count == zip_count @@ -103,12 +99,10 @@ def test_gtfs_to_parquet_compression() -> None: ) pq_end_count = 0 if os.path.exists(end_path): - pq_filter = ( - pc.field("gtfs_active_date") <= active_end_date - ) & (pc.field("gtfs_end_date") >= active_end_date) - pq_end_count = ( - pd.dataset(end_path).filter(pq_filter).count_rows() + pq_filter = (pc.field("gtfs_active_date") <= active_end_date) & ( + pc.field("gtfs_end_date") >= active_end_date ) + pq_end_count = pd.dataset(end_path).filter(pq_filter).count_rows() assert ( pq_end_count == zip_count diff --git a/tests/ingestion/test_gtfs_converter.py b/tests/ingestion/test_gtfs_converter.py index 57a69fa0..c643178e 100644 --- a/tests/ingestion/test_gtfs_converter.py +++ b/tests/ingestion/test_gtfs_converter.py @@ -378,9 +378,7 @@ def test_schedule_conversion(_s3_patch: Callable[..., None]) -> None: """ # generate a schedule converter with an empty queue to inspect later metadata_queue: Queue = Queue() - converter = GtfsConverter( - config_type=ConfigType.SCHEDULE, metadata_queue=metadata_queue - ) + converter = GtfsConverter(config_type=ConfigType.SCHEDULE, metadata_queue=metadata_queue) # pass in the test schedule file and convert. the monkey patched parquet # writer function will check that all tables generated by conversion were diff --git a/tests/ingestion/test_gtfs_rt_converter.py b/tests/ingestion/test_gtfs_rt_converter.py index 3e017a2f..e3058f4d 100644 --- a/tests/ingestion/test_gtfs_rt_converter.py +++ b/tests/ingestion/test_gtfs_rt_converter.py @@ -21,9 +21,7 @@ def test_bad_conversion_local() -> None: be added to the error files list """ # dummy config to avoid mypy errors - converter = GtfsRtConverter( - config_type=ConfigType.RT_ALERTS, metadata_queue=Queue() - ) + converter = GtfsRtConverter(config_type=ConfigType.RT_ALERTS, metadata_queue=Queue()) converter.add_files(["badfile"]) # process the bad file and get the table out @@ -41,9 +39,7 @@ def test_bad_conversion_s3() -> None: will be added to the error files list """ with patch("pyarrow.fs.S3FileSystem", return_value=fs.LocalFileSystem): - converter = GtfsRtConverter( - config_type=ConfigType.RT_ALERTS, metadata_queue=Queue() - ) + converter = GtfsRtConverter(config_type=ConfigType.RT_ALERTS, metadata_queue=Queue()) converter.add_files(["s3://badfile"]) # process the bad file and get the table out @@ -64,9 +60,7 @@ def test_empty_files() -> None: ) for config_type in configs_to_test: - converter = GtfsRtConverter( - config_type=config_type, metadata_queue=Queue() - ) + converter = GtfsRtConverter(config_type=config_type, metadata_queue=Queue()) converter.thread_init() empty_file = os.path.join(incoming_dir, "empty.json.gz") @@ -74,8 +68,7 @@ def test_empty_files() -> None: assert filename == empty_file assert table.to_pandas().shape == ( 0, - len(converter.detail.import_schema) - + 4, # add 4 for header timestamp columns + len(converter.detail.import_schema) + 4, # add 4 for header timestamp columns ) one_blank_file = os.path.join(incoming_dir, "one_blank_record.json.gz") @@ -83,8 +76,7 @@ def test_empty_files() -> None: assert filename == one_blank_file assert table.to_pandas().shape == ( 1, - len(converter.detail.import_schema) - + 4, # add 4 for header timestamp columns + len(converter.detail.import_schema) + 4, # add 4 for header timestamp columns ) @@ -133,9 +125,7 @@ def test_vehicle_positions_file_conversion() -> None: # 426 records in 'entity' for 2022-01-01T00:00:03Z_https_cdn.mbta.com_realtime_VehiclePositions_enhanced.json.gz assert table.num_rows == 426 - assert ( - table.num_columns == len(converter.detail.import_schema) + 4 - ) # add 4 for header timestamp columns + assert table.num_columns == len(converter.detail.import_schema) + 4 # add 4 for header timestamp columns np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) @@ -161,9 +151,7 @@ def test_vehicle_positions_file_conversion() -> None: np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) - parquet_file = os.path.join( - test_files_dir, "ingestion_GTFS-RT_VP_OLD.parquet" - ) + parquet_file = os.path.join(test_files_dir, "ingestion_GTFS-RT_VP_OLD.parquet") parquet_df = pandas.read_parquet(parquet_file) parquet_df = drop_list_columns(parquet_df) @@ -196,16 +184,12 @@ def test_rt_alert_file_conversion() -> None: # 144 records in 'entity' for 2022-05-04T15:59:48Z_https_cdn.mbta.com_realtime_Alerts_enhanced.json.gz assert table.num_rows == 144 - assert ( - table.num_columns == len(converter.detail.import_schema) + 4 - ) # add 4 for header timestamp columns + assert table.num_columns == len(converter.detail.import_schema) + 4 # add 4 for header timestamp columns np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) - parquet_file = os.path.join( - test_files_dir, "ingestion_GTFS-RT_ALERT.parquet" - ) + parquet_file = os.path.join(test_files_dir, "ingestion_GTFS-RT_ALERT.parquet") parquet_df = pandas.read_parquet(parquet_file) parquet_df = drop_list_columns(parquet_df) @@ -239,9 +223,7 @@ def test_rt_trip_file_conversion() -> None: # 79 records in 'entity' for # 2022-05-08T06:04:57Z_https_cdn.mbta.com_realtime_TripUpdates_enhanced.json.gz assert table.num_rows == 79 - assert ( - table.num_columns == len(converter.detail.import_schema) + 4 - ) # add 4 for header timestamp columns + assert table.num_columns == len(converter.detail.import_schema) + 4 # add 4 for header timestamp columns np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) @@ -266,9 +248,7 @@ def test_rt_trip_file_conversion() -> None: np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) - parquet_file = os.path.join( - test_files_dir, "ingestion_GTFS-RT_TU_OLD.parquet" - ) + parquet_file = os.path.join(test_files_dir, "ingestion_GTFS-RT_TU_OLD.parquet") parquet_df = pandas.read_parquet(parquet_file) parquet_df = drop_list_columns(parquet_df) @@ -301,9 +281,7 @@ def test_bus_vehicle_positions_file_conversion() -> None: # 844 records in 'entity' for 2022-05-05T16_00_15Z_https_mbta_busloc_s3.s3.amazonaws.com_prod_VehiclePositions_enhanced.json.gz assert table.num_rows == 844 - assert ( - table.num_columns == len(converter.detail.import_schema) + 4 - ) # add 4 for header timestamp columns + assert table.num_columns == len(converter.detail.import_schema) + 4 # add 4 for header timestamp columns np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) @@ -341,9 +319,7 @@ def test_bus_trip_updates_file_conversion() -> None: # 157 records in 'entity' for 2022-06-28T10_03_18Z_https_mbta_busloc_s3.s3.amazonaws.com_prod_TripUpdates_enhanced.json.gz assert table.num_rows == 157 - assert ( - table.num_columns == len(converter.detail.import_schema) + 4 - ) # add 4 for header timestamp columns + assert table.num_columns == len(converter.detail.import_schema) + 4 # add 4 for header timestamp columns np_df = flatten_schema(table).to_pandas() np_df = drop_list_columns(np_df) diff --git a/tests/ingestion_tm/test_ingest.py b/tests/ingestion_tm/test_ingest.py index 381385d7..2fe55959 100644 --- a/tests/ingestion_tm/test_ingest.py +++ b/tests/ingestion_tm/test_ingest.py @@ -35,6 +35,4 @@ def test_ingestion_job_count() -> None: assert all_job_types # ensure all job types are accounted for in ingestion - assert ( - all_job_types == job_types - ), f"Missing instances for subclasses: {all_job_types - job_types}" + assert all_job_types == job_types, f"Missing instances for subclasses: {all_job_types - job_types}" diff --git a/tests/performance_manager/test_alerts.py b/tests/performance_manager/test_alerts.py index 98bc890d..930bbf17 100644 --- a/tests/performance_manager/test_alerts.py +++ b/tests/performance_manager/test_alerts.py @@ -81,9 +81,7 @@ def test_transform_translations() -> None: # check that the number of records that have english translations is # the same as the number of transformed translations. - raw_en_translation_count = ( - alerts_raw[old_name].apply(lambda x: '"en"' in json.dumps(x)).sum() - ) + raw_en_translation_count = alerts_raw[old_name].apply(lambda x: '"en"' in json.dumps(x)).sum() processed_translation_count = alerts_processed[new_name].notna().sum() assert raw_en_translation_count == processed_translation_count @@ -106,23 +104,17 @@ def generate_sample_timestamps(start_ts: int, end_ts: int) -> pandas.DataFrame: created_timestamp = random.randint(start_ts, end_ts) record["created_timestamp"] = created_timestamp if random.choice([True, False]): - record["last_modified_timestamp"] = ( - created_timestamp + random.randint(0, 3600 * 24 * 2) - ) + record["last_modified_timestamp"] = created_timestamp + random.randint(0, 3600 * 24 * 2) else: record["last_modified_timestamp"] = created_timestamp if random.choice([True, False]): - record["last_push_notification_timestamp"] = ( - created_timestamp + random.randint(0, 3600 * 24 * 2) - ) + record["last_push_notification_timestamp"] = created_timestamp + random.randint(0, 3600 * 24 * 2) else: record["last_push_notification_timestamp"] = None if random.choice([True, False]): - record["closed_timestamp"] = created_timestamp + random.randint( - 0, 3600 * 24 * 2 - ) + record["closed_timestamp"] = created_timestamp + random.randint(0, 3600 * 24 * 2) else: record["closed_timestamp"] = None @@ -130,18 +122,10 @@ def generate_sample_timestamps(start_ts: int, end_ts: int) -> pandas.DataFrame: # convert sample data to formatted dataframe alerts_raw = pandas.DataFrame(sample_data) - alerts_raw["created_timestamp"] = alerts_raw["created_timestamp"].astype( - "Int64" - ) - alerts_raw["last_modified_timestamp"] = alerts_raw[ - "last_modified_timestamp" - ].astype("Int64") - alerts_raw["last_push_notification_timestamp"] = alerts_raw[ - "last_push_notification_timestamp" - ].astype("Int64") - alerts_raw["closed_timestamp"] = alerts_raw["closed_timestamp"].astype( - "Int64" - ) + alerts_raw["created_timestamp"] = alerts_raw["created_timestamp"].astype("Int64") + alerts_raw["last_modified_timestamp"] = alerts_raw["last_modified_timestamp"].astype("Int64") + alerts_raw["last_push_notification_timestamp"] = alerts_raw["last_push_notification_timestamp"].astype("Int64") + alerts_raw["closed_timestamp"] = alerts_raw["closed_timestamp"].astype("Int64") return alerts_raw @@ -174,9 +158,7 @@ def ranged_timestamp_test(start_ts: int, end_ts: int) -> None: if base in ["created", "last_modified"]: assert datetime_count == len(alerts_processed) else: - assert ( - datetime_count < len(alerts_processed) * 0.75 - ), alerts_processed[[old_name, new_name]].head() + assert datetime_count < len(alerts_processed) * 0.75, alerts_processed[[old_name, new_name]].head() non_null = alerts_processed[new_name].dropna() assert len(non_null) > 0 @@ -222,16 +204,12 @@ def generate_sample_active_periods( # generate sample data for index in range(1000): - record: Dict[str, list[Dict[str, int | None]] | int | None] = { - "id": index - } + record: Dict[str, list[Dict[str, int | None]] | int | None] = {"id": index} periods: List[Dict[str, int | None]] = [] for __ in range(random.randint(0, 15)): exploded_count += 1 start: Optional[int] = random.randint(range_start_ts, range_end_ts) - end: Optional[int] = ( - start + random.randint(3600, max_end_seconds) if start else None - ) + end: Optional[int] = start + random.randint(3600, max_end_seconds) if start else None if random.randint(1, 100) < 5: start = None @@ -270,11 +248,7 @@ def test_explode_active_period() -> None: alerts_processed = explode_active_periods(alerts_raw) assert "active_period" not in alerts_processed.columns - nan_active_periods = ( - alerts_raw["active_period"] - .apply(lambda x: 1 if x is None or len(x) == 0 else 0) - .sum() - ) + nan_active_periods = alerts_raw["active_period"].apply(lambda x: 1 if x is None or len(x) == 0 else 0).sum() total_record_count = nan_active_periods + active_period_count assert len(alerts_processed) == total_record_count @@ -314,21 +288,13 @@ def generate_sample_informed_entity( informed_entity_count = 0 for index in range(100): informed_entity = [] - for route_id in random.sample( - choices["route_id"], random.randint(1, 4) - ): + for route_id in random.sample(choices["route_id"], random.randint(1, 4)): route_type = random.choice([0, 1, 2, 3, None]) - for direction_id in random.sample( - choices["direction_id"], random.randint(1, 2) - ): - for stop_id in random.sample( - choices["stop_id"], random.randint(1, 3) - ): + for direction_id in random.sample(choices["direction_id"], random.randint(1, 2)): + for stop_id in random.sample(choices["stop_id"], random.randint(1, 3)): facility_id = random.choice(choices["facility_id"]) - activities = random.sample( - choices["activities"], random.randint(0, 4) - ) + activities = random.sample(choices["activities"], random.randint(0, 4)) record = {} record["route_id"] = route_id @@ -351,9 +317,7 @@ def test_explode_informed_entity() -> None: """ test that exploding around the informed entity column works as expected """ - choices: Dict[ - str, Union[List[str | None], List[float | None], List[str]] - ] = { + choices: Dict[str, Union[List[str | None], List[float | None], List[str]]] = { "route_id": [ "1234", "Blue", @@ -388,9 +352,7 @@ def test_explode_informed_entity() -> None: if column == "route_type": filtered_options = [o for o in options if not pandas.isna(o)] filtered_values = [v for v in values if not pandas.isna(v)] - assert set(filtered_values) == set( - filtered_options - ), f"{column} has different values" + assert set(filtered_values) == set(filtered_options), f"{column} has different values" elif column == "activities": for value in values: if value == "": @@ -420,9 +382,7 @@ def test_etl() -> None: key_columns = ["id", "last_modified_timestamp"] existing = pandas.DataFrame(columns=key_columns) - alerts = extract_alerts( - alert_files=[test_file], existing_id_timestamp_pairs=existing - ) + alerts = extract_alerts(alert_files=[test_file], existing_id_timestamp_pairs=existing) alerts = transform_translations(alerts) alerts = transform_timestamps(alerts) alerts = explode_active_periods(alerts) @@ -430,9 +390,7 @@ def test_etl() -> None: # process it a second time with some of the id / lm timestamp pairs to filter against. existing = alerts[key_columns].drop_duplicates().head(5) - alerts_2 = extract_alerts( - alert_files=[test_file], existing_id_timestamp_pairs=existing - ) + alerts_2 = extract_alerts(alert_files=[test_file], existing_id_timestamp_pairs=existing) alerts_2 = transform_translations(alerts_2) alerts_2 = transform_timestamps(alerts_2) alerts_2 = explode_active_periods(alerts_2) diff --git a/tests/performance_manager/test_l0_gtfs_rt_events.py b/tests/performance_manager/test_l0_gtfs_rt_events.py index 893511c7..3fe91a22 100644 --- a/tests/performance_manager/test_l0_gtfs_rt_events.py +++ b/tests/performance_manager/test_l0_gtfs_rt_events.py @@ -62,9 +62,7 @@ def test_vp_missing_service_date(tmp_path: pathlib.Path) -> None: """ csv_file = os.path.join(test_files_dir, "vp_missing_start_date.csv") - parquet_folder = tmp_path.joinpath( - "RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11" - ) + parquet_folder = tmp_path.joinpath("RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11") parquet_folder.mkdir(parents=True) parquet_file = str(parquet_folder.joinpath("flat_file.parquet")) @@ -77,9 +75,7 @@ def test_vp_missing_service_date(tmp_path: pathlib.Path) -> None: assert events["service_date"].hasnans # add the service dates that are missing - events = add_missing_service_dates( - events, timestamp_key="vehicle_timestamp" - ) + events = add_missing_service_dates(events, timestamp_key="vehicle_timestamp") # check that new service dates match existing and are numbers assert len(events["service_date"].unique()) == 1 @@ -97,9 +93,7 @@ def test_tu_missing_service_date() -> None: # check that NaN service dates exist from reading the file assert events["service_date"].hasnans - events = add_missing_service_dates( - events_dataframe=events, timestamp_key="timestamp" - ) + events = add_missing_service_dates(events_dataframe=events, timestamp_key="timestamp") # check that all service dates exist and are the same assert not events["service_date"].hasnans diff --git a/tests/performance_manager/test_performance_manager.py b/tests/performance_manager/test_performance_manager.py index ceb11e99..b83118da 100644 --- a/tests/performance_manager/test_performance_manager.py +++ b/tests/performance_manager/test_performance_manager.py @@ -101,9 +101,7 @@ def set_env_vars() -> None: if int(os.environ.get("BOOTSTRAPPED", 0)) == 1: logging.warning("already bootstrapped") else: - env_file = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "..", "..", ".env" - ) + env_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".env") logging.debug("bootstrapping with env file %s", env_file) with open(env_file, "r", encoding="utf8") as reader: @@ -123,9 +121,7 @@ def fixture_rpm_db_manager() -> DatabaseManager: generate a database manager for all of our tests """ set_env_vars() - db_manager = DatabaseManager( - db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER - ) + db_manager = DatabaseManager(db_index=DatabaseIndex.RAIL_PERFORMANCE_MANAGER) db_name = os.getenv("ALEMBIC_DB_NAME", "performance_manager_prod") alembic_downgrade_to_base(db_name) alembic_upgrade_to_head(db_name) @@ -152,9 +148,7 @@ def fixture_s3_patch(monkeypatch: MonkeyPatch) -> Iterator[None]: files instead of s3, so we read these files differently """ - def mock__get_static_parquet_paths( - table_type: str, feed_info_path: str - ) -> List[str]: + def mock__get_static_parquet_paths(table_type: str, feed_info_path: str) -> List[str]: """ instead of mocking up s3 responses, just rewrite this method and monkeypatch it @@ -219,10 +213,7 @@ def mock__object_metadata(obj: str) -> Dict[str, str]: return the current value, implying that we're not testing the logic that will reset old versions. """ - assert ( - obj - == f"{test_archive_value}/lamp/subway-on-time-performance-v1/index.csv" - ) + assert obj == f"{test_archive_value}/lamp/subway-on-time-performance-v1/index.csv" return {S3Archive.VERSION_KEY: test_version_value} monkeypatch.setattr( @@ -230,9 +221,7 @@ def mock__object_metadata(obj: str) -> Dict[str, str]: mock__object_metadata, ) - def mock__file_list_from_s3( - bucket_name: str, file_prefix: str, max_list_size: int = 250_000 - ) -> List[str]: + def mock__file_list_from_s3(bucket_name: str, file_prefix: str, max_list_size: int = 250_000) -> List[str]: """ this is used to get all of the files that are already on s3. """ @@ -252,9 +241,7 @@ def mock__file_list_from_s3( mock__file_list_from_s3, ) - def mock__file_list_from_s3_with_details( - bucket_name: str, file_prefix: str - ) -> List[Dict]: + def mock__file_list_from_s3_with_details(bucket_name: str, file_prefix: str) -> List[Dict]: """ this is used to write the index csv in the flat file """ @@ -268,9 +255,7 @@ def mock__file_list_from_s3_with_details( mock__file_list_from_s3_with_details, ) - def mock__upload_file( - file_name: str, object_path: str, extra_args: Optional[Dict] = None - ) -> bool: + def mock__upload_file(file_name: str, object_path: str, extra_args: Optional[Dict] = None) -> bool: """ this is used by the flat file writer to move parquet and index csv files to s3 @@ -287,9 +272,7 @@ def inspect_csv(filepath: str) -> None: "size_bytes", "last_modified", ] - assert set(index_data.columns) == set( - expected_columns - ), "index.csv has incorrect columns" + assert set(index_data.columns) == set(expected_columns), "index.csv has incorrect columns" # ensure that the index didn't refer to itself assert not ( @@ -331,15 +314,11 @@ def inspect_parquet(filepath: str) -> None: "scheduled_headway_trunk", ] - assert set(flat_data.columns) == set( - expected_columns - ), "flat parquet file has incorrect columns" + assert set(flat_data.columns) == set(expected_columns), "flat parquet file has incorrect columns" assert not flat_data.empty, "flat parquet file has no data" - expected_extra_args = { - "Metadata": {S3Archive.VERSION_KEY: test_version_value} - } + expected_extra_args = {"Metadata": {S3Archive.VERSION_KEY: test_version_value}} assert extra_args == expected_extra_args if object_path.endswith("index.csv"): @@ -382,9 +361,7 @@ def message_to_dict(message: str) -> Dict[str, str]: (key, value) = part.split("=", 1) message_dict[key] = value except Exception as exception: - pytest.fail( - f"Unable to parse log message {message}. Reason {exception}" - ) + pytest.fail(f"Unable to parse log message {message}. Reason {exception}") return message_dict # keep track of logged processes to ensure order is correct @@ -415,9 +392,7 @@ def message_to_dict(message: str) -> Dict[str, str]: if log["status"] == "complete": # process should be at the end of the stack if process_stack[-1]["uuid"] != log["uuid"]: - pytest.fail( - f"Improper Ordering of Log Statements {caplog.text}" - ) + pytest.fail(f"Improper Ordering of Log Statements {caplog.text}") if "duration" not in log: pytest.fail(f"Log missing duration key - {record.message}") process_stack.pop() @@ -452,14 +427,11 @@ def test_static_tables( unprocessed_static_schedules = md_db_manager.select_as_list( sa.select(MetadataLog.path).where( - (MetadataLog.rail_pm_processed == sa.false()) - & (MetadataLog.path.contains("FEED_INFO")) + (MetadataLog.rail_pm_processed == sa.false()) & (MetadataLog.path.contains("FEED_INFO")) ) ) - process_static_tables( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_static_tables(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) # these are the row counts in the parquet files computed in a jupyter # notebook without using any of our module. our module should be taking @@ -481,14 +453,11 @@ def test_static_tables( for table, should_count in row_counts.items(): actual_count = session.query(table).count() tablename = table.__tablename__ - assert ( - actual_count == should_count - ), f"Table {tablename} has incorrect row count" + assert actual_count == should_count, f"Table {tablename} has incorrect row count" unprocessed_static_schedules = md_db_manager.select_as_list( sa.select(MetadataLog.path).where( - (MetadataLog.rail_pm_processed == sa.false()) - & (MetadataLog.path.contains("FEED_INFO")) + (MetadataLog.rail_pm_processed == sa.false()) & (MetadataLog.path.contains("FEED_INFO")) ) ) @@ -534,17 +503,12 @@ def test_gtfs_rt_processing( rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) paths = [ file for file in test_files() - if ("RT_VEHICLE_POSITIONS" in file or "RT_TRIP_UPDATES" in file) - and ("hour=12" in file or "hour=13" in file) + if ("RT_VEHICLE_POSITIONS" in file or "RT_TRIP_UPDATES" in file) and ("hour=12" in file or "hour=13" in file) ] seed_metadata(md_db_manager, paths) @@ -648,22 +612,12 @@ def test_vp_only( rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) - paths = [ - p - for p in test_files() - if "RT_VEHICLE_POSITIONS" in p and ("hourt=12" in p or "hour=13" in p) - ] + paths = [p for p in test_files() if "RT_VEHICLE_POSITIONS" in p and ("hourt=12" in p or "hour=13" in p)] seed_metadata(md_db_manager, paths) - process_gtfs_rt_files( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_gtfs_rt_files(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) check_logs(caplog) @@ -680,22 +634,12 @@ def test_tu_only( rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) - paths = [ - p - for p in test_files() - if "RT_TRIP_UPDATES" in p and ("hourt=12" in p or "hour=13" in p) - ] + paths = [p for p in test_files() if "RT_TRIP_UPDATES" in p and ("hourt=12" in p or "hour=13" in p)] seed_metadata(md_db_manager, paths) - process_gtfs_rt_files( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_gtfs_rt_files(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) check_logs(caplog) @@ -712,18 +656,12 @@ def test_vp_and_tu( rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) paths = [p for p in test_files() if "hourt=12" in p or "hour=13" in p] seed_metadata(md_db_manager, paths) - process_gtfs_rt_files( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_gtfs_rt_files(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) check_logs(caplog) @@ -743,18 +681,12 @@ def test_missing_start_time( # clear out old data from the database rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) # create a new parquet file from ths missing start times csv and add it to # the metadata table for processing csv_file = os.path.join(test_files_dir, "vp_missing_start_time.csv") - parquet_folder = tmp_path.joinpath( - "RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11" - ) + parquet_folder = tmp_path.joinpath("RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11") parquet_folder.mkdir(parents=True) parquet_file = str(parquet_folder.joinpath("flat_file.parquet")) @@ -762,9 +694,7 @@ def test_missing_start_time( seed_metadata(md_db_manager, paths=[parquet_file]) # process the parquet file - process_gtfs_rt_files( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_gtfs_rt_files(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) # check that all trips have an int convertible start time that is in # seconds after midnight. @@ -781,9 +711,7 @@ def test_missing_start_time( # there is an added trip in the csv data who's first move time is 1683547198 # or 7:59:58 AM. that is 25200 + 3540 + 58 = 28798 seconds after midnight. added_trip_start_time = rpm_db_manager.select_as_list( - sa.select(VehicleTrips.start_time).where( - VehicleTrips.trip_id == "ADDED-1581518546" - ) + sa.select(VehicleTrips.start_time).where(VehicleTrips.trip_id == "ADDED-1581518546") ) assert len(added_trip_start_time) == 1 assert added_trip_start_time[0]["start_time"] == 28798 @@ -804,9 +732,7 @@ def test_process_vp_files( caplog.set_level(logging.INFO) csv_file = os.path.join(test_files_dir, "vehicle_positions_flat_input.csv") - parquet_folder = tmp_path.joinpath( - "RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11" - ) + parquet_folder = tmp_path.joinpath("RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11") parquet_folder.mkdir(parents=True) parquet_file = str(parquet_folder.joinpath("flat_file.parquet")) @@ -847,9 +773,7 @@ def test_process_vp_files( column_exceptions = [] for column in csv_result_df.columns: try: - pandas.testing.assert_series_equal( - result_df[column], csv_result_df[column] - ) + pandas.testing.assert_series_equal(result_df[column], csv_result_df[column]) except Exception as exception: logging.error( "Pipeline values in %s column do not match process_vp_files_flat_out.csv CSV file", @@ -877,25 +801,17 @@ def test_whole_table( rpm_db_manager.truncate_table(VehicleEvents, restart_identity=True) rpm_db_manager.truncate_table(VehicleTrips, restart_identity=True) - md_db_manager.execute( - sa.delete(MetadataLog.__table__).where( - ~MetadataLog.path.contains("FEED_INFO") - ) - ) + md_db_manager.execute(sa.delete(MetadataLog.__table__).where(~MetadataLog.path.contains("FEED_INFO"))) csv_file = os.path.join(test_files_dir, "vehicle_positions_flat_input.csv") - parquet_folder = tmp_path.joinpath( - "RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11" - ) + parquet_folder = tmp_path.joinpath("RT_VEHICLE_POSITIONS/year=2023/month=5/day=8/hour=11") parquet_folder.mkdir(parents=True) parquet_file = str(parquet_folder.joinpath("flat_file.parquet")) csv_to_vp_parquet(csv_file, parquet_file) seed_metadata(md_db_manager, paths=[parquet_file]) - process_gtfs_rt_files( - rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager - ) + process_gtfs_rt_files(rpm_db_manager=rpm_db_manager, md_db_manager=md_db_manager) result_select = ( sa.select( @@ -975,9 +891,7 @@ def test_whole_table( column_exceptions = [] for column in csv_result_df.columns: try: - pandas.testing.assert_series_equal( - db_result_df[column], csv_result_df[column] - ) + pandas.testing.assert_series_equal(db_result_df[column], csv_result_df[column]) except Exception as exception: logging.error( "Pipeline values in %s column do not match pipeline_flat_out.csv CSV file", diff --git a/tests/test_resources.py b/tests/test_resources.py index 95989979..60806b62 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -66,15 +66,7 @@ def s3_uri(self) -> str: bucket=S3_SPRINGBOARD, prefix="TM/STOP_CROSSING", ) -tm_geo_node_file = LocalS3Location( - bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_GEO_NODE.parquet" -) -tm_route_file = LocalS3Location( - bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_ROUTE.parquet" -) -tm_trip_file = LocalS3Location( - bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_TRIP.parquet" -) -tm_vehicle_file = LocalS3Location( - bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_VEHICLE.parquet" -) +tm_geo_node_file = LocalS3Location(bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_GEO_NODE.parquet") +tm_route_file = LocalS3Location(bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_ROUTE.parquet") +tm_trip_file = LocalS3Location(bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_TRIP.parquet") +tm_vehicle_file = LocalS3Location(bucket=S3_SPRINGBOARD, prefix="TM/TMMAIN_VEHICLE.parquet")